lly/umami
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'dev' into feat-funnel-per-step-filters

Browse Source
This commit is contained in:
Mike Cao
2026-03-25 13:40:12 -07:00
committed by GitHub
parent 5246e2f3aa 54dbfe1ca9
commit fb785180d5
675 changed files with 42211 additions and 143313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/queries/sql/events/getEventData.ts
+124 -35
View File
@@ -1,63 +1,152 @@
import type { EventData } from '@/generated/prisma/client';
import clickhouse from '@/lib/clickhouse';
import { DEFAULT_PAGE_SIZE } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
const FUNCTION_NAME = 'getEventData';
export async function getEventData(
...args: [websiteId: string, eventId: string]
): Promise<EventData[]> {
export async function getEventData(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, eventId: string) {
const { rawQuery } = prisma;
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { rawQuery, parseFilters } = prisma;
const { page = 1, pageSize } = filters;
const size = +pageSize || DEFAULT_PAGE_SIZE;
const offset = +size * (+page - 1);
return rawQuery(
const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
// Selects distinct event IDs matching all filters — reused for count and paged data
const eventQuery = `
select website_event.event_id
from website_event
join event_data on event_data.website_event_id = website_event.event_id
and event_data.website_id = {{websiteId::uuid}}
and event_data.created_at between {{startDate}} and {{endDate}}
${cohortQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by website_event.event_id
`;
const count = await rawQuery(
`select count(*) as num from (${eventQuery}) t`,
queryParams,
).then((res: any) => res[0].num);
const data = await rawQuery(
`
select event_data.website_id as "websiteId",
event_data.website_event_id as "eventId",
website_event.event_name as "eventName",
event_data.data_key as "dataKey",
event_data.string_value as "stringValue",
event_data.number_value as "numberValue",
event_data.date_value as "dateValue",
event_data.data_type as "dataType",
event_data.created_at as "createdAt"
with paged_events as (
${eventQuery}
order by max(website_event.created_at) desc
limit ${size} offset ${offset}
)
select
event_data.website_id as "websiteId",
event_data.website_event_id as "eventId",
website_event.event_name as "eventName",
event_data.data_key as "dataKey",
event_data.string_value as "stringValue",
event_data.number_value as "numberValue",
event_data.date_value as "dateValue",
event_data.data_type as "dataType",
event_data.created_at as "createdAt"
from event_data
join website_event on website_event.event_id = event_data.website_event_id
and website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
join paged_events on paged_events.event_id = event_data.website_event_id
where event_data.website_id = {{websiteId::uuid}}
and event_data.website_event_id = {{eventId::uuid}}
and event_data.created_at between {{startDate}} and {{endDate}}
order by event_data.created_at desc
`,
{ websiteId, eventId },
queryParams,
FUNCTION_NAME,
);
return { data, count, page: +page, pageSize: size };
}
async function clickhouseQuery(websiteId: string, eventId: string): Promise<EventData[]> {
const { rawQuery } = clickhouse;
async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { rawQuery, parseFilters } = clickhouse;
const { page = 1, pageSize } = filters;
const size = +pageSize || DEFAULT_PAGE_SIZE;
const offset = +size * (+page - 1);
return rawQuery(
const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId });
// Selects distinct event IDs matching all filters — reused for count and paged data
const eventQuery = `
select event_data.event_id
from event_data
any left join (
select event_id, session_id, website_id, event_name, created_at
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2
) website_event
on website_event.event_id = event_data.event_id
and website_event.session_id = event_data.session_id
and website_event.website_id = event_data.website_id
${cohortQuery}
where event_data.website_id = {websiteId:UUID}
and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
group by event_data.event_id
`;
const count = await rawQuery(
`select count(*) as num from (${eventQuery}) t`,
queryParams,
).then((res: any) => res[0].num);
const data = await rawQuery(
`
select website_id as websiteId,
event_id as eventId,
event_name as eventName,
data_key as dataKey,
string_value as stringValue,
number_value as numberValue,
date_value as dateValue,
data_type as dataType,
created_at as createdAt
from event_data
where website_id = {websiteId:UUID}
and event_id = {eventId:UUID}
with paged_events as (
${eventQuery}
order by max(event_data.created_at) desc
limit ${size} offset ${offset}
)
select
event_data.website_id as websiteId,
event_data.event_id as eventId,
website_event.event_name as eventName,
data_key as dataKey,
string_value as stringValue,
number_value as numberValue,
date_value as dateValue,
data_type as dataType,
event_data.created_at as createdAt
from event_data
any left join (
select event_id, session_id, website_id, event_name
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2
) website_event
on website_event.event_id = event_data.event_id
and website_event.session_id = event_data.session_id
and website_event.website_id = event_data.website_id
inner join paged_events on paged_events.event_id = event_data.event_id
where event_data.website_id = {websiteId:UUID}
and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}
order by event_data.created_at desc
`,
{ websiteId, eventId },
queryParams,
FUNCTION_NAME,
);
}
return { data, count, page: +page, pageSize: size };
}
src/queries/sql/events/getEventDataById.ts
+63
View File
@@ -0,0 +1,63 @@
import type { EventData } from '@/generated/prisma/client';
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
const FUNCTION_NAME = 'getEventDataById';
export async function getEventDataById(
...args: [websiteId: string, eventId: string]
): Promise<EventData[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, eventId: string) {
const { rawQuery } = prisma;
return rawQuery(
`
select event_data.website_id as "websiteId",
event_data.website_event_id as "eventId",
website_event.event_name as "eventName",
event_data.data_key as "dataKey",
event_data.string_value as "stringValue",
event_data.number_value as "numberValue",
event_data.date_value as "dateValue",
event_data.data_type as "dataType",
event_data.created_at as "createdAt"
from event_data
join website_event on website_event.event_id = event_data.website_event_id
and website_event.website_id = {{websiteId::uuid}}
where event_data.website_id = {{websiteId::uuid}}
and event_data.website_event_id = {{eventId::uuid}}
`,
{ websiteId, eventId },
FUNCTION_NAME,
);
}
async function clickhouseQuery(websiteId: string, eventId: string): Promise<EventData[]> {
const { rawQuery } = clickhouse;
return rawQuery(
`
select website_id as websiteId,
event_id as eventId,
event_name as eventName,
data_key as dataKey,
string_value as stringValue,
number_value as numberValue,
date_value as dateValue,
data_type as dataType,
created_at as createdAt
from event_data
where website_id = {websiteId:UUID}
and event_id = {eventId:UUID}
`,
{ websiteId, eventId },
FUNCTION_NAME,
);
}
src/queries/sql/events/getEventDataEvents.ts
+21 -9
View File
@@ -25,7 +25,7 @@ export async function getEventDataEvents(
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { rawQuery, parseFilters } = prisma;
const { event } = filters;
const { queryParams } = parseFilters({
const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
@@ -42,9 +42,11 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
from event_data
inner join website_event
on website_event.event_id = event_data.website_event_id
${cohortQuery}
${joinSessionQuery}
where event_data.website_id = {{websiteId::uuid}}
and event_data.created_at between {{startDate}} and {{endDate}}
and website_event.event_name = {{event}}
${filterQuery}
group by website_event.event_name, event_data.data_key, event_data.data_type, event_data.string_value
order by 1 asc, 2 asc, 3 asc, 5 desc
`,
@@ -63,8 +65,11 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
from event_data
inner join website_event
on website_event.event_id = event_data.website_event_id
${cohortQuery}
${joinSessionQuery}
where event_data.website_id = {{websiteId::uuid}}
and event_data.created_at between {{startDate}} and {{endDate}}
${filterQuery}
limit 500
`,
queryParams,
@@ -93,15 +98,18 @@ async function clickhouseQuery(
string_value as propertyValue,
count(*) as total
from event_data
join website_event
any left join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2) website_event
on website_event.event_id = event_data.event_id
and website_event.session_id = event_data.session_id
and website_event.website_id = event_data.website_id
and website_event.website_id = {websiteId:UUID}
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${cohortQuery}
where event_data.website_id = {websiteId:UUID}
and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_data.event_name = {event:String}
${filterQuery}
group by data_key, data_type, string_value, event_name
order by 1 asc, 2 asc, 3 asc, 5 desc
@@ -120,11 +128,15 @@ async function clickhouseQuery(
data_type as dataType,
count(*) as total
from event_data
join website_event
any left join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2) website_event
on website_event.event_id = event_data.event_id
and website_event.session_id = event_data.session_id
and website_event.website_id = event_data.website_id
and website_event.website_id = {websiteId:UUID}
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${cohortQuery}
where event_data.website_id = {websiteId:UUID}
and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}
src/queries/sql/events/getEventDataFields.ts
+7 -3
View File
@@ -65,11 +65,15 @@ async function clickhouseQuery(
string_value) as "value",
count(*) as "total"
from event_data
join website_event
any left join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2) website_event
on website_event.event_id = event_data.event_id
and website_event.session_id = event_data.session_id
and website_event.website_id = event_data.website_id
and website_event.website_id = {websiteId:UUID}
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${cohortQuery}
where event_data.website_id = {websiteId:UUID}
and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}
src/queries/sql/events/getEventDataProperties.ts
+7 -3
View File
@@ -69,11 +69,15 @@ async function clickhouseQuery(
data_key as propertyName,
count(*) as total
from event_data
join website_event
any left join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2) website_event
on website_event.event_id = event_data.event_id
and website_event.session_id = event_data.session_id
and website_event.website_id = event_data.website_id
and website_event.website_id = {websiteId:UUID}
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${cohortQuery}
where event_data.website_id = {websiteId:UUID}
and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}
src/queries/sql/events/getEventDataStats.ts
+7 -3
View File
@@ -72,11 +72,15 @@ async function clickhouseQuery(
data_key,
count(*) as "total"
from event_data
join website_event
any left join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2) website_event
on website_event.event_id = event_data.event_id
and website_event.session_id = event_data.session_id
and website_event.website_id = event_data.website_id
and website_event.website_id = {websiteId:UUID}
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${cohortQuery}
where event_data.website_id = {websiteId:UUID}
and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}
src/queries/sql/events/getEventDataValues.ts
+7 -4
View File
@@ -72,16 +72,19 @@ async function clickhouseQuery(
string_value) as "value",
count(*) as "total"
from event_data
join website_event
any left join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2) website_event
on website_event.event_id = event_data.event_id
and website_event.session_id = event_data.session_id
and website_event.website_id = event_data.website_id
and website_event.website_id = {websiteId:UUID}
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${cohortQuery}
where event_data.website_id = {websiteId:UUID}
and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_data.data_key = {propertyName:String}
and event_data.event_name = {event:String}
${filterQuery}
group by value
order by 2 desc
src/queries/sql/events/getEventExpandedMetrics.ts
+2 -1
View File
@@ -58,7 +58,7 @@ async function relationalQuery(
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime"
from (
select
${column} name,
${column} as "name",
website_event.session_id,
website_event.visit_id,
count(*) as "c",
@@ -72,6 +72,7 @@ async function relationalQuery(
${filterQuery}
group by name, website_event.session_id, website_event.visit_id
) as t
where name != ''
group by name
order by visitors desc, visits desc
limit ${limit}
src/queries/sql/events/getEventStats.ts
+45 -6
View File
@@ -1,11 +1,14 @@
import clickhouse from '@/lib/clickhouse';
import { EVENT_TYPE } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
const FUNCTION_NAME = 'getEventStats';
export interface EventStatsParameters {
limit?: number | string;
}
interface WebsiteEventMetric {
x: string;
t: string;
@@ -13,7 +16,7 @@ interface WebsiteEventMetric {
}
export async function getEventStats(
...args: [websiteId: string, filters: QueryFilters]
...args: [websiteId: string, parameters: EventStatsParameters, filters: QueryFilters]
): Promise<WebsiteEventMetric[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@@ -21,15 +24,32 @@ export async function getEventStats(
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
async function relationalQuery(
websiteId: string,
parameters: EventStatsParameters,
filters: QueryFilters,
) {
const { limit } = parameters;
const { timezone = 'utc', unit = 'day' } = filters;
const { rawQuery, getDateSQL, parseFilters } = prisma;
const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({
...filters,
websiteId,
eventType: EVENT_TYPE.customEvent,
});
const limitQuery = limit
? `and event_name in (
select event_name
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and event_type = 2
group by event_name
order by count(*) desc
limit ${limit}
)`
: '';
return rawQuery(
`
select
@@ -41,7 +61,9 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type = 2
${filterQuery}
${limitQuery}
group by 1, 2
order by 2
`,
@@ -52,16 +74,30 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
async function clickhouseQuery(
websiteId: string,
parameters: EventStatsParameters,
filters: QueryFilters,
): Promise<{ x: string; t: string; y: number }[]> {
const { limit } = parameters;
const { timezone = 'UTC', unit = 'day' } = filters;
const { rawQuery, getDateSQL, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
eventType: EVENT_TYPE.customEvent,
});
const limitQuery = limit
? `and event_name in (
select event_name
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2
group by event_name
order by count(*) desc
limit ${limit}
)`
: '';
let sql = '';
if (filterQuery || cohortQuery) {
@@ -74,7 +110,9 @@ async function clickhouseQuery(
${cohortQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2
${filterQuery}
${limitQuery}
group by x, t
order by t
`;
@@ -90,7 +128,8 @@ async function clickhouseQuery(
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
and event_type = 2
${limitQuery}
) as g
group by x, t
order by t
src/queries/sql/events/getWebsiteEventStats.ts
+97
View File
@@ -0,0 +1,97 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
const FUNCTION_NAME = 'getWebsiteEventStats';
export interface WebsiteEventStatsData {
events: number;
visitors: number;
visits: number;
uniqueEvents: number;
}
export async function getWebsiteEventStats(
...args: [websiteId: string, filters: QueryFilters]
): Promise<WebsiteEventStatsData[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: QueryFilters,
): Promise<WebsiteEventStatsData[]> {
const { parseFilters, rawQuery } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
return rawQuery(
`
select
cast(coalesce(sum(t.c), 0) as bigint) as "events",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
count(distinct t.event_name) as "uniqueEvents"
from (
select
website_event.session_id,
website_event.visit_id,
website_event.event_name,
count(*) as "c"
from website_event
${cohortQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type = 2
${filterQuery}
group by 1, 2, 3
) as t
`,
queryParams,
FUNCTION_NAME,
).then(result => result?.[0]);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<WebsiteEventStatsData[]> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
return rawQuery(
`
select
sum(t.c) as "events",
uniq(t.session_id) as "visitors",
uniq(t.visit_id) as "visits",
count(distinct t.event_name) as "uniqueEvents"
from (
select
session_id,
visit_id,
event_name,
count(*) c
from website_event
${cohortQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2
${filterQuery}
group by session_id, visit_id, event_name
) as t;
`,
queryParams,
FUNCTION_NAME,
).then(result => result?.[0]);
}
src/queries/sql/events/saveEvent.ts
+33 -6
View File
@@ -53,6 +53,13 @@ export interface SaveEventArgs {
ttclid?: string;
lifatid?: string;
twclid?: string;
// Performance
lcp?: number;
inp?: number;
cls?: number;
fcp?: number;
ttfb?: number;
}
export async function saveEvent(args: SaveEventArgs) {
@@ -89,6 +96,11 @@ async function relationalQuery({
ttclid,
lifatid,
twclid,
lcp,
inp,
cls,
fcp,
ttfb,
}: SaveEventArgs) {
const websiteEventId = uuid();
@@ -119,6 +131,11 @@ async function relationalQuery({
eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
tag,
hostname,
lcp,
inp,
cls,
fcp,
ttfb,
createdAt,
},
});
@@ -186,6 +203,11 @@ async function clickhouseQuery({
ttclid,
lifatid,
twclid,
lcp,
inp,
cls,
fcp,
ttfb,
}: SaveEventArgs) {
const { insert, getUTCString } = clickhouse;
const { sendMessage } = kafka;
@@ -221,12 +243,17 @@ async function clickhouseQuery({
tag: tag,
distinct_id: distinctId,
created_at: getUTCString(createdAt),
browser,
os,
device,
screen,
language,
hostname,
browser: browser,
os: os,
device: device,
screen: screen,
language: language,
hostname: hostname,
lcp: lcp,
inp: inp,
cls: cls,
fcp: fcp,
ttfb: ttfb,
};
if (kafka.enabled) {
src/queries/sql/getChannelExpandedMetrics.ts
+33 -25
View File
@@ -41,10 +41,11 @@ async function relationalQuery(
filters: QueryFilters,
): Promise<ChannelExpandedMetricsData[]> {
const { rawQuery, parseFilters, getTimestampDiffSQL } = prisma;
const { queryParams, filterQuery, joinSessionQuery, cohortQuery, dateQuery } = parseFilters({
...filters,
websiteId,
});
const { queryParams, filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, dateQuery } =
parseFilters({
...filters,
websiteId,
});
return rawQuery(
`
@@ -59,14 +60,16 @@ async function relationalQuery(
website_event.utm_source,
website_event.session_id,
website_event.visit_id,
website_event.hostname,
count(*) c,
min(website_event.created_at) min_time,
max(website_event.created_at) max_time
from website_event
${cohortQuery}
${excludeBounceQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${dateQuery}
${filterQuery}
group by prefix,
@@ -75,7 +78,8 @@ async function relationalQuery(
website_event.utm_medium,
website_event.utm_source,
website_event.session_id,
website_event.visit_id),
website_event.visit_id,
website_event.hostname),
channels as (
select case
@@ -89,7 +93,8 @@ async function relationalQuery(
when ${toPostgresPositionClause('referrer_domain', EMAIL_DOMAINS)} or utm_medium ilike '%mail%' then 'email'
when ${toPostgresPositionClause('referrer_domain', SHOPPING_DOMAINS)} or utm_medium ilike '%shop%' then concat(prefix, 'Shopping')
when ${toPostgresPositionClause('referrer_domain', VIDEO_DOMAINS)} or utm_medium ilike '%video%' then concat(prefix, 'Video')
else '' end AS name,
when referrer_domain != regexp_replace(hostname, '^www.', '') and referrer_domain != '' then 'referral'
else '' end as "name",
session_id,
visit_id,
c,
@@ -119,7 +124,7 @@ async function clickhouseQuery(
filters: QueryFilters,
): Promise<ChannelExpandedMetricsData[]> {
const { rawQuery, parseFilters } = clickhouse;
const { queryParams, filterQuery, cohortQuery } = parseFilters({
const { queryParams, filterQuery, cohortQuery, excludeBounceQuery } = parseFilters({
...filters,
websiteId,
});
@@ -134,31 +139,33 @@ async function clickhouseQuery(
sum(if(t.c = 1, 1, 0)) as "bounces",
sum(max_time-min_time) as "totaltime"
from (
select case when multiSearchAny(utm_medium, ['cp', 'ppc', 'retargeting', 'paid']) != 0 then 'paid' else 'organic' end prefix,
case
select
case when multiSearchAny(lower(utm_medium), ['cp', 'ppc', 'retargeting', 'paid']) != 0 then 'paid' else 'organic' end prefix,
case
when referrer_domain = '' and url_query = '' then 'direct'
when multiSearchAny(url_query, [${toClickHouseStringArray(
when multiSearchAny(lower(url_query), [${toClickHouseStringArray(
PAID_AD_PARAMS,
)}]) != 0 then 'paidAds'
when multiSearchAny(utm_medium, ['referral', 'app','link']) != 0 then 'referral'
when position(utm_medium, 'affiliate') > 0 then 'affiliate'
when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms'
when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
when multiSearchAny(lower(utm_medium), ['referral', 'app','link']) != 0 then 'referral'
when position(lower(utm_medium), 'affiliate') > 0 then 'affiliate'
when position(lower(utm_medium), 'sms') > 0 or position(lower(utm_source), 'sms') > 0 then 'sms'
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
SEARCH_DOMAINS,
)}]) != 0 or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search')
when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
)}]) != 0 or position(lower(utm_medium), 'organic') > 0 then concat(prefix, 'Search')
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
SOCIAL_DOMAINS,
)}]) != 0 then concat(prefix, 'Social')
when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
EMAIL_DOMAINS,
)}]) != 0 or position(utm_medium, 'mail') > 0 then 'email'
when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
)}]) != 0 or position(lower(utm_medium), 'mail') > 0 then 'email'
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
SHOPPING_DOMAINS,
)}]) != 0 or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping')
when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
)}]) != 0 or position(lower(utm_medium), 'shop') > 0 then concat(prefix, 'Shopping')
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
VIDEO_DOMAINS,
)}]) != 0 or position(utm_medium, 'video') > 0 then concat(prefix, 'Video')
else '' end AS name,
)}]) != 0 or position(lower(utm_medium), 'video') > 0 then concat(prefix, 'Video')
when referrer_domain != hostname and referrer_domain != '' then 'referral'
else '' end AS "name",
session_id,
visit_id,
count(*) c,
@@ -166,9 +173,10 @@ async function clickhouseQuery(
max(created_at) max_time
from website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
and name != ''
${filterQuery}
group by prefix, name, session_id, visit_id
src/queries/sql/getChannelMetrics.ts
+31 -24
View File
@@ -22,10 +22,11 @@ export async function getChannelMetrics(...args: [websiteId: string, filters?: Q
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { rawQuery, parseFilters } = prisma;
const { queryParams, filterQuery, joinSessionQuery, cohortQuery, dateQuery } = parseFilters({
...filters,
websiteId,
});
const { queryParams, filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, dateQuery } =
parseFilters({
...filters,
websiteId,
});
return rawQuery(
`
@@ -38,12 +39,14 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
website_event.url_query,
website_event.utm_medium,
website_event.utm_source,
website_event.session_id
website_event.session_id,
website_event.hostname
from website_event
${cohortQuery}
${excludeBounceQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${dateQuery}
${filterQuery}),
@@ -59,6 +62,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
when ${toPostgresLikeClause('referrer_domain', EMAIL_DOMAINS)} or utm_medium ilike '%mail%' then 'email'
when ${toPostgresLikeClause('referrer_domain', SHOPPING_DOMAINS)} or utm_medium ilike '%shop%' then concat(prefix, 'Shopping')
when ${toPostgresLikeClause('referrer_domain', VIDEO_DOMAINS)} or utm_medium ilike '%video%' then concat(prefix, 'Video')
when referrer_domain != regexp_replace(hostname, '^www.', '') and referrer_domain != '' then 'referral'
else '' end AS x,
count(distinct session_id) y
from prefix
@@ -81,43 +85,46 @@ async function clickhouseQuery(
filters: QueryFilters,
): Promise<{ x: string; y: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { queryParams, filterQuery, cohortQuery, dateQuery } = parseFilters({
const { queryParams, filterQuery, cohortQuery, excludeBounceQuery, dateQuery } = parseFilters({
...filters,
websiteId,
});
const sql = `
WITH channels as (
select case when multiSearchAny(utm_medium, ['cp', 'ppc', 'retargeting', 'paid']) != 0 then 'paid' else 'organic' end prefix,
case
select
case when multiSearchAny(lower(utm_medium), ['cp', 'ppc', 'retargeting', 'paid']) != 0 then 'paid' else 'organic' end prefix,
case
when referrer_domain = '' and url_query = '' then 'direct'
when multiSearchAny(url_query, [${toClickHouseStringArray(
when multiSearchAny(lower(url_query), [${toClickHouseStringArray(
PAID_AD_PARAMS,
)}]) != 0 then 'paidAds'
when multiSearchAny(utm_medium, ['referral', 'app','link']) != 0 then 'referral'
when position(utm_medium, 'affiliate') > 0 then 'affiliate'
when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms'
when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
when multiSearchAny(lower(utm_medium), ['referral', 'app','link']) != 0 then 'referral'
when position(lower(utm_medium), 'affiliate') > 0 then 'affiliate'
when position(lower(utm_medium), 'sms') > 0 or position(lower(utm_source), 'sms') > 0 then 'sms'
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
SEARCH_DOMAINS,
)}]) != 0 or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search')
when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
)}]) != 0 or position(lower(utm_medium), 'organic') > 0 then concat(prefix, 'Search')
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
SOCIAL_DOMAINS,
)}]) != 0 then concat(prefix, 'Social')
when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
EMAIL_DOMAINS,
)}]) != 0 or position(utm_medium, 'mail') > 0 then 'email'
when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
)}]) != 0 or position(lower(utm_medium), 'mail') > 0 then 'email'
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
SHOPPING_DOMAINS,
)}]) != 0 or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping')
when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
)}]) != 0 or position(lower(utm_medium), 'shop') > 0 then concat(prefix, 'Shopping')
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
VIDEO_DOMAINS,
)}]) != 0 or position(utm_medium, 'video') > 0 then concat(prefix, 'Video')
else '' end AS x,
)}]) != 0 or position(lower(utm_medium), 'video') > 0 then concat(prefix, 'Video')
when referrer_domain != hostname and referrer_domain != '' then 'referral'
else '' end AS x,
count(distinct session_id) y
from website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and event_type != 2
and event_type NOT IN (2, 5)
${dateQuery}
${filterQuery}
group by 1, 2
src/queries/sql/getRealtimeActivity.ts
+4 -2
View File
@@ -30,7 +30,8 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
session.device,
session.country,
website_event.url_path as "urlPath",
website_event.referrer_domain as "referrerDomain"
website_event.referrer_domain as "referrerDomain",
website_event.hostname
from website_event
${cohortQuery}
inner join session
@@ -65,7 +66,8 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters): Promis
device,
country,
url_path as urlPath,
referrer_domain as referrerDomain
referrer_domain as referrerDomain,
hostname
from website_event
${cohortQuery}
where website_id = {websiteId:UUID}
src/queries/sql/getValues.ts
+1 -1
View File
@@ -23,7 +23,7 @@ async function relationalQuery(websiteId: string, column: string, filters: Query
let excludeDomain = '';
if (column === 'referrer_domain') {
excludeDomain = `and website_event.referrer_domain != website_event.hostname
excludeDomain = `and website_event.referrer_domain != regexp_replace(website_event.hostname, '^www.', '')
and website_event.referrer_domain != ''`;
}
src/queries/sql/getWebsiteStats.ts
+21 -11
View File
@@ -28,10 +28,14 @@ async function relationalQuery(
filters: QueryFilters,
): Promise<WebsiteStatsData[]> {
const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
const { filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, queryParams } =
parseFilters({
...filters,
websiteId,
});
const { excludeBounce } = filters;
const bounceQuery = excludeBounce ? '0' : 'coalesce(sum(case when t.c = 1 then 1 else 0 end), 0)';
return rawQuery(
`
@@ -39,7 +43,7 @@ async function relationalQuery(
cast(coalesce(sum(t.c), 0) as bigint) as "pageviews",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
coalesce(sum(case when t.c = 1 then 1 else 0 end), 0) as "bounces",
${bounceQuery} as "bounces",
cast(coalesce(sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}), 0) as bigint) as "totaltime"
from (
select
@@ -50,12 +54,14 @@ async function relationalQuery(
max(website_event.created_at) as "max_time"
from website_event
${cohortQuery}
${excludeBounceQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by 1, 2
) as t
`,
queryParams,
@@ -68,12 +74,14 @@ async function clickhouseQuery(
filters: QueryFilters,
): Promise<WebsiteStatsData[]> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
let sql = '';
const { excludeBounce } = filters;
const bounceQuery = excludeBounce ? '0' : 'sumIf(1, t.c = 1)';
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
sql = `
@@ -81,7 +89,7 @@ async function clickhouseQuery(
sum(t.c) as "pageviews",
uniq(t.session_id) as "visitors",
uniq(t.visit_id) as "visits",
sum(if(t.c = 1, 1, 0)) as "bounces",
${bounceQuery} as "bounces",
sum(max_time-min_time) as "totaltime"
from (
select
@@ -92,9 +100,10 @@ async function clickhouseQuery(
max(created_at) max_time
from website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by session_id, visit_id
) as t;
@@ -105,7 +114,7 @@ async function clickhouseQuery(
sum(t.c) as "pageviews",
uniq(session_id) as "visitors",
uniq(visit_id) as "visits",
sumIf(1, t.c = 1) as "bounces",
${bounceQuery} as "bounces",
sum(max_time-min_time) as "totaltime"
from (select
session_id,
@@ -115,9 +124,10 @@ async function clickhouseQuery(
max(max_time) max_time
from website_event_stats_hourly "website_event"
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by session_id, visit_id
) as t;
src/queries/sql/getWeeklyTraffic.ts
+17 -7
View File
@@ -14,12 +14,13 @@ export async function getWeeklyTraffic(...args: [websiteId: string, filters: Que
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const timezone = 'utc';
const { timezone = 'utc' } = filters;
const { rawQuery, getDateWeeklySQL, parseFilters } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
const { filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, queryParams } =
parseFilters({
...filters,
websiteId,
});
return rawQuery(
`
@@ -28,12 +29,14 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
count(distinct website_event.session_id) as value
from website_event
${cohortQuery}
${excludeBounceQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by time
order by 2
order by 1
`,
queryParams,
FUNCTION_NAME,
@@ -43,7 +46,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc' } = filters;
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = await parseFilters({ ...filters, websiteId });
const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = await parseFilters({
...filters,
websiteId,
});
let sql = '';
@@ -54,8 +60,10 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(distinct session_id) as value
from website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type NOT IN (2, 5)
${filterQuery}
group by time
order by time
@@ -67,8 +75,10 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(distinct session_id) as value
from website_event_stats_hourly website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type NOT IN (2, 5)
${filterQuery}
group by time
order by time
src/queries/sql/index.ts
+5
View File
@@ -1,3 +1,4 @@
export * from './events/getEventDataById';
export * from './events/getEventDataEvents';
export * from './events/getEventDataFields';
export * from './events/getEventDataProperties';
@@ -22,6 +23,10 @@ export * from './getWeeklyTraffic';
export * from './pageviews/getPageviewExpandedMetrics';
export * from './pageviews/getPageviewMetrics';
export * from './pageviews/getPageviewStats';
export * from './performance/getPerformanceStats';
export * from './replays/getReplayChunks';
export * from './replays/getSessionReplays';
export * from './replays/saveRecording';
export * from './reports/getBreakdown';
export * from './reports/getFunnel';
export * from './reports/getJourney';
src/queries/sql/pageviews/getPageviewExpandedMetrics.ts
+17 -14
View File
@@ -38,19 +38,20 @@ async function relationalQuery(
const { type, limit = 500, offset = 0 } = parameters;
let column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters, getTimestampDiffSQL } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(
{
...filters,
websiteId,
},
{ joinSession: SESSION_COLUMNS.includes(type) },
);
const { filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, queryParams } =
parseFilters(
{
...filters,
websiteId,
},
{ joinSession: SESSION_COLUMNS.includes(type) },
);
let entryExitQuery = '';
let excludeDomain = '';
if (column === 'referrer_domain') {
excludeDomain = `and website_event.referrer_domain != website_event.hostname
excludeDomain = `and website_event.referrer_domain != regexp_replace(website_event.hostname, '^www.', '')
and website_event.referrer_domain != ''`;
if (type === 'domain') {
column = toPostgresGroupedReferrer(GROUPED_DOMAINS);
@@ -67,7 +68,7 @@ async function relationalQuery(
from website_event
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
group by visit_id
) x
on x.visit_id = website_event.visit_id
@@ -86,7 +87,7 @@ async function relationalQuery(
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime"
from (
select
${column} as name,
${column} as "name",
website_event.session_id,
website_event.visit_id,
count(*) as "c",
@@ -94,11 +95,12 @@ async function relationalQuery(
max(website_event.created_at) as "max_time"
from website_event
${cohortQuery}
${excludeBounceQuery}
${joinSessionQuery}
${entryExitQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${excludeDomain}
${filterQuery}
group by ${column}, website_event.session_id, website_event.visit_id
@@ -122,7 +124,7 @@ async function clickhouseQuery(
const { type, limit = 500, offset = 0 } = parameters;
let column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
@@ -147,7 +149,7 @@ async function clickhouseQuery(
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
group by visit_id) x
ON x.visit_id = website_event.visit_id`;
}
@@ -171,10 +173,11 @@ async function clickhouseQuery(
max(created_at) max_time
from website_event
${cohortQuery}
${excludeBounceQuery}
${entryExitQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
and name != ''
${excludeDomain}
${filterQuery}
src/queries/sql/pageviews/getPageviewMetrics.ts
+18 -14
View File
@@ -34,19 +34,20 @@ async function relationalQuery(
const { type, limit = 500, offset = 0 } = parameters;
let column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(
{
...filters,
websiteId,
},
{ joinSession: SESSION_COLUMNS.includes(type) },
);
const { filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, queryParams } =
parseFilters(
{
...filters,
websiteId,
},
{ joinSession: SESSION_COLUMNS.includes(type) },
);
let entryExitQuery = '';
let excludeDomain = '';
if (column === 'referrer_domain') {
excludeDomain = `and website_event.referrer_domain != website_event.hostname
excludeDomain = `and website_event.referrer_domain != regexp_replace(website_event.hostname, '^www.', '')
and website_event.referrer_domain != ''`;
}
@@ -62,7 +63,7 @@ async function relationalQuery(
from website_event
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
order by visit_id, created_at ${order}
) x
on x.visit_id = website_event.visit_id
@@ -75,11 +76,12 @@ async function relationalQuery(
count(distinct website_event.session_id) as y
from website_event
${cohortQuery}
${excludeBounceQuery}
${joinSessionQuery}
${entryExitQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${excludeDomain}
${filterQuery}
group by 1
@@ -100,7 +102,7 @@ async function clickhouseQuery(
const { type, limit = 500, offset = 0 } = parameters;
let column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
@@ -125,7 +127,7 @@ async function clickhouseQuery(
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
group by visit_id) x
ON x.visit_id = website_event.visit_id`;
}
@@ -135,10 +137,11 @@ async function clickhouseQuery(
uniq(website_event.session_id) as y
from website_event
${cohortQuery}
${excludeBounceQuery}
${entryExitQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${excludeDomain}
${filterQuery}
group by x
@@ -174,9 +177,10 @@ async function clickhouseQuery(
${columnQuery} as t
from website_event_stats_hourly as website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${excludeDomain}
${filterQuery}
${groupByQuery}) as g
src/queries/sql/pageviews/getPageviewStats.ts
+12 -8
View File
@@ -16,10 +16,11 @@ export async function getPageviewStats(...args: [websiteId: string, filters: Que
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc', unit = 'day' } = filters;
const { getDateSQL, parseFilters, rawQuery } = prisma;
const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
const { filterQuery, cohortQuery, excludeBounceQuery, joinSessionQuery, queryParams } =
parseFilters({
...filters,
websiteId,
});
return rawQuery(
`
@@ -28,10 +29,11 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
count(*) y
from website_event
${cohortQuery}
${excludeBounceQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by 1
order by 1
@@ -47,7 +49,7 @@ async function clickhouseQuery(
): Promise<{ x: string; y: number }[]> {
const { timezone = 'UTC', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateSQL } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
@@ -65,9 +67,10 @@ async function clickhouseQuery(
count(*) as y
from website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by t
) as g
@@ -84,9 +87,10 @@ async function clickhouseQuery(
sum(views) as y
from website_event_stats_hourly as website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by t
) as g
src/queries/sql/performance/getPerformanceStats.ts
+82
View File
@@ -0,0 +1,82 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
export interface PerformanceStatsResult {
lcp: number;
inp: number;
cls: number;
fcp: number;
ttfb: number;
count: number;
}
export async function getPerformanceStats(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: QueryFilters,
): Promise<PerformanceStatsResult> {
const { rawQuery, parseFilters } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
const result = await rawQuery(
`
select
percentile_cont(0.75) within group (order by lcp) as lcp,
percentile_cont(0.75) within group (order by inp) as inp,
percentile_cont(0.75) within group (order by cls) as cls,
percentile_cont(0.75) within group (order by fcp) as fcp,
percentile_cont(0.75) within group (order by ttfb) as ttfb,
count(*) as count
from website_event
${cohortQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.event_type = 5
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
`,
queryParams,
);
return result?.[0] || { lcp: 0, inp: 0, cls: 0, fcp: 0, ttfb: 0, count: 0 };
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<PerformanceStatsResult> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId });
const result = await rawQuery<PerformanceStatsResult>(
`
select
quantile(0.75)(lcp) as lcp,
quantile(0.75)(inp) as inp,
quantile(0.75)(cls) as cls,
quantile(0.75)(fcp) as fcp,
quantile(0.75)(ttfb) as ttfb,
count() as count
from website_event
${cohortQuery}
where website_event.website_id = {websiteId:UUID}
and website_event.event_type = 5
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
`,
queryParams,
);
return result?.[0] || { lcp: 0, inp: 0, cls: 0, fcp: 0, ttfb: 0, count: 0 };
}
src/queries/sql/replays/getReplayChunks.ts
+102
View File
@@ -0,0 +1,102 @@
import { gunzipSync } from 'node:zlib';
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
const FUNCTION_NAME = 'getReplayChunks';
export interface ReplayChunk {
sessionId: string;
visitId: string;
events: any[];
chunkIndex: number;
eventCount: number;
startedAt: Date;
endedAt: Date;
}
export async function getReplayChunks(websiteId: string, visitId: string): Promise<ReplayChunk[]> {
return runQuery({
[PRISMA]: () => relationalQuery(websiteId, visitId),
[CLICKHOUSE]: () => clickhouseQuery(websiteId, visitId),
});
}
async function relationalQuery(websiteId: string, visitId: string): Promise<ReplayChunk[]> {
const { rawQuery } = prisma;
const chunks: {
sessionId: string;
visitId: string;
events: Buffer;
chunkIndex: number;
eventCount: number;
startedAt: Date;
endedAt: Date;
}[] = await rawQuery(
`
select
session_id as "sessionId",
visit_id as "visitId",
events,
chunk_index as "chunkIndex",
event_count as "eventCount",
started_at as "startedAt",
ended_at as "endedAt"
from session_replay
where website_id = {{websiteId::uuid}}
and visit_id = {{visitId::uuid}}
order by chunk_index asc
`,
{ websiteId, visitId },
FUNCTION_NAME,
);
return chunks.map(chunk => ({
...chunk,
events: JSON.parse(gunzipSync(Buffer.from(chunk.events)).toString('utf-8')),
}));
}
async function clickhouseQuery(websiteId: string, visitId: string): Promise<ReplayChunk[]> {
const { rawQuery } = clickhouse;
const results = await rawQuery<
{
sessionId: string;
visitId: string;
events: string;
chunk_index: number;
event_count: number;
started_at: string;
ended_at: string;
}[]
>(
`
select
session_id as sessionId,
visit_id as visitId,
events,
chunk_index,
event_count,
started_at,
ended_at
from session_replay
where website_id = {websiteId:UUID}
and visit_id = {visitId:UUID}
order by chunk_index asc
`,
{ websiteId, visitId },
FUNCTION_NAME,
);
return results.map(row => ({
sessionId: row.sessionId,
visitId: row.visitId,
events: JSON.parse(row.events),
chunkIndex: row.chunk_index,
eventCount: row.event_count,
startedAt: new Date(row.started_at),
endedAt: new Date(row.ended_at),
}));
}
src/queries/sql/replays/getSessionReplays.ts
+148
View File
@@ -0,0 +1,148 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
const FUNCTION_NAME = 'getSessionReplays';
export function getSessionReplays(
...args: [websiteId: string, filters: QueryFilters, sessionId?: string]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters, sessionId?: string) {
const { pagedRawQuery, parseFilters } = prisma;
const { search } = filters;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
search: search ? `%${search}%` : undefined,
});
const joinQuery =
filterQuery || cohortQuery
? `join (select distinct website_id, session_id, visit_id
from website_event
${cohortQuery}
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
${filterQuery}) website_event
on website_event.website_id = sr.website_id
and website_event.session_id = sr.session_id
and website_event.visit_id = sr.visit_id`
: '';
const sessionFilter = sessionId ? 'and sr.session_id = {{sessionId::uuid}}' : '';
const searchQuery = search
? `and (session.distinct_id ilike {{search}}
or session.city ilike {{search}}
or session.browser ilike {{search}}
or session.os ilike {{search}}
or session.device ilike {{search}})`
: '';
return pagedRawQuery(
`
select
sr.visit_id as "id",
sr.session_id as "sessionId",
sr.website_id as "websiteId",
session.browser,
session.os,
session.device,
session.country,
session.city,
sum(sr.event_count) as "eventCount",
count(sr.replay_id) as "chunkCount",
min(sr.started_at) as "startedAt",
max(sr.ended_at) as "endedAt",
sum(extract(epoch from sr.ended_at - sr.started_at) * 1000)::bigint as "duration",
max(sr.created_at) as "createdAt"
from session_replay sr
join session on session.session_id = sr.session_id
and session.website_id = sr.website_id
${joinQuery}
where sr.website_id = {{websiteId::uuid}}
and sr.created_at between {{startDate}} and {{endDate}}
${sessionFilter}
${searchQuery}
group by sr.visit_id,
sr.session_id,
sr.website_id,
session.browser,
session.os,
session.device,
session.country,
session.city
order by max(sr.created_at) desc
`,
{ ...queryParams, sessionId },
filters,
FUNCTION_NAME,
);
}
async function clickhouseQuery(websiteId: string, filters: QueryFilters, sessionId?: string) {
const { pagedRawQuery, parseFilters } = clickhouse;
const { search } = filters;
const { queryParams, cohortQuery, filterQuery } = parseFilters({
...filters,
websiteId,
});
const sessionFilter = sessionId ? 'and session_replay.session_id = {sessionId:UUID}' : '';
const searchQuery = search
? `and ((positionCaseInsensitive(distinct_id, {search:String}) > 0)
or (positionCaseInsensitive(city, {search:String}) > 0)
or (positionCaseInsensitive(browser, {search:String}) > 0)
or (positionCaseInsensitive(os, {search:String}) > 0)
or (positionCaseInsensitive(device, {search:String}) > 0))`
: '';
return pagedRawQuery(
`
select
session_replay.visit_id as id,
session_replay.session_id as sessionId,
session_replay.website_id as websiteId,
website_event.browser,
website_event.os,
website_event.device,
website_event.country,
website_event.city,
sum(session_replay.event_count) as eventCount,
count(session_replay.replay_id) as chunkCount,
min(session_replay.started_at) as startedAt,
max(session_replay.ended_at) as endedAt,
toInt64(sum(dateDiff('millisecond', session_replay.started_at, session_replay.ended_at))) as duration,
max(session_replay.created_at) as createdAt
from session_replay
join (
select distinct website_id, session_id, visit_id, browser, os, device, country, city
from website_event
${cohortQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
${searchQuery}
) website_event
on website_event.session_id = session_replay.session_id
and website_event.website_id = session_replay.website_id
and website_event.visit_id = session_replay.visit_id
where session_replay.website_id = {websiteId:UUID}
and session_replay.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${sessionFilter}
group by session_replay.visit_id, session_replay.session_id, session_replay.website_id, website_event.browser, website_event.os, website_event.device, website_event.country, website_event.city
order by max(created_at) desc
`,
{ ...queryParams, sessionId },
filters,
FUNCTION_NAME,
);
}
src/queries/sql/replays/saveRecording.ts
+83
View File
@@ -0,0 +1,83 @@
import { gzipSync } from 'node:zlib';
import clickhouse from '@/lib/clickhouse';
import { uuid } from '@/lib/crypto';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import kafka from '@/lib/kafka';
import prisma from '@/lib/prisma';
export interface SaveRecordingArgs {
websiteId: string;
sessionId: string;
visitId: string;
chunkIndex: number;
events: any[];
eventCount: number;
startedAt: Date;
endedAt: Date;
}
export async function saveRecording(args: SaveRecordingArgs) {
return runQuery({
[PRISMA]: () => relationalQuery(args),
[CLICKHOUSE]: () => clickhouseQuery(args),
});
}
async function relationalQuery({
websiteId,
sessionId,
visitId,
chunkIndex,
events,
eventCount,
startedAt,
endedAt,
}: SaveRecordingArgs) {
const compressed = gzipSync(Buffer.from(JSON.stringify(events), 'utf-8'));
return prisma.client.sessionReplay.create({
data: {
id: uuid(),
websiteId,
sessionId,
visitId,
chunkIndex,
events: compressed as any,
eventCount,
startedAt,
endedAt,
},
});
}
async function clickhouseQuery({
websiteId,
sessionId,
visitId,
chunkIndex,
events,
eventCount,
startedAt,
endedAt,
}: SaveRecordingArgs) {
const { insert, getUTCString } = clickhouse;
const { sendMessage } = kafka;
const message = {
replay_id: uuid(),
website_id: websiteId,
session_id: sessionId,
visit_id: visitId,
chunk_index: chunkIndex,
events: JSON.stringify(events),
event_count: eventCount,
started_at: getUTCString(startedAt),
ended_at: getUTCString(endedAt),
};
if (kafka.enabled) {
return sendMessage('session_replay', message);
}
return insert('session_replay', [message]);
}
src/queries/sql/reports/getAttribution.ts
+82 -123
View File
@@ -10,7 +10,6 @@ export interface AttributionParameters {
model: string;
type: string;
step: string;
currency?: string;
}
export interface AttributionResult {
@@ -38,7 +37,7 @@ async function relationalQuery(
parameters: AttributionParameters,
filters: QueryFilters,
): Promise<AttributionResult> {
const { model, type, currency } = parameters;
const { model, type } = parameters;
const { rawQuery, parseFilters } = prisma;
const eventType = type === 'path' ? EVENT_TYPE.pageView : EVENT_TYPE.customEvent;
const column = type === 'path' ? 'url_path' : 'event_name';
@@ -51,17 +50,16 @@ async function relationalQuery(
function getUTMQuery(utmColumn: string) {
return `
select
coalesce(we.${utmColumn}, '') name,
${currency ? 'sum(e.value)' : 'count(distinct we.session_id)'} value
select
coalesce(we.${utmColumn}, '') as "name",
count(distinct we.session_id) as "value"
from model m
join website_event we
on we.created_at = m.created_at
and we.session_id = m.session_id
${currency ? 'join events e on e.session_id = m.session_id' : ''}
where we.website_id = {{websiteId::uuid}}
and we.created_at between {{startDate}} and {{endDate}}
${currency ? '' : `and we.${utmColumn} != ''`}
and we.${utmColumn} != ''
group by 1
order by 2 desc
limit 20`;
@@ -80,30 +78,9 @@ async function relationalQuery(
${filterQuery}
group by 1),`;
const revenueEventQuery = `WITH events AS (
select
revenue.session_id,
max(revenue.created_at) max_dt,
sum(revenue.revenue) value
from revenue
join website_event
on website_event.website_id = revenue.website_id
and website_event.session_id = revenue.session_id
and website_event.event_id = revenue.event_id
and website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
${cohortQuery}
${joinSessionQuery}
where revenue.website_id = {{websiteId::uuid}}
and revenue.created_at between {{startDate}} and {{endDate}}
and revenue.${column} = {{step}}
and revenue.currency = {{currency}}
${filterQuery}
group by 1),`;
function getModelQuery(model: string) {
return model === 'first-click'
? `\n
? `\n
model AS (select e.session_id,
min(we.created_at) created_at
from events e
@@ -112,39 +89,34 @@ async function relationalQuery(
where we.website_id = {{websiteId::uuid}}
and we.created_at between {{startDate}} and {{endDate}}
group by e.session_id)`
: `\n
: `\n
model AS (select e.session_id,
max(we.created_at) created_at
from events e
join website_event we
on we.session_id = e.session_id
where we.website_id = {{websiteId::uuid}}
and we.created_at between {{startDate}} and {{endDate}}
and we.created_at between {{startDate}} and {{endDate}}
and we.created_at < e.max_dt
group by e.session_id)`;
}
const referrerRes = await rawQuery(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
select coalesce(we.referrer_domain, '') name,
${currency ? 'sum(e.value)' : 'count(distinct we.session_id)'} value
select coalesce(we.referrer_domain, '') as "name",
count(distinct we.session_id) value
from model m
join website_event we
on we.created_at = m.created_at
and we.session_id = m.session_id
join session s
on s.session_id = m.session_id
${currency ? 'join events e on e.session_id = m.session_id' : ''}
where we.website_id = {{websiteId::uuid}}
and we.created_at between {{startDate}} and {{endDate}}
${
currency
? ''
: `and we.referrer_domain != hostname
and we.referrer_domain != ''`
}
and we.referrer_domain != regexp_replace(we.hostname, '^www.', '')
and we.referrer_domain != ''
group by 1
order by 2 desc
limit 20
@@ -154,40 +126,39 @@ async function relationalQuery(
const paidAdsres = await rawQuery(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)},
results AS (
select case
when coalesce(gclid, '') != '' then 'Google Ads'
when coalesce(fbclid, '') != '' then 'Facebook / Meta'
when coalesce(msclkid, '') != '' then 'Microsoft Ads'
when coalesce(ttclid, '') != '' then 'TikTok Ads'
when coalesce(li_fat_id, '') != '' then 'LinkedIn Ads'
when coalesce(gclid, '') != '' then 'Google Ads'
when coalesce(fbclid, '') != '' then 'Facebook / Meta'
when coalesce(msclkid, '') != '' then 'Microsoft Ads'
when coalesce(ttclid, '') != '' then 'TikTok Ads'
when coalesce(li_fat_id, '') != '' then 'LinkedIn Ads'
when coalesce(twclid, '') != '' then 'Twitter Ads (X)'
else ''
end name,
${currency ? 'sum(e.value)' : 'count(distinct we.session_id)'} value
end as "name",
count(distinct we.session_id) as "value"
from model m
join website_event we
on we.created_at = m.created_at
and we.session_id = m.session_id
${currency ? 'join events e on e.session_id = m.session_id' : ''}
where we.website_id = {{websiteId::uuid}}
and we.created_at between {{startDate}} and {{endDate}}
group by 1
order by 2 desc
limit 20)
SELECT *
SELECT *
FROM results
${currency ? '' : `WHERE name != ''`}
WHERE name != ''
`,
queryParams,
);
const sourceRes = await rawQuery(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
${getUTMQuery('utm_source')}
`,
@@ -196,7 +167,7 @@ async function relationalQuery(
const mediumRes = await rawQuery(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
${getUTMQuery('utm_medium')}
`,
@@ -205,7 +176,7 @@ async function relationalQuery(
const campaignRes = await rawQuery(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
${getUTMQuery('utm_campaign')}
`,
@@ -214,7 +185,7 @@ async function relationalQuery(
const contentRes = await rawQuery(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
${getUTMQuery('utm_content')}
`,
@@ -223,7 +194,7 @@ async function relationalQuery(
const termRes = await rawQuery(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
${getUTMQuery('utm_term')}
`,
@@ -232,7 +203,7 @@ async function relationalQuery(
const totalRes = await rawQuery(
`
select
select
count(*) as "pageviews",
count(distinct website_event.session_id) as "visitors",
count(distinct website_event.visit_id) as "visits"
@@ -264,7 +235,7 @@ async function clickhouseQuery(
parameters: AttributionParameters,
filters: QueryFilters,
): Promise<AttributionResult> {
const { model, type, currency } = parameters;
const { model, type } = parameters;
const { rawQuery, parseFilters } = clickhouse;
const eventType = type === 'path' ? EVENT_TYPE.pageView : EVENT_TYPE.customEvent;
const column = type === 'path' ? 'url_path' : 'event_name';
@@ -277,17 +248,19 @@ async function clickhouseQuery(
function getUTMQuery(utmColumn: string) {
return `
select
select
we.${utmColumn} name,
${currency ? 'sum(e.value)' : 'uniqExact(we.session_id)'} value
uniqExact(we.session_id) value
from model m
join website_event we
join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
) we
on we.created_at = m.created_at
and we.session_id = m.session_id
${currency ? 'join events e on e.session_id = m.session_id' : ''}
where we.website_id = {websiteId:UUID}
and we.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${currency ? '' : `and we.${utmColumn} != ''`}
where we.${utmColumn} != ''
group by 1
order by 2 desc
limit 20
@@ -296,14 +269,17 @@ async function clickhouseQuery(
function getModelQuery(model: string) {
if (model === 'first-click') {
return `
return `
model AS (select e.session_id,
min(we.created_at) created_at
from events e
join website_event we
join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
) we
on we.session_id = e.session_id
where we.website_id = {websiteId:UUID}
and we.created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by e.session_id)
`;
}
@@ -312,11 +288,14 @@ async function clickhouseQuery(
model AS (select e.session_id,
max(we.created_at) created_at
from events e
join website_event we
join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
) we
on we.session_id = e.session_id
where we.website_id = {websiteId:UUID}
and we.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and we.created_at < e.max_dt
where we.created_at < e.max_dt
group by e.session_id)
`;
}
@@ -333,26 +312,6 @@ async function clickhouseQuery(
${filterQuery}
group by 1),`;
const revenueEventQuery = `WITH events AS (
select
website_revenue.session_id,
max(website_revenue.created_at) max_dt,
sum(website_revenue.revenue) as value
from website_revenue
join website_event
on website_event.website_id = website_revenue.website_id
and website_event.session_id = website_revenue.session_id
and website_event.event_id = website_revenue.event_id
and website_event.website_id = {websiteId:UUID}
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${cohortQuery}
where website_revenue.website_id = {websiteId:UUID}
and website_revenue.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and website_revenue.${column} = {step:String}
and website_revenue.currency = {currency:String}
${filterQuery}
group by 1),`;
const referrerRes = await rawQuery<
{
name: string;
@@ -360,23 +319,21 @@ async function clickhouseQuery(
}[]
>(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
select we.referrer_domain name,
${currency ? 'sum(e.value)' : 'uniqExact(we.session_id)'} value
uniqExact(we.session_id) value
from model m
join website_event we
join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
) we
on we.created_at = m.created_at
and we.session_id = m.session_id
${currency ? 'join events e on e.session_id = m.session_id' : ''}
where we.website_id = {websiteId:UUID}
and we.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${
currency
? ''
: `and we.referrer_domain != hostname
and we.referrer_domain != ''`
}
where we.referrer_domain != hostname
and we.referrer_domain != ''
group by 1
order by 2 desc
limit 20
@@ -391,23 +348,25 @@ async function clickhouseQuery(
}[]
>(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
select multiIf(gclid != '', 'Google Ads',
fbclid != '', 'Facebook / Meta',
msclkid != '', 'Microsoft Ads',
ttclid != '', 'TikTok Ads',
li_fat_id != '', 'LinkedIn Ads',
select multiIf(gclid != '', 'Google Ads',
fbclid != '', 'Facebook / Meta',
msclkid != '', 'Microsoft Ads',
ttclid != '', 'TikTok Ads',
li_fat_id != '', 'LinkedIn Ads',
twclid != '', 'Twitter Ads (X)','') name,
${currency ? 'sum(e.value)' : 'uniqExact(we.session_id)'} value
uniqExact(we.session_id) value
from model m
join website_event we
join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
) we
on we.created_at = m.created_at
and we.session_id = m.session_id
${currency ? 'join events e on e.session_id = m.session_id' : ''}
where we.website_id = {websiteId:UUID}
and we.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${currency ? '' : `and name != ''`}
where name != ''
group by 1
order by 2 desc
limit 20
@@ -422,7 +381,7 @@ async function clickhouseQuery(
}[]
>(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
${getUTMQuery('utm_source')}
`,
@@ -436,7 +395,7 @@ async function clickhouseQuery(
}[]
>(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
${getUTMQuery('utm_medium')}
`,
@@ -450,7 +409,7 @@ async function clickhouseQuery(
}[]
>(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
${getUTMQuery('utm_campaign')}
`,
@@ -464,7 +423,7 @@ async function clickhouseQuery(
}[]
>(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
${getUTMQuery('utm_content')}
`,
@@ -478,7 +437,7 @@ async function clickhouseQuery(
}[]
>(
`
${currency ? revenueEventQuery : eventQuery}
${eventQuery}
${getModelQuery(model)}
${getUTMQuery('utm_term')}
`,
src/queries/sql/reports/getBreakdown.ts
+3 -3
View File
@@ -71,7 +71,7 @@ async function relationalQuery(
website_event.session_id, website_event.visit_id
) as t
group by ${parseFieldsByName(fields)}
order by 1 desc, 2 desc
order by 2 desc, 1 desc
limit 500
`,
queryParams,
@@ -119,7 +119,7 @@ async function clickhouseQuery(
session_id, visit_id
) as t
group by ${parseFieldsByName(fields)}
order by 1 desc, 2 desc
order by 2 desc, 1 desc
limit 500
`,
queryParams,
@@ -131,5 +131,5 @@ function parseFields(fields: string[]) {
}
function parseFieldsByName(fields: string[]) {
return `${fields.map(name => name).join(',')}`;
return `${fields.map(name => `"${name}"`).join(',')}`;
}
src/queries/sql/reports/getGoal.ts
+34 -8
View File
@@ -9,8 +9,6 @@ export interface GoalParameters {
endDate: Date;
type: string;
value: string;
operator?: string;
property?: string;
}
export async function getGoal(
@@ -31,15 +29,29 @@ async function relationalQuery(
const { rawQuery, parseFilters } = prisma;
const eventType = type === 'path' ? EVENT_TYPE.pageView : EVENT_TYPE.customEvent;
const column = type === 'path' ? 'url_path' : 'event_name';
let operator = '=';
let paramValue = value;
if (value.startsWith('*') || value.endsWith('*')) {
operator = 'like';
paramValue = value.replace(/^\*|\*$/g, '%');
}
const { filterQuery, dateQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
value,
value: paramValue,
startDate,
endDate,
eventType,
});
const excludeEventTypeFilterQuery = filterQuery
.split('\n')
.filter(filter => !filter.includes('event_type'))
.join('\n')
.trim();
return rawQuery(
`
select count(distinct website_event.session_id) as num,
@@ -50,13 +62,13 @@ async function relationalQuery(
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
${dateQuery}
${filterQuery}
${excludeEventTypeFilterQuery}
) as total
from website_event
${cohortQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and ${column} = {{value}}
and ${column} ${operator} {{value}}
${dateQuery}
${filterQuery}
`,
@@ -73,15 +85,29 @@ async function clickhouseQuery(
const { rawQuery, parseFilters } = clickhouse;
const eventType = type === 'path' ? EVENT_TYPE.pageView : EVENT_TYPE.customEvent;
const column = type === 'path' ? 'url_path' : 'event_name';
let operator = '=';
let paramValue = value;
if (value.startsWith('*') || value.endsWith('*')) {
operator = 'like';
paramValue = value.replace(/^\*|\*$/g, '%');
}
const { filterQuery, dateQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
value,
value: paramValue,
startDate,
endDate,
eventType,
});
const excludeEventTypeFilterQuery = filterQuery
.split('\n')
.filter(filter => !filter.includes('event_type'))
.join('\n')
.trim();
return rawQuery(
`
select count(distinct session_id) as num,
@@ -91,12 +117,12 @@ async function clickhouseQuery(
${cohortQuery}
where website_id = {websiteId:UUID}
${dateQuery}
${filterQuery}
${excludeEventTypeFilterQuery}
) as total
from website_event
${cohortQuery}
where website_id = {websiteId:UUID}
and ${column} = {value:String}
and ${column} ${operator} {value:String}
${dateQuery}
${filterQuery}
`,
src/queries/sql/reports/getJourney.ts
+3 -3
View File
@@ -60,7 +60,7 @@ async function relationalQuery(
endStepQuery: string;
params: Record<string, string>;
} {
const params = {};
const params: { startStep?: string; endStep?: string } = {};
let sequenceQuery = '';
let startStepQuery = '';
let endStepQuery = '';
@@ -119,7 +119,7 @@ async function relationalQuery(
select distinct
website_event.visit_id,
website_event.referrer_path,
coalesce(nullIf(website_event.event_name, ''), website_event.url_path) event,
coalesce(nullIf(website_event.event_name, ''), website_event.url_path) "event",
row_number() OVER (PARTITION BY visit_id ORDER BY website_event.created_at) AS event_number
from website_event
${cohortQuery}
@@ -172,7 +172,7 @@ async function clickhouseQuery(
endStepQuery: string;
params: Record<string, string>;
} {
const params = {};
const params: { startStep?: string; endStep?: string } = {};
let sequenceQuery = '';
let startStepQuery = '';
let endStepQuery = '';
src/queries/sql/reports/getPerformance.ts
+216
View File
@@ -0,0 +1,216 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
export interface PerformanceParameters {
startDate: Date;
endDate: Date;
unit: string;
timezone: string;
metric: string;
}
export interface PerformanceResult {
chart: { t: string; p50: number; p75: number; p95: number }[];
summary: {
lcp: { p50: number; p75: number; p95: number };
inp: { p50: number; p75: number; p95: number };
cls: { p50: number; p75: number; p95: number };
fcp: { p50: number; p75: number; p95: number };
ttfb: { p50: number; p75: number; p95: number };
count: number;
};
}
export async function getPerformance(
...args: [websiteId: string, parameters: PerformanceParameters, filters: QueryFilters]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
parameters: PerformanceParameters,
filters: QueryFilters,
): Promise<PerformanceResult> {
const { startDate, endDate, unit = 'day', timezone = 'utc', metric = 'lcp' } = parameters;
const { getDateSQL, rawQuery, parseFilters } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
const chart = await rawQuery(
`
select
${getDateSQL('created_at', unit, timezone)} t,
percentile_cont(0.5) within group (order by ${metric}) as p50,
percentile_cont(0.75) within group (order by ${metric}) as p75,
percentile_cont(0.95) within group (order by ${metric}) as p95
from website_event
${cohortQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.event_type = 5
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by t
order by t
`,
{ ...queryParams, startDate, endDate },
);
const summaryResult = await rawQuery(
`
select
percentile_cont(0.5) within group (order by lcp) as lcp_p50,
percentile_cont(0.75) within group (order by lcp) as lcp_p75,
percentile_cont(0.95) within group (order by lcp) as lcp_p95,
percentile_cont(0.5) within group (order by inp) as inp_p50,
percentile_cont(0.75) within group (order by inp) as inp_p75,
percentile_cont(0.95) within group (order by inp) as inp_p95,
percentile_cont(0.5) within group (order by cls) as cls_p50,
percentile_cont(0.75) within group (order by cls) as cls_p75,
percentile_cont(0.95) within group (order by cls) as cls_p95,
percentile_cont(0.5) within group (order by fcp) as fcp_p50,
percentile_cont(0.75) within group (order by fcp) as fcp_p75,
percentile_cont(0.95) within group (order by fcp) as fcp_p95,
percentile_cont(0.5) within group (order by ttfb) as ttfb_p50,
percentile_cont(0.75) within group (order by ttfb) as ttfb_p75,
percentile_cont(0.95) within group (order by ttfb) as ttfb_p95,
count(*) as count
from website_event
${cohortQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.event_type = 5
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
`,
{ ...queryParams, startDate, endDate },
).then(result => result?.[0]);
const summary = {
lcp: {
p50: Number(summaryResult?.lcp_p50 || 0),
p75: Number(summaryResult?.lcp_p75 || 0),
p95: Number(summaryResult?.lcp_p95 || 0),
},
inp: {
p50: Number(summaryResult?.inp_p50 || 0),
p75: Number(summaryResult?.inp_p75 || 0),
p95: Number(summaryResult?.inp_p95 || 0),
},
cls: {
p50: Number(summaryResult?.cls_p50 || 0),
p75: Number(summaryResult?.cls_p75 || 0),
p95: Number(summaryResult?.cls_p95 || 0),
},
fcp: {
p50: Number(summaryResult?.fcp_p50 || 0),
p75: Number(summaryResult?.fcp_p75 || 0),
p95: Number(summaryResult?.fcp_p95 || 0),
},
ttfb: {
p50: Number(summaryResult?.ttfb_p50 || 0),
p75: Number(summaryResult?.ttfb_p75 || 0),
p95: Number(summaryResult?.ttfb_p95 || 0),
},
count: Number(summaryResult?.count || 0),
};
return { chart, summary };
}
async function clickhouseQuery(
websiteId: string,
parameters: PerformanceParameters,
filters: QueryFilters,
): Promise<PerformanceResult> {
const { startDate, endDate, unit = 'day', timezone = 'utc', metric = 'lcp' } = parameters;
const { getDateSQL, rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId });
const chart = await rawQuery<{ t: string; p50: number; p75: number; p95: number }[]>(
`
select
${getDateSQL('created_at', unit, timezone)} t,
quantile(0.5)(${metric}) as p50,
quantile(0.75)(${metric}) as p75,
quantile(0.95)(${metric}) as p95
from website_event
${cohortQuery}
where website_event.website_id = {websiteId:UUID}
and website_event.event_type = 5
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
group by t
order by t
`,
{ ...queryParams, startDate, endDate },
);
const summaryResult = await rawQuery<any>(
`
select
quantile(0.5)(lcp) as lcp_p50,
quantile(0.75)(lcp) as lcp_p75,
quantile(0.95)(lcp) as lcp_p95,
quantile(0.5)(inp) as inp_p50,
quantile(0.75)(inp) as inp_p75,
quantile(0.95)(inp) as inp_p95,
quantile(0.5)(cls) as cls_p50,
quantile(0.75)(cls) as cls_p75,
quantile(0.95)(cls) as cls_p95,
quantile(0.5)(fcp) as fcp_p50,
quantile(0.75)(fcp) as fcp_p75,
quantile(0.95)(fcp) as fcp_p95,
quantile(0.5)(ttfb) as ttfb_p50,
quantile(0.75)(ttfb) as ttfb_p75,
quantile(0.95)(ttfb) as ttfb_p95,
count() as count
from website_event
${cohortQuery}
where website_event.website_id = {websiteId:UUID}
and website_event.event_type = 5
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
`,
{ ...queryParams, startDate, endDate },
).then(result => result?.[0]);
const summary = {
lcp: {
p50: Number(summaryResult?.lcp_p50 || 0),
p75: Number(summaryResult?.lcp_p75 || 0),
p95: Number(summaryResult?.lcp_p95 || 0),
},
inp: {
p50: Number(summaryResult?.inp_p50 || 0),
p75: Number(summaryResult?.inp_p75 || 0),
p95: Number(summaryResult?.inp_p95 || 0),
},
cls: {
p50: Number(summaryResult?.cls_p50 || 0),
p75: Number(summaryResult?.cls_p75 || 0),
p95: Number(summaryResult?.cls_p95 || 0),
},
fcp: {
p50: Number(summaryResult?.fcp_p50 || 0),
p75: Number(summaryResult?.fcp_p75 || 0),
p95: Number(summaryResult?.fcp_p95 || 0),
},
ttfb: {
p50: Number(summaryResult?.ttfb_p50 || 0),
p75: Number(summaryResult?.ttfb_p75 || 0),
p95: Number(summaryResult?.ttfb_p95 || 0),
},
count: Number(summaryResult?.count || 0),
};
return { chart, summary };
}
src/queries/sql/reports/getPerformanceMetrics.ts
+99
View File
@@ -0,0 +1,99 @@
import clickhouse from '@/lib/clickhouse';
import { SESSION_COLUMNS } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
import type { PerformanceParameters } from './getPerformance';
export interface PerformanceMetricsData {
name: string;
p50: number;
p75: number;
p95: number;
count: number;
}
export async function getPerformanceMetrics(
...args: [
websiteId: string,
parameters: PerformanceParameters,
filters: QueryFilters,
column: string,
limit?: number,
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
parameters: PerformanceParameters,
filters: QueryFilters,
column: string,
limit?: number,
): Promise<PerformanceMetricsData[]> {
const { startDate, endDate, metric = 'lcp' } = parameters;
const { rawQuery, parseFilters } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(
{ ...filters, websiteId },
{ joinSession: SESSION_COLUMNS.includes(column) },
);
return rawQuery(
`
select
${column} as "name",
percentile_cont(0.5) within group (order by ${metric}) as p50,
percentile_cont(0.75) within group (order by ${metric}) as p75,
percentile_cont(0.95) within group (order by ${metric}) as p95,
count(*) as count
from website_event
${cohortQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.event_type = 5
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by ${column}
order by p75 desc
${limit ? `limit ${limit}` : ''}
`,
{ ...queryParams, startDate, endDate },
);
}
async function clickhouseQuery(
websiteId: string,
parameters: PerformanceParameters,
filters: QueryFilters,
column: string,
limit?: number,
): Promise<PerformanceMetricsData[]> {
const { startDate, endDate, metric = 'lcp' } = parameters;
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId });
return rawQuery<PerformanceMetricsData[]>(
`
select
${column} as "name",
quantile(0.5)(${metric}) as p50,
quantile(0.75)(${metric}) as p75,
quantile(0.95)(${metric}) as p95,
count() as count
from website_event
${cohortQuery}
where website_event.website_id = {websiteId:UUID}
and website_event.event_type = 5
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
group by ${column}
order by p75 desc
${limit ? `limit ${limit}` : ''}
`,
{ ...queryParams, startDate, endDate },
);
}
src/queries/sql/reports/getRetention.ts
+1 -1
View File
@@ -132,7 +132,7 @@ async function clickhouseQuery(
),
user_activities AS (
select distinct
website_event.session_id,
website_event.session_id as session_id,
toInt32((${getDateSQL('created_at', unit, timezone)} - cohort_items.cohort_date) / 86400) as day_number
from website_event
join cohort_items
src/queries/sql/reports/getRevenue.ts
+32 -126
View File
@@ -9,12 +9,7 @@ export interface RevenuParameters {
unit: string;
timezone: string;
currency: string;
}
export interface RevenueResult {
chart: { x: string; t: string; y: number }[];
country: { name: string; value: number }[];
total: { sum: number; count: number; average: number; unique_count: number };
compare?: string;
}
export async function getRevenue(
@@ -30,7 +25,7 @@ async function relationalQuery(
websiteId: string,
parameters: RevenuParameters,
filters: QueryFilters,
): Promise<RevenueResult> {
) {
const { startDate, endDate, unit = 'day', timezone = 'utc', currency } = parameters;
const { getDateSQL, rawQuery, parseFilters } = prisma;
const { queryParams, filterQuery, cohortQuery, joinSessionQuery } = parseFilters({
@@ -41,28 +36,32 @@ async function relationalQuery(
currency,
});
const joinQuery = filterQuery
? `join website_event
on website_event.website_id = revenue.website_id
and website_event.session_id = revenue.session_id
and website_event.event_id = revenue.event_id
and website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}`
: '';
const joinQuery =
filterQuery || cohortQuery
? `join (select *
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and event_type = 2) website_event
on website_event.website_id = revenue.website_id
and website_event.session_id = revenue.session_id
and website_event.event_id = revenue.event_id`
: '';
const chart = await rawQuery(
`
select
revenue.event_name x,
${getDateSQL('revenue.created_at', unit, timezone)} t,
sum(revenue.revenue) y
sum(revenue.revenue) y,
count(revenue.event_id) count
from revenue
${joinQuery}
${cohortQuery}
${joinSessionQuery}
where revenue.website_id = {{websiteId::uuid}}
and revenue.created_at between {{startDate}} and {{endDate}}
and revenue.currency = upper({{currency}})
and upper(revenue.currency) = {{currency}}
${filterQuery}
group by x, t
order by t
@@ -70,54 +69,14 @@ async function relationalQuery(
queryParams,
);
const country = await rawQuery(
`
select
session.country as name,
sum(revenue) value
from revenue
${joinQuery}
join session
on session.website_id = revenue.website_id
and session.session_id = revenue.session_id
${cohortQuery}
where revenue.website_id = {{websiteId::uuid}}
and revenue.created_at between {{startDate}} and {{endDate}}
and revenue.currency = upper({{currency}})
${filterQuery}
group by session.country
`,
queryParams,
);
const total = await rawQuery(
`
select
sum(revenue.revenue) as sum,
count(distinct revenue.event_id) as count,
count(distinct revenue.session_id) as unique_count
from revenue
${joinQuery}
${cohortQuery}
${joinSessionQuery}
where revenue.website_id = {{websiteId::uuid}}
and revenue.created_at between {{startDate}} and {{endDate}}
and revenue.currency = upper({{currency}})
${filterQuery}
`,
queryParams,
).then(result => result?.[0]);
total.average = total.count > 0 ? Number(total.sum) / Number(total.count) : 0;
return { chart, country, total };
return { chart };
}
async function clickhouseQuery(
websiteId: string,
parameters: RevenuParameters,
filters: QueryFilters,
): Promise<RevenueResult> {
) {
const { startDate, endDate, unit = 'day', timezone = 'utc', currency } = parameters;
const { getDateSQL, rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
@@ -129,32 +88,30 @@ async function clickhouseQuery(
});
const joinQuery = filterQuery
? `join website_event
on website_event.website_id = website_revenue.website_id
and website_event.session_id = website_revenue.session_id
and website_event.event_id = website_revenue.event_id
and website_event.website_id = {websiteId:UUID}
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}`
? `any left join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2) website_event
on website_event.website_id = website_revenue.website_id
and website_event.session_id = website_revenue.session_id
and website_event.event_id = website_revenue.event_id`
: '';
const chart = await rawQuery<
{
x: string;
t: string;
y: number;
}[]
>(
const chart = await rawQuery<{ x: string; t: string; y: number; count: number }[]>(
`
select
website_revenue.event_name x,
${getDateSQL('website_revenue.created_at', unit, timezone)} t,
sum(website_revenue.revenue) y
sum(website_revenue.revenue) y,
count(website_revenue.event_id) count
from website_revenue
${joinQuery}
${cohortQuery}
where website_revenue.website_id = {websiteId:UUID}
and website_revenue.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and website_revenue.currency = upper({currency:String})
and upper(website_revenue.currency) = {currency:String}
${filterQuery}
group by x, t
order by t
@@ -162,56 +119,5 @@ async function clickhouseQuery(
queryParams,
);
const country = await rawQuery<
{
name: string;
value: number;
}[]
>(
`
select
website_event.country as name,
sum(website_revenue.revenue) as value
from website_revenue
join website_event
on website_event.website_id = website_revenue.website_id
and website_event.session_id = website_revenue.session_id
and website_event.event_id = website_revenue.event_id
and website_event.website_id = {websiteId:UUID}
and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}
${cohortQuery}
where website_revenue.website_id = {websiteId:UUID}
and website_revenue.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and website_revenue.currency = upper({currency:String})
${filterQuery}
group by website_event.country
order by value desc
`,
queryParams,
);
const total = await rawQuery<{
sum: number;
count: number;
unique_count: number;
}>(
`
select
sum(website_revenue.revenue) as sum,
uniqExact(website_revenue.event_id) as count,
uniqExact(website_revenue.session_id) as unique_count
from website_revenue
${joinQuery}
${cohortQuery}
where website_revenue.website_id = {websiteId:UUID}
and website_revenue.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and website_revenue.currency = upper({currency:String})
${filterQuery}
`,
queryParams,
).then(result => result?.[0]);
total.average = total.count > 0 ? total.sum / total.count : 0;
return { chart, country, total };
return { chart };
}
src/queries/sql/reports/getRevenueMetrics.ts
+451
View File
@@ -0,0 +1,451 @@
import clickhouse from '@/lib/clickhouse';
import {
EMAIL_DOMAINS,
PAID_AD_PARAMS,
SEARCH_DOMAINS,
SHOPPING_DOMAINS,
SOCIAL_DOMAINS,
VIDEO_DOMAINS,
} from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
import type { RevenuParameters } from './getRevenue';
export interface RevenueMetricsResult {
country: { name: string; value: number }[];
region: { name: string; value: number; country: string }[];
referrer: { name: string; value: number }[];
channel: { name: string; value: number }[];
}
export async function getRevenueMetrics(
...args: [websiteId: string, parameters: RevenuParameters, filters: QueryFilters]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
parameters: RevenuParameters,
filters: QueryFilters,
): Promise<RevenueMetricsResult> {
const { startDate, endDate, currency } = parameters;
const { rawQuery, parseFilters } = prisma;
const { queryParams, filterQuery, cohortQuery, joinSessionQuery } = parseFilters({
...filters,
websiteId,
startDate,
endDate,
currency,
});
const joinQuery =
filterQuery || cohortQuery
? `join (select *
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and event_type = 2) website_event
on website_event.website_id = revenue.website_id
and website_event.session_id = revenue.session_id
and website_event.event_id = revenue.event_id`
: '';
const country = await rawQuery(
`
select
session.country as "name",
sum(revenue) as "value"
from revenue
${joinQuery}
join session
on session.website_id = revenue.website_id
and session.session_id = revenue.session_id
${cohortQuery}
where revenue.website_id = {{websiteId::uuid}}
and revenue.created_at between {{startDate}} and {{endDate}}
and upper(revenue.currency) = {{currency}}
${filterQuery}
group by session.country
order by value desc
`,
queryParams,
);
const region = await rawQuery(
`
select
session.country,
session.region as "name",
sum(revenue.revenue) as "value"
from revenue
${joinQuery}
join session
on session.website_id = revenue.website_id
and session.session_id = revenue.session_id
${cohortQuery}
where revenue.website_id = {{websiteId::uuid}}
and revenue.created_at between {{startDate}} and {{endDate}}
and upper(revenue.currency) = {{currency}}
${filterQuery}
group by session.country, session.region
order by value desc
`,
queryParams,
);
const referrer = await rawQuery(
`
WITH events AS (
select
revenue.website_id,
revenue.session_id,
sum(revenue.revenue) as "value"
from revenue
${joinQuery}
${cohortQuery}
${joinSessionQuery}
where revenue.website_id = {{websiteId::uuid}}
and revenue.created_at between {{startDate}} and {{endDate}}
and upper(revenue.currency) = {{currency}}
${filterQuery}
group by revenue.website_id, revenue.session_id),
revenue_data AS (
select
e.website_id,
e.session_id,
e.value,
we.min_date as created_at
from events e
join (
select session_id, min(created_at) as min_date
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
group by session_id
) we on we.session_id = e.session_id)
select
we.referrer_domain as "name",
sum(revenue_data.value) as "value"
from revenue_data
join (
select website_id, session_id, referrer_domain, created_at
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}) we
on we.website_id = revenue_data.website_id
and we.session_id = revenue_data.session_id
and we.created_at = revenue_data.created_at
group by we.referrer_domain
order by value desc
`,
queryParams,
);
const channel = await rawQuery(
`
WITH events AS (
select
revenue.website_id,
revenue.session_id,
sum(revenue.revenue) as "value"
from revenue
${joinQuery}
${cohortQuery}
${joinSessionQuery}
where revenue.website_id = {{websiteId::uuid}}
and revenue.created_at between {{startDate}} and {{endDate}}
and upper(revenue.currency) = {{currency}}
${filterQuery}
group by revenue.website_id, revenue.session_id),
revenue_data AS (
select
e.website_id,
e.session_id,
e.value,
we.min_date as created_at
from events e
join (
select session_id, min(created_at) as min_date
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
group by session_id
) we on we.session_id = e.session_id),
revenue_prefix AS (
select
case when we.utm_medium ilike '%cp%' OR
we.utm_medium ilike '%ppc%' OR
we.utm_medium ilike '%retargeting%' OR
we.utm_medium ilike '%paid%' then 'paid' else 'organic' end AS prefix,
we.referrer_domain,
we.url_query,
we.utm_medium,
we.utm_source,
we.hostname,
r.value
from revenue_data r
join (
select website_id, session_id, referrer_domain, url_query, utm_medium, utm_source, hostname, created_at
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}) we
on we.website_id = r.website_id
and we.session_id = r.session_id
and we.created_at = r.created_at),
channels AS (
select
case
when referrer_domain = '' and url_query = '' then 'direct'
when ${toPostgresLikeClause('url_query', PAID_AD_PARAMS)} then 'paidAds'
when ${toPostgresLikeClause('utm_medium', ['referral', 'app', 'link'])} then 'referral'
when utm_medium ilike '%affiliate%' then 'affiliate'
when utm_medium ilike '%sms%' or utm_source ilike '%sms%' then 'sms'
when ${toPostgresLikeClause('referrer_domain', SEARCH_DOMAINS)} or utm_medium ilike '%organic%' then concat(prefix, 'Search')
when ${toPostgresLikeClause('referrer_domain', SOCIAL_DOMAINS)} then concat(prefix, 'Social')
when ${toPostgresLikeClause('referrer_domain', EMAIL_DOMAINS)} or utm_medium ilike '%mail%' then 'email'
when ${toPostgresLikeClause('referrer_domain', SHOPPING_DOMAINS)} or utm_medium ilike '%shop%' then concat(prefix, 'Shopping')
when ${toPostgresLikeClause('referrer_domain', VIDEO_DOMAINS)} or utm_medium ilike '%video%' then concat(prefix, 'Video')
when referrer_domain != regexp_replace(hostname, '^www.', '') and referrer_domain != '' then 'referral'
else 'Unknown' end AS "name",
value
from revenue_prefix)
select name, sum(value) as value
from channels
group by name
order by value desc
`,
queryParams,
);
return { country, region, referrer, channel };
}
async function clickhouseQuery(
websiteId: string,
parameters: RevenuParameters,
filters: QueryFilters,
): Promise<RevenueMetricsResult> {
const { startDate, endDate, currency } = parameters;
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
startDate,
endDate,
currency,
});
const joinQuery = filterQuery
? `any left join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2) website_event
on website_event.website_id = website_revenue.website_id
and website_event.session_id = website_revenue.session_id
and website_event.event_id = website_revenue.event_id`
: '';
const country = await rawQuery<{ name: string; value: number }[]>(
`
select
website_event.country as "name",
sum(website_revenue.revenue) as "value"
from website_revenue
any left join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2) website_event
on website_event.website_id = website_revenue.website_id
and website_event.session_id = website_revenue.session_id
and website_event.event_id = website_revenue.event_id
${cohortQuery}
where website_revenue.website_id = {websiteId:UUID}
and website_revenue.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and upper(website_revenue.currency) = {currency:String}
${filterQuery}
group by website_event.country
order by value desc
`,
queryParams,
);
const region = await rawQuery<{ name: string; value: number; country: string }[]>(
`
select
website_event.country,
website_event.region as "name",
sum(website_revenue.revenue) as "value"
from website_revenue
any left join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2) website_event
on website_event.website_id = website_revenue.website_id
and website_event.session_id = website_revenue.session_id
and website_event.event_id = website_revenue.event_id
${cohortQuery}
where website_revenue.website_id = {websiteId:UUID}
and website_revenue.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and upper(website_revenue.currency) = {currency:String}
${filterQuery}
group by 1,2
order by value desc
`,
queryParams,
);
const referrer = await rawQuery<{ name: string; value: number }[]>(
`
WITH events AS (
select distinct
website_id,
session_id,
sum(revenue) as "value"
from website_revenue
${joinQuery}
${cohortQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and upper(currency) = {currency:String}
${filterQuery}
group by 1,2),
revenue AS (
select
e.website_id,
e.session_id,
e.value,
we.min_date as created_at
from events e
join (select session_id, min(created_at) min_date
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by 1
) we
on we.session_id = e.session_id)
select
website_event.referrer_domain as "name",
sum(revenue.value) as "value"
from revenue
any left join (
select website_id, session_id, referrer_domain, created_at
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}) website_event
on website_event.website_id = revenue.website_id
and website_event.session_id = revenue.session_id
and website_event.created_at = revenue.created_at
group by 1
order by value desc
`,
queryParams,
);
const channel = await rawQuery<{ name: string; value: number }[]>(
`
WITH events AS (
select distinct
website_id,
session_id,
sum(revenue) as "value"
from website_revenue
${joinQuery}
${cohortQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and upper(currency) = {currency:String}
${filterQuery}
group by 1,2),
revenue AS (
select
e.website_id,
e.session_id,
e.value,
we.min_date as created_at
from events e
join (select session_id, min(created_at) min_date
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by 1
) we
on we.session_id = e.session_id),
channels AS (
select
case when multiSearchAny(lower(utm_medium), ['cp', 'ppc', 'retargeting', 'paid']) != 0 then 'paid' else 'organic' end prefix,
case
when referrer_domain = '' and url_query = '' then 'direct'
when multiSearchAny(lower(url_query), [${toClickHouseStringArray(
PAID_AD_PARAMS,
)}]) != 0 then 'paidAds'
when multiSearchAny(lower(utm_medium), ['referral', 'app','link']) != 0 then 'referral'
when position(lower(utm_medium), 'affiliate') > 0 then 'affiliate'
when position(lower(utm_medium), 'sms') > 0 or position(lower(utm_source), 'sms') > 0 then 'sms'
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
SEARCH_DOMAINS,
)}]) != 0 or position(lower(utm_medium), 'organic') > 0 then concat(prefix, 'Search')
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
SOCIAL_DOMAINS,
)}]) != 0 then concat(prefix, 'Social')
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
EMAIL_DOMAINS,
)}]) != 0 or position(lower(utm_medium), 'mail') > 0 then 'email'
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
SHOPPING_DOMAINS,
)}]) != 0 or position(lower(utm_medium), 'shop') > 0 then concat(prefix, 'Shopping')
when multiSearchAny(lower(referrer_domain), [${toClickHouseStringArray(
VIDEO_DOMAINS,
)}]) != 0 or position(lower(utm_medium), 'video') > 0 then concat(prefix, 'Video')
when referrer_domain != hostname and referrer_domain != '' then 'referral'
else 'Unknown' end AS "name",
sum(revenue.value) as "value"
from revenue
any left join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}) website_event
on website_event.website_id = revenue.website_id
and website_event.session_id = revenue.session_id
and website_event.created_at = revenue.created_at
group by 1, 2)
select name, sum(value) value
from channels
group by 1
order by value desc;
`,
queryParams,
);
return { country, region, referrer, channel };
}
function toClickHouseStringArray(arr: string[]): string {
return arr.map(p => `'${p.replace(/'/g, "\\'")}'`).join(', ');
}
function toPostgresLikeClause(column: string, arr: string[]) {
return arr.map(val => `${column} ilike '%${val.replace(/'/g, "''")}%'`).join(' OR\n ');
}
src/queries/sql/reports/getRevenueSessions.ts
+146
View File
@@ -0,0 +1,146 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
const FUNCTION_NAME = 'getRevenueSessions';
export async function getRevenueSessions(
...args: [websiteId: string, currency: string, filters: QueryFilters]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, currency: string, filters: QueryFilters) {
const { pagedRawQuery, parseFilters } = prisma;
const { search } = filters;
const { filterQuery, dateQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
currency,
search: search ? `%${search}%` : undefined,
});
const searchQuery = search
? `and (session.browser ilike {{search}}
or session.os ilike {{search}}
or session.device ilike {{search}}
or session.city ilike {{search}})`
: '';
return pagedRawQuery(
`
select
session.session_id as "id",
session.website_id as "websiteId",
website_event.hostname,
session.browser,
session.os,
session.device,
session.screen,
session.language,
session.country,
session.region,
session.city,
min(website_event.created_at) as "firstAt",
max(website_event.created_at) as "lastAt",
count(distinct website_event.visit_id) as "visits",
sum(case when website_event.event_type = 1 then 1 else 0 end) as "views",
sum(case when website_event.event_type = 2 then 1 else 0 end) as "events",
max(website_event.created_at) as "createdAt"
from website_event
${cohortQuery}
join session
on session.session_id = website_event.session_id
and session.website_id = website_event.website_id
join (
select distinct session_id
from revenue
where website_id = {{websiteId::uuid}}
and revenue.created_at between {{startDate}} and {{endDate}}
and upper(currency) = {{currency}}
) rev on rev.session_id = website_event.session_id
where website_event.website_id = {{websiteId::uuid}}
${dateQuery}
${filterQuery}
${searchQuery}
group by
session.session_id,
session.website_id,
website_event.hostname,
session.browser,
session.os,
session.device,
session.screen,
session.language,
session.country,
session.region,
session.city
order by max(website_event.created_at) desc
`,
queryParams,
filters,
FUNCTION_NAME,
);
}
async function clickhouseQuery(websiteId: string, currency: string, filters: QueryFilters) {
const { pagedRawQuery, parseFilters, getDateStringSQL } = clickhouse;
const { search } = filters;
const { filterQuery, dateQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
currency,
});
const searchQuery = search
? `and ((positionCaseInsensitive(browser, {search:String}) > 0)
or (positionCaseInsensitive(city, {search:String}) > 0)
or (positionCaseInsensitive(os, {search:String}) > 0)
or (positionCaseInsensitive(device, {search:String}) > 0))`
: '';
return pagedRawQuery(
`
select
session_id as id,
website_id as websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
${getDateStringSQL('min(created_at)')} as firstAt,
${getDateStringSQL('max(created_at)')} as lastAt,
uniq(visit_id) as visits,
sumIf(1, event_type = 1) as views,
sumIf(1, event_type = 2) as events,
max(created_at) as createdAt
from website_event
${cohortQuery}
where website_id = {websiteId:UUID}
${dateQuery}
${filterQuery}
${searchQuery}
and session_id in (
select distinct session_id
from website_revenue
where website_id = {websiteId:UUID}
${dateQuery}
and upper(currency) = {currency:String}
)
group by session_id, website_id, hostname, browser, os, device, screen, language, country, region, city
order by max(created_at) desc
`,
queryParams,
filters,
FUNCTION_NAME,
);
}
src/queries/sql/reports/getRevenueStats.ts
+136
View File
@@ -0,0 +1,136 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
import type { RevenuParameters } from './getRevenue';
export interface RevenueStatsResult {
sum: number;
count: number;
average: number;
unique_count: number;
arpu: number;
}
export async function getRevenueStats(
...args: [websiteId: string, parameters: RevenuParameters, filters: QueryFilters]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
parameters: RevenuParameters,
filters: QueryFilters,
): Promise<RevenueStatsResult> {
const { startDate, endDate, currency } = parameters;
const { rawQuery, parseFilters } = prisma;
const { queryParams, filterQuery, cohortQuery, joinSessionQuery } = parseFilters({
...filters,
websiteId,
startDate,
endDate,
currency,
});
const joinQuery =
filterQuery || cohortQuery
? `join (select *
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and event_type = 2) website_event
on website_event.website_id = revenue.website_id
and website_event.session_id = revenue.session_id
and website_event.event_id = revenue.event_id`
: '';
const total = await rawQuery(
`
select
sum(revenue.revenue) as sum,
count(distinct revenue.event_id) as count,
count(distinct revenue.session_id) as unique_count,
(select count(distinct session_id)
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}) as total_sessions
from revenue
${joinQuery}
${cohortQuery}
${joinSessionQuery}
where revenue.website_id = {{websiteId::uuid}}
and revenue.created_at between {{startDate}} and {{endDate}}
and upper(revenue.currency) = {{currency}}
${filterQuery}
`,
queryParams,
).then(result => result?.[0]);
total.average = total.count > 0 ? Number(total.sum) / Number(total.count) : 0;
total.arpu = total.total_sessions > 0 ? Number(total.sum) / Number(total.total_sessions) : 0;
return total;
}
async function clickhouseQuery(
websiteId: string,
parameters: RevenuParameters,
filters: QueryFilters,
): Promise<RevenueStatsResult> {
const { startDate, endDate, currency } = parameters;
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
startDate,
endDate,
currency,
});
const joinQuery = filterQuery
? `any left join (
select *
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = 2) website_event
on website_event.website_id = website_revenue.website_id
and website_event.session_id = website_revenue.session_id
and website_event.event_id = website_revenue.event_id`
: '';
const total = await rawQuery<{
sum: number;
count: number;
unique_count: number;
total_sessions: number;
}>(
`
select
sum(website_revenue.revenue) as sum,
uniqExact(website_revenue.event_id) as count,
uniqExact(website_revenue.session_id) as unique_count,
(select uniqExact(session_id)
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}) as total_sessions
from website_revenue
${joinQuery}
${cohortQuery}
where website_revenue.website_id = {websiteId:UUID}
and website_revenue.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and upper(website_revenue.currency) = {currency:String}
${filterQuery}
`,
queryParams,
).then(result => result?.[0]);
total.average = total.count > 0 ? total.sum / total.count : 0;
total.arpu = total.total_sessions > 0 ? total.sum / total.total_sessions : 0;
return total;
}
src/queries/sql/reports/getUTM.ts
+2
View File
@@ -47,6 +47,7 @@ async function relationalQuery(
${filterQuery}
group by 1
order by 2 desc
limit 50
`,
queryParams,
);
@@ -78,6 +79,7 @@ async function clickhouseQuery(
${filterQuery}
group by 1
order by 2 desc
limit 50
`,
queryParams,
);
src/queries/sql/sessions/getSessionActivity.ts
+2
View File
@@ -29,6 +29,7 @@ async function relationalQuery(websiteId: string, sessionId: string, filters: Qu
event_type as "eventType",
event_name as "eventName",
visit_id as "visitId",
hostname,
event_id IN (select website_event_id
from event_data
where website_id = {{websiteId::uuid}}
@@ -60,6 +61,7 @@ async function clickhouseQuery(websiteId: string, sessionId: string, filters: Qu
event_type as eventType,
event_name as eventName,
visit_id as visitId,
hostname,
event_id IN (select event_id
from event_data
where website_id = {websiteId:UUID}
src/queries/sql/sessions/getSessionExpandedMetrics.ts
+17 -13
View File
@@ -38,15 +38,16 @@ async function relationalQuery(
const { type, limit = 500, offset = 0 } = parameters;
let column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery, getTimestampDiffSQL } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(
{
...filters,
websiteId,
},
{
joinSession: SESSION_COLUMNS.includes(type),
},
);
const { filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, queryParams } =
parseFilters(
{
...filters,
websiteId,
},
{
joinSession: SESSION_COLUMNS.includes(type),
},
);
const includeCountry = column === 'city' || column === 'region';
if (type === 'language') {
@@ -65,7 +66,7 @@ async function relationalQuery(
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime"
from (
select
${column} name,
${column} as "name",
${includeCountry ? 'country,' : ''}
website_event.session_id,
website_event.visit_id,
@@ -74,14 +75,16 @@ async function relationalQuery(
max(website_event.created_at) as "max_time"
from website_event
${cohortQuery}
${excludeBounceQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by name, website_event.session_id, website_event.visit_id
${includeCountry ? ', country' : ''}
) as t
where name != ''
group by name
${includeCountry ? ', country' : ''}
order by visitors desc, visits desc
@@ -101,7 +104,7 @@ async function clickhouseQuery(
const { type, limit = 500, offset = 0 } = parameters;
let column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
@@ -132,9 +135,10 @@ async function clickhouseQuery(
max(created_at) max_time
from website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
and name != ''
${filterQuery}
group by name, session_id, visit_id
src/queries/sql/sessions/getSessionMetrics.ts
+17 -13
View File
@@ -29,15 +29,16 @@ async function relationalQuery(
const { type, limit = 500, offset = 0 } = parameters;
let column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(
{
...filters,
websiteId,
},
{
joinSession: SESSION_COLUMNS.includes(type),
},
);
const { filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, queryParams } =
parseFilters(
{
...filters,
websiteId,
},
{
joinSession: SESSION_COLUMNS.includes(type),
},
);
const includeCountry = column === 'city' || column === 'region';
if (type === 'language') {
@@ -52,10 +53,11 @@ async function relationalQuery(
${includeCountry ? ', country' : ''}
from website_event
${cohortQuery}
${excludeBounceQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by 1
${includeCountry ? ', 3' : ''}
@@ -76,7 +78,7 @@ async function clickhouseQuery(
const { type, limit = 500, offset = 0 } = parameters;
let column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
@@ -96,9 +98,10 @@ async function clickhouseQuery(
${includeCountry ? ', country' : ''}
from website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by x
${includeCountry ? ', country' : ''}
@@ -114,9 +117,10 @@ async function clickhouseQuery(
${includeCountry ? ', country' : ''}
from website_event_stats_hourly as website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by x
${includeCountry ? ', country' : ''}
src/queries/sql/sessions/getSessionStats.ts
+12 -8
View File
@@ -16,10 +16,11 @@ export async function getSessionStats(...args: [websiteId: string, filters: Quer
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc', unit = 'day' } = filters;
const { getDateSQL, parseFilters, rawQuery } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
const { filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, queryParams } =
parseFilters({
...filters,
websiteId,
});
return rawQuery(
`
@@ -28,10 +29,11 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
count(distinct website_event.session_id) y
from website_event
${cohortQuery}
${excludeBounceQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type != 2
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by 1
order by 1
@@ -47,7 +49,7 @@ async function clickhouseQuery(
): Promise<{ x: string; y: number }[]> {
const { timezone = 'UTC', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateSQL } = clickhouse;
const { filterQuery, cohortQuery, queryParams } = parseFilters({
const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({
...filters,
websiteId,
});
@@ -65,9 +67,10 @@ async function clickhouseQuery(
count(distinct session_id) as y
from website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by t
) as g
@@ -84,9 +87,10 @@ async function clickhouseQuery(
uniq(session_id) as y
from website_event_stats_hourly as website_event
${cohortQuery}
${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type != 2
and event_type NOT IN (2, 5)
${filterQuery}
group by t
) as g
src/queries/sql/sessions/getWebsiteSessionStats.ts
+1 -1
View File
@@ -70,7 +70,7 @@ async function clickhouseQuery(
uniq(session_id) as "visitors",
uniq(visit_id) as "visits",
uniq(country) as "countries",
sum(length(event_name)) as "events"
sumIf(1, event_type = 2) as "events"
from website_event
${cohortQuery}
where website_id = {websiteId:UUID}
src/queries/sql/sessions/getWebsiteSessions.ts
+3
View File
@@ -48,6 +48,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
max(website_event.created_at) as "lastAt",
count(distinct website_event.visit_id) as "visits",
sum(case when website_event.event_type = 1 then 1 else 0 end) as "views",
sum(case when website_event.event_type = 2 then 1 else 0 end) as "events",
max(website_event.created_at) as "createdAt"
from website_event
${cohortQuery}
@@ -112,6 +113,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
${getDateStringSQL('max(created_at)')} as lastAt,
uniq(visit_id) as visits,
sumIf(1, event_type = 1) as views,
sumIf(1, event_type = 2) as events,
lastAt as createdAt
from website_event
${cohortQuery}
@@ -140,6 +142,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
${getDateStringSQL('max(max_time)')} as lastAt,
uniq(visit_id) as visits,
sumIf(views, event_type = 1) as views,
sum(length(event_name)) as events,
lastAt as createdAt
from website_event_stats_hourly as website_event
${cohortQuery}
src/queries/sql/sessions/saveSessionData.ts
+14 -22
View File
@@ -46,31 +46,23 @@ export async function relationalQuery({
createdAt,
}));
const existing = await client.sessionData.findMany({
where: {
sessionId,
},
select: {
id: true,
sessionId: true,
dataKey: true,
},
});
for (const data of flattenedData) {
const { sessionId, dataKey, ...props } = data;
const record = existing.find(e => e.sessionId === sessionId && e.dataKey === dataKey);
if (record) {
await client.sessionData.update({
where: {
id: record.id,
},
data: {
...props,
},
});
} else {
// Try to update existing record using compound where clause
// This is safer than using id from a previous query due to race conditions
const updateResult = await client.sessionData.updateMany({
where: {
sessionId,
dataKey,
},
data: {
...props,
},
});
// If no record was updated, create a new one
if (updateResult.count === 0) {
await client.sessionData.create({
data,
});