heatmap migration/code clean-up
This commit is contained in:
Francis Cao
@@ -7,7 +7,6 @@ CREATE TABLE umami.heatmap_event
|
||||
visit_id UUID,
|
||||
url_path String,
|
||||
event_type UInt8,
|
||||
node_id Nullable(Int32),
|
||||
x Nullable(Int32),
|
||||
y Nullable(Int32),
|
||||
page_x Nullable(Int32),
|
||||
@@ -17,34 +16,9 @@ CREATE TABLE umami.heatmap_event
|
||||
viewport_h Nullable(Int32),
|
||||
page_h Nullable(Int32),
|
||||
scroll_pct Nullable(UInt8),
|
||||
replay_chunk_index Nullable(UInt32),
|
||||
replay_event_index Nullable(UInt32),
|
||||
replay_time_ms Nullable(Int64),
|
||||
created_at DateTime('UTC')
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(created_at)
|
||||
ORDER BY (website_id, url_path, event_type, created_at)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- Create heatmap_snapshot
|
||||
CREATE TABLE umami.heatmap_snapshot
|
||||
(
|
||||
snapshot_id UUID,
|
||||
website_id UUID,
|
||||
url_path String,
|
||||
viewport_w UInt32,
|
||||
viewport_h UInt32,
|
||||
page_w UInt32,
|
||||
page_h UInt32,
|
||||
status UInt8,
|
||||
mime_type LowCardinality(String),
|
||||
object_key String,
|
||||
image_size Nullable(UInt32),
|
||||
error Nullable(String),
|
||||
created_at DateTime('UTC')
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(created_at)
|
||||
ORDER BY (website_id, url_path, viewport_w, viewport_h, created_at)
|
||||
SETTINGS index_granularity = 8192;
|
||||
SETTINGS index_granularity = 8192;
|
||||
@@ -409,7 +409,6 @@ CREATE TABLE umami.heatmap_event
|
||||
visit_id UUID,
|
||||
url_path String,
|
||||
event_type UInt8,
|
||||
node_id Nullable(Int32),
|
||||
x Nullable(Int32),
|
||||
y Nullable(Int32),
|
||||
page_x Nullable(Int32),
|
||||
@@ -419,34 +418,9 @@ CREATE TABLE umami.heatmap_event
|
||||
viewport_h Nullable(Int32),
|
||||
page_h Nullable(Int32),
|
||||
scroll_pct Nullable(UInt8),
|
||||
replay_chunk_index Nullable(UInt32),
|
||||
replay_event_index Nullable(UInt32),
|
||||
replay_time_ms Nullable(Int64),
|
||||
created_at DateTime('UTC')
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(created_at)
|
||||
ORDER BY (website_id, url_path, event_type, created_at)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
-- Create heatmap_snapshot
|
||||
CREATE TABLE umami.heatmap_snapshot
|
||||
(
|
||||
snapshot_id UUID,
|
||||
website_id UUID,
|
||||
url_path String,
|
||||
viewport_w UInt32,
|
||||
viewport_h UInt32,
|
||||
page_w UInt32,
|
||||
page_h UInt32,
|
||||
status UInt8,
|
||||
mime_type LowCardinality(String),
|
||||
object_key String,
|
||||
image_size Nullable(UInt32),
|
||||
error Nullable(String),
|
||||
created_at DateTime('UTC')
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(created_at)
|
||||
ORDER BY (website_id, url_path, viewport_w, viewport_h, created_at)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
@@ -12,7 +12,6 @@ CREATE TABLE "heatmap_event" (
|
||||
"visit_id" UUID NOT NULL,
|
||||
"url_path" VARCHAR(500) NOT NULL,
|
||||
"event_type" INTEGER NOT NULL,
|
||||
"node_id" INTEGER,
|
||||
"x" INTEGER,
|
||||
"y" INTEGER,
|
||||
"page_x" INTEGER,
|
||||
@@ -22,9 +21,6 @@ CREATE TABLE "heatmap_event" (
|
||||
"viewport_h" INTEGER,
|
||||
"page_h" INTEGER,
|
||||
"scroll_pct" INTEGER,
|
||||
"replay_chunk_index" INTEGER,
|
||||
"replay_event_index" INTEGER,
|
||||
"replay_time_ms" BIGINT,
|
||||
"created_at" TIMESTAMPTZ(6) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "heatmap_event_pkey" PRIMARY KEY ("heatmap_event_id")
|
||||
@@ -34,32 +30,4 @@ CREATE TABLE "heatmap_event" (
|
||||
CREATE INDEX "heatmap_event_website_id_idx" ON "heatmap_event"("website_id");
|
||||
CREATE INDEX "heatmap_event_visit_id_idx" ON "heatmap_event"("visit_id");
|
||||
CREATE INDEX "heatmap_event_website_id_created_at_idx" ON "heatmap_event"("website_id", "created_at");
|
||||
CREATE INDEX "heatmap_event_website_id_url_path_event_type_created_at_idx" ON "heatmap_event"("website_id", "url_path", "event_type", "created_at");
|
||||
CREATE INDEX "heatmap_event_website_id_visit_id_replay_chunk_index_replay_event_index_idx" ON "heatmap_event"("website_id", "visit_id", "replay_chunk_index", "replay_event_index");
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "heatmap_snapshot" (
|
||||
"snapshot_id" UUID NOT NULL,
|
||||
"website_id" UUID NOT NULL,
|
||||
"url_path" VARCHAR(500) NOT NULL,
|
||||
"viewport_w" INTEGER NOT NULL,
|
||||
"viewport_h" INTEGER NOT NULL,
|
||||
"page_w" INTEGER NOT NULL,
|
||||
"page_h" INTEGER NOT NULL,
|
||||
"status" VARCHAR(20) NOT NULL,
|
||||
"mime_type" VARCHAR(100),
|
||||
"image_data" BYTEA,
|
||||
"image_size" INTEGER,
|
||||
"error" VARCHAR(500),
|
||||
"created_at" TIMESTAMPTZ(6) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" TIMESTAMPTZ(6),
|
||||
|
||||
CONSTRAINT "heatmap_snapshot_pkey" PRIMARY KEY ("snapshot_id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "heatmap_snapshot_website_id_url_path_viewport_w_viewport_h_key"
|
||||
ON "heatmap_snapshot"("website_id", "url_path", "viewport_w", "viewport_h");
|
||||
CREATE INDEX "heatmap_snapshot_website_id_idx" ON "heatmap_snapshot"("website_id");
|
||||
CREATE INDEX "heatmap_snapshot_website_id_url_path_idx" ON "heatmap_snapshot"("website_id", "url_path");
|
||||
CREATE INDEX "heatmap_snapshot_website_id_updated_at_idx" ON "heatmap_snapshot"("website_id", "updated_at");
|
||||
CREATE INDEX "heatmap_event_website_id_url_path_event_type_created_at_idx" ON "heatmap_event"("website_id", "url_path", "event_type", "created_at");
|
||||
@@ -1,38 +0,0 @@
|
||||
-- AlterTable
|
||||
ALTER TABLE "heatmap_snapshot" ALTER COLUMN "created_at" DROP NOT NULL;
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE "heatmap_replay_preview" (
|
||||
"preview_id" UUID NOT NULL,
|
||||
"website_id" UUID NOT NULL,
|
||||
"session_id" UUID NOT NULL,
|
||||
"visit_id" UUID NOT NULL,
|
||||
"url_path" VARCHAR(500) NOT NULL,
|
||||
"viewport_w" INTEGER NOT NULL,
|
||||
"viewport_h" INTEGER NOT NULL,
|
||||
"replay_chunk_index" INTEGER NOT NULL,
|
||||
"replay_event_index" INTEGER NOT NULL,
|
||||
"replay_time_ms" BIGINT,
|
||||
"created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" TIMESTAMPTZ(6),
|
||||
|
||||
CONSTRAINT "heatmap_replay_preview_pkey" PRIMARY KEY ("preview_id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "heatmap_replay_preview_website_id_idx" ON "heatmap_replay_preview"("website_id");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "heatmap_replay_preview_visit_id_idx" ON "heatmap_replay_preview"("visit_id");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "heatmap_replay_preview_website_id_url_path_idx" ON "heatmap_replay_preview"("website_id", "url_path");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX "heatmap_replay_preview_website_id_url_path_viewport_w_viewp_key" ON "heatmap_replay_preview"("website_id", "url_path", "viewport_w", "viewport_h");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "session_replay_visit_id_idx" ON "session_replay"("visit_id");
|
||||
|
||||
-- RenameIndex
|
||||
ALTER INDEX "heatmap_event_website_id_visit_id_replay_chunk_index_replay_eve" RENAME TO "heatmap_event_website_id_visit_id_replay_chunk_index_replay_idx";
|
||||
@@ -89,8 +89,6 @@ model Website {
|
||||
sessionReplays SessionReplay[]
|
||||
sessionReplaysSaved SessionReplaySaved[]
|
||||
heatmapEvents HeatmapEvent[]
|
||||
heatmapReplayPreviews HeatmapReplayPreview[]
|
||||
heatmapSnapshots HeatmapSnapshot[]
|
||||
|
||||
@@index([userId])
|
||||
@@index([teamId])
|
||||
@@ -410,7 +408,6 @@ model HeatmapEvent {
|
||||
visitId String @map("visit_id") @db.Uuid
|
||||
urlPath String @map("url_path") @db.VarChar(500)
|
||||
eventType Int @map("event_type") @db.Integer
|
||||
nodeId Int? @map("node_id") @db.Integer
|
||||
x Int? @db.Integer
|
||||
y Int? @db.Integer
|
||||
pageX Int? @map("page_x") @db.Integer
|
||||
@@ -420,9 +417,6 @@ model HeatmapEvent {
|
||||
viewportH Int? @map("viewport_h") @db.Integer
|
||||
pageH Int? @map("page_h") @db.Integer
|
||||
scrollPct Int? @map("scroll_pct") @db.Integer
|
||||
replayChunkIndex Int? @map("replay_chunk_index") @db.Integer
|
||||
replayEventIndex Int? @map("replay_event_index") @db.Integer
|
||||
replayTimeMs BigInt? @map("replay_time_ms") @db.BigInt
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
|
||||
|
||||
website Website @relation(fields: [websiteId], references: [id])
|
||||
@@ -431,54 +425,5 @@ model HeatmapEvent {
|
||||
@@index([visitId])
|
||||
@@index([websiteId, createdAt])
|
||||
@@index([websiteId, urlPath, eventType, createdAt])
|
||||
@@index([websiteId, visitId, replayChunkIndex, replayEventIndex])
|
||||
@@map("heatmap_event")
|
||||
}
|
||||
|
||||
model HeatmapReplayPreview {
|
||||
id String @id() @map("preview_id") @db.Uuid
|
||||
websiteId String @map("website_id") @db.Uuid
|
||||
sessionId String @map("session_id") @db.Uuid
|
||||
visitId String @map("visit_id") @db.Uuid
|
||||
urlPath String @map("url_path") @db.VarChar(500)
|
||||
viewportW Int @map("viewport_w") @db.Integer
|
||||
viewportH Int @map("viewport_h") @db.Integer
|
||||
replayChunkIndex Int @map("replay_chunk_index") @db.Integer
|
||||
replayEventIndex Int @map("replay_event_index") @db.Integer
|
||||
replayTimeMs BigInt? @map("replay_time_ms") @db.BigInt
|
||||
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
|
||||
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz(6)
|
||||
|
||||
website Website @relation(fields: [websiteId], references: [id])
|
||||
|
||||
@@unique([websiteId, urlPath, viewportW, viewportH])
|
||||
@@index([websiteId])
|
||||
@@index([visitId])
|
||||
@@index([websiteId, urlPath])
|
||||
@@map("heatmap_replay_preview")
|
||||
}
|
||||
|
||||
model HeatmapSnapshot {
|
||||
id String @id() @map("snapshot_id") @db.Uuid
|
||||
websiteId String @map("website_id") @db.Uuid
|
||||
urlPath String @map("url_path") @db.VarChar(500)
|
||||
viewportW Int @map("viewport_w") @db.Integer
|
||||
viewportH Int @map("viewport_h") @db.Integer
|
||||
pageW Int @map("page_w") @db.Integer
|
||||
pageH Int @map("page_h") @db.Integer
|
||||
status String @db.VarChar(20)
|
||||
mimeType String? @map("mime_type") @db.VarChar(100)
|
||||
imageData Bytes? @map("image_data")
|
||||
imageSize Int? @map("image_size") @db.Integer
|
||||
error String? @db.VarChar(500)
|
||||
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
|
||||
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz(6)
|
||||
|
||||
website Website @relation(fields: [websiteId], references: [id])
|
||||
|
||||
@@unique([websiteId, urlPath, viewportW, viewportH])
|
||||
@@index([websiteId])
|
||||
@@index([websiteId, urlPath])
|
||||
@@index([websiteId, updatedAt])
|
||||
@@map("heatmap_snapshot")
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@ import { Laptop, Monitor, Smartphone, Tablet } from 'lucide-react';
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { LoadingPanel } from '@/components/common/LoadingPanel';
|
||||
import { useResultQuery } from '@/components/hooks';
|
||||
import { getClientAuthToken } from '@/lib/client';
|
||||
import { formatLongNumber } from '@/lib/format';
|
||||
import type { HeatmapMode, HeatmapPoint, HeatmapResult, HeatmapSnapshot } from '@/queries/sql';
|
||||
import styles from './Heatmap.module.css';
|
||||
@@ -882,90 +881,14 @@ function SnapshotPreview({
|
||||
snapshot: HeatmapSnapshot;
|
||||
onReady: () => void;
|
||||
}) {
|
||||
if (snapshot.kind === 'iframe') {
|
||||
return <IframeSnapshot snapshot={snapshot} onReady={onReady} />;
|
||||
}
|
||||
|
||||
return <SnapshotImage snapshot={snapshot} onReady={onReady} />;
|
||||
}
|
||||
|
||||
function SnapshotImage({
|
||||
snapshot,
|
||||
onReady,
|
||||
}: {
|
||||
snapshot: Extract<HeatmapSnapshot, { kind: 'image' }>;
|
||||
onReady: () => void;
|
||||
}) {
|
||||
const [src, setSrc] = useState<string | null>(null);
|
||||
const imageUrl = snapshot.imageUrl;
|
||||
|
||||
useEffect(() => {
|
||||
if (!imageUrl) {
|
||||
setSrc(null);
|
||||
onReady();
|
||||
return;
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const token = getClientAuthToken();
|
||||
let objectUrl: string | null = null;
|
||||
|
||||
setSrc(null);
|
||||
|
||||
fetch(imageUrl, {
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
...(token ? { authorization: `Bearer ${token}` } : {}),
|
||||
},
|
||||
})
|
||||
.then(async response => {
|
||||
if (!response.ok) {
|
||||
throw new Error(`Snapshot image request failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
objectUrl = URL.createObjectURL(blob);
|
||||
setSrc(objectUrl);
|
||||
})
|
||||
.catch(() => {
|
||||
setSrc(null);
|
||||
onReady();
|
||||
});
|
||||
|
||||
return () => {
|
||||
controller.abort();
|
||||
|
||||
if (objectUrl) {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
}
|
||||
};
|
||||
}, [imageUrl, onReady, snapshot.id]);
|
||||
|
||||
const handleLoad = useCallback(() => onReady(), [onReady]);
|
||||
const imageWidth = Math.max(snapshot.pageW, snapshot.viewportW);
|
||||
|
||||
return (
|
||||
<div className={styles.snapshot}>
|
||||
<img
|
||||
className={styles.snapshotImage}
|
||||
src={src || undefined}
|
||||
alt=""
|
||||
draggable={false}
|
||||
onLoad={handleLoad}
|
||||
style={{
|
||||
width: imageWidth,
|
||||
height: snapshot.pageH,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
return <IframeSnapshot snapshot={snapshot} onReady={onReady} />;
|
||||
}
|
||||
|
||||
function IframeSnapshot({
|
||||
snapshot,
|
||||
onReady,
|
||||
}: {
|
||||
snapshot: Extract<HeatmapSnapshot, { kind: 'iframe' }>;
|
||||
snapshot: HeatmapSnapshot;
|
||||
onReady: () => void;
|
||||
}) {
|
||||
const [available, setAvailable] = useState(true);
|
||||
|
||||
@@ -217,7 +217,6 @@ export async function POST(request: Request) {
|
||||
sessionId,
|
||||
visitId,
|
||||
eventType: event.type === 'click' ? HEATMAP_EVENT_TYPE.click : HEATMAP_EVENT_TYPE.scroll,
|
||||
nodeId: null,
|
||||
x: event.type === 'click' ? (event.x ?? null) : null,
|
||||
y: event.type === 'click' ? (event.y ?? null) : null,
|
||||
pageX: event.type === 'click' ? (event.pageX ?? null) : null,
|
||||
@@ -229,9 +228,6 @@ export async function POST(request: Request) {
|
||||
scrollPct: event.type === 'scroll' ? (event.scrollPct ?? null) : null,
|
||||
urlPath: getUrlPath(event.url),
|
||||
createdAt: new Date(event.timestamp ?? fallbackMs),
|
||||
replayChunkIndex: null,
|
||||
replayEventIndex: null,
|
||||
replayTimeMs: null,
|
||||
}));
|
||||
|
||||
if (heatmapRows.length) {
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
import { parseRequest } from '@/lib/request';
|
||||
import { notFound, unauthorized } from '@/lib/response';
|
||||
import { canViewAuthenticatedWebsite } from '@/permissions';
|
||||
import { getHeatmapSnapshotImage } from '@/queries/sql/heatmap/ensureHeatmapSnapshot';
|
||||
|
||||
export async function GET(
|
||||
request: Request,
|
||||
{ params }: { params: Promise<{ websiteId: string; snapshotId: string }> },
|
||||
) {
|
||||
const { auth, error } = await parseRequest(request);
|
||||
const { websiteId, snapshotId } = await params;
|
||||
|
||||
if (error) {
|
||||
return error();
|
||||
}
|
||||
|
||||
if (!(await canViewAuthenticatedWebsite(auth, websiteId))) {
|
||||
return unauthorized();
|
||||
}
|
||||
|
||||
const snapshot = await getHeatmapSnapshotImage(websiteId, snapshotId);
|
||||
|
||||
if (!snapshot) {
|
||||
return notFound({ message: 'Snapshot not found.' });
|
||||
}
|
||||
|
||||
return new Response(new Uint8Array(snapshot.imageData), {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': snapshot.mimeType,
|
||||
'Cache-Control': 'private, max-age=300',
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||
|
||||
let client: S3Client | null = null;
|
||||
|
||||
function getBucket() {
|
||||
const bucket = process.env.R2_BUCKET;
|
||||
|
||||
if (!bucket) {
|
||||
throw new Error('R2_BUCKET is not set.');
|
||||
}
|
||||
|
||||
return bucket;
|
||||
}
|
||||
|
||||
function getAccountId() {
|
||||
const accountId = process.env.R2_ACCOUNT_ID;
|
||||
|
||||
if (!accountId) {
|
||||
throw new Error('R2_ACCOUNT_ID is not set.');
|
||||
}
|
||||
|
||||
return accountId;
|
||||
}
|
||||
|
||||
function getCredentials() {
|
||||
const accessKeyId = process.env.R2_ACCESS_KEY_ID;
|
||||
const secretAccessKey = process.env.R2_SECRET_ACCESS_KEY;
|
||||
|
||||
if (!accessKeyId) {
|
||||
throw new Error('R2_ACCESS_KEY_ID is not set.');
|
||||
}
|
||||
|
||||
if (!secretAccessKey) {
|
||||
throw new Error('R2_SECRET_ACCESS_KEY is not set.');
|
||||
}
|
||||
|
||||
return {
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
};
|
||||
}
|
||||
|
||||
function getClient() {
|
||||
if (!client) {
|
||||
client = new S3Client({
|
||||
region: 'auto',
|
||||
endpoint: `https://${getAccountId()}.r2.cloudflarestorage.com`,
|
||||
credentials: getCredentials(),
|
||||
});
|
||||
}
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
export async function putHeatmapSnapshot(objectKey: string, imageData: Buffer, mimeType: string) {
|
||||
await getClient().send(
|
||||
new PutObjectCommand({
|
||||
Bucket: getBucket(),
|
||||
Key: objectKey,
|
||||
Body: imageData,
|
||||
ContentType: mimeType,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function getHeatmapSnapshot(objectKey: string) {
|
||||
let response;
|
||||
|
||||
try {
|
||||
response = await getClient().send(
|
||||
new GetObjectCommand({
|
||||
Bucket: getBucket(),
|
||||
Key: objectKey,
|
||||
}),
|
||||
);
|
||||
} catch (error: any) {
|
||||
if (
|
||||
error?.name === 'NoSuchKey' ||
|
||||
error?.Code === 'NoSuchKey' ||
|
||||
error?.$metadata?.httpStatusCode === 404
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (!response.Body) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
mimeType: response.ContentType || 'application/octet-stream',
|
||||
imageData: Buffer.from(await response.Body.transformToByteArray()),
|
||||
};
|
||||
}
|
||||
@@ -1,877 +0,0 @@
|
||||
import { serializeError } from 'serialize-error';
|
||||
import { getApiUrl } from '@/lib/api-url';
|
||||
import clickhouse from '@/lib/clickhouse';
|
||||
import { uuid } from '@/lib/crypto';
|
||||
import { getHeatmapSnapshot, putHeatmapSnapshot } from '@/lib/heatmap-r2';
|
||||
import prisma from '@/lib/prisma';
|
||||
import { getWebsite } from '@/queries/prisma';
|
||||
|
||||
const SNAPSHOT_STATUS = {
|
||||
pending: 'pending',
|
||||
ready: 'ready',
|
||||
failed: 'failed',
|
||||
} as const;
|
||||
|
||||
const SNAPSHOT_RETRY_DELAY_MS = 15 * 60 * 1000;
|
||||
const SNAPSHOT_PENDING_WINDOW_MS = 30 * 1000;
|
||||
const SNAPSHOT_DEVICE_SCALE_FACTOR = 1;
|
||||
const SNAPSHOT_UNAVAILABLE_ERROR = 'Page screenshot unavailable.';
|
||||
const SNAPSHOT_ERROR_MAX_LENGTH = 500;
|
||||
export type HeatmapSnapshotStatus = (typeof SNAPSHOT_STATUS)[keyof typeof SNAPSHOT_STATUS];
|
||||
|
||||
export interface HeatmapSnapshotImage {
|
||||
kind: 'image';
|
||||
id: string;
|
||||
imageUrl: string | null;
|
||||
status: HeatmapSnapshotStatus;
|
||||
mimeType: string | null;
|
||||
pageW: number;
|
||||
pageH: number;
|
||||
viewportW: number;
|
||||
viewportH: number;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
interface SnapshotRecord {
|
||||
id: string;
|
||||
websiteId: string;
|
||||
urlPath: string;
|
||||
viewportW: number;
|
||||
viewportH: number;
|
||||
pageW: number;
|
||||
pageH: number;
|
||||
status: HeatmapSnapshotStatus;
|
||||
mimeType: string | null;
|
||||
objectKey: string | null;
|
||||
imageSize: number | null;
|
||||
error: string | null;
|
||||
hasImage: boolean;
|
||||
updatedAt: Date | string | null;
|
||||
}
|
||||
|
||||
interface EnsureHeatmapSnapshotOptions {
|
||||
websiteId: string;
|
||||
urlPath: string;
|
||||
viewportW: number | null;
|
||||
viewportH: number | null;
|
||||
pageW: number | null;
|
||||
pageH: number | null;
|
||||
}
|
||||
|
||||
interface CaptureResult {
|
||||
imageData: Buffer;
|
||||
mimeType: string;
|
||||
pageW: number;
|
||||
pageH: number;
|
||||
}
|
||||
|
||||
const CLICKHOUSE_SNAPSHOT_STATUS = {
|
||||
pending: 0,
|
||||
ready: 1,
|
||||
failed: 2,
|
||||
} as const;
|
||||
|
||||
async function measurePage(page: any) {
|
||||
return page.evaluate(() => {
|
||||
const doc = document.documentElement;
|
||||
const body = document.body;
|
||||
const root = document.scrollingElement || doc || body;
|
||||
const rootClientWidth = root?.clientWidth || 0;
|
||||
const docClientWidth = doc?.clientWidth || 0;
|
||||
const bodyClientWidth = body?.clientWidth || 0;
|
||||
const visibleWidth = Math.max(
|
||||
window.innerWidth,
|
||||
rootClientWidth,
|
||||
docClientWidth,
|
||||
bodyClientWidth,
|
||||
);
|
||||
const rootScrollWidth = root?.scrollWidth || 0;
|
||||
const docScrollWidth = doc?.scrollWidth || 0;
|
||||
const bodyScrollWidth = body?.scrollWidth || 0;
|
||||
const horizontalOverflow = Math.max(
|
||||
rootScrollWidth - rootClientWidth,
|
||||
docScrollWidth - docClientWidth,
|
||||
bodyScrollWidth - bodyClientWidth,
|
||||
0,
|
||||
);
|
||||
let maxRight = 0;
|
||||
let maxBottom = 0;
|
||||
|
||||
if (body) {
|
||||
const walker = document.createTreeWalker(body, NodeFilter.SHOW_ELEMENT);
|
||||
let node = walker.currentNode as Element | null;
|
||||
|
||||
while (node) {
|
||||
const rect = node.getBoundingClientRect?.();
|
||||
|
||||
if (rect && (rect.width > 0 || rect.height > 0)) {
|
||||
maxRight = Math.max(maxRight, rect.right);
|
||||
maxBottom = Math.max(maxBottom, rect.bottom);
|
||||
}
|
||||
|
||||
node = walker.nextNode() as Element | null;
|
||||
}
|
||||
}
|
||||
|
||||
const pageW =
|
||||
horizontalOverflow > 24
|
||||
? Math.max(visibleWidth, rootScrollWidth, docScrollWidth, bodyScrollWidth)
|
||||
: visibleWidth;
|
||||
const pageH = Math.max(
|
||||
window.innerHeight,
|
||||
root?.scrollHeight || 0,
|
||||
doc?.scrollHeight || 0,
|
||||
body?.scrollHeight || 0,
|
||||
Math.ceil(maxBottom),
|
||||
);
|
||||
|
||||
return {
|
||||
pageW: Math.ceil(pageW),
|
||||
pageH: Math.ceil(pageH),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function warmLazyContent(page: any) {
|
||||
await page.evaluate(async () => {
|
||||
const root = document.scrollingElement || document.documentElement;
|
||||
|
||||
if (!root) {
|
||||
return;
|
||||
}
|
||||
|
||||
const maxScrollTop = Math.max(0, root.scrollHeight - window.innerHeight);
|
||||
|
||||
if (maxScrollTop <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const targets = [
|
||||
Math.min(window.innerHeight, maxScrollTop),
|
||||
Math.min(Math.round(maxScrollTop * 0.25), maxScrollTop),
|
||||
Math.min(Math.round(maxScrollTop * 0.5), maxScrollTop),
|
||||
Math.min(Math.round(maxScrollTop * 0.75), maxScrollTop),
|
||||
maxScrollTop,
|
||||
].filter((value, index, values) => value > 0 && values.indexOf(value) === index);
|
||||
|
||||
for (const top of targets) {
|
||||
window.scrollTo(0, top);
|
||||
await new Promise(resolve => window.setTimeout(resolve, 350));
|
||||
}
|
||||
|
||||
window.scrollTo(0, 0);
|
||||
await new Promise(resolve => window.setTimeout(resolve, 250));
|
||||
});
|
||||
}
|
||||
|
||||
function getSchema() {
|
||||
const databaseUrl = process.env.DATABASE_URL;
|
||||
|
||||
if (!databaseUrl) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const connectionUrl = new URL(databaseUrl);
|
||||
|
||||
return connectionUrl.searchParams.get('schema');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function rawExecute(sql: string, data: Record<string, any> = {}) {
|
||||
const params: any[] = [];
|
||||
const schema = getSchema();
|
||||
|
||||
if (schema) {
|
||||
await prisma.client.$executeRawUnsafe(`SET search_path TO "${schema}";`);
|
||||
}
|
||||
|
||||
const query = sql.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*}}/g, (...args) => {
|
||||
const [, name, type] = args;
|
||||
|
||||
params.push(data[name]);
|
||||
|
||||
return `$${params.length}${type ?? ''}`;
|
||||
});
|
||||
|
||||
return prisma.client.$executeRawUnsafe(query, ...params);
|
||||
}
|
||||
|
||||
async function findSnapshot(
|
||||
websiteId: string,
|
||||
urlPath: string,
|
||||
viewportW: number,
|
||||
viewportH: number,
|
||||
): Promise<SnapshotRecord | null> {
|
||||
if (clickhouse.enabled) {
|
||||
return findClickhouseSnapshot(websiteId, urlPath, viewportW, viewportH);
|
||||
}
|
||||
|
||||
return findRelationalSnapshot(websiteId, urlPath, viewportW, viewportH);
|
||||
}
|
||||
|
||||
async function findRelationalSnapshot(
|
||||
websiteId: string,
|
||||
urlPath: string,
|
||||
viewportW: number,
|
||||
viewportH: number,
|
||||
): Promise<SnapshotRecord | null> {
|
||||
const rows = await prisma.rawQuery(
|
||||
`
|
||||
select
|
||||
snapshot_id as id,
|
||||
website_id as "websiteId",
|
||||
url_path as "urlPath",
|
||||
viewport_w as "viewportW",
|
||||
viewport_h as "viewportH",
|
||||
page_w as "pageW",
|
||||
page_h as "pageH",
|
||||
status,
|
||||
mime_type as "mimeType",
|
||||
null as "objectKey",
|
||||
image_size as "imageSize",
|
||||
error,
|
||||
image_data is not null as "hasImage",
|
||||
updated_at as "updatedAt"
|
||||
from heatmap_snapshot
|
||||
where website_id = {{websiteId::uuid}}
|
||||
and url_path = {{urlPath}}
|
||||
and viewport_w = {{viewportW}}
|
||||
and viewport_h = {{viewportH}}
|
||||
limit 1
|
||||
`,
|
||||
{ websiteId, urlPath, viewportW, viewportH },
|
||||
'findHeatmapSnapshot',
|
||||
);
|
||||
|
||||
return rows?.[0] ?? null;
|
||||
}
|
||||
|
||||
async function findClickhouseSnapshot(
|
||||
websiteId: string,
|
||||
urlPath: string,
|
||||
viewportW: number,
|
||||
viewportH: number,
|
||||
): Promise<SnapshotRecord | null> {
|
||||
const rows = await clickhouse.rawQuery<
|
||||
{
|
||||
id: string;
|
||||
websiteId: string;
|
||||
urlPath: string;
|
||||
viewportW: number;
|
||||
viewportH: number;
|
||||
pageW: number;
|
||||
pageH: number;
|
||||
status: number;
|
||||
mimeType: string | null;
|
||||
objectKey: string;
|
||||
imageSize: number | null;
|
||||
error: string | null;
|
||||
createdAt: string;
|
||||
}[]
|
||||
>(
|
||||
`
|
||||
select
|
||||
snapshot_id as id,
|
||||
website_id as websiteId,
|
||||
url_path as urlPath,
|
||||
viewport_w as viewportW,
|
||||
viewport_h as viewportH,
|
||||
page_w as pageW,
|
||||
page_h as pageH,
|
||||
status,
|
||||
mime_type as mimeType,
|
||||
object_key as objectKey,
|
||||
image_size as imageSize,
|
||||
error,
|
||||
created_at as createdAt
|
||||
from heatmap_snapshot
|
||||
where website_id = {websiteId:UUID}
|
||||
and url_path = {urlPath:String}
|
||||
and viewport_w = {viewportW:UInt32}
|
||||
and viewport_h = {viewportH:UInt32}
|
||||
order by created_at desc
|
||||
limit 1
|
||||
`,
|
||||
{ websiteId, urlPath, viewportW, viewportH },
|
||||
'findHeatmapSnapshot',
|
||||
);
|
||||
|
||||
const row = rows?.[0];
|
||||
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const status = Object.entries(CLICKHOUSE_SNAPSHOT_STATUS).find(
|
||||
([, value]) => value === row.status,
|
||||
)?.[0];
|
||||
|
||||
if (!status) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
...row,
|
||||
status: SNAPSHOT_STATUS[status as keyof typeof SNAPSHOT_STATUS],
|
||||
mimeType: row.mimeType || null,
|
||||
objectKey: row.objectKey || null,
|
||||
hasImage: row.status === CLICKHOUSE_SNAPSHOT_STATUS.ready && Boolean(row.objectKey),
|
||||
updatedAt: row.createdAt,
|
||||
};
|
||||
}
|
||||
|
||||
function getSnapshotImageUrl(websiteId: string, snapshotId: string) {
|
||||
return getApiUrl(`/websites/${websiteId}/heatmaps/snapshots/${snapshotId}`, {
|
||||
apiUrl: process.env.API_URL,
|
||||
basePath: process.env.BASE_PATH,
|
||||
});
|
||||
}
|
||||
|
||||
function mapSnapshot(websiteId: string, row: SnapshotRecord): HeatmapSnapshotImage {
|
||||
return {
|
||||
kind: 'image',
|
||||
id: row.id,
|
||||
imageUrl:
|
||||
row.status === SNAPSHOT_STATUS.ready && row.hasImage
|
||||
? getSnapshotImageUrl(websiteId, row.id)
|
||||
: null,
|
||||
status: row.status,
|
||||
mimeType: row.mimeType,
|
||||
pageW: Number(row.pageW),
|
||||
pageH: Number(row.pageH),
|
||||
viewportW: Number(row.viewportW),
|
||||
viewportH: Number(row.viewportH),
|
||||
error: row.error,
|
||||
};
|
||||
}
|
||||
|
||||
function getFirstDomain(domain?: string | null) {
|
||||
return domain?.split(',')[0]?.trim() || null;
|
||||
}
|
||||
|
||||
function getWebsiteOrigin(domain?: string | null) {
|
||||
const host = getFirstDomain(domain);
|
||||
|
||||
if (!host) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (host.startsWith('http://') || host.startsWith('https://')) {
|
||||
return new URL(host);
|
||||
}
|
||||
|
||||
const protocol =
|
||||
host.startsWith('localhost') || host.startsWith('127.0.0.1') || host.startsWith('[::1]')
|
||||
? 'http'
|
||||
: 'https';
|
||||
|
||||
return new URL(`${protocol}://${host}`);
|
||||
}
|
||||
|
||||
export function buildHeatmapPageUrl(domain: string | null | undefined, urlPath: string) {
|
||||
try {
|
||||
const origin = getWebsiteOrigin(domain);
|
||||
|
||||
if (!origin) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new URL(urlPath || '/', origin).toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function shouldSkipSnapshot(urlPath: string) {
|
||||
// Internal Umami app routes cannot be rendered from the tracked website domain.
|
||||
return urlPath.startsWith('/teams/');
|
||||
}
|
||||
|
||||
async function upsertSnapshotRecord({
|
||||
id,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status,
|
||||
mimeType,
|
||||
imageData,
|
||||
objectKey,
|
||||
error,
|
||||
}: {
|
||||
id: string;
|
||||
websiteId: string;
|
||||
urlPath: string;
|
||||
viewportW: number;
|
||||
viewportH: number;
|
||||
pageW: number;
|
||||
pageH: number;
|
||||
status: HeatmapSnapshotStatus;
|
||||
mimeType: string | null;
|
||||
imageData: Buffer | null;
|
||||
objectKey?: string | null;
|
||||
error: string | null;
|
||||
}) {
|
||||
if (clickhouse.enabled) {
|
||||
return insertClickhouseSnapshotRecord({
|
||||
id,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status,
|
||||
mimeType,
|
||||
objectKey: objectKey ?? null,
|
||||
imageSize: imageData?.byteLength ?? null,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
return upsertRelationalSnapshotRecord({
|
||||
id,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status,
|
||||
mimeType,
|
||||
imageData,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
async function upsertRelationalSnapshotRecord({
|
||||
id,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status,
|
||||
mimeType,
|
||||
imageData,
|
||||
error,
|
||||
}: {
|
||||
id: string;
|
||||
websiteId: string;
|
||||
urlPath: string;
|
||||
viewportW: number;
|
||||
viewportH: number;
|
||||
pageW: number;
|
||||
pageH: number;
|
||||
status: HeatmapSnapshotStatus;
|
||||
mimeType: string | null;
|
||||
imageData: Buffer | null;
|
||||
error: string | null;
|
||||
}) {
|
||||
return rawExecute(
|
||||
`
|
||||
insert into heatmap_snapshot (
|
||||
snapshot_id,
|
||||
website_id,
|
||||
url_path,
|
||||
viewport_w,
|
||||
viewport_h,
|
||||
page_w,
|
||||
page_h,
|
||||
status,
|
||||
mime_type,
|
||||
image_data,
|
||||
image_size,
|
||||
error,
|
||||
created_at,
|
||||
updated_at
|
||||
)
|
||||
values (
|
||||
{{id::uuid}},
|
||||
{{websiteId::uuid}},
|
||||
{{urlPath}},
|
||||
{{viewportW}},
|
||||
{{viewportH}},
|
||||
{{pageW}},
|
||||
{{pageH}},
|
||||
{{status}},
|
||||
{{mimeType}},
|
||||
{{imageData}},
|
||||
{{imageSize}},
|
||||
{{error}},
|
||||
now(),
|
||||
now()
|
||||
)
|
||||
on conflict (website_id, url_path, viewport_w, viewport_h) do update
|
||||
set
|
||||
page_w = excluded.page_w,
|
||||
page_h = excluded.page_h,
|
||||
status = excluded.status,
|
||||
mime_type = excluded.mime_type,
|
||||
image_data = excluded.image_data,
|
||||
image_size = excluded.image_size,
|
||||
error = excluded.error,
|
||||
updated_at = now()
|
||||
`,
|
||||
{
|
||||
id,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status,
|
||||
mimeType,
|
||||
imageData,
|
||||
imageSize: imageData?.byteLength ?? null,
|
||||
error,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function insertClickhouseSnapshotRecord({
|
||||
id,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status,
|
||||
mimeType,
|
||||
objectKey,
|
||||
imageSize,
|
||||
error,
|
||||
}: {
|
||||
id: string;
|
||||
websiteId: string;
|
||||
urlPath: string;
|
||||
viewportW: number;
|
||||
viewportH: number;
|
||||
pageW: number;
|
||||
pageH: number;
|
||||
status: HeatmapSnapshotStatus;
|
||||
mimeType: string | null;
|
||||
objectKey: string | null;
|
||||
imageSize: number | null;
|
||||
error: string | null;
|
||||
}) {
|
||||
return clickhouse.insert('heatmap_snapshot', [
|
||||
{
|
||||
snapshot_id: id,
|
||||
website_id: websiteId,
|
||||
url_path: urlPath,
|
||||
viewport_w: viewportW,
|
||||
viewport_h: viewportH,
|
||||
page_w: pageW,
|
||||
page_h: pageH,
|
||||
status: CLICKHOUSE_SNAPSHOT_STATUS[status],
|
||||
mime_type: mimeType || '',
|
||||
object_key: objectKey || '',
|
||||
image_size: imageSize,
|
||||
error,
|
||||
created_at: clickhouse.getUTCString(),
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
function getSnapshotObjectKey(
|
||||
websiteId: string,
|
||||
snapshotId: string,
|
||||
viewportW: number,
|
||||
viewportH: number,
|
||||
) {
|
||||
return `${websiteId}/${viewportW}x${viewportH}/${snapshotId}.png`;
|
||||
}
|
||||
|
||||
function getSnapshotErrorMessage(error: unknown) {
|
||||
const message =
|
||||
error instanceof Error
|
||||
? [error.name, error.message].filter(Boolean).join(': ')
|
||||
: typeof error === 'string'
|
||||
? error
|
||||
: SNAPSHOT_UNAVAILABLE_ERROR;
|
||||
|
||||
return message.slice(0, SNAPSHOT_ERROR_MAX_LENGTH) || SNAPSHOT_UNAVAILABLE_ERROR;
|
||||
}
|
||||
|
||||
async function createSnapshotBrowser() {
|
||||
const endpoint = process.env.PLAYWRIGHT_URL?.trim();
|
||||
|
||||
if (!endpoint) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const { chromium } = await import('playwright-core');
|
||||
return chromium.connect(endpoint);
|
||||
}
|
||||
|
||||
async function captureSnapshot(
|
||||
url: string,
|
||||
viewportW: number,
|
||||
viewportH: number,
|
||||
): Promise<CaptureResult> {
|
||||
const browser = await createSnapshotBrowser();
|
||||
|
||||
if (!browser) {
|
||||
throw new Error(SNAPSHOT_UNAVAILABLE_ERROR);
|
||||
}
|
||||
|
||||
const initialViewportW = viewportW;
|
||||
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: initialViewportW, height: viewportH },
|
||||
screen: { width: initialViewportW, height: viewportH },
|
||||
deviceScaleFactor: SNAPSHOT_DEVICE_SCALE_FACTOR,
|
||||
ignoreHTTPSErrors: true,
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
|
||||
await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => undefined);
|
||||
await page.waitForTimeout(500);
|
||||
await warmLazyContent(page);
|
||||
await page.waitForLoadState('networkidle', { timeout: 3000 }).catch(() => undefined);
|
||||
await page.waitForTimeout(300);
|
||||
|
||||
let dimensions = await measurePage(page);
|
||||
let currentWidth = initialViewportW;
|
||||
let captureWidth =
|
||||
dimensions.pageW > initialViewportW + 24
|
||||
? Math.max(initialViewportW, dimensions.pageW)
|
||||
: initialViewportW;
|
||||
|
||||
for (let i = 0; i < 3; i++) {
|
||||
if (captureWidth <= currentWidth) {
|
||||
break;
|
||||
}
|
||||
|
||||
currentWidth = captureWidth;
|
||||
|
||||
await page.setViewportSize({
|
||||
width: currentWidth,
|
||||
height: viewportH,
|
||||
});
|
||||
await page.waitForLoadState('networkidle', { timeout: 3000 }).catch(() => undefined);
|
||||
await page.waitForTimeout(300);
|
||||
|
||||
dimensions = await measurePage(page);
|
||||
captureWidth = Math.max(currentWidth, dimensions.pageW);
|
||||
}
|
||||
|
||||
const imageData = Buffer.from(await page.screenshot({ fullPage: true, type: 'png' }));
|
||||
|
||||
await context.close();
|
||||
|
||||
return {
|
||||
imageData,
|
||||
mimeType: 'image/png',
|
||||
pageW: dimensions.pageW,
|
||||
pageH: dimensions.pageH,
|
||||
};
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureHeatmapSnapshot({
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
}: EnsureHeatmapSnapshotOptions): Promise<HeatmapSnapshotImage | null> {
|
||||
if (!urlPath || !viewportW || !viewportH || !pageW || !pageH) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (shouldSkipSnapshot(urlPath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!process.env.PLAYWRIGHT_URL?.trim()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const existing = await findSnapshot(websiteId, urlPath, viewportW, viewportH);
|
||||
|
||||
if (existing?.status === SNAPSHOT_STATUS.ready && existing.hasImage) {
|
||||
return mapSnapshot(websiteId, existing);
|
||||
}
|
||||
|
||||
const updatedAt = existing?.updatedAt ? new Date(existing.updatedAt) : null;
|
||||
const ageMs = updatedAt ? Date.now() - updatedAt.getTime() : Number.POSITIVE_INFINITY;
|
||||
|
||||
if (existing?.status === SNAPSHOT_STATUS.pending && ageMs < SNAPSHOT_PENDING_WINDOW_MS) {
|
||||
return mapSnapshot(websiteId, existing);
|
||||
}
|
||||
|
||||
if (existing?.status === SNAPSHOT_STATUS.failed && ageMs < SNAPSHOT_RETRY_DELAY_MS) {
|
||||
return mapSnapshot(websiteId, existing);
|
||||
}
|
||||
|
||||
const snapshotId = existing?.id ?? uuid();
|
||||
const website = await getWebsite(websiteId);
|
||||
const captureUrl = buildHeatmapPageUrl(website?.domain, urlPath);
|
||||
|
||||
if (!captureUrl) {
|
||||
await upsertSnapshotRecord({
|
||||
id: snapshotId,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status: SNAPSHOT_STATUS.failed,
|
||||
mimeType: null,
|
||||
imageData: null,
|
||||
error: SNAPSHOT_UNAVAILABLE_ERROR,
|
||||
});
|
||||
|
||||
const failed = await findSnapshot(websiteId, urlPath, viewportW, viewportH);
|
||||
|
||||
return failed ? mapSnapshot(websiteId, failed) : null;
|
||||
}
|
||||
|
||||
await upsertSnapshotRecord({
|
||||
id: snapshotId,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status: SNAPSHOT_STATUS.pending,
|
||||
mimeType: null,
|
||||
imageData: null,
|
||||
error: null,
|
||||
});
|
||||
|
||||
try {
|
||||
const capture = await captureSnapshot(captureUrl, viewportW, viewportH);
|
||||
const objectKey = clickhouse.enabled
|
||||
? getSnapshotObjectKey(websiteId, snapshotId, viewportW, viewportH)
|
||||
: null;
|
||||
|
||||
if (objectKey) {
|
||||
await putHeatmapSnapshot(objectKey, capture.imageData, capture.mimeType);
|
||||
}
|
||||
|
||||
await upsertSnapshotRecord({
|
||||
id: snapshotId,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW: capture.pageW,
|
||||
pageH: capture.pageH,
|
||||
status: SNAPSHOT_STATUS.ready,
|
||||
mimeType: capture.mimeType,
|
||||
imageData: clickhouse.enabled ? null : capture.imageData,
|
||||
objectKey,
|
||||
error: null,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Heatmap snapshot capture failed', {
|
||||
captureUrl,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
error: serializeError(error),
|
||||
});
|
||||
|
||||
await upsertSnapshotRecord({
|
||||
id: snapshotId,
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
status: SNAPSHOT_STATUS.failed,
|
||||
mimeType: null,
|
||||
imageData: null,
|
||||
error: getSnapshotErrorMessage(error),
|
||||
});
|
||||
}
|
||||
|
||||
const snapshot = await findSnapshot(websiteId, urlPath, viewportW, viewportH);
|
||||
|
||||
return snapshot ? mapSnapshot(websiteId, snapshot) : null;
|
||||
}
|
||||
|
||||
export async function getHeatmapSnapshotImage(
|
||||
websiteId: string,
|
||||
snapshotId: string,
|
||||
): Promise<{ mimeType: string; imageData: Buffer } | null> {
|
||||
if (clickhouse.enabled) {
|
||||
const rows = await clickhouse.rawQuery<{ mimeType: string; objectKey: string }[]>(
|
||||
`
|
||||
select
|
||||
mime_type as mimeType,
|
||||
object_key as objectKey
|
||||
from heatmap_snapshot
|
||||
where snapshot_id = {snapshotId:UUID}
|
||||
and website_id = {websiteId:UUID}
|
||||
and status = {status:UInt8}
|
||||
and object_key != ''
|
||||
order by created_at desc
|
||||
limit 1
|
||||
`,
|
||||
{
|
||||
websiteId,
|
||||
snapshotId,
|
||||
status: CLICKHOUSE_SNAPSHOT_STATUS.ready,
|
||||
},
|
||||
'getHeatmapSnapshotImage',
|
||||
);
|
||||
|
||||
const row = rows?.[0];
|
||||
|
||||
if (!row?.objectKey) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return getHeatmapSnapshot(row.objectKey);
|
||||
}
|
||||
|
||||
const rows = await prisma.rawQuery(
|
||||
`
|
||||
select
|
||||
mime_type as "mimeType",
|
||||
image_data as "imageData"
|
||||
from heatmap_snapshot
|
||||
where snapshot_id = {{snapshotId::uuid}}
|
||||
and website_id = {{websiteId::uuid}}
|
||||
and status = 'ready'
|
||||
and image_data is not null
|
||||
limit 1
|
||||
`,
|
||||
{ websiteId, snapshotId },
|
||||
'getHeatmapSnapshotImage',
|
||||
);
|
||||
|
||||
const row = rows?.[0];
|
||||
|
||||
if (!row?.imageData || !row?.mimeType) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
mimeType: row.mimeType,
|
||||
imageData: Buffer.from(row.imageData),
|
||||
};
|
||||
}
|
||||
@@ -9,7 +9,6 @@ const RRWEB_MOUSE_CLICK = 2;
|
||||
|
||||
export interface ExtractedHeatmapEvent {
|
||||
eventType: number;
|
||||
nodeId: number | null;
|
||||
x: number | null;
|
||||
y: number | null;
|
||||
viewportW: number | null;
|
||||
@@ -18,9 +17,6 @@ export interface ExtractedHeatmapEvent {
|
||||
scrollPct: number | null;
|
||||
urlPath: string;
|
||||
createdAt: Date;
|
||||
replayChunkIndex: number | null;
|
||||
replayEventIndex: number | null;
|
||||
replayTimeMs: number | null;
|
||||
}
|
||||
|
||||
interface ExtractHeatmapEventOptions {
|
||||
@@ -36,10 +32,7 @@ function safePathname(href: unknown): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
export function extractHeatmapEvents(
|
||||
events: any[],
|
||||
{ chunkIndex }: ExtractHeatmapEventOptions = {},
|
||||
): ExtractedHeatmapEvent[] {
|
||||
export function extractHeatmapEvents(events: any[], _options: ExtractHeatmapEventOptions = {}) {
|
||||
if (!Array.isArray(events) || events.length === 0) return [];
|
||||
|
||||
let urlPath: string | null = null;
|
||||
@@ -47,7 +40,7 @@ export function extractHeatmapEvents(
|
||||
let viewportH: number | null = null;
|
||||
const out: ExtractedHeatmapEvent[] = [];
|
||||
|
||||
for (const [eventIndex, ev] of events.entries()) {
|
||||
for (const ev of events) {
|
||||
if (!ev || typeof ev !== 'object') continue;
|
||||
const replayTimeMs =
|
||||
typeof ev.timestamp === 'number' && Number.isFinite(ev.timestamp) ? ev.timestamp : null;
|
||||
@@ -73,7 +66,6 @@ export function extractHeatmapEvents(
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
eventType: HEATMAP_EVENT_TYPE.scroll,
|
||||
nodeId: null,
|
||||
x: null,
|
||||
y: null,
|
||||
viewportW: typeof p.viewportW === 'number' ? p.viewportW : viewportW,
|
||||
@@ -85,9 +77,6 @@ export function extractHeatmapEvents(
|
||||
: null,
|
||||
urlPath: path,
|
||||
createdAt: new Date(replayTimeMs ?? Date.now()),
|
||||
replayChunkIndex: chunkIndex ?? null,
|
||||
replayEventIndex: eventIndex,
|
||||
replayTimeMs,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
@@ -110,7 +99,6 @@ export function extractHeatmapEvents(
|
||||
) {
|
||||
out.push({
|
||||
eventType: HEATMAP_EVENT_TYPE.click,
|
||||
nodeId: typeof d.id === 'number' ? d.id : null,
|
||||
x: typeof d.x === 'number' ? Math.round(d.x) : null,
|
||||
y: typeof d.y === 'number' ? Math.round(d.y) : null,
|
||||
viewportW,
|
||||
@@ -119,9 +107,6 @@ export function extractHeatmapEvents(
|
||||
scrollPct: null,
|
||||
urlPath,
|
||||
createdAt: new Date(replayTimeMs ?? Date.now()),
|
||||
replayChunkIndex: chunkIndex ?? null,
|
||||
replayEventIndex: eventIndex,
|
||||
replayTimeMs,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,12 +5,6 @@ import { filtersObjectToArray } from '@/lib/params';
|
||||
import prisma from '@/lib/prisma';
|
||||
import type { QueryFilters } from '@/lib/types';
|
||||
import { getWebsite } from '@/queries/prisma';
|
||||
import {
|
||||
buildHeatmapPageUrl,
|
||||
ensureHeatmapSnapshot,
|
||||
type HeatmapSnapshotImage,
|
||||
shouldSkipSnapshot,
|
||||
} from './ensureHeatmapSnapshot';
|
||||
|
||||
const FUNCTION_NAME = 'getHeatmap';
|
||||
|
||||
@@ -32,7 +26,6 @@ export interface HeatmapPage {
|
||||
}
|
||||
|
||||
export interface HeatmapPoint {
|
||||
nodeId: number | null;
|
||||
x: number;
|
||||
y: number;
|
||||
pageX: number;
|
||||
@@ -63,7 +56,7 @@ export interface HeatmapSnapshotIframe {
|
||||
viewportH: number;
|
||||
}
|
||||
|
||||
export type HeatmapSnapshot = HeatmapSnapshotImage | HeatmapSnapshotIframe;
|
||||
export type HeatmapSnapshot = HeatmapSnapshotIframe;
|
||||
|
||||
export interface HeatmapResult {
|
||||
mode: HeatmapMode;
|
||||
@@ -234,7 +227,6 @@ async function relationalQuery(
|
||||
const rawPoints: HeatmapPoint[] = await rawQuery(
|
||||
`
|
||||
select
|
||||
h.node_id as "nodeId",
|
||||
h.x,
|
||||
h.y,
|
||||
h.page_x as "pageX",
|
||||
@@ -260,7 +252,6 @@ async function relationalQuery(
|
||||
and h.viewport_w is not null
|
||||
and h.viewport_h is not null
|
||||
group by
|
||||
h.node_id,
|
||||
h.x,
|
||||
h.y,
|
||||
h.page_x,
|
||||
@@ -441,7 +432,6 @@ async function clickhouseQuery(
|
||||
|
||||
const pointRows = await rawQuery<
|
||||
{
|
||||
nodeId: number | null;
|
||||
x: number;
|
||||
y: number;
|
||||
pageX: number;
|
||||
@@ -455,7 +445,6 @@ async function clickhouseQuery(
|
||||
>(
|
||||
`
|
||||
select
|
||||
h.node_id as nodeId,
|
||||
h.x,
|
||||
h.y,
|
||||
h.page_x as pageX,
|
||||
@@ -481,7 +470,6 @@ async function clickhouseQuery(
|
||||
and h.viewport_w is not null
|
||||
and h.viewport_h is not null
|
||||
group by
|
||||
h.node_id,
|
||||
h.x,
|
||||
h.y,
|
||||
h.page_x,
|
||||
@@ -498,7 +486,6 @@ async function clickhouseQuery(
|
||||
);
|
||||
|
||||
const points: HeatmapPoint[] = pointRows.map(p => ({
|
||||
nodeId: p.nodeId === null || p.nodeId === undefined ? null : Number(p.nodeId),
|
||||
x: Number(p.x),
|
||||
y: Number(p.y),
|
||||
pageX: Number(p.pageX),
|
||||
@@ -549,19 +536,6 @@ async function resolveHeatmapSnapshot({
|
||||
pageW: number | null;
|
||||
pageH: number | null;
|
||||
}): Promise<HeatmapSnapshot | null> {
|
||||
const imageSnapshot = await ensureHeatmapSnapshot({
|
||||
websiteId,
|
||||
urlPath,
|
||||
viewportW,
|
||||
viewportH,
|
||||
pageW,
|
||||
pageH,
|
||||
});
|
||||
|
||||
if (imageSnapshot?.status === 'ready' && imageSnapshot.imageUrl) {
|
||||
return imageSnapshot;
|
||||
}
|
||||
|
||||
return getIframeSnapshot({
|
||||
websiteId,
|
||||
urlPath,
|
||||
@@ -612,6 +586,48 @@ async function getIframeSnapshot({
|
||||
};
|
||||
}
|
||||
|
||||
function getFirstDomain(domain?: string | null) {
|
||||
return domain?.split(',')[0]?.trim() || null;
|
||||
}
|
||||
|
||||
function getWebsiteOrigin(domain?: string | null) {
|
||||
const host = getFirstDomain(domain);
|
||||
|
||||
if (!host) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (host.startsWith('http://') || host.startsWith('https://')) {
|
||||
return new URL(host);
|
||||
}
|
||||
|
||||
const protocol =
|
||||
host.startsWith('localhost') || host.startsWith('127.0.0.1') || host.startsWith('[::1]')
|
||||
? 'http'
|
||||
: 'https';
|
||||
|
||||
return new URL(`${protocol}://${host}`);
|
||||
}
|
||||
|
||||
function buildHeatmapPageUrl(domain: string | null | undefined, urlPath: string) {
|
||||
try {
|
||||
const origin = getWebsiteOrigin(domain);
|
||||
|
||||
if (!origin) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new URL(urlPath || '/', origin).toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function shouldSkipSnapshot(urlPath: string) {
|
||||
// Internal Umami app routes cannot be rendered from the tracked website domain.
|
||||
return urlPath.startsWith('/teams/');
|
||||
}
|
||||
|
||||
function pickSnapshotViewport(
|
||||
points: HeatmapPoint[],
|
||||
): { width: number; height: number; pageW: number; pageH: number } | null {
|
||||
|
||||
@@ -10,7 +10,6 @@ export interface HeatmapEventRow {
|
||||
visitId: string;
|
||||
urlPath: string;
|
||||
eventType: number;
|
||||
nodeId: number | null;
|
||||
x: number | null;
|
||||
y: number | null;
|
||||
pageX: number | null;
|
||||
@@ -21,9 +20,6 @@ export interface HeatmapEventRow {
|
||||
pageH: number | null;
|
||||
scrollPct: number | null;
|
||||
createdAt: Date;
|
||||
replayChunkIndex: number | null;
|
||||
replayEventIndex: number | null;
|
||||
replayTimeMs: number | null;
|
||||
}
|
||||
|
||||
export async function saveHeatmapEvents(rows: HeatmapEventRow[]) {
|
||||
@@ -31,7 +27,6 @@ export async function saveHeatmapEvents(rows: HeatmapEventRow[]) {
|
||||
|
||||
const normalizedRows = rows.map(r => ({
|
||||
...r,
|
||||
nodeId: toInt(r.nodeId),
|
||||
x: toInt(r.x),
|
||||
y: toInt(r.y),
|
||||
pageX: toInt(r.pageX),
|
||||
@@ -70,7 +65,6 @@ async function relationalQuery(rows: HeatmapEventRow[]) {
|
||||
visitId: r.visitId,
|
||||
urlPath: r.urlPath,
|
||||
eventType: r.eventType,
|
||||
nodeId: r.nodeId,
|
||||
x: r.x,
|
||||
y: r.y,
|
||||
pageX: r.pageX,
|
||||
@@ -81,9 +75,6 @@ async function relationalQuery(rows: HeatmapEventRow[]) {
|
||||
pageH: r.pageH,
|
||||
scrollPct: r.scrollPct,
|
||||
createdAt: r.createdAt,
|
||||
replayChunkIndex: r.replayChunkIndex,
|
||||
replayEventIndex: r.replayEventIndex,
|
||||
replayTimeMs: r.replayTimeMs,
|
||||
})) as any,
|
||||
});
|
||||
}
|
||||
@@ -99,7 +90,6 @@ async function clickhouseQuery(rows: HeatmapEventRow[]) {
|
||||
visit_id: r.visitId,
|
||||
url_path: r.urlPath,
|
||||
event_type: r.eventType,
|
||||
node_id: r.nodeId,
|
||||
x: r.x,
|
||||
y: r.y,
|
||||
page_x: r.pageX,
|
||||
@@ -110,9 +100,6 @@ async function clickhouseQuery(rows: HeatmapEventRow[]) {
|
||||
page_h: r.pageH,
|
||||
scroll_pct: r.scrollPct,
|
||||
created_at: getUTCString(r.createdAt),
|
||||
replay_chunk_index: r.replayChunkIndex,
|
||||
replay_event_index: r.replayEventIndex,
|
||||
replay_time_ms: r.replayTimeMs,
|
||||
}));
|
||||
|
||||
if (kafka.enabled) {
|
||||
|
||||
Reference in New Issue
Block a user