diff --git a/db/clickhouse/migrations/12_add_heatmap.sql b/db/clickhouse/migrations/12_add_heatmap.sql index ba5f93cb7..db44666b3 100644 --- a/db/clickhouse/migrations/12_add_heatmap.sql +++ b/db/clickhouse/migrations/12_add_heatmap.sql @@ -26,3 +26,25 @@ ENGINE = MergeTree PARTITION BY toYYYYMM(created_at) ORDER BY (website_id, url_path, event_type, created_at) SETTINGS index_granularity = 8192; + +-- Create heatmap_snapshot +CREATE TABLE umami.heatmap_snapshot +( + snapshot_id UUID, + website_id UUID, + url_path String, + viewport_w UInt32, + viewport_h UInt32, + page_w UInt32, + page_h UInt32, + status UInt8, + mime_type LowCardinality(String), + object_key String, + image_size Nullable(UInt32), + error Nullable(String), + created_at DateTime('UTC') +) +ENGINE = MergeTree + PARTITION BY toYYYYMM(created_at) + ORDER BY (website_id, url_path, viewport_w, viewport_h, created_at) + SETTINGS index_granularity = 8192; diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index a075977f1..75943488c 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -428,3 +428,25 @@ ENGINE = MergeTree PARTITION BY toYYYYMM(created_at) ORDER BY (website_id, url_path, event_type, created_at) SETTINGS index_granularity = 8192; + +-- Create heatmap_snapshot +CREATE TABLE umami.heatmap_snapshot +( + snapshot_id UUID, + website_id UUID, + url_path String, + viewport_w UInt32, + viewport_h UInt32, + page_w UInt32, + page_h UInt32, + status UInt8, + mime_type LowCardinality(String), + object_key String, + image_size Nullable(UInt32), + error Nullable(String), + created_at DateTime('UTC') +) +ENGINE = MergeTree + PARTITION BY toYYYYMM(created_at) + ORDER BY (website_id, url_path, viewport_w, viewport_h, created_at) + SETTINGS index_granularity = 8192; diff --git a/package.json b/package.json index fd263afcf..72f9a47b5 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ ".next/cache" ], "dependencies": { + "@aws-sdk/client-s3": "^3.924.0", "@clickhouse/client": "^1.18.5", "@date-fns/utc": "^2.1.1", "@dicebear/collection": "^9.4.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c4f918851..7bf76d6af 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + '@aws-sdk/client-s3': + specifier: ^3.924.0 + version: 3.1050.0 '@clickhouse/client': specifier: ^1.18.5 version: 1.18.5 @@ -352,6 +355,125 @@ packages: '@asamuzakjp/nwsapi@2.3.9': resolution: {integrity: sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==} + '@aws-crypto/crc32@5.2.0': + resolution: {integrity: sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==} + engines: {node: '>=16.0.0'} + + '@aws-crypto/crc32c@5.2.0': + resolution: {integrity: sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==} + + '@aws-crypto/sha1-browser@5.2.0': + resolution: {integrity: sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==} + + '@aws-crypto/sha256-browser@5.2.0': + resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} + + '@aws-crypto/sha256-js@5.2.0': + resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==} + engines: {node: '>=16.0.0'} + + '@aws-crypto/supports-web-crypto@5.2.0': + resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==} + + '@aws-crypto/util@5.2.0': + resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==} + + '@aws-sdk/client-s3@3.1050.0': + resolution: {integrity: sha512-9kgtv+bXZQrOIJT2INPPBCezrJu1FlgGrzEat/ut4A4V53IT00LynsBZgp12eFKbjJuNCeTo7iPSKjPsX8ub+A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.974.12': + resolution: {integrity: sha512-qrqgioqYFjwR6LatVNS1L2Vk++EwRIxqSQXPKNv5Ofux2D8UNgqMQ1znnMyEImXquVPTtbf71fc128pvmU6y9A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/crc64-nvme@3.972.8': + resolution: {integrity: sha512-fVfUCL/Xh2zINYMPZvj+iBn6XWouQf0DAnjaWCI9MkmqXzL2Iy5FoQB8O7syFe6gN6AH1ecDDU58T51Ou0kFkA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.38': + resolution: {integrity: sha512-m3WjZEgPtioMhPmwqUt+DhlTJ2i9ufR6DhfkyXojb9puEvfR+ur2U5shavu5/Cc9WHHsDCvALi6UFHgcqjhQ5w==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.40': + resolution: {integrity: sha512-D78L/m2Dr6cJnnSvWoAudPhQmCwmJ7j6APXsPYmFpPaKfQTfCSu0rdm8j14Np+VmXF9z8Aj8HE3xFpsrwtfgeg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.972.42': + resolution: {integrity: sha512-Mu5ESvFXeinafVM8jTIvRqcvK2Ehj4kz3auT39yUcHwu1Vfxo6xRlmUafdKLW4tusjAJukQwK09sCSMgOm7OKg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.42': + resolution: {integrity: sha512-O6WkZga3kf0yqyJYd1dbeJqVhEgJx/x1UaLgtbR+XuL/YP+K5y6QTxQKL7ka9z3jnQASESKGAPnRyt4D5hQrxA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.43': + resolution: {integrity: sha512-D/DJmbrWRP5BXEO3FH+ar4el+2n6OlGofiud7dQun2jES+AQEJjczenp1jBb4MBN7CpGpS8nsWGQLtuzc9tQbA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.38': + resolution: {integrity: sha512-EnbYVajGgbkb24s0K1eo4VNAPV5mHIET7LSvirTaFCwkfrfaOJxtSE+wY/tJdKDS21cEYkZs2ruCaAm+W4iblg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.972.42': + resolution: {integrity: sha512-RVV/9NbFwI8ZHEH5dn39lGyFmSbSVj1+orZdr6QsOe1mW9DCglmlen0cFaNZmCcqkqc7erNRHNBduxbeZuHAnw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.42': + resolution: {integrity: sha512-/67fXX0ddllD4u2Nujc5PvT4byHgpMUfz6+RxIKi/0nFIckeorm7JvXgzBuDyVKw0s58EbofmETDWUf9vTEuHQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-bucket-endpoint@3.972.14': + resolution: {integrity: sha512-Aaj0d+xbo1jJquBWJP0/9V/XZRYukO3LWIRp3dOLHmoFrYKb4YZ0aLefgVHfGcNOVBS2ZTq7L/n5JcrE7DaC+Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-expect-continue@3.972.12': + resolution: {integrity: sha512-dA5pKTom/Ls9mgeyeaRBNQrRIVOLVjv4AmKOB0/e4yaiXEUy0gSz2d3liP8JHtYoCAEWySU1jWnyzwLOREN+4g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-flexible-checksums@3.974.20': + resolution: {integrity: sha512-NdnMVQCR1YjIcqFAiNLdBiOwr2DyQDB2IiXQrBhzolKOv32ae4d4Ll7IzLMi04eMHiq/o/Y/GjFuVjF9HuG0QA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-location-constraint@3.972.10': + resolution: {integrity: sha512-rI3NZvJcEvjoD0+0PI0iUAwlPw2IlSlhyvgBK/3WkKJQE/YiKFedd9dMN2lVacdNxPNhxL/jzQaKQdrGtQagjQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-s3@3.972.41': + resolution: {integrity: sha512-M4T2I2WPuH5WQpU8Tsp+u2bcO29zGRkU14ATzuqb9I4xh8tzsLqtp4hzaJM5aO2dhMZnHDzyQwSFVgc3XbnoGg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-ssec@3.972.10': + resolution: {integrity: sha512-Gli9A0u8EVVb+5bFDGS/QbSVg28w/wpEidg1ggVcSj65BDTdGR6punsOcVjqdiu1i42WHWo51MCvARPIIz9juw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.997.10': + resolution: {integrity: sha512-FtQ/Bt327peZJuyo4WZSOLVUTw9ujRxntepiC7L65FxA2P82Xlq0g14T22BuqBUeMjDoxa9nvwiMHjLIfP3eUg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.27': + resolution: {integrity: sha512-0Phbz4t6HI3D3skxvG2uI+VWU034/nSIw1T8d+FPzzQG9EQTrw94o9mOKO2Gv3n3Oc8P7JD7RAUxkoneLWv5Eg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1049.0': + resolution: {integrity: sha512-r7+d0lQMTHKypkmaF5jRTBYLYHCUHzt3gaVoN9SidLhQeWhCmHk3AKrboDTpPF5b7Pt7vKu3+oeMjznM2Eu1ow==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.973.8': + resolution: {integrity: sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/util-locate-window@3.965.5': + resolution: {integrity: sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.24': + resolution: {integrity: sha512-V8z5YcDPfsvzrBlj0xR1vhRtocblhYbqdreCJB/voGd4Sr5zjNAeWxexbnqVtskTJe0vFb5KMqbSL++ePl+zRw==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.2.4': + resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==} + engines: {node: '>=18.0.0'} + '@babel/code-frame@7.27.1': resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==} engines: {node: '>=6.9.0'} @@ -1753,6 +1875,9 @@ packages: cpu: [x64] os: [win32] + '@nodable/entities@2.1.0': + resolution: {integrity: sha512-nyT7T3nbMyBI/lvr6L5TyWbFJAI9FTgVRakNoBqCD+PmID8DzFrrNdLLtHMwMszOtqZa8PAOV24ZqDnQrhQINA==} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -2430,6 +2555,42 @@ packages: resolution: {integrity: sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==} engines: {node: '>=18'} + '@smithy/core@3.24.3': + resolution: {integrity: sha512-Ep/7tPamGY8mgESE3LyLKtxJyy6U52WWAqr/3wial47Sj4u3PiIF73AOGI27UyLy9duTkhZbgzodOfLV4TduZg==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.3.3': + resolution: {integrity: sha512-I2Bti0DKFo2IJyN28ijCsx51BAumEYR4/1yZ1FXyBygy9MqbnMqCev4JPth/MbpRfBSRAX35hITSnAdJRo1u5w==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.4.3': + resolution: {integrity: sha512-F+DRf8IJazRJgYog2A/yJK7eYVc0rqTlRzO+5ZxjJd4WkZoKz0IJRncf7G6t1pdVT3kryJcwuTFhN1c5m6N47A==} + engines: {node: '>=18.0.0'} + + '@smithy/is-array-buffer@2.2.0': + resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==} + engines: {node: '>=14.0.0'} + + '@smithy/node-http-handler@4.7.3': + resolution: {integrity: sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.4.3': + resolution: {integrity: sha512-53+75QuPl6DL+ct6vVEB51FDO5oulXr20TPV46VvJZg76lIlXNWfxi8j+G2V/t0I2qxCBOa3vX/8bmjrpFVo9g==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.14.2': + resolution: {integrity: sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==} + engines: {node: '>=18.0.0'} + + '@smithy/util-buffer-from@2.2.0': + resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==} + engines: {node: '>=14.0.0'} + + '@smithy/util-utf8@2.3.0': + resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==} + engines: {node: '>=14.0.0'} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -2906,6 +3067,9 @@ packages: boolbase@1.0.0: resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + brace-expansion@1.1.12: resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} @@ -3551,6 +3715,13 @@ packages: fast-wrap-ansi@0.2.0: resolution: {integrity: sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w==} + fast-xml-builder@1.2.0: + resolution: {integrity: sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==} + + fast-xml-parser@5.7.3: + resolution: {integrity: sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==} + hasBin: true + fastq@1.19.1: resolution: {integrity: sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==} @@ -4512,6 +4683,10 @@ packages: path-browserify@1.0.1: resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==} + path-expression-matcher@1.5.0: + resolution: {integrity: sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==} + engines: {node: '>=14.0.0'} + path-is-absolute@1.0.1: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} @@ -5615,6 +5790,9 @@ packages: resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==} engines: {node: '>=8'} + strnum@2.3.0: + resolution: {integrity: sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==} + style-inject@0.3.0: resolution: {integrity: sha512-IezA2qp+vcdlhJaVm5SOdPPTUu0FCEqfNSli2vRuSIBbu5Nq5UvygTk/VzeCqfLz2Atj3dVII5QBKGZRZ0edzw==} @@ -6065,6 +6243,10 @@ packages: resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==} engines: {node: '>=18'} + xml-naming@0.1.0: + resolution: {integrity: sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==} + engines: {node: '>=16.0.0'} + xmlchars@2.2.0: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} @@ -6152,6 +6334,271 @@ snapshots: '@asamuzakjp/nwsapi@2.3.9': {} + '@aws-crypto/crc32@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.8 + tslib: 2.8.1 + + '@aws-crypto/crc32c@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.8 + tslib: 2.8.1 + + '@aws-crypto/sha1-browser@5.2.0': + dependencies: + '@aws-crypto/supports-web-crypto': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.8 + '@aws-sdk/util-locate-window': 3.965.5 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-crypto/sha256-browser@5.2.0': + dependencies: + '@aws-crypto/sha256-js': 5.2.0 + '@aws-crypto/supports-web-crypto': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.8 + '@aws-sdk/util-locate-window': 3.965.5 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-crypto/sha256-js@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.8 + tslib: 2.8.1 + + '@aws-crypto/supports-web-crypto@5.2.0': + dependencies: + tslib: 2.8.1 + + '@aws-crypto/util@5.2.0': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-sdk/client-s3@3.1050.0': + dependencies: + '@aws-crypto/sha1-browser': 5.2.0 + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.12 + '@aws-sdk/credential-provider-node': 3.972.43 + '@aws-sdk/middleware-bucket-endpoint': 3.972.14 + '@aws-sdk/middleware-expect-continue': 3.972.12 + '@aws-sdk/middleware-flexible-checksums': 3.974.20 + '@aws-sdk/middleware-location-constraint': 3.972.10 + '@aws-sdk/middleware-sdk-s3': 3.972.41 + '@aws-sdk/middleware-ssec': 3.972.10 + '@aws-sdk/signature-v4-multi-region': 3.996.27 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/fetch-http-handler': 5.4.3 + '@smithy/node-http-handler': 4.7.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/core@3.974.12': + dependencies: + '@aws-sdk/types': 3.973.8 + '@aws-sdk/xml-builder': 3.972.24 + '@aws/lambda-invoke-store': 0.2.4 + '@smithy/core': 3.24.3 + '@smithy/signature-v4': 5.4.3 + '@smithy/types': 4.14.2 + bowser: 2.14.1 + tslib: 2.8.1 + + '@aws-sdk/crc64-nvme@3.972.8': + dependencies: + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.38': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.40': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/fetch-http-handler': 5.4.3 + '@smithy/node-http-handler': 4.7.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.972.42': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/credential-provider-env': 3.972.38 + '@aws-sdk/credential-provider-http': 3.972.40 + '@aws-sdk/credential-provider-login': 3.972.42 + '@aws-sdk/credential-provider-process': 3.972.38 + '@aws-sdk/credential-provider-sso': 3.972.42 + '@aws-sdk/credential-provider-web-identity': 3.972.42 + '@aws-sdk/nested-clients': 3.997.10 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/credential-provider-imds': 4.3.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-login@3.972.42': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/nested-clients': 3.997.10 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-node@3.972.43': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.38 + '@aws-sdk/credential-provider-http': 3.972.40 + '@aws-sdk/credential-provider-ini': 3.972.42 + '@aws-sdk/credential-provider-process': 3.972.38 + '@aws-sdk/credential-provider-sso': 3.972.42 + '@aws-sdk/credential-provider-web-identity': 3.972.42 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/credential-provider-imds': 4.3.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.38': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.972.42': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/nested-clients': 3.997.10 + '@aws-sdk/token-providers': 3.1049.0 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.42': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/nested-clients': 3.997.10 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/middleware-bucket-endpoint@3.972.14': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/middleware-expect-continue@3.972.12': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/middleware-flexible-checksums@3.974.20': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@aws-crypto/crc32c': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/core': 3.974.12 + '@aws-sdk/crc64-nvme': 3.972.8 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/middleware-location-constraint@3.972.10': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-s3@3.972.41': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/signature-v4-multi-region': 3.996.27 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/signature-v4': 5.4.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/middleware-ssec@3.972.10': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.997.10': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.12 + '@aws-sdk/signature-v4-multi-region': 3.996.27 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/fetch-http-handler': 5.4.3 + '@smithy/node-http-handler': 4.7.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.996.27': + dependencies: + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/signature-v4': 5.4.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1049.0': + dependencies: + '@aws-sdk/core': 3.974.12 + '@aws-sdk/nested-clients': 3.997.10 + '@aws-sdk/types': 3.973.8 + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/types@3.973.8': + dependencies: + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/util-locate-window@3.965.5': + dependencies: + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.24': + dependencies: + '@nodable/entities': 2.1.0 + '@smithy/types': 4.14.2 + fast-xml-parser: 5.7.3 + tslib: 2.8.1 + + '@aws/lambda-invoke-store@0.2.4': {} + '@babel/code-frame@7.27.1': dependencies: '@babel/helper-validator-identifier': 7.27.1 @@ -7381,6 +7828,8 @@ snapshots: '@next/swc-win32-x64-msvc@16.2.6': optional: true + '@nodable/entities@2.1.0': {} + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -7941,6 +8390,54 @@ snapshots: '@sindresorhus/merge-streams@2.3.0': {} + '@smithy/core@3.24.3': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.3.3': + dependencies: + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@smithy/fetch-http-handler@5.4.3': + dependencies: + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@smithy/is-array-buffer@2.2.0': + dependencies: + tslib: 2.8.1 + + '@smithy/node-http-handler@4.7.3': + dependencies: + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@smithy/signature-v4@5.4.3': + dependencies: + '@smithy/core': 3.24.3 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@smithy/types@4.14.2': + dependencies: + tslib: 2.8.1 + + '@smithy/util-buffer-from@2.2.0': + dependencies: + '@smithy/is-array-buffer': 2.2.0 + tslib: 2.8.1 + + '@smithy/util-utf8@2.3.0': + dependencies: + '@smithy/util-buffer-from': 2.2.0 + tslib: 2.8.1 + '@standard-schema/spec@1.1.0': {} '@svgr/babel-plugin-add-jsx-attribute@8.0.0(@babel/core@7.28.3)': @@ -8412,6 +8909,8 @@ snapshots: boolbase@1.0.0: {} + bowser@2.14.1: {} + brace-expansion@1.1.12: dependencies: balanced-match: 1.0.2 @@ -9169,6 +9668,18 @@ snapshots: dependencies: fast-string-width: 3.0.2 + fast-xml-builder@1.2.0: + dependencies: + path-expression-matcher: 1.5.0 + xml-naming: 0.1.0 + + fast-xml-parser@5.7.3: + dependencies: + '@nodable/entities': 2.1.0 + fast-xml-builder: 1.2.0 + path-expression-matcher: 1.5.0 + strnum: 2.3.0 + fastq@1.19.1: dependencies: reusify: 1.1.0 @@ -10109,6 +10620,8 @@ snapshots: path-browserify@1.0.1: {} + path-expression-matcher@1.5.0: {} + path-is-absolute@1.0.1: {} path-key@2.0.1: {} @@ -11342,6 +11855,8 @@ snapshots: dependencies: min-indent: 1.0.1 + strnum@2.3.0: {} + style-inject@0.3.0: {} styled-jsx@5.1.6(@babel/core@7.29.0)(react@19.2.6): @@ -11786,6 +12301,8 @@ snapshots: xml-name-validator@5.0.0: {} + xml-naming@0.1.0: {} + xmlchars@2.2.0: {} xtend@4.0.2: {} diff --git a/src/lib/heatmap-s3.ts b/src/lib/heatmap-s3.ts new file mode 100644 index 000000000..ccb234d47 --- /dev/null +++ b/src/lib/heatmap-s3.ts @@ -0,0 +1,56 @@ +import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; + +let client: S3Client | null = null; + +function getBucket() { + const bucket = process.env.S3_HEATMAP_BUCKET; + + if (!bucket) { + throw new Error('S3_HEATMAP_BUCKET is not set.'); + } + + return bucket; +} + +function getClient() { + if (!client) { + client = new S3Client({ + region: process.env.AWS_REGION, + }); + } + + return client; +} + +export async function putHeatmapSnapshot( + objectKey: string, + imageData: Buffer, + mimeType: string, +) { + await getClient().send( + new PutObjectCommand({ + Bucket: getBucket(), + Key: objectKey, + Body: imageData, + ContentType: mimeType, + }), + ); +} + +export async function getHeatmapSnapshot(objectKey: string) { + const response = await getClient().send( + new GetObjectCommand({ + Bucket: getBucket(), + Key: objectKey, + }), + ); + + if (!response.Body) { + return null; + } + + return { + mimeType: response.ContentType || 'application/octet-stream', + imageData: Buffer.from(await response.Body.transformToByteArray()), + }; +} diff --git a/src/queries/sql/heatmap/ensureHeatmapSnapshot.ts b/src/queries/sql/heatmap/ensureHeatmapSnapshot.ts index 90496563a..49beb663a 100644 --- a/src/queries/sql/heatmap/ensureHeatmapSnapshot.ts +++ b/src/queries/sql/heatmap/ensureHeatmapSnapshot.ts @@ -1,5 +1,7 @@ +import clickhouse from '@/lib/clickhouse'; import prisma from '@/lib/prisma'; import { uuid } from '@/lib/crypto'; +import { getHeatmapSnapshot, putHeatmapSnapshot } from '@/lib/heatmap-s3'; import { getWebsite } from '@/queries/prisma'; const SNAPSHOT_STATUS = { @@ -10,7 +12,6 @@ const SNAPSHOT_STATUS = { const SNAPSHOT_RETRY_DELAY_MS = 15 * 60 * 1000; const SNAPSHOT_PENDING_WINDOW_MS = 30 * 1000; - export type HeatmapSnapshotStatus = (typeof SNAPSHOT_STATUS)[keyof typeof SNAPSHOT_STATUS]; export interface HeatmapSnapshotImage { @@ -35,6 +36,7 @@ interface SnapshotRecord { pageH: number; status: HeatmapSnapshotStatus; mimeType: string | null; + objectKey: string | null; imageSize: number | null; error: string | null; hasImage: boolean; @@ -57,6 +59,12 @@ interface CaptureResult { pageH: number; } +const CLICKHOUSE_SNAPSHOT_STATUS = { + pending: 0, + ready: 1, + failed: 2, +} as const; + async function measurePage(page: any) { return page.evaluate(() => { const doc = document.documentElement; @@ -143,6 +151,19 @@ async function findSnapshot( urlPath: string, viewportW: number, viewportH: number, +): Promise { + if (clickhouse.enabled) { + return findClickhouseSnapshot(websiteId, urlPath, viewportW, viewportH); + } + + return findRelationalSnapshot(websiteId, urlPath, viewportW, viewportH); +} + +async function findRelationalSnapshot( + websiteId: string, + urlPath: string, + viewportW: number, + viewportH: number, ): Promise { const rows = await prisma.rawQuery( ` @@ -156,6 +177,7 @@ async function findSnapshot( page_h as "pageH", status, mime_type as "mimeType", + null as "objectKey", image_size as "imageSize", error, image_data is not null as "hasImage", @@ -174,6 +196,78 @@ async function findSnapshot( return rows?.[0] ?? null; } +async function findClickhouseSnapshot( + websiteId: string, + urlPath: string, + viewportW: number, + viewportH: number, +): Promise { + const rows = await clickhouse.rawQuery< + { + id: string; + websiteId: string; + urlPath: string; + viewportW: number; + viewportH: number; + pageW: number; + pageH: number; + status: number; + mimeType: string | null; + objectKey: string; + imageSize: number | null; + error: string | null; + createdAt: string; + }[] + >( + ` + select + snapshot_id as id, + website_id as websiteId, + url_path as urlPath, + viewport_w as viewportW, + viewport_h as viewportH, + page_w as pageW, + page_h as pageH, + status, + mime_type as mimeType, + object_key as objectKey, + image_size as imageSize, + error, + created_at as createdAt + from heatmap_snapshot + where website_id = {websiteId:UUID} + and url_path = {urlPath:String} + and viewport_w = {viewportW:UInt32} + and viewport_h = {viewportH:UInt32} + order by created_at desc + limit 1 + `, + { websiteId, urlPath, viewportW, viewportH }, + 'findHeatmapSnapshot', + ); + + const row = rows?.[0]; + + if (!row) { + return null; + } + + const status = Object.entries(CLICKHOUSE_SNAPSHOT_STATUS).find(([, value]) => value === row.status)?.[0]; + + if (!status) { + return null; + } + + return { + ...row, + status: SNAPSHOT_STATUS[status as keyof typeof SNAPSHOT_STATUS], + mimeType: row.mimeType || null, + objectKey: row.objectKey || null, + hasImage: row.status === CLICKHOUSE_SNAPSHOT_STATUS.ready && Boolean(row.objectKey), + updatedAt: row.createdAt, + }; +} + function getSnapshotImageUrl(websiteId: string, snapshotId: string) { return `/api/websites/${websiteId}/heatmaps/snapshots/${snapshotId}`; } @@ -231,6 +325,65 @@ export function shouldSkipSnapshot(urlPath: string) { } async function upsertSnapshotRecord({ + id, + websiteId, + urlPath, + viewportW, + viewportH, + pageW, + pageH, + status, + mimeType, + imageData, + objectKey, + error, +}: { + id: string; + websiteId: string; + urlPath: string; + viewportW: number; + viewportH: number; + pageW: number; + pageH: number; + status: HeatmapSnapshotStatus; + mimeType: string | null; + imageData: Buffer | null; + objectKey?: string | null; + error: string | null; +}) { + if (clickhouse.enabled) { + return insertClickhouseSnapshotRecord({ + id, + websiteId, + urlPath, + viewportW, + viewportH, + pageW, + pageH, + status, + mimeType, + objectKey: objectKey ?? null, + imageSize: imageData?.byteLength ?? null, + error, + }); + } + + return upsertRelationalSnapshotRecord({ + id, + websiteId, + urlPath, + viewportW, + viewportH, + pageW, + pageH, + status, + mimeType, + imageData, + error, + }); +} + +async function upsertRelationalSnapshotRecord({ id, websiteId, urlPath, @@ -317,6 +470,56 @@ async function upsertSnapshotRecord({ ); } +async function insertClickhouseSnapshotRecord({ + id, + websiteId, + urlPath, + viewportW, + viewportH, + pageW, + pageH, + status, + mimeType, + objectKey, + imageSize, + error, +}: { + id: string; + websiteId: string; + urlPath: string; + viewportW: number; + viewportH: number; + pageW: number; + pageH: number; + status: HeatmapSnapshotStatus; + mimeType: string | null; + objectKey: string | null; + imageSize: number | null; + error: string | null; +}) { + return clickhouse.insert('heatmap_snapshot', [ + { + snapshot_id: id, + website_id: websiteId, + url_path: urlPath, + viewport_w: viewportW, + viewport_h: viewportH, + page_w: pageW, + page_h: pageH, + status: CLICKHOUSE_SNAPSHOT_STATUS[status], + mime_type: mimeType || '', + object_key: objectKey || '', + image_size: imageSize, + error, + created_at: clickhouse.getUTCString(), + }, + ]); +} + +function getSnapshotObjectKey(websiteId: string, snapshotId: string, viewportW: number, viewportH: number) { + return `heatmaps/${websiteId}/${viewportW}x${viewportH}/${snapshotId}.png`; +} + async function captureSnapshot( url: string, viewportW: number, @@ -456,6 +659,14 @@ export async function ensureHeatmapSnapshot({ try { const capture = await captureSnapshot(captureUrl, viewportW, viewportH, pageW); + const objectKey = + clickhouse.enabled + ? getSnapshotObjectKey(websiteId, snapshotId, viewportW, viewportH) + : null; + + if (objectKey) { + await putHeatmapSnapshot(objectKey, capture.imageData, capture.mimeType); + } await upsertSnapshotRecord({ id: snapshotId, @@ -467,7 +678,8 @@ export async function ensureHeatmapSnapshot({ pageH: capture.pageH, status: SNAPSHOT_STATUS.ready, mimeType: capture.mimeType, - imageData: capture.imageData, + imageData: clickhouse.enabled ? null : capture.imageData, + objectKey, error: null, }); } catch (error) { @@ -495,6 +707,39 @@ export async function getHeatmapSnapshotImage( websiteId: string, snapshotId: string, ): Promise<{ mimeType: string; imageData: Buffer } | null> { + if (clickhouse.enabled) { + const rows = await clickhouse.rawQuery< + { mimeType: string; objectKey: string }[] + >( + ` + select + mime_type as mimeType, + object_key as objectKey + from heatmap_snapshot + where snapshot_id = {snapshotId:UUID} + and website_id = {websiteId:UUID} + and status = {status:UInt8} + and object_key != '' + order by created_at desc + limit 1 + `, + { + websiteId, + snapshotId, + status: CLICKHOUSE_SNAPSHOT_STATUS.ready, + }, + 'getHeatmapSnapshotImage', + ); + + const row = rows?.[0]; + + if (!row?.objectKey) { + return null; + } + + return getHeatmapSnapshot(row.objectKey); + } + const rows = await prisma.rawQuery( ` select