From 500460d0712ac588bad6d911d8627ae0579f8cfa Mon Sep 17 00:00:00 2001
From: crosstyan <crosstyan@outlook.com>
Date: Mon, 24 Mar 2025 15:39:43 +0800
Subject: [PATCH] Implement camera view handling with extrinsic and intrinsic
 matrices, add text rendering capabilities, and refactor Scene component for
 improved structure.

---
 src/App.tsx | 112 +++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 106 insertions(+), 6 deletions(-)

diff --git a/src/App.tsx b/src/App.tsx
index c8d34df..049370c 100644
--- a/src/App.tsx
+++ b/src/App.tsx
@@ -1,10 +1,71 @@
 import { Grid, useBVH, useGLTF, CameraControls, AccumulativeShadows, OrbitControls, Stats } from '@react-three/drei'
 import { Camera, Canvas, useFrame, useThree, useLoader } from '@react-three/fiber'
 import * as THREE from 'three'
-import { useState } from 'react'
+import { FontLoader } from 'three/addons/loaders/FontLoader.js'
+import { TextGeometry } from 'three/addons/geometries/TextGeometry.js'
+import HelvetikerRegular from "three/examples/fonts/helvetiker_regular.typeface.json"
+import { useEffect, useRef, useState, JSX } from 'react'
+
+const THREE_ADDONS = {
+  FontLoader,
+  TextGeometry,
+} as const
+
 
 const DEFAULT_NEAR = 0.1
-const DEFAULT_FAR = 5
+const DEFAULT_FAR = 1
+const CAMERA_EXTRINSIC_MATRIX_MAP: Record<string, number[]> = {
+  "AE_01": [
+    0.2321047, -0.97264263, -0.0096808, -0.96323585, 0.06882254,
+    0.02634936, -0.99728089, 0.03661007, 0.97025299, 0.23080732,
+    0.07305554, 3.34933242, 0., 0., 0.,
+    1.
+  ],
+  "AE_1A": [
+    0.93194049, -0.35571886, -0.07036343, -0.92123075, 0.01084819,
+    0.22131041, -0.97514308, 0.24922173, 0.36244895, 0.908012,
+    0.21010704, 4.87284891, 0., 0., 0.,
+    1.
+  ],
+  "AE_08": [
+    0.66806102, -0.74355508, -0.02864123, -1.10173496, 0.05931037,
+    0.09157787, -0.99403007, 0.26760438, 0.74173901, 0.66237402,
+    0.10528013, 6.92372493, 0., 0., 0.,
+    1.
+  ]
+} as const
+const CAMERA_INTRINSIC_MATRIX_MAP: Record<string, number[]> = {
+  "AE_01": [
+    1806.82137617, 0., 1230.53175624, 0.,
+    1809.75580378, 766.36204406, 0., 0.,
+    1.
+  ],
+  "AE_1A": [
+    3467.39715751, 0., 1000.62548655, 0.,
+    3473.7168112, 831.64048503, 0., 0.,
+    1.
+  ],
+  "AE_08": [
+    2785.43931794, 0., 1254.98272372, 0.,
+    2788.10437965, 738.82985324, 0., 0.,
+    1.
+  ]
+} as const
+
+const IMAGE_WIDTH = 2560
+const IMAGE_HEIGHT = 1440
+
+const intrinsicToFov = (intrinsic: number[], image_size: { width: number, height: number }) => {
+  console.assert(intrinsic.length === 9, "intrinsic must be a 3x3 matrix")
+  const fx = intrinsic[0]
+  const fy = intrinsic[4]
+  const cx = intrinsic[2]
+  const cy = intrinsic[5]
+  // in degrees
+  const fov_x = 2 * Math.atan(image_size.width / (2 * fx)) * (180 / Math.PI)
+  const fov_y = 2 * Math.atan(image_size.height / (2 * fy)) * (180 / Math.PI)
+  return { fov_x, fov_y }
+}
 
 const Scene = () => {
   function Floor() {
@@ -18,12 +79,47 @@ const Scene = () => {
   const Axes = () => {
     return <axesHelper args={[15]} />
   }
-  const ExampleCameraView = () => {
-    const camera = new THREE.PerspectiveCamera(60, 4 / 3, DEFAULT_NEAR, DEFAULT_FAR)
+  interface CameraViewFromExtrinsicProps {
+    extrinsic: number[]
+    aspect?: number
+    name?: string
+    near?: number
+    far?: number
+    fov?: number
+    textSize?: number
+  }
+  // https://threejs.org/docs/#examples/en/loaders/FontLoader
+  // https://www.ilyameerovich.com/simple-3d-text-meshes-in-three-js/
+
+  const CameraViewFromExtrinsic = ({ extrinsic, name, near, far, fov, textSize, aspect }: CameraViewFromExtrinsicProps) => {
+    const font = new FontLoader().parse(HelvetikerRegular)
+    const camera = new THREE.PerspectiveCamera(fov ?? 60, aspect ?? 4 / 3, near ?? DEFAULT_NEAR, far ?? DEFAULT_FAR)
     const helper = <cameraHelper args={[camera]} />
-    camera.position.set(0, 5, 0)
+    // @ts-expect-error 16 elements
+    camera.applyMatrix4(new THREE.Matrix4(...extrinsic))
+    let text: JSX.Element | null = null
+    if (name) {
+      // https://github.com/pmndrs/react-three-fiber/discussions/832
+      const geo = new THREE_ADDONS.TextGeometry(name ?? "", { font, size: textSize ?? 0.1, depth: 0.001 })
+
+      // Extract camera position from extrinsic matrix
+      const matrix = new THREE.Matrix4()
+      // @ts-expect-error 16 elements
+      matrix.set(...extrinsic)
+      const position = new THREE.Vector3()
+      position.setFromMatrixPosition(matrix)
+
+      // Create text without applying camera matrix transformation
+      text = (
+        <mesh position={position}>
+          <primitive object={geo} />
+          <meshStandardMaterial color="black" />
+        </mesh>
+      )
+    }
     return (
       <group>
+        {text}
         <primitive object={camera} />
         {helper}
       </group>
@@ -41,7 +137,11 @@ const Scene = () => {
       <ambientLight intensity={0.05} />
       <directionalLight castShadow position={[3.3, 6, 4.4]} intensity={5} />
       <Floor />
-      <ExampleCameraView />
+      {Object.entries(CAMERA_EXTRINSIC_MATRIX_MAP).map(([key, value]) => {
+        const intrinsic = CAMERA_INTRINSIC_MATRIX_MAP[key]
+        const fov = intrinsicToFov(intrinsic, { width: IMAGE_WIDTH, height: IMAGE_HEIGHT })
+        return <CameraViewFromExtrinsic key={key} name={key} extrinsic={value} fov={fov.fov_x} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} />
+      })}
       <Axes />
       <Stats />
     </>