From f70c25a5f667487d64f6e71170d4202e171f4be1 Mon Sep 17 00:00:00 2001
From: crosstyan <crosstyan@outlook.com>
Date: Mon, 24 Mar 2025 17:25:05 +0800
Subject: [PATCH] Add OpenCV to OpenGL conversion matrix and update
 CameraViewFromExtrinsic to accept both extrinsic arrays and Matrix4.
 Introduce pre-processing for extrinsic matrices to ensure correct camera
 orientation.

---
 src/App.tsx | 41 +++++++++++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/src/App.tsx b/src/App.tsx
index 607c290..29b15e0 100644
--- a/src/App.tsx
+++ b/src/App.tsx
@@ -11,6 +11,16 @@ const THREE_ADDONS = {
   TextGeometry,
 } as const
 
+// Create OpenCV to OpenGL conversion matrix
+// OpenCV: X right, Y down, Z forward
+// OpenGL: X right, Y up, Z backward
+const CV_TO_GL_MAT = new THREE.Matrix4().set(
+  1, 0, 0, 0,
+  0, -1, 0, 0,
+  0, 0, -1, 0,
+  0, 0, 0, 1
+)
+
 
 const DEFAULT_TRANSFORMATION_MATRIX = [
   1, 0, 0, 0,
@@ -86,7 +96,7 @@ const Scene = () => {
     return <axesHelper args={[15]} />
   }
   interface CameraViewFromExtrinsicProps {
-    extrinsic: number[]
+    extrinsic: number[] | THREE.Matrix4
     aspect?: number
     name?: string
     near?: number
@@ -98,16 +108,18 @@ const Scene = () => {
   // https://www.ilyameerovich.com/simple-3d-text-meshes-in-three-js/
 
   const CameraViewFromExtrinsic = ({ extrinsic, name, near, far, fov, textSize, aspect }: CameraViewFromExtrinsicProps) => {
-    console.assert(extrinsic.length === 16, "extrinsic must be a 4x4 matrix")
+    let Rt: THREE.Matrix4
+    if (extrinsic instanceof THREE.Matrix4) {
+      Rt = extrinsic
+    } else {
+      Rt = new THREE.Matrix4()
+      // @ts-expect-error 16 elements
+      Rt.set(...extrinsic)
+    }
+    console.assert(Rt.elements.length === 16, "extrinsic must be a 4x4 matrix")
     const font = new FontLoader().parse(HelvetikerRegular)
     const camera = new THREE.PerspectiveCamera(fov ?? 60, aspect ?? 4 / 3, near ?? DEFAULT_NEAR, far ?? DEFAULT_FAR)
-    // the camera by default is looking up to the y-axis
-    // we want it to look at the origin
-    camera.rotation.x = -Math.PI / 2
     const helper = <cameraHelper args={[camera]} />
-    const Rt = new THREE.Matrix4()
-    // @ts-expect-error 16 elements
-    Rt.set(...extrinsic)
     camera.applyMatrix4(Rt)
 
     const textRef = useRef<THREE.Mesh>(null)
@@ -141,17 +153,26 @@ const Scene = () => {
     )
   }
 
+  const preProcessExtrinsic = (extrinsic: number[]) => {
+    const Rt = new THREE.Matrix4()
+    // @ts-expect-error 16 elements
+    Rt.set(...extrinsic)
+    Rt.invert()
+    Rt.multiply(CV_TO_GL_MAT)
+    return Rt
+  }
+
   const scene = (<group>
     {/* <OrbitControls /> */}
     <ambientLight intensity={0.05} />
     <directionalLight castShadow position={[3.3, 6, 4.4]} intensity={5} />
-    <Floor />
+    {/* <Floor /> */}
     {Object.entries(CAMERA_EXTRINSIC_MATRIX_MAP).map(([key, value]) => {
       const intrinsic = CAMERA_INTRINSIC_MATRIX_MAP[key]
       const { fov_x, fov_y } = intrinsicToFov(intrinsic, { width: IMAGE_WIDTH, height: IMAGE_HEIGHT })
       // make the far reverse proportional to the fov
       const far = (1 / fov_x) * 20
-      return <CameraViewFromExtrinsic key={key} name={`${key}(${fov_x.toFixed(1)})`} extrinsic={value} fov={fov_x} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} far={far} />
+      return <CameraViewFromExtrinsic key={key} name={`${key}(${fov_x.toFixed(1)})`} extrinsic={preProcessExtrinsic(value)} fov={fov_x} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} far={far} />
     })}
     <CameraViewFromExtrinsic name="default" extrinsic={DEFAULT_TRANSFORMATION_MATRIX} fov={60} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} far={0.4} />
     <Axes />