Add OpenCV to OpenGL conversion matrix and update CameraViewFromExtrinsic to accept both extrinsic arrays and Matrix4. Introduce pre-processing for extrinsic matrices to ensure correct camera orientation.

src/App.tsx

@@ -11,7 +11,23 @@ const THREE_ADDONS = {
   TextGeometry,
 } as const
 
+// Create OpenCV to OpenGL conversion matrix
+// OpenCV: X right, Y down, Z forward
+// OpenGL: X right, Y up, Z backward
+const CV_TO_GL_MAT = new THREE.Matrix4().set(
+  1, 0, 0, 0,
+  0, -1, 0, 0,
+  0, 0, -1, 0,
+  0, 0, 0, 1
+)
 
+
+const DEFAULT_TRANSFORMATION_MATRIX = [
+  1, 0, 0, 0,
+  0, 1, 0, 0,
+  0, 0, 1, 0,
+  0, 0, 0, 1,
+]
 const DEFAULT_NEAR = 0.05
 const DEFAULT_FAR = 1
 const CAMERA_EXTRINSIC_MATRIX_MAP: Record<string, number[]> = {
@@ -80,7 +96,7 @@ const Scene = () => {
     return <axesHelper args={[15]} />
   }
   interface CameraViewFromExtrinsicProps {
-    extrinsic: number[]
+    extrinsic: number[] | THREE.Matrix4
     aspect?: number
     name?: string
     near?: number
@@ -92,16 +108,18 @@ const Scene = () => {
   // https://www.ilyameerovich.com/simple-3d-text-meshes-in-three-js/
 
   const CameraViewFromExtrinsic = ({ extrinsic, name, near, far, fov, textSize, aspect }: CameraViewFromExtrinsicProps) => {
-    console.assert(extrinsic.length === 16, "extrinsic must be a 4x4 matrix")
-    const font = new FontLoader().parse(HelvetikerRegular)
-    const camera = new THREE.PerspectiveCamera(fov ?? 60, aspect ?? 4 / 3, near ?? DEFAULT_NEAR, far ?? DEFAULT_FAR)
-    // the camera by default is looking up to the y-axis
-    // we want it to look at the origin
-    camera.rotation.x = -Math.PI / 2
-    const helper = <cameraHelper args={[camera]} />
-    const Rt = new THREE.Matrix4()
+    let Rt: THREE.Matrix4
+    if (extrinsic instanceof THREE.Matrix4) {
+      Rt = extrinsic
+    } else {
+      Rt = new THREE.Matrix4()
       // @ts-expect-error 16 elements
       Rt.set(...extrinsic)
+    }
+    console.assert(Rt.elements.length === 16, "extrinsic must be a 4x4 matrix")
+    const font = new FontLoader().parse(HelvetikerRegular)
+    const camera = new THREE.PerspectiveCamera(fov ?? 60, aspect ?? 4 / 3, near ?? DEFAULT_NEAR, far ?? DEFAULT_FAR)
+    const helper = <cameraHelper args={[camera]} />
     camera.applyMatrix4(Rt)
 
     const textRef = useRef<THREE.Mesh>(null)
@@ -135,18 +153,28 @@ const Scene = () => {
     )
   }
 
+  const preProcessExtrinsic = (extrinsic: number[]) => {
+    const Rt = new THREE.Matrix4()
+    // @ts-expect-error 16 elements
+    Rt.set(...extrinsic)
+    Rt.invert()
+    Rt.multiply(CV_TO_GL_MAT)
+    return Rt
+  }
+
   const scene = (<group>
     {/* <OrbitControls /> */}
     <ambientLight intensity={0.05} />
     <directionalLight castShadow position={[3.3, 6, 4.4]} intensity={5} />
-    <Floor />
+    {/* <Floor /> */}
     {Object.entries(CAMERA_EXTRINSIC_MATRIX_MAP).map(([key, value]) => {
       const intrinsic = CAMERA_INTRINSIC_MATRIX_MAP[key]
       const { fov_x, fov_y } = intrinsicToFov(intrinsic, { width: IMAGE_WIDTH, height: IMAGE_HEIGHT })
       // make the far reverse proportional to the fov
       const far = (1 / fov_x) * 20
-      return <CameraViewFromExtrinsic key={key} name={`${key}(${fov_x.toFixed(1)})`} extrinsic={value} fov={fov_x} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} far={far} />
+      return <CameraViewFromExtrinsic key={key} name={`${key}(${fov_x.toFixed(1)})`} extrinsic={preProcessExtrinsic(value)} fov={fov_x} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} far={far} />
     })}
+    <CameraViewFromExtrinsic name="default" extrinsic={DEFAULT_TRANSFORMATION_MATRIX} fov={60} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} far={0.4} />
     <Axes />
   </group>)
   return (