camera-extrinsic-play/src/App.tsx

import { Grid, useBVH, useGLTF, CameraControls, AccumulativeShadows, OrbitControls, Stats } from '@react-three/drei'
import { Camera, Canvas, useFrame, useThree, useLoader } from '@react-three/fiber'
import * as THREE from 'three'
import { FontLoader } from 'three/addons/loaders/FontLoader.js'
import { TextGeometry } from 'three/addons/geometries/TextGeometry.js'
import HelvetikerRegular from "three/examples/fonts/helvetiker_regular.typeface.json"
import { useEffect, useRef, useState, JSX } from 'react'

const THREE_ADDONS = {
  FontLoader,
  TextGeometry,
} as const


const DEFAULT_NEAR = 0.1
const DEFAULT_FAR = 1
const CAMERA_EXTRINSIC_MATRIX_MAP: Record<string, number[]> = {
  "AE_01": [
    0.2321047, -0.97264263, -0.0096808, -0.96323585, 0.06882254,
    0.02634936, -0.99728089, 0.03661007, 0.97025299, 0.23080732,
    0.07305554, 3.34933242, 0., 0., 0.,
    1.
  ],
  "AE_1A": [
    0.93194049, -0.35571886, -0.07036343, -0.92123075, 0.01084819,
    0.22131041, -0.97514308, 0.24922173, 0.36244895, 0.908012,
    0.21010704, 4.87284891, 0., 0., 0.,
    1.
  ],
  "AE_08": [
    0.66806102, -0.74355508, -0.02864123, -1.10173496, 0.05931037,
    0.09157787, -0.99403007, 0.26760438, 0.74173901, 0.66237402,
    0.10528013, 6.92372493, 0., 0., 0.,
    1.
  ]
} as const
const CAMERA_INTRINSIC_MATRIX_MAP: Record<string, number[]> = {
  "AE_01": [
    1806.82137617, 0., 1230.53175624, 0.,
    1809.75580378, 766.36204406, 0., 0.,
    1.
  ],
  "AE_1A": [
    3467.39715751, 0., 1000.62548655, 0.,
    3473.7168112, 831.64048503, 0., 0.,
    1.
  ],
  "AE_08": [
    2785.43931794, 0., 1254.98272372, 0.,
    2788.10437965, 738.82985324, 0., 0.,
    1.
  ]
} as const

const IMAGE_WIDTH = 2560
const IMAGE_HEIGHT = 1440

const intrinsicToFov = (intrinsic: number[], image_size: { width: number, height: number }) => {
  console.assert(intrinsic.length === 9, "intrinsic must be a 3x3 matrix")
  const fx = intrinsic[0]
  const fy = intrinsic[4]
  const cx = intrinsic[2]
  const cy = intrinsic[5]
  // in degrees
  const fov_x = 2 * Math.atan(image_size.width / (2 * fx)) * (180 / Math.PI)
  const fov_y = 2 * Math.atan(image_size.height / (2 * fy)) * (180 / Math.PI)
  return { fov_x, fov_y }
}

const Scene = () => {
  function Floor() {
    return (
      <mesh rotation-x={-Math.PI / 2} position-y={-0.05} receiveShadow>
        <planeGeometry args={[15, 15]} />
        <meshStandardMaterial color="#ccc" />
      </mesh>
    )
  }
  const Axes = () => {
    return <axesHelper args={[15]} />
  }
  interface CameraViewFromExtrinsicProps {
    extrinsic: number[]
    aspect?: number
    name?: string
    near?: number
    far?: number
    fov?: number
    textSize?: number
  }
  // https://threejs.org/docs/#examples/en/loaders/FontLoader
  // https://www.ilyameerovich.com/simple-3d-text-meshes-in-three-js/

  const CameraViewFromExtrinsic = ({ extrinsic, name, near, far, fov, textSize, aspect }: CameraViewFromExtrinsicProps) => {
    const font = new FontLoader().parse(HelvetikerRegular)
    const camera = new THREE.PerspectiveCamera(fov ?? 60, aspect ?? 4 / 3, near ?? DEFAULT_NEAR, far ?? DEFAULT_FAR)
    const helper = <cameraHelper args={[camera]} />
    // @ts-expect-error 16 elements
    camera.applyMatrix4(new THREE.Matrix4(...extrinsic))
    let text: JSX.Element | null = null
    if (name) {
      // https://github.com/pmndrs/react-three-fiber/discussions/832
      const geo = new THREE_ADDONS.TextGeometry(name ?? "", { font, size: textSize ?? 0.1, depth: 0.001 })

      // Extract camera position from extrinsic matrix
      const matrix = new THREE.Matrix4()
      // @ts-expect-error 16 elements
      matrix.set(...extrinsic)
      const position = new THREE.Vector3()
      position.setFromMatrixPosition(matrix)

      // Create text without applying camera matrix transformation
      text = (
        <mesh position={position}>
          <primitive object={geo} />
          <meshStandardMaterial color="black" />
        </mesh>
      )
    }
    return (
      <group>
        {text}
        <primitive object={camera} />
        {helper}
      </group>
    )
  }

  return (
    // Note that we don't need to import anything, All three.js objects will be treated
    // as native JSX elements, just like you can just write <div /> or <span /> in
    // regular ReactDOM. The general rule is that Fiber components are available under
    // the camel-case version of their name in three.js.
    <>
      <CameraControls />
      {/* <OrbitControls /> */}
      <ambientLight intensity={0.05} />
      <directionalLight castShadow position={[3.3, 6, 4.4]} intensity={5} />
      <Floor />
      {Object.entries(CAMERA_EXTRINSIC_MATRIX_MAP).map(([key, value]) => {
        const intrinsic = CAMERA_INTRINSIC_MATRIX_MAP[key]
        const fov = intrinsicToFov(intrinsic, { width: IMAGE_WIDTH, height: IMAGE_HEIGHT })
        return <CameraViewFromExtrinsic key={key} name={key} extrinsic={value} fov={fov.fov_x} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} />
      })}
      <Axes />
      <Stats />
    </>
  )
}

function App() {
  return (
    <Canvas shadows style={{ background: "#e9e9e9", width: "100vw", height: "100vh" }}>
      <Scene />
    </Canvas>
  )
}

export default App