Update App.tsx to introduce a new pose data structure for skeleton animation, enhancing the Human3DSkeleton component to accept multiple skeletons. Adjust type definitions and update skeleton instances in the Scene for improved flexibility in pose rendering.

.gitignore

@@ -22,3 +22,5 @@ dist-ssr
 *.njsproj
 *.sln
 *.sw?
+public/*.json
+src/assets/*.json

note.md

@@ -107,6 +107,56 @@ The actual sequence is:
 
 Thank you for catching this! It's crucial to understand the correct order of operations, especially when dealing with coordinate system transformations.
 
+Let me break this down more clearly:
+
+## Why do we need these conversions?
+
+1. **World Coordinate Conversion (Z-up to Y-up)**:
+   - Your original data uses a Z-up coordinate system (Z is "up" in the real world)
+   - Three.js uses a Y-up coordinate system (Y is "up" in Three.js)
+   - We need to convert between these two world coordinate systems
+
+2. **Camera Coordinate Conversion (OpenCV to OpenGL)**:
+   - Camera coordinate systems define how the camera "sees" the world
+   - OpenCV uses a right-handed system where Z points forward from the camera
+   - OpenGL/Three.js uses a system where Z points backward from the camera
+   - We need to convert between these camera conventions
+
+## What's the difference?
+
+- **World conversion** (Z-up to Y-up) affects how we represent points in the world
+- **Camera conversion** (OpenCV to OpenGL) affects how the camera interprets what it sees
+
+These are completely separate transforms that operate on different coordinate spaces:
+- One operates on the world before the camera sees it
+- The other operates on the camera's view of the world
+
+## Why this specific order?
+
+The order matters because we're dealing with a transformation pipeline:
+
+1. First, we want to convert the world from Z-up to Y-up (`worldCvt`)
+   - This makes the world match Three.js's expectations
+
+2. Then, apply the camera's extrinsic matrix (`Rt`)
+   - This transforms world points into the camera's coordinate system
+   - At this point, we're in OpenCV camera space
+
+3. Finally, convert from OpenCV to OpenGL camera conventions (`cameraCvt`)
+   - This converts the camera space to match Three.js's expectations
+
+If we changed the order:
+- Applying camera conversion before the camera transformation would incorrectly transform world points
+- Applying world conversion after the camera transformation would transform points that are already in camera space
+
+Think of it as a pipeline:
+1. Fix the world coordinates (world conversion)
+2. View the world through the camera (camera extrinsic)
+3. Adjust how the camera interprets what it sees (camera conversion)
+
+The fact that matrix multiplication is associative means we can compute this entire pipeline as a single matrix operation, but the conceptual order still matters for getting the correct result.
+
+
 ---
 
 [Matrix4.multiply](https://threejs.org/docs/#api/en/math/Matrix4.multiply)

src/App.tsx

@@ -1,10 +1,29 @@
 import { Grid, useBVH, useGLTF, CameraControls, AccumulativeShadows, OrbitControls, Stats } from '@react-three/drei'
-import { Camera, Canvas, useFrame, useThree, useLoader } from '@react-three/fiber'
+import { Camera, Canvas, useFrame, useThree, useLoader, RenderCallback, RootState } from '@react-three/fiber'
 import * as THREE from 'three'
 import { FontLoader } from 'three/addons/loaders/FontLoader.js'
 import { TextGeometry } from 'three/addons/geometries/TextGeometry.js'
 import HelvetikerRegular from "three/examples/fonts/helvetiker_regular.typeface.json"
 import { useEffect, useRef, useState, JSX } from 'react'
+// import POSE_3D_ from "./assets/result_ae_01_ae_08.json"
+import POSE_3D_ from "./assets/temp_result.json"
+import POSE_3D_MANY_ from "./assets/many_people_all_3d_pose.json"
+import POSE_3D_04_02_ from "./assets/res.json"
+
+// 133, 3
+type PosePoints3D = [number, number, number][]
+
+// F, 133, 3
+type AnimePosePoints3D = PosePoints3D[]
+
+interface Skeleton0402 {
+  "a": PosePoints3D
+  "b": PosePoints3D
+}
+
+const POSE_3D = POSE_3D_ as AnimePosePoints3D
+const POSE_3D_MANY = POSE_3D_MANY_ as AnimePosePoints3D[] // N F 133 3
+const POSE_3D_04_02 = POSE_3D_04_02_ as Skeleton0402
 
 const THREE_ADDONS = {
   FontLoader,
@@ -24,57 +43,110 @@ const CV_TO_GL_MAT = new THREE.Matrix4().set(
 // Z-up to Y-up conversion matrix
 // Rotate -90 degrees around X axis to convert from Z-up to Y-up
 const Z_UP_TO_Y_UP = new THREE.Matrix4().set(
-  1, 0, 0, 0,
+  -1, 0, 0, 0,
-  0, 0, 1, 0,
+  0, 0, -1, 0,
   0, -1, 0, 0,
   0, 0, 0, 1
 )
 
+const Z_UP_TO_Y_UP_PRIME = new THREE.Matrix4().set(
+  1, 0, 0, 0,
+  0, 0, 1, 0,
+  0, 1, 0, 0,
+  0, 0, 0, 1
+)
+
+// Color definitions for different body parts
+const COLOR_SPINE = new THREE.Color(138 / 255, 201 / 255, 38 / 255)  // green, spine & head
+const COLOR_ARMS = new THREE.Color(255 / 255, 202 / 255, 58 / 255)   // yellow, arms & shoulders
+const COLOR_LEGS = new THREE.Color(25 / 255, 130 / 255, 196 / 255)   // blue, legs & hips
+const COLOR_FINGERS = new THREE.Color(255 / 255, 0, 0)           // red, fingers
+const COLOR_FACE = new THREE.Color(255 / 255, 200 / 255, 0)        // yellow, face
+const COLOR_FOOT = new THREE.Color(255 / 255, 128 / 255, 0)        // orange, foot
+const COLOR_HEAD = new THREE.Color(255 / 255, 0, 255 / 255)        // purple, head
+
+// Body bone connections
+const BODY_BONES = [
+  // legs
+  [15, 13], [13, 11], [16, 14], [14, 12], [11, 12], // legs
+  [5, 11], [6, 12], [5, 6],                         // torso
+  [5, 7], [7, 9], [6, 8], [8, 10],                  // arms
+  [1, 2], [0, 1], [0, 2], [1, 3], [2, 4],           // head
+  [15, 17], [15, 18], [15, 19],                     // left foot
+  [16, 20], [16, 21], [16, 22],                     // right foot
+] as const
+
+// Body bone colors
+const BODY_BONE_COLORS = [
+  COLOR_LEGS, COLOR_LEGS, COLOR_LEGS, COLOR_LEGS, COLOR_LEGS,
+  COLOR_SPINE, COLOR_SPINE, COLOR_SPINE,
+  COLOR_ARMS, COLOR_ARMS, COLOR_ARMS, COLOR_ARMS,
+  COLOR_HEAD, COLOR_HEAD, COLOR_HEAD, COLOR_HEAD, COLOR_HEAD,
+  COLOR_FOOT, COLOR_FOOT, COLOR_FOOT,
+  COLOR_FOOT, COLOR_FOOT, COLOR_FOOT,
+] as const
+
+// Hand bone connections (in pairs of [start, end] indices)
+const HAND_BONES = [
+  // right hand
+  [91, 92], [92, 93], [93, 94], [94, 95],           // right thumb
+  [91, 96], [96, 97], [97, 98], [98, 99],           // right index
+  [91, 100], [100, 101], [101, 102], [102, 103],    // right middle
+  [91, 104], [104, 105], [105, 106], [106, 107],    // right ring
+  [91, 108], [108, 109], [109, 110], [110, 111],    // right pinky
+  // left hand
+  [112, 113], [113, 114], [114, 115], [115, 116],   // left thumb
+  [112, 117], [117, 118], [118, 119], [119, 120],   // left index
+  [112, 121], [121, 122], [122, 123], [123, 124],   // left middle
+  [112, 125], [125, 126], [126, 127], [127, 128],   // left ring
+  [112, 129], [129, 130], [130, 131], [131, 132]    // left pinky
+] as const
+
 const DEFAULT_TRANSFORMATION_MATRIX = [
   1, 0, 0, 0,
   0, 1, 0, 0,
   0, 0, 1, 0,
   0, 0, 0, 1,
-]
+] as const
 const DEFAULT_NEAR = 0.05
 const DEFAULT_FAR = 1
 const CAMERA_EXTRINSIC_MATRIX_MAP: Record<string, number[]> = {
   "AE_01": [
-    0.2321047, -0.97264263, -0.0096808, -0.96323585, 0.06882254,
+    0.37408302, -0.91907411, 0.12395429, 1.18976111, 0.17243349,
-    0.02634936, -0.99728089, 0.03661007, 0.97025299, 0.23080732,
+    -0.06239751, -0.98304285, -0.06429779, 0.91122367, 0.38911351,
-    0.07305554, 3.34933242, 0., 0., 0.,
+    0.13513731, 2.51940833, 0., 0., 0.,
     1.
-  ],
+  ] as const,
   "AE_1A": [
-    0.93194049, -0.35571886, -0.07036343, -0.92123075, 0.01084819,
+    0.92998171, -0.36696694, -0.02166301, 2.21643671, -0.05110403,
-    0.22131041, -0.97514308, 0.24922173, 0.36244895, 0.908012,
+    -0.07070226, -0.99618752, -0.72948697, 0.36403626, 0.92754324,
-    0.21010704, 4.87284891, 0., 0., 0.,
+    -0.0845053, 6.45800206, 0., 0., 0.,
     1.
-  ],
+  ] as const,
   "AE_08": [
-    0.66806102, -0.74355508, -0.02864123, -1.10173496, 0.05931037,
+    0.98195914, -0.18888337, -0.00890642, 1.43011854, -0.02247979,
-    0.09157787, -0.99403007, 0.26760438, 0.74173901, 0.66237402,
+    -0.06984105, -0.99730481, -0.61678831, 0.18775226, 0.97951279,
-    0.10528013, 6.92372493, 0., 0., 0.,
+    -0.07282712, 5.81983825, 0., 0., 0.,
     1.
-  ]
+  ] as const
-} as const
+}
 const CAMERA_INTRINSIC_MATRIX_MAP: Record<string, number[]> = {
   "AE_01": [
     1806.82137617, 0., 1230.53175624, 0.,
     1809.75580378, 766.36204406, 0., 0.,
     1.
-  ],
+  ] as const,
   "AE_1A": [
     3467.39715751, 0., 1000.62548655, 0.,
     3473.7168112, 831.64048503, 0., 0.,
     1.
-  ],
+  ] as const,
   "AE_08": [
     2785.43931794, 0., 1254.98272372, 0.,
     2788.10437965, 738.82985324, 0., 0.,
     1.
-  ]
+  ] as const
-} as const
+}
 
 const IMAGE_WIDTH = 2560
 const IMAGE_HEIGHT = 1440
@@ -119,12 +191,14 @@ const Scene = () => {
     let Rt: THREE.Matrix4
     if (extrinsic instanceof THREE.Matrix4) {
       Rt = extrinsic
-    } else {
+    } else if (Array.isArray(extrinsic)) {
+      console.assert(extrinsic.length === 16, "extrinsic must be a 4x4 matrix")
       Rt = new THREE.Matrix4()
       // @ts-expect-error 16 elements
       Rt.set(...extrinsic)
+    } else {
+      throw new Error("extrinsic must be a 4x4 matrix or an array of 16 elements")
     }
-    console.assert(Rt.elements.length === 16, "extrinsic must be a 4x4 matrix")
     const font = new FontLoader().parse(HelvetikerRegular)
     const camera = new THREE.PerspectiveCamera(fov ?? 60, aspect ?? 4 / 3, near ?? DEFAULT_NEAR, far ?? DEFAULT_FAR)
     const helper = <cameraHelper args={[camera]} />
@@ -161,17 +235,25 @@ const Scene = () => {
     )
   }
 
-  const preProcessExtrinsic = (extrinsic: number[]) => {
+  const preProcessExtrinsic = (extrinsic: number[] | THREE.Matrix4) => {
-    // Create the initial world-to-camera transform
+    let Rt: THREE.Matrix4
-    const worldToCamera = new THREE.Matrix4()
+    if (extrinsic instanceof THREE.Matrix4) {
+      Rt = extrinsic
+    } else if (Array.isArray(extrinsic)) {
+      console.assert(extrinsic.length === 16, "extrinsic must be a 4x4 matrix")
+      Rt = new THREE.Matrix4()
       // @ts-expect-error 16 elements
-    worldToCamera.set(...extrinsic)
+      Rt.set(...extrinsic)
+    } else {
+      throw new Error("extrinsic must be a 4x4 matrix or an array of 16 elements")
+    }
 
-    // Convert from Z-up to Y-up first (this affects world coordinates)
-    const worldZupToYup = Z_UP_TO_Y_UP.clone()
 
     // Then handle OpenCV to OpenGL camera convention
-    const cameraConversion = CV_TO_GL_MAT.clone()
+    const cameraCvt = CV_TO_GL_MAT.clone()
+
+    // Convert from Z-up to Y-up first (this affects world coordinates)
+    const worldCvt = Z_UP_TO_Y_UP.clone()
 
     // Final transformation:
     // 1. Convert world from Z-up to Y-up
@@ -179,36 +261,195 @@ const Scene = () => {
     // 3. Convert camera coordinates from OpenCV to OpenGL
     const final = new THREE.Matrix4()
     final
-      .multiply(cameraConversion)
+      .multiply(cameraCvt)
-      .multiply(worldToCamera)
+      .multiply(Rt)
-      .multiply(worldZupToYup)
+      .multiply(worldCvt)
 
     // Invert to get the camera-to-world transform
     final.invert()
     return final
   }
 
-  const scene = (<group>
+  interface Human3DSkeletonProps {
-    {/* <OrbitControls /> */}
+    skeleton: AnimePosePoints3D
-    <ambientLight intensity={0.05} />
+    startFrame?: number
-    <directionalLight castShadow position={[3.3, 6, 4.4]} intensity={5} />
+    jointRadius?: number
-    {/* <Floor /> */}
+    boneRadius?: number
-    {Object.entries(CAMERA_EXTRINSIC_MATRIX_MAP).map(([key, value]) => {
+    showJoints?: boolean
+    showBones?: boolean
+    frameRate?: number
+  }
+
+  const Human3DSkeleton = ({
+    skeleton,
+    startFrame = 0,
+    jointRadius = 0.01,
+    boneRadius = 0.005,
+    showJoints = true,
+    showBones = true,
+    frameRate = 30
+  }: Human3DSkeletonProps) => {
+    const [frameIndex, setFrameIndex] = useState(startFrame)
+    const totalFrames = skeleton.length
+    const onFrame: RenderCallback = (totalFrames === 0) ? (state, delta) => { } : (state: RootState, delta: number) => {
+      // Calculate next frame based on desired frame rate and delta time
+      setFrameIndex(prevFrame => {
+        // Calculate next frame
+        const nextFrame = prevFrame + frameRate * delta
+        // Loop back to start if we reach the end
+        return nextFrame >= totalFrames ? 0 : nextFrame
+      })
+      return null
+    }
+
+    // Use frame to animate through the skeleton poses
+    useFrame(onFrame)
+
+    // Get the current frame joints - use Math.floor to get the nearest frame
+    const currentFrame = Math.floor(frameIndex) % totalFrames
+    const joints = skeleton[currentFrame]
+
+    // Function to get appropriate color for a joint index
+    const getJointColor = (idx: number) => {
+      // Face joints (23-90)
+      if (idx >= 23 && idx <= 90) return COLOR_FACE
+      // Hand joints (91-132)
+      if (idx >= 91 && idx <= 132) return COLOR_FINGERS
+      // Foot joints (17-22)
+      if (idx >= 17 && idx <= 22) return COLOR_FOOT
+      // Head (0-4)
+      if (idx <= 4) return COLOR_HEAD
+      // Arms (5-10)
+      if (idx >= 5 && idx <= 10) return COLOR_ARMS
+      // Legs (11-16)
+      if (idx >= 11 && idx <= 16) return COLOR_LEGS
+      // Default
+      return COLOR_SPINE
+    }
+
+    // Transform a joint position using the coordinate system conversion
+    const transformJointPosition = (j: [number, number, number]) => {
+      const [x, y, z] = j
+      const V = new THREE.Vector3(x, y, z)
+      const worldCvt = Z_UP_TO_Y_UP_PRIME.clone()
+      V.applyMatrix4(worldCvt)
+      return V
+    }
+
+    // Create the joint spheres
+    const jointMeshes = showJoints ? joints.map((j, idx) => {
+      const position = transformJointPosition(j)
+      const color = getJointColor(idx)
+
+      return (
+        <mesh key={`joint-${idx}`} position={position}>
+          <sphereGeometry args={[jointRadius, 16, 16]} />
+          <meshStandardMaterial color={color} />
+        </mesh>
+      )
+    }) : null
+
+    // Create the bone cylinders
+    const boneMeshes = showBones ? (
+      <>
+        {BODY_BONES.map((bone, idx) => {
+          const [startIdx, endIdx] = bone
+          if (startIdx >= joints.length || endIdx >= joints.length) return null
+
+          const startPos = transformJointPosition(joints[startIdx])
+          const endPos = transformJointPosition(joints[endIdx])
+          const color = BODY_BONE_COLORS[idx]
+
+          // Calculate midpoint and length
+          const midpoint = new THREE.Vector3().addVectors(startPos, endPos).multiplyScalar(0.5)
+          const length = startPos.distanceTo(endPos)
+
+          // Calculate rotation
+          const direction = new THREE.Vector3().subVectors(endPos, startPos).normalize()
+          const quaternion = new THREE.Quaternion()
+          const up = new THREE.Vector3(0, 1, 0)
+          quaternion.setFromUnitVectors(up, direction)
+
+          return (
+            <mesh key={`bone-body-${idx}`} position={midpoint} quaternion={quaternion}>
+              <cylinderGeometry args={[boneRadius, boneRadius, length, 8]} />
+              <meshStandardMaterial color={color} />
+            </mesh>
+          )
+        })}
+
+        {HAND_BONES.map((bone, idx) => {
+          const [startIdx, endIdx] = bone
+          if (startIdx >= joints.length || endIdx >= joints.length) return null
+
+          const startPos = transformJointPosition(joints[startIdx])
+          const endPos = transformJointPosition(joints[endIdx])
+
+          // Calculate midpoint and length
+          const midpoint = new THREE.Vector3().addVectors(startPos, endPos).multiplyScalar(0.5)
+          const length = startPos.distanceTo(endPos)
+
+          // Calculate rotation
+          const direction = new THREE.Vector3().subVectors(endPos, startPos).normalize()
+          const quaternion = new THREE.Quaternion()
+          const up = new THREE.Vector3(0, 1, 0)
+          quaternion.setFromUnitVectors(up, direction)
+
+          return (
+            <mesh key={`bone-hand-${idx}`} position={midpoint} quaternion={quaternion}>
+              <cylinderGeometry args={[boneRadius, boneRadius, length, 8]} />
+              <meshStandardMaterial color={COLOR_FINGERS} />
+            </mesh>
+          )
+        })}
+      </>
+    ) : null
+
+    return (
+      <group>
+        {jointMeshes}
+        {boneMeshes}
+      </group>
+    )
+  }
+
+  // const S0 = [POSE_3D_MANY[0][0]]
+  // const S1 = [POSE_3D_MANY[0][1]]
+  // const skeletons = POSE_3D_MANY.map((el) => <Human3DSkeleton jointRadius={0.005} boneRadius={0.0025} frameRate={24} skeleton={el} />)
+  // const skeletons = [<Human3DSkeleton jointRadius={0.05} boneRadius={0.025} frameRate={1} skeleton={S0} />,
+  // <Human3DSkeleton jointRadius={0.05} boneRadius={0.025} frameRate={1} skeleton={S1} />
+  // ]
+  // const skeletons = [
+  //   <Human3DSkeleton jointRadius={0.005} boneRadius={0.0025} frameRate={24} skeleton={POSE_3D} />,
+  // ]
+  const skeletons = [
+    <Human3DSkeleton jointRadius={0.005} boneRadius={0.0025} frameRate={24} skeleton={[POSE_3D_04_02.a]} />,
+    <Human3DSkeleton jointRadius={0.005} boneRadius={0.0025} frameRate={24} skeleton={[POSE_3D_04_02.b]} />,
+  ]
+
+  const cameras = Object.entries(CAMERA_EXTRINSIC_MATRIX_MAP).map(([key, value]) => {
     const intrinsic = CAMERA_INTRINSIC_MATRIX_MAP[key]
     const { fov_x, fov_y } = intrinsicToFov(intrinsic, { width: IMAGE_WIDTH, height: IMAGE_HEIGHT })
     // make the far reverse proportional to the fov
     const far = (1 / fov_x) * 20
     return <CameraViewFromExtrinsic key={key} name={`${key}(${fov_x.toFixed(1)})`} extrinsic={preProcessExtrinsic(value)} fov={fov_x} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} far={far} />
-    })}
+  })
-    <CameraViewFromExtrinsic name="default" extrinsic={DEFAULT_TRANSFORMATION_MATRIX} fov={60} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} far={0.4} />
+
+  const scene = (<group>
+    {/* <OrbitControls /> */}
+    <ambientLight intensity={0.05} />
+    <directionalLight castShadow position={[3.3, 6, 4.4]} intensity={5} />
+    {/* <Floor /> */}
+    { }
     <Axes />
+    {cameras}
+    {skeletons}
   </group>)
   return (
     // Note that we don't need to import anything, All three.js objects will be treated
     // as native JSX elements, just like you can just write <div /> or <span /> in
     // regular ReactDOM. The general rule is that Fiber components are available under
     // the camel-case version of their name in three.js.
-
     <>
       <CameraControls />
       <Stats />