Update App.tsx to introduce a new pose data structure for skeleton animation, enhancing the Human3DSkeleton component to accept multiple skeletons. Adjust type definitions and update skeleton instances in the Scene for improved flexibility in pose rendering.

src/App.tsx

@@ -1,14 +1,29 @@
 import { Grid, useBVH, useGLTF, CameraControls, AccumulativeShadows, OrbitControls, Stats } from '@react-three/drei'
-import { Camera, Canvas, useFrame, useThree, useLoader } from '@react-three/fiber'
+import { Camera, Canvas, useFrame, useThree, useLoader, RenderCallback, RootState } from '@react-three/fiber'
 import * as THREE from 'three'
 import { FontLoader } from 'three/addons/loaders/FontLoader.js'
 import { TextGeometry } from 'three/addons/geometries/TextGeometry.js'
 import HelvetikerRegular from "three/examples/fonts/helvetiker_regular.typeface.json"
 import { useEffect, useRef, useState, JSX } from 'react'
-import POSE_3D_ from "./assets/result_ae_01_ae_08.json"
+// import POSE_3D_ from "./assets/result_ae_01_ae_08.json"
+import POSE_3D_ from "./assets/temp_result.json"
+import POSE_3D_MANY_ from "./assets/many_people_all_3d_pose.json"
+import POSE_3D_04_02_ from "./assets/res.json"
+
+// 133, 3
+type PosePoints3D = [number, number, number][]
 
 // F, 133, 3
-const POSE_3D: Array<Array<[number, number, number]>> = POSE_3D_ as [number, number, number][][]
+type AnimePosePoints3D = PosePoints3D[]
+
+interface Skeleton0402 {
+  "a": PosePoints3D
+  "b": PosePoints3D
+}
+
+const POSE_3D = POSE_3D_ as AnimePosePoints3D
+const POSE_3D_MANY = POSE_3D_MANY_ as AnimePosePoints3D[] // N F 133 3
+const POSE_3D_04_02 = POSE_3D_04_02_ as Skeleton0402
 
 const THREE_ADDONS = {
   FontLoader,
@@ -35,9 +50,9 @@ const Z_UP_TO_Y_UP = new THREE.Matrix4().set(
 )
 
 const Z_UP_TO_Y_UP_PRIME = new THREE.Matrix4().set(
-  -1, 0, 0, 0,
+  1, 0, 0, 0,
   0, 0, 1, 0,
-  0, -1, 0, 0,
+  0, 1, 0, 0,
   0, 0, 0, 1
 )
 
@@ -59,7 +74,7 @@ const BODY_BONES = [
   [1, 2], [0, 1], [0, 2], [1, 3], [2, 4],           // head
   [15, 17], [15, 18], [15, 19],                     // left foot
   [16, 20], [16, 21], [16, 22],                     // right foot
-]
+] as const
 
 // Body bone colors
 const BODY_BONE_COLORS = [
@@ -69,7 +84,7 @@ const BODY_BONE_COLORS = [
   COLOR_HEAD, COLOR_HEAD, COLOR_HEAD, COLOR_HEAD, COLOR_HEAD,
   COLOR_FOOT, COLOR_FOOT, COLOR_FOOT,
   COLOR_FOOT, COLOR_FOOT, COLOR_FOOT,
-]
+] as const
 
 // Hand bone connections (in pairs of [start, end] indices)
 const HAND_BONES = [
@@ -85,53 +100,53 @@ const HAND_BONES = [
   [112, 121], [121, 122], [122, 123], [123, 124],   // left middle
   [112, 125], [125, 126], [126, 127], [127, 128],   // left ring
   [112, 129], [129, 130], [130, 131], [131, 132]    // left pinky
-]
+] as const
 
 const DEFAULT_TRANSFORMATION_MATRIX = [
   1, 0, 0, 0,
   0, 1, 0, 0,
   0, 0, 1, 0,
   0, 0, 0, 1,
-]
+] as const
 const DEFAULT_NEAR = 0.05
 const DEFAULT_FAR = 1
 const CAMERA_EXTRINSIC_MATRIX_MAP: Record<string, number[]> = {
   "AE_01": [
-    0.2321047, -0.97264263, -0.0096808, -0.96323585, 0.06882254,
-    0.02634936, -0.99728089, 0.03661007, 0.97025299, 0.23080732,
-    0.07305554, 3.34933242, 0., 0., 0.,
+    0.37408302, -0.91907411, 0.12395429, 1.18976111, 0.17243349,
+    -0.06239751, -0.98304285, -0.06429779, 0.91122367, 0.38911351,
+    0.13513731, 2.51940833, 0., 0., 0.,
     1.
-  ],
+  ] as const,
   "AE_1A": [
-    0.93194049, -0.35571886, -0.07036343, -0.92123075, 0.01084819,
-    0.22131041, -0.97514308, 0.24922173, 0.36244895, 0.908012,
-    0.21010704, 4.87284891, 0., 0., 0.,
+    0.92998171, -0.36696694, -0.02166301, 2.21643671, -0.05110403,
+    -0.07070226, -0.99618752, -0.72948697, 0.36403626, 0.92754324,
+    -0.0845053, 6.45800206, 0., 0., 0.,
     1.
-  ],
+  ] as const,
   "AE_08": [
-    0.66806102, -0.74355508, -0.02864123, -1.10173496, 0.05931037,
-    0.09157787, -0.99403007, 0.26760438, 0.74173901, 0.66237402,
-    0.10528013, 6.92372493, 0., 0., 0.,
+    0.98195914, -0.18888337, -0.00890642, 1.43011854, -0.02247979,
+    -0.06984105, -0.99730481, -0.61678831, 0.18775226, 0.97951279,
+    -0.07282712, 5.81983825, 0., 0., 0.,
     1.
-  ]
-} as const
+  ] as const
+}
 const CAMERA_INTRINSIC_MATRIX_MAP: Record<string, number[]> = {
   "AE_01": [
     1806.82137617, 0., 1230.53175624, 0.,
     1809.75580378, 766.36204406, 0., 0.,
     1.
-  ],
+  ] as const,
   "AE_1A": [
     3467.39715751, 0., 1000.62548655, 0.,
     3473.7168112, 831.64048503, 0., 0.,
     1.
-  ],
+  ] as const,
   "AE_08": [
     2785.43931794, 0., 1254.98272372, 0.,
     2788.10437965, 738.82985324, 0., 0.,
     1.
-  ]
-} as const
+  ] as const
+}
 
 const IMAGE_WIDTH = 2560
 const IMAGE_HEIGHT = 1440
@@ -256,6 +271,7 @@ const Scene = () => {
   }
 
   interface Human3DSkeletonProps {
+    skeleton: AnimePosePoints3D
     startFrame?: number
     jointRadius?: number
     boneRadius?: number
@@ -265,6 +281,7 @@ const Scene = () => {
   }
 
   const Human3DSkeleton = ({
+    skeleton,
     startFrame = 0,
     jointRadius = 0.01,
     boneRadius = 0.005,
@@ -273,10 +290,8 @@ const Scene = () => {
     frameRate = 30
   }: Human3DSkeletonProps) => {
     const [frameIndex, setFrameIndex] = useState(startFrame)
-    const totalFrames = POSE_3D.length
-
-    // Use frame to animate through the skeleton poses
-    useFrame((state, delta) => {
+    const totalFrames = skeleton.length
+    const onFrame: RenderCallback = (totalFrames === 0) ? (state, delta) => { } : (state: RootState, delta: number) => {
       // Calculate next frame based on desired frame rate and delta time
       setFrameIndex(prevFrame => {
         // Calculate next frame
@@ -284,11 +299,15 @@ const Scene = () => {
         // Loop back to start if we reach the end
         return nextFrame >= totalFrames ? 0 : nextFrame
       })
-    })
+      return null
+    }
+
+    // Use frame to animate through the skeleton poses
+    useFrame(onFrame)
 
     // Get the current frame joints - use Math.floor to get the nearest frame
     const currentFrame = Math.floor(frameIndex) % totalFrames
-    const joints = POSE_3D[currentFrame]
+    const joints = skeleton[currentFrame]
 
     // Function to get appropriate color for a joint index
     const getJointColor = (idx: number) => {
@@ -394,20 +413,37 @@ const Scene = () => {
     )
   }
 
-  const scene = (<group>
-    {/* <OrbitControls /> */}
-    <ambientLight intensity={0.05} />
-    <directionalLight castShadow position={[3.3, 6, 4.4]} intensity={5} />
-    {/* <Floor /> */}
-    {Object.entries(CAMERA_EXTRINSIC_MATRIX_MAP).map(([key, value]) => {
+  // const S0 = [POSE_3D_MANY[0][0]]
+  // const S1 = [POSE_3D_MANY[0][1]]
+  // const skeletons = POSE_3D_MANY.map((el) => <Human3DSkeleton jointRadius={0.005} boneRadius={0.0025} frameRate={24} skeleton={el} />)
+  // const skeletons = [<Human3DSkeleton jointRadius={0.05} boneRadius={0.025} frameRate={1} skeleton={S0} />,
+  // <Human3DSkeleton jointRadius={0.05} boneRadius={0.025} frameRate={1} skeleton={S1} />
+  // ]
+  // const skeletons = [
+  //   <Human3DSkeleton jointRadius={0.005} boneRadius={0.0025} frameRate={24} skeleton={POSE_3D} />,
+  // ]
+  const skeletons = [
+    <Human3DSkeleton jointRadius={0.005} boneRadius={0.0025} frameRate={24} skeleton={[POSE_3D_04_02.a]} />,
+    <Human3DSkeleton jointRadius={0.005} boneRadius={0.0025} frameRate={24} skeleton={[POSE_3D_04_02.b]} />,
+  ]
+
+  const cameras = Object.entries(CAMERA_EXTRINSIC_MATRIX_MAP).map(([key, value]) => {
     const intrinsic = CAMERA_INTRINSIC_MATRIX_MAP[key]
     const { fov_x, fov_y } = intrinsicToFov(intrinsic, { width: IMAGE_WIDTH, height: IMAGE_HEIGHT })
     // make the far reverse proportional to the fov
     const far = (1 / fov_x) * 20
     return <CameraViewFromExtrinsic key={key} name={`${key}(${fov_x.toFixed(1)})`} extrinsic={preProcessExtrinsic(value)} fov={fov_x} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} far={far} />
-    })}
+  })
+
+  const scene = (<group>
+    {/* <OrbitControls /> */}
+    <ambientLight intensity={0.05} />
+    <directionalLight castShadow position={[3.3, 6, 4.4]} intensity={5} />
+    {/* <Floor /> */}
+    { }
     <Axes />
-    <Human3DSkeleton jointRadius={0.005} boneRadius={0.0025} frameRate={24} />
+    {cameras}
+    {skeletons}
   </group>)
   return (
     // Note that we don't need to import anything, All three.js objects will be treated