Refactor App.tsx to enhance skeleton animation capabilities by introducing a new pose data structure and updating the Human3DSkeleton component to accept customizable skeletons. Adjust camera extrinsic parameters for improved scene rendering.

src/App.tsx

@@ -1,5 +1,5 @@
 import { Grid, useBVH, useGLTF, CameraControls, AccumulativeShadows, OrbitControls, Stats } from '@react-three/drei'
-import { Camera, Canvas, useFrame, useThree, useLoader } from '@react-three/fiber'
+import { Camera, Canvas, useFrame, useThree, useLoader, RenderCallback, RootState } from '@react-three/fiber'
 import * as THREE from 'three'
 import { FontLoader } from 'three/addons/loaders/FontLoader.js'
 import { TextGeometry } from 'three/addons/geometries/TextGeometry.js'
@@ -7,9 +7,13 @@ import HelvetikerRegular from "three/examples/fonts/helvetiker_regular.typeface.
 import { useEffect, useRef, useState, JSX } from 'react'
 // import POSE_3D_ from "./assets/result_ae_01_ae_08.json"
 import POSE_3D_ from "./assets/temp_result.json"
+import POSE_3D_MANY_ from "./assets/many_people_all_3d_pose.json"
 
 // F, 133, 3
-const POSE_3D: Array<Array<[number, number, number]>> = POSE_3D_ as [number, number, number][][]
+type SkeletonType = [number, number, number][][]
+
+const POSE_3D = POSE_3D_ as SkeletonType
+const POSE_3D_MANY = POSE_3D_MANY_ as SkeletonType[] // N F 133 3
 
 const THREE_ADDONS = {
   FontLoader,
@@ -98,21 +102,21 @@ const DEFAULT_NEAR = 0.05
 const DEFAULT_FAR = 1
 const CAMERA_EXTRINSIC_MATRIX_MAP: Record<string, number[]> = {
   "AE_01": [
-    0.2321047, -0.97264263, -0.0096808, -0.96323585, 0.06882254,
+    0.37408302, -0.91907411, 0.12395429, 1.18976111, 0.17243349,
-    0.02634936, -0.99728089, 0.03661007, 0.97025299, 0.23080732,
+    -0.06239751, -0.98304285, -0.06429779, 0.91122367, 0.38911351,
-    0.07305554, 3.34933242, 0., 0., 0.,
+    0.13513731, 2.51940833, 0., 0., 0.,
     1.
   ] as const,
   "AE_1A": [
-    0.93194049, -0.35571886, -0.07036343, -0.92123075, 0.01084819,
+    0.92998171, -0.36696694, -0.02166301, 2.21643671, -0.05110403,
-    0.22131041, -0.97514308, 0.24922173, 0.36244895, 0.908012,
+    -0.07070226, -0.99618752, -0.72948697, 0.36403626, 0.92754324,
-    0.21010704, 4.87284891, 0., 0., 0.,
+    -0.0845053, 6.45800206, 0., 0., 0.,
     1.
   ] as const,
   "AE_08": [
-    0.66806102, -0.74355508, -0.02864123, -1.10173496, 0.05931037,
+    0.98195914, -0.18888337, -0.00890642, 1.43011854, -0.02247979,
-    0.09157787, -0.99403007, 0.26760438, 0.74173901, 0.66237402,
+    -0.06984105, -0.99730481, -0.61678831, 0.18775226, 0.97951279,
-    0.10528013, 6.92372493, 0., 0., 0.,
+    -0.07282712, 5.81983825, 0., 0., 0.,
     1.
   ] as const
 }
@@ -257,6 +261,7 @@ const Scene = () => {
   }
 
   interface Human3DSkeletonProps {
+    skeleton: SkeletonType
     startFrame?: number
     jointRadius?: number
     boneRadius?: number
@@ -266,6 +271,7 @@ const Scene = () => {
   }
 
   const Human3DSkeleton = ({
+    skeleton,
     startFrame = 0,
     jointRadius = 0.01,
     boneRadius = 0.005,
@@ -274,10 +280,8 @@ const Scene = () => {
     frameRate = 30
   }: Human3DSkeletonProps) => {
     const [frameIndex, setFrameIndex] = useState(startFrame)
-    const totalFrames = POSE_3D.length
+    const totalFrames = skeleton.length
-
+    const onFrame: RenderCallback = (totalFrames === 0) ? (state, delta) => { } : (state: RootState, delta: number) => {
-    // Use frame to animate through the skeleton poses
-    useFrame((state, delta) => {
       // Calculate next frame based on desired frame rate and delta time
       setFrameIndex(prevFrame => {
         // Calculate next frame
@@ -285,11 +289,15 @@ const Scene = () => {
         // Loop back to start if we reach the end
         return nextFrame >= totalFrames ? 0 : nextFrame
       })
-    })
+      return null
+    }
+
+    // Use frame to animate through the skeleton poses
+    useFrame(onFrame)
 
     // Get the current frame joints - use Math.floor to get the nearest frame
     const currentFrame = Math.floor(frameIndex) % totalFrames
-    const joints = POSE_3D[currentFrame]
+    const joints = skeleton[currentFrame]
 
     // Function to get appropriate color for a joint index
     const getJointColor = (idx: number) => {
@@ -395,20 +403,33 @@ const Scene = () => {
     )
   }
 
+  // const S0 = [POSE_3D_MANY[0][0]]
+  // const S1 = [POSE_3D_MANY[0][1]]
+  // const skeletons = POSE_3D_MANY.map((el) => <Human3DSkeleton jointRadius={0.005} boneRadius={0.0025} frameRate={24} skeleton={el} />)
+  // const skeletons = [<Human3DSkeleton jointRadius={0.05} boneRadius={0.025} frameRate={1} skeleton={S0} />,
+  // <Human3DSkeleton jointRadius={0.05} boneRadius={0.025} frameRate={1} skeleton={S1} />
+  // ]
+  const skeletons = [
+    <Human3DSkeleton jointRadius={0.005} boneRadius={0.0025} frameRate={24} skeleton={POSE_3D} />,
+  ]
+
+  const cameras = Object.entries(CAMERA_EXTRINSIC_MATRIX_MAP).map(([key, value]) => {
+    const intrinsic = CAMERA_INTRINSIC_MATRIX_MAP[key]
+    const { fov_x, fov_y } = intrinsicToFov(intrinsic, { width: IMAGE_WIDTH, height: IMAGE_HEIGHT })
+    // make the far reverse proportional to the fov
+    const far = (1 / fov_x) * 20
+    return <CameraViewFromExtrinsic key={key} name={`${key}(${fov_x.toFixed(1)})`} extrinsic={preProcessExtrinsic(value)} fov={fov_x} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} far={far} />
+  })
+
   const scene = (<group>
     {/* <OrbitControls /> */}
     <ambientLight intensity={0.05} />
     <directionalLight castShadow position={[3.3, 6, 4.4]} intensity={5} />
     {/* <Floor /> */}
-    {Object.entries(CAMERA_EXTRINSIC_MATRIX_MAP).map(([key, value]) => {
+    { }
-      const intrinsic = CAMERA_INTRINSIC_MATRIX_MAP[key]
-      const { fov_x, fov_y } = intrinsicToFov(intrinsic, { width: IMAGE_WIDTH, height: IMAGE_HEIGHT })
-      // make the far reverse proportional to the fov
-      const far = (1 / fov_x) * 20
-      return <CameraViewFromExtrinsic key={key} name={`${key}(${fov_x.toFixed(1)})`} extrinsic={preProcessExtrinsic(value)} fov={fov_x} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} far={far} />
-    })}
     <Axes />
-    <Human3DSkeleton jointRadius={0.005} boneRadius={0.0025} frameRate={24} />
+    {cameras}
+    {skeletons}
   </group>)
   return (
     // Note that we don't need to import anything, All three.js objects will be treated