Enhance Human3DSkeleton component to visualize 3D skeleton with color-coded joints and bones. Introduce body and hand bone connections, and update joint rendering logic to use specific colors based on joint types. Maintain compatibility with the previous Joints component.

src/App.tsx

@@ -7,7 +7,7 @@ import HelvetikerRegular from "three/examples/fonts/helvetiker_regular.typeface.
 import { useEffect, useRef, useState, JSX } from 'react'
 import POSE_3D_ from "../public/result_ae_01_ae_08.json"
 
-// F, 16, 3
+// F, 133, 3
 const POSE_3D: Array<Array<[number, number, number]>> = POSE_3D_ as [number, number, number][][]
 
 const THREE_ADDONS = {
@@ -41,6 +41,52 @@ const Z_UP_TO_Y_UP_PRIME = new THREE.Matrix4().set(
   0, 0, 0, 1
 )
 
+// Color definitions for different body parts
+const COLOR_SPINE = new THREE.Color(138 / 255, 201 / 255, 38 / 255)  // green, spine & head
+const COLOR_ARMS = new THREE.Color(255 / 255, 202 / 255, 58 / 255)   // yellow, arms & shoulders
+const COLOR_LEGS = new THREE.Color(25 / 255, 130 / 255, 196 / 255)   // blue, legs & hips
+const COLOR_FINGERS = new THREE.Color(255 / 255, 0, 0)           // red, fingers
+const COLOR_FACE = new THREE.Color(255 / 255, 200 / 255, 0)        // yellow, face
+const COLOR_FOOT = new THREE.Color(255 / 255, 128 / 255, 0)        // orange, foot
+const COLOR_HEAD = new THREE.Color(255 / 255, 0, 255 / 255)        // purple, head
+
+// Body bone connections
+const BODY_BONES = [
+  // legs
+  [15, 13], [13, 11], [16, 14], [14, 12], [11, 12], // legs
+  [5, 11], [6, 12], [5, 6],                         // torso
+  [5, 7], [7, 9], [6, 8], [8, 10],                  // arms
+  [1, 2], [0, 1], [0, 2], [1, 3], [2, 4],           // head
+  [15, 17], [15, 18], [15, 19],                     // left foot
+  [16, 20], [16, 21], [16, 22],                     // right foot
+]
+
+// Body bone colors
+const BODY_BONE_COLORS = [
+  COLOR_LEGS, COLOR_LEGS, COLOR_LEGS, COLOR_LEGS, COLOR_LEGS,
+  COLOR_SPINE, COLOR_SPINE, COLOR_SPINE,
+  COLOR_ARMS, COLOR_ARMS, COLOR_ARMS, COLOR_ARMS,
+  COLOR_HEAD, COLOR_HEAD, COLOR_HEAD, COLOR_HEAD, COLOR_HEAD,
+  COLOR_FOOT, COLOR_FOOT, COLOR_FOOT,
+  COLOR_FOOT, COLOR_FOOT, COLOR_FOOT,
+]
+
+// Hand bone connections (in pairs of [start, end] indices)
+const HAND_BONES = [
+  // right hand
+  [91, 92], [92, 93], [93, 94], [94, 95],           // right thumb
+  [91, 96], [96, 97], [97, 98], [98, 99],           // right index
+  [91, 100], [100, 101], [101, 102], [102, 103],    // right middle
+  [91, 104], [104, 105], [105, 106], [106, 107],    // right ring
+  [91, 108], [108, 109], [109, 110], [110, 111],    // right pinky
+  // left hand
+  [112, 113], [113, 114], [114, 115], [115, 116],   // left thumb
+  [112, 117], [117, 118], [118, 119], [119, 120],   // left index
+  [112, 121], [121, 122], [122, 123], [123, 124],   // left middle
+  [112, 125], [125, 126], [126, 127], [127, 128],   // left ring
+  [112, 129], [129, 130], [130, 131], [131, 132]    // left pinky
+]
+
 const DEFAULT_TRANSFORMATION_MATRIX = [
   1, 0, 0, 0,
   0, 1, 0, 0,
@@ -209,26 +255,129 @@ const Scene = () => {
     return final
   }
 
-  interface JointsProps {
+  interface Human3DSkeletonProps {
     index: number
-    radius?: number
+    jointRadius?: number
+    boneRadius?: number
+    showJoints?: boolean
+    showBones?: boolean
   }
-  const Joints = ({ index, radius }: JointsProps) => {
+
-    const js = POSE_3D[index]
+  const Human3DSkeleton = ({ index, jointRadius = 0.01, boneRadius = 0.005, showJoints = true, showBones = true }: Human3DSkeletonProps) => {
-    const r = radius ?? 0.1
+    const joints = POSE_3D[index]
-    const joint = js.map((j) => {
+
+    // Function to get appropriate color for a joint index
+    const getJointColor = (idx: number) => {
+      // Face joints (23-90)
+      if (idx >= 23 && idx <= 90) return COLOR_FACE
+      // Hand joints (91-132)
+      if (idx >= 91 && idx <= 132) return COLOR_FINGERS
+      // Foot joints (17-22)
+      if (idx >= 17 && idx <= 22) return COLOR_FOOT
+      // Head (0-4)
+      if (idx <= 4) return COLOR_HEAD
+      // Arms (5-10)
+      if (idx >= 5 && idx <= 10) return COLOR_ARMS
+      // Legs (11-16)
+      if (idx >= 11 && idx <= 16) return COLOR_LEGS
+      // Default
+      return COLOR_SPINE
+    }
+
+    // Transform a joint position using the coordinate system conversion
+    const transformJointPosition = (j: [number, number, number]) => {
       const [x, y, z] = j
       const V = new THREE.Vector3(x, y, z)
       const worldCvt = Z_UP_TO_Y_UP_PRIME.clone()
       V.applyMatrix4(worldCvt)
-      // random pick a color
+      return V
-      const color = `hsl(${Math.random() * 360}, 100%, 50%)`
+    }
-      return <mesh position={V}>
+
-        <sphereGeometry args={[r, 32, 32]} />
+    // Create the joint spheres
+    const jointMeshes = showJoints ? joints.map((j, idx) => {
+      const position = transformJointPosition(j)
+      const color = getJointColor(idx)
+
+      return (
+        <mesh key={`joint-${idx}`} position={position}>
+          <sphereGeometry args={[jointRadius, 16, 16]} />
           <meshStandardMaterial color={color} />
         </mesh>
-    })
+      )
-    return <group>{joint}</group>
+    }) : null
+
+    // Create the bone cylinders
+    const boneMeshes = showBones ? (
+      <>
+        {BODY_BONES.map((bone, idx) => {
+          const [startIdx, endIdx] = bone
+          if (startIdx >= joints.length || endIdx >= joints.length) return null
+
+          const startPos = transformJointPosition(joints[startIdx])
+          const endPos = transformJointPosition(joints[endIdx])
+          const color = BODY_BONE_COLORS[idx]
+
+          // Calculate midpoint and length
+          const midpoint = new THREE.Vector3().addVectors(startPos, endPos).multiplyScalar(0.5)
+          const length = startPos.distanceTo(endPos)
+
+          // Calculate rotation
+          const direction = new THREE.Vector3().subVectors(endPos, startPos).normalize()
+          const quaternion = new THREE.Quaternion()
+          const up = new THREE.Vector3(0, 1, 0)
+          quaternion.setFromUnitVectors(up, direction)
+
+          return (
+            <mesh key={`bone-body-${idx}`} position={midpoint} quaternion={quaternion}>
+              <cylinderGeometry args={[boneRadius, boneRadius, length, 8]} />
+              <meshStandardMaterial color={color} />
+            </mesh>
+          )
+        })}
+
+        {HAND_BONES.map((bone, idx) => {
+          const [startIdx, endIdx] = bone
+          if (startIdx >= joints.length || endIdx >= joints.length) return null
+
+          const startPos = transformJointPosition(joints[startIdx])
+          const endPos = transformJointPosition(joints[endIdx])
+
+          // Calculate midpoint and length
+          const midpoint = new THREE.Vector3().addVectors(startPos, endPos).multiplyScalar(0.5)
+          const length = startPos.distanceTo(endPos)
+
+          // Calculate rotation
+          const direction = new THREE.Vector3().subVectors(endPos, startPos).normalize()
+          const quaternion = new THREE.Quaternion()
+          const up = new THREE.Vector3(0, 1, 0)
+          quaternion.setFromUnitVectors(up, direction)
+
+          return (
+            <mesh key={`bone-hand-${idx}`} position={midpoint} quaternion={quaternion}>
+              <cylinderGeometry args={[boneRadius, boneRadius, length, 8]} />
+              <meshStandardMaterial color={COLOR_FINGERS} />
+            </mesh>
+          )
+        })}
+      </>
+    ) : null
+
+    return (
+      <group>
+        {jointMeshes}
+        {boneMeshes}
+      </group>
+    )
+  }
+
+  interface JointsProps {
+    index: number
+    radius?: number
+  }
+
+  // Keep the old Joints component for compatibility
+  const Joints = ({ index, radius }: JointsProps) => {
+    return <Human3DSkeleton index={index} jointRadius={radius} />
   }
 
   const scene = (<group>
@@ -244,7 +393,7 @@ const Scene = () => {
       return <CameraViewFromExtrinsic key={key} name={`${key}(${fov_x.toFixed(1)})`} extrinsic={preProcessExtrinsic(value)} fov={fov_x} aspect={IMAGE_WIDTH / IMAGE_HEIGHT} far={far} />
     })}
     <Axes />
-    <Joints index={12} radius={0.01} />
+    <Human3DSkeleton index={12} jointRadius={0.01} boneRadius={0.005} />
   </group>)
   return (
     // Note that we don't need to import anything, All three.js objects will be treated