From 42c5ff6540ee7a1ec12ec408d15cf5042316146e Mon Sep 17 00:00:00 2001 From: crosstyan Date: Wed, 16 Apr 2025 10:24:45 +0800 Subject: [PATCH] Refactor App.tsx to enhance skeleton animation capabilities by introducing a new pose data structure and updating the Human3DSkeleton component to accept customizable skeletons. Adjust camera extrinsic parameters for improved scene rendering. --- src/App.tsx | 71 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/src/App.tsx b/src/App.tsx index d0d126c..7318ac2 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,5 +1,5 @@ import { Grid, useBVH, useGLTF, CameraControls, AccumulativeShadows, OrbitControls, Stats } from '@react-three/drei' -import { Camera, Canvas, useFrame, useThree, useLoader } from '@react-three/fiber' +import { Camera, Canvas, useFrame, useThree, useLoader, RenderCallback, RootState } from '@react-three/fiber' import * as THREE from 'three' import { FontLoader } from 'three/addons/loaders/FontLoader.js' import { TextGeometry } from 'three/addons/geometries/TextGeometry.js' @@ -7,9 +7,13 @@ import HelvetikerRegular from "three/examples/fonts/helvetiker_regular.typeface. import { useEffect, useRef, useState, JSX } from 'react' // import POSE_3D_ from "./assets/result_ae_01_ae_08.json" import POSE_3D_ from "./assets/temp_result.json" +import POSE_3D_MANY_ from "./assets/many_people_all_3d_pose.json" // F, 133, 3 -const POSE_3D: Array> = POSE_3D_ as [number, number, number][][] +type SkeletonType = [number, number, number][][] + +const POSE_3D = POSE_3D_ as SkeletonType +const POSE_3D_MANY = POSE_3D_MANY_ as SkeletonType[] // N F 133 3 const THREE_ADDONS = { FontLoader, @@ -98,21 +102,21 @@ const DEFAULT_NEAR = 0.05 const DEFAULT_FAR = 1 const CAMERA_EXTRINSIC_MATRIX_MAP: Record = { "AE_01": [ - 0.2321047, -0.97264263, -0.0096808, -0.96323585, 0.06882254, - 0.02634936, -0.99728089, 0.03661007, 0.97025299, 0.23080732, - 0.07305554, 3.34933242, 0., 0., 0., + 0.37408302, -0.91907411, 0.12395429, 1.18976111, 0.17243349, + -0.06239751, -0.98304285, -0.06429779, 0.91122367, 0.38911351, + 0.13513731, 2.51940833, 0., 0., 0., 1. ] as const, "AE_1A": [ - 0.93194049, -0.35571886, -0.07036343, -0.92123075, 0.01084819, - 0.22131041, -0.97514308, 0.24922173, 0.36244895, 0.908012, - 0.21010704, 4.87284891, 0., 0., 0., + 0.92998171, -0.36696694, -0.02166301, 2.21643671, -0.05110403, + -0.07070226, -0.99618752, -0.72948697, 0.36403626, 0.92754324, + -0.0845053, 6.45800206, 0., 0., 0., 1. ] as const, "AE_08": [ - 0.66806102, -0.74355508, -0.02864123, -1.10173496, 0.05931037, - 0.09157787, -0.99403007, 0.26760438, 0.74173901, 0.66237402, - 0.10528013, 6.92372493, 0., 0., 0., + 0.98195914, -0.18888337, -0.00890642, 1.43011854, -0.02247979, + -0.06984105, -0.99730481, -0.61678831, 0.18775226, 0.97951279, + -0.07282712, 5.81983825, 0., 0., 0., 1. ] as const } @@ -257,6 +261,7 @@ const Scene = () => { } interface Human3DSkeletonProps { + skeleton: SkeletonType startFrame?: number jointRadius?: number boneRadius?: number @@ -266,6 +271,7 @@ const Scene = () => { } const Human3DSkeleton = ({ + skeleton, startFrame = 0, jointRadius = 0.01, boneRadius = 0.005, @@ -274,10 +280,8 @@ const Scene = () => { frameRate = 30 }: Human3DSkeletonProps) => { const [frameIndex, setFrameIndex] = useState(startFrame) - const totalFrames = POSE_3D.length - - // Use frame to animate through the skeleton poses - useFrame((state, delta) => { + const totalFrames = skeleton.length + const onFrame: RenderCallback = (totalFrames === 0) ? (state, delta) => { } : (state: RootState, delta: number) => { // Calculate next frame based on desired frame rate and delta time setFrameIndex(prevFrame => { // Calculate next frame @@ -285,11 +289,15 @@ const Scene = () => { // Loop back to start if we reach the end return nextFrame >= totalFrames ? 0 : nextFrame }) - }) + return null + } + + // Use frame to animate through the skeleton poses + useFrame(onFrame) // Get the current frame joints - use Math.floor to get the nearest frame const currentFrame = Math.floor(frameIndex) % totalFrames - const joints = POSE_3D[currentFrame] + const joints = skeleton[currentFrame] // Function to get appropriate color for a joint index const getJointColor = (idx: number) => { @@ -395,20 +403,33 @@ const Scene = () => { ) } + // const S0 = [POSE_3D_MANY[0][0]] + // const S1 = [POSE_3D_MANY[0][1]] + // const skeletons = POSE_3D_MANY.map((el) => ) + // const skeletons = [, + // + // ] + const skeletons = [ + , + ] + + const cameras = Object.entries(CAMERA_EXTRINSIC_MATRIX_MAP).map(([key, value]) => { + const intrinsic = CAMERA_INTRINSIC_MATRIX_MAP[key] + const { fov_x, fov_y } = intrinsicToFov(intrinsic, { width: IMAGE_WIDTH, height: IMAGE_HEIGHT }) + // make the far reverse proportional to the fov + const far = (1 / fov_x) * 20 + return + }) + const scene = ( {/* */} {/* */} - {Object.entries(CAMERA_EXTRINSIC_MATRIX_MAP).map(([key, value]) => { - const intrinsic = CAMERA_INTRINSIC_MATRIX_MAP[key] - const { fov_x, fov_y } = intrinsicToFov(intrinsic, { width: IMAGE_WIDTH, height: IMAGE_HEIGHT }) - // make the far reverse proportional to the fov - const far = (1 / fov_x) * 20 - return - })} + { } - + {cameras} + {skeletons} ) return ( // Note that we don't need to import anything, All three.js objects will be treated