feat: initialize offline multiview pose tracking experiment

Set up pose_tracking_exp as a uv-managed Python package for offline multiview body tracking experiments. This initial commit includes: - the typed package scaffold, CLI entrypoints, and repo-local uv configuration - scene and replay loaders for generic JSON replays and ActualTest parquet inputs - ParaJumping payload conversion and RTMPose-to-body20 normalization - a custom articulated tracker with tentative, active, and lost lifecycle handling - RPT-backed proposal generation, camera convention handling, and multiview reprojection updates - regression tests for normalization, camera conventions, ActualTest ingestion, seeding, and tracker smoke flows - project documentation covering extrinsic formats and the ActualTest calibration caveat
2026-03-26 13:03:56 +08:00
commit f1a2372b3c
27 changed files with 3791 additions and 0 deletions
@@ -0,0 +1,77 @@
+from pathlib import Path
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from pose_tracking_exp.actualtest import load_actualtest_scene, load_actualtest_segment_bundles
+from pose_tracking_exp.joints import BODY20_INDEX_BY_NAME
+
+
+def _write_parquet(path: Path, rows: list[dict[str, object]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    pq.write_table(pa.Table.from_pylist(rows), path)
+
+
+def _sample_rtmpose_detection() -> tuple[list[float], list[list[float]], list[float]]:
+    keypoints_xy = np.zeros((133, 2), dtype=np.float64)
+    scores = np.zeros((133,), dtype=np.float64)
+    keypoints_xy[5] = [10.0, 20.0]
+    keypoints_xy[6] = [30.0, 20.0]
+    keypoints_xy[11] = [12.0, 60.0]
+    keypoints_xy[12] = [28.0, 60.0]
+    keypoints_xy[0] = [20.0, 8.0]
+    scores[[0, 5, 6, 11, 12]] = 1.0
+    return [8.0, 4.0, 32.0, 64.0], keypoints_xy.tolist(), scores.tolist()
+
+
+def test_load_actualtest_parquet_scene_and_segment(tmp_path: Path) -> None:
+    root = tmp_path / "ActualTest_WeiHua"
+    _write_parquet(
+        root / "camera_params" / "camera_params.parquet",
+        [
+            {
+                "name": "AF_02",
+                "port": 5602,
+                "intrinsic": {
+                    "camera_matrix": [[500.0, 0.0, 320.0], [0.0, 500.0, 240.0], [0.0, 0.0, 1.0]],
+                    "distortion_coefficients": [0.0, 0.0, 0.0, 0.0, 0.0],
+                },
+                "extrinsic": {"rvec": [0.0, 0.0, 0.0], "tvec": [0.0, 0.0, 0.0]},
+                "resolution": {"width": 640, "height": 480},
+            },
+            {
+                "name": "AF_03",
+                "port": 5603,
+                "intrinsic": {
+                    "camera_matrix": [[500.0, 0.0, 320.0], [0.0, 500.0, 240.0], [0.0, 0.0, 1.0]],
+                    "distortion_coefficients": [0.0, 0.0, 0.0, 0.0, 0.0],
+                },
+                "extrinsic": {"rvec": [0.0, 0.0, 0.0], "tvec": [1.0, 0.0, 0.0]},
+                "resolution": {"width": 640, "height": 480},
+            },
+        ],
+    )
+    box, keypoints_xy, scores = _sample_rtmpose_detection()
+    for camera_name in ("5602", "5603"):
+        _write_parquet(
+            root / "Segment_1" / f"{camera_name}_detected.parquet",
+            [
+                {"frame_index": 689, "boxes": [], "kps": [], "kps_scores": []},
+                {"frame_index": 690, "boxes": [box], "kps": [keypoints_xy], "kps_scores": [scores]},
+            ],
+        )
+
+    scene = load_actualtest_scene(root)
+    bundles = load_actualtest_segment_bundles(root, "Segment_1", frame_start=690, max_frames=1)
+
+    assert [camera.name for camera in scene.cameras] == ["5602", "5603"]
+    np.testing.assert_allclose(scene.cameras[0].pose_T, [0.0, 0.0, 0.0])
+    np.testing.assert_allclose(scene.cameras[1].pose_T, [-1.0, 0.0, 0.0])
+    assert len(bundles) == 1
+    assert [view.camera_name for view in bundles[0].views] == ["5602", "5603"]
+    assert bundles[0].views[0].frame_index == 690
+    np.testing.assert_allclose(
+        bundles[0].views[0].detections[0].keypoints[BODY20_INDEX_BY_NAME["hip_middle"], :2],
+        [20.0, 60.0],
+    )