feat(rgbd): add RGB-D reconstruction pipeline

Add end-to-end RGB-D reconstruction support across the C++ core and Python API.

- add a native merge_rgbd_views path, view-aware 3D pose containers, and nanobind bindings

- expose Python helpers to sample aligned depth, apply per-joint offsets, lift UVD poses to world space, and run reconstruct_rgbd

- add RGB-D regression tests for merging, manual pipeline parity, symmetric depth sampling windows, and out-of-bounds joints

- bump the project version from 0.1.0 to 0.2.0 for the new feature surface
This commit is contained in:
2026-03-26 13:04:57 +08:00
parent 6c09f7044b
commit ed721729fd
12 changed files with 1932 additions and 5 deletions
+46
View File
@@ -23,6 +23,8 @@ using CountArray = nb::ndarray<nb::numpy, const uint32_t, nb::shape<-1>, nb::c_c
using TrackIdArray = nb::ndarray<nb::numpy, const int64_t, nb::shape<-1>, nb::c_contig>;
using PoseArray3DConst =
nb::ndarray<nb::numpy, const float, nb::shape<-1, -1, 4>, nb::c_contig>;
using PoseArray3DByViewConst =
nb::ndarray<nb::numpy, const float, nb::shape<-1, -1, -1, 4>, nb::c_contig>;
using PoseArray3D = nb::ndarray<nb::numpy, float, nb::shape<-1, -1, 4>, nb::c_contig>;
using PoseArray2DOut = nb::ndarray<nb::numpy, float, nb::shape<-1, 4>, nb::c_contig>;
@@ -59,6 +61,32 @@ PoseBatch3DView pose_batch3d_view_from_numpy(const PoseArray3DConst &poses_3d)
};
}
PoseBatch3DByViewView pose_batch3d_by_view_from_numpy(
const PoseArray3DByViewConst &poses_3d,
const CountArray &person_counts)
{
if (poses_3d.shape(0) != person_counts.shape(0))
{
throw std::invalid_argument("poses_3d and person_counts must have the same number of views.");
}
for (size_t i = 0; i < static_cast<size_t>(person_counts.shape(0)); ++i)
{
if (person_counts(i) > poses_3d.shape(1))
{
throw std::invalid_argument("person_counts entries must not exceed the padded person dimension.");
}
}
return PoseBatch3DByViewView {
poses_3d.data(),
person_counts.data(),
static_cast<size_t>(poses_3d.shape(0)),
static_cast<size_t>(poses_3d.shape(1)),
static_cast<size_t>(poses_3d.shape(2)),
};
}
TrackedPoseBatch3DView tracked_pose_batch_view_from_numpy(
const PoseArray3DConst &poses_3d,
const TrackIdArray &track_ids)
@@ -432,6 +460,24 @@ NB_MODULE(_core, m)
"person_counts"_a,
"config"_a);
m.def(
"merge_rgbd_views",
[](const PoseArray3DByViewConst &poses_3d,
const CountArray &person_counts,
const TriangulationConfig &config,
float max_distance)
{
const PoseBatch3D merged = merge_rgbd_views(
pose_batch3d_by_view_from_numpy(poses_3d, person_counts),
config,
max_distance);
return pose_batch_to_numpy(merged);
},
"poses_3d"_a,
"person_counts"_a,
"config"_a,
"max_distance"_a = 0.5f);
m.def(
"triangulate_with_report",
[](const PoseArray2D &poses_2d,