feat(calibrate): integrate multi-frame depth pooling with --depth-pool-size flag

This commit is contained in:
2026-02-07 08:10:01 +00:00
parent dad1f2a69f
commit 4fc8de4bdc
6 changed files with 774 additions and 82 deletions
@@ -67,7 +67,7 @@ def test_benchmark_matrix(mock_dependencies):
"frame_index": 100,
}
verification_frames = {serial_int: vf}
verification_frames = {serial_int: [vf]}
first_frames = {serial_int: vf}
marker_geometry = {1: np.zeros((4, 3))}
camera_matrices = {serial_int: np.eye(3)}
@@ -100,6 +100,7 @@ def test_verify_only(mock_dependencies, tmp_path):
# Setup inputs
serial = "123456"
serial_int = int(serial)
results = {
serial: {
"pose": "1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1", # Identity matrix flattened
@@ -107,16 +108,18 @@ def test_verify_only(mock_dependencies, tmp_path):
}
}
verification_frames = {
serial: {
"frame": MagicMock(
depth_map=np.zeros((10, 10)), confidence_map=np.zeros((10, 10))
),
"ids": np.array([[1]]),
"corners": np.zeros((1, 4, 2)),
}
serial_int: [
{
"frame": MagicMock(
depth_map=np.zeros((10, 10)), confidence_map=np.zeros((10, 10))
),
"ids": np.array([[1]]),
"corners": np.zeros((1, 4, 2)),
}
]
}
marker_geometry = {1: np.zeros((4, 3))}
camera_matrices = {serial: np.eye(3)}
camera_matrices = {serial_int: np.eye(3)}
updated_results, csv_rows = apply_depth_verify_refine_postprocess(
results=results,
@@ -146,18 +149,21 @@ def test_refine_depth(mock_dependencies):
# Setup inputs
serial = "123456"
serial_int = int(serial)
results = {serial: {"pose": "1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1", "stats": {}}}
verification_frames = {
serial: {
"frame": MagicMock(
depth_map=np.zeros((10, 10)), confidence_map=np.zeros((10, 10))
),
"ids": np.array([[1]]),
"corners": np.zeros((1, 4, 2)),
}
serial_int: [
{
"frame": MagicMock(
depth_map=np.zeros((10, 10)), confidence_map=np.zeros((10, 10))
),
"ids": np.array([[1]]),
"corners": np.zeros((1, 4, 2)),
}
]
}
marker_geometry = {1: np.zeros((4, 3))}
camera_matrices = {serial: np.eye(3)}
camera_matrices = {serial_int: np.eye(3)}
# Mock verify to return different values for pre and post
# First call (pre-refine)
@@ -199,15 +205,18 @@ def test_refine_depth_warning_negligible_improvement(mock_dependencies):
mock_verify, mock_refine, mock_echo = mock_dependencies
serial = "123456"
serial_int = int(serial)
results = {serial: {"pose": "1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1", "stats": {}}}
verification_frames = {
serial: {
"frame": MagicMock(depth_map=np.zeros((10, 10))),
"ids": np.array([[1]]),
}
serial_int: [
{
"frame": MagicMock(depth_map=np.zeros((10, 10))),
"ids": np.array([[1]]),
}
]
}
marker_geometry = {1: np.zeros((4, 3))}
camera_matrices = {serial: np.eye(3)}
camera_matrices = {serial_int: np.eye(3)}
# RMSE stays almost same
res_pre = MagicMock(rmse=0.1, n_valid=10, residuals=[])
@@ -249,15 +258,18 @@ def test_refine_depth_warning_failed_or_stalled(mock_dependencies):
mock_verify, mock_refine, mock_echo = mock_dependencies
serial = "123456"
serial_int = int(serial)
results = {serial: {"pose": "1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1", "stats": {}}}
verification_frames = {
serial: {
"frame": MagicMock(depth_map=np.zeros((10, 10))),
"ids": np.array([[1]]),
}
serial_int: [
{
"frame": MagicMock(depth_map=np.zeros((10, 10))),
"ids": np.array([[1]]),
}
]
}
marker_geometry = {1: np.zeros((4, 3))}
camera_matrices = {serial: np.eye(3)}
camera_matrices = {serial_int: np.eye(3)}
res_pre = MagicMock(rmse=0.1, n_valid=10, residuals=[])
res_post = MagicMock(rmse=0.1, n_valid=10, residuals=[])
@@ -298,18 +310,21 @@ def test_csv_output(mock_dependencies, tmp_path):
csv_path = tmp_path / "report.csv"
serial = "123456"
serial_int = int(serial)
results = {serial: {"pose": "1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1", "stats": {}}}
verification_frames = {
serial: {
"frame": MagicMock(
depth_map=np.zeros((10, 10)), confidence_map=np.zeros((10, 10))
),
"ids": np.array([[1]]),
"corners": np.zeros((1, 4, 2)),
}
serial_int: [
{
"frame": MagicMock(
depth_map=np.zeros((10, 10)), confidence_map=np.zeros((10, 10))
),
"ids": np.array([[1]]),
"corners": np.zeros((1, 4, 2)),
}
]
}
marker_geometry = {1: np.zeros((4, 3))}
camera_matrices = {serial: np.eye(3)}
camera_matrices = {serial_int: np.eye(3)}
updated_results, csv_rows = apply_depth_verify_refine_postprocess(
results=results,
@@ -324,11 +339,11 @@ def test_csv_output(mock_dependencies, tmp_path):
)
assert len(csv_rows) == 2 # From mock_verify_res.residuals
assert csv_rows[0] == [serial, 1, 0, 0.01]
assert csv_rows[0] == [serial_int, 1, 0, 0.01]
# Verify file content
assert csv_path.exists()
content = csv_path.read_text().splitlines()
assert len(content) == 3 # Header + 2 rows
assert content[0] == "serial,marker_id,corner_idx,residual"
assert content[1] == f"{serial},1,0,0.01"
assert content[1] == f"{serial_int},1,0,0.01"
+134
View File
@@ -0,0 +1,134 @@
import numpy as np
import pytest
from aruco.depth_pool import pool_depth_maps
def test_pool_depth_maps_empty():
with pytest.raises(ValueError, match="depth_maps list cannot be empty"):
_ = pool_depth_maps([])
def test_pool_depth_maps_shape_mismatch():
dm1 = np.ones((10, 10))
dm2 = np.ones((10, 11))
with pytest.raises(ValueError, match="inconsistent shape"):
_ = pool_depth_maps([dm1, dm2])
def test_pool_depth_maps_confidence_mismatch():
dm1 = np.ones((10, 10))
cm1 = np.ones((10, 10))
with pytest.raises(ValueError, match="must match number of depth maps"):
_ = pool_depth_maps([dm1], confidence_maps=[cm1, cm1])
def test_pool_depth_maps_confidence_shape_mismatch():
dm1 = np.ones((10, 10))
cm1 = np.ones((10, 11))
with pytest.raises(ValueError, match="inconsistent shape"):
_ = pool_depth_maps([dm1], confidence_maps=[cm1])
def test_pool_depth_maps_single_map():
# N=1 returns masked copy behavior
dm = np.array([[1.0, -1.0], [np.nan, 2.0]])
pooled, conf = pool_depth_maps([dm])
expected = np.array([[1.0, np.nan], [np.nan, 2.0]])
np.testing.assert_allclose(pooled, expected)
assert conf is None
# Test min_valid_count > 1 for single map
pooled, _ = pool_depth_maps([dm], min_valid_count=2)
assert np.all(np.isnan(pooled))
def test_pool_depth_maps_median():
# Median pooling with clean values
dm1 = np.array([[1.0, 2.0], [3.0, 4.0]])
dm2 = np.array([[1.2, 1.8], [3.2, 3.8]])
dm3 = np.array([[0.8, 2.2], [2.8, 4.2]])
pooled, _ = pool_depth_maps([dm1, dm2, dm3])
# Median of [1.0, 1.2, 0.8] is 1.0
# Median of [2.0, 1.8, 2.2] is 2.0
# Median of [3.0, 3.2, 2.8] is 3.0
# Median of [4.0, 3.8, 4.2] is 4.0
expected = np.array([[1.0, 2.0], [3.0, 4.0]])
np.testing.assert_allclose(pooled, expected)
def test_pool_depth_maps_invalid_handling():
# NaN/invalid handling (non-finite or <=0)
dm1 = np.array([[1.0, np.nan], [0.0, -1.0]])
dm2 = np.array([[1.2, 2.0], [3.0, 4.0]])
pooled, _ = pool_depth_maps([dm1, dm2])
# (0,0): median(1.0, 1.2) = 1.1
# (0,1): median(nan, 2.0) = 2.0
# (1,0): median(0.0, 3.0) = 3.0 (0.0 is invalid)
# (1,1): median(-1.0, 4.0) = 4.0 (-1.0 is invalid)
expected = np.array([[1.1, 2.0], [3.0, 4.0]])
np.testing.assert_allclose(pooled, expected)
def test_pool_depth_maps_confidence_gating():
# Confidence gating (confidence > threshold excluded)
dm1 = np.array([[1.0, 1.0], [1.0, 1.0]])
dm2 = np.array([[2.0, 2.0], [2.0, 2.0]])
cm1 = np.array([[10, 60], [10, 60]])
cm2 = np.array([[60, 10], [10, 10]])
# threshold = 50
pooled, pooled_conf = pool_depth_maps(
[dm1, dm2], confidence_maps=[cm1, cm2], confidence_thresh=50.0
)
# (0,0): dm1 valid (10), dm2 invalid (60) -> 1.0
# (0,1): dm1 invalid (60), dm2 valid (10) -> 2.0
# (1,0): dm1 valid (10), dm2 valid (10) -> 1.5
# (1,1): dm1 invalid (60), dm2 valid (10) -> 2.0
expected_depth = np.array([[1.0, 2.0], [1.5, 2.0]])
expected_conf = np.array([[10, 10], [10, 10]])
np.testing.assert_allclose(pooled, expected_depth)
assert pooled_conf is not None
np.testing.assert_allclose(pooled_conf, expected_conf)
def test_pool_depth_maps_all_invalid():
# All invalid -> NaN outputs
dm1 = np.array([[np.nan, 0.0], [-1.0, 1.0]])
cm1 = np.array([[10, 10], [10, 100]]) # 100 > 50
pooled, _ = pool_depth_maps([dm1], confidence_maps=[cm1], confidence_thresh=50.0)
assert np.all(np.isnan(pooled))
def test_pool_depth_maps_min_valid_count():
# min_valid_count enforcement
dm1 = np.array([[1.0, 1.0], [1.0, 1.0]])
dm2 = np.array([[2.0, 2.0], [np.nan, np.nan]])
# min_valid_count = 2
pooled, _ = pool_depth_maps([dm1, dm2], min_valid_count=2)
# (0,0): 2 valid -> 1.5
# (0,1): 2 valid -> 1.5
# (1,0): 1 valid -> nan
# (1,1): 1 valid -> nan
expected = np.array([[1.5, 1.5], [np.nan, np.nan]])
np.testing.assert_allclose(pooled, expected)
def test_pool_depth_maps_confidence_none():
# confidence_maps None behavior
dm1 = np.ones((2, 2))
dm2 = np.ones((2, 2)) * 2
pooled, conf = pool_depth_maps([dm1, dm2])
assert conf is None
np.testing.assert_allclose(pooled, np.ones((2, 2)) * 1.5)
@@ -0,0 +1,253 @@
import pytest
import numpy as np
from unittest.mock import MagicMock, patch
import sys
from pathlib import Path
# Add py_workspace to path
sys.path.append(str(Path(__file__).parent.parent))
from calibrate_extrinsics import apply_depth_verify_refine_postprocess
@pytest.fixture
def mock_dependencies():
with (
patch("calibrate_extrinsics.verify_extrinsics_with_depth") as mock_verify,
patch("calibrate_extrinsics.refine_extrinsics_with_depth") as mock_refine,
patch("calibrate_extrinsics.click.echo") as mock_echo,
):
# Setup mock return values
mock_verify_res = MagicMock()
mock_verify_res.rmse = 0.05
mock_verify_res.mean_abs = 0.04
mock_verify_res.median = 0.03
mock_verify_res.depth_normalized_rmse = 0.02
mock_verify_res.n_valid = 100
mock_verify_res.n_total = 120
mock_verify_res.residuals = []
mock_verify.return_value = mock_verify_res
mock_refine.return_value = (np.eye(4), {"success": True})
yield mock_verify, mock_refine, mock_echo
def test_pool_size_1_equivalence(mock_dependencies):
"""
Regression test: Ensure pool_size=1 behaves exactly like the old single-frame path.
"""
mock_verify, _, _ = mock_dependencies
serial = "123456"
results = {serial: {"pose": "1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1"}}
# Create a frame with specific depth values
depth_map = np.ones((10, 10)) * 2.0
conf_map = np.zeros((10, 10))
frame_mock = MagicMock()
frame_mock.depth_map = depth_map
frame_mock.confidence_map = conf_map
vf = {
"frame": frame_mock,
"ids": np.array([[1]]),
"corners": np.zeros((1, 4, 2)),
"score": 100.0,
}
# Structure for new implementation: list of frames
verification_frames = {serial: [vf]}
marker_geometry = {1: np.zeros((4, 3))}
camera_matrices = {serial: np.eye(3)}
# Run with pool_size=1
apply_depth_verify_refine_postprocess(
results=results,
verification_frames=verification_frames,
marker_geometry=marker_geometry,
camera_matrices=camera_matrices,
verify_depth=True,
refine_depth=False,
use_confidence_weights=False,
depth_confidence_threshold=50,
depth_pool_size=1,
)
# Verify that verify_extrinsics_with_depth was called with the exact depth map from the frame
args, _ = mock_verify.call_args
passed_depth_map = args[2]
np.testing.assert_array_equal(passed_depth_map, depth_map)
assert passed_depth_map is depth_map
def test_pool_size_5_integration(mock_dependencies):
"""
Test that pool_size > 1 actually calls pooling and uses the result.
"""
mock_verify, _, mock_echo = mock_dependencies
serial = "123456"
results = {serial: {"pose": "1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1"}}
# Create 3 frames with different depth values
# Frame 1: 2.0m
# Frame 2: 2.2m
# Frame 3: 1.8m
# Median should be 2.0m
frames = []
for d in [2.0, 2.2, 1.8]:
f = MagicMock()
f.depth_map = np.ones((10, 10)) * d
f.confidence_map = np.zeros((10, 10))
frames.append(f)
vfs = []
for i, f in enumerate(frames):
vfs.append(
{
"frame": f,
"ids": np.array([[1]]),
"corners": np.zeros((1, 4, 2)),
"score": 100.0 - i,
}
)
verification_frames = {serial: vfs}
marker_geometry = {1: np.zeros((4, 3))}
camera_matrices = {serial: np.eye(3)}
# Run with pool_size=3
apply_depth_verify_refine_postprocess(
results=results,
verification_frames=verification_frames,
marker_geometry=marker_geometry,
camera_matrices=camera_matrices,
verify_depth=True,
refine_depth=False,
use_confidence_weights=False,
depth_confidence_threshold=50,
depth_pool_size=3,
)
# Check that "Using pooled depth" was logged
any_pooled = any(
"Using pooled depth" in str(call.args[0]) for call in mock_echo.call_args_list
)
assert any_pooled
# Check that the depth map passed to verify is the median (2.0)
args, _ = mock_verify.call_args
passed_depth_map = args[2]
expected_median = np.ones((10, 10)) * 2.0
np.testing.assert_allclose(passed_depth_map, expected_median)
# Verify metadata was added
assert "depth_pool" in results[serial]
assert results[serial]["depth_pool"]["pooled"] is True
assert results[serial]["depth_pool"]["pool_size_actual"] == 3
def test_pool_fallback_insufficient_valid(mock_dependencies):
"""
Test fallback to single frame when pooled result has too few valid points.
"""
mock_verify, _, mock_echo = mock_dependencies
serial = "123456"
results = {serial: {"pose": "1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1"}}
# Frame 1: Good depth
f1 = MagicMock()
f1.depth_map = np.ones((10, 10)) * 2.0
f1.confidence_map = np.zeros((10, 10))
# Frame 2: NaN depth (simulating misalignment or noise)
f2 = MagicMock()
f2.depth_map = np.full((10, 10), np.nan)
f2.confidence_map = np.zeros((10, 10))
# Frame 3: NaN depth
f3 = MagicMock()
f3.depth_map = np.full((10, 10), np.nan)
f3.confidence_map = np.zeros((10, 10))
# With median pooling, if >50% are NaN, result is NaN (standard median behavior with NaNs usually propagates or ignores)
# Our pool_depth_maps uses nanmedian, which ignores NaNs.
# But if we have [2.0, NaN, NaN], median of [2.0] is 2.0.
# Wait, let's make it so they are valid but inconsistent to cause variance?
# Or just force the pooled result to be bad by making them all different and sparse?
# Let's use the fact that we can patch pool_depth_maps in the test!
with patch("calibrate_extrinsics.pool_depth_maps") as mock_pool:
# Return empty/invalid map
mock_pool.return_value = (
np.zeros((10, 10)),
None,
) # Zeros are invalid depth (<=0)
# Frame 1: Valid on left half
d1 = np.full((10, 10), np.nan)
d1[:, :5] = 2.0
f1.depth_map = d1
f1.confidence_map = np.zeros((10, 10))
# Frame 2: Valid on right half
d2 = np.full((10, 10), np.nan)
d2[:, 5:] = 2.0
f2.depth_map = d2
f2.confidence_map = np.zeros((10, 10))
vfs = [
{
"frame": f1,
"ids": np.array([[1]]),
"corners": np.zeros((1, 4, 2)),
"score": 100,
},
{
"frame": f2,
"ids": np.array([[1]]),
"corners": np.zeros((1, 4, 2)),
"score": 90,
},
]
verification_frames = {serial: vfs}
marker_geometry = {1: np.zeros((4, 3))}
camera_matrices = {serial: np.eye(3)}
apply_depth_verify_refine_postprocess(
results=results,
verification_frames=verification_frames,
marker_geometry=marker_geometry,
camera_matrices=camera_matrices,
verify_depth=True,
refine_depth=False,
use_confidence_weights=False,
depth_confidence_threshold=50,
depth_pool_size=2,
)
# Check for fallback message
any_fallback = any(
"Falling back to best single frame" in str(call.args[0])
for call in mock_echo.call_args_list
)
assert any_fallback
# Verify we used the best frame (f1)
args, _ = mock_verify.call_args
passed_depth_map = args[2]
assert passed_depth_map is d1
# Verify metadata
assert results[serial]["depth_pool"]["pooled"] is False
assert (
results[serial]["depth_pool"]["fallback_reason"]
== "insufficient_valid_points"
)