gyf/CVTH3PE
1
0
Fork 0
forked from HQU-gxy/CVTH3PE
Code Pull Requests Activity

feat: Enhance play notebook and camera module with new unprojection functionalities

Browse Source 
- Updated the play notebook to include new methods for unprojecting 2D points onto a 3D plane.
- Introduced `unproject_points_onto_plane` and `unproject_points_to_z_plane` functions in the camera module for improved point handling.
- Enhanced the `Camera` class with a method for unprojecting points to a specified z-plane.
- Cleaned up execution counts in the notebook for better organization and clarity.
This commit is contained in:
crosstyan
2025-04-24 18:55:24 +08:00
parent 00481a0d6f
commit c3c93f6ca6
3 changed files with 160 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
play.ipynb
View File

@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
@ -43,7 +43,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
@ -75,7 +75,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
@ -97,7 +97,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 33,
"metadata": {},
"outputs": [],
"source": [
@ -174,7 +174,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
@ -246,7 +246,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
@ -339,7 +339,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
@ -348,7 +348,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
@ -432,7 +432,7 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 43,
"metadata": {},
"outputs": [],
"source": [
@ -530,21 +530,32 @@
"\n",
"\n",
"@jaxtyped(typechecker=beartype)\n",
"@dataclass\n",
"@dataclass(frozen=True)\n",
"class Tracking:\n",
" id: int\n",
" keypoints: Float[Array, \"J 3\"]\n",
" last_active_timestamp: datetime\n",
"\n",
" def __repr__(self) -> str:\n",
" return f\"Tracking({self.id}, {self.last_active_timestamp})\"\n",
"\n",
"\n",
"@jaxtyped(typechecker=beartype)\n",
"def triangle_from_cluster(cluster: list[Detection]) -> Float[Array, \"N 3\"]:\n",
"def triangle_from_cluster(\n",
" cluster: list[Detection],\n",
") -> tuple[Float[Array, \"N 3\"], datetime]:\n",
" proj_matrices = jnp.array([el.camera.params.projection_matrix for el in cluster])\n",
" points = jnp.array([el.keypoints_undistorted for el in cluster])\n",
" confidences = jnp.array([el.confidences for el in cluster])\n",
" return triangulate_points_from_multiple_views_linear(\n",
" proj_matrices, points, confidences=confidences\n",
" latest_timestamp = max(el.timestamp for el in cluster)\n",
" return (\n",
" triangulate_points_from_multiple_views_linear(\n",
" proj_matrices, points, confidences=confidences\n",
" ),\n",
" latest_timestamp,\n",
" )\n",
"\n",
"\n",
"# res = {\n",
"# \"a\": triangle_from_cluster(clusters_detections[0]).tolist(),\n",
"# \"b\": triangle_from_cluster(clusters_detections[1]).tolist(),\n",
@ -552,28 +563,39 @@
"# with open(\"samples/res.json\", \"wb\") as f:\n",
"# f.write(orjson.dumps(res))\n",
"\n",
"\n",
"class GlobalTrackingState:\n",
" _last_id: int\n",
" _trackings: list[Tracking]\n",
" _trackings: dict[int, Tracking]\n",
"\n",
" def __init__(self):\n",
" self._last_id = 0\n",
" self._trackings = []\n",
" self._trackings = {}\n",
"\n",
" def __repr__(self) -> str:\n",
" return (\n",
" f\"GlobalTrackingState(last_id={self._last_id}, trackings={self._trackings})\"\n",
" )\n",
"\n",
" @property\n",
" def trackings(self) -> list[Tracking]:\n",
" def trackings(self) -> dict[int, Tracking]:\n",
" return shallow_copy(self._trackings)\n",
"\n",
" def add_tracking(self, cluster: list[Detection]) -> Tracking:\n",
" tracking = Tracking(id=self._last_id, keypoints=triangle_from_cluster(cluster))\n",
" self._last_id += 1\n",
" self._trackings.append(tracking)\n",
" kps_3d, latest_timestamp = triangle_from_cluster(cluster)\n",
" next_id = self._last_id + 1\n",
" tracking = Tracking(\n",
" id=next_id, keypoints=kps_3d, last_active_timestamp=latest_timestamp\n",
" )\n",
" self._trackings[next_id] = tracking\n",
" self._last_id = next_id\n",
" return tracking\n",
"\n",
"\n",
"global_tracking_state = GlobalTrackingState()\n",
"for cluster in clusters_detections:\n",
" global_tracking_state.add_tracking(cluster)"
" global_tracking_state.add_tracking(cluster)\n",
"display(global_tracking_state)"
]
},
{
@ -585,6 +607,28 @@
"next_group = next(sync_gen)\n",
"display(next_group)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from app.camera import classify_by_camera\n",
"\n",
"# let's do cross-view association\n",
"trackings = sorted(global_tracking_state.trackings.values(), key=lambda x: x.id)\n",
"detections = shallow_copy(next_group)\n",
"# cross-view association matrix with shape (T, D), where T is the number of trackings, D is the number of detections\n",
"affinity = np.zeros((len(trackings), len(detections)))\n",
"detection_by_camera = classify_by_camera(detections)\n",
"for i, tracking in enumerate(trackings):\n",
" for c, detections in detection_by_camera.items():\n",
" camera = next(iter(detections)).camera\n",
" # pixel space, unnormalized\n",
" tracking_2d_projection = camera.project(tracking.keypoints)\n",
" \n"
]
}
],
"metadata": {