diff --git a/README.md b/README.md index 169f5fb..0d4167b 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Fast triangulation of multiple persons from multiple camera views. -isystem /onnxruntime/include/onnxruntime/core/session/ \ -isystem /onnxruntime/include/onnxruntime/core/providers/tensorrt/ \ -L /onnxruntime/build/Linux/Release/ \ - test_skelda_dataset_cpp.cpp \ + test_skelda_dataset.cpp \ /RapidPoseTriangulation/rpt/*.cpp \ -o test_skelda_dataset.bin \ -Wl,--start-group \ diff --git a/extras/ros/docker-compose.yml b/extras/ros/docker-compose.yml index 491c6b2..0e239d5 100644 --- a/extras/ros/docker-compose.yml +++ b/extras/ros/docker-compose.yml @@ -35,7 +35,6 @@ services: - DISPLAY - QT_X11_NO_MITSHM=1 - "PYTHONUNBUFFERED=1" - # command: /bin/bash -i -c 'export ROS_DOMAIN_ID=18 && ros2 run rpt2D_wrapper_py rpt2D_wrapper' command: /bin/bash -i -c 'export ROS_DOMAIN_ID=18 && ros2 run rpt2D_wrapper_cpp rpt2D_wrapper' pose_visualizer: diff --git a/extras/ros/dockerfile b/extras/ros/dockerfile index ec65f90..169e0a2 100644 --- a/extras/ros/dockerfile +++ b/extras/ros/dockerfile @@ -63,12 +63,10 @@ RUN pip3 install --no-cache-dir "setuptools<=58.2.0" COPY ./extras/include/ /RapidPoseTriangulation/extras/include/ COPY ./scripts/ /RapidPoseTriangulation/scripts/ COPY ./extras/ros/pose2D_visualizer/ /RapidPoseTriangulation/extras/ros/pose2D_visualizer/ -COPY ./extras/ros/rpt2D_wrapper_py/ /RapidPoseTriangulation/extras/ros/rpt2D_wrapper_py/ COPY ./extras/ros/rpt2D_wrapper_cpp/ /RapidPoseTriangulation/extras/ros/rpt2D_wrapper_cpp/ # Link and build as ros package RUN ln -s /RapidPoseTriangulation/extras/ros/pose2D_visualizer/ /project/dev_ws/src/pose2D_visualizer -RUN ln -s /RapidPoseTriangulation/extras/ros/rpt2D_wrapper_py/ /project/dev_ws/src/rpt2D_wrapper_py RUN ln -s /RapidPoseTriangulation/extras/ros/rpt2D_wrapper_cpp/ /project/dev_ws/src/rpt2D_wrapper_cpp RUN /bin/bash -i -c 'cd /project/dev_ws/; colcon build --symlink-install --cmake-args -DCMAKE_BUILD_TYPE=Release' diff --git a/extras/ros/rpt2D_wrapper_py/package.xml b/extras/ros/rpt2D_wrapper_py/package.xml deleted file mode 100644 index 178e790..0000000 --- a/extras/ros/rpt2D_wrapper_py/package.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - rpt2D_wrapper_py - 0.0.0 - TODO: Package description - root - TODO: License declaration - - ament_copyright - ament_flake8 - ament_pep257 - python3-pytest - - - ament_python - - diff --git a/extras/ros/rpt2D_wrapper_py/resource/rpt2D_wrapper_py b/extras/ros/rpt2D_wrapper_py/resource/rpt2D_wrapper_py deleted file mode 100644 index e69de29..0000000 diff --git a/extras/ros/rpt2D_wrapper_py/rpt2D_wrapper_py/__init__.py b/extras/ros/rpt2D_wrapper_py/rpt2D_wrapper_py/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/extras/ros/rpt2D_wrapper_py/rpt2D_wrapper_py/rpt2D_wrapper.py b/extras/ros/rpt2D_wrapper_py/rpt2D_wrapper_py/rpt2D_wrapper.py deleted file mode 100644 index dfd7cdb..0000000 --- a/extras/ros/rpt2D_wrapper_py/rpt2D_wrapper_py/rpt2D_wrapper.py +++ /dev/null @@ -1,196 +0,0 @@ -import json -import os -import sys -import threading -import time - -import numpy as np -import rclpy -from cv_bridge import CvBridge -from rclpy.qos import QoSHistoryPolicy, QoSProfile, QoSReliabilityPolicy -from sensor_msgs.msg import Image -from std_msgs.msg import String - -filepath = os.path.dirname(os.path.realpath(__file__)) + "/" -sys.path.append(filepath + "../../../scripts/") -import test_triangulate -import utils_2d_pose - -# ================================================================================================== - -bridge = CvBridge() -node = None -publisher_pose = None - -cam_id = "camera01" -img_input_topic = "/" + cam_id + "/pylon_ros2_camera_node/image_raw" -pose_out_topic = "/poses/" + cam_id - -last_input_image = None -last_input_time = 0.0 -kpt_model = None -joint_names_2d = test_triangulate.joint_names_2d - -lock = threading.Lock() -stop_flag = False - -# Model config -min_bbox_score = 0.4 -min_bbox_area = 0.1 * 0.1 -batch_poses = True - - -# ================================================================================================== - - -def callback_images(image_data): - global last_input_image, last_input_time, lock - - # Convert into cv images from image string - if image_data.encoding == "bayer_rggb8": - bayer_image = bridge.imgmsg_to_cv2(image_data, "bayer_rggb8") - elif image_data.encoding == "mono8": - bayer_image = bridge.imgmsg_to_cv2(image_data, "mono8") - elif image_data.encoding == "rgb8": - color_image = bridge.imgmsg_to_cv2(image_data, "rgb8") - bayer_image = test_triangulate.rgb2bayer(color_image) - else: - raise ValueError("Unknown image encoding:", image_data.encoding) - - time_stamp = image_data.header.stamp.sec + image_data.header.stamp.nanosec / 1.0e9 - - with lock: - last_input_image = bayer_image - last_input_time = time_stamp - - -# ================================================================================================== - - -def callback_model(): - global last_input_image, last_input_time, kpt_model, lock - - ptime = time.time() - if last_input_time == 0.0: - time.sleep(0.0001) - return - - # Collect inputs - images_2d = [] - timestamps = [] - with lock: - img = last_input_image - ts = last_input_time - images_2d.append(img) - timestamps.append(ts) - last_input_image = None - last_input_time = 0.0 - - # Predict 2D poses - images_2d = [test_triangulate.bayer2rgb(img) for img in images_2d] - poses_2d = utils_2d_pose.get_2d_pose(kpt_model, images_2d) - poses_2d = test_triangulate.update_keypoints(poses_2d, joint_names_2d) - poses_2d = poses_2d[0] - - # Drop persons with no joints - poses_2d = np.asarray(poses_2d) - mask = np.sum(poses_2d[..., 2], axis=1) > 0 - poses_2d = poses_2d[mask] - - # Round poses - poses2D = [np.array(p).round(3).tolist() for p in poses_2d] - - # Publish poses - ts_pose = time.time() - poses = { - "bodies2D": poses2D, - "joints": joint_names_2d, - "num_persons": len(poses2D), - "timestamps": { - "image": timestamps[0], - "pose": ts_pose, - "z-images-pose": ts_pose - timestamps[0], - }, - } - publish(poses) - - msg = "Detected persons: {} - Prediction time: {:.4f}s" - print(msg.format(poses["num_persons"], time.time() - ptime)) - - -# ================================================================================================== - - -def callback_wrapper(): - global stop_flag - while not stop_flag: - callback_model() - time.sleep(0.001) - - -# ================================================================================================== - - -def publish(data): - # Publish json data - msg = String() - msg.data = json.dumps(data) - publisher_pose.publish(msg) - - -# ================================================================================================== - - -def main(): - global node, publisher_pose, kpt_model, stop_flag - - # Start node - rclpy.init(args=sys.argv) - node = rclpy.create_node("rpt2D_wrapper") - - # Quality of service settings - qos_profile = QoSProfile( - reliability=QoSReliabilityPolicy.RELIABLE, - history=QoSHistoryPolicy.KEEP_LAST, - depth=1, - ) - - # Create subscribers - _ = node.create_subscription( - Image, - img_input_topic, - callback_images, - qos_profile, - ) - - # Create publishers - publisher_pose = node.create_publisher(String, pose_out_topic, qos_profile) - - # Load 2D pose model - whole_body = test_triangulate.whole_body - if any((whole_body[k] for k in whole_body)): - kpt_model = utils_2d_pose.load_wb_model( - min_bbox_score, min_bbox_area, batch_poses - ) - else: - kpt_model = utils_2d_pose.load_model(min_bbox_score, min_bbox_area, batch_poses) - - node.get_logger().info("Finished initialization of pose estimator") - - # Start prediction thread - p1 = threading.Thread(target=callback_wrapper) - p1.start() - - # Run ros update thread - rclpy.spin(node) - - stop_flag = True - p1.join() - node.destroy_node() - rclpy.shutdown() - - -# ================================================================================================== - -if __name__ == "__main__": - main() diff --git a/extras/ros/rpt2D_wrapper_py/setup.cfg b/extras/ros/rpt2D_wrapper_py/setup.cfg deleted file mode 100644 index 80d9b5d..0000000 --- a/extras/ros/rpt2D_wrapper_py/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[develop] -script_dir=$base/lib/rpt2D_wrapper_py -[install] -install_scripts=$base/lib/rpt2D_wrapper_py diff --git a/extras/ros/rpt2D_wrapper_py/setup.py b/extras/ros/rpt2D_wrapper_py/setup.py deleted file mode 100644 index cb32e53..0000000 --- a/extras/ros/rpt2D_wrapper_py/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -from setuptools import setup - -package_name = "rpt2D_wrapper_py" - -setup( - name=package_name, - version="0.0.0", - packages=[package_name], - data_files=[ - ("share/ament_index/resource_index/packages", ["resource/" + package_name]), - ("share/" + package_name, ["package.xml"]), - ], - install_requires=["setuptools"], - zip_safe=True, - maintainer="root", - maintainer_email="root@todo.todo", - description="TODO: Package description", - license="TODO: License declaration", - tests_require=["pytest"], - entry_points={ - "console_scripts": ["rpt2D_wrapper = rpt2D_wrapper_py.rpt2D_wrapper:main"], - }, -) diff --git a/extras/ros/rpt2D_wrapper_py/test/test_copyright.py b/extras/ros/rpt2D_wrapper_py/test/test_copyright.py deleted file mode 100644 index 8f18fa4..0000000 --- a/extras/ros/rpt2D_wrapper_py/test/test_copyright.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2015 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from ament_copyright.main import main - - -# Remove the `skip` decorator once the source file(s) have a copyright header -@pytest.mark.skip( - reason="No copyright header has been placed in the generated source file." -) -@pytest.mark.copyright -@pytest.mark.linter -def test_copyright(): - rc = main(argv=[".", "test"]) - assert rc == 0, "Found errors" diff --git a/extras/ros/rpt2D_wrapper_py/test/test_flake8.py b/extras/ros/rpt2D_wrapper_py/test/test_flake8.py deleted file mode 100644 index f494570..0000000 --- a/extras/ros/rpt2D_wrapper_py/test/test_flake8.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2017 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from ament_flake8.main import main_with_errors - - -@pytest.mark.flake8 -@pytest.mark.linter -def test_flake8(): - rc, errors = main_with_errors(argv=[]) - assert rc == 0, "Found %d code style errors / warnings:\n" % len( - errors - ) + "\n".join(errors) diff --git a/extras/ros/rpt2D_wrapper_py/test/test_pep257.py b/extras/ros/rpt2D_wrapper_py/test/test_pep257.py deleted file mode 100644 index 4eddb46..0000000 --- a/extras/ros/rpt2D_wrapper_py/test/test_pep257.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2015 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from ament_pep257.main import main - - -@pytest.mark.linter -@pytest.mark.pep257 -def test_pep257(): - rc = main(argv=[".", "test"]) - assert rc == 0, "Found code style errors / warnings" diff --git a/scripts/test_skelda_dataset_cpp.cpp b/scripts/test_skelda_dataset.cpp similarity index 100% rename from scripts/test_skelda_dataset_cpp.cpp rename to scripts/test_skelda_dataset.cpp diff --git a/scripts/test_skelda_dataset_cpp.py b/scripts/test_skelda_dataset.py similarity index 100% rename from scripts/test_skelda_dataset_cpp.py rename to scripts/test_skelda_dataset.py diff --git a/scripts/test_skelda_dataset_py.py b/scripts/test_skelda_dataset_py.py deleted file mode 100644 index 501d1a2..0000000 --- a/scripts/test_skelda_dataset_py.py +++ /dev/null @@ -1,489 +0,0 @@ -import json -import os -import sys -import time - -import cv2 -import matplotlib -import numpy as np -import tqdm - -import utils_2d_pose -import utils_pipeline -from skelda import evals - -sys.path.append("/RapidPoseTriangulation/swig/") -import rpt - -# ================================================================================================== - -whole_body = { - "foots": False, - "face": False, - "hands": False, -} - -dataset_use = "human36m" -# dataset_use = "panoptic" -# dataset_use = "mvor" -# dataset_use = "shelf" -# dataset_use = "campus" -# dataset_use = "ikeaasm" -# dataset_use = "chi3d" -# dataset_use = "tsinghua" -# dataset_use = "human36m_wb" -# dataset_use = "egohumans_tagging" -# dataset_use = "egohumans_legoassemble" -# dataset_use = "egohumans_fencing" -# dataset_use = "egohumans_basketball" -# dataset_use = "egohumans_volleyball" -# dataset_use = "egohumans_badminton" -# dataset_use = "egohumans_tennis" -# dataset_use = "ntu" -# dataset_use = "koarob" - - -# Describes the minimum area as fraction of the image size for a 2D bounding box to be considered -# If the persons are small in the image, use a lower value -default_min_bbox_area = 0.1 * 0.1 - -# Describes how confident a 2D bounding box needs to be to be considered -# If the persons are small in the image, or poorly recognizable, use a lower value -default_min_bbox_score = 0.3 - -# Describes how good two 2D poses need to match each other to create a valid triangulation -# If the quality of the 2D detections is poor, use a lower value -default_min_match_score = 0.94 - -# Describes the minimum number of camera pairs that need to detect the same person -# If the number of cameras is high, and the views are not occluded, use a higher value -default_min_group_size = 1 - -# Batch poses per image for faster processing -# If most of the time only one person is in a image, disable it, because it is slightly slower then -default_batch_poses = True - -datasets = { - "human36m": { - "path": "/datasets/human36m/skelda/pose_test.json", - "take_interval": 5, - "min_match_score": 0.95, - "min_group_size": 1, - "min_bbox_score": 0.4, - "min_bbox_area": 0.1 * 0.1, - "batch_poses": False, - }, - "panoptic": { - "path": "/datasets/panoptic/skelda/test.json", - "cams": ["00_03", "00_06", "00_12", "00_13", "00_23"], - # "cams": ["00_03", "00_06", "00_12"], - # "cams": ["00_03", "00_06", "00_12", "00_13", "00_23", "00_15", "00_10", "00_21", "00_09", "00_01"], - "take_interval": 3, - "min_match_score": 0.95, - "use_scenes": ["160906_pizza1", "160422_haggling1", "160906_ian5"], - "min_group_size": 1, - # "min_group_size": 4, - "min_bbox_area": 0.05 * 0.05, - }, - "mvor": { - "path": "/datasets/mvor/skelda/all.json", - "take_interval": 1, - "with_depth": False, - "min_match_score": 0.85, - "min_bbox_score": 0.25, - }, - "campus": { - "path": "/datasets/campus/skelda/test.json", - "take_interval": 1, - "min_match_score": 0.90, - "min_bbox_score": 0.5, - }, - "shelf": { - "path": "/datasets/shelf/skelda/test.json", - "take_interval": 1, - "min_match_score": 0.96, - "min_group_size": 2, - }, - "ikeaasm": { - "path": "/datasets/ikeaasm/skelda/test.json", - "take_interval": 2, - "min_match_score": 0.92, - "min_bbox_score": 0.20, - }, - "chi3d": { - "path": "/datasets/chi3d/skelda/all.json", - "take_interval": 5, - }, - "tsinghua": { - "path": "/datasets/tsinghua/skelda/test.json", - "take_interval": 3, - "min_match_score": 0.95, - "min_group_size": 2, - }, - "human36m_wb": { - "path": "/datasets/human36m/skelda/wb/test.json", - "take_interval": 100, - "min_bbox_score": 0.4, - "batch_poses": False, - }, - "egohumans_tagging": { - "path": "/datasets/egohumans/skelda/all.json", - "take_interval": 2, - "subset": "tagging", - "min_group_size": 2, - "min_bbox_score": 0.2, - "min_bbox_area": 0.05 * 0.05, - }, - "egohumans_legoassemble": { - "path": "/datasets/egohumans/skelda/all.json", - "take_interval": 2, - "subset": "legoassemble", - "min_group_size": 2, - }, - "egohumans_fencing": { - "path": "/datasets/egohumans/skelda/all.json", - "take_interval": 2, - "subset": "fencing", - "min_group_size": 7, - "min_bbox_score": 0.5, - "min_bbox_area": 0.05 * 0.05, - }, - "egohumans_basketball": { - "path": "/datasets/egohumans/skelda/all.json", - "take_interval": 2, - "subset": "basketball", - "min_group_size": 7, - "min_bbox_score": 0.25, - "min_bbox_area": 0.025 * 0.025, - }, - "egohumans_volleyball": { - "path": "/datasets/egohumans/skelda/all.json", - "take_interval": 2, - "subset": "volleyball", - "min_group_size": 11, - "min_bbox_score": 0.25, - "min_bbox_area": 0.05 * 0.05, - }, - "egohumans_badminton": { - "path": "/datasets/egohumans/skelda/all.json", - "take_interval": 2, - "subset": "badminton", - "min_group_size": 7, - "min_bbox_score": 0.25, - "min_bbox_area": 0.05 * 0.05, - }, - "egohumans_tennis": { - "path": "/datasets/egohumans/skelda/all.json", - "take_interval": 2, - "subset": "tennis", - "min_group_size": 11, - "min_bbox_area": 0.025 * 0.025, - }, -} - -joint_names_2d = utils_pipeline.get_joint_names(whole_body) -joint_names_3d = list(joint_names_2d) -eval_joints = [ - "head", - "shoulder_left", - "shoulder_right", - "elbow_left", - "elbow_right", - "wrist_left", - "wrist_right", - "hip_left", - "hip_right", - "knee_left", - "knee_right", - "ankle_left", - "ankle_right", -] -if dataset_use == "human36m": - eval_joints[eval_joints.index("head")] = "nose" -if dataset_use == "panoptic": - eval_joints[eval_joints.index("head")] = "nose" -if dataset_use == "human36m_wb": - if utils_pipeline.use_whole_body(whole_body): - eval_joints = list(joint_names_2d) - else: - eval_joints[eval_joints.index("head")] = "nose" - -# output_dir = "/RapidPoseTriangulation/data/testoutput/" -output_dir = "" - -# ================================================================================================== - - -def load_json(path: str): - with open(path, "r", encoding="utf-8") as file: - data = json.load(file) - return data - - -# ================================================================================================== - - -def load_labels(dataset: dict): - """Load labels by dataset description""" - - if "panoptic" in dataset: - labels = load_json(dataset["panoptic"]["path"]) - labels = [lb for i, lb in enumerate(labels) if i % 1500 < 90] - - # Filter by maximum number of persons - labels = [l for l in labels if len(l["bodies3D"]) <= 10] - - # Filter scenes - if "use_scenes" in dataset["panoptic"]: - labels = [ - l for l in labels if l["scene"] in dataset["panoptic"]["use_scenes"] - ] - - # Filter cameras - if not "cameras_depth" in labels[0]: - for label in labels: - for i, cam in reversed(list(enumerate(label["cameras"]))): - if cam["name"] not in dataset["panoptic"]["cams"]: - label["cameras"].pop(i) - label["imgpaths"].pop(i) - - elif "human36m" in dataset: - labels = load_json(dataset["human36m"]["path"]) - labels = [lb for lb in labels if lb["subject"] == "S9"] - labels = [lb for i, lb in enumerate(labels) if i % 4000 < 150] - - for label in labels: - label.pop("action") - label.pop("frame") - - elif "mvor" in dataset: - labels = load_json(dataset["mvor"]["path"]) - - # Rename keys - for label in labels: - label["cameras_color"] = label["cameras"] - label["imgpaths_color"] = label["imgpaths"] - - elif "ikeaasm" in dataset: - labels = load_json(dataset["ikeaasm"]["path"]) - cams0 = str(labels[0]["cameras"]) - labels = [lb for lb in labels if str(lb["cameras"]) == cams0] - - elif "shelf" in dataset: - labels = load_json(dataset["shelf"]["path"]) - labels = [lb for lb in labels if "test" in lb["splits"]] - - elif "campus" in dataset: - labels = load_json(dataset["campus"]["path"]) - labels = [lb for lb in labels if "test" in lb["splits"]] - - elif "tsinghua" in dataset: - labels = load_json(dataset["tsinghua"]["path"]) - labels = [lb for lb in labels if "test" in lb["splits"]] - labels = [lb for lb in labels if lb["seq"] == "seq_1"] - labels = [lb for i, lb in enumerate(labels) if i % 300 < 90] - - for label in labels: - label["bodyids"] = list(range(len(label["bodies3D"]))) - - elif "chi3d" in dataset: - labels = load_json(dataset["chi3d"]["path"]) - labels = [lb for lb in labels if lb["setup"] == "s03"] - labels = [lb for i, lb in enumerate(labels) if i % 2000 < 150] - - elif "human36m_wb" in dataset: - labels = load_json(dataset["human36m_wb"]["path"]) - - elif any(("egohumans" in key for key in dataset)): - labels = load_json(dataset[dataset_use]["path"]) - labels = [lb for lb in labels if "test" in lb["splits"]] - labels = [lb for lb in labels if dataset[dataset_use]["subset"] in lb["seq"]] - if dataset[dataset_use]["subset"] in ["volleyball", "tennis"]: - labels = [lb for i, lb in enumerate(labels) if i % 150 < 60] - - else: - raise ValueError("Dataset not available") - - # Optionally drop samples to speed up train/eval - if "take_interval" in dataset: - take_interval = dataset["take_interval"] - if take_interval > 1: - labels = [l for i, l in enumerate(labels) if i % take_interval == 0] - - return labels - - -# ================================================================================================== - - -def main(): - global joint_names_3d, eval_joints - - # Load dataset specific parameters - min_match_score = datasets[dataset_use].get( - "min_match_score", default_min_match_score - ) - min_group_size = datasets[dataset_use].get("min_group_size", default_min_group_size) - min_bbox_score = datasets[dataset_use].get("min_bbox_score", default_min_bbox_score) - min_bbox_area = datasets[dataset_use].get("min_bbox_area", default_min_bbox_area) - batch_poses = datasets[dataset_use].get("batch_poses", default_batch_poses) - - # Load 2D pose model - if utils_pipeline.use_whole_body(whole_body): - kpt_model = utils_2d_pose.load_wb_model( - min_bbox_score, min_bbox_area, batch_poses - ) - else: - kpt_model = utils_2d_pose.load_model(min_bbox_score, min_bbox_area, batch_poses) - - # Manually set matplotlib backend - try: - matplotlib.use("TkAgg") - except ImportError: - print("WARNING: Using headless mode, no visualizations will be shown.") - - print("Loading dataset ...") - labels = load_labels( - { - dataset_use: datasets[dataset_use], - "take_interval": datasets[dataset_use]["take_interval"], - } - ) - - # Print a dataset sample for debugging - print(labels[0]) - - print("\nPrefetching images ...") - for label in tqdm.tqdm(labels): - # If the images are stored on a HDD, it sometimes takes a while to load them - # Prefetching them results in more stable timings of the following steps - # To prevent memory overflow, the code only loads the images, but does not store them - try: - for i in range(len(label["imgpaths"])): - imgpath = label["imgpaths"][i] - img = utils_pipeline.load_image(imgpath) - except cv2.error: - print("One of the paths not found:", label["imgpaths"]) - continue - time.sleep(3) - - print("\nCalculating 2D predictions ...") - all_poses_2d = [] - times = [] - for label in tqdm.tqdm(labels): - images_2d = [] - - start = time.time() - try: - for i in range(len(label["imgpaths"])): - imgpath = label["imgpaths"][i] - img = utils_pipeline.load_image(imgpath) - images_2d.append(img) - except cv2.error: - print("One of the paths not found:", label["imgpaths"]) - continue - - if dataset_use == "human36m": - for i in range(len(images_2d)): - # Since the images don't have the same shape, rescale some of them - img = images_2d[i] - ishape = img.shape - if ishape != (1000, 1000, 3): - cam = label["cameras"][i] - cam["K"][1][1] = cam["K"][1][1] * (1000 / ishape[0]) - cam["K"][1][2] = cam["K"][1][2] * (1000 / ishape[0]) - cam["K"][0][0] = cam["K"][0][0] * (1000 / ishape[1]) - cam["K"][0][2] = cam["K"][0][2] * (1000 / ishape[1]) - images_2d[i] = cv2.resize(img, (1000, 1000)) - - # Convert image format to Bayer encoding to simulate real camera input - # This also resulted in notably better MPJPE results in most cases, presumbly since the - # demosaicing algorithm from OpenCV is better than the default one from the cameras - for i in range(len(images_2d)): - images_2d[i] = utils_pipeline.rgb2bayer(images_2d[i]) - time_imgs = time.time() - start - - start = time.time() - for i in range(len(images_2d)): - images_2d[i] = utils_pipeline.bayer2rgb(images_2d[i]) - poses_2d = utils_2d_pose.get_2d_pose(kpt_model, images_2d) - poses_2d = utils_pipeline.update_keypoints(poses_2d, joint_names_2d, whole_body) - time_2d = time.time() - start - - all_poses_2d.append(poses_2d) - times.append([time_imgs, time_2d, 0]) - - print("\nCalculating 3D predictions ...") - all_poses_3d = [] - all_ids = [] - triangulator = rpt.Triangulator( - min_match_score=min_match_score, min_group_size=min_group_size - ) - old_scene = "" - old_index = -1 - for i in tqdm.tqdm(range(len(labels))): - label = labels[i] - poses_2d = all_poses_2d[i] - - if old_scene != label.get("scene", "") or ( - old_index + datasets[dataset_use]["take_interval"] < label["index"] - ): - # Reset last poses if scene changes - old_scene = label.get("scene", "") - triangulator.reset() - - start = time.time() - if sum(np.sum(p) for p in poses_2d) == 0: - poses3D = np.zeros([1, len(joint_names_3d), 4]).tolist() - else: - rpt_cameras = rpt.convert_cameras(label["cameras"]) - roomparams = [label["room_size"], label["room_center"]] - poses3D = triangulator.triangulate_poses( - poses_2d, rpt_cameras, roomparams, joint_names_2d - ) - time_3d = time.time() - start - - old_index = label["index"] - all_poses_3d.append(np.array(poses3D).tolist()) - all_ids.append(label["id"]) - times[i][2] = time_3d - - # Print per-step timings - warmup_iters = 10 - if len(times) > warmup_iters: - times = times[warmup_iters:] - avg_time_im = np.mean([t[0] for t in times]) - avg_time_2d = np.mean([t[1] for t in times]) - avg_time_3d = np.mean([t[2] for t in times]) - tstats = { - "img_loading": avg_time_im, - "avg_time_2d": avg_time_2d, - "avg_time_3d": avg_time_3d, - "avg_fps": 1.0 / (avg_time_2d + avg_time_3d), - } - print("\nMetrics:") - print(json.dumps(tstats, indent=2)) - triangulator.print_stats() - - _ = evals.mpjpe.run_eval( - labels, - all_poses_3d, - all_ids, - joint_names_net=joint_names_3d, - joint_names_use=eval_joints, - save_error_imgs=output_dir, - ) - _ = evals.pcp.run_eval( - labels, - all_poses_3d, - all_ids, - joint_names_net=joint_names_3d, - joint_names_use=eval_joints, - replace_head_with_nose=True, - ) - - -# ================================================================================================== - -if __name__ == "__main__": - main()