diff --git a/media/RESULTS.md b/media/RESULTS.md index a35d6af..9219abc 100644 --- a/media/RESULTS.md +++ b/media/RESULTS.md @@ -278,9 +278,9 @@ Results of the model in various experiments on different datasets. (duration 00:00:56) ```json { - "avg_time_2d": 0.0996503477653687, - "avg_time_3d": 0.003785594632125802, - "avg_fps": 9.667819297832613 + "avg_time_2d": 0.10053871915102824, + "avg_time_3d": 0.0020945760392651115, + "avg_fps": 9.743426810431163 } { "person_nums": { diff --git a/spt/triangulator.cpp b/spt/triangulator.cpp index 77d062b..82a9fe4 100644 --- a/spt/triangulator.cpp +++ b/spt/triangulator.cpp @@ -1,5 +1,6 @@ #include +#include #include #include "camera.hpp" @@ -180,6 +181,7 @@ std::vector>> TriangulatorInternal::triangulat } // Undistort 2D poses + #pragma omp parallel for for (size_t i = 0; i < cameras.size(); ++i) { undistort_poses(poses_2d_mats[i], internal_cameras[i]); @@ -206,11 +208,12 @@ std::vector>> TriangulatorInternal::triangulat } // Project - last_poses_2d.reserve(cameras.size()); + last_poses_2d.resize(cameras.size()); + #pragma omp parallel for for (size_t i = 0; i < cameras.size(); ++i) { auto [poses2d, dists] = project_poses(last_core_poses, internal_cameras[i], true); - last_poses_2d.emplace_back(poses2d, dists); + last_poses_2d[i] = std::make_tuple(poses2d, dists); } } @@ -219,38 +222,51 @@ std::vector>> TriangulatorInternal::triangulat std::map>> scored_pasts; if (!last_poses_3d.empty()) { + // Calculate index pairs and initialize vectors + std::vector> indices; for (size_t i = 0; i < cameras.size(); ++i) { + size_t num_persons = std::get<0>(last_poses_2d[i]).size(); scored_pasts[i] = std::map>(); + + for (size_t j = 0; j < num_persons; ++j) + { + scored_pasts[i][j] = std::vector(); + for (size_t k = 0; k < poses_2d_mats[i].size(); ++k) + { + indices.push_back({i, j, k}); + } + } + } + + #pragma omp parallel for ordered schedule(dynamic) + for (size_t e = 0; e < indices.size(); ++e) + { + const auto &i = indices[e][0]; + const auto &j = indices[e][1]; + const auto &k = indices[e][2]; + const std::vector &poses = poses_2d_mats[i]; const std::vector &last_poses = std::get<0>(last_poses_2d[i]); const std::vector &dists = std::get<1>(last_poses_2d[i]); - for (size_t j = 0; j < last_poses.size(); ++j) + // Select core joints + const cv::Mat &last_pose = last_poses[j]; + const cv::Mat &pose = poses[k]; + std::vector dims = {(int)core_joint_idx.size(), 3}; + cv::Mat pose_core(dims, pose.type()); + for (size_t l = 0; l < core_joint_idx.size(); ++l) { - scored_pasts[i][j] = std::vector(); - const cv::Mat &last_pose = last_poses[j]; + size_t idx = core_joint_idx[l]; + pose.row(idx).copyTo(pose_core.row(l)); + } - for (size_t k = 0; k < poses.size(); ++k) - { - // Select core joints - const cv::Mat &pose = poses[k]; - std::vector dims = {(int)core_joint_idx.size(), 3}; - cv::Mat pose_core(dims, pose.type()); - for (size_t l = 0; l < core_joint_idx.size(); ++l) - { - size_t idx = core_joint_idx[l]; - pose.row(idx).copyTo(pose_core.row(l)); - } - - // Calculate score - double score = calc_pose_score( - pose_core, last_pose, dists[j], internal_cameras[i]); - if (score > threshold) - { - scored_pasts[i][j].push_back(k); - } - } + // Calculate score + double score = calc_pose_score(pose_core, last_pose, dists[j], internal_cameras[i]); + if (score > threshold) + { + #pragma omp ordered + scored_pasts[i][j].push_back(k); } } } @@ -264,6 +280,7 @@ std::vector>> TriangulatorInternal::triangulat num_persons.push_back(poses_2d[i].size()); } std::vector, std::pair>> all_pairs; + std::vector> indices; for (size_t i = 0; i < cameras.size(); ++i) { for (size_t j = i + 1; j < cameras.size(); ++j) @@ -272,44 +289,54 @@ std::vector>> TriangulatorInternal::triangulat { for (size_t l = 0; l < poses_2d[j].size(); ++l) { - int pid1 = std::accumulate(num_persons.begin(), num_persons.begin() + i, 0) + k; - int pid2 = std::accumulate(num_persons.begin(), num_persons.begin() + j, 0) + l; - bool match = false; - - if (!last_poses_3d.empty()) - { - for (size_t m = 0; m < last_poses_3d.size(); ++m) - { - auto &smi = scored_pasts[i][m]; - auto &smj = scored_pasts[j][m]; - if (std::find(smi.begin(), smi.end(), k) != smi.end() && - std::find(smj.begin(), smj.end(), l) != smj.end()) - { - match = true; - all_pairs.emplace_back( - std::make_tuple(i, j, k, l), std::make_pair(pid1, pid2)); - } - else if (std::find(smi.begin(), smi.end(), k) != smi.end() || - std::find(smj.begin(), smj.end(), l) != smj.end()) - { - match = true; - } - } - } - - if (!match) - { - all_pairs.emplace_back( - std::make_tuple(i, j, k, l), std::make_pair(pid1, pid2)); - } + indices.push_back({i, j, k, l}); } } } } + #pragma omp parallel for ordered schedule(dynamic) + for (size_t e = 0; e < indices.size(); ++e) + { + const auto [i, j, k, l] = indices[e]; + + int pid1 = std::accumulate(num_persons.begin(), num_persons.begin() + i, 0) + k; + int pid2 = std::accumulate(num_persons.begin(), num_persons.begin() + j, 0) + l; + bool match = false; + + if (!last_poses_3d.empty()) + { + for (size_t m = 0; m < last_poses_3d.size(); ++m) + { + auto &smi = scored_pasts[i][m]; + auto &smj = scored_pasts[j][m]; + if (std::find(smi.begin(), smi.end(), k) != smi.end() && + std::find(smj.begin(), smj.end(), l) != smj.end()) + { + match = true; + #pragma omp ordered + all_pairs.emplace_back( + std::make_tuple(i, j, k, l), std::make_pair(pid1, pid2)); + } + else if (std::find(smi.begin(), smi.end(), k) != smi.end() || + std::find(smj.begin(), smj.end(), l) != smj.end()) + { + match = true; + } + } + } + + if (!match) + { + #pragma omp ordered + all_pairs.emplace_back( + std::make_tuple(i, j, k, l), std::make_pair(pid1, pid2)); + } + } // Calculate pair scores std::vector> all_scored_poses; - all_scored_poses.reserve(all_pairs.size()); + all_scored_poses.resize(all_pairs.size()); + #pragma omp parallel for for (size_t i = 0; i < all_pairs.size(); ++i) { const auto &pids = all_pairs[i].first; @@ -336,7 +363,7 @@ std::vector>> TriangulatorInternal::triangulat // Triangulate and score auto [pose3d, score] = triangulate_and_score( pose1_core, pose2_core, cam1, cam2, roomparams, core_limbs_idx); - all_scored_poses.emplace_back(pose3d, score); + all_scored_poses[i] = std::make_pair(pose3d, score); } // Drop low scoring poses @@ -363,7 +390,8 @@ std::vector>> TriangulatorInternal::triangulat // Calculate full 3D poses std::vector all_full_poses; - all_full_poses.reserve(all_pairs.size()); + all_full_poses.resize(all_pairs.size()); + #pragma omp parallel for for (size_t i = 0; i < all_pairs.size(); ++i) { const auto &pids = all_pairs[i].first; @@ -372,13 +400,15 @@ std::vector>> TriangulatorInternal::triangulat const auto &pose1 = poses_2d_mats[std::get<0>(pids)][std::get<2>(pids)]; const auto &pose2 = poses_2d_mats[std::get<1>(pids)][std::get<3>(pids)]; - auto [pose3d, score] = triangulate_and_score(pose1, pose2, cam1, cam2, roomparams, {}); - all_full_poses.push_back(pose3d); + auto [pose3d, score] = triangulate_and_score( + pose1, pose2, cam1, cam2, roomparams, {}); + all_full_poses[i] = (pose3d); } // Merge groups std::vector all_merged_poses; - all_merged_poses.reserve(groups.size()); + all_merged_poses.resize(groups.size()); + #pragma omp parallel for for (size_t i = 0; i < groups.size(); ++i) { const auto &group = groups[i]; @@ -389,7 +419,7 @@ std::vector>> TriangulatorInternal::triangulat } auto merged_pose = merge_group(poses, min_score); - all_merged_poses.push_back(merged_pose); + all_merged_poses[i] = (merged_pose); } // Convert to output format diff --git a/swig/Makefile b/swig/Makefile index 8ac0b95..add0b44 100644 --- a/swig/Makefile +++ b/swig/Makefile @@ -7,7 +7,7 @@ PYTHONL = -Xlinker -export-dynamic # Default super-target all: - cd ../spt/ && g++ -fPIC -std=c++2a -Wall -I/usr/include/opencv4 -c *.cpp ; cd ../swig/ + cd ../spt/ && g++ -fPIC -std=c++2a -Wall -fopenmp -I/usr/include/opencv4 -c *.cpp ; cd ../swig/ swig -c++ -python -keyword -o spt_wrap.cxx spt.i - g++ $(FLAGS) $(PYTHONI) -I/usr/include/opencv4 -c spt_wrap.cxx -o spt_wrap.o - g++ $(PYTHONL) $(LIBFLAGS) -shared ../spt/*.o spt_wrap.o -lopencv_core -lopencv_imgproc -lopencv_calib3d -o _spt.so + g++ $(FLAGS) $(PYTHONI) -fopenmp -I/usr/include/opencv4 -c spt_wrap.cxx -o spt_wrap.o + g++ $(PYTHONL) $(LIBFLAGS) -fopenmp -shared ../spt/*.o spt_wrap.o -lopencv_core -lopencv_imgproc -lopencv_calib3d -o _spt.so