From b80ad139e5d0d798c7c22f070c984b667ef2ee5a Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 10 Oct 2024 12:46:11 +0200 Subject: [PATCH] Improved group merging. --- media/RESULTS.md | 278 +++++++++++++++++++++---------------------- spt/triangulator.cpp | 126 ++++++++++++++++++-- 2 files changed, 257 insertions(+), 147 deletions(-) diff --git a/media/RESULTS.md b/media/RESULTS.md index 654bbdc..22f1eda 100644 --- a/media/RESULTS.md +++ b/media/RESULTS.md @@ -5998,58 +5998,58 @@ Results of the model in various experiments on different datasets. (volleyball) ```json { - "avg_time_2d": 0.3441050417788394, - "avg_time_3d": 0.0033576402578267966, - "avg_fps": 2.878006910378003 + "avg_time_2d": 0.3424515831577885, + "avg_time_3d": 0.003363656568097639, + "avg_fps": 2.8917175564404274 } { "person_nums": { "total_frames": 121, "total_labels": 484, - "total_preds": 633, + "total_preds": 634, "considered_empty": 0, "valid_preds": 483, - "invalid_preds": 150, + "invalid_preds": 151, "missing": 1, - "invalid_fraction": 0.23697, - "precision": 0.76303, + "invalid_fraction": 0.23817, + "precision": 0.76183, "recall": 0.99793, - "f1": 0.86482, - "non_empty": 633 + "f1": 0.86404, + "non_empty": 634 }, "mpjpe": { "count": 483, - "mean": 0.035428, - "median": 0.032584, - "std": 0.013121, - "sem": 0.000598, - "min": 0.017908, - "max": 0.12169, - "recall-0.025": 0.216942, - "recall-0.05": 0.876033, - "recall-0.1": 0.993802, + "mean": 0.03745, + "median": 0.035063, + "std": 0.014194, + "sem": 0.000647, + "min": 0.01794, + "max": 0.136715, + "recall-0.025": 0.163223, + "recall-0.05": 0.842975, + "recall-0.1": 0.991736, "recall-0.15": 0.997934, "recall-0.25": 0.997934, "recall-0.5": 0.997934, "num_labels": 484, - "ap-0.025": 0.041041, - "ap-0.05": 0.626763, - "ap-0.1": 0.797009, - "ap-0.15": 0.803699, - "ap-0.25": 0.803699, - "ap-0.5": 0.803699 + "ap-0.025": 0.023957, + "ap-0.05": 0.580664, + "ap-0.1": 0.79534, + "ap-0.15": 0.804355, + "ap-0.25": 0.804355, + "ap-0.5": 0.804355 }, "head": { "count": 483, - "mean": 0.036562, - "median": 0.030386, - "std": 0.022881, - "sem": 0.001042, + "mean": 0.037576, + "median": 0.030619, + "std": 0.023474, + "sem": 0.001069, "min": 0.004382, "max": 0.136584, - "recall-0.025": 0.392562, - "recall-0.05": 0.739669, - "recall-0.1": 0.979339, + "recall-0.025": 0.378099, + "recall-0.05": 0.721074, + "recall-0.1": 0.977273, "recall-0.15": 0.997934, "recall-0.25": 0.997934, "recall-0.5": 0.997934, @@ -6057,15 +6057,15 @@ Results of the model in various experiments on different datasets. }, "shoulder_left": { "count": 483, - "mean": 0.034722, - "median": 0.029123, - "std": 0.020485, - "sem": 0.000933, + "mean": 0.039129, + "median": 0.032213, + "std": 0.022975, + "sem": 0.001047, "min": 0.004214, "max": 0.142662, - "recall-0.025": 0.369835, - "recall-0.05": 0.81405, - "recall-0.1": 0.987603, + "recall-0.025": 0.31405, + "recall-0.05": 0.745868, + "recall-0.1": 0.979339, "recall-0.15": 0.997934, "recall-0.25": 0.997934, "recall-0.5": 0.997934, @@ -6073,111 +6073,111 @@ Results of the model in various experiments on different datasets. }, "shoulder_right": { "count": 483, - "mean": 0.031759, - "median": 0.026109, - "std": 0.018407, - "sem": 0.000838, - "min": 0.00347, - "max": 0.110679, - "recall-0.025": 0.466942, - "recall-0.05": 0.826446, - "recall-0.1": 0.993802, - "recall-0.15": 0.997934, + "mean": 0.035508, + "median": 0.030929, + "std": 0.021175, + "sem": 0.000965, + "min": 0.003681, + "max": 0.180338, + "recall-0.025": 0.384298, + "recall-0.05": 0.778926, + "recall-0.1": 0.987603, + "recall-0.15": 0.995868, "recall-0.25": 0.997934, "recall-0.5": 0.997934, "num_labels": 484 }, "elbow_left": { "count": 483, - "mean": 0.035706, - "median": 0.029341, - "std": 0.024522, - "sem": 0.001117, - "min": 0.004211, + "mean": 0.039093, + "median": 0.031479, + "std": 0.027871, + "sem": 0.001269, + "min": 0.005996, "max": 0.237523, - "recall-0.025": 0.386364, - "recall-0.05": 0.77686, - "recall-0.1": 0.981405, - "recall-0.15": 0.993802, + "recall-0.025": 0.347107, + "recall-0.05": 0.737603, + "recall-0.1": 0.966942, + "recall-0.15": 0.987603, "recall-0.25": 0.997934, "recall-0.5": 0.997934, "num_labels": 484 }, "elbow_right": { "count": 483, - "mean": 0.03838, - "median": 0.031125, - "std": 0.034151, - "sem": 0.001556, + "mean": 0.041455, + "median": 0.033784, + "std": 0.038189, + "sem": 0.001739, "min": 0.003944, - "max": 0.362552, - "recall-0.025": 0.407025, - "recall-0.05": 0.774793, - "recall-0.1": 0.964876, - "recall-0.15": 0.981405, + "max": 0.443462, + "recall-0.025": 0.367769, + "recall-0.05": 0.741736, + "recall-0.1": 0.954545, + "recall-0.15": 0.979339, "recall-0.25": 0.993802, "recall-0.5": 0.997934, "num_labels": 484 }, "wrist_left": { "count": 483, - "mean": 0.047139, - "median": 0.03772, - "std": 0.035565, - "sem": 0.00162, - "min": 0.003421, - "max": 0.314461, - "recall-0.025": 0.274793, - "recall-0.05": 0.669421, - "recall-0.1": 0.923554, - "recall-0.15": 0.979339, + "mean": 0.04895, + "median": 0.039104, + "std": 0.036788, + "sem": 0.001676, + "min": 0.002618, + "max": 0.309556, + "recall-0.025": 0.258264, + "recall-0.05": 0.650826, + "recall-0.1": 0.913223, + "recall-0.15": 0.977273, "recall-0.25": 0.995868, "recall-0.5": 0.997934, "num_labels": 484 }, "wrist_right": { - "count": 482, - "mean": 0.047946, - "median": 0.036566, - "std": 0.041584, - "sem": 0.001896, - "min": 0.002446, - "max": 0.39908, - "recall-0.025": 0.289256, - "recall-0.05": 0.652893, - "recall-0.1": 0.919421, - "recall-0.15": 0.969008, + "count": 481, + "mean": 0.050009, + "median": 0.040081, + "std": 0.040668, + "sem": 0.001856, + "min": 0.003069, + "max": 0.35488, + "recall-0.025": 0.262397, + "recall-0.05": 0.63843, + "recall-0.1": 0.900826, + "recall-0.15": 0.964876, "recall-0.25": 0.987603, - "recall-0.5": 0.995868, + "recall-0.5": 0.993802, "num_labels": 484 }, "hip_left": { "count": 483, - "mean": 0.038758, - "median": 0.036117, - "std": 0.019053, - "sem": 0.000868, - "min": 0.005977, - "max": 0.133493, - "recall-0.025": 0.239669, - "recall-0.05": 0.774793, - "recall-0.1": 0.985537, - "recall-0.15": 0.997934, + "mean": 0.040026, + "median": 0.036626, + "std": 0.021724, + "sem": 0.00099, + "min": 0.004385, + "max": 0.191835, + "recall-0.025": 0.231405, + "recall-0.05": 0.756198, + "recall-0.1": 0.979339, + "recall-0.15": 0.995868, "recall-0.25": 0.997934, "recall-0.5": 0.997934, "num_labels": 484 }, "hip_right": { "count": 483, - "mean": 0.040711, - "median": 0.033421, - "std": 0.026405, - "sem": 0.001203, - "min": 0.005419, + "mean": 0.042645, + "median": 0.034828, + "std": 0.027596, + "sem": 0.001257, + "min": 0.004215, "max": 0.149838, - "recall-0.025": 0.291322, - "recall-0.05": 0.75, - "recall-0.1": 0.958678, + "recall-0.025": 0.27686, + "recall-0.05": 0.71281, + "recall-0.1": 0.948347, "recall-0.15": 0.997934, "recall-0.25": 0.997934, "recall-0.5": 0.997934, @@ -6185,14 +6185,14 @@ Results of the model in various experiments on different datasets. }, "knee_left": { "count": 483, - "mean": 0.025004, - "median": 0.0211, - "std": 0.018866, - "sem": 0.000859, - "min": 0.001613, + "mean": 0.025053, + "median": 0.021076, + "std": 0.018987, + "sem": 0.000865, + "min": 0.001403, "max": 0.27912, - "recall-0.025": 0.595041, - "recall-0.05": 0.93595, + "recall-0.025": 0.603306, + "recall-0.05": 0.923554, "recall-0.1": 0.993802, "recall-0.15": 0.995868, "recall-0.25": 0.995868, @@ -6201,14 +6201,14 @@ Results of the model in various experiments on different datasets. }, "knee_right": { "count": 483, - "mean": 0.025695, - "median": 0.022859, - "std": 0.014864, - "sem": 0.000677, + "mean": 0.026099, + "median": 0.022868, + "std": 0.015211, + "sem": 0.000693, "min": 0.001415, "max": 0.094234, - "recall-0.025": 0.576446, - "recall-0.05": 0.92562, + "recall-0.025": 0.557851, + "recall-0.05": 0.919421, "recall-0.1": 0.997934, "recall-0.15": 0.997934, "recall-0.25": 0.997934, @@ -6217,15 +6217,15 @@ Results of the model in various experiments on different datasets. }, "ankle_left": { "count": 483, - "mean": 0.028829, - "median": 0.023676, - "std": 0.033788, - "sem": 0.001539, - "min": 0.001752, + "mean": 0.029784, + "median": 0.023999, + "std": 0.034378, + "sem": 0.001566, + "min": 0.002215, "max": 0.497796, "recall-0.025": 0.530992, - "recall-0.05": 0.92562, - "recall-0.1": 0.981405, + "recall-0.05": 0.909091, + "recall-0.1": 0.979339, "recall-0.15": 0.991736, "recall-0.25": 0.993802, "recall-0.5": 0.997934, @@ -6233,14 +6233,14 @@ Results of the model in various experiments on different datasets. }, "ankle_right": { "count": 483, - "mean": 0.028136, - "median": 0.02508, - "std": 0.016445, - "sem": 0.000749, + "mean": 0.029202, + "median": 0.026139, + "std": 0.017443, + "sem": 0.000795, "min": 0.001964, "max": 0.103825, - "recall-0.025": 0.495868, - "recall-0.05": 0.890496, + "recall-0.025": 0.464876, + "recall-0.05": 0.880165, "recall-0.1": 0.993802, "recall-0.15": 0.997934, "recall-0.25": 0.997934, @@ -6249,18 +6249,18 @@ Results of the model in various experiments on different datasets. }, "joint_recalls": { "num_labels": 6292, - "recall-0.025": 0.40782, - "recall-0.05": 0.8034, - "recall-0.1": 0.9733, - "recall-0.15": 0.99189, + "recall-0.025": 0.38207, + "recall-0.05": 0.77718, + "recall-0.1": 0.96615, + "recall-0.15": 0.99031, "recall-0.25": 0.99603, - "recall-0.5": 0.99777 + "recall-0.5": 0.99762 } } { "total_parts": 6776, - "correct_parts": 6740, - "pcp": 0.994687 + "correct_parts": 6729, + "pcp": 0.993064 } ``` diff --git a/spt/triangulator.cpp b/spt/triangulator.cpp index ac2b585..8b3986f 100644 --- a/spt/triangulator.cpp +++ b/spt/triangulator.cpp @@ -1220,8 +1220,7 @@ std::vector>> TriangulatorInte cv::Mat &group_pose = std::get<1>(group); // Calculate average joint distance - float dist_sum = 0.0; - size_t count = 0; + std::vector dists; for (size_t row = 0; row < num_joints; ++row) { const float *pose_3d_ptr = pose_3d.ptr(row); @@ -1236,15 +1235,20 @@ std::vector>> TriangulatorInte float dy = pose_3d_ptr[1] - group_pose_ptr[1]; float dz = pose_3d_ptr[2] - group_pose_ptr[2]; float dist_sq = dx * dx + dy * dy + dz * dz; - dist_sum += std::sqrt(dist_sq); - count++; + dists.push_back(std::sqrt(dist_sq)); } } - - if (count > 0) + if (dists.size() >= 5) + { + // Drop highest value to reduce influence of outliers + auto max_it = std::max_element(dists.begin(), dists.end()); + dists.erase(max_it); + } + if (dists.size() > 0) { // Check if the average joint distance is close enough - float avg_dist = dist_sum / count; + float avg_dist = std::accumulate(dists.begin(), dists.end(), 0.0); + avg_dist /= static_cast(dists.size()); if (avg_dist < max_joint_avg_dist && avg_dist < best_dist) { best_dist = avg_dist; @@ -1310,7 +1314,113 @@ std::vector>> TriangulatorInte } } - return groups; + // Merge close groups + // Depending on the inital group creation, one or more groups can be created that in the end + // share the same persons, even if they had a larger distance at the beginning + // So merge them similar to the group assignment before + std::vector>> merged_groups; + for (size_t i = 0; i < groups.size(); ++i) + { + size_t num_joints = std::get<1>(groups[i]).rows; + auto &group = groups[i]; + auto &group_visible_counts = per_group_visible_counts[i]; + + float best_dist = std::numeric_limits::infinity(); + int best_group = -1; + + for (size_t j = 0; j < merged_groups.size(); ++j) + { + auto &merged_group = merged_groups[j]; + + // Calculate average joint distance + std::vector dists; + for (size_t row = 0; row < num_joints; ++row) + { + const float *group_pose_ptr = std::get<1>(group).ptr(row); + const float *merged_pose_ptr = std::get<1>(merged_group).ptr(row); + + float score1 = group_pose_ptr[3]; + float score2 = merged_pose_ptr[3]; + + if (score1 > min_score && score2 > min_score) + { + float dx = group_pose_ptr[0] - merged_pose_ptr[0]; + float dy = group_pose_ptr[1] - merged_pose_ptr[1]; + float dz = group_pose_ptr[2] - merged_pose_ptr[2]; + float dist_sq = dx * dx + dy * dy + dz * dz; + dists.push_back(std::sqrt(dist_sq)); + } + } + if (dists.size() >= 5) + { + // Drop highest value to reduce influence of outliers + auto max_it = std::max_element(dists.begin(), dists.end()); + dists.erase(max_it); + } + if (dists.size() > 0) + { + // Check if the average joint distance is close enough + float avg_dist = std::accumulate(dists.begin(), dists.end(), 0.0); + avg_dist /= static_cast(dists.size()); + if (avg_dist < max_joint_avg_dist && avg_dist < best_dist) + { + best_dist = avg_dist; + best_group = static_cast(j); + } + } + } + + if (best_group == -1) + { + // Create a new group + merged_groups.push_back(group); + } + else + { + // Update existing group + auto &merged_group = merged_groups[best_group]; + cv::Point3f &merged_center = std::get<0>(merged_group); + cv::Mat &merged_group_pose = std::get<1>(merged_group); + std::vector &merged_group_indices = std::get<2>(merged_group); + + float n_elems1 = static_cast(merged_group_indices.size()); + float n_elems2 = static_cast(std::get<2>(group).size()); + float inv1 = n_elems1 / (n_elems1 + n_elems2); + float inv2 = n_elems2 / (n_elems1 + n_elems2); + + // Update group center + merged_center.x = (merged_center.x * inv1 + std::get<0>(group).x * inv2); + merged_center.y = (merged_center.y * inv1 + std::get<0>(group).y * inv2); + merged_center.z = (merged_center.z * inv1 + std::get<0>(group).z * inv2); + + // Update group pose + for (size_t row = 0; row < num_joints; ++row) + { + const float *group_pose_ptr = std::get<1>(group).ptr(row); + float *merged_pose_ptr = merged_group_pose.ptr(row); + + if (group_pose_ptr[3] > min_score) + { + float j_elems1 = static_cast(group_visible_counts[row]); + float j_elems2 = static_cast(per_group_visible_counts[best_group][row]); + float inv1 = j_elems1 / (j_elems1 + j_elems2); + float inv2 = j_elems2 / (j_elems1 + j_elems2); + + merged_pose_ptr[0] = (merged_pose_ptr[0] * inv1 + group_pose_ptr[0] * inv2); + merged_pose_ptr[1] = (merged_pose_ptr[1] * inv1 + group_pose_ptr[1] * inv2); + merged_pose_ptr[2] = (merged_pose_ptr[2] * inv1 + group_pose_ptr[2] * inv2); + merged_pose_ptr[3] = (merged_pose_ptr[3] * inv1 + group_pose_ptr[3] * inv2); + group_visible_counts[row]++; + } + } + + // Merge indices + merged_group_indices.insert( + merged_group_indices.end(), std::get<2>(group).begin(), std::get<2>(group).end()); + } + } + + return merged_groups; } // =================================================================================================