From 3350e2c4afb3bdff5d8103d67d7314c3e158db69 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 13 Sep 2024 16:56:53 +0200 Subject: [PATCH] Some mixed performance improvements. --- media/RESULTS.md | 1184 +++++++++++++++++++++--------------------- spt/triangulator.cpp | 190 +++++-- spt/triangulator.hpp | 13 +- 3 files changed, 745 insertions(+), 642 deletions(-) diff --git a/media/RESULTS.md b/media/RESULTS.md index 90c510f..a35d6af 100644 --- a/media/RESULTS.md +++ b/media/RESULTS.md @@ -7,9 +7,9 @@ Results of the model in various experiments on different datasets. (duration 00:01:20) ```json { - "avg_time_2d": 0.07016186108023434, - "avg_time_3d": 0.001361288458614026, - "avg_fps": 13.981487203060627 + "avg_time_2d": 0.07047960596569514, + "avg_time_3d": 0.0013772115869037176, + "avg_fps": 13.916563995726705 } { "person_nums": { @@ -28,53 +28,53 @@ Results of the model in various experiments on different datasets. }, "mpjpe": { "count": 600, - "mean": 0.066851, - "median": 0.059172, - "std": 0.027766, - "sem": 0.001134, + "mean": 0.067041, + "median": 0.059381, + "std": 0.027778, + "sem": 0.001135, "min": 0.043707, - "max": 0.189867, + "max": 0.190247, "recall-0.025": 0.0, - "recall-0.05": 0.035, + "recall-0.05": 0.033333, "recall-0.1": 0.933333, "recall-0.15": 0.95, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 600, "ap-0.025": 0.0, - "ap-0.05": 0.002244, - "ap-0.1": 0.895131, - "ap-0.15": 0.917269, + "ap-0.05": 0.002029, + "ap-0.1": 0.893791, + "ap-0.15": 0.916756, "ap-0.25": 1.0, "ap-0.5": 1.0 }, "nose": { "count": 600, - "mean": 0.11743, - "median": 0.101455, - "std": 0.042492, - "sem": 0.001736, + "mean": 0.117446, + "median": 0.101523, + "std": 0.042638, + "sem": 0.001742, "min": 0.046467, "max": 0.271134, "recall-0.025": 0.0, "recall-0.05": 0.003333, - "recall-0.1": 0.486667, - "recall-0.15": 0.806667, + "recall-0.1": 0.485, + "recall-0.15": 0.808333, "recall-0.25": 0.995, "recall-0.5": 1.0, "num_labels": 600 }, "shoulder_left": { "count": 600, - "mean": 0.033355, - "median": 0.025557, - "std": 0.032207, - "sem": 0.001316, + "mean": 0.033391, + "median": 0.025711, + "std": 0.032125, + "sem": 0.001313, "min": 0.003037, "max": 0.181945, - "recall-0.025": 0.488333, - "recall-0.05": 0.875, - "recall-0.1": 0.946667, + "recall-0.025": 0.486667, + "recall-0.05": 0.871667, + "recall-0.1": 0.948333, "recall-0.15": 0.965, "recall-0.25": 1.0, "recall-0.5": 1.0, @@ -82,30 +82,30 @@ Results of the model in various experiments on different datasets. }, "shoulder_right": { "count": 600, - "mean": 0.048134, - "median": 0.033305, - "std": 0.04268, - "sem": 0.001744, + "mean": 0.048216, + "median": 0.033552, + "std": 0.042119, + "sem": 0.001721, "min": 0.003882, "max": 0.247363, - "recall-0.025": 0.238333, - "recall-0.05": 0.766667, + "recall-0.025": 0.235, + "recall-0.05": 0.758333, "recall-0.1": 0.901667, - "recall-0.15": 0.936667, + "recall-0.15": 0.943333, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 600 }, "elbow_left": { "count": 600, - "mean": 0.042548, - "median": 0.034439, - "std": 0.034603, - "sem": 0.001414, + "mean": 0.042931, + "median": 0.034598, + "std": 0.034814, + "sem": 0.001422, "min": 0.002682, "max": 0.193558, - "recall-0.025": 0.273333, - "recall-0.05": 0.818333, + "recall-0.025": 0.271667, + "recall-0.05": 0.806667, "recall-0.1": 0.943333, "recall-0.15": 0.956667, "recall-0.25": 1.0, @@ -114,31 +114,31 @@ Results of the model in various experiments on different datasets. }, "elbow_right": { "count": 600, - "mean": 0.04325, - "median": 0.03319, - "std": 0.035165, - "sem": 0.001437, + "mean": 0.043934, + "median": 0.033341, + "std": 0.036065, + "sem": 0.001474, "min": 0.00667, "max": 0.285024, - "recall-0.025": 0.251667, - "recall-0.05": 0.81, - "recall-0.1": 0.935, - "recall-0.15": 0.946667, + "recall-0.025": 0.245, + "recall-0.05": 0.806667, + "recall-0.1": 0.928333, + "recall-0.15": 0.945, "recall-0.25": 0.998333, "recall-0.5": 1.0, "num_labels": 600 }, "wrist_left": { "count": 600, - "mean": 0.040641, - "median": 0.024203, - "std": 0.042385, - "sem": 0.001732, + "mean": 0.040738, + "median": 0.024321, + "std": 0.04224, + "sem": 0.001726, "min": 0.001517, - "max": 0.18543, - "recall-0.025": 0.516667, + "max": 0.18631, + "recall-0.025": 0.511667, "recall-0.05": 0.746667, - "recall-0.1": 0.908333, + "recall-0.1": 0.91, "recall-0.15": 0.945, "recall-0.25": 1.0, "recall-0.5": 1.0, @@ -146,31 +146,31 @@ Results of the model in various experiments on different datasets. }, "wrist_right": { "count": 600, - "mean": 0.042497, - "median": 0.025673, - "std": 0.046851, - "sem": 0.001914, + "mean": 0.043205, + "median": 0.025854, + "std": 0.04796, + "sem": 0.00196, "min": 0.001698, "max": 0.425617, - "recall-0.025": 0.481667, - "recall-0.05": 0.775, - "recall-0.1": 0.896667, - "recall-0.15": 0.921667, + "recall-0.025": 0.471667, + "recall-0.05": 0.77, + "recall-0.1": 0.893333, + "recall-0.15": 0.92, "recall-0.25": 0.998333, "recall-0.5": 1.0, "num_labels": 600 }, "hip_left": { "count": 600, - "mean": 0.089623, - "median": 0.086723, - "std": 0.032973, - "sem": 0.001347, + "mean": 0.089857, + "median": 0.087197, + "std": 0.033035, + "sem": 0.00135, "min": 0.01535, "max": 0.236764, "recall-0.025": 0.006667, "recall-0.05": 0.043333, - "recall-0.1": 0.83, + "recall-0.1": 0.823333, "recall-0.15": 0.95, "recall-0.25": 1.0, "recall-0.5": 1.0, @@ -178,30 +178,30 @@ Results of the model in various experiments on different datasets. }, "hip_right": { "count": 600, - "mean": 0.114452, - "median": 0.114981, - "std": 0.026143, - "sem": 0.001068, + "mean": 0.114593, + "median": 0.11502, + "std": 0.026334, + "sem": 0.001076, "min": 0.046173, "max": 0.234935, "recall-0.025": 0.0, "recall-0.05": 0.001667, - "recall-0.1": 0.25, - "recall-0.15": 0.948333, + "recall-0.1": 0.253333, + "recall-0.15": 0.946667, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 600 }, "knee_left": { "count": 600, - "mean": 0.060815, - "median": 0.045004, - "std": 0.061356, - "sem": 0.002507, + "mean": 0.06083, + "median": 0.045088, + "std": 0.061419, + "sem": 0.00251, "min": 0.01946, "max": 0.425374, - "recall-0.025": 0.045, - "recall-0.05": 0.605, + "recall-0.025": 0.046667, + "recall-0.05": 0.6, "recall-0.1": 0.92, "recall-0.15": 0.921667, "recall-0.25": 0.98, @@ -210,13 +210,13 @@ Results of the model in various experiments on different datasets. }, "knee_right": { "count": 600, - "mean": 0.051079, - "median": 0.043279, - "std": 0.034968, - "sem": 0.001429, + "mean": 0.051154, + "median": 0.04325, + "std": 0.035226, + "sem": 0.001439, "min": 0.017082, "max": 0.259899, - "recall-0.025": 0.038333, + "recall-0.025": 0.036667, "recall-0.05": 0.733333, "recall-0.1": 0.943333, "recall-0.15": 0.945, @@ -226,15 +226,15 @@ Results of the model in various experiments on different datasets. }, "ankle_left": { "count": 598, - "mean": 0.098276, - "median": 0.086303, - "std": 0.049778, - "sem": 0.002037, - "min": 0.053264, + "mean": 0.098141, + "median": 0.086284, + "std": 0.049814, + "sem": 0.002039, + "min": 0.039216, "max": 0.495347, "recall-0.025": 0.0, - "recall-0.05": 0.0, - "recall-0.1": 0.841667, + "recall-0.05": 0.001667, + "recall-0.1": 0.843333, "recall-0.15": 0.938333, "recall-0.25": 0.98, "recall-0.5": 0.996667, @@ -242,34 +242,34 @@ Results of the model in various experiments on different datasets. }, "ankle_right": { "count": 600, - "mean": 0.08545, - "median": 0.068594, - "std": 0.058449, - "sem": 0.002388, + "mean": 0.085589, + "median": 0.068683, + "std": 0.05862, + "sem": 0.002395, "min": 0.032987, "max": 0.463238, "recall-0.025": 0.0, - "recall-0.05": 0.015, + "recall-0.05": 0.016667, "recall-0.1": 0.885, - "recall-0.15": 0.908333, + "recall-0.15": 0.906667, "recall-0.25": 0.978333, "recall-0.5": 1.0, "num_labels": 600 }, "joint_recalls": { "num_labels": 7800, - "recall-0.025": 0.17949, - "recall-0.05": 0.4759, - "recall-0.1": 0.82179, - "recall-0.15": 0.92962, + "recall-0.025": 0.17782, + "recall-0.05": 0.47333, + "recall-0.1": 0.82038, + "recall-0.15": 0.92974, "recall-0.25": 0.99397, "recall-0.5": 0.99974 } } { "total_parts": 8400, - "correct_parts": 8092, - "pcp": 0.963333 + "correct_parts": 8090, + "pcp": 0.963095 } ``` @@ -278,56 +278,56 @@ Results of the model in various experiments on different datasets. (duration 00:00:56) ```json { - "avg_time_2d": 0.10439699048438843, - "avg_time_3d": 0.003671272513792687, - "avg_fps": 9.253410504218346 + "avg_time_2d": 0.0996503477653687, + "avg_time_3d": 0.003785594632125802, + "avg_fps": 9.667819297832613 } { "person_nums": { "total_frames": 301, "total_labels": 477, - "total_preds": 882, - "considered_empty": 3, + "total_preds": 889, + "considered_empty": 2, "valid_preds": 477, - "invalid_preds": 402, + "invalid_preds": 410, "missing": 0, - "invalid_fraction": 0.45734, - "precision": 0.54266, + "invalid_fraction": 0.46223, + "precision": 0.53777, "recall": 1.0, - "f1": 0.70354, - "non_empty": 879 + "f1": 0.69941, + "non_empty": 887 }, "mpjpe": { "count": 477, - "mean": 0.04824, + "mean": 0.048309, "median": 0.04301, - "std": 0.014639, - "sem": 0.000671, + "std": 0.014754, + "sem": 0.000676, "min": 0.029586, "max": 0.106389, "recall-0.025": 0.0, "recall-0.05": 0.677149, - "recall-0.1": 0.991614, + "recall-0.1": 0.989518, "recall-0.15": 1.0, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 477, "ap-0.025": 0.0, - "ap-0.05": 0.342452, - "ap-0.1": 0.692315, - "ap-0.15": 0.702141, - "ap-0.25": 0.702141, - "ap-0.5": 0.702141 + "ap-0.05": 0.342622, + "ap-0.1": 0.692726, + "ap-0.15": 0.703931, + "ap-0.25": 0.703931, + "ap-0.5": 0.703931 }, "head": { "count": 477, - "mean": 0.055391, + "mean": 0.05522, "median": 0.049788, - "std": 0.026004, - "sem": 0.001192, + "std": 0.025981, + "sem": 0.001191, "min": 0.006374, "max": 0.147303, - "recall-0.025": 0.075472, + "recall-0.025": 0.077568, "recall-0.05": 0.505241, "recall-0.1": 0.91195, "recall-0.15": 1.0, @@ -337,9 +337,9 @@ Results of the model in various experiments on different datasets. }, "shoulder_left": { "count": 477, - "mean": 0.044421, + "mean": 0.04442, "median": 0.038938, - "std": 0.020657, + "std": 0.020658, "sem": 0.000947, "min": 0.006714, "max": 0.138512, @@ -353,14 +353,14 @@ Results of the model in various experiments on different datasets. }, "shoulder_right": { "count": 477, - "mean": 0.050386, - "median": 0.045185, - "std": 0.024435, - "sem": 0.00112, + "mean": 0.050591, + "median": 0.045337, + "std": 0.024506, + "sem": 0.001123, "min": 0.00639, "max": 0.15065, "recall-0.025": 0.106918, - "recall-0.05": 0.557652, + "recall-0.05": 0.555556, "recall-0.1": 0.947589, "recall-0.15": 0.997904, "recall-0.25": 1.0, @@ -369,9 +369,9 @@ Results of the model in various experiments on different datasets. }, "elbow_left": { "count": 477, - "mean": 0.040857, + "mean": 0.040846, "median": 0.034194, - "std": 0.025942, + "std": 0.025936, "sem": 0.001189, "min": 0.005533, "max": 0.152226, @@ -385,14 +385,14 @@ Results of the model in various experiments on different datasets. }, "elbow_right": { "count": 477, - "mean": 0.053459, - "median": 0.043749, - "std": 0.041996, - "sem": 0.001925, + "mean": 0.053511, + "median": 0.044068, + "std": 0.041975, + "sem": 0.001924, "min": 0.002858, "max": 0.244706, "recall-0.025": 0.259958, - "recall-0.05": 0.561845, + "recall-0.05": 0.559748, "recall-0.1": 0.890985, "recall-0.15": 0.958071, "recall-0.25": 1.0, @@ -401,30 +401,30 @@ Results of the model in various experiments on different datasets. }, "wrist_left": { "count": 477, - "mean": 0.060765, + "mean": 0.061566, "median": 0.053276, - "std": 0.043274, - "sem": 0.001983, + "std": 0.046136, + "sem": 0.002115, "min": 0.003745, "max": 0.359369, "recall-0.025": 0.12369, "recall-0.05": 0.412998, "recall-0.1": 0.914046, "recall-0.15": 0.962264, - "recall-0.25": 0.985325, + "recall-0.25": 0.981132, "recall-0.5": 1.0, "num_labels": 477 }, "wrist_right": { "count": 477, - "mean": 0.057975, - "median": 0.053722, - "std": 0.02995, - "sem": 0.001373, + "mean": 0.058012, + "median": 0.05377, + "std": 0.029967, + "sem": 0.001374, "min": 0.009013, "max": 0.202256, - "recall-0.025": 0.106918, - "recall-0.05": 0.448637, + "recall-0.025": 0.109015, + "recall-0.05": 0.446541, "recall-0.1": 0.897275, "recall-0.15": 0.989518, "recall-0.25": 1.0, @@ -433,15 +433,15 @@ Results of the model in various experiments on different datasets. }, "hip_left": { "count": 477, - "mean": 0.0479, - "median": 0.042849, - "std": 0.026619, - "sem": 0.00122, + "mean": 0.047812, + "median": 0.043075, + "std": 0.02636, + "sem": 0.001208, "min": 0.005757, "max": 0.140667, "recall-0.025": 0.213836, "recall-0.05": 0.620545, - "recall-0.1": 0.951782, + "recall-0.1": 0.958071, "recall-0.15": 1.0, "recall-0.25": 1.0, "recall-0.5": 1.0, @@ -449,14 +449,14 @@ Results of the model in various experiments on different datasets. }, "hip_right": { "count": 477, - "mean": 0.057871, - "median": 0.056052, - "std": 0.025477, - "sem": 0.001168, + "mean": 0.057973, + "median": 0.056146, + "std": 0.025504, + "sem": 0.001169, "min": 0.005189, "max": 0.136207, "recall-0.025": 0.109015, - "recall-0.05": 0.421384, + "recall-0.05": 0.419287, "recall-0.1": 0.930818, "recall-0.15": 1.0, "recall-0.25": 1.0, @@ -465,9 +465,9 @@ Results of the model in various experiments on different datasets. }, "knee_left": { "count": 477, - "mean": 0.040016, + "mean": 0.040006, "median": 0.03745, - "std": 0.024403, + "std": 0.024382, "sem": 0.001118, "min": 0.005407, "max": 0.195502, @@ -481,9 +481,9 @@ Results of the model in various experiments on different datasets. }, "knee_right": { "count": 477, - "mean": 0.039374, + "mean": 0.039398, "median": 0.034816, - "std": 0.02337, + "std": 0.023356, "sem": 0.001071, "min": 0.006865, "max": 0.186559, @@ -497,14 +497,14 @@ Results of the model in various experiments on different datasets. }, "ankle_left": { "count": 477, - "mean": 0.037256, + "mean": 0.037236, "median": 0.031, - "std": 0.030824, + "std": 0.030833, "sem": 0.001413, "min": 0.001693, "max": 0.224454, "recall-0.025": 0.433962, - "recall-0.05": 0.819706, + "recall-0.05": 0.821803, "recall-0.1": 0.943396, "recall-0.15": 0.983229, "recall-0.25": 1.0, @@ -513,11 +513,11 @@ Results of the model in various experiments on different datasets. }, "ankle_right": { "count": 477, - "mean": 0.041454, + "mean": 0.041431, "median": 0.031787, - "std": 0.038012, - "sem": 0.001742, - "min": 0.005439, + "std": 0.037989, + "sem": 0.001741, + "min": 0.004747, "max": 0.298397, "recall-0.025": 0.312369, "recall-0.05": 0.792453, @@ -529,18 +529,18 @@ Results of the model in various experiments on different datasets. }, "joint_recalls": { "num_labels": 6201, - "recall-0.025": 0.21077, - "recall-0.05": 0.62038, - "recall-0.1": 0.93807, + "recall-0.025": 0.21109, + "recall-0.05": 0.6199, + "recall-0.1": 0.9384, "recall-0.15": 0.98742, - "recall-0.25": 0.99871, + "recall-0.25": 0.99823, "recall-0.5": 1.0 } } { "total_parts": 6678, - "correct_parts": 6618, - "pcp": 0.991015 + "correct_parts": 6617, + "pcp": 0.990866 } ``` @@ -549,53 +549,53 @@ Results of the model in various experiments on different datasets. (duration 00:00:29) ```json { - "avg_time_2d": 0.05871032543902127, - "avg_time_3d": 0.0016240331361878594, - "avg_fps": 16.574303988886548 + "avg_time_2d": 0.05880933100322507, + "avg_time_3d": 0.0016649657825254045, + "avg_fps": 16.535950860955353 } { "person_nums": { "total_frames": 222, "total_labels": 376, - "total_preds": 464, + "total_preds": 473, "considered_empty": 1, "valid_preds": 376, - "invalid_preds": 87, + "invalid_preds": 96, "missing": 0, - "invalid_fraction": 0.1879, - "precision": 0.8121, + "invalid_fraction": 0.20339, + "precision": 0.79661, "recall": 1.0, - "f1": 0.89631, - "non_empty": 463 + "f1": 0.88679, + "non_empty": 472 }, "mpjpe": { "count": 376, - "mean": 0.0776, - "median": 0.075327, - "std": 0.017994, - "sem": 0.000929, + "mean": 0.078386, + "median": 0.076036, + "std": 0.018832, + "sem": 0.000972, "min": 0.048684, "max": 0.158395, "recall-0.025": 0.0, "recall-0.05": 0.010638, - "recall-0.1": 0.906915, + "recall-0.1": 0.890957, "recall-0.15": 0.989362, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 376, "ap-0.025": 0.0, - "ap-0.05": 0.000331, - "ap-0.1": 0.795398, - "ap-0.15": 0.888884, - "ap-0.25": 0.897608, - "ap-0.5": 0.897608 + "ap-0.05": 0.000324, + "ap-0.1": 0.774956, + "ap-0.15": 0.89001, + "ap-0.25": 0.898696, + "ap-0.5": 0.898696 }, "head": { "count": 376, - "mean": 0.059377, - "median": 0.060875, - "std": 0.025299, - "sem": 0.001306, + "mean": 0.059335, + "median": 0.06095, + "std": 0.025332, + "sem": 0.001308, "min": 0.006936, "max": 0.161995, "recall-0.025": 0.098404, @@ -608,13 +608,13 @@ Results of the model in various experiments on different datasets. }, "shoulder_left": { "count": 376, - "mean": 0.061281, - "median": 0.056412, - "std": 0.028295, - "sem": 0.001461, + "mean": 0.061447, + "median": 0.056892, + "std": 0.028318, + "sem": 0.001462, "min": 0.003738, "max": 0.154144, - "recall-0.025": 0.053191, + "recall-0.025": 0.050532, "recall-0.05": 0.401596, "recall-0.1": 0.906915, "recall-0.15": 0.989362, @@ -624,14 +624,14 @@ Results of the model in various experiments on different datasets. }, "shoulder_right": { "count": 376, - "mean": 0.063652, - "median": 0.058472, - "std": 0.026828, + "mean": 0.063637, + "median": 0.057984, + "std": 0.026815, "sem": 0.001385, "min": 0.016753, "max": 0.185574, "recall-0.025": 0.039894, - "recall-0.05": 0.369681, + "recall-0.05": 0.367021, "recall-0.1": 0.893617, "recall-0.15": 0.992021, "recall-0.25": 1.0, @@ -640,31 +640,31 @@ Results of the model in various experiments on different datasets. }, "elbow_left": { "count": 376, - "mean": 0.085614, - "median": 0.069707, - "std": 0.047296, - "sem": 0.002442, + "mean": 0.08585, + "median": 0.069926, + "std": 0.047388, + "sem": 0.002447, "min": 0.007605, "max": 0.205639, "recall-0.025": 0.029255, - "recall-0.05": 0.268617, - "recall-0.1": 0.643617, - "recall-0.15": 0.864362, + "recall-0.05": 0.265957, + "recall-0.1": 0.640957, + "recall-0.15": 0.861702, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 376 }, "elbow_right": { "count": 376, - "mean": 0.07041, + "mean": 0.070551, "median": 0.060437, - "std": 0.035305, - "sem": 0.001823, + "std": 0.035349, + "sem": 0.001825, "min": 0.00548, "max": 0.237233, "recall-0.025": 0.023936, - "recall-0.05": 0.327128, - "recall-0.1": 0.797872, + "recall-0.05": 0.324468, + "recall-0.1": 0.795213, "recall-0.15": 0.973404, "recall-0.25": 1.0, "recall-0.5": 1.0, @@ -672,10 +672,10 @@ Results of the model in various experiments on different datasets. }, "wrist_left": { "count": 376, - "mean": 0.114571, + "mean": 0.114922, "median": 0.099655, - "std": 0.062882, - "sem": 0.003247, + "std": 0.06375, + "sem": 0.003292, "min": 0.010314, "max": 0.431497, "recall-0.025": 0.013298, @@ -688,130 +688,130 @@ Results of the model in various experiments on different datasets. }, "wrist_right": { "count": 376, - "mean": 0.097637, + "mean": 0.097856, "median": 0.08983, - "std": 0.042502, - "sem": 0.002195, + "std": 0.042276, + "sem": 0.002183, "min": 0.014228, "max": 0.309971, "recall-0.025": 0.007979, - "recall-0.05": 0.101064, + "recall-0.05": 0.095745, "recall-0.1": 0.603723, - "recall-0.15": 0.880319, - "recall-0.25": 0.994681, + "recall-0.15": 0.87766, + "recall-0.25": 0.99734, "recall-0.5": 1.0, "num_labels": 376 }, "hip_left": { "count": 376, - "mean": 0.080917, - "median": 0.082729, - "std": 0.028998, - "sem": 0.001497, + "mean": 0.081482, + "median": 0.082897, + "std": 0.02958, + "sem": 0.001527, "min": 0.018902, "max": 0.179369, "recall-0.025": 0.029255, "recall-0.05": 0.170213, - "recall-0.1": 0.723404, - "recall-0.15": 0.994681, + "recall-0.1": 0.710106, + "recall-0.15": 0.992021, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 376 }, "hip_right": { "count": 376, - "mean": 0.086686, - "median": 0.083672, - "std": 0.02543, - "sem": 0.001313, + "mean": 0.087186, + "median": 0.083851, + "std": 0.026263, + "sem": 0.001356, "min": 0.024343, "max": 0.207002, "recall-0.025": 0.00266, "recall-0.05": 0.058511, - "recall-0.1": 0.755319, - "recall-0.15": 0.973404, + "recall-0.1": 0.75, + "recall-0.15": 0.968085, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 376 }, "knee_left": { "count": 376, - "mean": 0.05488, - "median": 0.05203, - "std": 0.025318, - "sem": 0.001307, + "mean": 0.055923, + "median": 0.05234, + "std": 0.027946, + "sem": 0.001443, "min": 0.007197, - "max": 0.15096, + "max": 0.228932, "recall-0.025": 0.082447, - "recall-0.05": 0.468085, - "recall-0.1": 0.941489, - "recall-0.15": 0.99734, + "recall-0.05": 0.465426, + "recall-0.1": 0.933511, + "recall-0.15": 0.989362, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 376 }, "knee_right": { "count": 376, - "mean": 0.058292, - "median": 0.05652, - "std": 0.022861, - "sem": 0.001181, + "mean": 0.059604, + "median": 0.056881, + "std": 0.026175, + "sem": 0.001352, "min": 0.017224, - "max": 0.157863, + "max": 0.210945, "recall-0.025": 0.053191, - "recall-0.05": 0.37766, - "recall-0.1": 0.941489, - "recall-0.15": 0.99734, + "recall-0.05": 0.37234, + "recall-0.1": 0.930851, + "recall-0.15": 0.989362, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 376 }, "ankle_left": { "count": 372, - "mean": 0.089987, - "median": 0.05905, - "std": 0.099776, - "sem": 0.00518, + "mean": 0.091736, + "median": 0.060062, + "std": 0.100196, + "sem": 0.005202, "min": 0.007394, "max": 0.493162, "recall-0.025": 0.06383, - "recall-0.05": 0.305851, - "recall-0.1": 0.827128, - "recall-0.15": 0.890957, - "recall-0.25": 0.909574, + "recall-0.05": 0.297872, + "recall-0.1": 0.816489, + "recall-0.15": 0.882979, + "recall-0.25": 0.904255, "recall-0.5": 0.989362, "num_labels": 376 }, "ankle_right": { "count": 372, - "mean": 0.076338, - "median": 0.052354, - "std": 0.085003, - "sem": 0.004413, + "mean": 0.080393, + "median": 0.052772, + "std": 0.090361, + "sem": 0.004691, "min": 0.006245, "max": 0.496601, "recall-0.025": 0.071809, - "recall-0.05": 0.478723, - "recall-0.1": 0.859043, - "recall-0.15": 0.914894, - "recall-0.25": 0.930851, + "recall-0.05": 0.470745, + "recall-0.1": 0.845745, + "recall-0.15": 0.901596, + "recall-0.25": 0.920213, "recall-0.5": 0.989362, "num_labels": 376 }, "joint_recalls": { "num_labels": 4888, "recall-0.025": 0.04255, - "recall-0.05": 0.2901, - "recall-0.1": 0.79439, - "recall-0.15": 0.9376, - "recall-0.25": 0.98363, + "recall-0.05": 0.28703, + "recall-0.1": 0.78969, + "recall-0.15": 0.93412, + "recall-0.25": 0.98261, "recall-0.5": 0.99836 } } { "total_parts": 5264, - "correct_parts": 4969, - "pcp": 0.943959 + "correct_parts": 4962, + "pcp": 0.942629 } ``` @@ -820,189 +820,189 @@ Results of the model in various experiments on different datasets. (duration 00:02:31) ```json { - "avg_time_2d": 0.05640246679401552, - "avg_time_3d": 0.0013555948876795361, - "avg_fps": 17.31360040284947 + "avg_time_2d": 0.055016697512304646, + "avg_time_3d": 0.0014002407887448018, + "avg_fps": 17.72517315037279 } { "person_nums": { "total_frames": 629, "total_labels": 1061, - "total_preds": 1047, - "considered_empty": 118, - "valid_preds": 710, - "invalid_preds": 219, - "missing": 351, - "invalid_fraction": 0.23574, - "precision": 0.76426, - "recall": 0.66918, - "f1": 0.71357, - "non_empty": 929 + "total_preds": 1085, + "considered_empty": 85, + "valid_preds": 761, + "invalid_preds": 239, + "missing": 300, + "invalid_fraction": 0.239, + "precision": 0.761, + "recall": 0.71725, + "f1": 0.73848, + "non_empty": 1000 }, "mpjpe": { - "count": 710, - "mean": 0.11365, - "median": 0.094542, - "std": 0.066461, - "sem": 0.002496, + "count": 761, + "mean": 0.116647, + "median": 0.095682, + "std": 0.069225, + "sem": 0.002511, "min": 0.040489, "max": 0.497874, "recall-0.025": 0.0, - "recall-0.05": 0.006598, - "recall-0.1": 0.366635, - "recall-0.15": 0.570217, - "recall-0.25": 0.64279, - "recall-0.5": 0.66918, + "recall-0.05": 0.005655, + "recall-0.1": 0.385485, + "recall-0.15": 0.589067, + "recall-0.25": 0.686145, + "recall-0.5": 0.717248, "num_labels": 1061, "ap-0.025": 0.0, - "ap-0.05": 0.000155, - "ap-0.1": 0.237214, - "ap-0.15": 0.475129, - "ap-0.25": 0.571172, - "ap-0.5": 0.602435 + "ap-0.05": 9.9e-05, + "ap-0.1": 0.247799, + "ap-0.15": 0.48848, + "ap-0.25": 0.614452, + "ap-0.5": 0.654049 }, "head": { - "count": 709, - "mean": 0.062285, - "median": 0.049078, - "std": 0.052765, - "sem": 0.001983, - "min": 0.006565, + "count": 760, + "mean": 0.065587, + "median": 0.050599, + "std": 0.057824, + "sem": 0.002099, + "min": 0.005944, "max": 0.436815, - "recall-0.025": 0.080113, - "recall-0.05": 0.341188, - "recall-0.1": 0.591894, - "recall-0.15": 0.632422, - "recall-0.25": 0.6541, - "recall-0.5": 0.668238, + "recall-0.025": 0.085768, + "recall-0.05": 0.354383, + "recall-0.1": 0.62394, + "recall-0.15": 0.667295, + "recall-0.25": 0.69934, + "recall-0.5": 0.716305, "num_labels": 1061 }, "shoulder_left": { - "count": 708, - "mean": 0.062058, - "median": 0.043195, - "std": 0.066177, - "sem": 0.002489, + "count": 759, + "mean": 0.064625, + "median": 0.04426, + "std": 0.068399, + "sem": 0.002484, "min": 0.003155, "max": 0.498748, - "recall-0.025": 0.142319, - "recall-0.05": 0.401508, - "recall-0.1": 0.577757, - "recall-0.15": 0.625825, - "recall-0.25": 0.647502, - "recall-0.5": 0.667295, + "recall-0.025": 0.150801, + "recall-0.05": 0.413761, + "recall-0.1": 0.604147, + "recall-0.15": 0.662582, + "recall-0.25": 0.693685, + "recall-0.5": 0.715363, "num_labels": 1061 }, "shoulder_right": { - "count": 708, - "mean": 0.05826, - "median": 0.039432, - "std": 0.064247, - "sem": 0.002416, + "count": 758, + "mean": 0.062384, + "median": 0.040803, + "std": 0.069916, + "sem": 0.002541, "min": 0.001884, - "max": 0.470146, - "recall-0.025": 0.147031, - "recall-0.05": 0.428841, - "recall-0.1": 0.588124, - "recall-0.15": 0.630537, - "recall-0.25": 0.64656, - "recall-0.5": 0.667295, + "max": 0.496442, + "recall-0.025": 0.150801, + "recall-0.05": 0.443921, + "recall-0.1": 0.615457, + "recall-0.15": 0.667295, + "recall-0.25": 0.688973, + "recall-0.5": 0.71442, "num_labels": 1061 }, "elbow_left": { - "count": 696, - "mean": 0.100812, - "median": 0.073006, - "std": 0.081769, - "sem": 0.003102, - "min": 0.005709, + "count": 745, + "mean": 0.102191, + "median": 0.075708, + "std": 0.082833, + "sem": 0.003037, + "min": 0.005382, "max": 0.481769, - "recall-0.025": 0.067861, - "recall-0.05": 0.216777, - "recall-0.1": 0.399623, - "recall-0.15": 0.516494, - "recall-0.25": 0.615457, - "recall-0.5": 0.655985, + "recall-0.025": 0.065033, + "recall-0.05": 0.220547, + "recall-0.1": 0.429783, + "recall-0.15": 0.557964, + "recall-0.25": 0.6541, + "recall-0.5": 0.702168, "num_labels": 1061 }, "elbow_right": { - "count": 699, - "mean": 0.079988, - "median": 0.05676, - "std": 0.074464, - "sem": 0.002819, + "count": 748, + "mean": 0.083851, + "median": 0.057555, + "std": 0.078679, + "sem": 0.002879, "min": 0.003313, "max": 0.498068, - "recall-0.025": 0.087653, - "recall-0.05": 0.295947, - "recall-0.1": 0.499529, - "recall-0.15": 0.585297, - "recall-0.25": 0.632422, - "recall-0.5": 0.658812, + "recall-0.025": 0.095193, + "recall-0.05": 0.30066, + "recall-0.1": 0.520264, + "recall-0.15": 0.61263, + "recall-0.25": 0.66918, + "recall-0.5": 0.704995, "num_labels": 1061 }, "wrist_left": { - "count": 702, - "mean": 0.12747, - "median": 0.091245, - "std": 0.104152, - "sem": 0.003934, + "count": 752, + "mean": 0.127271, + "median": 0.090832, + "std": 0.102889, + "sem": 0.003754, "min": 0.004181, "max": 0.497384, - "recall-0.025": 0.06409, - "recall-0.05": 0.173421, - "recall-0.1": 0.351555, - "recall-0.15": 0.446748, - "recall-0.25": 0.567389, - "recall-0.5": 0.66164, + "recall-0.025": 0.061263, + "recall-0.05": 0.180961, + "recall-0.1": 0.378888, + "recall-0.15": 0.472196, + "recall-0.25": 0.61263, + "recall-0.5": 0.708765, "num_labels": 1061 }, "wrist_right": { - "count": 697, - "mean": 0.112895, - "median": 0.078236, - "std": 0.100097, - "sem": 0.003794, + "count": 746, + "mean": 0.112637, + "median": 0.076428, + "std": 0.101177, + "sem": 0.003707, "min": 0.006423, "max": 0.494757, "recall-0.025": 0.071631, - "recall-0.05": 0.218662, - "recall-0.1": 0.397738, - "recall-0.15": 0.485391, - "recall-0.25": 0.581527, - "recall-0.5": 0.656927, + "recall-0.05": 0.241282, + "recall-0.1": 0.429783, + "recall-0.15": 0.526861, + "recall-0.25": 0.618285, + "recall-0.5": 0.70311, "num_labels": 1061 }, "hip_left": { - "count": 696, - "mean": 0.186824, - "median": 0.167605, - "std": 0.084772, - "sem": 0.003216, + "count": 742, + "mean": 0.188889, + "median": 0.17064, + "std": 0.084308, + "sem": 0.003097, "min": 0.021758, "max": 0.499358, "recall-0.025": 0.000943, - "recall-0.05": 0.005655, - "recall-0.1": 0.065975, - "recall-0.15": 0.249764, - "recall-0.25": 0.540999, - "recall-0.5": 0.655985, + "recall-0.05": 0.00377, + "recall-0.1": 0.065033, + "recall-0.15": 0.259189, + "recall-0.25": 0.572102, + "recall-0.5": 0.69934, "num_labels": 1061 }, "hip_right": { - "count": 695, - "mean": 0.178149, - "median": 0.163969, - "std": 0.078021, - "sem": 0.002962, - "min": 0.024563, + "count": 741, + "mean": 0.182767, + "median": 0.167996, + "std": 0.079854, + "sem": 0.002935, + "min": 0.036054, "max": 0.492847, - "recall-0.025": 0.000943, + "recall-0.025": 0.0, "recall-0.05": 0.00377, - "recall-0.1": 0.076343, - "recall-0.15": 0.271442, - "recall-0.25": 0.557964, - "recall-0.5": 0.655042, + "recall-0.1": 0.069746, + "recall-0.15": 0.27427, + "recall-0.25": 0.583412, + "recall-0.5": 0.698398, "num_labels": 1061 }, "knee_left": {}, @@ -1011,18 +1011,18 @@ Results of the model in various experiments on different datasets. "ankle_right": {}, "joint_recalls": { "num_labels": 9549, - "recall-0.025": 0.0732, - "recall-0.05": 0.23123, - "recall-0.1": 0.39376, - "recall-0.15": 0.49314, - "recall-0.25": 0.60425, - "recall-0.5": 0.66028 + "recall-0.025": 0.0753, + "recall-0.05": 0.24003, + "recall-0.1": 0.41481, + "recall-0.15": 0.52152, + "recall-0.25": 0.64321, + "recall-0.5": 0.70636 } } { "total_parts": 10610, - "correct_parts": 5503, - "pcp": 0.518662 + "correct_parts": 5825, + "pcp": 0.54901 } ``` @@ -1031,269 +1031,269 @@ Results of the model in various experiments on different datasets. (duration 00:02:28) ```json { - "avg_time_2d": 0.11131117460204334, - "avg_time_3d": 0.004892377737091808, - "avg_fps": 8.605588898707179 + "avg_time_2d": 0.10984443804112877, + "avg_time_3d": 0.004945937598623881, + "avg_fps": 8.71153173275002 } { "person_nums": { "total_frames": 420, "total_labels": 1466, - "total_preds": 1545, - "considered_empty": 6, - "valid_preds": 1463, - "invalid_preds": 76, - "missing": 3, - "invalid_fraction": 0.04938, - "precision": 0.95062, - "recall": 0.99795, - "f1": 0.97371, - "non_empty": 1539 + "total_preds": 1549, + "considered_empty": 7, + "valid_preds": 1464, + "invalid_preds": 78, + "missing": 2, + "invalid_fraction": 0.05058, + "precision": 0.94942, + "recall": 0.99864, + "f1": 0.9734, + "non_empty": 1542 }, "mpjpe": { - "count": 1463, - "mean": 0.042929, - "median": 0.038037, - "std": 0.02037, - "sem": 0.000533, + "count": 1464, + "mean": 0.042745, + "median": 0.037965, + "std": 0.020222, + "sem": 0.000529, "min": 0.01189, - "max": 0.154352, - "recall-0.025": 0.152115, - "recall-0.05": 0.738063, - "recall-0.1": 0.981583, - "recall-0.15": 0.997271, - "recall-0.25": 0.997954, - "recall-0.5": 0.997954, + "max": 0.147865, + "recall-0.025": 0.153479, + "recall-0.05": 0.742156, + "recall-0.1": 0.982265, + "recall-0.15": 0.998636, + "recall-0.25": 0.998636, + "recall-0.5": 0.998636, "num_labels": 1466, - "ap-0.025": 0.063073, - "ap-0.05": 0.663867, - "ap-0.1": 0.960176, - "ap-0.15": 0.982657, - "ap-0.25": 0.98331, - "ap-0.5": 0.98331 + "ap-0.025": 0.064846, + "ap-0.05": 0.664636, + "ap-0.1": 0.96085, + "ap-0.15": 0.983955, + "ap-0.25": 0.983955, + "ap-0.5": 0.983955 }, "nose": { - "count": 1461, - "mean": 0.016212, - "median": 0.012492, - "std": 0.01876, - "sem": 0.000491, + "count": 1462, + "mean": 0.016118, + "median": 0.012438, + "std": 0.019038, + "sem": 0.000498, "min": 0.001407, "max": 0.282177, - "recall-0.025": 0.899522, - "recall-0.05": 0.963773, + "recall-0.025": 0.900889, + "recall-0.05": 0.96514, "recall-0.1": 0.995215, "recall-0.15": 0.995899, - "recall-0.25": 0.995899, - "recall-0.5": 0.998633, + "recall-0.25": 0.996582, + "recall-0.5": 0.999316, "num_labels": 1463 }, "shoulder_left": { - "count": 1463, - "mean": 0.021455, - "median": 0.019023, - "std": 0.011436, - "sem": 0.000299, + "count": 1464, + "mean": 0.021387, + "median": 0.019035, + "std": 0.011197, + "sem": 0.000293, "min": 0.002677, "max": 0.095794, - "recall-0.025": 0.702592, - "recall-0.05": 0.975443, - "recall-0.1": 0.997954, - "recall-0.15": 0.997954, - "recall-0.25": 0.997954, - "recall-0.5": 0.997954, + "recall-0.025": 0.705321, + "recall-0.05": 0.97749, + "recall-0.1": 0.998636, + "recall-0.15": 0.998636, + "recall-0.25": 0.998636, + "recall-0.5": 0.998636, "num_labels": 1466 }, "shoulder_right": { - "count": 1462, - "mean": 0.022691, - "median": 0.019716, - "std": 0.014235, - "sem": 0.000372, + "count": 1463, + "mean": 0.022631, + "median": 0.019541, + "std": 0.014416, + "sem": 0.000377, "min": 0.00112, "max": 0.146597, - "recall-0.025": 0.711945, - "recall-0.05": 0.954949, + "recall-0.025": 0.714676, + "recall-0.05": 0.953584, "recall-0.1": 0.995904, - "recall-0.15": 0.997952, - "recall-0.25": 0.997952, - "recall-0.5": 0.997952, + "recall-0.15": 0.998635, + "recall-0.25": 0.998635, + "recall-0.5": 0.998635, "num_labels": 1465 }, "elbow_left": { - "count": 1462, - "mean": 0.026381, - "median": 0.017863, - "std": 0.028393, - "sem": 0.000743, - "min": 0.001846, + "count": 1463, + "mean": 0.02639, + "median": 0.017858, + "std": 0.02901, + "sem": 0.000759, + "min": 0.001259, "max": 0.323155, - "recall-0.025": 0.701706, - "recall-0.05": 0.880546, - "recall-0.1": 0.976792, - "recall-0.15": 0.985666, + "recall-0.025": 0.704437, + "recall-0.05": 0.882594, + "recall-0.1": 0.978157, + "recall-0.15": 0.986348, "recall-0.25": 0.996587, - "recall-0.5": 0.997952, + "recall-0.5": 0.998635, "num_labels": 1465 }, "elbow_right": { - "count": 1461, - "mean": 0.023727, - "median": 0.018347, - "std": 0.018642, - "sem": 0.000488, + "count": 1462, + "mean": 0.023401, + "median": 0.018196, + "std": 0.018141, + "sem": 0.000475, "min": 0.001407, "max": 0.219498, - "recall-0.025": 0.684211, - "recall-0.05": 0.920711, - "recall-0.1": 0.992481, - "recall-0.15": 0.997266, - "recall-0.25": 0.998633, - "recall-0.5": 0.998633, + "recall-0.025": 0.687628, + "recall-0.05": 0.926179, + "recall-0.1": 0.993848, + "recall-0.15": 0.997949, + "recall-0.25": 0.999316, + "recall-0.5": 0.999316, "num_labels": 1463 }, "wrist_left": { - "count": 1431, - "mean": 0.036984, - "median": 0.017216, - "std": 0.05458, - "sem": 0.001443, + "count": 1430, + "mean": 0.035377, + "median": 0.016959, + "std": 0.051088, + "sem": 0.001351, "min": 0.000713, "max": 0.397255, - "recall-0.025": 0.668759, - "recall-0.05": 0.816597, - "recall-0.1": 0.904463, - "recall-0.15": 0.948396, - "recall-0.25": 0.986053, - "recall-0.5": 0.997908, + "recall-0.025": 0.67643, + "recall-0.05": 0.824268, + "recall-0.1": 0.90795, + "recall-0.15": 0.954672, + "recall-0.25": 0.987448, + "recall-0.5": 0.997211, "num_labels": 1434 }, "wrist_right": { - "count": 1455, - "mean": 0.025496, - "median": 0.016754, - "std": 0.027589, - "sem": 0.000724, + "count": 1456, + "mean": 0.024355, + "median": 0.016346, + "std": 0.024627, + "sem": 0.000646, "min": 0.001905, - "max": 0.394822, - "recall-0.025": 0.68956, - "recall-0.05": 0.882555, - "recall-0.1": 0.970467, - "recall-0.15": 0.993132, - "recall-0.25": 0.99794, - "recall-0.5": 0.999313, + "max": 0.304244, + "recall-0.025": 0.70261, + "recall-0.05": 0.89217, + "recall-0.1": 0.979396, + "recall-0.15": 0.993819, + "recall-0.25": 0.999313, + "recall-0.5": 1.0, "num_labels": 1456 }, "hip_left": { - "count": 1462, - "mean": 0.059122, - "median": 0.053514, - "std": 0.029657, - "sem": 0.000776, + "count": 1463, + "mean": 0.059443, + "median": 0.053492, + "std": 0.030125, + "sem": 0.000788, "min": 0.005563, "max": 0.17979, "recall-0.025": 0.068259, - "recall-0.05": 0.423891, - "recall-0.1": 0.901706, - "recall-0.15": 0.969283, - "recall-0.25": 0.997952, - "recall-0.5": 0.997952, + "recall-0.05": 0.424573, + "recall-0.1": 0.893515, + "recall-0.15": 0.968601, + "recall-0.25": 0.998635, + "recall-0.5": 0.998635, "num_labels": 1465 }, "hip_right": { - "count": 1463, - "mean": 0.058599, - "median": 0.054934, - "std": 0.028583, - "sem": 0.000748, + "count": 1464, + "mean": 0.058641, + "median": 0.054688, + "std": 0.029005, + "sem": 0.000758, "min": 0.003776, "max": 0.304524, - "recall-0.025": 0.05457, - "recall-0.05": 0.412688, - "recall-0.1": 0.912688, - "recall-0.15": 0.984993, - "recall-0.25": 0.997271, - "recall-0.5": 0.997954, + "recall-0.025": 0.055252, + "recall-0.05": 0.417462, + "recall-0.1": 0.91337, + "recall-0.15": 0.985675, + "recall-0.25": 0.997954, + "recall-0.5": 0.998636, "num_labels": 1466 }, "knee_left": { - "count": 1462, - "mean": 0.051285, - "median": 0.042752, - "std": 0.04192, - "sem": 0.001097, + "count": 1463, + "mean": 0.0519, + "median": 0.042755, + "std": 0.042761, + "sem": 0.001118, "min": 0.001938, "max": 0.353024, - "recall-0.025": 0.185666, - "recall-0.05": 0.634812, - "recall-0.1": 0.931058, - "recall-0.15": 0.969966, + "recall-0.025": 0.186348, + "recall-0.05": 0.635495, + "recall-0.1": 0.927645, + "recall-0.15": 0.967235, "recall-0.25": 0.9843, - "recall-0.5": 0.997952, + "recall-0.5": 0.998635, "num_labels": 1465 }, "knee_right": { - "count": 1456, - "mean": 0.048348, - "median": 0.042693, - "std": 0.029301, - "sem": 0.000768, + "count": 1457, + "mean": 0.048275, + "median": 0.042583, + "std": 0.029088, + "sem": 0.000762, "min": 0.003542, - "max": 0.305725, - "recall-0.025": 0.210418, - "recall-0.05": 0.590816, + "max": 0.324496, + "recall-0.025": 0.214531, + "recall-0.05": 0.588759, "recall-0.1": 0.938999, - "recall-0.15": 0.991775, - "recall-0.25": 0.995888, - "recall-0.5": 0.997944, + "recall-0.15": 0.993146, + "recall-0.25": 0.997258, + "recall-0.5": 0.998629, "num_labels": 1459 }, "ankle_left": { - "count": 1453, - "mean": 0.083857, - "median": 0.037654, - "std": 0.106287, + "count": 1456, + "mean": 0.083562, + "median": 0.037346, + "std": 0.106393, "sem": 0.002789, "min": 0.002453, "max": 0.495773, - "recall-0.025": 0.360902, - "recall-0.05": 0.583732, - "recall-0.1": 0.726589, - "recall-0.15": 0.817498, - "recall-0.25": 0.910458, - "recall-0.5": 0.993165, + "recall-0.025": 0.361586, + "recall-0.05": 0.58715, + "recall-0.1": 0.729323, + "recall-0.15": 0.820232, + "recall-0.25": 0.913192, + "recall-0.5": 0.995215, "num_labels": 1463 }, "ankle_right": { - "count": 1446, - "mean": 0.077158, - "median": 0.033178, - "std": 0.103249, + "count": 1448, + "mean": 0.077433, + "median": 0.033338, + "std": 0.10331, "sem": 0.002716, "min": 0.001061, - "max": 0.493086, - "recall-0.025": 0.372603, - "recall-0.05": 0.685616, - "recall-0.1": 0.777397, - "recall-0.15": 0.815753, - "recall-0.25": 0.893151, - "recall-0.5": 0.990411, + "max": 0.4981, + "recall-0.025": 0.369863, + "recall-0.05": 0.686301, + "recall-0.1": 0.773288, + "recall-0.15": 0.815068, + "recall-0.25": 0.896575, + "recall-0.5": 0.991781, "num_labels": 1460 }, "joint_recalls": { "num_labels": 18990, - "recall-0.025": 0.48468, - "recall-0.05": 0.74771, - "recall-0.1": 0.92428, - "recall-0.15": 0.95856, - "recall-0.25": 0.98052, - "recall-0.5": 0.99695 + "recall-0.025": 0.48768, + "recall-0.05": 0.75034, + "recall-0.1": 0.9247, + "recall-0.15": 0.95935, + "recall-0.25": 0.98146, + "recall-0.5": 0.99774 } } { "total_parts": 20444, - "correct_parts": 19964, - "pcp": 0.976521 + "correct_parts": 19975, + "pcp": 0.977059 } ``` diff --git a/spt/triangulator.cpp b/spt/triangulator.cpp index d8caeab..77d062b 100644 --- a/spt/triangulator.cpp +++ b/spt/triangulator.cpp @@ -170,6 +170,14 @@ std::vector>> TriangulatorInternal::triangulat auto it = std::find(joint_names.begin(), joint_names.end(), joint); core_joint_idx.push_back(std::distance(joint_names.begin(), it)); } + std::vector> core_limbs_idx; + for (const auto &limb : core_limbs) + { + auto it1 = std::find(core_joints.begin(), core_joints.end(), limb.first); + auto it2 = std::find(core_joints.begin(), core_joints.end(), limb.second); + core_limbs_idx.push_back({(size_t)std::distance(core_joints.begin(), it1), + (size_t)std::distance(core_joints.begin(), it2)}); + } // Undistort 2D poses for (size_t i = 0; i < cameras.size(); ++i) @@ -236,7 +244,8 @@ std::vector>> TriangulatorInternal::triangulat } // Calculate score - double score = calc_pose_score(pose_core, last_pose, dists[j], internal_cameras[i]); + double score = calc_pose_score( + pose_core, last_pose, dists[j], internal_cameras[i]); if (score > threshold) { scored_pasts[i][j].push_back(k); @@ -325,7 +334,8 @@ std::vector>> TriangulatorInternal::triangulat } // Triangulate and score - auto [pose3d, score] = triangulate_and_score(pose1_core, pose2_core, cam1, cam2, roomparams); + auto [pose3d, score] = triangulate_and_score( + pose1_core, pose2_core, cam1, cam2, roomparams, core_limbs_idx); all_scored_poses.emplace_back(pose3d, score); } @@ -362,7 +372,7 @@ std::vector>> TriangulatorInternal::triangulat const auto &pose1 = poses_2d_mats[std::get<0>(pids)][std::get<2>(pids)]; const auto &pose2 = poses_2d_mats[std::get<1>(pids)][std::get<3>(pids)]; - auto [pose3d, score] = triangulate_and_score(pose1, pose2, cam1, cam2, roomparams); + auto [pose3d, score] = triangulate_and_score(pose1, pose2, cam1, cam2, roomparams, {}); all_full_poses.push_back(pose3d); } @@ -451,13 +461,7 @@ void TriangulatorInternal::undistort_poses(std::vector &poses, CameraIn int num_joints = poses[p].rows; // Extract the (x, y) coordinates - std::vector dims = {num_joints, 2}; - cv::Mat points = cv::Mat(dims, CV_64F); - for (int j = 0; j < num_joints; ++j) - { - points.at(j, 0) = poses[p].at(j, 0); - points.at(j, 1) = poses[p].at(j, 1); - } + cv::Mat points = poses[p].colRange(0, 2).clone(); // Undistort the points cv::Mat newK = undistort_points(points, icam); @@ -508,14 +512,7 @@ std::tuple, std::vector> TriangulatorInternal::pro size_t num_joints = body3D.size[0]; // Split up vector - std::vector dimsA = {(int)num_joints, 3}; - cv::Mat points3d = cv::Mat(dimsA, CV_64F); - for (size_t i = 0; i < num_joints; ++i) - { - points3d.at(i, 0) = body3D.at(i, 0); - points3d.at(i, 1) = body3D.at(i, 1); - points3d.at(i, 2) = body3D.at(i, 2); - } + cv::Mat points3d = body3D.colRange(0, 3).clone(); // Project from world to camera coordinate system cv::Mat T_repeated = cv::repeat(icam.T, 1, num_joints); @@ -609,15 +606,36 @@ double TriangulatorInternal::calc_pose_score( double iscale = (icam.cam.width + icam.cam.height) / 2; cv::Mat scores = score_projection(pose1, pose2, dist1, mask, iscale); - double score = 0.0; + // Drop lowest scores + size_t drop_k = static_cast(pose1.rows * 0.2); + size_t min_k = 3; + std::vector scores_vec; for (int i = 0; i < scores.rows; ++i) { if (mask.at(i) > 0) { - score += scores.at(i); + scores_vec.push_back(scores.at(i)); } } - score /= valid_count; + std::sort(scores_vec.begin(), scores_vec.end()); + drop_k = (scores_vec.size() >= min_k) ? std::min(drop_k, scores_vec.size() - min_k) : 0; + if (drop_k > 0) + { + scores_vec.erase(scores_vec.begin(), scores_vec.begin() + drop_k); + } + + // Calculate final score + double score = 0.0; + size_t items = 0; + for (size_t i = 0; i < scores_vec.size(); i++) + { + score += scores_vec[i]; + items++; + } + if (items > 0) + { + score /= (double)items; + } return score; } @@ -635,8 +653,7 @@ cv::Mat TriangulatorInternal::score_projection( // Calculate error cv::Mat diff = pose1.colRange(0, 2) - repro1.colRange(0, 2); - cv::Mat error1; - error1 = diff.mul(diff); + cv::Mat error1 = diff.mul(diff); cv::reduce(error1, error1, 1, cv::REDUCE_SUM, CV_64F); cv::sqrt(error1, error1); error1.setTo(0.0, ~mask); @@ -665,7 +682,8 @@ std::pair TriangulatorInternal::triangulate_and_score( const cv::Mat &pose2, const CameraInternal &cam1, const CameraInternal &cam2, - const std::array, 2> &roomparams) + const std::array, 2> &roomparams, + const std::vector> &core_limbs_idx) { // Mask out invisible joints double min_score = 0.1; @@ -720,10 +738,8 @@ std::pair TriangulatorInternal::triangulate_and_score( } } - // Check if points are inside the room bounds + // Check if mean of the points is inside the room bounds std::array mean = {0, 0, 0}; - std::array mins = {std::numeric_limits::max(), std::numeric_limits::max(), std::numeric_limits::max()}; - std::array maxs = {std::numeric_limits::lowest(), std::numeric_limits::lowest(), std::numeric_limits::lowest()}; for (int i = 0; i < pose1.rows; ++i) { if (mask.at(i) > 0) @@ -731,8 +747,6 @@ std::pair TriangulatorInternal::triangulate_and_score( for (int j = 0; j < 3; ++j) { mean[j] += pose3d.at(i, j); - mins[j] = std::min(mins[j], pose3d.at(i, j)); - maxs[j] = std::max(maxs[j], pose3d.at(i, j)); } } } @@ -740,15 +754,12 @@ std::pair TriangulatorInternal::triangulate_and_score( { mean[j] /= num_visible; } - double wdist = 0.1; std::array center = roomparams[0]; std::array size = roomparams[1]; for (int i = 0; i < 3; ++i) { if (mean[i] > center[i] + size[i] / 2 || - mean[i] < center[i] - size[i] / 2 || - maxs[i] > center[i] + size[i] / 2 + wdist || - mins[i] < center[i] - size[i] / 2 - wdist) + mean[i] < center[i] - size[i] / 2) { // Very low score if outside room for (int j = 0; j < pose1.rows; ++j) @@ -786,22 +797,77 @@ std::pair TriangulatorInternal::triangulate_and_score( } } - // Drop lowest scores - int drop_k = static_cast(pose1.rows * 0.2); - std::vector scores_vec; + // Set scores outside the room to a low value + double wdist = 0.1; for (int i = 0; i < pose1.rows; ++i) { if (mask.at(i) > 0) { - scores_vec.push_back(scores.at(i)); + for (int j = 0; j < 3; ++j) + { + if (pose3d.at(i, j) > center[j] + size[j] / 2 + wdist || + pose3d.at(i, j) < center[j] - size[j] / 2 - wdist) + { + pose3d.at(i, 3) = 0.001; + } + } + } + } + + // Filter clearly wrong limbs + if (core_limbs_idx.size() > 0) + { + double max_length = 0.9; + std::vector invalid_joints; + for (size_t i = 0; i < core_limbs_idx.size(); ++i) + { + auto limb = core_limbs_idx[i]; + if (pose3d.at(limb[0], 3) > min_score && + pose3d.at(limb[1], 3) > min_score) + { + cv::Point3d p1 = cv::Point3d( + pose3d.at(limb[0], 0), + pose3d.at(limb[0], 1), + pose3d.at(limb[0], 2)); + cv::Point3d p2 = cv::Point3d( + pose3d.at(limb[1], 0), + pose3d.at(limb[1], 1), + pose3d.at(limb[1], 2)); + double length = cv::norm(p1 - p2); + if (length > max_length) + { + invalid_joints.push_back(limb[1]); + } + } + } + for (size_t i = 0; i < invalid_joints.size(); ++i) + { + pose3d.at(invalid_joints[i], 3) = 0.001; + } + } + + // Drop lowest scores + size_t drop_k = static_cast(pose1.rows * 0.2); + size_t min_k = 3; + std::vector scores_vec; + for (int i = 0; i < pose1.rows; ++i) + { + if (pose3d.at(i, 3) > min_score) + { + scores_vec.push_back(pose3d.at(i, 3)); } } std::sort(scores_vec.begin(), scores_vec.end()); + drop_k = (scores_vec.size() >= min_k) ? std::min(drop_k, scores_vec.size() - min_k) : 0; + if (drop_k > 0) + { + scores_vec.erase(scores_vec.begin(), scores_vec.begin() + drop_k); + } // Calculate final score double score = 0.0; size_t items = 0; - for (size_t i = drop_k; i < scores_vec.size(); i++) + for (size_t i = 0; i < scores_vec.size(); i++) { score += scores_vec[i]; items++; @@ -823,13 +889,13 @@ std::vector>> TriangulatorInte { double max_center_dist = 0.6; double max_joint_avg_dist = 0.3; - size_t num_joints = all_scored_poses[0].first.rows; // Calculate pose centers std::vector centers; for (size_t i = 0; i < all_pairs.size(); ++i) { const cv::Mat &pose_3d = all_scored_poses[i].first; + size_t num_joints = pose_3d.rows; cv::Point3d center(0, 0, 0); size_t num_valid = 0; for (size_t j = 0; j < num_joints; ++j) @@ -857,6 +923,7 @@ std::vector>> TriangulatorInte for (size_t i = 0; i < all_pairs.size(); ++i) { const cv::Mat &pose_3d = all_scored_poses[i].first; + size_t num_joints = pose_3d.rows; const cv::Point3d ¢er = centers[i]; double best_dist = std::numeric_limits::infinity(); int best_group = -1; @@ -874,10 +941,17 @@ std::vector>> TriangulatorInte std::vector dists; for (size_t row = 0; row < num_joints; row++) { - if (pose_3d.at(row, 3) > min_score && group_pose.at(row, 3) > min_score) + if (pose_3d.at(row, 3) > min_score && + group_pose.at(row, 3) > min_score) { - cv::Point3d p1(pose_3d.at(row, 0), pose_3d.at(row, 1), pose_3d.at(row, 2)); - cv::Point3d p2(group_pose.at(row, 0), group_pose.at(row, 1), group_pose.at(row, 2)); + cv::Point3d p1 = cv::Point3d( + pose_3d.at(row, 0), + pose_3d.at(row, 1), + pose_3d.at(row, 2)); + cv::Point3d p2 = cv::Point3d( + group_pose.at(row, 0), + group_pose.at(row, 1), + group_pose.at(row, 2)); dists.push_back(cv::norm(p1 - p2)); } } @@ -917,15 +991,24 @@ std::vector>> TriangulatorInte for (size_t row = 0; row < num_joints; row++) { - new_pose.at(row, 0) = (group_pose.at(row, 0) * n_elems + pose_3d.at(row, 0)) / (n_elems + 1); - new_pose.at(row, 1) = (group_pose.at(row, 1) * n_elems + pose_3d.at(row, 1)) / (n_elems + 1); - new_pose.at(row, 2) = (group_pose.at(row, 2) * n_elems + pose_3d.at(row, 2)) / (n_elems + 1); - new_pose.at(row, 3) = (group_pose.at(row, 3) * n_elems + pose_3d.at(row, 3)) / (n_elems + 1); + new_pose.at(row, 0) = + (group_pose.at(row, 0) * n_elems + pose_3d.at(row, 0)) / + (n_elems + 1); + new_pose.at(row, 1) = + (group_pose.at(row, 1) * n_elems + pose_3d.at(row, 1)) / + (n_elems + 1); + new_pose.at(row, 2) = + (group_pose.at(row, 2) * n_elems + pose_3d.at(row, 2)) / + (n_elems + 1); + new_pose.at(row, 3) = + (group_pose.at(row, 3) * n_elems + pose_3d.at(row, 3)) / + (n_elems + 1); } group_pose = new_pose; group_indices.push_back(i); } } + return groups; } @@ -970,7 +1053,10 @@ cv::Mat TriangulatorInternal::merge_group(const std::vector &poses_3d, if (initial_pose_3d.at(j, 3) > 0.0) { jmask.at(j) = 1; - center += cv::Point3d(initial_pose_3d.at(j, 0), initial_pose_3d.at(j, 1), initial_pose_3d.at(j, 2)); + center += cv::Point3d( + initial_pose_3d.at(j, 0), + initial_pose_3d.at(j, 1), + initial_pose_3d.at(j, 2)); valid_joints++; } } @@ -997,8 +1083,14 @@ cv::Mat TriangulatorInternal::merge_group(const std::vector &poses_3d, { for (int j = 0; j < num_joints; ++j) { - cv::Point3d joint_i(poses_3d[i].at(j, 0), poses_3d[i].at(j, 1), poses_3d[i].at(j, 2)); - cv::Point3d joint_initial(initial_pose_3d.at(j, 0), initial_pose_3d.at(j, 1), initial_pose_3d.at(j, 2)); + cv::Point3d joint_i = cv::Point3d( + poses_3d[i].at(j, 0), + poses_3d[i].at(j, 1), + poses_3d[i].at(j, 2)); + cv::Point3d joint_initial = cv::Point3d( + initial_pose_3d.at(j, 0), + initial_pose_3d.at(j, 1), + initial_pose_3d.at(j, 2)); double distance = cv::norm(joint_i - joint_initial); distances.at(i, j) = distance; if (distance <= max_dist && poses_3d[i].at(j, 3) > (min_score - offset)) diff --git a/spt/triangulator.hpp b/spt/triangulator.hpp index 0987430..d66bea8 100644 --- a/spt/triangulator.hpp +++ b/spt/triangulator.hpp @@ -57,6 +57,16 @@ private: "ankle_left", "ankle_right", }; + const std::vector> core_limbs = { + {"knee_left", "ankle_left"}, + {"hip_left", "knee_left"}, + {"hip_right", "knee_right"}, + {"knee_right", "ankle_right"}, + {"elbow_left", "wrist_left"}, + {"elbow_right", "wrist_right"}, + {"shoulder_left", "elbow_left"}, + {"shoulder_right", "elbow_right"}, + }; std::vector last_poses_3d; @@ -84,7 +94,8 @@ private: const cv::Mat &pose2, const CameraInternal &cam1, const CameraInternal &cam2, - const std::array, 2> &roomparams); + const std::array, 2> &roomparams, + const std::vector> &core_limbs_idx); std::vector>> calc_grouping( const std::vector, std::pair>> &all_pairs,