diff --git a/media/RESULTS.md b/media/RESULTS.md index b88d55a..226a858 100644 --- a/media/RESULTS.md +++ b/media/RESULTS.md @@ -7,9 +7,9 @@ Results of the model in various experiments on different datasets. (duration 00:01:20) ```json { - "avg_time_2d": 0.0706314975932493, - "avg_time_3d": 0.002891659332534014, - "avg_fps": 13.6011569934277 + "avg_time_2d": 0.07194638131028515, + "avg_time_3d": 0.0031490806805885446, + "avg_fps": 13.316383886439496 } { "person_nums": { @@ -30,7 +30,7 @@ Results of the model in various experiments on different datasets. "count": 600, "mean": 0.066878, "median": 0.059221, - "std": 0.027766, + "std": 0.027763, "sem": 0.001134, "min": 0.043707, "max": 0.189867, @@ -42,23 +42,23 @@ Results of the model in various experiments on different datasets. "recall-0.5": 1.0, "num_labels": 600, "ap-0.025": 0.0, - "ap-0.05": 0.00225, - "ap-0.1": 0.895105, - "ap-0.15": 0.917278, + "ap-0.05": 0.002245, + "ap-0.1": 0.895107, + "ap-0.15": 0.917238, "ap-0.25": 1.0, "ap-0.5": 1.0 }, "nose": { "count": 600, - "mean": 0.117723, - "median": 0.101523, - "std": 0.04276, - "sem": 0.001747, + "mean": 0.117653, + "median": 0.101455, + "std": 0.042711, + "sem": 0.001745, "min": 0.046466, "max": 0.271134, "recall-0.025": 0.0, "recall-0.05": 0.003333, - "recall-0.1": 0.485, + "recall-0.1": 0.486667, "recall-0.15": 0.803333, "recall-0.25": 0.995, "recall-0.5": 1.0, @@ -66,9 +66,9 @@ Results of the model in various experiments on different datasets. }, "shoulder_left": { "count": 600, - "mean": 0.033334, + "mean": 0.033344, "median": 0.02547, - "std": 0.032213, + "std": 0.032209, "sem": 0.001316, "min": 0.003037, "max": 0.181945, @@ -82,9 +82,9 @@ Results of the model in various experiments on different datasets. }, "shoulder_right": { "count": 600, - "mean": 0.048002, - "median": 0.033262, - "std": 0.04254, + "mean": 0.048035, + "median": 0.033333, + "std": 0.04253, "sem": 0.001738, "min": 0.003882, "max": 0.247363, @@ -98,14 +98,14 @@ Results of the model in various experiments on different datasets. }, "elbow_left": { "count": 600, - "mean": 0.042568, + "mean": 0.042537, "median": 0.034439, - "std": 0.034643, - "sem": 0.001415, + "std": 0.0346, + "sem": 0.001414, "min": 0.002682, "max": 0.193558, "recall-0.025": 0.273333, - "recall-0.05": 0.818333, + "recall-0.05": 0.82, "recall-0.1": 0.943333, "recall-0.15": 0.956667, "recall-0.25": 1.0, @@ -114,14 +114,14 @@ Results of the model in various experiments on different datasets. }, "elbow_right": { "count": 600, - "mean": 0.04321, - "median": 0.03319, - "std": 0.035166, + "mean": 0.043251, + "median": 0.033208, + "std": 0.035165, "sem": 0.001437, "min": 0.00667, "max": 0.285024, "recall-0.025": 0.251667, - "recall-0.05": 0.811667, + "recall-0.05": 0.81, "recall-0.1": 0.935, "recall-0.15": 0.946667, "recall-0.25": 0.998333, @@ -130,12 +130,12 @@ Results of the model in various experiments on different datasets. }, "wrist_left": { "count": 600, - "mean": 0.040636, + "mean": 0.040583, "median": 0.024203, - "std": 0.042374, - "sem": 0.001731, + "std": 0.04223, + "sem": 0.001725, "min": 0.001517, - "max": 0.18543, + "max": 0.184684, "recall-0.025": 0.516667, "recall-0.05": 0.746667, "recall-0.1": 0.908333, @@ -146,14 +146,14 @@ Results of the model in various experiments on different datasets. }, "wrist_right": { "count": 600, - "mean": 0.042567, + "mean": 0.042641, "median": 0.025715, - "std": 0.046868, - "sem": 0.001915, + "std": 0.046903, + "sem": 0.001916, "min": 0.001698, "max": 0.425617, "recall-0.025": 0.48, - "recall-0.05": 0.773333, + "recall-0.05": 0.771667, "recall-0.1": 0.896667, "recall-0.15": 0.921667, "recall-0.25": 0.998333, @@ -162,9 +162,9 @@ Results of the model in various experiments on different datasets. }, "hip_left": { "count": 600, - "mean": 0.08963, + "mean": 0.089628, "median": 0.086723, - "std": 0.03297, + "std": 0.032971, "sem": 0.001347, "min": 0.01535, "max": 0.236764, @@ -178,10 +178,10 @@ Results of the model in various experiments on different datasets. }, "hip_right": { "count": 600, - "mean": 0.114502, + "mean": 0.11452, "median": 0.114981, - "std": 0.026108, - "sem": 0.001067, + "std": 0.026088, + "sem": 0.001066, "min": 0.046173, "max": 0.234935, "recall-0.025": 0.0, @@ -194,9 +194,9 @@ Results of the model in various experiments on different datasets. }, "knee_left": { "count": 600, - "mean": 0.060821, + "mean": 0.060824, "median": 0.045004, - "std": 0.061355, + "std": 0.061352, "sem": 0.002507, "min": 0.01946, "max": 0.425374, @@ -210,14 +210,14 @@ Results of the model in various experiments on different datasets. }, "knee_right": { "count": 600, - "mean": 0.051083, - "median": 0.043279, - "std": 0.034967, + "mean": 0.051047, + "median": 0.043249, + "std": 0.034972, "sem": 0.001429, "min": 0.017082, "max": 0.259899, "recall-0.025": 0.038333, - "recall-0.05": 0.733333, + "recall-0.05": 0.735, "recall-0.1": 0.943333, "recall-0.15": 0.945, "recall-0.25": 0.998333, @@ -226,7 +226,7 @@ Results of the model in various experiments on different datasets. }, "ankle_left": { "count": 598, - "mean": 0.098277, + "mean": 0.098282, "median": 0.086303, "std": 0.049778, "sem": 0.002037, @@ -242,9 +242,9 @@ Results of the model in various experiments on different datasets. }, "ankle_right": { "count": 600, - "mean": 0.085551, + "mean": 0.085558, "median": 0.068594, - "std": 0.05855, + "std": 0.058547, "sem": 0.002392, "min": 0.032987, "max": 0.463238, @@ -259,8 +259,8 @@ Results of the model in various experiments on different datasets. "joint_recalls": { "num_labels": 7800, "recall-0.025": 0.17962, - "recall-0.05": 0.47577, - "recall-0.1": 0.82141, + "recall-0.05": 0.47603, + "recall-0.1": 0.82154, "recall-0.15": 0.92923, "recall-0.25": 0.99397, "recall-0.5": 0.99974 @@ -278,33 +278,33 @@ Results of the model in various experiments on different datasets. (duration 00:00:56) ```json { - "avg_time_2d": 0.10373067446181045, - "avg_time_3d": 0.01558385436067876, - "avg_fps": 8.381208976551004 + "avg_time_2d": 0.10030525082984741, + "avg_time_3d": 0.00994173521848069, + "avg_fps": 9.07054274990917 } { "person_nums": { "total_frames": 301, "total_labels": 477, - "total_preds": 909, + "total_preds": 882, "considered_empty": 3, "valid_preds": 477, - "invalid_preds": 429, + "invalid_preds": 402, "missing": 0, - "invalid_fraction": 0.47351, - "precision": 0.52649, + "invalid_fraction": 0.45734, + "precision": 0.54266, "recall": 1.0, - "f1": 0.6898, - "non_empty": 906 + "f1": 0.70354, + "non_empty": 879 }, "mpjpe": { "count": 477, - "mean": 0.04826, + "mean": 0.048235, "median": 0.04301, - "std": 0.014673, - "sem": 0.000673, + "std": 0.014642, + "sem": 0.000671, "min": 0.029586, - "max": 0.115693, + "max": 0.106122, "recall-0.025": 0.0, "recall-0.05": 0.675052, "recall-0.1": 0.991614, @@ -313,23 +313,23 @@ Results of the model in various experiments on different datasets. "recall-0.5": 1.0, "num_labels": 477, "ap-0.025": 0.0, - "ap-0.05": 0.340336, - "ap-0.1": 0.692986, - "ap-0.15": 0.703021, - "ap-0.25": 0.703021, - "ap-0.5": 0.703021 + "ap-0.05": 0.340912, + "ap-0.1": 0.6919, + "ap-0.15": 0.701795, + "ap-0.25": 0.701795, + "ap-0.5": 0.701795 }, "head": { "count": 477, - "mean": 0.055426, - "median": 0.049823, - "std": 0.026032, - "sem": 0.001193, + "mean": 0.055384, + "median": 0.049788, + "std": 0.025996, + "sem": 0.001192, "min": 0.006374, "max": 0.147303, - "recall-0.025": 0.077568, - "recall-0.05": 0.503145, - "recall-0.1": 0.914046, + "recall-0.025": 0.075472, + "recall-0.05": 0.505241, + "recall-0.1": 0.91195, "recall-0.15": 1.0, "recall-0.25": 1.0, "recall-0.5": 1.0, @@ -353,13 +353,13 @@ Results of the model in various experiments on different datasets. }, "shoulder_right": { "count": 477, - "mean": 0.05039, + "mean": 0.050381, "median": 0.045185, - "std": 0.02441, - "sem": 0.001119, + "std": 0.024432, + "sem": 0.00112, "min": 0.00639, "max": 0.15065, - "recall-0.025": 0.109015, + "recall-0.025": 0.106918, "recall-0.05": 0.557652, "recall-0.1": 0.947589, "recall-0.15": 0.997904, @@ -369,30 +369,30 @@ Results of the model in various experiments on different datasets. }, "elbow_left": { "count": 477, - "mean": 0.04107, + "mean": 0.040845, "median": 0.034194, - "std": 0.026964, - "sem": 0.001236, + "std": 0.025928, + "sem": 0.001188, "min": 0.005533, - "max": 0.210838, + "max": 0.152226, "recall-0.025": 0.301887, "recall-0.05": 0.746331, - "recall-0.1": 0.955975, - "recall-0.15": 0.995807, + "recall-0.1": 0.958071, + "recall-0.15": 0.997904, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 477 }, "elbow_right": { "count": 477, - "mean": 0.053503, - "median": 0.044068, - "std": 0.041992, - "sem": 0.001925, + "mean": 0.053498, + "median": 0.043749, + "std": 0.042037, + "sem": 0.001927, "min": 0.002858, "max": 0.244706, "recall-0.025": 0.259958, - "recall-0.05": 0.559748, + "recall-0.05": 0.561845, "recall-0.1": 0.888889, "recall-0.15": 0.958071, "recall-0.25": 1.0, @@ -401,10 +401,10 @@ Results of the model in various experiments on different datasets. }, "wrist_left": { "count": 477, - "mean": 0.060714, + "mean": 0.060633, "median": 0.053257, - "std": 0.043226, - "sem": 0.001981, + "std": 0.043196, + "sem": 0.00198, "min": 0.003745, "max": 0.359369, "recall-0.025": 0.12369, @@ -417,9 +417,9 @@ Results of the model in various experiments on different datasets. }, "wrist_right": { "count": 477, - "mean": 0.057941, + "mean": 0.057929, "median": 0.053722, - "std": 0.029951, + "std": 0.029957, "sem": 0.001373, "min": 0.009013, "max": 0.202256, @@ -433,31 +433,31 @@ Results of the model in various experiments on different datasets. }, "hip_left": { "count": 477, - "mean": 0.048053, + "mean": 0.048001, "median": 0.042849, - "std": 0.027148, - "sem": 0.001244, + "std": 0.026773, + "sem": 0.001227, "min": 0.005757, - "max": 0.193725, + "max": 0.140667, "recall-0.025": 0.213836, "recall-0.05": 0.618449, - "recall-0.1": 0.951782, - "recall-0.15": 0.997904, + "recall-0.1": 0.949686, + "recall-0.15": 1.0, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 477 }, "hip_right": { "count": 477, - "mean": 0.057889, + "mean": 0.057906, "median": 0.056146, - "std": 0.025477, + "std": 0.025472, "sem": 0.001168, "min": 0.005189, "max": 0.136207, "recall-0.025": 0.109015, - "recall-0.05": 0.417191, - "recall-0.1": 0.932914, + "recall-0.05": 0.419287, + "recall-0.1": 0.930818, "recall-0.15": 1.0, "recall-0.25": 1.0, "recall-0.5": 1.0, @@ -465,14 +465,14 @@ Results of the model in various experiments on different datasets. }, "knee_left": { "count": 477, - "mean": 0.039815, + "mean": 0.039986, "median": 0.037315, - "std": 0.024224, - "sem": 0.00111, + "std": 0.02438, + "sem": 0.001117, "min": 0.005407, "max": 0.195502, - "recall-0.025": 0.262055, - "recall-0.05": 0.765199, + "recall-0.025": 0.259958, + "recall-0.05": 0.763103, "recall-0.1": 0.974843, "recall-0.15": 0.989518, "recall-0.25": 1.0, @@ -481,15 +481,15 @@ Results of the model in various experiments on different datasets. }, "knee_right": { "count": 477, - "mean": 0.039432, - "median": 0.034665, - "std": 0.02349, - "sem": 0.001077, + "mean": 0.039374, + "median": 0.034816, + "std": 0.02337, + "sem": 0.001071, "min": 0.006865, "max": 0.186559, "recall-0.025": 0.29979, "recall-0.05": 0.727463, - "recall-0.1": 0.976939, + "recall-0.1": 0.979036, "recall-0.15": 0.997904, "recall-0.25": 1.0, "recall-0.5": 1.0, @@ -497,10 +497,10 @@ Results of the model in various experiments on different datasets. }, "ankle_left": { "count": 477, - "mean": 0.037263, + "mean": 0.037256, "median": 0.031, - "std": 0.030843, - "sem": 0.001414, + "std": 0.030824, + "sem": 0.001413, "min": 0.001693, "max": 0.224454, "recall-0.025": 0.433962, @@ -513,10 +513,10 @@ Results of the model in various experiments on different datasets. }, "ankle_right": { "count": 477, - "mean": 0.041461, + "mean": 0.041445, "median": 0.031787, - "std": 0.038055, - "sem": 0.001744, + "std": 0.038031, + "sem": 0.001743, "min": 0.005439, "max": 0.298397, "recall-0.025": 0.312369, @@ -529,18 +529,18 @@ Results of the model in various experiments on different datasets. }, "joint_recalls": { "num_labels": 6201, - "recall-0.025": 0.21109, - "recall-0.05": 0.61974, - "recall-0.1": 0.93807, - "recall-0.15": 0.98694, + "recall-0.025": 0.21045, + "recall-0.05": 0.62006, + "recall-0.1": 0.93824, + "recall-0.15": 0.98742, "recall-0.25": 0.99871, "recall-0.5": 1.0 } } { "total_parts": 6678, - "correct_parts": 6615, - "pcp": 0.990566 + "correct_parts": 6618, + "pcp": 0.991015 } ``` @@ -549,53 +549,53 @@ Results of the model in various experiments on different datasets. (duration 00:00:29) ```json { - "avg_time_2d": 0.059855266562047996, - "avg_time_3d": 0.004884123802185059, - "avg_fps": 15.446546443731663 + "avg_time_2d": 0.056824864081616674, + "avg_time_3d": 0.0035331103036988455, + "avg_fps": 16.56781908577916 } { "person_nums": { "total_frames": 222, "total_labels": 376, - "total_preds": 481, - "considered_empty": 6, + "total_preds": 466, + "considered_empty": 3, "valid_preds": 376, - "invalid_preds": 99, + "invalid_preds": 87, "missing": 0, - "invalid_fraction": 0.20842, - "precision": 0.79158, + "invalid_fraction": 0.1879, + "precision": 0.8121, "recall": 1.0, - "f1": 0.88367, - "non_empty": 475 + "f1": 0.89631, + "non_empty": 463 }, "mpjpe": { "count": 376, - "mean": 0.078225, - "median": 0.075936, - "std": 0.018756, - "sem": 0.000969, + "mean": 0.0776, + "median": 0.075327, + "std": 0.017994, + "sem": 0.000929, "min": 0.048684, "max": 0.158395, "recall-0.025": 0.0, "recall-0.05": 0.010638, - "recall-0.1": 0.893617, + "recall-0.1": 0.906915, "recall-0.15": 0.989362, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 376, "ap-0.025": 0.0, - "ap-0.05": 0.000329, - "ap-0.1": 0.769608, - "ap-0.15": 0.880589, - "ap-0.25": 0.88912, - "ap-0.5": 0.88912 + "ap-0.05": 0.000331, + "ap-0.1": 0.795398, + "ap-0.15": 0.888884, + "ap-0.25": 0.897608, + "ap-0.5": 0.897608 }, "head": { "count": 376, - "mean": 0.059335, - "median": 0.06095, - "std": 0.025332, - "sem": 0.001308, + "mean": 0.059377, + "median": 0.060875, + "std": 0.025299, + "sem": 0.001306, "min": 0.006936, "max": 0.161995, "recall-0.025": 0.098404, @@ -608,13 +608,13 @@ Results of the model in various experiments on different datasets. }, "shoulder_left": { "count": 376, - "mean": 0.061364, - "median": 0.056692, - "std": 0.028308, - "sem": 0.001462, + "mean": 0.061281, + "median": 0.056412, + "std": 0.028295, + "sem": 0.001461, "min": 0.003738, "max": 0.154144, - "recall-0.025": 0.050532, + "recall-0.025": 0.053191, "recall-0.05": 0.401596, "recall-0.1": 0.906915, "recall-0.15": 0.989362, @@ -624,10 +624,10 @@ Results of the model in various experiments on different datasets. }, "shoulder_right": { "count": 376, - "mean": 0.063608, - "median": 0.057984, - "std": 0.026836, - "sem": 0.001386, + "mean": 0.063652, + "median": 0.058472, + "std": 0.026828, + "sem": 0.001385, "min": 0.016753, "max": 0.185574, "recall-0.025": 0.039894, @@ -640,14 +640,14 @@ Results of the model in various experiments on different datasets. }, "elbow_left": { "count": 376, - "mean": 0.085619, - "median": 0.069926, - "std": 0.047256, - "sem": 0.00244, + "mean": 0.085614, + "median": 0.069707, + "std": 0.047296, + "sem": 0.002442, "min": 0.007605, "max": 0.205639, "recall-0.025": 0.029255, - "recall-0.05": 0.265957, + "recall-0.05": 0.268617, "recall-0.1": 0.643617, "recall-0.15": 0.864362, "recall-0.25": 1.0, @@ -656,10 +656,10 @@ Results of the model in various experiments on different datasets. }, "elbow_right": { "count": 376, - "mean": 0.070277, - "median": 0.060336, - "std": 0.035319, - "sem": 0.001824, + "mean": 0.07041, + "median": 0.060437, + "std": 0.035305, + "sem": 0.001823, "min": 0.00548, "max": 0.237233, "recall-0.025": 0.023936, @@ -672,10 +672,10 @@ Results of the model in various experiments on different datasets. }, "wrist_left": { "count": 376, - "mean": 0.114624, + "mean": 0.114571, "median": 0.099655, - "std": 0.062815, - "sem": 0.003244, + "std": 0.062882, + "sem": 0.003247, "min": 0.010314, "max": 0.431497, "recall-0.025": 0.013298, @@ -688,31 +688,31 @@ Results of the model in various experiments on different datasets. }, "wrist_right": { "count": 376, - "mean": 0.097785, + "mean": 0.097637, "median": 0.08983, - "std": 0.042341, - "sem": 0.002186, + "std": 0.042502, + "sem": 0.002195, "min": 0.014228, "max": 0.309971, "recall-0.025": 0.007979, - "recall-0.05": 0.098404, + "recall-0.05": 0.101064, "recall-0.1": 0.603723, - "recall-0.15": 0.87766, - "recall-0.25": 0.99734, + "recall-0.15": 0.880319, + "recall-0.25": 0.994681, "recall-0.5": 1.0, "num_labels": 376 }, "hip_left": { "count": 376, - "mean": 0.081185, - "median": 0.082897, - "std": 0.029202, - "sem": 0.001508, + "mean": 0.080917, + "median": 0.082729, + "std": 0.028998, + "sem": 0.001497, "min": 0.018902, "max": 0.179369, "recall-0.025": 0.029255, "recall-0.05": 0.170213, - "recall-0.1": 0.715426, + "recall-0.1": 0.723404, "recall-0.15": 0.994681, "recall-0.25": 1.0, "recall-0.5": 1.0, @@ -720,15 +720,15 @@ Results of the model in various experiments on different datasets. }, "hip_right": { "count": 376, - "mean": 0.086746, - "median": 0.083762, - "std": 0.025452, - "sem": 0.001314, + "mean": 0.086686, + "median": 0.083672, + "std": 0.02543, + "sem": 0.001313, "min": 0.024343, "max": 0.207002, "recall-0.025": 0.00266, "recall-0.05": 0.058511, - "recall-0.1": 0.75266, + "recall-0.1": 0.755319, "recall-0.15": 0.973404, "recall-0.25": 1.0, "recall-0.5": 1.0, @@ -736,82 +736,82 @@ Results of the model in various experiments on different datasets. }, "knee_left": { "count": 376, - "mean": 0.055947, - "median": 0.05234, - "std": 0.027994, - "sem": 0.001446, + "mean": 0.05488, + "median": 0.05203, + "std": 0.025318, + "sem": 0.001307, "min": 0.007197, - "max": 0.228932, + "max": 0.15096, "recall-0.025": 0.082447, - "recall-0.05": 0.465426, - "recall-0.1": 0.933511, - "recall-0.15": 0.989362, + "recall-0.05": 0.468085, + "recall-0.1": 0.941489, + "recall-0.15": 0.99734, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 376 }, "knee_right": { "count": 376, - "mean": 0.05944, - "median": 0.056702, - "std": 0.026041, - "sem": 0.001345, + "mean": 0.058292, + "median": 0.05652, + "std": 0.022861, + "sem": 0.001181, "min": 0.017224, - "max": 0.210945, + "max": 0.157863, "recall-0.025": 0.053191, - "recall-0.05": 0.37234, - "recall-0.1": 0.933511, - "recall-0.15": 0.989362, + "recall-0.05": 0.37766, + "recall-0.1": 0.941489, + "recall-0.15": 0.99734, "recall-0.25": 1.0, "recall-0.5": 1.0, "num_labels": 376 }, "ankle_left": { "count": 372, - "mean": 0.091609, - "median": 0.059987, - "std": 0.100211, - "sem": 0.005203, + "mean": 0.089987, + "median": 0.05905, + "std": 0.099776, + "sem": 0.00518, "min": 0.007394, "max": 0.493162, "recall-0.025": 0.06383, - "recall-0.05": 0.297872, - "recall-0.1": 0.819149, - "recall-0.15": 0.882979, - "recall-0.25": 0.904255, + "recall-0.05": 0.305851, + "recall-0.1": 0.827128, + "recall-0.15": 0.890957, + "recall-0.25": 0.909574, "recall-0.5": 0.989362, "num_labels": 376 }, "ankle_right": { "count": 372, - "mean": 0.080282, - "median": 0.052603, - "std": 0.09038, - "sem": 0.004692, + "mean": 0.076338, + "median": 0.052354, + "std": 0.085003, + "sem": 0.004413, "min": 0.006245, "max": 0.496601, "recall-0.025": 0.071809, - "recall-0.05": 0.473404, - "recall-0.1": 0.845745, - "recall-0.15": 0.901596, - "recall-0.25": 0.920213, + "recall-0.05": 0.478723, + "recall-0.1": 0.859043, + "recall-0.15": 0.914894, + "recall-0.25": 0.930851, "recall-0.5": 0.989362, "num_labels": 376 }, "joint_recalls": { "num_labels": 4888, "recall-0.025": 0.04255, - "recall-0.05": 0.28785, - "recall-0.1": 0.79153, - "recall-0.15": 0.93535, - "recall-0.25": 0.98261, + "recall-0.05": 0.2901, + "recall-0.1": 0.79439, + "recall-0.15": 0.9376, + "recall-0.25": 0.98363, "recall-0.5": 0.99836 } } { "total_parts": 5264, - "correct_parts": 4964, - "pcp": 0.943009 + "correct_parts": 4969, + "pcp": 0.943959 } ``` @@ -820,189 +820,189 @@ Results of the model in various experiments on different datasets. (duration 00:02:31) ```json { - "avg_time_2d": 0.05529122129203044, - "avg_time_3d": 0.0028780115247735685, - "avg_fps": 17.191218649029846 + "avg_time_2d": 0.054314954600542156, + "avg_time_3d": 0.0029571287466366573, + "avg_fps": 17.460513771396784 } { "person_nums": { "total_frames": 629, "total_labels": 1061, - "total_preds": 1107, - "considered_empty": 106, - "valid_preds": 754, - "invalid_preds": 247, - "missing": 307, - "invalid_fraction": 0.24675, - "precision": 0.75325, - "recall": 0.71065, - "f1": 0.73133, - "non_empty": 1001 + "total_preds": 1046, + "considered_empty": 118, + "valid_preds": 711, + "invalid_preds": 217, + "missing": 350, + "invalid_fraction": 0.23384, + "precision": 0.76616, + "recall": 0.67012, + "f1": 0.71493, + "non_empty": 928 }, "mpjpe": { - "count": 754, - "mean": 0.112533, - "median": 0.093582, - "std": 0.064237, - "sem": 0.002341, + "count": 711, + "mean": 0.1134, + "median": 0.094476, + "std": 0.065779, + "sem": 0.002469, "min": 0.040489, "max": 0.497874, "recall-0.025": 0.0, - "recall-0.05": 0.004713, - "recall-0.1": 0.39868, - "recall-0.15": 0.602262, - "recall-0.25": 0.687088, - "recall-0.5": 0.71065, + "recall-0.05": 0.006598, + "recall-0.1": 0.367578, + "recall-0.15": 0.570217, + "recall-0.25": 0.645617, + "recall-0.5": 0.670123, "num_labels": 1061, "ap-0.025": 0.0, - "ap-0.05": 9.5e-05, - "ap-0.1": 0.257803, - "ap-0.15": 0.500043, - "ap-0.25": 0.610091, - "ap-0.5": 0.638298 + "ap-0.05": 0.000155, + "ap-0.1": 0.237957, + "ap-0.15": 0.474506, + "ap-0.25": 0.573657, + "ap-0.5": 0.602532 }, "head": { - "count": 753, - "mean": 0.061147, - "median": 0.048418, - "std": 0.051506, - "sem": 0.001878, + "count": 710, + "mean": 0.062213, + "median": 0.049204, + "std": 0.05213, + "sem": 0.001958, "min": 0.006565, "max": 0.436815, - "recall-0.025": 0.091423, - "recall-0.05": 0.36852, - "recall-0.1": 0.63525, - "recall-0.15": 0.675778, - "recall-0.25": 0.69557, - "recall-0.5": 0.709708, + "recall-0.025": 0.081056, + "recall-0.05": 0.340245, + "recall-0.1": 0.592837, + "recall-0.15": 0.633365, + "recall-0.25": 0.655985, + "recall-0.5": 0.66918, "num_labels": 1061 }, "shoulder_left": { - "count": 752, - "mean": 0.059702, - "median": 0.042396, - "std": 0.060756, - "sem": 0.002217, - "min": 0.002877, + "count": 709, + "mean": 0.061416, + "median": 0.043163, + "std": 0.064096, + "sem": 0.002409, + "min": 0.003155, "max": 0.48718, - "recall-0.025": 0.161169, - "recall-0.05": 0.424128, - "recall-0.1": 0.618285, - "recall-0.15": 0.670123, - "recall-0.25": 0.689915, - "recall-0.5": 0.708765, + "recall-0.025": 0.144204, + "recall-0.05": 0.399623, + "recall-0.1": 0.579642, + "recall-0.15": 0.62771, + "recall-0.25": 0.649387, + "recall-0.5": 0.668238, "num_labels": 1061 }, "shoulder_right": { - "count": 752, - "mean": 0.058725, - "median": 0.038927, - "std": 0.064616, - "sem": 0.002358, + "count": 709, + "mean": 0.058409, + "median": 0.039678, + "std": 0.063479, + "sem": 0.002386, "min": 0.001883, "max": 0.470146, - "recall-0.025": 0.154571, - "recall-0.05": 0.454288, - "recall-0.1": 0.621112, - "recall-0.15": 0.668238, - "recall-0.25": 0.685203, - "recall-0.5": 0.708765, + "recall-0.025": 0.147974, + "recall-0.05": 0.427898, + "recall-0.1": 0.587182, + "recall-0.15": 0.630537, + "recall-0.25": 0.647502, + "recall-0.5": 0.668238, "num_labels": 1061 }, "elbow_left": { - "count": 740, - "mean": 0.100132, - "median": 0.07137, - "std": 0.082644, - "sem": 0.00304, + "count": 697, + "mean": 0.100689, + "median": 0.073042, + "std": 0.081503, + "sem": 0.003089, "min": 0.005709, "max": 0.481769, - "recall-0.025": 0.071631, - "recall-0.05": 0.234684, - "recall-0.1": 0.428841, - "recall-0.15": 0.551367, - "recall-0.25": 0.655042, - "recall-0.5": 0.697455, + "recall-0.025": 0.067861, + "recall-0.05": 0.216777, + "recall-0.1": 0.399623, + "recall-0.15": 0.518379, + "recall-0.25": 0.617342, + "recall-0.5": 0.656927, "num_labels": 1061 }, "elbow_right": { - "count": 744, - "mean": 0.081124, - "median": 0.056996, - "std": 0.075451, - "sem": 0.002768, - "min": 0.003759, + "count": 700, + "mean": 0.07996, + "median": 0.056737, + "std": 0.074426, + "sem": 0.002815, + "min": 0.003313, "max": 0.498068, - "recall-0.025": 0.093308, - "recall-0.05": 0.310085, - "recall-0.1": 0.525919, - "recall-0.15": 0.617342, - "recall-0.25": 0.672008, - "recall-0.5": 0.701225, + "recall-0.025": 0.088596, + "recall-0.05": 0.29689, + "recall-0.1": 0.499529, + "recall-0.15": 0.585297, + "recall-0.25": 0.633365, + "recall-0.5": 0.659755, "num_labels": 1061 }, "wrist_left": { - "count": 746, - "mean": 0.124492, - "median": 0.087532, - "std": 0.102378, - "sem": 0.003751, + "count": 703, + "mean": 0.127245, + "median": 0.091393, + "std": 0.103887, + "sem": 0.003921, "min": 0.004181, "max": 0.497384, - "recall-0.025": 0.070688, - "recall-0.05": 0.190386, - "recall-0.1": 0.37606, - "recall-0.15": 0.479736, - "recall-0.25": 0.609802, - "recall-0.5": 0.70311, + "recall-0.025": 0.06409, + "recall-0.05": 0.173421, + "recall-0.1": 0.351555, + "recall-0.15": 0.448633, + "recall-0.25": 0.569274, + "recall-0.5": 0.662582, "num_labels": 1061 }, "wrist_right": { - "count": 739, - "mean": 0.110218, - "median": 0.076683, - "std": 0.097189, - "sem": 0.003578, + "count": 698, + "mean": 0.112388, + "median": 0.07829, + "std": 0.098963, + "sem": 0.003748, "min": 0.006423, "max": 0.485199, - "recall-0.025": 0.073516, - "recall-0.05": 0.235627, - "recall-0.1": 0.430726, - "recall-0.15": 0.518379, - "recall-0.25": 0.621112, - "recall-0.5": 0.696513, + "recall-0.025": 0.071631, + "recall-0.05": 0.218662, + "recall-0.1": 0.39868, + "recall-0.15": 0.485391, + "recall-0.25": 0.583412, + "recall-0.5": 0.65787, "num_labels": 1061 }, "hip_left": { - "count": 739, - "mean": 0.186663, - "median": 0.16686, - "std": 0.08462, - "sem": 0.003115, + "count": 697, + "mean": 0.18639, + "median": 0.167596, + "std": 0.084505, + "sem": 0.003203, "min": 0.021758, "max": 0.499358, "recall-0.025": 0.000943, - "recall-0.05": 0.004713, - "recall-0.1": 0.069746, - "recall-0.15": 0.263902, - "recall-0.25": 0.574929, - "recall-0.5": 0.696513, + "recall-0.05": 0.005655, + "recall-0.1": 0.066918, + "recall-0.15": 0.249764, + "recall-0.25": 0.542884, + "recall-0.5": 0.656927, "num_labels": 1061 }, "hip_right": { - "count": 742, - "mean": 0.180059, - "median": 0.164071, - "std": 0.079758, - "sem": 0.00293, + "count": 696, + "mean": 0.177823, + "median": 0.163941, + "std": 0.077814, + "sem": 0.002952, "min": 0.024563, "max": 0.492847, "recall-0.025": 0.000943, "recall-0.05": 0.00377, - "recall-0.1": 0.074458, - "recall-0.15": 0.28558, - "recall-0.25": 0.590952, - "recall-0.5": 0.69934, + "recall-0.1": 0.077286, + "recall-0.15": 0.272385, + "recall-0.25": 0.558907, + "recall-0.5": 0.655985, "num_labels": 1061 }, "knee_left": {}, @@ -1011,18 +1011,18 @@ Results of the model in various experiments on different datasets. "ankle_right": {}, "joint_recalls": { "num_labels": 9549, - "recall-0.025": 0.07938, - "recall-0.05": 0.24673, - "recall-0.1": 0.41973, - "recall-0.15": 0.5254, - "recall-0.25": 0.64321, - "recall-0.5": 0.70175 + "recall-0.025": 0.07404, + "recall-0.05": 0.23091, + "recall-0.1": 0.39449, + "recall-0.15": 0.49419, + "recall-0.25": 0.60593, + "recall-0.5": 0.66133 } } { "total_parts": 10610, - "correct_parts": 5861, - "pcp": 0.552403 + "correct_parts": 5510, + "pcp": 0.519321 } ``` @@ -1031,72 +1031,72 @@ Results of the model in various experiments on different datasets. (duration 00:02:28) ```json { - "avg_time_2d": 0.10983607013051103, - "avg_time_3d": 0.023344272520483995, - "avg_fps": 7.508615611693869 + "avg_time_2d": 0.11225916583363603, + "avg_time_3d": 0.012731300912252287, + "avg_fps": 8.000610174798759 } { "person_nums": { "total_frames": 420, "total_labels": 1466, - "total_preds": 1568, - "considered_empty": 8, + "total_preds": 1509, + "considered_empty": 5, "valid_preds": 1463, - "invalid_preds": 97, + "invalid_preds": 41, "missing": 3, - "invalid_fraction": 0.06218, - "precision": 0.93782, + "invalid_fraction": 0.02726, + "precision": 0.97274, "recall": 0.99795, - "f1": 0.96695, - "non_empty": 1560 + "f1": 0.98519, + "non_empty": 1504 }, "mpjpe": { "count": 1463, - "mean": 0.044122, - "median": 0.0376, - "std": 0.024349, - "sem": 0.000637, + "mean": 0.044136, + "median": 0.038064, + "std": 0.023929, + "sem": 0.000626, "min": 0.01199, "max": 0.383741, - "recall-0.025": 0.162347, - "recall-0.05": 0.712824, - "recall-0.1": 0.96794, - "recall-0.15": 0.995907, + "recall-0.025": 0.158254, + "recall-0.05": 0.712142, + "recall-0.1": 0.969304, + "recall-0.15": 0.996589, "recall-0.25": 0.997271, "recall-0.5": 0.997954, "num_labels": 1466, - "ap-0.025": 0.063858, - "ap-0.05": 0.642449, - "ap-0.1": 0.949904, - "ap-0.15": 0.982339, - "ap-0.25": 0.983697, - "ap-0.5": 0.984345 + "ap-0.025": 0.061059, + "ap-0.05": 0.64118, + "ap-0.1": 0.951321, + "ap-0.15": 0.983556, + "ap-0.25": 0.984229, + "ap-0.5": 0.9849 }, "nose": { "count": 1461, - "mean": 0.018639, - "median": 0.012435, - "std": 0.032832, - "sem": 0.000859, + "mean": 0.018933, + "median": 0.01246, + "std": 0.033585, + "sem": 0.000879, "min": 0.00163, "max": 0.316534, - "recall-0.025": 0.894053, - "recall-0.05": 0.955571, - "recall-0.1": 0.985646, - "recall-0.15": 0.986329, - "recall-0.25": 0.986329, + "recall-0.025": 0.891319, + "recall-0.05": 0.952837, + "recall-0.1": 0.984962, + "recall-0.15": 0.985646, + "recall-0.25": 0.985646, "recall-0.5": 0.998633, "num_labels": 1463 }, "shoulder_left": { "count": 1463, - "mean": 0.021907, - "median": 0.019325, - "std": 0.012953, - "sem": 0.000339, + "mean": 0.021887, + "median": 0.019127, + "std": 0.013093, + "sem": 0.000342, "min": 0.002677, "max": 0.124087, - "recall-0.025": 0.714188, + "recall-0.025": 0.715553, "recall-0.05": 0.967258, "recall-0.1": 0.993861, "recall-0.15": 0.997954, @@ -1106,15 +1106,15 @@ Results of the model in various experiments on different datasets. }, "shoulder_right": { "count": 1462, - "mean": 0.023838, - "median": 0.019815, - "std": 0.016961, - "sem": 0.000444, + "mean": 0.023863, + "median": 0.019858, + "std": 0.016912, + "sem": 0.000442, "min": 0.002471, "max": 0.146597, - "recall-0.025": 0.690102, + "recall-0.025": 0.688055, "recall-0.05": 0.94198, - "recall-0.1": 0.988396, + "recall-0.1": 0.989078, "recall-0.15": 0.997952, "recall-0.25": 0.997952, "recall-0.5": 0.997952, @@ -1122,15 +1122,15 @@ Results of the model in various experiments on different datasets. }, "elbow_left": { "count": 1462, - "mean": 0.024992, - "median": 0.017082, - "std": 0.027098, + "mean": 0.025007, + "median": 0.017079, + "std": 0.027085, "sem": 0.000709, "min": 0.001846, "max": 0.323155, - "recall-0.025": 0.720137, - "recall-0.05": 0.891468, - "recall-0.1": 0.98157, + "recall-0.025": 0.720819, + "recall-0.05": 0.89215, + "recall-0.1": 0.980887, "recall-0.15": 0.991809, "recall-0.25": 0.994539, "recall-0.5": 0.997952, @@ -1138,63 +1138,63 @@ Results of the model in various experiments on different datasets. }, "elbow_right": { "count": 1461, - "mean": 0.024015, + "mean": 0.024025, "median": 0.01806, - "std": 0.01946, - "sem": 0.000509, + "std": 0.019691, + "sem": 0.000515, "min": 0.001407, "max": 0.233543, - "recall-0.025": 0.681476, - "recall-0.05": 0.917977, - "recall-0.1": 0.991798, - "recall-0.15": 0.996582, + "recall-0.025": 0.683527, + "recall-0.05": 0.920711, + "recall-0.1": 0.991114, + "recall-0.15": 0.995899, "recall-0.25": 0.998633, "recall-0.5": 0.998633, "num_labels": 1463 }, "wrist_left": { "count": 1429, - "mean": 0.035263, - "median": 0.016885, - "std": 0.051671, - "sem": 0.001367, - "min": 0.001303, + "mean": 0.035061, + "median": 0.016859, + "std": 0.051336, + "sem": 0.001359, + "min": 0.001333, "max": 0.496738, - "recall-0.025": 0.666667, - "recall-0.05": 0.822873, - "recall-0.1": 0.912134, - "recall-0.15": 0.958856, + "recall-0.025": 0.665969, + "recall-0.05": 0.82357, + "recall-0.1": 0.912831, + "recall-0.15": 0.960251, "recall-0.25": 0.987448, "recall-0.5": 0.996513, "num_labels": 1434 }, "wrist_right": { "count": 1454, - "mean": 0.026128, - "median": 0.017008, - "std": 0.026801, - "sem": 0.000703, + "mean": 0.026296, + "median": 0.017106, + "std": 0.028279, + "sem": 0.000742, "min": 0.001042, - "max": 0.202973, - "recall-0.025": 0.683379, - "recall-0.05": 0.880495, - "recall-0.1": 0.963599, - "recall-0.15": 0.991071, - "recall-0.25": 0.998626, + "max": 0.394822, + "recall-0.025": 0.681319, + "recall-0.05": 0.879808, + "recall-0.1": 0.962912, + "recall-0.15": 0.992445, + "recall-0.25": 0.99794, "recall-0.5": 0.998626, "num_labels": 1456 }, "hip_left": { "count": 1461, - "mean": 0.058624, - "median": 0.052631, - "std": 0.029266, - "sem": 0.000766, + "mean": 0.058879, + "median": 0.052826, + "std": 0.029358, + "sem": 0.000768, "min": 0.004253, "max": 0.17979, "recall-0.025": 0.070307, - "recall-0.05": 0.445051, - "recall-0.1": 0.895563, + "recall-0.05": 0.440273, + "recall-0.1": 0.894198, "recall-0.15": 0.974061, "recall-0.25": 0.99727, "recall-0.5": 0.99727, @@ -1202,15 +1202,15 @@ Results of the model in various experiments on different datasets. }, "hip_right": { "count": 1462, - "mean": 0.060147, - "median": 0.057418, - "std": 0.02837, - "sem": 0.000742, + "mean": 0.060436, + "median": 0.057528, + "std": 0.028518, + "sem": 0.000746, "min": 0.010398, "max": 0.304524, - "recall-0.025": 0.045703, - "recall-0.05": 0.379263, - "recall-0.1": 0.912005, + "recall-0.025": 0.046385, + "recall-0.05": 0.375853, + "recall-0.1": 0.906548, "recall-0.15": 0.987722, "recall-0.25": 0.996589, "recall-0.5": 0.997271, @@ -1218,82 +1218,82 @@ Results of the model in various experiments on different datasets. }, "knee_left": { "count": 1461, - "mean": 0.05085, - "median": 0.04337, - "std": 0.038015, - "sem": 0.000995, + "mean": 0.050771, + "median": 0.043452, + "std": 0.03768, + "sem": 0.000986, "min": 0.001938, "max": 0.349881, - "recall-0.025": 0.178157, - "recall-0.05": 0.619113, + "recall-0.025": 0.17884, + "recall-0.05": 0.614334, "recall-0.1": 0.936519, - "recall-0.15": 0.971331, + "recall-0.15": 0.972696, "recall-0.25": 0.988396, "recall-0.5": 0.99727, "num_labels": 1465 }, "knee_right": { "count": 1456, - "mean": 0.050085, - "median": 0.04265, - "std": 0.035134, - "sem": 0.000921, + "mean": 0.049844, + "median": 0.042843, + "std": 0.034357, + "sem": 0.000901, "min": 0.000605, "max": 0.480558, - "recall-0.025": 0.189856, + "recall-0.025": 0.186429, "recall-0.05": 0.590816, - "recall-0.1": 0.921864, + "recall-0.1": 0.925977, "recall-0.15": 0.986292, - "recall-0.25": 0.991775, + "recall-0.25": 0.992461, "recall-0.5": 0.997944, "num_labels": 1459 }, "ankle_left": { - "count": 1448, - "mean": 0.08721, - "median": 0.035703, - "std": 0.109209, - "sem": 0.002871, + "count": 1449, + "mean": 0.087757, + "median": 0.035935, + "std": 0.108924, + "sem": 0.002862, "min": 0.002453, - "max": 0.495773, - "recall-0.025": 0.376623, - "recall-0.05": 0.578264, - "recall-0.1": 0.712235, - "recall-0.15": 0.789474, + "max": 0.498193, + "recall-0.025": 0.375256, + "recall-0.05": 0.572796, + "recall-0.1": 0.708134, + "recall-0.15": 0.788107, "recall-0.25": 0.900205, - "recall-0.5": 0.989747, + "recall-0.5": 0.990431, "num_labels": 1463 }, "ankle_right": { - "count": 1443, - "mean": 0.079468, - "median": 0.032419, - "std": 0.107852, - "sem": 0.00284, + "count": 1445, + "mean": 0.079798, + "median": 0.032499, + "std": 0.108043, + "sem": 0.002843, "min": 0.001061, - "max": 0.4981, - "recall-0.025": 0.378082, - "recall-0.05": 0.686301, - "recall-0.1": 0.771918, - "recall-0.15": 0.810274, - "recall-0.25": 0.889041, - "recall-0.5": 0.988356, + "max": 0.492817, + "recall-0.025": 0.378767, + "recall-0.05": 0.682877, + "recall-0.1": 0.771233, + "recall-0.15": 0.810959, + "recall-0.25": 0.891096, + "recall-0.5": 0.989726, "num_labels": 1460 }, "joint_recalls": { "num_labels": 18990, - "recall-0.025": 0.48294, - "recall-0.05": 0.74397, - "recall-0.1": 0.92038, - "recall-0.15": 0.95656, - "recall-0.25": 0.97841, - "recall-0.5": 0.9961 + "recall-0.025": 0.48257, + "recall-0.05": 0.74234, + "recall-0.1": 0.91954, + "recall-0.15": 0.95682, + "recall-0.25": 0.97873, + "recall-0.5": 0.99631 } } { "total_parts": 20444, - "correct_parts": 19935, - "pcp": 0.975103 + "correct_parts": 19937, + "pcp": 0.975201 } ``` diff --git a/scripts/test_skelda_dataset.py b/scripts/test_skelda_dataset.py index e69092c..c226dc4 100644 --- a/scripts/test_skelda_dataset.py +++ b/scripts/test_skelda_dataset.py @@ -319,6 +319,7 @@ def main(): all_ids = [] all_paths = [] times = [] + last_poses_3d = np.array([]) for label in tqdm.tqdm(labels): images_2d = [] @@ -377,7 +378,7 @@ def main(): poses2D = np.zeros([len(images_2d), 1, len(joint_names_3d), 3]) else: poses3D = triangulate_poses.get_3d_pose( - poses_2d, label["cameras"], roomparams, joint_names_2d, minscore + poses_2d, label["cameras"], roomparams, joint_names_2d, last_poses_3d, minscore ) poses2D = [] for cam in label["cameras"]: @@ -392,6 +393,7 @@ def main(): drop_few_limbs=(dataset_use != "mvor"), ) poses3D = add_missing_joints(poses3D, joint_names_3d) + last_poses_3d = poses3D time_3d = time.time() - start print("3D time:", time_3d) diff --git a/scripts/triangulate_poses.py b/scripts/triangulate_poses.py index 8360e3d..7cf1a90 100644 --- a/scripts/triangulate_poses.py +++ b/scripts/triangulate_poses.py @@ -1,6 +1,5 @@ import copy import math -import time import cv2 import numpy as np @@ -66,6 +65,29 @@ def get_camera_P(cam): # ================================================================================================== +def calc_pose_score(pose1, pose2, dist1, cam1, joint_names, use_joints): + """Calculates the score between two poses""" + + # Select core joints + jids = [joint_names.index(j) for j in use_joints] + pose1 = pose1[jids] + pose2 = pose2[jids] + dist1 = dist1[jids] + + mask = (pose1[:, 2] > 0.1) & (pose2[:, 2] > 0.1) + if np.sum(mask) < 3: + return 0.0 + + iscale = (cam1["width"] + cam1["height"]) / 2 + scores = score_projection(pose1, pose2, dist1, mask, iscale) + score = np.mean(scores) + + return score + + +# ================================================================================================== + + def calc_pair_score(pair, poses_2d, camparams, roomparams, joint_names_2d, use_joints): """Triangulates a pair of persons and scores them based on the reprojection error""" @@ -79,14 +101,38 @@ def calc_pair_score(pair, poses_2d, camparams, roomparams, joint_names_2d, use_j pose1 = pose1[jids] pose2 = pose2[jids] - poses_3d, score = calc_pose_scored(pose1, pose2, cam1, cam2, roomparams) + poses_3d, score = triangulate_and_score(pose1, pose2, cam1, cam2, roomparams) return poses_3d, score # ================================================================================================== -def calc_pose_scored(pose1, pose2, cam1, cam2, roomparams): +def score_projection(pose1, repro1, dists1, mask, iscale): + + min_score = 0.1 + error1 = np.linalg.norm(pose1[mask, 0:2] - repro1[mask, 0:2], axis=1) + + # Set errors of invisible reprojections to a high value + penalty = iscale + mask1b = (repro1[:, 2] < min_score)[mask] + error1[mask1b] = penalty + + # Scale error by image size and distance to the camera + error1 = error1.clip(0, iscale / 4) / iscale + dscale1 = np.sqrt(np.mean(dists1[mask]) / 3.5) + error1 = error1 * dscale1 + + # Convert errors to a score + score1 = 1.0 / (1.0 + error1 * 10) + + return score1 + + +# ================================================================================================== + + +def triangulate_and_score(pose1, pose2, cam1, cam2, roomparams): """Triangulates a pair of persons and scores them based on the reprojection error""" # Mask out invisible joints @@ -129,27 +175,16 @@ def calc_pose_scored(pose1, pose2, cam1, cam2, roomparams): poses_3d = np.expand_dims(pose3d, axis=0) repro1, dists1 = utils_pose.project_poses(poses_3d, cam1, calc_dists=True) repro2, dists2 = utils_pose.project_poses(poses_3d, cam2, calc_dists=True) - error1 = np.linalg.norm(pose1[mask, 0:2] - repro1[0, mask, 0:2], axis=1) - error2 = np.linalg.norm(pose2[mask, 0:2] - repro2[0, mask, 0:2], axis=1) + repro1, dists1 = repro1[0], dists1[0] + repro2, dists2 = repro2[0], dists2[0] - # Set errors of invisible reprojections to a high value - penalty = (cam1["width"] + cam1["height"]) / 2 - mask1b = (repro1[0, :, 2] < min_score)[mask] - mask2b = (repro2[0, :, 2] < min_score)[mask] - error1[mask1b] = penalty - error2[mask2b] = penalty - - # Convert errors to a score - # Scale by image size and distance to the camera + # Calculate scores for each view iscale = (cam1["width"] + cam1["height"]) / 2 - error1 = error1.clip(0, iscale / 4) / iscale - error2 = error2.clip(0, iscale / 4) / iscale - dscale1 = np.sqrt(np.mean(dists1[0, mask]) / 3.5) - dscale2 = np.sqrt(np.mean(dists2[0, mask]) / 3.5) - error1 = error1 * dscale1 - error2 = error2 * dscale2 - error = (error1 + error2) / 2 - scores = 1.0 / (1.0 + error * 10) + score1 = score_projection(pose1, repro1, dists1, mask, iscale) + score2 = score_projection(pose2, repro2, dists2, mask, iscale) + + # Combine scores + scores = (score1 + score2) / 2 # Drop lowest scores drop_k = math.floor(len(pose1) * 0.2) @@ -283,7 +318,14 @@ def merge_group(poses_3d: np.ndarray, min_score: float): # ================================================================================================== -def get_3d_pose(poses_2d, camparams, roomparams, joint_names_2d, min_score=0.95): +def get_3d_pose( + poses_2d, + camparams, + roomparams, + joint_names_2d, + last_poses_3d=np.array([]), + min_score=0.95, +): """Triangulates 3D poses from 2D poses of multiple views""" # Convert poses and camparams to numpy arrays @@ -314,10 +356,42 @@ def get_3d_pose(poses_2d, camparams, roomparams, joint_names_2d, min_score=0.95) cam["P"] = get_camera_P(cam) camparams[i] = cam + # Project last 3D poses to 2D + last_poses_2d = [] + last_poses_3d = np.asarray(last_poses_3d) + if last_poses_3d.size > 0: + for i in range(len(camparams)): + poses2d, dists = utils_pose.project_poses(last_poses_3d, camparams[i]) + last_poses_2d.append((poses2d, dists)) + + # Check matches to old poses + threshold = min_score - 0.2 + scored_pasts = {} + if last_poses_3d.size > 0: + for i in range(len(camparams)): + scored_pasts[i] = {} + poses = poses_2d[i] + last_poses, dists = last_poses_2d[i] + for j in range(len(last_poses)): + scored_pasts[i][j] = [] + for k in range(len(poses)): + score = calc_pose_score( + poses[k], + last_poses[j], + dists[j], + camparams[i], + joint_names_2d, + core_joints, + ) + if score > threshold: + scored_pasts[i][j].append(k) + # Create pairs of persons + # Checks if the person was already matched to the last frame and if so only creates pairs with those + # Else it creates all possible pairs num_persons = [len(p) for p in poses_2d] all_pairs = [] - for i in range(len(poses_2d)): + for i in range(len(camparams)): poses = poses_2d[i] for j in range(i + 1, len(poses_2d)): poses2 = poses_2d[j] @@ -325,7 +399,16 @@ def get_3d_pose(poses_2d, camparams, roomparams, joint_names_2d, min_score=0.95) for l in range(len(poses2)): pid1 = sum(num_persons[:i]) + k pid2 = sum(num_persons[:j]) + l - all_pairs.append([(i, j, k, l), (pid1, pid2)]) + match = False + if last_poses_3d.size > 0: + for m in range(len(last_poses_3d)): + if k in scored_pasts[i][m] and l in scored_pasts[j][m]: + match = True + all_pairs.append([(i, j, k, l), (pid1, pid2)]) + elif k in scored_pasts[i][m] or l in scored_pasts[j][m]: + match = True + if not match: + all_pairs.append([(i, j, k, l), (pid1, pid2)]) # Calculate pair scores for i in range(len(all_pairs)): @@ -356,7 +439,7 @@ def get_3d_pose(poses_2d, camparams, roomparams, joint_names_2d, min_score=0.95) pose1 = poses_2d[pair[0][0]][pair[0][2]] pose2 = poses_2d[pair[0][1]][pair[0][3]] - pose_3d, _ = calc_pose_scored(pose1, pose2, cam1, cam2, roomparams) + pose_3d, _ = triangulate_and_score(pose1, pose2, cam1, cam2, roomparams) pair.append(pose_3d) # Merge groups