From b41380b033ca5e4875032aef6014247278b5ba8e Mon Sep 17 00:00:00 2001
From: Daniel <daniel@mail.de>
Date: Tue, 28 Jan 2025 15:30:23 +0100
Subject: [PATCH] Eval with all panoptic cameras.

---
 media/RESULTS.md               | 287 +++++++++++++++++++++++++++++++++
 scripts/test_skelda_dataset.py |   4 +-
 2 files changed, 290 insertions(+), 1 deletion(-)

diff --git a/media/RESULTS.md b/media/RESULTS.md
index 6cb4b1a..02ca0e8 100644
--- a/media/RESULTS.md
+++ b/media/RESULTS.md
@@ -1948,6 +1948,293 @@ Results of the model in various experiments on different datasets. \
 }
 ```
 
+##### 31 cameras
+
+```json
+{
+  "img_loading": 0.316871,
+  "demosaicing": 0.0110794,
+  "avg_time_2d": 0.0985203,
+  "avg_time_3d": 0.0129658,
+  "fps": 8.15891
+}
+{
+  "triangulator_calls": 420,
+  "init_time": 6.01526e-05,
+  "undistort_time": 0.000318677,
+  "project_time": 0.000553076,
+  "match_time": 0.000529147,
+  "pairs_time": 0.00223077,
+  "pair_scoring_time": 0.00300736,
+  "grouping_time": 0.00276694,
+  "full_time": 0.002134,
+  "merge_time": 0.000522887,
+  "post_time": 1.79106e-05,
+  "convert_time": 9.62974e-07,
+  "total_time": 0.0121423
+}
+{
+  "person_nums": {
+    "total_frames": 420,
+    "total_labels": 1466,
+    "total_preds": 1505,
+    "considered_empty": 0,
+    "valid_preds": 1465,
+    "invalid_preds": 40,
+    "missing": 1,
+    "invalid_fraction": 0.02658,
+    "precision": 0.97342,
+    "recall": 0.99932,
+    "f1": 0.9862,
+    "non_empty": 1505
+  },
+  "mpjpe": {
+    "count": 1465,
+    "mean": 0.021378,
+    "median": 0.019413,
+    "std": 0.007745,
+    "sem": 0.000202,
+    "min": 0.009697,
+    "max": 0.102079,
+    "recall-0.025": 0.733288,
+    "recall-0.05": 0.995225,
+    "recall-0.1": 0.998636,
+    "recall-0.15": 0.999318,
+    "recall-0.25": 0.999318,
+    "recall-0.5": 0.999318,
+    "num_labels": 1466,
+    "ap-0.025": 0.660592,
+    "ap-0.05": 0.983876,
+    "ap-0.1": 0.988209,
+    "ap-0.15": 0.989728,
+    "ap-0.25": 0.989728,
+    "ap-0.5": 0.989728
+  },
+  "nose": {
+    "count": 1462,
+    "mean": 0.009366,
+    "median": 0.008127,
+    "std": 0.007158,
+    "sem": 0.000187,
+    "min": 0.001832,
+    "max": 0.089174,
+    "recall-0.025": 0.982912,
+    "recall-0.05": 0.98838,
+    "recall-0.1": 0.999316,
+    "recall-0.15": 0.999316,
+    "recall-0.25": 0.999316,
+    "recall-0.5": 0.999316,
+    "num_labels": 1463
+  },
+  "shoulder_left": {
+    "count": 1465,
+    "mean": 0.010461,
+    "median": 0.009022,
+    "std": 0.006253,
+    "sem": 0.000163,
+    "min": 0.000486,
+    "max": 0.042895,
+    "recall-0.025": 0.965211,
+    "recall-0.05": 0.999318,
+    "recall-0.1": 0.999318,
+    "recall-0.15": 0.999318,
+    "recall-0.25": 0.999318,
+    "recall-0.5": 0.999318,
+    "num_labels": 1466
+  },
+  "shoulder_right": {
+    "count": 1464,
+    "mean": 0.011754,
+    "median": 0.010296,
+    "std": 0.006613,
+    "sem": 0.000173,
+    "min": 0.000586,
+    "max": 0.082849,
+    "recall-0.025": 0.963823,
+    "recall-0.05": 0.99727,
+    "recall-0.1": 0.999317,
+    "recall-0.15": 0.999317,
+    "recall-0.25": 0.999317,
+    "recall-0.5": 0.999317,
+    "num_labels": 1465
+  },
+  "elbow_left": {
+    "count": 1464,
+    "mean": 0.013819,
+    "median": 0.012248,
+    "std": 0.008099,
+    "sem": 0.000212,
+    "min": 0.001903,
+    "max": 0.065253,
+    "recall-0.025": 0.92628,
+    "recall-0.05": 0.992491,
+    "recall-0.1": 0.999317,
+    "recall-0.15": 0.999317,
+    "recall-0.25": 0.999317,
+    "recall-0.5": 0.999317,
+    "num_labels": 1465
+  },
+  "elbow_right": {
+    "count": 1462,
+    "mean": 0.013849,
+    "median": 0.012421,
+    "std": 0.008812,
+    "sem": 0.000231,
+    "min": 0.000687,
+    "max": 0.167748,
+    "recall-0.025": 0.935065,
+    "recall-0.05": 0.991114,
+    "recall-0.1": 0.998633,
+    "recall-0.15": 0.998633,
+    "recall-0.25": 0.999316,
+    "recall-0.5": 0.999316,
+    "num_labels": 1463
+  },
+  "wrist_left": {
+    "count": 1433,
+    "mean": 0.015001,
+    "median": 0.010714,
+    "std": 0.016661,
+    "sem": 0.00044,
+    "min": 0.000797,
+    "max": 0.148407,
+    "recall-0.025": 0.887727,
+    "recall-0.05": 0.967225,
+    "recall-0.1": 0.990237,
+    "recall-0.15": 0.999303,
+    "recall-0.25": 0.999303,
+    "recall-0.5": 0.999303,
+    "num_labels": 1434
+  },
+  "wrist_right": {
+    "count": 1456,
+    "mean": 0.014295,
+    "median": 0.010251,
+    "std": 0.012861,
+    "sem": 0.000337,
+    "min": 0.000738,
+    "max": 0.093478,
+    "recall-0.025": 0.876374,
+    "recall-0.05": 0.972527,
+    "recall-0.1": 1.0,
+    "recall-0.15": 1.0,
+    "recall-0.25": 1.0,
+    "recall-0.5": 1.0,
+    "num_labels": 1456
+  },
+  "hip_left": {
+    "count": 1464,
+    "mean": 0.02108,
+    "median": 0.01849,
+    "std": 0.011596,
+    "sem": 0.000303,
+    "min": 0.001987,
+    "max": 0.084873,
+    "recall-0.025": 0.746075,
+    "recall-0.05": 0.967918,
+    "recall-0.1": 0.999317,
+    "recall-0.15": 0.999317,
+    "recall-0.25": 0.999317,
+    "recall-0.5": 0.999317,
+    "num_labels": 1465
+  },
+  "hip_right": {
+    "count": 1465,
+    "mean": 0.022031,
+    "median": 0.021111,
+    "std": 0.010293,
+    "sem": 0.000269,
+    "min": 0.001982,
+    "max": 0.184573,
+    "recall-0.025": 0.651432,
+    "recall-0.05": 0.992497,
+    "recall-0.1": 0.998636,
+    "recall-0.15": 0.998636,
+    "recall-0.25": 0.999318,
+    "recall-0.5": 0.999318,
+    "num_labels": 1466
+  },
+  "knee_left": {
+    "count": 1464,
+    "mean": 0.023031,
+    "median": 0.020769,
+    "std": 0.012813,
+    "sem": 0.000335,
+    "min": 0.002548,
+    "max": 0.113513,
+    "recall-0.025": 0.640273,
+    "recall-0.05": 0.958362,
+    "recall-0.1": 0.997952,
+    "recall-0.15": 0.999317,
+    "recall-0.25": 0.999317,
+    "recall-0.5": 0.999317,
+    "num_labels": 1465
+  },
+  "knee_right": {
+    "count": 1458,
+    "mean": 0.024958,
+    "median": 0.022737,
+    "std": 0.016621,
+    "sem": 0.000435,
+    "min": 0.001307,
+    "max": 0.175266,
+    "recall-0.025": 0.579849,
+    "recall-0.05": 0.957505,
+    "recall-0.1": 0.986292,
+    "recall-0.15": 0.998629,
+    "recall-0.25": 0.999315,
+    "recall-0.5": 0.999315,
+    "num_labels": 1459
+  },
+  "ankle_left": {
+    "count": 1462,
+    "mean": 0.048153,
+    "median": 0.032151,
+    "std": 0.042385,
+    "sem": 0.001109,
+    "min": 0.001567,
+    "max": 0.240114,
+    "recall-0.025": 0.386193,
+    "recall-0.05": 0.67054,
+    "recall-0.1": 0.859193,
+    "recall-0.15": 0.966507,
+    "recall-0.25": 0.999316,
+    "recall-0.5": 0.999316,
+    "num_labels": 1463
+  },
+  "ankle_right": {
+    "count": 1458,
+    "mean": 0.049114,
+    "median": 0.028611,
+    "std": 0.047326,
+    "sem": 0.00124,
+    "min": 0.001747,
+    "max": 0.286754,
+    "recall-0.025": 0.444521,
+    "recall-0.05": 0.670548,
+    "recall-0.1": 0.854795,
+    "recall-0.15": 0.95274,
+    "recall-0.25": 0.99589,
+    "recall-0.5": 0.99863,
+    "num_labels": 1460
+  },
+  "joint_recalls": {
+    "num_labels": 18990,
+    "recall-0.025": 0.76777,
+    "recall-0.05": 0.93244,
+    "recall-0.1": 0.9752,
+    "recall-0.15": 0.99273,
+    "recall-0.25": 0.99868,
+    "recall-0.5": 0.99889
+  }
+}
+{
+  "total_parts": 20444,
+  "correct_parts": 20400,
+  "pcp": 0.997848
+}
+```
+
 ### IkeaAsm
 
 (duration 00:01:45)
diff --git a/scripts/test_skelda_dataset.py b/scripts/test_skelda_dataset.py
index 837b602..e445f8a 100644
--- a/scripts/test_skelda_dataset.py
+++ b/scripts/test_skelda_dataset.py
@@ -66,11 +66,13 @@ datasets = {
         "cams": ["00_03", "00_06", "00_12", "00_13", "00_23"],
         # "cams": ["00_03", "00_06", "00_12"],
         # "cams": ["00_03", "00_06", "00_12", "00_13", "00_23", "00_15", "00_10", "00_21", "00_09", "00_01"],
+        # "cams": [],
         "take_interval": 3,
         "min_match_score": 0.95,
         "use_scenes": ["160906_pizza1", "160422_haggling1", "160906_ian5"],
         "min_group_size": 1,
         # "min_group_size": 4,
+        # "min_group_size": 11,
         "min_bbox_area": 0.05 * 0.05,
     },
     "mvor": {
@@ -220,7 +222,7 @@ def load_labels(dataset: dict):
             ]
 
         # Filter cameras
-        if not "cameras_depth" in labels[0]:
+        if not "cameras_depth" in labels[0] and len(dataset["panoptic"]["cams"]) > 0:
             for label in labels:
                 for i, cam in reversed(list(enumerate(label["cameras"]))):
                     if cam["name"] not in dataset["panoptic"]["cams"]: