From 13aa790de1c02e832962c2d40e3e57dfda94d8a9 Mon Sep 17 00:00:00 2001
From: Daniel <daniel@mail.de>
Date: Wed, 7 Aug 2024 10:20:35 +0200
Subject: [PATCH] Testing different camera numbers.

---
 media/RESULTS.md               | 538 +++++++++++++++++++++++++++++++++
 scripts/test_skelda_dataset.py |   2 +
 2 files changed, 540 insertions(+)

diff --git a/media/RESULTS.md b/media/RESULTS.md
index a46184c..5a25338 100644
--- a/media/RESULTS.md
+++ b/media/RESULTS.md
@@ -1297,6 +1297,544 @@ Results of the model in various experiments on different datasets.
 }
 ```
 
+(3 cameras)
+```json
+{
+  "avg_time_2d": 0.06710797461067758,
+  "avg_time_3d": 0.00547288278254067,
+  "avg_fps": 13.777737490511337
+}
+{
+  "person_nums": {
+    "total_frames": 420,
+    "total_labels": 1466,
+    "total_preds": 1441,
+    "considered_empty": 15,
+    "valid_preds": 1410,
+    "invalid_preds": 16,
+    "missing": 56,
+    "invalid_fraction": 0.01122,
+    "precision": 0.98878,
+    "recall": 0.9618,
+    "f1": 0.9751,
+    "non_empty": 1426
+  },
+  "mpjpe": {
+    "count": 1410,
+    "mean": 0.074643,
+    "median": 0.064929,
+    "std": 0.041159,
+    "sem": 0.001096,
+    "min": 0.020973,
+    "max": 0.309055,
+    "recall-0.025": 0.004775,
+    "recall-0.05": 0.274898,
+    "recall-0.1": 0.800819,
+    "recall-0.15": 0.909959,
+    "recall-0.25": 0.950205,
+    "recall-0.5": 0.961801,
+    "num_labels": 1466,
+    "ap-0.025": 0.001596,
+    "ap-0.05": 0.185761,
+    "ap-0.1": 0.740582,
+    "ap-0.15": 0.888662,
+    "ap-0.25": 0.937571,
+    "ap-0.5": 0.951842
+  },
+  "nose": {
+    "count": 1409,
+    "mean": 0.023912,
+    "median": 0.018335,
+    "std": 0.018202,
+    "sem": 0.000485,
+    "min": 0.001916,
+    "max": 0.23494,
+    "recall-0.025": 0.625427,
+    "recall-0.05": 0.904306,
+    "recall-0.1": 0.954887,
+    "recall-0.15": 0.961039,
+    "recall-0.25": 0.96309,
+    "recall-0.5": 0.96309,
+    "num_labels": 1463
+  },
+  "shoulder_left": {
+    "count": 1410,
+    "mean": 0.024111,
+    "median": 0.021119,
+    "std": 0.017353,
+    "sem": 0.000462,
+    "min": 0.002094,
+    "max": 0.31592,
+    "recall-0.025": 0.570941,
+    "recall-0.05": 0.912005,
+    "recall-0.1": 0.95839,
+    "recall-0.15": 0.960437,
+    "recall-0.25": 0.960437,
+    "recall-0.5": 0.961801,
+    "num_labels": 1466
+  },
+  "shoulder_right": {
+    "count": 1409,
+    "mean": 0.029808,
+    "median": 0.024305,
+    "std": 0.023739,
+    "sem": 0.000633,
+    "min": 0.00406,
+    "max": 0.302527,
+    "recall-0.025": 0.498976,
+    "recall-0.05": 0.866894,
+    "recall-0.1": 0.946758,
+    "recall-0.15": 0.955631,
+    "recall-0.25": 0.959727,
+    "recall-0.5": 0.961775,
+    "num_labels": 1465
+  },
+  "elbow_left": {
+    "count": 1409,
+    "mean": 0.036301,
+    "median": 0.023575,
+    "std": 0.034416,
+    "sem": 0.000917,
+    "min": 0.001498,
+    "max": 0.359129,
+    "recall-0.025": 0.512628,
+    "recall-0.05": 0.752901,
+    "recall-0.1": 0.894881,
+    "recall-0.15": 0.94744,
+    "recall-0.25": 0.961092,
+    "recall-0.5": 0.961775,
+    "num_labels": 1465
+  },
+  "elbow_right": {
+    "count": 1409,
+    "mean": 0.037177,
+    "median": 0.022519,
+    "std": 0.043006,
+    "sem": 0.001146,
+    "min": 0.001047,
+    "max": 0.414541,
+    "recall-0.025": 0.548189,
+    "recall-0.05": 0.783322,
+    "recall-0.1": 0.900889,
+    "recall-0.15": 0.920711,
+    "recall-0.25": 0.961039,
+    "recall-0.5": 0.96309,
+    "num_labels": 1463
+  },
+  "wrist_left": {
+    "count": 1378,
+    "mean": 0.057001,
+    "median": 0.022718,
+    "std": 0.076028,
+    "sem": 0.002049,
+    "min": 0.001727,
+    "max": 0.4856,
+    "recall-0.025": 0.508368,
+    "recall-0.05": 0.640865,
+    "recall-0.1": 0.801255,
+    "recall-0.15": 0.866109,
+    "recall-0.25": 0.932357,
+    "recall-0.5": 0.960948,
+    "num_labels": 1434
+  },
+  "wrist_right": {
+    "count": 1399,
+    "mean": 0.051695,
+    "median": 0.028091,
+    "std": 0.06539,
+    "sem": 0.001749,
+    "min": 0.0006,
+    "max": 0.449266,
+    "recall-0.025": 0.430632,
+    "recall-0.05": 0.644231,
+    "recall-0.1": 0.843407,
+    "recall-0.15": 0.90728,
+    "recall-0.25": 0.93956,
+    "recall-0.5": 0.960852,
+    "num_labels": 1456
+  },
+  "hip_left": {
+    "count": 1409,
+    "mean": 0.06923,
+    "median": 0.063829,
+    "std": 0.034073,
+    "sem": 0.000908,
+    "min": 0.004913,
+    "max": 0.205973,
+    "recall-0.025": 0.038908,
+    "recall-0.05": 0.259386,
+    "recall-0.1": 0.840956,
+    "recall-0.15": 0.913311,
+    "recall-0.25": 0.961775,
+    "recall-0.5": 0.961775,
+    "num_labels": 1465
+  },
+  "hip_right": {
+    "count": 1410,
+    "mean": 0.079903,
+    "median": 0.071168,
+    "std": 0.038933,
+    "sem": 0.001037,
+    "min": 0.010114,
+    "max": 0.304524,
+    "recall-0.025": 0.012278,
+    "recall-0.05": 0.203956,
+    "recall-0.1": 0.748295,
+    "recall-0.15": 0.892224,
+    "recall-0.25": 0.960437,
+    "recall-0.5": 0.961801,
+    "num_labels": 1466
+  },
+  "knee_left": {
+    "count": 1393,
+    "mean": 0.081516,
+    "median": 0.055603,
+    "std": 0.077943,
+    "sem": 0.002089,
+    "min": 0.006728,
+    "max": 0.491598,
+    "recall-0.025": 0.096928,
+    "recall-0.05": 0.392491,
+    "recall-0.1": 0.745392,
+    "recall-0.15": 0.815017,
+    "recall-0.25": 0.908532,
+    "recall-0.5": 0.950853,
+    "num_labels": 1465
+  },
+  "knee_right": {
+    "count": 1391,
+    "mean": 0.080744,
+    "median": 0.05609,
+    "std": 0.075146,
+    "sem": 0.002016,
+    "min": 0.005476,
+    "max": 0.494565,
+    "recall-0.025": 0.078821,
+    "recall-0.05": 0.381768,
+    "recall-0.1": 0.753941,
+    "recall-0.15": 0.831391,
+    "recall-0.25": 0.912269,
+    "recall-0.5": 0.953393,
+    "num_labels": 1459
+  },
+  "ankle_left": {
+    "count": 1321,
+    "mean": 0.161113,
+    "median": 0.128889,
+    "std": 0.139661,
+    "sem": 0.003844,
+    "min": 0.000769,
+    "max": 0.49962,
+    "recall-0.025": 0.198906,
+    "recall-0.05": 0.295284,
+    "recall-0.1": 0.375256,
+    "recall-0.15": 0.501709,
+    "recall-0.25": 0.652085,
+    "recall-0.5": 0.902939,
+    "num_labels": 1463
+  },
+  "ankle_right": {
+    "count": 1312,
+    "mean": 0.149017,
+    "median": 0.106396,
+    "std": 0.131149,
+    "sem": 0.003622,
+    "min": 0.001734,
+    "max": 0.496945,
+    "recall-0.025": 0.163014,
+    "recall-0.05": 0.296575,
+    "recall-0.1": 0.430137,
+    "recall-0.15": 0.536301,
+    "recall-0.25": 0.683562,
+    "recall-0.5": 0.89863,
+    "num_labels": 1460
+  },
+  "joint_recalls": {
+    "num_labels": 18990,
+    "recall-0.025": 0.32886,
+    "recall-0.05": 0.56367,
+    "recall-0.1": 0.78394,
+    "recall-0.15": 0.84655,
+    "recall-0.25": 0.90411,
+    "recall-0.5": 0.95076
+  }
+}
+{
+  "total_parts": 20444,
+  "correct_parts": 18326,
+  "pcp": 0.8964
+}
+```
+
+(10 cameras)
+```json
+{
+  "avg_time_2d": 0.20916332849642125,
+  "avg_time_3d": 0.04559231851159073,
+  "avg_fps": 3.925330063315733
+}
+{
+  "person_nums": {
+    "total_frames": 420,
+    "total_labels": 1466,
+    "total_preds": 1710,
+    "considered_empty": 32,
+    "valid_preds": 1465,
+    "invalid_preds": 213,
+    "missing": 1,
+    "invalid_fraction": 0.12694,
+    "precision": 0.87306,
+    "recall": 0.99932,
+    "f1": 0.93193,
+    "non_empty": 1678
+  },
+  "mpjpe": {
+    "count": 1465,
+    "mean": 0.037378,
+    "median": 0.03232,
+    "std": 0.01817,
+    "sem": 0.000475,
+    "min": 0.013848,
+    "max": 0.134262,
+    "recall-0.025": 0.188267,
+    "recall-0.05": 0.850614,
+    "recall-0.1": 0.982265,
+    "recall-0.15": 0.999318,
+    "recall-0.25": 0.999318,
+    "recall-0.5": 0.999318,
+    "num_labels": 1466,
+    "ap-0.025": 0.082813,
+    "ap-0.05": 0.796959,
+    "ap-0.1": 0.956531,
+    "ap-0.15": 0.977531,
+    "ap-0.25": 0.977531,
+    "ap-0.5": 0.977531
+  },
+  "nose": {
+    "count": 1462,
+    "mean": 0.012332,
+    "median": 0.010836,
+    "std": 0.007658,
+    "sem": 0.0002,
+    "min": 0.001275,
+    "max": 0.124832,
+    "recall-0.025": 0.955571,
+    "recall-0.05": 0.993848,
+    "recall-0.1": 0.998633,
+    "recall-0.15": 0.999316,
+    "recall-0.25": 0.999316,
+    "recall-0.5": 0.999316,
+    "num_labels": 1463
+  },
+  "shoulder_left": {
+    "count": 1465,
+    "mean": 0.018735,
+    "median": 0.016428,
+    "std": 0.01034,
+    "sem": 0.00027,
+    "min": 0.001104,
+    "max": 0.11138,
+    "recall-0.025": 0.804911,
+    "recall-0.05": 0.988404,
+    "recall-0.1": 0.998636,
+    "recall-0.15": 0.999318,
+    "recall-0.25": 0.999318,
+    "recall-0.5": 0.999318,
+    "num_labels": 1466
+  },
+  "shoulder_right": {
+    "count": 1464,
+    "mean": 0.020356,
+    "median": 0.0193,
+    "std": 0.008406,
+    "sem": 0.00022,
+    "min": 0.002013,
+    "max": 0.087962,
+    "recall-0.025": 0.776792,
+    "recall-0.05": 0.992491,
+    "recall-0.1": 0.999317,
+    "recall-0.15": 0.999317,
+    "recall-0.25": 0.999317,
+    "recall-0.5": 0.999317,
+    "num_labels": 1465
+  },
+  "elbow_left": {
+    "count": 1464,
+    "mean": 0.021276,
+    "median": 0.018298,
+    "std": 0.013204,
+    "sem": 0.000345,
+    "min": 0.002104,
+    "max": 0.147682,
+    "recall-0.025": 0.716041,
+    "recall-0.05": 0.964505,
+    "recall-0.1": 0.998635,
+    "recall-0.15": 0.999317,
+    "recall-0.25": 0.999317,
+    "recall-0.5": 0.999317,
+    "num_labels": 1465
+  },
+  "elbow_right": {
+    "count": 1462,
+    "mean": 0.019708,
+    "median": 0.017321,
+    "std": 0.012077,
+    "sem": 0.000316,
+    "min": 0.001927,
+    "max": 0.132247,
+    "recall-0.025": 0.791524,
+    "recall-0.05": 0.969925,
+    "recall-0.1": 0.997266,
+    "recall-0.15": 0.999316,
+    "recall-0.25": 0.999316,
+    "recall-0.5": 0.999316,
+    "num_labels": 1463
+  },
+  "wrist_left": {
+    "count": 1433,
+    "mean": 0.020952,
+    "median": 0.015145,
+    "std": 0.021486,
+    "sem": 0.000568,
+    "min": 0.001105,
+    "max": 0.195801,
+    "recall-0.025": 0.761506,
+    "recall-0.05": 0.939331,
+    "recall-0.1": 0.981869,
+    "recall-0.15": 0.993724,
+    "recall-0.25": 0.999303,
+    "recall-0.5": 0.999303,
+    "num_labels": 1434
+  },
+  "wrist_right": {
+    "count": 1456,
+    "mean": 0.020586,
+    "median": 0.013908,
+    "std": 0.023086,
+    "sem": 0.000605,
+    "min": 0.000284,
+    "max": 0.437929,
+    "recall-0.025": 0.76717,
+    "recall-0.05": 0.932005,
+    "recall-0.1": 0.986264,
+    "recall-0.15": 0.995879,
+    "recall-0.25": 0.999313,
+    "recall-0.5": 1.0,
+    "num_labels": 1456
+  },
+  "hip_left": {
+    "count": 1464,
+    "mean": 0.049993,
+    "median": 0.048091,
+    "std": 0.01908,
+    "sem": 0.000499,
+    "min": 0.00494,
+    "max": 0.161097,
+    "recall-0.025": 0.047782,
+    "recall-0.05": 0.556314,
+    "recall-0.1": 0.975427,
+    "recall-0.15": 0.998635,
+    "recall-0.25": 0.999317,
+    "recall-0.5": 0.999317,
+    "num_labels": 1465
+  },
+  "hip_right": {
+    "count": 1465,
+    "mean": 0.0503,
+    "median": 0.049116,
+    "std": 0.017259,
+    "sem": 0.000451,
+    "min": 0.007258,
+    "max": 0.161485,
+    "recall-0.025": 0.055935,
+    "recall-0.05": 0.514325,
+    "recall-0.1": 0.98704,
+    "recall-0.15": 0.998636,
+    "recall-0.25": 0.999318,
+    "recall-0.5": 0.999318,
+    "num_labels": 1466
+  },
+  "knee_left": {
+    "count": 1464,
+    "mean": 0.038607,
+    "median": 0.031936,
+    "std": 0.027955,
+    "sem": 0.000731,
+    "min": 0.001986,
+    "max": 0.263932,
+    "recall-0.025": 0.331741,
+    "recall-0.05": 0.767235,
+    "recall-0.1": 0.967918,
+    "recall-0.15": 0.989761,
+    "recall-0.25": 0.998635,
+    "recall-0.5": 0.999317,
+    "num_labels": 1465
+  },
+  "knee_right": {
+    "count": 1458,
+    "mean": 0.042193,
+    "median": 0.035821,
+    "std": 0.02818,
+    "sem": 0.000738,
+    "min": 0.003916,
+    "max": 0.268463,
+    "recall-0.025": 0.239205,
+    "recall-0.05": 0.742975,
+    "recall-0.1": 0.957505,
+    "recall-0.15": 0.986977,
+    "recall-0.25": 0.998629,
+    "recall-0.5": 0.999315,
+    "num_labels": 1459
+  },
+  "ankle_left": {
+    "count": 1456,
+    "mean": 0.087391,
+    "median": 0.042765,
+    "std": 0.105221,
+    "sem": 0.002758,
+    "min": 0.000814,
+    "max": 0.499075,
+    "recall-0.025": 0.350649,
+    "recall-0.05": 0.549556,
+    "recall-0.1": 0.7054,
+    "recall-0.15": 0.822967,
+    "recall-0.25": 0.909774,
+    "recall-0.5": 0.995215,
+    "num_labels": 1463
+  },
+  "ankle_right": {
+    "count": 1445,
+    "mean": 0.076361,
+    "median": 0.04236,
+    "std": 0.09602,
+    "sem": 0.002527,
+    "min": 0.001734,
+    "max": 0.49866,
+    "recall-0.025": 0.320548,
+    "recall-0.05": 0.563699,
+    "recall-0.1": 0.791781,
+    "recall-0.15": 0.85274,
+    "recall-0.25": 0.912329,
+    "recall-0.5": 0.989726,
+    "num_labels": 1460
+  },
+  "joint_recalls": {
+    "num_labels": 18990,
+    "recall-0.025": 0.5316,
+    "recall-0.05": 0.80527,
+    "recall-0.1": 0.9495,
+    "recall-0.15": 0.97162,
+    "recall-0.25": 0.9852,
+    "recall-0.5": 0.99789
+  }
+}
+{
+  "total_parts": 20444,
+  "correct_parts": 20190,
+  "pcp": 0.987576
+}
+```
+
 ### IkeaAsm
 
 (duration 00:01:45)
diff --git a/scripts/test_skelda_dataset.py b/scripts/test_skelda_dataset.py
index 02df3c9..67065f4 100644
--- a/scripts/test_skelda_dataset.py
+++ b/scripts/test_skelda_dataset.py
@@ -26,6 +26,8 @@ datasets = {
     "panoptic": {
         "path": "/datasets/panoptic/skelda/test.json",
         "cams": ["00_03", "00_06", "00_12", "00_13", "00_23"],
+        # "cams": ["00_03", "00_06", "00_12"],
+        # "cams": ["00_03", "00_06", "00_12", "00_13", "00_23", "00_15", "00_10", "00_21", "00_09", "00_01"],
         "take_interval": 3,
         "use_scenes": ["160906_pizza1", "160422_haggling1", "160906_ian5"],
     },