diff --git a/.gitignore b/.gitignore index fceb75b..ea206b4 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,7 @@ dmypy.json # Cython debug symbols cython_debug/ + +output/* + +GREW_result/* diff --git a/configs/skeletongait/README.md b/configs/skeletongait/README.md index 5f7b325..7563a68 100644 --- a/configs/skeletongait/README.md +++ b/configs/skeletongait/README.md @@ -2,7 +2,9 @@ This [paper](https://arxiv.org/abs/2311.13444) has been accepted by AAAI 2024. -## Step 1: Generating Heatmap +## Generating Heatmap and Training Steps + +### Step 1: Generating Heatmap Leveraging the power of Distributed Data Parallel (DDP), we've streamlined the heatmap generation process. Below is the script to initiate the generation: ``` CUDA_VISIBLE_DEVICES=0,1,2,3 \ @@ -21,9 +23,12 @@ Parameter Guide: - `--ext_name`: An **optional** suffix for the 'save_root' directory to facilitate identification. Defaults to an empty string. - `--heatmap_cfg_path`: Path to the configuration file of the heatmap generator. The default setting is `configs/skeletongait/pretreatment_heatmap.yaml`. +Note: If your pose data follows the COCO 18 format (for instance, OU-MVLP pose data or data extracted using [OpenPose](https://github.com/CMU-Perceptual-Computing-Lab/openpose) in COCO format), ensure to set `transfer_to_coco17` to True in the configuration file `configs/skeletongait/pretreatment_heatmap.yaml`. + + **Optional** -## Step 2: Creating Symbolic Links for Heatmap and Silhouette Data +### Step 2: Creating Symbolic Links for Heatmap and Silhouette Data The script to symlink heatmaps and silouettes is as follows: @@ -38,9 +43,9 @@ Parameter Guide: - `--heatmap_data_path`: The **absolute** path to your heatmap data. This is **required**. - `--silhouette_data_path`: The **absolute** path to your silhouette data. This is **required**. - `--output_path`: Designates the directory for linked output data. This is **required**. -- `--dataset_pkl_ext_name`: An **optional** parameter to specify the extension for `.pkl` silhouette files. Defaults to `.pkl`. +- `--dataset_pkl_ext_name`: An **optional** parameter to specify the extension for `.pkl` silhouette files. Defaults to `.pkl`. CCPG is `aligned-sils.pkl`, SUSTech-1K is `Camera-Sils_aligned.pkl`, and other is `.pkl`. -## Step3: Training SkeletonGait or SkeletonGait++ +### Step3: Training SkeletonGait or SkeletonGait++ The script to SkeletonGait is as follows: @@ -61,3 +66,24 @@ CUDA_VISIBLE_DEVICES=0,1,2,3 \ --cfgs ./configs/skeletongait/skeletongait++_Gait3D.yaml \ --phase train --log_to_file ``` + +## Performance for SkeletonGait and SkeletonGait++ + +### SkeletonGait +| Datasets | `Rank1` | Configuration | +|---------------------|---------|----------------------------------------------| +| CCPG | CL: 52.4, UP: 65.4, DN: 72.8, BG: 80.9 | [skeletongait_CCPG.yaml](./skeletongait_CCPG.yaml) | +| OU-MVLP (AlphaPose) | TODO | [skeletongait_OUMVLP.yaml](./skeletongait_OUMVLP.yaml) | +| SUSTech-1K | Normal: 54.2, Bag: 51.7, Clothing: 21.34, Carrying: 51.59, Umberalla: 44.5, Uniform: 53.37, Occlusion: 67.07, Night: 44.15, Overall: 51.46 | [skeletongait_SUSTech1K.yaml](./skeletongait_SUSTech1K.yaml) | +| Gait3D | 38.1 | [skeletongait_Gait3D.yaml](./skeletongait_Gait3D.yaml) | +| GREW | TODO | [skeletongait_GREW.yaml](./skeletongait_GREW.yaml) | + +### SkeletonGait++ +| Datasets | `Rank1` | Configuration | +|---------------------|---------|-------------------------------------------------| +| CCPG | CL: 90.1, UP: 95.0, DN: 92.9, BG: 97.0 | [skeletongait++_CCPG.yaml](./skeletongait++_CCPG.yaml) | +| SUSTech-1K | Normal: 85.09, Bag: 82.90, Clothing: 46.53, Carrying: 81.88, Umberalla: 80.76, Uniform: 82.50, Occlusion: 86.16, Night: 47.48, Overall: 81.33 | [skeletongait++_SUSTech1K.yaml](./skeletongait++_SUSTech1K.yaml) | +| Gait3D | 77.40 | [skeletongait++_Gait3D.yaml](./skeletongait++_Gait3D.yaml) | +| GREW | 87.04 | [skeletongait++_GREW.yaml](./skeletongait++_GREW.yaml) | + + diff --git a/configs/skeletongait/skeletongait++_CCPG.yaml b/configs/skeletongait/skeletongait++_CCPG.yaml index c051c05..86f6ecc 100644 --- a/configs/skeletongait/skeletongait++_CCPG.yaml +++ b/configs/skeletongait/skeletongait++_CCPG.yaml @@ -1,6 +1,6 @@ data_cfg: dataset_name: CCPG - dataset_root: your_path + dataset_root: /data3/gait_heatmap_data/CCPG/heatmap_sil dataset_partition: ./datasets/CCPG/CCPG.json num_workers: 1 data_in_use: [True, True] # heatmap, sil @@ -10,7 +10,7 @@ data_cfg: evaluator_cfg: enable_float16: true restore_ckpt_strict: true - restore_hint: 60000 + restore_hint: /home/jdy/fanchao/OpenGait_230701/output/CCPG/DeepGaitV2_P3D_GaitMap_MM_FeatureLevel/DeepGaitV2_P3D_GaitMap_B1C2_Sigma-8.0_Hot_False_Align-True_OpenGaitDA-True_ML_LowLevel/checkpoints/DeepGaitV2_P3D_GaitMap_B1C2_Sigma-8.0_Hot_False_Align-True_OpenGaitDA-True_ML_LowLevel-60000.pt save_name: SkeletonGaitPP eval_func: evaluate_CCPG sampler: diff --git a/configs/skeletongait/skeletongait++_GREW.yaml b/configs/skeletongait/skeletongait++_GREW.yaml index 0cbe899..07c1911 100644 --- a/configs/skeletongait/skeletongait++_GREW.yaml +++ b/configs/skeletongait/skeletongait++_GREW.yaml @@ -1,6 +1,6 @@ data_cfg: dataset_name: GREW - dataset_root: your_path + dataset_root: /data3/gait_heatmap_data/GREW/heatmap_sil dataset_partition: ./datasets/GREW/GREW.json num_workers: 1 data_in_use: [True, True] # heatmap, sil @@ -10,7 +10,7 @@ data_cfg: evaluator_cfg: enable_float16: true restore_ckpt_strict: true - restore_hint: 180000 + restore_hint: /home/jdy/fanchao/OpenGait_230701/output/GREW/DeepGaitV2_P3D_GaitMap_MM_FeatureLevel/DeepGaitV2_P3D_GaitMap_B4C2_Sigma-8.0_Hot_False_Align-True_OpenGaitDA-True_LowLevel-AttentionFusion/checkpoints/DeepGaitV2_P3D_GaitMap_B4C2_Sigma-8.0_Hot_False_Align-True_OpenGaitDA-True_LowLevel-AttentionFusion-180000.pt save_name: SkeletonGaitPP eval_func: GREW_submission sampler: diff --git a/configs/skeletongait/skeletongait++_Gait3D.yaml b/configs/skeletongait/skeletongait++_Gait3D.yaml index 327476d..1d82809 100644 --- a/configs/skeletongait/skeletongait++_Gait3D.yaml +++ b/configs/skeletongait/skeletongait++_Gait3D.yaml @@ -10,7 +10,7 @@ data_cfg: evaluator_cfg: enable_float16: true restore_ckpt_strict: true - restore_hint: 60000 + restore_hint: /home/jdy/fanchao/OpenGait_230701/output/Gait3D/DeepGaitV2_P3D_GaitMap_MM_FeatureLevel/DeepGaitV2_P3D_GaitMap_B4C2_Sigma-8.0_Hot_False_Align-True_OpenGaitDA-True_MM_FeatureLevel_AttentionFusion/checkpoints/DeepGaitV2_P3D_GaitMap_B4C2_Sigma-8.0_Hot_False_Align-True_OpenGaitDA-True_MM_FeatureLevel_AttentionFusion-60000.pt save_name: SkeletonGaitPP # LowLevel eval_func: evaluate_Gait3D sampler: @@ -73,7 +73,7 @@ trainer_cfg: sampler: batch_shuffle: true batch_size: - - 4 # TripletSampler, batch_size[0] indicates Number of Identity + - 32 # TripletSampler, batch_size[0] indicates Number of Identity - 4 # batch_size[1] indicates Samples sequqnce for each Identity frames_num_fixed: 30 # fixed frames number for training frames_skip_num: 4 diff --git a/configs/skeletongait/skeletongait++_SUSTech1K.yaml b/configs/skeletongait/skeletongait++_SUSTech1K.yaml index 65be0c8..709d5a6 100644 --- a/configs/skeletongait/skeletongait++_SUSTech1K.yaml +++ b/configs/skeletongait/skeletongait++_SUSTech1K.yaml @@ -1,6 +1,6 @@ data_cfg: dataset_name: SUSTech1K - dataset_root: your_path + dataset_root: /data3/gait_heatmap_data/SUSTech1K/heatmap_sil dataset_partition: ./datasets/SUSTech1K/SUSTech1K.json num_workers: 4 data_in_use: [True, True] # heatmap, sil @@ -10,7 +10,7 @@ data_cfg: evaluator_cfg: enable_float16: true restore_ckpt_strict: true - restore_hint: 50000 + restore_hint: /home/jdy/fanchao/OpenGait_230701/output/SUSTech1K/DeepGaitV2_P3D_GaitMap_MM_FeatureLevel/DeepGaitV2_P3D_GaitMap_B1C2_Sigma-8.0_Hot_False_Align-True_OpenGaitDA-True_ML_LowLevel/checkpoints/DeepGaitV2_P3D_GaitMap_B1C2_Sigma-8.0_Hot_False_Align-True_OpenGaitDA-True_ML_LowLevel-50000.pt save_name: SkeletonGaitPP eval_func: evaluate_indoor_dataset #evaluate_Gait3D sampler: diff --git a/configs/skeletongait/skeletongait_Gait3D.yaml b/configs/skeletongait/skeletongait_Gait3D.yaml index 78bcb69..027f683 100644 --- a/configs/skeletongait/skeletongait_Gait3D.yaml +++ b/configs/skeletongait/skeletongait_Gait3D.yaml @@ -1,6 +1,6 @@ data_cfg: dataset_name: Gait3D - dataset_root: /data3/gait_heatmap_data/Gait3D/heatmap_sil + dataset_root: your_path dataset_partition: ./datasets/Gait3D/Gait3D.json num_workers: 1 data_in_use: [True, False] # heatmap, sil @@ -78,7 +78,7 @@ trainer_cfg: sampler: batch_shuffle: true batch_size: - - 4 # TripletSampler, batch_size[0] indicates Number of Identity + - 32 # TripletSampler, batch_size[0] indicates Number of Identity - 4 # batch_size[1] indicates Samples sequqnce for each Identity frames_num_fixed: 30 # fixed frames number for training frames_skip_num: 4 diff --git a/configs/skeletongait/skeletongait_OUMVLP.yaml b/configs/skeletongait/skeletongait_OUMVLP.yaml index b02b2dc..0aa8960 100644 --- a/configs/skeletongait/skeletongait_OUMVLP.yaml +++ b/configs/skeletongait/skeletongait_OUMVLP.yaml @@ -1,6 +1,6 @@ data_cfg: dataset_name: OUMVLP - dataset_root: your_path + dataset_root: /data3/gait_heatmap_data/OUMVLP/OUMVLP_sigma_8.0_/pkl dataset_partition: ./datasets/OUMVLP/OUMVLP.json num_workers: 1 data_in_use: [True, False] # heatmap, sil @@ -10,7 +10,7 @@ data_cfg: evaluator_cfg: enable_float16: true restore_ckpt_strict: true - restore_hint: 120000 + restore_hint: /home/jdy/fanchao/OpenGait_230701/output/OUMVLP/DeepGaitV2_P3D_GaitMap/DeepGaitV2_P3D_GaitMap_B4C2_Sigma-8.0_Hot_False_Align-True/checkpoints/DeepGaitV2_P3D_GaitMap_B4C2_Sigma-8.0_Hot_False_Align-True-120000.pt save_name: SkeletonGait sampler: batch_shuffle: false diff --git a/configs/skeletongait/skeletongait_SUSTech1K.yaml b/configs/skeletongait/skeletongait_SUSTech1K.yaml index 76d51ec..e1a0aa0 100644 --- a/configs/skeletongait/skeletongait_SUSTech1K.yaml +++ b/configs/skeletongait/skeletongait_SUSTech1K.yaml @@ -1,6 +1,6 @@ data_cfg: dataset_name: SUSTech1K - dataset_root: your_path + dataset_root: /data3/gait_heatmap_data/SUSTech1K/SUSTech1K_sigma_8.0_/pkl dataset_partition: ./datasets/SUSTech1K/SUSTech1K.json num_workers: 4 data_in_use: [True, False] # heatmap, sil @@ -10,7 +10,7 @@ data_cfg: evaluator_cfg: enable_float16: true restore_ckpt_strict: true - restore_hint: 50000 + restore_hint: /home/jdy/fanchao/OpenGait_230701/output/SUSTech1K/DeepGaitV2_P3D_GaitMap/DeepGaitV2_P3D_GaitMap_B1C2_Sigma-8.0_Hot_False_Align-True_OpenGaitDA-True/checkpoints/DeepGaitV2_P3D_GaitMap_B1C2_Sigma-8.0_Hot_False_Align-True_OpenGaitDA-True-50000.pt save_name: SkeletonGait eval_func: evaluate_indoor_dataset #evaluate_Gait3D sampler: diff --git a/datasets/ln_sil_heatmap.py b/datasets/ln_sil_heatmap.py index dbbbd3d..7348443 100644 --- a/datasets/ln_sil_heatmap.py +++ b/datasets/ln_sil_heatmap.py @@ -6,8 +6,8 @@ from glob import glob def get_args(): parser = argparse.ArgumentParser(description='Symlink silouette data and pose data into the same folder for SkeletonGait++ training.') - parser.add_argument('--heatmapt_data_path', type=str, required=True, help="path of heatmap data, must be the absolute path.") - parser.add_argument('--silouette_data_path', type=str, required=True, help="path of silouette data, must be the absolute path.") + parser.add_argument('--heatmap_data_path', type=str, required=True, help="path of heatmap data, must be the absolute path.") + parser.add_argument('--silhouette_data_path', type=str, required=True, help="path of silouette data, must be the absolute path.") parser.add_argument('--dataset_pkl_ext_name', type=str, default='.pkl', help="The extent name for .pkl files of silouettes data.") parser.add_argument('--output_path', type=str, required=True, help="path of output data") opt = parser.parse_args() @@ -15,24 +15,24 @@ def get_args(): def main(): opt = get_args() - heatmap_data_path = opt.heatmapt_data_path - silouette_data_path = opt.silouette_data_path + heatmap_data_path = opt.heatmap_data_path + silhouette_data_path = opt.silhouette_data_path if not os.path.exists(heatmap_data_path): print(f"heatmap data path {heatmap_data_path} does not exist.") sys.exit(1) - if not os.path.exists(silouette_data_path): - print(f"silouette data path {silouette_data_path} does not exist.") + if not os.path.exists(silhouette_data_path): + print(f"silouette data path {silhouette_data_path} does not exist.") sys.exit(1) all_heatmap_files = sorted(glob(os.path.join(heatmap_data_path, "*/*/*/*.pkl"))) - all_silouette_files = sorted(glob(os.path.join(silouette_data_path, f"*/*/*/*{opt.dataset_pkl_ext_name}"))) + all_silouette_files = sorted(glob(os.path.join(silhouette_data_path, f"*/*/*/*{opt.dataset_pkl_ext_name}"))) # print(len(all_heatmap_files), len(all_silouette_files)) # assert len(all_heatmap_files) == len(all_silouette_files), "The number of heatmap files and silouette files are not equal." if len(all_heatmap_files) >= len(all_silouette_files): for heatmap_file in tqdm(all_heatmap_files): tmp_list = heatmap_file.split('/') - sil_folder = os.path.join(silouette_data_path, *tmp_list[-4:-1]) + sil_folder = os.path.join(silhouette_data_path, *tmp_list[-4:-1]) if not os.path.exists(sil_folder): print(f"silouette folder {sil_folder} does not exist.") continue diff --git a/datasets/pretreatment_heatmap.py b/datasets/pretreatment_heatmap.py index 0c2b7f7..effa922 100644 --- a/datasets/pretreatment_heatmap.py +++ b/datasets/pretreatment_heatmap.py @@ -647,8 +647,8 @@ class TransferDataset(Dataset): os.makedirs(save_path_img, exist_ok=True) # save_heatemapimg_index = random.choice(list(range(heatmap_img.shape[0]))) for save_heatemapimg_index in range(heatmap_img.shape[0]): - cv2.imwrite(os.path.join(save_path_img, f'pose_{save_heatemapimg_index}.jpg'), heatmap_img[save_heatemapimg_index, 0]) - cv2.imwrite(os.path.join(save_path_img, f'bone_{save_heatemapimg_index}.jpg'), heatmap_img[save_heatemapimg_index, 1]) + cv2.imwrite(os.path.join(save_path_img, f'bone_{save_heatemapimg_index}.jpg'), heatmap_img[save_heatemapimg_index, 0]) + cv2.imwrite(os.path.join(save_path_img, f'pose_{save_heatemapimg_index}.jpg'), heatmap_img[save_heatemapimg_index, 1]) pickle.dump(heatmap_img, open(os.path.join(save_path_pkl, tmp_split[-1]), 'wb')) return None diff --git a/test.sh b/test.sh index 443b7bc..414b252 100644 --- a/test.sh +++ b/test.sh @@ -1,6 +1,6 @@ # # **************** For CASIA-B **************** # # Baseline -CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 opengait/main.py --cfgs ./configs/baseline/baseline.yaml --phase test +CUDA_VISIBLE_DEVICES=0,1,2,3 NCCL_P2P_DISABLE=1 python -m torch.distributed.launch --nproc_per_node=4 opengait/main.py --cfgs ./configs/skeletongait/skeletongait++_GREW.yaml --phase test --log_to_file # # GaitSet # CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 opengait/main.py --cfgs ./configs/gaitset/gaitset.yaml --phase test