From 78171dba917c9704c22210f335f9e2ee8ba7328d Mon Sep 17 00:00:00 2001 From: darkliang <11710911@mail.sustech.edu.cn> Date: Tue, 27 Jun 2023 21:25:38 +0800 Subject: [PATCH] support CCPG --- configs/gaitbase/gaitbase_ccpg.yaml | 104 ++++++++++++++ datasets/CCPG/CCPG.json | 206 ++++++++++++++++++++++++++++ datasets/CCPG/README.md | 27 ++++ datasets/CCPG/organize_ccpg.py | 101 ++++++++++++++ opengait/evaluation/evaluator.py | 126 ++++++++++++++++- opengait/evaluation/metric.py | 63 +++++++++ 6 files changed, 625 insertions(+), 2 deletions(-) create mode 100644 configs/gaitbase/gaitbase_ccpg.yaml create mode 100644 datasets/CCPG/CCPG.json create mode 100644 datasets/CCPG/README.md create mode 100644 datasets/CCPG/organize_ccpg.py diff --git a/configs/gaitbase/gaitbase_ccpg.yaml b/configs/gaitbase/gaitbase_ccpg.yaml new file mode 100644 index 0000000..6344293 --- /dev/null +++ b/configs/gaitbase/gaitbase_ccpg.yaml @@ -0,0 +1,104 @@ +data_cfg: + dataset_name: CCPG + dataset_root: your_path + dataset_partition: ./datasets/CCPG/CCPG.json + num_workers: 1 + data_in_use: [True,False,False,False] + remove_no_gallery: false # Remove probe if no gallery for it + test_dataset_name: CCPG + +evaluator_cfg: + enable_float16: true + restore_ckpt_strict: true + restore_hint: 60000 + save_name: GaitBase + eval_func: evaluate_CCPG + sampler: + batch_shuffle: false + batch_size: 16 + sample_type: all_ordered # all indicates whole sequence used to test, while ordered means input sequence by its natural order; Other options: fixed_unordered + frames_all_limit: 720 # limit the number of sampled frames to prevent out of memory + metric: euc # cos + transform: + - type: BaseSilCuttingTransform + img_w: 64 +loss_cfg: + - loss_term_weight: 1.0 + margin: 0.2 + type: TripletLoss + log_prefix: triplet + - loss_term_weight: 1.0 + scale: 16 + type: CrossEntropyLoss + log_prefix: softmax + log_accuracy: true + +model_cfg: + model: Baseline + backbone_cfg: + type: ResNet9 + block: BasicBlock + channels: # Layers configuration for automatically model construction + - 64 + - 128 + - 256 + - 512 + layers: + - 1 + - 1 + - 1 + - 1 + strides: + - 1 + - 2 + - 2 + - 1 + maxpool: false + SeparateFCs: + in_channels: 512 + out_channels: 256 + parts_num: 16 + SeparateBNNecks: + class_num: 100 + in_channels: 256 + parts_num: 16 + bin_num: + - 16 + +optimizer_cfg: + lr: 0.1 + momentum: 0.9 + solver: SGD + weight_decay: 0.0005 + +scheduler_cfg: + gamma: 0.1 + milestones: # Learning Rate Reduction at each milestones + - 20000 + - 40000 + - 50000 + scheduler: MultiStepLR +trainer_cfg: + enable_float16: true # half_percesion float for memory reduction and speedup + fix_BN: false + with_test: false + log_iter: 100 + restore_ckpt_strict: true + restore_hint: 0 + save_iter: 60000 + save_name: GaitBase + sync_BN: true + total_iter: 60000 + sampler: + batch_shuffle: true + batch_size: + - 8 # TripletSampler, batch_size[0] indicates Number of Identity + - 16 # batch_size[1] indicates Samples sequqnce for each Identity + frames_num_fixed: 30 # fixed frames number for training + frames_num_max: 50 # max frames number for unfixed training + frames_num_min: 10 # min frames number for unfixed traing + sample_type: fixed_unordered # fixed control input frames number, unordered for controlling order of input tensor; Other options: unfixed_ordered or all_ordered + type: TripletSampler + transform: + - type: BaseSilCuttingTransform + img_w: 64 \ No newline at end of file diff --git a/datasets/CCPG/CCPG.json b/datasets/CCPG/CCPG.json new file mode 100644 index 0000000..24876e6 --- /dev/null +++ b/datasets/CCPG/CCPG.json @@ -0,0 +1,206 @@ +{ + "TRAIN_SET": [ + "000", + "001", + "002", + "003", + "004", + "005", + "006", + "007", + "008", + "009", + "010", + "011", + "012", + "013", + "014", + "015", + "016", + "017", + "018", + "019", + "020", + "021", + "022", + "023", + "024", + "025", + "026", + "027", + "028", + "029", + "030", + "031", + "032", + "033", + "034", + "035", + "036", + "037", + "038", + "039", + "040", + "041", + "042", + "043", + "044", + "045", + "046", + "047", + "048", + "049", + "050", + "051", + "052", + "053", + "054", + "055", + "056", + "057", + "058", + "059", + "060", + "061", + "062", + "063", + "064", + "065", + "066", + "067", + "068", + "069", + "070", + "071", + "072", + "073", + "074", + "075", + "076", + "077", + "078", + "079", + "080", + "081", + "082", + "083", + "084", + "085", + "086", + "087", + "088", + "089", + "090", + "091", + "092", + "093", + "094", + "095", + "096", + "097", + "098", + "099" + ], + "TEST_SET": [ + "100", + "101", + "102", + "103", + "104", + "105", + "106", + "107", + "108", + "109", + "110", + "111", + "112", + "113", + "114", + "115", + "116", + "117", + "118", + "119", + "120", + "121", + "122", + "123", + "124", + "125", + "126", + "127", + "128", + "129", + "130", + "131", + "132", + "133", + "134", + "135", + "136", + "137", + "138", + "139", + "140", + "141", + "142", + "143", + "144", + "145", + "146", + "147", + "148", + "149", + "150", + "151", + "152", + "153", + "154", + "155", + "156", + "157", + "158", + "159", + "160", + "161", + "162", + "163", + "164", + "165", + "166", + "167", + "168", + "169", + "170", + "171", + "172", + "173", + "174", + "175", + "176", + "177", + "178", + "179", + "180", + "181", + "182", + "183", + "184", + "185", + "186", + "187", + "188", + "189", + "190", + "191", + "192", + "193", + "194", + "195", + "196", + "197", + "198", + "199" + ] +} \ No newline at end of file diff --git a/datasets/CCPG/README.md b/datasets/CCPG/README.md new file mode 100644 index 0000000..f041cc6 --- /dev/null +++ b/datasets/CCPG/README.md @@ -0,0 +1,27 @@ +# The CCPG Benchmark + +A Cloth-Changing Benchmark for Person re-identification and Gait Recognition (CCPG). + +The original dataset can be found [here](https://github.com/BNU-IVC/CCPG). The original dataset is not publicly available. You need to request access to the dataset in order to download it. +## Data Pretreatment +```python +python datasets/CCPG/organize_ccpg.py --sil_path 'CCPG/CCPG_D_MASK_FACE_SHOE' --rgb_path 'CCPG/CCPG_G_SIL' --output_path 'CCPG/CCPG-end2end-pkl' +``` + +## Train +### GatiBase model: +`CUDA_VISIBLE_DEVICES=0,1,2,3 python -m torch.distributed.launch --nproc_per_node=4 opengait/main.py --cfgs ./configs/gaitbase/gaitbase_ccpg.yaml --phase train` + + +## Citation +If you use this dataset in your research, please cite the following paper: +``` +@InProceedings{Li_2023_CVPR, + author = {Li, Weijia and Hou, Saihui and Zhang, Chunjie and Cao, Chunshui and Liu, Xu and Huang, Yongzhen and Zhao, Yao}, + title = {An In-Depth Exploration of Person Re-Identification and Gait Recognition in Cloth-Changing Conditions}, + booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, + month = {June}, + year = {2023}, + pages = {13824-13833} +} +``` diff --git a/datasets/CCPG/organize_ccpg.py b/datasets/CCPG/organize_ccpg.py new file mode 100644 index 0000000..73c1ff6 --- /dev/null +++ b/datasets/CCPG/organize_ccpg.py @@ -0,0 +1,101 @@ +import os +import pickle +import numpy as np +import cv2 +from tqdm import tqdm +import argparse + + +T_W = 64 +T_H = 64 + + +def cut_img(img): + # A silhouette contains too little white pixels + # might be not valid for identification. + # Get the top and bottom point + y = img.sum(axis=1) + y_top = (y != 0).argmax(axis=0) + y_btm = (y != 0).cumsum(axis=0).argmax(axis=0) + img = img[y_top:y_btm + 1, :] + # As the height of a person is larger than the width, + # use the height to calculate resize ratio. + _r = img.shape[1] / img.shape[0] + _t_w = int(T_H * _r) + img = cv2.resize(img, (_t_w, T_H), interpolation=cv2.INTER_AREA) + # Get the median of x axis and regard it as the x center of the person. + sum_point = img.sum() + sum_column = img.sum(axis=0).cumsum() + x_center = -1 + for i in range(sum_column.size): + if sum_column[i] > sum_point / 2: + x_center = i + break + if x_center < 0: + return None + h_T_W = int(T_W / 2) + left = x_center - h_T_W + right = x_center + h_T_W + if left <= 0 or right >= img.shape[1]: + left += h_T_W + right += h_T_W + _ = np.zeros((img.shape[0], h_T_W)) + img = np.concatenate([_, img, _], axis=1) + img = img[:, left:right] + return img.astype('uint8') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='CCPG dataset Preprocessing.') + parser.add_argument('--sil_path', default='', type=str, + help='Root path of raw silhouette dataset.') + parser.add_argument('--rgb_path', default='', type=str, + help='Root path of raw RGB dataset.') + parser.add_argument('-o', '--output_path', default='', + type=str, help='Output path of pickled dataset.') + args = parser.parse_args() + + RGB_SIZE = (128, 128) + for _id in tqdm(sorted(os.listdir(args.sil_path))): + for _type in sorted(os.listdir(os.path.join(args.rgb_path, _id))): + for _view in sorted(os.listdir(os.path.join(args.rgb_path, _id, _type))): + imgs = [] + segs = [] + ratios = [] + aligned_segs = [] + for img_file in sorted(os.listdir(os.path.join(args.rgb_path, _id, _type, _view))): + seg_file = img_file.split(".")[0]+".png" + img_path = os.path.join( + args.rgb_path, _id, _type, _view, img_file) + seg_path = os.path.join( + args.rgb_path, _id, _type, _view, seg_file) + if not os.path.exists(seg_path): + print("Not Found: "+seg_path) + continue + img = cv2.imread(img_path) + seg = cv2.imread(seg_path, cv2.IMREAD_GRAYSCALE) + ratio = img.shape[1]/img.shape[0] + aligned_seg = cut_img(seg) + img = np.transpose(cv2.cvtColor(cv2.resize( + img, RGB_SIZE), cv2.COLOR_BGR2RGB), (2, 0, 1)) + imgs.append(img) + segs.append(cv2.resize( + seg, RGB_SIZE)) + aligned_segs.append(aligned_seg) + ratios.append(ratio) + if len(imgs) > 0: + output_path = os.path.join( + args.output_path, _id, _type, _view) + os.makedirs(output_path, exist_ok=True) + pickle.dump(np.asarray(imgs), open(os.path.join( + output_path, _view+"-rgbs.pkl"), "wb")) + pickle.dump(np.asarray(segs), open(os.path.join( + output_path, _view+"-sils.pkl"), "wb")) + pickle.dump(np.asarray(ratios), open(os.path.join( + output_path, _view+"-ratios.pkl"), "wb")) + pickle.dump(np.asarray(aligned_segs), open(os.path.join( + output_path, _view+"-aligned-sils.pkl"), "wb")) + else: + print("No imgs Found: " + + os.path.join(args.rgb_path, _id, _type, _view)) + continue diff --git a/opengait/evaluation/evaluator.py b/opengait/evaluation/evaluator.py index c030339..896e4ae 100644 --- a/opengait/evaluation/evaluator.py +++ b/opengait/evaluation/evaluator.py @@ -3,7 +3,7 @@ from time import strftime, localtime import numpy as np from utils import get_msg_mgr, mkdir -from .metric import mean_iou, cuda_dist, compute_ACC_mAP, evaluate_rank +from .metric import mean_iou, cuda_dist, compute_ACC_mAP, evaluate_rank, evaluate_many from .re_rank import re_ranking @@ -71,7 +71,7 @@ def cross_view_gallery_evaluation(feature, label, seq_type, view, dataset, metri def single_view_gallery_evaluation(feature, label, seq_type, view, dataset, metric): probe_seq_dict = {'CASIA-B': {'NM': ['nm-05', 'nm-06'], 'BG': ['bg-01', 'bg-02'], 'CL': ['cl-01', 'cl-02']}, 'OUMVLP': {'NM': ['00']}, - 'CASIA-E': {'NM': ['H-scene2-nm-1', 'H-scene2-nm-2', 'L-scene2-nm-1', 'L-scene2-nm-2', 'H-scene3-nm-1', 'H-scene3-nm-2', 'L-scene3-nm-1', 'L-scene3-nm-2', 'H-scene3_s-nm-1', 'H-scene3_s-nm-2', 'L-scene3_s-nm-1', 'L-scene3_s-nm-2',], + 'CASIA-E': {'NM': ['H-scene2-nm-1', 'H-scene2-nm-2', 'L-scene2-nm-1', 'L-scene2-nm-2', 'H-scene3-nm-1', 'H-scene3-nm-2', 'L-scene3-nm-1', 'L-scene3-nm-2', 'H-scene3_s-nm-1', 'H-scene3_s-nm-2', 'L-scene3_s-nm-1', 'L-scene3_s-nm-2', ], 'BG': ['H-scene2-bg-1', 'H-scene2-bg-2', 'L-scene2-bg-1', 'L-scene2-bg-2', 'H-scene3-bg-1', 'H-scene3-bg-2', 'L-scene3-bg-1', 'L-scene3-bg-2', 'H-scene3_s-bg-1', 'H-scene3_s-bg-2', 'L-scene3_s-bg-1', 'L-scene3_s-bg-2'], 'CL': ['H-scene2-cl-1', 'H-scene2-cl-2', 'L-scene2-cl-1', 'L-scene2-cl-2', 'H-scene3-cl-1', 'H-scene3-cl-2', 'L-scene3-cl-1', 'L-scene3-cl-2', 'H-scene3_s-cl-1', 'H-scene3_s-cl-2', 'L-scene3_s-cl-1', 'L-scene3_s-cl-2'] } @@ -280,3 +280,125 @@ def evaluate_Gait3D(data, dataset, metric='euc'): # print_csv_format(dataset_name, results) msg_mgr.log_info(results) return results + + +def evaluate_CCPG(data, dataset, metric='euc'): + msg_mgr = get_msg_mgr() + + feature, label, seq_type, view = data['embeddings'], data['labels'], data['types'], data['views'] + + label = np.array(label) + for i in range(len(view)): + view[i] = view[i].split("_")[0] + view_np = np.array(view) + view_list = list(set(view)) + view_list.sort() + + view_num = len(view_list) + + probe_seq_dict = {'CCPG': [["U0_D0_BG", "U0_D0"], [ + "U3_D3"], ["U1_D0"], ["U0_D0_BG"]]} + + gallery_seq_dict = { + 'CCPG': [["U1_D1", "U2_D2", "U3_D3"], ["U0_D3"], ["U1_D1"], ["U0_D0"]]} + if dataset not in (probe_seq_dict or gallery_seq_dict): + raise KeyError("DataSet %s hasn't been supported !" % dataset) + num_rank = 5 + acc = np.zeros([len(probe_seq_dict[dataset]), + view_num, view_num, num_rank]) - 1. + + ap_save = [] + cmc_save = [] + minp = [] + for (p, probe_seq) in enumerate(probe_seq_dict[dataset]): + # for gallery_seq in gallery_seq_dict[dataset]: + gallery_seq = gallery_seq_dict[dataset][p] + gseq_mask = np.isin(seq_type, gallery_seq) + gallery_x = feature[gseq_mask, :] + # print("gallery_x", gallery_x.shape) + gallery_y = label[gseq_mask] + gallery_view = view_np[gseq_mask] + + pseq_mask = np.isin(seq_type, probe_seq) + probe_x = feature[pseq_mask, :] + probe_y = label[pseq_mask] + probe_view = view_np[pseq_mask] + + msg_mgr.log_info( + ("gallery length", len(gallery_y), gallery_seq, "probe length", len(probe_y), probe_seq)) + distmat = cuda_dist(probe_x, gallery_x, metric).cpu().numpy() + # cmc, ap = evaluate(distmat, probe_y, gallery_y, probe_view, gallery_view) + cmc, ap, inp = evaluate_many( + distmat, probe_y, gallery_y, probe_view, gallery_view) + ap_save.append(ap) + cmc_save.append(cmc[0]) + minp.append(inp) + + # print(ap_save, cmc_save) + + msg_mgr.log_info( + '===Rank-1 (Exclude identical-view cases for Person Re-Identification)===') + msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % ( + cmc_save[0]*100, cmc_save[1]*100, cmc_save[2]*100, cmc_save[3]*100)) + + msg_mgr.log_info( + '===mAP (Exclude identical-view cases for Person Re-Identification)===') + msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % ( + ap_save[0]*100, ap_save[1]*100, ap_save[2]*100, ap_save[3]*100)) + + msg_mgr.log_info( + '===mINP (Exclude identical-view cases for Person Re-Identification)===') + msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % + (minp[0]*100, minp[1]*100, minp[2]*100, minp[3]*100)) + + for (p, probe_seq) in enumerate(probe_seq_dict[dataset]): + # for gallery_seq in gallery_seq_dict[dataset]: + gallery_seq = gallery_seq_dict[dataset][p] + for (v1, probe_view) in enumerate(view_list): + for (v2, gallery_view) in enumerate(view_list): + gseq_mask = np.isin(seq_type, gallery_seq) & np.isin( + view, [gallery_view]) + gallery_x = feature[gseq_mask, :] + gallery_y = label[gseq_mask] + + pseq_mask = np.isin(seq_type, probe_seq) & np.isin( + view, [probe_view]) + probe_x = feature[pseq_mask, :] + probe_y = label[pseq_mask] + + dist = cuda_dist(probe_x, gallery_x, metric) + idx = dist.sort(1)[1].cpu().numpy() + # print(p, v1, v2, "\n") + acc[p, v1, v2, :] = np.round( + np.sum(np.cumsum(np.reshape(probe_y, [-1, 1]) == gallery_y[idx[:, 0:num_rank]], 1) > 0, + 0) * 100 / dist.shape[0], 2) + result_dict = {} + for i in range(1): + msg_mgr.log_info( + '===Rank-%d (Include identical-view cases)===' % (i + 1)) + msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % ( + np.mean(acc[0, :, :, i]), + np.mean(acc[1, :, :, i]), + np.mean(acc[2, :, :, i]), + np.mean(acc[3, :, :, i]))) + for i in range(1): + msg_mgr.log_info( + '===Rank-%d (Exclude identical-view cases)===' % (i + 1)) + msg_mgr.log_info('CL: %.3f,\tUP: %.3f,\tDN: %.3f,\tBG: %.3f' % ( + de_diag(acc[0, :, :, i]), + de_diag(acc[1, :, :, i]), + de_diag(acc[2, :, :, i]), + de_diag(acc[3, :, :, i]))) + result_dict["scalar/test_accuracy/CL"] = acc[0, :, :, i] + result_dict["scalar/test_accuracy/UP"] = acc[1, :, :, i] + result_dict["scalar/test_accuracy/DN"] = acc[2, :, :, i] + result_dict["scalar/test_accuracy/BG"] = acc[3, :, :, i] + np.set_printoptions(precision=2, floatmode='fixed') + for i in range(1): + msg_mgr.log_info( + '===Rank-%d of each angle (Exclude identical-view cases)===' % (i + 1)) + msg_mgr.log_info('CL: {}'.format(de_diag(acc[0, :, :, i], True))) + msg_mgr.log_info('UP: {}'.format(de_diag(acc[1, :, :, i], True))) + msg_mgr.log_info('DN: {}'.format(de_diag(acc[2, :, :, i], True))) + msg_mgr.log_info('BG: {}'.format(de_diag(acc[3, :, :, i], True))) + return result_dict diff --git a/opengait/evaluation/metric.py b/opengait/evaluation/metric.py index 627ed62..2aa43c6 100644 --- a/opengait/evaluation/metric.py +++ b/opengait/evaluation/metric.py @@ -156,3 +156,66 @@ def evaluate_rank(distmat, p_lbls, g_lbls, max_rank=50): all_cmc = all_cmc.sum(0) / num_valid_p return all_cmc, all_AP, all_INP + + +def evaluate_many(distmat, q_pids, g_pids, q_camids, g_camids, max_rank=50): + num_q, num_g = distmat.shape + if num_g < max_rank: + max_rank = num_g + print("Note: number of gallery samples is quite small, got {}".format(num_g)) + indices = np.argsort(distmat, axis=1) # 对应位置变成从小到大的序号 + matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype( + np.int32) # 根据indices调整顺序 g_pids[indices] + # print(matches) + + # compute cmc curve for each query + all_cmc = [] + all_AP = [] + all_INP = [] + num_valid_q = 0. + for q_idx in range(num_q): + # get query pid and camid + q_pid = q_pids[q_idx] + q_camid = q_camids[q_idx] + + # remove gallery samples that have the same pid and camid with query + order = indices[q_idx] + remove = (g_pids[order] == q_pid) & (g_camids[order] == q_camid) + keep = np.invert(remove) + + # compute cmc curve + # binary vector, positions with value 1 are correct matches + orig_cmc = matches[q_idx][keep] + if not np.any(orig_cmc): + # this condition is true when query identity does not appear in gallery + continue + + cmc = orig_cmc.cumsum() + + pos_idx = np.where(orig_cmc == 1) + max_pos_idx = np.max(pos_idx) + inp = cmc[max_pos_idx] / (max_pos_idx + 1.0) + all_INP.append(inp) + + cmc[cmc > 1] = 1 + + all_cmc.append(cmc[:max_rank]) + num_valid_q += 1. + + # compute average precision + # reference: https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Average_precision + num_rel = orig_cmc.sum() + tmp_cmc = orig_cmc.cumsum() + tmp_cmc = [x / (i+1.) for i, x in enumerate(tmp_cmc)] + tmp_cmc = np.asarray(tmp_cmc) * orig_cmc + AP = tmp_cmc.sum() / num_rel + all_AP.append(AP) + + assert num_valid_q > 0, "Error: all query identities do not appear in gallery" + + all_cmc = np.asarray(all_cmc).astype(np.float32) + all_cmc = all_cmc.sum(0) / num_valid_q + mAP = np.mean(all_AP) + mINP = np.mean(all_INP) + + return all_cmc, mAP, mINP