From 793405ad7cc19630e445b3c72c9a29901e084ca9 Mon Sep 17 00:00:00 2001 From: darkliang <11710911@mail.sustech.edu.cn> Date: Thu, 24 Nov 2022 22:07:50 +0800 Subject: [PATCH] refactor evaluation and add cross view gallery --- configs/baseline/baseline.yaml | 1 - configs/baseline/baseline_GREW.yaml | 2 +- configs/baseline/baseline_OUMVLP.yaml | 1 - configs/baseline/baseline_hid.yaml | 2 +- configs/default.yaml | 3 +- configs/gaitgl/gaitgl_GREW.yaml | 2 +- configs/gaitgl/gaitgl_GREW_BNNeck.yaml | 2 +- configs/gaitpart/gaitpart_GREW.yaml | 2 +- configs/gaitset/gaitset_GREW.yaml | 2 +- opengait/evaluation/__init__.py | 3 + opengait/evaluation/evaluator.py | 222 +++++++++++++++++++ opengait/evaluation/metric.py | 88 ++++++++ opengait/evaluation/re_rank.py | 64 ++++++ opengait/modeling/base_model.py | 2 +- opengait/modeling/losses/bce.py | 4 +- opengait/utils/__init__.py | 1 - opengait/utils/common.py | 12 -- opengait/utils/evaluation.py | 284 ------------------------- 18 files changed, 388 insertions(+), 309 deletions(-) create mode 100644 opengait/evaluation/__init__.py create mode 100644 opengait/evaluation/evaluator.py create mode 100644 opengait/evaluation/metric.py create mode 100644 opengait/evaluation/re_rank.py delete mode 100644 opengait/utils/evaluation.py diff --git a/configs/baseline/baseline.yaml b/configs/baseline/baseline.yaml index f03335b..a7aecc1 100644 --- a/configs/baseline/baseline.yaml +++ b/configs/baseline/baseline.yaml @@ -11,7 +11,6 @@ evaluator_cfg: restore_ckpt_strict: true restore_hint: 60000 save_name: Baseline - eval_func: identification sampler: batch_shuffle: false batch_size: 16 diff --git a/configs/baseline/baseline_GREW.yaml b/configs/baseline/baseline_GREW.yaml index 222c495..8dd893e 100644 --- a/configs/baseline/baseline_GREW.yaml +++ b/configs/baseline/baseline_GREW.yaml @@ -11,7 +11,7 @@ evaluator_cfg: restore_ckpt_strict: true restore_hint: 250000 save_name: Baseline - eval_func: identification_GREW_submission # identification_real_scene # identification_GREW_submission + eval_func: GREW_submission # identification_real_scene # identification_GREW_submission sampler: batch_shuffle: false batch_size: 64 diff --git a/configs/baseline/baseline_OUMVLP.yaml b/configs/baseline/baseline_OUMVLP.yaml index b53b5dc..4e6618d 100644 --- a/configs/baseline/baseline_OUMVLP.yaml +++ b/configs/baseline/baseline_OUMVLP.yaml @@ -11,7 +11,6 @@ evaluator_cfg: restore_ckpt_strict: true restore_hint: 150000 save_name: Baseline - eval_func: identification sampler: batch_shuffle: false batch_size: 4 diff --git a/configs/baseline/baseline_hid.yaml b/configs/baseline/baseline_hid.yaml index 6735d79..21809ce 100644 --- a/configs/baseline/baseline_hid.yaml +++ b/configs/baseline/baseline_hid.yaml @@ -9,7 +9,7 @@ evaluator_cfg: restore_ckpt_strict: true restore_hint: 60000 save_name: Baseline - eval_func: evaluate_HID + eval_func: HID_submission sampler: batch_shuffle: false batch_size: 8 diff --git a/configs/default.yaml b/configs/default.yaml index b5b772d..2c5e4b1 100644 --- a/configs/default.yaml +++ b/configs/default.yaml @@ -12,7 +12,7 @@ evaluator_cfg: restore_ckpt_strict: true restore_hint: 80000 save_name: tmp - eval_func: identification + eval_func: evaluate_indoor_dataset sampler: batch_size: 4 sample_type: all_ordered @@ -21,6 +21,7 @@ evaluator_cfg: - img_w: 64 type: BaseSilCuttingTransform metric: euc # cos + cross_view_gallery: false loss_cfg: loss_term_weight: 1.0 diff --git a/configs/gaitgl/gaitgl_GREW.yaml b/configs/gaitgl/gaitgl_GREW.yaml index aab8441..384a8d7 100644 --- a/configs/gaitgl/gaitgl_GREW.yaml +++ b/configs/gaitgl/gaitgl_GREW.yaml @@ -12,7 +12,7 @@ evaluator_cfg: restore_ckpt_strict: true restore_hint: 250000 save_name: GaitGL - eval_func: identification_GREW_submission # identification_real_scene # identification_GREW_submission + eval_func: GREW_submission # identification_real_scene # identification_GREW_submission sampler: batch_size: 4 sample_type: all_ordered diff --git a/configs/gaitgl/gaitgl_GREW_BNNeck.yaml b/configs/gaitgl/gaitgl_GREW_BNNeck.yaml index 84c784a..819edcc 100644 --- a/configs/gaitgl/gaitgl_GREW_BNNeck.yaml +++ b/configs/gaitgl/gaitgl_GREW_BNNeck.yaml @@ -12,7 +12,7 @@ evaluator_cfg: restore_ckpt_strict: true restore_hint: 250000 save_name: GaitGL_BNNeck - eval_func: identification_GREW_submission # identification_real_scene # identification_GREW_submission + eval_func: GREW_submission # identification_real_scene # identification_GREW_submission sampler: batch_size: 4 sample_type: all_ordered diff --git a/configs/gaitpart/gaitpart_GREW.yaml b/configs/gaitpart/gaitpart_GREW.yaml index be61bd7..62b15fc 100644 --- a/configs/gaitpart/gaitpart_GREW.yaml +++ b/configs/gaitpart/gaitpart_GREW.yaml @@ -11,7 +11,7 @@ evaluator_cfg: restore_ckpt_strict: true restore_hint: 250000 save_name: GaitPart - eval_func: identification_GREW_submission # identification_real_scene # identification_GREW_submission + eval_func: GREW_submission # identification_real_scene # identification_GREW_submission sampler: batch_size: 4 sample_type: all_ordered diff --git a/configs/gaitset/gaitset_GREW.yaml b/configs/gaitset/gaitset_GREW.yaml index f460940..f69f22c 100644 --- a/configs/gaitset/gaitset_GREW.yaml +++ b/configs/gaitset/gaitset_GREW.yaml @@ -11,7 +11,7 @@ evaluator_cfg: restore_ckpt_strict: true restore_hint: 250000 save_name: GaitSet - eval_func: identification_GREW_submission # identification_real_scene # identification_GREW_submission + eval_func: GREW_submission # identification_real_scene # identification_GREW_submission sampler: batch_size: 4 sample_type: all_ordered diff --git a/opengait/evaluation/__init__.py b/opengait/evaluation/__init__.py new file mode 100644 index 0000000..8a3a841 --- /dev/null +++ b/opengait/evaluation/__init__.py @@ -0,0 +1,3 @@ +from .metric import mean_iou +from numpy import set_printoptions +set_printoptions(suppress=True, formatter={'float': '{:0.2f}'.format}) diff --git a/opengait/evaluation/evaluator.py b/opengait/evaluation/evaluator.py new file mode 100644 index 0000000..2c0f609 --- /dev/null +++ b/opengait/evaluation/evaluator.py @@ -0,0 +1,222 @@ +import os +from time import strftime, localtime +import numpy as np +from utils import get_msg_mgr, mkdir + +from .metric import mean_iou, cuda_dist, compute_ACC_mAP +from .re_rank import re_ranking + + +def de_diag(acc, each_angle=False): + # Exclude identical-view cases + dividend = acc.shape[1] - 1. + result = np.sum(acc - np.diag(np.diag(acc)), 1) / dividend + if not each_angle: + result = np.mean(result) + return result + + +def cross_view_gallery_evaluation(feature, label, seq_type, view, probe_seq_dict, gallery_seq_list, metric): + msg_mgr = get_msg_mgr() + acc = {} + map = {} + view_list = sorted(np.unique(view)) + for (type_, probe_seq) in probe_seq_dict.items(): + acc[type_] = np.zeros(len(view_list)) - 1. + map[type_] = np.zeros(len(view_list)) - 1. + for (v1, probe_view) in enumerate(view_list): + pseq_mask = np.isin(seq_type, probe_seq) & np.isin( + view, [probe_view]) + probe_x = feature[pseq_mask, :] + probe_y = label[pseq_mask] + gseq_mask = np.isin(seq_type, gallery_seq_list) + gallery_y = label[gseq_mask] + gallery_x = feature[gseq_mask, :] + dist = cuda_dist(probe_x, gallery_x, metric) + eval_results = compute_ACC_mAP( + dist.cpu().numpy(), probe_y, gallery_y, np.asarray(view)[pseq_mask], np.asarray(view)[gseq_mask]) + acc[type_][v1] = np.round(eval_results[0] * 100, 2) + map[type_][v1] = np.round(eval_results[1] * 100, 2) + + result_dict = {} + msg_mgr.log_info( + '===Cross View Gallery Evaluation (Excluded identical-view cases)===') + out_acc_str = "========= Rank@1 Acc =========\n" + out_map_str = "============= mAP ============\n" + for type_ in probe_seq_dict.keys(): + avg_acc = np.mean(acc[type_]) + avg_map = np.mean(map[type_]) + result_dict[f'scalar/test_accuracy/{type_}-Rank@1'] = avg_acc + result_dict[f'scalar/test_accuracy/{type_}-mAP'] = avg_map + out_acc_str += f"{type_}:\t{acc[type_]}, mean: {avg_acc:.2f}%\n" + out_map_str += f"{type_}:\t{map[type_]}, mean: {avg_map:.2f}%\n" + # msg_mgr.log_info(f'========= Rank@1 Acc =========') + msg_mgr.log_info(f'{out_acc_str}') + # msg_mgr.log_info(f'========= mAP =========') + msg_mgr.log_info(f'{out_map_str}') + return result_dict + +# Modified From https://github.com/AbnerHqC/GaitSet/blob/master/model/utils/evaluator.py + + +def single_view_gallery_evaluation(feature, label, seq_type, view, probe_seq_dict, gallery_seq_list, metric): + msg_mgr = get_msg_mgr() + acc = {} + map = {} + view_list = sorted(np.unique(view)) + view_num = len(view_list) + num_rank = 1 + for (type_, probe_seq) in probe_seq_dict.items(): + acc[type_] = np.zeros((view_num, view_num)) - 1. + map[type_] = np.zeros((view_num, view_num)) - 1. + for (v1, probe_view) in enumerate(view_list): + pseq_mask = np.isin(seq_type, probe_seq) & np.isin( + view, [probe_view]) + probe_x = feature[pseq_mask, :] + probe_y = label[pseq_mask] + + for (v2, gallery_view) in enumerate(view_list): + gseq_mask = np.isin(seq_type, gallery_seq_list) & np.isin( + view, [gallery_view]) + gallery_y = label[gseq_mask] + gallery_x = feature[gseq_mask, :] + dist = cuda_dist(probe_x, gallery_x, metric) + idx = dist.cpu().sort(1)[1].numpy() + acc[type_][v1, v2] = np.round(np.sum(np.cumsum(np.reshape(probe_y, [-1, 1]) == gallery_y[idx[:, 0:num_rank]], 1) > 0, + 0) * 100 / dist.shape[0], 2) + + result_dict = {} + msg_mgr.log_info('===Rank-1 (Exclude identical-view cases)===') + out_str = "" + for type_ in probe_seq_dict.keys(): + sub_acc = de_diag(acc[type_], each_angle=True) + msg_mgr.log_info(f'{type_}: {sub_acc}') + result_dict[f'scalar/test_accuracy/{type_}'] = np.mean(sub_acc) + out_str += f"{type_}: {np.mean(sub_acc):.2f}%\t" + msg_mgr.log_info(out_str) + return result_dict + + +def evaluate_indoor_dataset(data, dataset, metric='euc', cross_view_gallery=False): + feature, label, seq_type, view = data['embeddings'], data['labels'], data['types'], data['views'] + label = np.array(label) + view = np.array(view) + + probe_seq_dict = {'CASIA-B': {'NM': ['nm-05', 'nm-06'], 'BG': ['bg-01', 'bg-02'], 'CL': ['cl-01', 'cl-02']}, + 'OUMVLP': {'NM': ['00']}} + + gallery_seq_dict = {'CASIA-B': ['nm-01', 'nm-02', 'nm-03', 'nm-04'], + 'OUMVLP': ['01']} + + if dataset not in (probe_seq_dict or gallery_seq_dict): + raise KeyError("DataSet %s hasn't been supported !" % dataset) + if cross_view_gallery: + return cross_view_gallery_evaluation( + feature, label, seq_type, view, probe_seq_dict[dataset], gallery_seq_dict[dataset], metric) + else: + return single_view_gallery_evaluation( + feature, label, seq_type, view, probe_seq_dict[dataset], gallery_seq_dict[dataset], metric) + + +def evaluate_real_scene(data, dataset, metric='euc'): + msg_mgr = get_msg_mgr() + feature, label, seq_type = data['embeddings'], data['labels'], data['types'] + label = np.array(label) + + gallery_seq_type = {'0001-1000': ['1', '2'], + "HID2021": ['0'], '0001-1000-test': ['0'], + 'GREW': ['01'], 'TTG-200': ['1']} + probe_seq_type = {'0001-1000': ['3', '4', '5', '6'], + "HID2021": ['1'], '0001-1000-test': ['1'], + 'GREW': ['02'], 'TTG-200': ['2', '3', '4', '5', '6']} + + num_rank = 20 + acc = np.zeros([num_rank]) - 1. + gseq_mask = np.isin(seq_type, gallery_seq_type[dataset]) + gallery_x = feature[gseq_mask, :] + gallery_y = label[gseq_mask] + pseq_mask = np.isin(seq_type, probe_seq_type[dataset]) + probe_x = feature[pseq_mask, :] + probe_y = label[pseq_mask] + + dist = cuda_dist(probe_x, gallery_x, metric) + idx = dist.cpu().sort(1)[1].numpy() + acc = np.round(np.sum(np.cumsum(np.reshape(probe_y, [-1, 1]) == gallery_y[idx[:, 0:num_rank]], 1) > 0, + 0) * 100 / dist.shape[0], 2) + msg_mgr.log_info('==Rank-1==') + msg_mgr.log_info('%.3f' % (np.mean(acc[0]))) + msg_mgr.log_info('==Rank-5==') + msg_mgr.log_info('%.3f' % (np.mean(acc[4]))) + msg_mgr.log_info('==Rank-10==') + msg_mgr.log_info('%.3f' % (np.mean(acc[9]))) + msg_mgr.log_info('==Rank-20==') + msg_mgr.log_info('%.3f' % (np.mean(acc[19]))) + return {"scalar/test_accuracy/Rank-1": np.mean(acc[0]), "scalar/test_accuracy/Rank-5": np.mean(acc[4])} + + +def GREW_submission(data, dataset, metric='euc'): + get_msg_mgr().log_info("Evaluating GREW") + feature, label, seq_type, view = data['embeddings'], data['labels'], data['types'], data['views'] + label = np.array(label) + view = np.array(view) + gallery_seq_type = {'GREW': ['01', '02']} + probe_seq_type = {'GREW': ['03']} + gseq_mask = np.isin(seq_type, gallery_seq_type[dataset]) + gallery_x = feature[gseq_mask, :] + gallery_y = label[gseq_mask] + pseq_mask = np.isin(seq_type, probe_seq_type[dataset]) + probe_x = feature[pseq_mask, :] + probe_y = view[pseq_mask] + + dist = cuda_dist(probe_x, gallery_x, metric) + idx = dist.cpu().sort(1)[1].numpy() + + save_path = os.path.join( + "GREW_result/"+strftime('%Y-%m%d-%H%M%S', localtime())+".csv") + mkdir("GREW_result") + with open(save_path, "w") as f: + f.write("videoId,rank1,rank2,rank3,rank4,rank5,rank6,rank7,rank8,rank9,rank10,rank11,rank12,rank13,rank14,rank15,rank16,rank17,rank18,rank19,rank20\n") + for i in range(len(idx)): + r_format = [int(idx) for idx in gallery_y[idx[i, 0:20]]] + output_row = '{}'+',{}'*20+'\n' + f.write(output_row.format(probe_y[i], *r_format)) + print("GREW result saved to {}/{}".format(os.getcwd(), save_path)) + return + + +def HID_submission(data, dataset, metric='euc'): + msg_mgr = get_msg_mgr() + msg_mgr.log_info("Evaluating HID") + feature, label, seq_type = data['embeddings'], data['labels'], data['types'] + label = np.array(label) + seq_type = np.array(seq_type) + probe_mask = (label == "probe") + gallery_mask = (label != "probe") + gallery_x = feature[gallery_mask, :] + gallery_y = label[gallery_mask] + probe_x = feature[probe_mask, :] + probe_y = seq_type[probe_mask] + + feat = np.concatenate([probe_x, gallery_x]) + dist = cuda_dist(feat, feat, metric).cpu().numpy() + msg_mgr.log_info("Starting Re-ranking") + re_rank = re_ranking(dist, probe_x.shape[0], k1=6, k2=6, lambda_value=0.3) + idx = np.argsort(re_rank, axis=1) + + save_path = os.path.join( + "HID_result/"+strftime('%Y-%m%d-%H%M%S', localtime())+".csv") + mkdir("HID_result") + with open(save_path, "w") as f: + f.write("videoID,label\n") + for i in range(len(idx)): + f.write("{},{}\n".format(probe_y[i], gallery_y[idx[i, 0]])) + print("HID result saved to {}/{}".format(os.getcwd(), save_path)) + return + + +def evaluate_segmentation(data, dataset): + labels = data['mask'] + pred = data['pred'] + miou = mean_iou(pred, labels) + get_msg_mgr().log_info('mIOU: %.3f' % (miou.mean())) + return {"scalar/test_accuracy/mIOU": miou} diff --git a/opengait/evaluation/metric.py b/opengait/evaluation/metric.py new file mode 100644 index 0000000..b9f2c85 --- /dev/null +++ b/opengait/evaluation/metric.py @@ -0,0 +1,88 @@ +import torch +import numpy as np +import torch.nn.functional as F + +from utils import is_tensor + + +def cuda_dist(x, y, metric='euc'): + x = torch.from_numpy(x).cuda() + y = torch.from_numpy(y).cuda() + if metric == 'cos': + x = F.normalize(x, p=2, dim=1) # n c p + y = F.normalize(y, p=2, dim=1) # n c p + num_bin = x.size(2) + n_x = x.size(0) + n_y = y.size(0) + dist = torch.zeros(n_x, n_y).cuda() + for i in range(num_bin): + _x = x[:, :, i] + _y = y[:, :, i] + if metric == 'cos': + dist += torch.matmul(_x, _y.transpose(0, 1)) + else: + _dist = torch.sum(_x ** 2, 1).unsqueeze(1) + torch.sum(_y ** 2, 1).unsqueeze( + 0) - 2 * torch.matmul(_x, _y.transpose(0, 1)) + dist += torch.sqrt(F.relu(_dist)) + return 1 - dist/num_bin if metric == 'cos' else dist / num_bin + + +def mean_iou(msk1, msk2, eps=1.0e-9): + if not is_tensor(msk1): + msk1 = torch.from_numpy(msk1).cuda() + if not is_tensor(msk2): + msk2 = torch.from_numpy(msk2).cuda() + n = msk1.size(0) + inter = msk1 * msk2 + union = ((msk1 + msk2) > 0.).float() + miou = inter.view(n, -1).sum(-1) / (union.view(n, -1).sum(-1) + eps) + return miou + + +def compute_ACC_mAP(distmat, q_pids, g_pids, q_views=None, g_views=None, rank=1): + num_q, _ = distmat.shape + # indices = np.argsort(distmat, axis=1) + # matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32) + + all_ACC = [] + all_AP = [] + num_valid_q = 0. # number of valid query + for q_idx in range(num_q): + q_idx_dist = distmat[q_idx] + q_idx_glabels = g_pids + if q_views is not None and g_views is not None: + q_idx_mask = np.isin(g_views, [q_views[q_idx]], invert=True) | np.isin( + g_pids, [q_pids[q_idx]], invert=True) + q_idx_dist = q_idx_dist[q_idx_mask] + q_idx_glabels = q_idx_glabels[q_idx_mask] + + assert(len(q_idx_glabels) > + 0), "No gallery after excluding identical-view cases!" + q_idx_indices = np.argsort(q_idx_dist) + q_idx_matches = (q_idx_glabels[q_idx_indices] + == q_pids[q_idx]).astype(np.int32) + + # binary vector, positions with value 1 are correct matches + # orig_cmc = matches[q_idx] + orig_cmc = q_idx_matches + cmc = orig_cmc.cumsum() + cmc[cmc > 1] = 1 + all_ACC.append(cmc[rank-1]) + + # compute average precision + # reference: https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Average_precision + num_rel = orig_cmc.sum() + + if num_rel > 0: + num_valid_q += 1. + tmp_cmc = orig_cmc.cumsum() + tmp_cmc = [x / (i + 1.) for i, x in enumerate(tmp_cmc)] + tmp_cmc = np.asarray(tmp_cmc) * orig_cmc + AP = tmp_cmc.sum() / num_rel + all_AP.append(AP) + + # all_ACC = np.asarray(all_ACC).astype(np.float32) + ACC = np.mean(all_ACC) + mAP = np.mean(all_AP) + + return ACC, mAP diff --git a/opengait/evaluation/re_rank.py b/opengait/evaluation/re_rank.py new file mode 100644 index 0000000..d5c5484 --- /dev/null +++ b/opengait/evaluation/re_rank.py @@ -0,0 +1,64 @@ +import numpy as np + + +def re_ranking(original_dist, query_num, k1, k2, lambda_value): + # Modified from https://github.com/michuanhaohao/reid-strong-baseline/blob/master/utils/re_ranking.py + all_num = original_dist.shape[0] + original_dist = np.transpose(original_dist / np.max(original_dist, axis=0)) + V = np.zeros_like(original_dist).astype(np.float16) + initial_rank = np.argsort(original_dist).astype(np.int32) + + for i in range(all_num): + # k-reciprocal neighbors + forward_k_neigh_index = initial_rank[i, :k1 + 1] + backward_k_neigh_index = initial_rank[forward_k_neigh_index, :k1 + 1] + fi = np.where(backward_k_neigh_index == i)[0] + k_reciprocal_index = forward_k_neigh_index[fi] + k_reciprocal_expansion_index = k_reciprocal_index + for j in range(len(k_reciprocal_index)): + candidate = k_reciprocal_index[j] + candidate_forward_k_neigh_index = initial_rank[candidate, :int( + np.around(k1 / 2)) + 1] + candidate_backward_k_neigh_index = initial_rank[candidate_forward_k_neigh_index, + :int(np.around(k1 / 2)) + 1] + fi_candidate = np.where( + candidate_backward_k_neigh_index == candidate)[0] + candidate_k_reciprocal_index = candidate_forward_k_neigh_index[fi_candidate] + if len(np.intersect1d(candidate_k_reciprocal_index, k_reciprocal_index)) > 2 / 3 * len( + candidate_k_reciprocal_index): + k_reciprocal_expansion_index = np.append( + k_reciprocal_expansion_index, candidate_k_reciprocal_index) + + k_reciprocal_expansion_index = np.unique(k_reciprocal_expansion_index) + weight = np.exp(-original_dist[i, k_reciprocal_expansion_index]) + V[i, k_reciprocal_expansion_index] = weight / np.sum(weight) + original_dist = original_dist[:query_num, ] + if k2 != 1: + V_qe = np.zeros_like(V, dtype=np.float16) + for i in range(all_num): + V_qe[i, :] = np.mean(V[initial_rank[i, :k2], :], axis=0) + V = V_qe + del V_qe + del initial_rank + invIndex = [] + for i in range(all_num): + invIndex.append(np.where(V[:, i] != 0)[0]) + + jaccard_dist = np.zeros_like(original_dist, dtype=np.float16) + + for i in range(query_num): + temp_min = np.zeros(shape=[1, all_num], dtype=np.float16) + indNonZero = np.where(V[i, :] != 0)[0] + indImages = [invIndex[ind] for ind in indNonZero] + for j in range(len(indNonZero)): + temp_min[0, indImages[j]] = temp_min[0, indImages[j]] + np.minimum(V[i, indNonZero[j]], + V[indImages[j], indNonZero[j]]) + jaccard_dist[i] = 1 - temp_min / (2 - temp_min) + + final_dist = jaccard_dist * (1 - lambda_value) + \ + original_dist * lambda_value + del original_dist + del V + del jaccard_dist + final_dist = final_dist[:query_num, query_num:] + return final_dist diff --git a/opengait/modeling/base_model.py b/opengait/modeling/base_model.py index cf3fdea..dff0b39 100644 --- a/opengait/modeling/base_model.py +++ b/opengait/modeling/base_model.py @@ -30,7 +30,7 @@ from data.dataset import DataSet import data.sampler as Samplers from utils import Odict, mkdir, ddp_all_gather from utils import get_valid_args, is_list, is_dict, np2var, ts2np, list2var, get_attr_from -from utils import evaluation as eval_functions +from evaluation import evaluator as eval_functions from utils import NoOp from utils import get_msg_mgr diff --git a/opengait/modeling/losses/bce.py b/opengait/modeling/losses/bce.py index 317738c..62034a9 100644 --- a/opengait/modeling/losses/bce.py +++ b/opengait/modeling/losses/bce.py @@ -1,6 +1,6 @@ import torch from .base import BaseLoss -from utils import MeanIOU +from evaluation import mean_iou class BinaryCrossEntropyLoss(BaseLoss): @@ -24,7 +24,7 @@ class BinaryCrossEntropyLoss(BaseLoss): loss = loss.view(n, -1) mean_loss = loss.mean() hard_loss = loss.max() - miou = MeanIOU((logits > 0.5).float(), labels) + miou = mean_iou((logits > 0.5).float(), labels) self.info.update({ 'loss': mean_loss.detach().clone(), 'hard_loss': hard_loss.detach().clone(), diff --git a/opengait/utils/__init__.py b/opengait/utils/__init__.py index d45a6da..8f72cd7 100644 --- a/opengait/utils/__init__.py +++ b/opengait/utils/__init__.py @@ -7,5 +7,4 @@ from .common import mkdir, clones from .common import MergeCfgsDict from .common import get_attr_from from .common import NoOp -from .common import MeanIOU from .msg_manager import get_msg_mgr \ No newline at end of file diff --git a/opengait/utils/common.py b/opengait/utils/common.py index ef1f9b6..4429174 100644 --- a/opengait/utils/common.py +++ b/opengait/utils/common.py @@ -203,15 +203,3 @@ def get_ddp_module(module, **kwargs): def params_count(net): n_parameters = sum(p.numel() for p in net.parameters()) return 'Parameters Count: {:.5f}M'.format(n_parameters / 1e6) - - -def MeanIOU(msk1, msk2, eps=1.0e-9): - if not is_tensor(msk1): - msk1 = torch.from_numpy(msk1).cuda() - if not is_tensor(msk2): - msk2 = torch.from_numpy(msk2).cuda() - n = msk1.size(0) - inter = msk1 * msk2 - union = ((msk1 + msk2) > 0.).float() - MeIOU = inter.view(n, -1).sum(-1) / (union.view(n, -1).sum(-1) + eps) - return MeIOU diff --git a/opengait/utils/evaluation.py b/opengait/utils/evaluation.py deleted file mode 100644 index 371517f..0000000 --- a/opengait/utils/evaluation.py +++ /dev/null @@ -1,284 +0,0 @@ -import os -from time import strftime, localtime -import torch -import numpy as np -import torch.nn.functional as F -from utils import get_msg_mgr, mkdir, MeanIOU - - -def cuda_dist(x, y, metric='euc'): - x = torch.from_numpy(x).cuda() - y = torch.from_numpy(y).cuda() - if metric == 'cos': - x = F.normalize(x, p=2, dim=1) # n c p - y = F.normalize(y, p=2, dim=1) # n c p - num_bin = x.size(2) - n_x = x.size(0) - n_y = y.size(0) - dist = torch.zeros(n_x, n_y).cuda() - for i in range(num_bin): - _x = x[:, :, i] - _y = y[:, :, i] - if metric == 'cos': - dist += torch.matmul(_x, _y.transpose(0, 1)) - else: - _dist = torch.sum(_x ** 2, 1).unsqueeze(1) + torch.sum(_y ** 2, 1).unsqueeze( - 0) - 2 * torch.matmul(_x, _y.transpose(0, 1)) - dist += torch.sqrt(F.relu(_dist)) - return 1 - dist/num_bin if metric == 'cos' else dist / num_bin - -# Exclude identical-view cases - - -def de_diag(acc, each_angle=False): - dividend = acc.shape[1] - 1. - result = np.sum(acc - np.diag(np.diag(acc)), 1) / dividend - if not each_angle: - result = np.mean(result) - return result - -# Modified From https://github.com/AbnerHqC/GaitSet/blob/master/model/utils/evaluator.py - - -def identification(data, dataset, metric='euc'): - msg_mgr = get_msg_mgr() - feature, label, seq_type, view = data['embeddings'], data['labels'], data['types'], data['views'] - label = np.array(label) - view_list = list(set(view)) - view_list.sort() - view_num = len(view_list) - # sample_num = len(feature) - - probe_seq_dict = {'CASIA-B': [['nm-05', 'nm-06'], ['bg-01', 'bg-02'], ['cl-01', 'cl-02']], - 'OUMVLP': [['00']]} - - gallery_seq_dict = {'CASIA-B': [['nm-01', 'nm-02', 'nm-03', 'nm-04']], - 'OUMVLP': [['01']]} - if dataset not in (probe_seq_dict or gallery_seq_dict): - raise KeyError("DataSet %s hasn't been supported !" % dataset) - num_rank = 5 - acc = np.zeros([len(probe_seq_dict[dataset]), - view_num, view_num, num_rank]) - 1. - for (p, probe_seq) in enumerate(probe_seq_dict[dataset]): - for gallery_seq in gallery_seq_dict[dataset]: - for (v1, probe_view) in enumerate(view_list): - for (v2, gallery_view) in enumerate(view_list): - gseq_mask = np.isin(seq_type, gallery_seq) & np.isin( - view, [gallery_view]) - gallery_x = feature[gseq_mask, :] - gallery_y = label[gseq_mask] - - pseq_mask = np.isin(seq_type, probe_seq) & np.isin( - view, [probe_view]) - probe_x = feature[pseq_mask, :] - probe_y = label[pseq_mask] - - dist = cuda_dist(probe_x, gallery_x, metric) - idx = dist.sort(1)[1].cpu().numpy() - acc[p, v1, v2, :] = np.round( - np.sum(np.cumsum(np.reshape(probe_y, [-1, 1]) == gallery_y[idx[:, 0:num_rank]], 1) > 0, - 0) * 100 / dist.shape[0], 2) - result_dict = {} - np.set_printoptions(precision=3, suppress=True) - if 'OUMVLP' not in dataset: - for i in range(1): - msg_mgr.log_info( - '===Rank-%d (Include identical-view cases)===' % (i + 1)) - msg_mgr.log_info('NM: %.3f,\tBG: %.3f,\tCL: %.3f' % ( - np.mean(acc[0, :, :, i]), - np.mean(acc[1, :, :, i]), - np.mean(acc[2, :, :, i]))) - for i in range(1): - msg_mgr.log_info( - '===Rank-%d (Exclude identical-view cases)===' % (i + 1)) - msg_mgr.log_info('NM: %.3f,\tBG: %.3f,\tCL: %.3f' % ( - de_diag(acc[0, :, :, i]), - de_diag(acc[1, :, :, i]), - de_diag(acc[2, :, :, i]))) - result_dict["scalar/test_accuracy/NM"] = de_diag(acc[0, :, :, i]) - result_dict["scalar/test_accuracy/BG"] = de_diag(acc[1, :, :, i]) - result_dict["scalar/test_accuracy/CL"] = de_diag(acc[2, :, :, i]) - np.set_printoptions(precision=2, floatmode='fixed') - for i in range(1): - msg_mgr.log_info( - '===Rank-%d of each angle (Exclude identical-view cases)===' % (i + 1)) - msg_mgr.log_info('NM: {}'.format(de_diag(acc[0, :, :, i], True))) - msg_mgr.log_info('BG: {}'.format(de_diag(acc[1, :, :, i], True))) - msg_mgr.log_info('CL: {}'.format(de_diag(acc[2, :, :, i], True))) - else: - msg_mgr.log_info('===Rank-1 (Include identical-view cases)===') - msg_mgr.log_info('NM: %.3f ' % (np.mean(acc[0, :, :, 0]))) - msg_mgr.log_info('===Rank-1 (Exclude identical-view cases)===') - msg_mgr.log_info('NM: %.3f ' % (de_diag(acc[0, :, :, 0]))) - msg_mgr.log_info( - '===Rank-1 of each angle (Exclude identical-view cases)===') - msg_mgr.log_info('NM: {}'.format(de_diag(acc[0, :, :, 0], True))) - result_dict["scalar/test_accuracy/NM"] = de_diag(acc[0, :, :, 0]) - return result_dict - - -def identification_real_scene(data, dataset, metric='euc'): - msg_mgr = get_msg_mgr() - feature, label, seq_type = data['embeddings'], data['labels'], data['types'] - label = np.array(label) - - gallery_seq_type = {'0001-1000': ['1', '2'], - "HID2021": ['0'], '0001-1000-test': ['0'], - 'GREW': ['01'], 'TTG-200': ['1']} - probe_seq_type = {'0001-1000': ['3', '4', '5', '6'], - "HID2021": ['1'], '0001-1000-test': ['1'], - 'GREW': ['02'], 'TTG-200': ['2', '3', '4', '5', '6']} - - num_rank = 20 - acc = np.zeros([num_rank]) - 1. - gseq_mask = np.isin(seq_type, gallery_seq_type[dataset]) - gallery_x = feature[gseq_mask, :] - gallery_y = label[gseq_mask] - pseq_mask = np.isin(seq_type, probe_seq_type[dataset]) - probe_x = feature[pseq_mask, :] - probe_y = label[pseq_mask] - - dist = cuda_dist(probe_x, gallery_x, metric) - idx = dist.cpu().sort(1)[1].numpy() - acc = np.round(np.sum(np.cumsum(np.reshape(probe_y, [-1, 1]) == gallery_y[idx[:, 0:num_rank]], 1) > 0, - 0) * 100 / dist.shape[0], 2) - msg_mgr.log_info('==Rank-1==') - msg_mgr.log_info('%.3f' % (np.mean(acc[0]))) - msg_mgr.log_info('==Rank-5==') - msg_mgr.log_info('%.3f' % (np.mean(acc[4]))) - msg_mgr.log_info('==Rank-10==') - msg_mgr.log_info('%.3f' % (np.mean(acc[9]))) - msg_mgr.log_info('==Rank-20==') - msg_mgr.log_info('%.3f' % (np.mean(acc[19]))) - return {"scalar/test_accuracy/Rank-1": np.mean(acc[0]), "scalar/test_accuracy/Rank-5": np.mean(acc[4])} - - -def identification_GREW_submission(data, dataset, metric='euc'): - get_msg_mgr().log_info("Evaluating GREW") - feature, label, seq_type, view = data['embeddings'], data['labels'], data['types'], data['views'] - label = np.array(label) - view = np.array(view) - gallery_seq_type = {'GREW': ['01', '02']} - probe_seq_type = {'GREW': ['03']} - gseq_mask = np.isin(seq_type, gallery_seq_type[dataset]) - gallery_x = feature[gseq_mask, :] - gallery_y = label[gseq_mask] - pseq_mask = np.isin(seq_type, probe_seq_type[dataset]) - probe_x = feature[pseq_mask, :] - probe_y = view[pseq_mask] - - dist = cuda_dist(probe_x, gallery_x, metric) - idx = dist.cpu().sort(1)[1].numpy() - - save_path = os.path.join( - "GREW_result/"+strftime('%Y-%m%d-%H%M%S', localtime())+".csv") - mkdir("GREW_result") - with open(save_path, "w") as f: - f.write("videoId,rank1,rank2,rank3,rank4,rank5,rank6,rank7,rank8,rank9,rank10,rank11,rank12,rank13,rank14,rank15,rank16,rank17,rank18,rank19,rank20\n") - for i in range(len(idx)): - r_format = [int(idx) for idx in gallery_y[idx[i, 0:20]]] - output_row = '{}'+',{}'*20+'\n' - f.write(output_row.format(probe_y[i], *r_format)) - print("GREW result saved to {}/{}".format(os.getcwd(), save_path)) - return - - -def evaluate_HID(data, dataset, metric='euc'): - msg_mgr = get_msg_mgr() - msg_mgr.log_info("Evaluating HID") - feature, label, seq_type = data['embeddings'], data['labels'], data['types'] - label = np.array(label) - seq_type = np.array(seq_type) - probe_mask = (label == "probe") - gallery_mask = (label != "probe") - gallery_x = feature[gallery_mask, :] - gallery_y = label[gallery_mask] - probe_x = feature[probe_mask, :] - probe_y = seq_type[probe_mask] - - feat = np.concatenate([probe_x, gallery_x]) - dist = cuda_dist(feat, feat, metric).cpu().numpy() - msg_mgr.log_info("Starting Re-ranking") - re_rank = re_ranking(dist, probe_x.shape[0], k1=6, k2=6, lambda_value=0.3) - idx = np.argsort(re_rank, axis=1) - - save_path = os.path.join( - "HID_result/"+strftime('%Y-%m%d-%H%M%S', localtime())+".csv") - mkdir("HID_result") - with open(save_path, "w") as f: - f.write("videoID,label\n") - for i in range(len(idx)): - f.write("{},{}\n".format(probe_y[i], gallery_y[idx[i, 0]])) - print("HID result saved to {}/{}".format(os.getcwd(), save_path)) - return - - -def re_ranking(original_dist, query_num, k1, k2, lambda_value): - # Modified from https://github.com/michuanhaohao/reid-strong-baseline/blob/master/utils/re_ranking.py - all_num = original_dist.shape[0] - original_dist = np.transpose(original_dist / np.max(original_dist, axis=0)) - V = np.zeros_like(original_dist).astype(np.float16) - initial_rank = np.argsort(original_dist).astype(np.int32) - - for i in range(all_num): - # k-reciprocal neighbors - forward_k_neigh_index = initial_rank[i, :k1 + 1] - backward_k_neigh_index = initial_rank[forward_k_neigh_index, :k1 + 1] - fi = np.where(backward_k_neigh_index == i)[0] - k_reciprocal_index = forward_k_neigh_index[fi] - k_reciprocal_expansion_index = k_reciprocal_index - for j in range(len(k_reciprocal_index)): - candidate = k_reciprocal_index[j] - candidate_forward_k_neigh_index = initial_rank[candidate, :int( - np.around(k1 / 2)) + 1] - candidate_backward_k_neigh_index = initial_rank[candidate_forward_k_neigh_index, - :int(np.around(k1 / 2)) + 1] - fi_candidate = np.where( - candidate_backward_k_neigh_index == candidate)[0] - candidate_k_reciprocal_index = candidate_forward_k_neigh_index[fi_candidate] - if len(np.intersect1d(candidate_k_reciprocal_index, k_reciprocal_index)) > 2 / 3 * len( - candidate_k_reciprocal_index): - k_reciprocal_expansion_index = np.append( - k_reciprocal_expansion_index, candidate_k_reciprocal_index) - - k_reciprocal_expansion_index = np.unique(k_reciprocal_expansion_index) - weight = np.exp(-original_dist[i, k_reciprocal_expansion_index]) - V[i, k_reciprocal_expansion_index] = weight / np.sum(weight) - original_dist = original_dist[:query_num, ] - if k2 != 1: - V_qe = np.zeros_like(V, dtype=np.float16) - for i in range(all_num): - V_qe[i, :] = np.mean(V[initial_rank[i, :k2], :], axis=0) - V = V_qe - del V_qe - del initial_rank - invIndex = [] - for i in range(all_num): - invIndex.append(np.where(V[:, i] != 0)[0]) - - jaccard_dist = np.zeros_like(original_dist, dtype=np.float16) - - for i in range(query_num): - temp_min = np.zeros(shape=[1, all_num], dtype=np.float16) - indNonZero = np.where(V[i, :] != 0)[0] - indImages = [invIndex[ind] for ind in indNonZero] - for j in range(len(indNonZero)): - temp_min[0, indImages[j]] = temp_min[0, indImages[j]] + np.minimum(V[i, indNonZero[j]], - V[indImages[j], indNonZero[j]]) - jaccard_dist[i] = 1 - temp_min / (2 - temp_min) - - final_dist = jaccard_dist * (1 - lambda_value) + \ - original_dist * lambda_value - del original_dist - del V - del jaccard_dist - final_dist = final_dist[:query_num, query_num:] - return final_dist - - -def mean_iou(data, dataset): - labels = data['mask'] - pred = data['pred'] - miou = MeanIOU(pred, labels) - get_msg_mgr().log_info('mIOU: %.3f' % (miou.mean())) - return {"scalar/test_accuracy/mIOU": miou}