From 7b4fa5dd1e290cbd15997e61ea04369b4c410dfa Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 14:20:09 +0100 Subject: [PATCH 01/78] initial commit --- test.py | 6 ++++- utils/metrics.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/test.py b/test.py index 32401ee62399..c4f440f1ceb1 100644 --- a/test.py +++ b/test.py @@ -14,7 +14,7 @@ from utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, box_iou, \ non_max_suppression, scale_coords, xyxy2xywh, xywh2xyxy, set_logging, increment_path from utils.loss import compute_loss -from utils.metrics import ap_per_class +from utils.metrics import ap_per_class, ConfusionMatrix from utils.plots import plot_images, output_to_target from utils.torch_utils import select_device, time_synchronized @@ -89,6 +89,7 @@ def test(data, dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt, pad=0.5, rect=True)[0] seen = 0 + confusion_matrix = ConfusionMatrix(num_classes=nc) names = {k: v for k, v in enumerate(model.names if hasattr(model, 'names') else model.module.names)} coco91class = coco80_to_coco91_class() s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP@.5', 'mAP@.5:.95') @@ -136,6 +137,9 @@ def test(data, predn = pred.clone() scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred + # Confusion Matrix + confusion_matrix.process_batch(pred, labels) + # Append to text file if save_txt: gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh diff --git a/utils/metrics.py b/utils/metrics.py index 62add1da1f8d..f87a93e5bd01 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -5,6 +5,8 @@ import matplotlib.pyplot as plt import numpy as np +from . import general + def fitness(x): # Model fitness as a weighted combination of metrics @@ -102,6 +104,69 @@ def compute_ap(recall, precision): return ap, mpre, mrec +class ConfusionMatrix: + # https://github.com/kaanakan/object_detection_confusion_matrix + def __init__(self, num_classes, CONF_THRESHOLD=0.3, IOU_THRESHOLD=0.5): + self.matrix = np.zeros((num_classes + 1, num_classes + 1)) + self.num_classes = num_classes + self.CONF_THRESHOLD = CONF_THRESHOLD + self.IOU_THRESHOLD = IOU_THRESHOLD + + def process_batch(self, detections, labels): + """ + Return intersection-over-union (Jaccard index) of boxes. + Both sets of boxes are expected to be in (x1, y1, x2, y2) format. + Arguments: + detections (Array[N, 6]), x1, y1, x2, y2, conf, class + labels (Array[M, 5]), class, x1, y1, x2, y2 + Returns: + None, updates confusion matrix accordingly + """ + detections = detections[detections[:, 4] > self.CONF_THRESHOLD] + gt_classes = labels[:, 0].astype(np.int16) + detection_classes = detections[:, 5].astype(np.int16) + + all_ious = general.box_iou(labels[:, 1:], detections[:, :4]) + want_idx = np.where(all_ious > self.IOU_THRESHOLD) + + all_matches = [] + for i in range(want_idx[0].shape[0]): + all_matches.append([want_idx[0][i], want_idx[1][i], all_ious[want_idx[0][i], want_idx[1][i]]]) + + all_matches = np.array(all_matches) + if all_matches.shape[0] > 0: # if there is match + all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] + + all_matches = all_matches[np.unique(all_matches[:, 1], return_index=True)[1]] + + all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] + + all_matches = all_matches[np.unique(all_matches[:, 0], return_index=True)[1]] + + for i, label in enumerate(labels): + if all_matches.shape[0] > 0 and all_matches[all_matches[:, 0] == i].shape[0] == 1: + gt_class = gt_classes[i] + detection_class = detection_classes[int(all_matches[all_matches[:, 0] == i, 1][0])] + self.matrix[(gt_class), detection_class] += 1 + else: + gt_class = gt_classes[i] + self.matrix[(gt_class), self.num_classes] += 1 + + for i, detection in enumerate(detections): + if all_matches.shape[0] and all_matches[all_matches[:, 1] == i].shape[0] == 0: + detection_class = detection_classes[i] + self.matrix[self.num_classes, detection_class] += 1 + + def return_matrix(self): + return self.matrix + + def print_matrix(self): + for i in range(self.num_classes + 1): + print(' '.join(map(str, self.matrix[i]))) + + +# Plots ---------------------------------------------------------------------------------------------------------------- + def plot_pr_curve(px, py, ap, save_dir='.', names=()): fig, ax = plt.subplots(1, 1, figsize=(9, 6)) py = np.stack(py, axis=1) From a156d4125b4bc25f6bdc3cea1e09fafaf856806c Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 14:51:06 +0100 Subject: [PATCH 02/78] add plotting --- test.py | 16 +++++++++------- utils/metrics.py | 29 ++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/test.py b/test.py index c4f440f1ceb1..d3c384899c51 100644 --- a/test.py +++ b/test.py @@ -137,9 +137,6 @@ def test(data, predn = pred.clone() scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred - # Confusion Matrix - confusion_matrix.process_batch(pred, labels) - # Append to text file if save_txt: gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh @@ -181,6 +178,9 @@ def test(data, tbox = xywh2xyxy(labels[:, 1:5]) * whwh scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1]) # native-space labels + # Confusion Matrix + confusion_matrix.process_batch(pred, torch.cat((labels[:, 0:1], tbox), 1)) + # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1) # prediction indices @@ -222,10 +222,12 @@ def test(data, else: nt = torch.zeros(1) - # W&B logging - if plots and wandb and wandb.run: - wandb.log({"Images": wandb_images}) - wandb.log({"Validation": [wandb.Image(str(x), caption=x.name) for x in sorted(save_dir.glob('test*.jpg'))]}) + # Plots + if plots: + confusion_matrix.plot(save_dir=save_dir) + if wandb and wandb.run: + wandb.log({"Images": wandb_images}) + wandb.log({"Validation": [wandb.Image(str(x), caption=x.name) for x in sorted(save_dir.glob('test*.jpg'))]}) # Print results pf = '%20s' + '%12.3g' * 6 # print format diff --git a/utils/metrics.py b/utils/metrics.py index f87a93e5bd01..7323771c5892 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -106,7 +106,7 @@ def compute_ap(recall, precision): class ConfusionMatrix: # https://github.com/kaanakan/object_detection_confusion_matrix - def __init__(self, num_classes, CONF_THRESHOLD=0.3, IOU_THRESHOLD=0.5): + def __init__(self, num_classes, CONF_THRESHOLD=0.0, IOU_THRESHOLD=0.45): self.matrix = np.zeros((num_classes + 1, num_classes + 1)) self.num_classes = num_classes self.CONF_THRESHOLD = CONF_THRESHOLD @@ -123,8 +123,8 @@ def process_batch(self, detections, labels): None, updates confusion matrix accordingly """ detections = detections[detections[:, 4] > self.CONF_THRESHOLD] - gt_classes = labels[:, 0].astype(np.int16) - detection_classes = detections[:, 5].astype(np.int16) + gt_classes = labels[:, 0].int() + detection_classes = detections[:, 5].int() all_ious = general.box_iou(labels[:, 1:], detections[:, :4]) want_idx = np.where(all_ious > self.IOU_THRESHOLD) @@ -157,10 +157,29 @@ def process_batch(self, detections, labels): detection_class = detection_classes[i] self.matrix[self.num_classes, detection_class] += 1 - def return_matrix(self): + def matrix(self): return self.matrix - def print_matrix(self): + def plot(self, save_dir=''): + import seaborn as sn + import pandas as pd + + # array = [[13, 1, 1, 0, 2, 0], + # [3, 9, 6, 0, 1, 0], + # [0, 0, 16, 2, 0, 0], + # [0, 0, 0, 13, 0, 0], + # [0, 0, 0, 0, 15, 0], + # [0, 0, 1, 0, 0, 15]] + + array = self.matrix.numpy() + df_cm = pd.DataFrame(array, range(6), range(6)) + plt.figure(figsize=(10,7)) + sn.color_palette("magma", as_cmap=True) + sn.set(font_scale=1.4) # for label size + sn.heatmap(df_cm, annot=True, annot_kws={"size": 16}, cmap='Blues') # font size + plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) + + def print(self): for i in range(self.num_classes + 1): print(' '.join(map(str, self.matrix[i]))) From 8f4a71daab3068bf8fa5d63452081afe9068fe94 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 14:52:34 +0100 Subject: [PATCH 03/78] matrix to cpu --- utils/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/metrics.py b/utils/metrics.py index 7323771c5892..405dd3961c07 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -171,7 +171,7 @@ def plot(self, save_dir=''): # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] - array = self.matrix.numpy() + array = self.matrix.cpu().numpy() df_cm = pd.DataFrame(array, range(6), range(6)) plt.figure(figsize=(10,7)) sn.color_palette("magma", as_cmap=True) From 08c73037c33bc1c88b89d99cf522712dd5391027 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 15:13:44 +0100 Subject: [PATCH 04/78] bug fix --- test.py | 2 +- utils/metrics.py | 38 +++++++++++++++++++------------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/test.py b/test.py index d3c384899c51..16e50f0b1914 100644 --- a/test.py +++ b/test.py @@ -89,7 +89,7 @@ def test(data, dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt, pad=0.5, rect=True)[0] seen = 0 - confusion_matrix = ConfusionMatrix(num_classes=nc) + confusion_matrix = ConfusionMatrix(nc=nc) names = {k: v for k, v in enumerate(model.names if hasattr(model, 'names') else model.module.names)} coco91class = coco80_to_coco91_class() s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP@.5', 'mAP@.5:.95') diff --git a/utils/metrics.py b/utils/metrics.py index 405dd3961c07..2f734eb2beec 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -4,6 +4,7 @@ import matplotlib.pyplot as plt import numpy as np +import torch from . import general @@ -106,9 +107,9 @@ def compute_ap(recall, precision): class ConfusionMatrix: # https://github.com/kaanakan/object_detection_confusion_matrix - def __init__(self, num_classes, CONF_THRESHOLD=0.0, IOU_THRESHOLD=0.45): - self.matrix = np.zeros((num_classes + 1, num_classes + 1)) - self.num_classes = num_classes + def __init__(self, nc, CONF_THRESHOLD=0.0, IOU_THRESHOLD=0.45): + self.matrix = np.zeros((nc + 1, nc + 1)) + self.nc = nc # number of classes self.CONF_THRESHOLD = CONF_THRESHOLD self.IOU_THRESHOLD = IOU_THRESHOLD @@ -127,21 +128,21 @@ def process_batch(self, detections, labels): detection_classes = detections[:, 5].int() all_ious = general.box_iou(labels[:, 1:], detections[:, :4]) - want_idx = np.where(all_ious > self.IOU_THRESHOLD) + want_idx = torch.where(all_ious > self.IOU_THRESHOLD) all_matches = [] for i in range(want_idx[0].shape[0]): all_matches.append([want_idx[0][i], want_idx[1][i], all_ious[want_idx[0][i], want_idx[1][i]]]) - all_matches = np.array(all_matches) - if all_matches.shape[0] > 0: # if there is match - all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] - - all_matches = all_matches[np.unique(all_matches[:, 1], return_index=True)[1]] - - all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] - - all_matches = all_matches[np.unique(all_matches[:, 0], return_index=True)[1]] + if all_matches: + all_matches = np.array(all_matches) + if all_matches.shape[0] > 0: # if there is match + all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] + all_matches = all_matches[np.unique(all_matches[:, 1], return_index=True)[1]] + all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] + all_matches = all_matches[np.unique(all_matches[:, 0], return_index=True)[1]] + else: + all_matches = torch.zeros([0, 1]) for i, label in enumerate(labels): if all_matches.shape[0] > 0 and all_matches[all_matches[:, 0] == i].shape[0] == 1: @@ -150,12 +151,12 @@ def process_batch(self, detections, labels): self.matrix[(gt_class), detection_class] += 1 else: gt_class = gt_classes[i] - self.matrix[(gt_class), self.num_classes] += 1 + self.matrix[(gt_class), self.nc] += 1 for i, detection in enumerate(detections): if all_matches.shape[0] and all_matches[all_matches[:, 1] == i].shape[0] == 0: detection_class = detection_classes[i] - self.matrix[self.num_classes, detection_class] += 1 + self.matrix[self.nc, detection_class] += 1 def matrix(self): return self.matrix @@ -171,16 +172,15 @@ def plot(self, save_dir=''): # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] - array = self.matrix.cpu().numpy() - df_cm = pd.DataFrame(array, range(6), range(6)) - plt.figure(figsize=(10,7)) + df_cm = pd.DataFrame(self.matrix, range(self.nc + 1), range(self.nc + 1)) + plt.figure(figsize=(10, 7)) sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.4) # for label size sn.heatmap(df_cm, annot=True, annot_kws={"size": 16}, cmap='Blues') # font size plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): - for i in range(self.num_classes + 1): + for i in range(self.nc + 1): print(' '.join(map(str, self.matrix[i]))) From 9ddafbde9a4658e920740622b06c4401e16f03a3 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 15:40:25 +0100 Subject: [PATCH 05/78] update plot --- utils/metrics.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 2f734eb2beec..7e18973933fb 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -107,7 +107,7 @@ def compute_ap(recall, precision): class ConfusionMatrix: # https://github.com/kaanakan/object_detection_confusion_matrix - def __init__(self, nc, CONF_THRESHOLD=0.0, IOU_THRESHOLD=0.45): + def __init__(self, nc, CONF_THRESHOLD=0.25, IOU_THRESHOLD=0.45): self.matrix = np.zeros((nc + 1, nc + 1)) self.nc = nc # number of classes self.CONF_THRESHOLD = CONF_THRESHOLD @@ -171,12 +171,13 @@ def plot(self, save_dir=''): # [0, 0, 0, 13, 0, 0], # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] + array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) - df_cm = pd.DataFrame(self.matrix, range(self.nc + 1), range(self.nc + 1)) - plt.figure(figsize=(10, 7)) + df_cm = pd.DataFrame(array, range(self.nc+1), range(self.nc+1)) + plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) - sn.set(font_scale=1.4) # for label size - sn.heatmap(df_cm, annot=True, annot_kws={"size": 16}, cmap='Blues') # font size + sn.set(font_scale=1.0) # for label size + sn.heatmap(df_cm, annot=self.nc < 20, annot_kws={"size": 12}, cmap='Blues', square=True) # font size plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 5e04f4a6a5c6980345a731c2e871738020cb51c4 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 16:24:59 +0100 Subject: [PATCH 06/78] update plot --- utils/metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 7e18973933fb..da3602bea45e 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -165,19 +165,19 @@ def plot(self, save_dir=''): import seaborn as sn import pandas as pd + array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # array = [[13, 1, 1, 0, 2, 0], # [3, 9, 6, 0, 1, 0], # [0, 0, 16, 2, 0, 0], # [0, 0, 0, 13, 0, 0], # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] - array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) - df_cm = pd.DataFrame(array, range(self.nc+1), range(self.nc+1)) + df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size - sn.heatmap(df_cm, annot=self.nc < 20, annot_kws={"size": 12}, cmap='Blues', square=True) # font size + sn.heatmap(df_cm, annot=self.nc < 25, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True) plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 20cb0f98cfe939123249cf23d91a292ae525ebd6 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 16:27:12 +0100 Subject: [PATCH 07/78] update plot --- utils/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/metrics.py b/utils/metrics.py index da3602bea45e..061a48f12943 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -177,7 +177,7 @@ def plot(self, save_dir=''): plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size - sn.heatmap(df_cm, annot=self.nc < 25, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True) + sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True) plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 85febbe4440a16648bd3af9adc5611c7f3510b60 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 16:32:48 +0100 Subject: [PATCH 08/78] update plot --- utils/metrics.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 061a48f12943..54820ae42b81 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -165,19 +165,20 @@ def plot(self, save_dir=''): import seaborn as sn import pandas as pd - array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) + array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize # array = [[13, 1, 1, 0, 2, 0], # [3, 9, 6, 0, 1, 0], # [0, 0, 16, 2, 0, 0], # [0, 0, 0, 13, 0, 0], # [0, 0, 0, 0, 15, 0], - # [0, 0, 1, 0, 0, 15]] - + # [0, 0, 1, 0, 0, 15]] # example + array[array == 0] = np.nan # don't annotate 0.0 df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) + plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size - sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True) + sn.heatmap(df_cm, annot=self.nc < 930, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True) plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 1da707bf9732334b399979c0934bd77df6ade688 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 16:47:36 +0100 Subject: [PATCH 09/78] update plot --- requirements.txt | 1 + utils/metrics.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c5e6a6119321..517372e3f5c1 100755 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ tqdm>=4.41.0 # extras -------------------------------------- # thop # FLOPS computation # seaborn # plotting +# pandas # plotting diff --git a/utils/metrics.py b/utils/metrics.py index 54820ae42b81..7f6be3ed256a 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -172,7 +172,7 @@ def plot(self, save_dir=''): # [0, 0, 0, 13, 0, 0], # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] # example - array[array == 0] = np.nan # don't annotate 0.0 + array[array < 0.005] = np.nan # don't annotate (will appear as 0.0) df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) plt.figure(figsize=(12, 9)) From cdfbcd808651a716191125ddd757faa42fff498f Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:06:04 +0100 Subject: [PATCH 10/78] update plot --- test.py | 2 +- utils/metrics.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test.py b/test.py index 16e50f0b1914..7793d65d11ed 100644 --- a/test.py +++ b/test.py @@ -224,7 +224,7 @@ def test(data, # Plots if plots: - confusion_matrix.plot(save_dir=save_dir) + confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) if wandb and wandb.run: wandb.log({"Images": wandb_images}) wandb.log({"Validation": [wandb.Image(str(x), caption=x.name) for x in sorted(save_dir.glob('test*.jpg'))]}) diff --git a/utils/metrics.py b/utils/metrics.py index 7f6be3ed256a..977429ab8552 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -161,9 +161,10 @@ def process_batch(self, detections, labels): def matrix(self): return self.matrix - def plot(self, save_dir=''): + def plot(self, save_dir='', names=[]): import seaborn as sn import pandas as pd + names = names + ['background'] if names else "auto" array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize # array = [[13, 1, 1, 0, 2, 0], @@ -178,7 +179,8 @@ def plot(self, save_dir=''): plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size - sn.heatmap(df_cm, annot=self.nc < 930, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True) + sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + xticklabels=names, yticklabels=names) plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From abc018fdf7bb3c881e2e8231e82eaada1a0ba279 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:11:54 +0100 Subject: [PATCH 11/78] update plot --- utils/metrics.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 977429ab8552..ebecdfb10bd3 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -164,7 +164,6 @@ def matrix(self): def plot(self, save_dir='', names=[]): import seaborn as sn import pandas as pd - names = names + ['background'] if names else "auto" array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize # array = [[13, 1, 1, 0, 2, 0], @@ -173,6 +172,7 @@ def plot(self, save_dir='', names=[]): # [0, 0, 0, 13, 0, 0], # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] # example + array[array < 0.005] = np.nan # don't annotate (will appear as 0.0) df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) @@ -180,7 +180,8 @@ def plot(self, save_dir='', names=[]): sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, - xticklabels=names, yticklabels=names) + xticklabels=names + ['background FN'] if names else "auto", + yticklabels=names + ['background FP'] if names else "auto") plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 88c5d722c98bad08d2a1e6e3405f7c3e604cdb09 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:19:31 +0100 Subject: [PATCH 12/78] update plot --- utils/metrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/metrics.py b/utils/metrics.py index ebecdfb10bd3..6dba70d5f66a 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -182,6 +182,7 @@ def plot(self, save_dir='', names=[]): sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", yticklabels=names + ['background FP'] if names else "auto") + plt.tight_layout() plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 8a4c2ac5a7ea61483d9c1adf52b9bccd62557abc Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:22:13 +0100 Subject: [PATCH 13/78] update plot --- utils/metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/metrics.py b/utils/metrics.py index 6dba70d5f66a..b56f3d0f4a0b 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -177,12 +177,13 @@ def plot(self, save_dir='', names=[]): df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) plt.figure(figsize=(12, 9)) + plt.tight_layout() sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", yticklabels=names + ['background FP'] if names else "auto") - plt.tight_layout() + plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 5ed4547d34897690af6d0a5712889921419ad684 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:22:30 +0100 Subject: [PATCH 14/78] update plot --- utils/metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index b56f3d0f4a0b..6dba70d5f66a 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -177,13 +177,12 @@ def plot(self, save_dir='', names=[]): df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) plt.figure(figsize=(12, 9)) - plt.tight_layout() sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", yticklabels=names + ['background FP'] if names else "auto") - + plt.tight_layout() plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 66be6d462bb07e8762e72de3abc6997cfd62a065 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:25:09 +0100 Subject: [PATCH 15/78] update plot --- utils/metrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/metrics.py b/utils/metrics.py index 6dba70d5f66a..7af9d5dc98e8 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -178,6 +178,7 @@ def plot(self, save_dir='', names=[]): plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) + sn.set_style("white", {'figure.facecolor': 'white'}) sn.set(font_scale=1.0) # for label size sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", From 4128c53ebd9a76423a2a633c4ff3ab00df20c094 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:32:26 +0100 Subject: [PATCH 16/78] update plot --- utils/metrics.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 7af9d5dc98e8..03e94e3e43a4 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -178,11 +178,11 @@ def plot(self, save_dir='', names=[]): plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) - sn.set_style("white", {'figure.facecolor': 'white'}) sn.set(font_scale=1.0) # for label size - sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, - xticklabels=names + ['background FN'] if names else "auto", - yticklabels=names + ['background FP'] if names else "auto") + g = sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + xticklabels=names + ['background FN'] if names else "auto", + yticklabels=names + ['background FP'] if names else "auto") + g.set_facecolor((1, 1, 1)) plt.tight_layout() plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) From 3bca0ea33b2c8a6de7a1e06a52d8c1c272ff4155 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:43:03 +0100 Subject: [PATCH 17/78] cleanup --- utils/metrics.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 03e94e3e43a4..88d54c4cd670 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -106,12 +106,12 @@ def compute_ap(recall, precision): class ConfusionMatrix: - # https://github.com/kaanakan/object_detection_confusion_matrix - def __init__(self, nc, CONF_THRESHOLD=0.25, IOU_THRESHOLD=0.45): + # From https://github.com/kaanakan/object_detection_confusion_matrix + def __init__(self, nc, conf=0.25, iou_thres=0.45): self.matrix = np.zeros((nc + 1, nc + 1)) self.nc = nc # number of classes - self.CONF_THRESHOLD = CONF_THRESHOLD - self.IOU_THRESHOLD = IOU_THRESHOLD + self.conf = conf + self.iou_thres = iou_thres def process_batch(self, detections, labels): """ @@ -123,12 +123,12 @@ def process_batch(self, detections, labels): Returns: None, updates confusion matrix accordingly """ - detections = detections[detections[:, 4] > self.CONF_THRESHOLD] + detections = detections[detections[:, 4] > self.conf] gt_classes = labels[:, 0].int() detection_classes = detections[:, 5].int() all_ious = general.box_iou(labels[:, 1:], detections[:, :4]) - want_idx = torch.where(all_ious > self.IOU_THRESHOLD) + want_idx = torch.where(all_ious > self.iou_thres) all_matches = [] for i in range(want_idx[0].shape[0]): @@ -136,7 +136,7 @@ def process_batch(self, detections, labels): if all_matches: all_matches = np.array(all_matches) - if all_matches.shape[0] > 0: # if there is match + if all_matches.shape[0]: # if there is match all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] all_matches = all_matches[np.unique(all_matches[:, 1], return_index=True)[1]] all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] @@ -145,13 +145,13 @@ def process_batch(self, detections, labels): all_matches = torch.zeros([0, 1]) for i, label in enumerate(labels): - if all_matches.shape[0] > 0 and all_matches[all_matches[:, 0] == i].shape[0] == 1: + if all_matches.shape[0] and all_matches[all_matches[:, 0] == i].shape[0] == 1: gt_class = gt_classes[i] detection_class = detection_classes[int(all_matches[all_matches[:, 0] == i, 1][0])] - self.matrix[(gt_class), detection_class] += 1 + self.matrix[gt_class, detection_class] += 1 else: gt_class = gt_classes[i] - self.matrix[(gt_class), self.nc] += 1 + self.matrix[gt_class, self.nc] += 1 for i, detection in enumerate(detections): if all_matches.shape[0] and all_matches[all_matches[:, 1] == i].shape[0] == 0: From 6c4e74f3f9292c563679aff1257f9c1d1bcdeb59 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:47:37 +0100 Subject: [PATCH 18/78] cleanup --- utils/metrics.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 88d54c4cd670..972ccbeb848b 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -130,33 +130,31 @@ def process_batch(self, detections, labels): all_ious = general.box_iou(labels[:, 1:], detections[:, :4]) want_idx = torch.where(all_ious > self.iou_thres) - all_matches = [] + matches = [] for i in range(want_idx[0].shape[0]): - all_matches.append([want_idx[0][i], want_idx[1][i], all_ious[want_idx[0][i], want_idx[1][i]]]) - - if all_matches: - all_matches = np.array(all_matches) - if all_matches.shape[0]: # if there is match - all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] - all_matches = all_matches[np.unique(all_matches[:, 1], return_index=True)[1]] - all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] - all_matches = all_matches[np.unique(all_matches[:, 0], return_index=True)[1]] + matches.append([want_idx[0][i], want_idx[1][i], all_ious[want_idx[0][i], want_idx[1][i]]]) + + if matches: + matches = np.array(matches) + if matches.shape[0]: # if there is match + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] else: - all_matches = torch.zeros([0, 1]) + matches = torch.zeros([0, 1]) for i, label in enumerate(labels): - if all_matches.shape[0] and all_matches[all_matches[:, 0] == i].shape[0] == 1: - gt_class = gt_classes[i] - detection_class = detection_classes[int(all_matches[all_matches[:, 0] == i, 1][0])] - self.matrix[gt_class, detection_class] += 1 + gti = gt_classes[i] + if matches.shape[0] and matches[matches[:, 0] == i].shape[0] == 1: + c = detection_classes[int(matches[matches[:, 0] == i, 1][0])] + self.matrix[gti, c] += 1 else: - gt_class = gt_classes[i] - self.matrix[gt_class, self.nc] += 1 + self.matrix[gti, self.nc] += 1 for i, detection in enumerate(detections): - if all_matches.shape[0] and all_matches[all_matches[:, 1] == i].shape[0] == 0: - detection_class = detection_classes[i] - self.matrix[self.nc, detection_class] += 1 + if matches.shape[0] and matches[matches[:, 1] == i].shape[0] == 0: + self.matrix[self.nc, detection_classes[i]] += 1 def matrix(self): return self.matrix From 73b7ab333c6233f381091a3d69fd5bdc68af032b Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:57:57 +0100 Subject: [PATCH 19/78] cleanup --- utils/metrics.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 972ccbeb848b..f76c02b9fb4e 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -127,12 +127,10 @@ def process_batch(self, detections, labels): gt_classes = labels[:, 0].int() detection_classes = detections[:, 5].int() - all_ious = general.box_iou(labels[:, 1:], detections[:, :4]) - want_idx = torch.where(all_ious > self.iou_thres) - matches = [] - for i in range(want_idx[0].shape[0]): - matches.append([want_idx[0][i], want_idx[1][i], all_ious[want_idx[0][i], want_idx[1][i]]]) + iou = general.box_iou(labels[:, 1:], detections[:, :4]) + for g1, g2 in zip(*torch.where(iou > self.iou_thres)): + matches.append([g1, g2, iou[g1, g2]]) if matches: matches = np.array(matches) @@ -142,7 +140,7 @@ def process_batch(self, detections, labels): matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 0], return_index=True)[1]] else: - matches = torch.zeros([0, 1]) + matches = np.zeros((0, 1)) for i, label in enumerate(labels): gti = gt_classes[i] From 0f8d209274158190642419f4dd1b9b69e25be269 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 18:00:29 +0100 Subject: [PATCH 20/78] cleanup --- utils/metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index f76c02b9fb4e..d1d8649fb13c 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -142,7 +142,7 @@ def process_batch(self, detections, labels): else: matches = np.zeros((0, 1)) - for i, label in enumerate(labels): + for i in range(len(labels)): gti = gt_classes[i] if matches.shape[0] and matches[matches[:, 0] == i].shape[0] == 1: c = detection_classes[int(matches[matches[:, 0] == i, 1][0])] @@ -150,7 +150,7 @@ def process_batch(self, detections, labels): else: self.matrix[gti, self.nc] += 1 - for i, detection in enumerate(detections): + for i in range(len(detections)): if matches.shape[0] and matches[matches[:, 1] == i].shape[0] == 0: self.matrix[self.nc, detection_classes[i]] += 1 From 08fc9f29711d2d7e3c5bd8917231466e52cf3bd5 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 18:05:10 +0100 Subject: [PATCH 21/78] cleanup --- requirements.txt | 8 ++++---- utils/metrics.py | 51 +++++++++++++++++++++++++----------------------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/requirements.txt b/requirements.txt index 517372e3f5c1..fb4f21a3a01b 100755 --- a/requirements.txt +++ b/requirements.txt @@ -16,8 +16,9 @@ tqdm>=4.41.0 # logging ------------------------------------- # wandb -# coco ---------------------------------------- -# pycocotools>=2.0 +# plotting ------------------------------------ +# seaborn +# pandas # export -------------------------------------- # coremltools==4.0 @@ -26,5 +27,4 @@ tqdm>=4.41.0 # extras -------------------------------------- # thop # FLOPS computation -# seaborn # plotting -# pandas # plotting +# pycocotools>=2.0 # COCO mAP diff --git a/utils/metrics.py b/utils/metrics.py index d1d8649fb13c..6536e8ce96a8 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -157,30 +157,33 @@ def process_batch(self, detections, labels): def matrix(self): return self.matrix - def plot(self, save_dir='', names=[]): - import seaborn as sn - import pandas as pd - - array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize - # array = [[13, 1, 1, 0, 2, 0], - # [3, 9, 6, 0, 1, 0], - # [0, 0, 16, 2, 0, 0], - # [0, 0, 0, 13, 0, 0], - # [0, 0, 0, 0, 15, 0], - # [0, 0, 1, 0, 0, 15]] # example - - array[array < 0.005] = np.nan # don't annotate (will appear as 0.0) - df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) - - plt.figure(figsize=(12, 9)) - sn.color_palette("magma", as_cmap=True) - sn.set(font_scale=1.0) # for label size - g = sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, - xticklabels=names + ['background FN'] if names else "auto", - yticklabels=names + ['background FP'] if names else "auto") - g.set_facecolor((1, 1, 1)) - plt.tight_layout() - plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) + def plot(self, save_dir='', names=()): + try: + import seaborn as sn + import pandas as pd + + array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize + # array = [[13, 1, 1, 0, 2, 0], + # [3, 9, 6, 0, 1, 0], + # [0, 0, 16, 2, 0, 0], + # [0, 0, 0, 13, 0, 0], + # [0, 0, 0, 0, 15, 0], + # [0, 0, 1, 0, 0, 15]] # example + + array[array < 0.005] = np.nan # don't annotate (will appear as 0.0) + df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) + + plt.figure(figsize=(12, 9)) + sn.color_palette("magma", as_cmap=True) + sn.set(font_scale=1.0) # for label size + g = sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + xticklabels=names + ['background FN'] if names else "auto", + yticklabels=names + ['background FP'] if names else "auto") + g.set_facecolor((1, 1, 1)) + plt.tight_layout() + plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) + except Exception as e: + pass def print(self): for i in range(self.nc + 1): From 53aa594a6c1ff1d3c2a9aba48af35e692edff11b Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 18:08:40 +0100 Subject: [PATCH 22/78] cleanup --- utils/metrics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/metrics.py b/utils/metrics.py index 6536e8ce96a8..79406bb84930 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -174,7 +174,6 @@ def plot(self, save_dir='', names=()): df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) plt.figure(figsize=(12, 9)) - sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size g = sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", From e0b6cab56e8756377a0618cbddd8b5ce7a668fc7 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 18:25:06 +0100 Subject: [PATCH 23/78] cleanup --- utils/metrics.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 79406bb84930..2e5de1c2b926 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -126,33 +126,33 @@ def process_batch(self, detections, labels): detections = detections[detections[:, 4] > self.conf] gt_classes = labels[:, 0].int() detection_classes = detections[:, 5].int() - - matches = [] iou = general.box_iou(labels[:, 1:], detections[:, :4]) - for g1, g2 in zip(*torch.where(iou > self.iou_thres)): - matches.append([g1, g2, iou[g1, g2]]) - - if matches: - matches = np.array(matches) - if matches.shape[0]: # if there is match - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique(matches[:, 1], return_index=True)[1]] - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + + # matches = [] + # for g1, g2 in zip(*torch.where(iou > self.iou_thres)): + # matches.append([g1, g2, iou[g1, g2]]) + + x = torch.where(iou > self.iou_thres) + if x[0].shape[0]: + matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).numpy() + + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] else: matches = np.zeros((0, 1)) - for i in range(len(labels)): - gti = gt_classes[i] + for i, gc in enumerate(gt_classes): if matches.shape[0] and matches[matches[:, 0] == i].shape[0] == 1: - c = detection_classes[int(matches[matches[:, 0] == i, 1][0])] - self.matrix[gti, c] += 1 + dc = detection_classes[int(matches[matches[:, 0] == i, 1][0])] + self.matrix[gc, dc] += 1 else: - self.matrix[gti, self.nc] += 1 + self.matrix[gc, self.nc] += 1 - for i in range(len(detections)): + for i, dc in enumerate(detection_classes): if matches.shape[0] and matches[matches[:, 1] == i].shape[0] == 0: - self.matrix[self.nc, detection_classes[i]] += 1 + self.matrix[self.nc, dc] += 1 def matrix(self): return self.matrix From 28f0c8f2cbb8492c4d33fcc0ff172287860bd022 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 18:26:26 +0100 Subject: [PATCH 24/78] cleanup --- utils/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/metrics.py b/utils/metrics.py index 2e5de1c2b926..fea4ef7dc655 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -134,7 +134,7 @@ def process_batch(self, detections, labels): x = torch.where(iou > self.iou_thres) if x[0].shape[0]: - matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).numpy() + matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 1], return_index=True)[1]] From 9ffc8f21e4b2e97825070b59c9eadc3c0409d949 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 18:52:46 +0100 Subject: [PATCH 25/78] cleanup --- utils/metrics.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index fea4ef7dc655..1e0e2a9e6798 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -128,10 +128,6 @@ def process_batch(self, detections, labels): detection_classes = detections[:, 5].int() iou = general.box_iou(labels[:, 1:], detections[:, :4]) - # matches = [] - # for g1, g2 in zip(*torch.where(iou > self.iou_thres)): - # matches.append([g1, g2, iou[g1, g2]]) - x = torch.where(iou > self.iou_thres) if x[0].shape[0]: matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() @@ -148,7 +144,8 @@ def process_batch(self, detections, labels): dc = detection_classes[int(matches[matches[:, 0] == i, 1][0])] self.matrix[gc, dc] += 1 else: - self.matrix[gc, self.nc] += 1 + self.matrix[gc, self.nc] += 1 # background + for i, dc in enumerate(detection_classes): if matches.shape[0] and matches[matches[:, 1] == i].shape[0] == 0: From 64ac27867a12e4266c96f30bbeab45b7dd85197f Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 19:09:53 +0100 Subject: [PATCH 26/78] cleanup --- utils/metrics.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 1e0e2a9e6798..d364a9f2bb5c 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -137,18 +137,20 @@ def process_batch(self, detections, labels): matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 0], return_index=True)[1]] else: - matches = np.zeros((0, 1)) + matches = np.zeros((0, 3)) + n = matches.shape[0] > 0 + m0, m1, _ = matches.transpose().astype(np.uint16) for i, gc in enumerate(gt_classes): - if matches.shape[0] and matches[matches[:, 0] == i].shape[0] == 1: - dc = detection_classes[int(matches[matches[:, 0] == i, 1][0])] + j = m0 == i + if n and sum(j) == 1: + dc = detection_classes[m1[j]] self.matrix[gc, dc] += 1 else: self.matrix[gc, self.nc] += 1 # background - for i, dc in enumerate(detection_classes): - if matches.shape[0] and matches[matches[:, 1] == i].shape[0] == 0: + if n and not any(m1 == i): self.matrix[self.nc, dc] += 1 def matrix(self): From e15bd1686f1959e098565b6ff1f5614b69c3d933 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 19:11:42 +0100 Subject: [PATCH 27/78] cleanup --- utils/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/metrics.py b/utils/metrics.py index d364a9f2bb5c..7bd897179cf1 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -140,7 +140,7 @@ def process_batch(self, detections, labels): matches = np.zeros((0, 3)) n = matches.shape[0] > 0 - m0, m1, _ = matches.transpose().astype(np.uint16) + m0, m1, _ = matches.transpose().astype(np.int16) for i, gc in enumerate(gt_classes): j = m0 == i if n and sum(j) == 1: From a9de0ddcfa766cf745f05866895f77913acd0702 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 19:18:55 +0100 Subject: [PATCH 28/78] cleanup --- utils/metrics.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 7bd897179cf1..59bbfbe4c17b 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -131,11 +131,11 @@ def process_batch(self, detections, labels): x = torch.where(iou > self.iou_thres) if x[0].shape[0]: matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() - - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique(matches[:, 1], return_index=True)[1]] - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + if x[0].shape[0] > 1: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] else: matches = np.zeros((0, 3)) @@ -147,11 +147,12 @@ def process_batch(self, detections, labels): dc = detection_classes[m1[j]] self.matrix[gc, dc] += 1 else: - self.matrix[gc, self.nc] += 1 # background + self.matrix[gc, self.nc] += 1 # background FP - for i, dc in enumerate(detection_classes): - if n and not any(m1 == i): - self.matrix[self.nc, dc] += 1 + if n: + for i, dc in enumerate(detection_classes): + if not any(m1 == i): + self.matrix[self.nc, dc] += 1 # background FN def matrix(self): return self.matrix From 34b14799047593c71d9a4ca415bdd46e7bef442e Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 19:38:12 +0100 Subject: [PATCH 29/78] cleanup --- utils/metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 59bbfbe4c17b..a5eddb9e94c6 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -144,8 +144,7 @@ def process_batch(self, detections, labels): for i, gc in enumerate(gt_classes): j = m0 == i if n and sum(j) == 1: - dc = detection_classes[m1[j]] - self.matrix[gc, dc] += 1 + self.matrix[gc, detection_classes[m1[j]]] += 1 # correct else: self.matrix[gc, self.nc] += 1 # background FP From e05efc607ef1549672f12888069e8e825aa9b830 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 19:47:30 +0100 Subject: [PATCH 30/78] seaborn pandas to requirements.txt --- requirements.txt | 4 ++-- train.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index fb4f21a3a01b..5e813fb0420c 100755 --- a/requirements.txt +++ b/requirements.txt @@ -17,8 +17,8 @@ tqdm>=4.41.0 # wandb # plotting ------------------------------------ -# seaborn -# pandas +seaborn +pandas # export -------------------------------------- # coremltools==4.0 diff --git a/train.py b/train.py index a2244fcbf395..c4aaa4c34eea 100644 --- a/train.py +++ b/train.py @@ -396,8 +396,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if plots: plot_results(save_dir=save_dir) # save as results.png if wandb: - wandb.log({"Results": [wandb.Image(str(save_dir / x), caption=x) for x in - ['results.png', 'precision_recall_curve.png']]}) + wandb.log({"Results": [wandb.Image(str(save_dir / x), caption=str(x)) for x in save_dir.glob('*.png')]}) logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600)) else: dist.destroy_process_group() From b552b43d535262e59032a1834c755c40a69c5741 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:06:35 +0100 Subject: [PATCH 31/78] seaborn pandas to requirements.txt --- train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/train.py b/train.py index c4aaa4c34eea..76e64e597edf 100644 --- a/train.py +++ b/train.py @@ -396,7 +396,8 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if plots: plot_results(save_dir=save_dir) # save as results.png if wandb: - wandb.log({"Results": [wandb.Image(str(save_dir / x), caption=str(x)) for x in save_dir.glob('*.png')]}) + files = ['results.png', 'precision_recall_curve.png', 'confusion_matrix.png'] + wandb.log({"Results": [wandb.Image(str(save_dir / x), caption=x) for x in files if os.path.exists(x)]}) logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600)) else: dist.destroy_process_group() From b602ad2a0998afc093861d6092836463242e95f3 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:19:04 +0100 Subject: [PATCH 32/78] update wandb plotting --- test.py | 2 +- train.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test.py b/test.py index 7793d65d11ed..faa1e8deb599 100644 --- a/test.py +++ b/test.py @@ -227,7 +227,7 @@ def test(data, confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) if wandb and wandb.run: wandb.log({"Images": wandb_images}) - wandb.log({"Validation": [wandb.Image(str(x), caption=x.name) for x in sorted(save_dir.glob('test*.jpg'))]}) + wandb.log({"Validation": [wandb.Image(str(f), caption=f.name) for f in sorted(save_dir.glob('test*.jpg'))]}) # Print results pf = '%20s' + '%12.3g' * 6 # print format diff --git a/train.py b/train.py index 76e64e597edf..60da58b91137 100644 --- a/train.py +++ b/train.py @@ -397,7 +397,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): plot_results(save_dir=save_dir) # save as results.png if wandb: files = ['results.png', 'precision_recall_curve.png', 'confusion_matrix.png'] - wandb.log({"Results": [wandb.Image(str(save_dir / x), caption=x) for x in files if os.path.exists(x)]}) + wandb.log({"Results": [wandb.Image(str(save_dir / f), caption=f) for f in files]}) logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600)) else: dist.destroy_process_group() From 0837504538d4fde150cebf7c70d908025c69670c Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:23:18 +0100 Subject: [PATCH 33/78] remove pandas --- utils/metrics.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index a5eddb9e94c6..8d3a3dc38e2d 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -159,7 +159,6 @@ def matrix(self): def plot(self, save_dir='', names=()): try: import seaborn as sn - import pandas as pd array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize # array = [[13, 1, 1, 0, 2, 0], @@ -168,13 +167,11 @@ def plot(self, save_dir='', names=()): # [0, 0, 0, 13, 0, 0], # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] # example - array[array < 0.005] = np.nan # don't annotate (will appear as 0.0) - df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) plt.figure(figsize=(12, 9)) sn.set(font_scale=1.0) # for label size - g = sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + g = sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", yticklabels=names + ['background FP'] if names else "auto") g.set_facecolor((1, 1, 1)) From 05d456d78fff425a9a94ef3f7d89495665e5cc52 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:27:59 +0100 Subject: [PATCH 34/78] if plots --- test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test.py b/test.py index faa1e8deb599..d7eef6d5e761 100644 --- a/test.py +++ b/test.py @@ -177,9 +177,8 @@ def test(data, # target boxes tbox = xywh2xyxy(labels[:, 1:5]) * whwh scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1]) # native-space labels - - # Confusion Matrix - confusion_matrix.process_batch(pred, torch.cat((labels[:, 0:1], tbox), 1)) + if plots: + confusion_matrix.process_batch(pred, torch.cat((labels[:, 0:1], tbox), 1)) # Per target class for cls in torch.unique(tcls_tensor): From a846da137bd244994079bbb015a31d8e0109564c Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:33:46 +0100 Subject: [PATCH 35/78] if plots --- utils/metrics.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 8d3a3dc38e2d..f0db5870b130 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -161,22 +161,15 @@ def plot(self, save_dir='', names=()): import seaborn as sn array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize - # array = [[13, 1, 1, 0, 2, 0], - # [3, 9, 6, 0, 1, 0], - # [0, 0, 16, 2, 0, 0], - # [0, 0, 0, 13, 0, 0], - # [0, 0, 0, 0, 15, 0], - # [0, 0, 1, 0, 0, 15]] # example - array[array < 0.005] = np.nan # don't annotate (will appear as 0.0) - - plt.figure(figsize=(12, 9)) + array[array < 0.005] = np.nan # don't annotate (would appear as 0.00) + + fig = plt.figure(figsize=(12, 9)) sn.set(font_scale=1.0) # for label size - g = sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, - xticklabels=names + ['background FN'] if names else "auto", - yticklabels=names + ['background FP'] if names else "auto") - g.set_facecolor((1, 1, 1)) - plt.tight_layout() - plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) + sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + xticklabels=names + ['background FN'] if names else "auto", + yticklabels=names + ['background FP'] if names else "auto").set_facecolor((1, 1, 1)) + fig.tight_layout() + fig.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) except Exception as e: pass From 36c6cdb12899d8e4d558d3cf6f21f6462dbad903 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:37:11 +0100 Subject: [PATCH 36/78] if plots --- utils/metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index f0db5870b130..cabb7c64c7b0 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -163,9 +163,9 @@ def plot(self, save_dir='', names=()): array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize array[array < 0.005] = np.nan # don't annotate (would appear as 0.00) - fig = plt.figure(figsize=(12, 9)) - sn.set(font_scale=1.0) # for label size - sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + fig, ax = plt.figure(figsize=(12, 9)) + # sn.set(font_scale=1.0) # for label size + sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, ax=ax, xticklabels=names + ['background FN'] if names else "auto", yticklabels=names + ['background FP'] if names else "auto").set_facecolor((1, 1, 1)) fig.tight_layout() From 4faf5dc9aed2825618621aaea2aa7af850928f2f Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:45:21 +0100 Subject: [PATCH 37/78] if plots --- test.py | 2 +- train.py | 3 ++- utils/metrics.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test.py b/test.py index d7eef6d5e761..062cce0f26d9 100644 --- a/test.py +++ b/test.py @@ -282,7 +282,7 @@ def test(data, parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model.pt path(s)') parser.add_argument('--data', type=str, default='data/coco128.yaml', help='*.data path') parser.add_argument('--batch-size', type=int, default=32, help='size of each image batch') - parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--img-size', type=int, default=256, help='inference size (pixels)') parser.add_argument('--conf-thres', type=float, default=0.001, help='object confidence threshold') parser.add_argument('--iou-thres', type=float, default=0.6, help='IOU threshold for NMS') parser.add_argument('--task', default='val', help="'val', 'test', 'study'") diff --git a/train.py b/train.py index 60da58b91137..ca8148a82176 100644 --- a/train.py +++ b/train.py @@ -397,7 +397,8 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): plot_results(save_dir=save_dir) # save as results.png if wandb: files = ['results.png', 'precision_recall_curve.png', 'confusion_matrix.png'] - wandb.log({"Results": [wandb.Image(str(save_dir / f), caption=f) for f in files]}) + wandb.log({"Results": [wandb.Image(str(save_dir / f), caption=f) for f in files + if (save_dir / f).exists()]}) logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600)) else: dist.destroy_process_group() diff --git a/utils/metrics.py b/utils/metrics.py index cabb7c64c7b0..c5dac2b6103b 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -163,9 +163,9 @@ def plot(self, save_dir='', names=()): array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize array[array < 0.005] = np.nan # don't annotate (would appear as 0.00) - fig, ax = plt.figure(figsize=(12, 9)) + fig = plt.figure(figsize=(12, 9)) # sn.set(font_scale=1.0) # for label size - sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, ax=ax, + sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", yticklabels=names + ['background FP'] if names else "auto").set_facecolor((1, 1, 1)) fig.tight_layout() From cdc2b18741941df74a2aa388c83ab34e574089d8 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:45:56 +0100 Subject: [PATCH 38/78] if plots --- test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.py b/test.py index 062cce0f26d9..d7eef6d5e761 100644 --- a/test.py +++ b/test.py @@ -282,7 +282,7 @@ def test(data, parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model.pt path(s)') parser.add_argument('--data', type=str, default='data/coco128.yaml', help='*.data path') parser.add_argument('--batch-size', type=int, default=32, help='size of each image batch') - parser.add_argument('--img-size', type=int, default=256, help='inference size (pixels)') + parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') parser.add_argument('--conf-thres', type=float, default=0.001, help='object confidence threshold') parser.add_argument('--iou-thres', type=float, default=0.6, help='IOU threshold for NMS') parser.add_argument('--task', default='val', help="'val', 'test', 'study'") From 6ac4bff6a7c6092ecc21698328ea77e996d85092 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 14:20:09 +0100 Subject: [PATCH 39/78] initial commit --- test.py | 6 ++++- utils/metrics.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/test.py b/test.py index e50fb117ef80..ed69b45bcfc0 100644 --- a/test.py +++ b/test.py @@ -14,7 +14,7 @@ from utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, box_iou, \ non_max_suppression, scale_coords, xyxy2xywh, xywh2xyxy, set_logging, increment_path from utils.loss import compute_loss -from utils.metrics import ap_per_class +from utils.metrics import ap_per_class, ConfusionMatrix from utils.plots import plot_images, output_to_target from utils.torch_utils import select_device, time_synchronized @@ -89,6 +89,7 @@ def test(data, dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt, pad=0.5, rect=True)[0] seen = 0 + confusion_matrix = ConfusionMatrix(num_classes=nc) names = {k: v for k, v in enumerate(model.names if hasattr(model, 'names') else model.module.names)} coco91class = coco80_to_coco91_class() s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP@.5', 'mAP@.5:.95') @@ -136,6 +137,9 @@ def test(data, predn = pred.clone() scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred + # Confusion Matrix + confusion_matrix.process_batch(pred, labels) + # Append to text file if save_txt: gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh diff --git a/utils/metrics.py b/utils/metrics.py index 62add1da1f8d..f87a93e5bd01 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -5,6 +5,8 @@ import matplotlib.pyplot as plt import numpy as np +from . import general + def fitness(x): # Model fitness as a weighted combination of metrics @@ -102,6 +104,69 @@ def compute_ap(recall, precision): return ap, mpre, mrec +class ConfusionMatrix: + # https://github.com/kaanakan/object_detection_confusion_matrix + def __init__(self, num_classes, CONF_THRESHOLD=0.3, IOU_THRESHOLD=0.5): + self.matrix = np.zeros((num_classes + 1, num_classes + 1)) + self.num_classes = num_classes + self.CONF_THRESHOLD = CONF_THRESHOLD + self.IOU_THRESHOLD = IOU_THRESHOLD + + def process_batch(self, detections, labels): + """ + Return intersection-over-union (Jaccard index) of boxes. + Both sets of boxes are expected to be in (x1, y1, x2, y2) format. + Arguments: + detections (Array[N, 6]), x1, y1, x2, y2, conf, class + labels (Array[M, 5]), class, x1, y1, x2, y2 + Returns: + None, updates confusion matrix accordingly + """ + detections = detections[detections[:, 4] > self.CONF_THRESHOLD] + gt_classes = labels[:, 0].astype(np.int16) + detection_classes = detections[:, 5].astype(np.int16) + + all_ious = general.box_iou(labels[:, 1:], detections[:, :4]) + want_idx = np.where(all_ious > self.IOU_THRESHOLD) + + all_matches = [] + for i in range(want_idx[0].shape[0]): + all_matches.append([want_idx[0][i], want_idx[1][i], all_ious[want_idx[0][i], want_idx[1][i]]]) + + all_matches = np.array(all_matches) + if all_matches.shape[0] > 0: # if there is match + all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] + + all_matches = all_matches[np.unique(all_matches[:, 1], return_index=True)[1]] + + all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] + + all_matches = all_matches[np.unique(all_matches[:, 0], return_index=True)[1]] + + for i, label in enumerate(labels): + if all_matches.shape[0] > 0 and all_matches[all_matches[:, 0] == i].shape[0] == 1: + gt_class = gt_classes[i] + detection_class = detection_classes[int(all_matches[all_matches[:, 0] == i, 1][0])] + self.matrix[(gt_class), detection_class] += 1 + else: + gt_class = gt_classes[i] + self.matrix[(gt_class), self.num_classes] += 1 + + for i, detection in enumerate(detections): + if all_matches.shape[0] and all_matches[all_matches[:, 1] == i].shape[0] == 0: + detection_class = detection_classes[i] + self.matrix[self.num_classes, detection_class] += 1 + + def return_matrix(self): + return self.matrix + + def print_matrix(self): + for i in range(self.num_classes + 1): + print(' '.join(map(str, self.matrix[i]))) + + +# Plots ---------------------------------------------------------------------------------------------------------------- + def plot_pr_curve(px, py, ap, save_dir='.', names=()): fig, ax = plt.subplots(1, 1, figsize=(9, 6)) py = np.stack(py, axis=1) From 3f1538ce6800c933e7503ad2c7b5dd6e0ebd6fd9 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 14:51:06 +0100 Subject: [PATCH 40/78] add plotting --- test.py | 16 +++++++++------- utils/metrics.py | 29 ++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/test.py b/test.py index ed69b45bcfc0..6519cffc8920 100644 --- a/test.py +++ b/test.py @@ -137,9 +137,6 @@ def test(data, predn = pred.clone() scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred - # Confusion Matrix - confusion_matrix.process_batch(pred, labels) - # Append to text file if save_txt: gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh @@ -181,6 +178,9 @@ def test(data, tbox = xywh2xyxy(labels[:, 1:5]) scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1]) # native-space labels + # Confusion Matrix + confusion_matrix.process_batch(pred, torch.cat((labels[:, 0:1], tbox), 1)) + # Per target class for cls in torch.unique(tcls_tensor): ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1) # prediction indices @@ -222,10 +222,12 @@ def test(data, else: nt = torch.zeros(1) - # W&B logging - if plots and wandb and wandb.run: - wandb.log({"Images": wandb_images}) - wandb.log({"Validation": [wandb.Image(str(x), caption=x.name) for x in sorted(save_dir.glob('test*.jpg'))]}) + # Plots + if plots: + confusion_matrix.plot(save_dir=save_dir) + if wandb and wandb.run: + wandb.log({"Images": wandb_images}) + wandb.log({"Validation": [wandb.Image(str(x), caption=x.name) for x in sorted(save_dir.glob('test*.jpg'))]}) # Print results pf = '%20s' + '%12.3g' * 6 # print format diff --git a/utils/metrics.py b/utils/metrics.py index f87a93e5bd01..7323771c5892 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -106,7 +106,7 @@ def compute_ap(recall, precision): class ConfusionMatrix: # https://github.com/kaanakan/object_detection_confusion_matrix - def __init__(self, num_classes, CONF_THRESHOLD=0.3, IOU_THRESHOLD=0.5): + def __init__(self, num_classes, CONF_THRESHOLD=0.0, IOU_THRESHOLD=0.45): self.matrix = np.zeros((num_classes + 1, num_classes + 1)) self.num_classes = num_classes self.CONF_THRESHOLD = CONF_THRESHOLD @@ -123,8 +123,8 @@ def process_batch(self, detections, labels): None, updates confusion matrix accordingly """ detections = detections[detections[:, 4] > self.CONF_THRESHOLD] - gt_classes = labels[:, 0].astype(np.int16) - detection_classes = detections[:, 5].astype(np.int16) + gt_classes = labels[:, 0].int() + detection_classes = detections[:, 5].int() all_ious = general.box_iou(labels[:, 1:], detections[:, :4]) want_idx = np.where(all_ious > self.IOU_THRESHOLD) @@ -157,10 +157,29 @@ def process_batch(self, detections, labels): detection_class = detection_classes[i] self.matrix[self.num_classes, detection_class] += 1 - def return_matrix(self): + def matrix(self): return self.matrix - def print_matrix(self): + def plot(self, save_dir=''): + import seaborn as sn + import pandas as pd + + # array = [[13, 1, 1, 0, 2, 0], + # [3, 9, 6, 0, 1, 0], + # [0, 0, 16, 2, 0, 0], + # [0, 0, 0, 13, 0, 0], + # [0, 0, 0, 0, 15, 0], + # [0, 0, 1, 0, 0, 15]] + + array = self.matrix.numpy() + df_cm = pd.DataFrame(array, range(6), range(6)) + plt.figure(figsize=(10,7)) + sn.color_palette("magma", as_cmap=True) + sn.set(font_scale=1.4) # for label size + sn.heatmap(df_cm, annot=True, annot_kws={"size": 16}, cmap='Blues') # font size + plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) + + def print(self): for i in range(self.num_classes + 1): print(' '.join(map(str, self.matrix[i]))) From 556c7f48aa1894e3880e5ff69768ec3734db2451 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 14:52:34 +0100 Subject: [PATCH 41/78] matrix to cpu --- utils/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/metrics.py b/utils/metrics.py index 7323771c5892..405dd3961c07 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -171,7 +171,7 @@ def plot(self, save_dir=''): # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] - array = self.matrix.numpy() + array = self.matrix.cpu().numpy() df_cm = pd.DataFrame(array, range(6), range(6)) plt.figure(figsize=(10,7)) sn.color_palette("magma", as_cmap=True) From 35f182aee8f18c21832d5d3fd0e45f0c2fe24312 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 15:13:44 +0100 Subject: [PATCH 42/78] bug fix --- test.py | 2 +- utils/metrics.py | 38 +++++++++++++++++++------------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/test.py b/test.py index 6519cffc8920..5bf05e3df31f 100644 --- a/test.py +++ b/test.py @@ -89,7 +89,7 @@ def test(data, dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt, pad=0.5, rect=True)[0] seen = 0 - confusion_matrix = ConfusionMatrix(num_classes=nc) + confusion_matrix = ConfusionMatrix(nc=nc) names = {k: v for k, v in enumerate(model.names if hasattr(model, 'names') else model.module.names)} coco91class = coco80_to_coco91_class() s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP@.5', 'mAP@.5:.95') diff --git a/utils/metrics.py b/utils/metrics.py index 405dd3961c07..2f734eb2beec 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -4,6 +4,7 @@ import matplotlib.pyplot as plt import numpy as np +import torch from . import general @@ -106,9 +107,9 @@ def compute_ap(recall, precision): class ConfusionMatrix: # https://github.com/kaanakan/object_detection_confusion_matrix - def __init__(self, num_classes, CONF_THRESHOLD=0.0, IOU_THRESHOLD=0.45): - self.matrix = np.zeros((num_classes + 1, num_classes + 1)) - self.num_classes = num_classes + def __init__(self, nc, CONF_THRESHOLD=0.0, IOU_THRESHOLD=0.45): + self.matrix = np.zeros((nc + 1, nc + 1)) + self.nc = nc # number of classes self.CONF_THRESHOLD = CONF_THRESHOLD self.IOU_THRESHOLD = IOU_THRESHOLD @@ -127,21 +128,21 @@ def process_batch(self, detections, labels): detection_classes = detections[:, 5].int() all_ious = general.box_iou(labels[:, 1:], detections[:, :4]) - want_idx = np.where(all_ious > self.IOU_THRESHOLD) + want_idx = torch.where(all_ious > self.IOU_THRESHOLD) all_matches = [] for i in range(want_idx[0].shape[0]): all_matches.append([want_idx[0][i], want_idx[1][i], all_ious[want_idx[0][i], want_idx[1][i]]]) - all_matches = np.array(all_matches) - if all_matches.shape[0] > 0: # if there is match - all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] - - all_matches = all_matches[np.unique(all_matches[:, 1], return_index=True)[1]] - - all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] - - all_matches = all_matches[np.unique(all_matches[:, 0], return_index=True)[1]] + if all_matches: + all_matches = np.array(all_matches) + if all_matches.shape[0] > 0: # if there is match + all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] + all_matches = all_matches[np.unique(all_matches[:, 1], return_index=True)[1]] + all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] + all_matches = all_matches[np.unique(all_matches[:, 0], return_index=True)[1]] + else: + all_matches = torch.zeros([0, 1]) for i, label in enumerate(labels): if all_matches.shape[0] > 0 and all_matches[all_matches[:, 0] == i].shape[0] == 1: @@ -150,12 +151,12 @@ def process_batch(self, detections, labels): self.matrix[(gt_class), detection_class] += 1 else: gt_class = gt_classes[i] - self.matrix[(gt_class), self.num_classes] += 1 + self.matrix[(gt_class), self.nc] += 1 for i, detection in enumerate(detections): if all_matches.shape[0] and all_matches[all_matches[:, 1] == i].shape[0] == 0: detection_class = detection_classes[i] - self.matrix[self.num_classes, detection_class] += 1 + self.matrix[self.nc, detection_class] += 1 def matrix(self): return self.matrix @@ -171,16 +172,15 @@ def plot(self, save_dir=''): # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] - array = self.matrix.cpu().numpy() - df_cm = pd.DataFrame(array, range(6), range(6)) - plt.figure(figsize=(10,7)) + df_cm = pd.DataFrame(self.matrix, range(self.nc + 1), range(self.nc + 1)) + plt.figure(figsize=(10, 7)) sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.4) # for label size sn.heatmap(df_cm, annot=True, annot_kws={"size": 16}, cmap='Blues') # font size plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): - for i in range(self.num_classes + 1): + for i in range(self.nc + 1): print(' '.join(map(str, self.matrix[i]))) From d3f997f846aa968e4615d147a6ab19391bff7088 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 15:40:25 +0100 Subject: [PATCH 43/78] update plot --- utils/metrics.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 2f734eb2beec..7e18973933fb 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -107,7 +107,7 @@ def compute_ap(recall, precision): class ConfusionMatrix: # https://github.com/kaanakan/object_detection_confusion_matrix - def __init__(self, nc, CONF_THRESHOLD=0.0, IOU_THRESHOLD=0.45): + def __init__(self, nc, CONF_THRESHOLD=0.25, IOU_THRESHOLD=0.45): self.matrix = np.zeros((nc + 1, nc + 1)) self.nc = nc # number of classes self.CONF_THRESHOLD = CONF_THRESHOLD @@ -171,12 +171,13 @@ def plot(self, save_dir=''): # [0, 0, 0, 13, 0, 0], # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] + array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) - df_cm = pd.DataFrame(self.matrix, range(self.nc + 1), range(self.nc + 1)) - plt.figure(figsize=(10, 7)) + df_cm = pd.DataFrame(array, range(self.nc+1), range(self.nc+1)) + plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) - sn.set(font_scale=1.4) # for label size - sn.heatmap(df_cm, annot=True, annot_kws={"size": 16}, cmap='Blues') # font size + sn.set(font_scale=1.0) # for label size + sn.heatmap(df_cm, annot=self.nc < 20, annot_kws={"size": 12}, cmap='Blues', square=True) # font size plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 50ceabbe53bdc1aa0ef8e8992d7090feaf5756d5 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 16:24:59 +0100 Subject: [PATCH 44/78] update plot --- utils/metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 7e18973933fb..da3602bea45e 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -165,19 +165,19 @@ def plot(self, save_dir=''): import seaborn as sn import pandas as pd + array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # array = [[13, 1, 1, 0, 2, 0], # [3, 9, 6, 0, 1, 0], # [0, 0, 16, 2, 0, 0], # [0, 0, 0, 13, 0, 0], # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] - array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) - df_cm = pd.DataFrame(array, range(self.nc+1), range(self.nc+1)) + df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size - sn.heatmap(df_cm, annot=self.nc < 20, annot_kws={"size": 12}, cmap='Blues', square=True) # font size + sn.heatmap(df_cm, annot=self.nc < 25, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True) plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 24fde43f5719ac41f2491ecdbda78ec3f2d9c779 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 16:27:12 +0100 Subject: [PATCH 45/78] update plot --- utils/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/metrics.py b/utils/metrics.py index da3602bea45e..061a48f12943 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -177,7 +177,7 @@ def plot(self, save_dir=''): plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size - sn.heatmap(df_cm, annot=self.nc < 25, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True) + sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True) plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 130cfc9de6f7d80f2e2b5ddb74b0b5a8c06b71f5 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 16:32:48 +0100 Subject: [PATCH 46/78] update plot --- utils/metrics.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 061a48f12943..54820ae42b81 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -165,19 +165,20 @@ def plot(self, save_dir=''): import seaborn as sn import pandas as pd - array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) + array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize # array = [[13, 1, 1, 0, 2, 0], # [3, 9, 6, 0, 1, 0], # [0, 0, 16, 2, 0, 0], # [0, 0, 0, 13, 0, 0], # [0, 0, 0, 0, 15, 0], - # [0, 0, 1, 0, 0, 15]] - + # [0, 0, 1, 0, 0, 15]] # example + array[array == 0] = np.nan # don't annotate 0.0 df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) + plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size - sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True) + sn.heatmap(df_cm, annot=self.nc < 930, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True) plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 8004b451b405d59fb6cd9b6c780dd4bde8c5b5eb Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 16:47:36 +0100 Subject: [PATCH 47/78] update plot --- requirements.txt | 1 + utils/metrics.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c5e6a6119321..517372e3f5c1 100755 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ tqdm>=4.41.0 # extras -------------------------------------- # thop # FLOPS computation # seaborn # plotting +# pandas # plotting diff --git a/utils/metrics.py b/utils/metrics.py index 54820ae42b81..7f6be3ed256a 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -172,7 +172,7 @@ def plot(self, save_dir=''): # [0, 0, 0, 13, 0, 0], # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] # example - array[array == 0] = np.nan # don't annotate 0.0 + array[array < 0.005] = np.nan # don't annotate (will appear as 0.0) df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) plt.figure(figsize=(12, 9)) From 0fd0b43dd59998874b3c2e3b3a2027700a5be05f Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:06:04 +0100 Subject: [PATCH 48/78] update plot --- test.py | 2 +- utils/metrics.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test.py b/test.py index 5bf05e3df31f..c3a42f8d7671 100644 --- a/test.py +++ b/test.py @@ -224,7 +224,7 @@ def test(data, # Plots if plots: - confusion_matrix.plot(save_dir=save_dir) + confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) if wandb and wandb.run: wandb.log({"Images": wandb_images}) wandb.log({"Validation": [wandb.Image(str(x), caption=x.name) for x in sorted(save_dir.glob('test*.jpg'))]}) diff --git a/utils/metrics.py b/utils/metrics.py index 7f6be3ed256a..977429ab8552 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -161,9 +161,10 @@ def process_batch(self, detections, labels): def matrix(self): return self.matrix - def plot(self, save_dir=''): + def plot(self, save_dir='', names=[]): import seaborn as sn import pandas as pd + names = names + ['background'] if names else "auto" array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize # array = [[13, 1, 1, 0, 2, 0], @@ -178,7 +179,8 @@ def plot(self, save_dir=''): plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size - sn.heatmap(df_cm, annot=self.nc < 930, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True) + sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + xticklabels=names, yticklabels=names) plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 860cce7108fcb36ec324e27e421a19726762ceb1 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:11:54 +0100 Subject: [PATCH 49/78] update plot --- utils/metrics.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 977429ab8552..ebecdfb10bd3 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -164,7 +164,6 @@ def matrix(self): def plot(self, save_dir='', names=[]): import seaborn as sn import pandas as pd - names = names + ['background'] if names else "auto" array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize # array = [[13, 1, 1, 0, 2, 0], @@ -173,6 +172,7 @@ def plot(self, save_dir='', names=[]): # [0, 0, 0, 13, 0, 0], # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] # example + array[array < 0.005] = np.nan # don't annotate (will appear as 0.0) df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) @@ -180,7 +180,8 @@ def plot(self, save_dir='', names=[]): sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, - xticklabels=names, yticklabels=names) + xticklabels=names + ['background FN'] if names else "auto", + yticklabels=names + ['background FP'] if names else "auto") plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 9a9845afa32dbb93cd8d4812e57d3ea972b2d48d Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:19:31 +0100 Subject: [PATCH 50/78] update plot --- utils/metrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/metrics.py b/utils/metrics.py index ebecdfb10bd3..6dba70d5f66a 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -182,6 +182,7 @@ def plot(self, save_dir='', names=[]): sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", yticklabels=names + ['background FP'] if names else "auto") + plt.tight_layout() plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From a71a9a431bae060811a58bd55f20d7df0a8a58c4 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:22:13 +0100 Subject: [PATCH 51/78] update plot --- utils/metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/metrics.py b/utils/metrics.py index 6dba70d5f66a..b56f3d0f4a0b 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -177,12 +177,13 @@ def plot(self, save_dir='', names=[]): df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) plt.figure(figsize=(12, 9)) + plt.tight_layout() sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", yticklabels=names + ['background FP'] if names else "auto") - plt.tight_layout() + plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 3a9ffe7438b10fd1e5458414e4ad864633d91568 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:22:30 +0100 Subject: [PATCH 52/78] update plot --- utils/metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index b56f3d0f4a0b..6dba70d5f66a 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -177,13 +177,12 @@ def plot(self, save_dir='', names=[]): df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) plt.figure(figsize=(12, 9)) - plt.tight_layout() sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", yticklabels=names + ['background FP'] if names else "auto") - + plt.tight_layout() plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) def print(self): From 370537aae88cb8d9c37469b3f3ce0b9eee6dff74 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:25:09 +0100 Subject: [PATCH 53/78] update plot --- utils/metrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/metrics.py b/utils/metrics.py index 6dba70d5f66a..7af9d5dc98e8 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -178,6 +178,7 @@ def plot(self, save_dir='', names=[]): plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) + sn.set_style("white", {'figure.facecolor': 'white'}) sn.set(font_scale=1.0) # for label size sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", From 9edbe6b447109b5f5e66ad3afb6eb77cbbd48542 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:32:26 +0100 Subject: [PATCH 54/78] update plot --- utils/metrics.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 7af9d5dc98e8..03e94e3e43a4 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -178,11 +178,11 @@ def plot(self, save_dir='', names=[]): plt.figure(figsize=(12, 9)) sn.color_palette("magma", as_cmap=True) - sn.set_style("white", {'figure.facecolor': 'white'}) sn.set(font_scale=1.0) # for label size - sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, - xticklabels=names + ['background FN'] if names else "auto", - yticklabels=names + ['background FP'] if names else "auto") + g = sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + xticklabels=names + ['background FN'] if names else "auto", + yticklabels=names + ['background FP'] if names else "auto") + g.set_facecolor((1, 1, 1)) plt.tight_layout() plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) From 526dfd381e105019dcd67944a415db143a746f35 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:43:03 +0100 Subject: [PATCH 55/78] cleanup --- utils/metrics.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 03e94e3e43a4..88d54c4cd670 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -106,12 +106,12 @@ def compute_ap(recall, precision): class ConfusionMatrix: - # https://github.com/kaanakan/object_detection_confusion_matrix - def __init__(self, nc, CONF_THRESHOLD=0.25, IOU_THRESHOLD=0.45): + # From https://github.com/kaanakan/object_detection_confusion_matrix + def __init__(self, nc, conf=0.25, iou_thres=0.45): self.matrix = np.zeros((nc + 1, nc + 1)) self.nc = nc # number of classes - self.CONF_THRESHOLD = CONF_THRESHOLD - self.IOU_THRESHOLD = IOU_THRESHOLD + self.conf = conf + self.iou_thres = iou_thres def process_batch(self, detections, labels): """ @@ -123,12 +123,12 @@ def process_batch(self, detections, labels): Returns: None, updates confusion matrix accordingly """ - detections = detections[detections[:, 4] > self.CONF_THRESHOLD] + detections = detections[detections[:, 4] > self.conf] gt_classes = labels[:, 0].int() detection_classes = detections[:, 5].int() all_ious = general.box_iou(labels[:, 1:], detections[:, :4]) - want_idx = torch.where(all_ious > self.IOU_THRESHOLD) + want_idx = torch.where(all_ious > self.iou_thres) all_matches = [] for i in range(want_idx[0].shape[0]): @@ -136,7 +136,7 @@ def process_batch(self, detections, labels): if all_matches: all_matches = np.array(all_matches) - if all_matches.shape[0] > 0: # if there is match + if all_matches.shape[0]: # if there is match all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] all_matches = all_matches[np.unique(all_matches[:, 1], return_index=True)[1]] all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] @@ -145,13 +145,13 @@ def process_batch(self, detections, labels): all_matches = torch.zeros([0, 1]) for i, label in enumerate(labels): - if all_matches.shape[0] > 0 and all_matches[all_matches[:, 0] == i].shape[0] == 1: + if all_matches.shape[0] and all_matches[all_matches[:, 0] == i].shape[0] == 1: gt_class = gt_classes[i] detection_class = detection_classes[int(all_matches[all_matches[:, 0] == i, 1][0])] - self.matrix[(gt_class), detection_class] += 1 + self.matrix[gt_class, detection_class] += 1 else: gt_class = gt_classes[i] - self.matrix[(gt_class), self.nc] += 1 + self.matrix[gt_class, self.nc] += 1 for i, detection in enumerate(detections): if all_matches.shape[0] and all_matches[all_matches[:, 1] == i].shape[0] == 0: From 76b74f3c9915826074782f1a0480edc6ceb50943 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:47:37 +0100 Subject: [PATCH 56/78] cleanup --- utils/metrics.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 88d54c4cd670..972ccbeb848b 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -130,33 +130,31 @@ def process_batch(self, detections, labels): all_ious = general.box_iou(labels[:, 1:], detections[:, :4]) want_idx = torch.where(all_ious > self.iou_thres) - all_matches = [] + matches = [] for i in range(want_idx[0].shape[0]): - all_matches.append([want_idx[0][i], want_idx[1][i], all_ious[want_idx[0][i], want_idx[1][i]]]) - - if all_matches: - all_matches = np.array(all_matches) - if all_matches.shape[0]: # if there is match - all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] - all_matches = all_matches[np.unique(all_matches[:, 1], return_index=True)[1]] - all_matches = all_matches[all_matches[:, 2].argsort()[::-1]] - all_matches = all_matches[np.unique(all_matches[:, 0], return_index=True)[1]] + matches.append([want_idx[0][i], want_idx[1][i], all_ious[want_idx[0][i], want_idx[1][i]]]) + + if matches: + matches = np.array(matches) + if matches.shape[0]: # if there is match + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] else: - all_matches = torch.zeros([0, 1]) + matches = torch.zeros([0, 1]) for i, label in enumerate(labels): - if all_matches.shape[0] and all_matches[all_matches[:, 0] == i].shape[0] == 1: - gt_class = gt_classes[i] - detection_class = detection_classes[int(all_matches[all_matches[:, 0] == i, 1][0])] - self.matrix[gt_class, detection_class] += 1 + gti = gt_classes[i] + if matches.shape[0] and matches[matches[:, 0] == i].shape[0] == 1: + c = detection_classes[int(matches[matches[:, 0] == i, 1][0])] + self.matrix[gti, c] += 1 else: - gt_class = gt_classes[i] - self.matrix[gt_class, self.nc] += 1 + self.matrix[gti, self.nc] += 1 for i, detection in enumerate(detections): - if all_matches.shape[0] and all_matches[all_matches[:, 1] == i].shape[0] == 0: - detection_class = detection_classes[i] - self.matrix[self.nc, detection_class] += 1 + if matches.shape[0] and matches[matches[:, 1] == i].shape[0] == 0: + self.matrix[self.nc, detection_classes[i]] += 1 def matrix(self): return self.matrix From 331c0026fc7b87229d14e314aa05d83cb48d47b9 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 17:57:57 +0100 Subject: [PATCH 57/78] cleanup --- utils/metrics.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 972ccbeb848b..f76c02b9fb4e 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -127,12 +127,10 @@ def process_batch(self, detections, labels): gt_classes = labels[:, 0].int() detection_classes = detections[:, 5].int() - all_ious = general.box_iou(labels[:, 1:], detections[:, :4]) - want_idx = torch.where(all_ious > self.iou_thres) - matches = [] - for i in range(want_idx[0].shape[0]): - matches.append([want_idx[0][i], want_idx[1][i], all_ious[want_idx[0][i], want_idx[1][i]]]) + iou = general.box_iou(labels[:, 1:], detections[:, :4]) + for g1, g2 in zip(*torch.where(iou > self.iou_thres)): + matches.append([g1, g2, iou[g1, g2]]) if matches: matches = np.array(matches) @@ -142,7 +140,7 @@ def process_batch(self, detections, labels): matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 0], return_index=True)[1]] else: - matches = torch.zeros([0, 1]) + matches = np.zeros((0, 1)) for i, label in enumerate(labels): gti = gt_classes[i] From 69c6b592a4ab6070733283fd703e3e6655510be7 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 18:00:29 +0100 Subject: [PATCH 58/78] cleanup --- utils/metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index f76c02b9fb4e..d1d8649fb13c 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -142,7 +142,7 @@ def process_batch(self, detections, labels): else: matches = np.zeros((0, 1)) - for i, label in enumerate(labels): + for i in range(len(labels)): gti = gt_classes[i] if matches.shape[0] and matches[matches[:, 0] == i].shape[0] == 1: c = detection_classes[int(matches[matches[:, 0] == i, 1][0])] @@ -150,7 +150,7 @@ def process_batch(self, detections, labels): else: self.matrix[gti, self.nc] += 1 - for i, detection in enumerate(detections): + for i in range(len(detections)): if matches.shape[0] and matches[matches[:, 1] == i].shape[0] == 0: self.matrix[self.nc, detection_classes[i]] += 1 From 8398cd509e2ca8165d3326b7e150a8e93f286af9 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 18:05:10 +0100 Subject: [PATCH 59/78] cleanup --- requirements.txt | 8 ++++---- utils/metrics.py | 51 +++++++++++++++++++++++++----------------------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/requirements.txt b/requirements.txt index 517372e3f5c1..fb4f21a3a01b 100755 --- a/requirements.txt +++ b/requirements.txt @@ -16,8 +16,9 @@ tqdm>=4.41.0 # logging ------------------------------------- # wandb -# coco ---------------------------------------- -# pycocotools>=2.0 +# plotting ------------------------------------ +# seaborn +# pandas # export -------------------------------------- # coremltools==4.0 @@ -26,5 +27,4 @@ tqdm>=4.41.0 # extras -------------------------------------- # thop # FLOPS computation -# seaborn # plotting -# pandas # plotting +# pycocotools>=2.0 # COCO mAP diff --git a/utils/metrics.py b/utils/metrics.py index d1d8649fb13c..6536e8ce96a8 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -157,30 +157,33 @@ def process_batch(self, detections, labels): def matrix(self): return self.matrix - def plot(self, save_dir='', names=[]): - import seaborn as sn - import pandas as pd - - array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize - # array = [[13, 1, 1, 0, 2, 0], - # [3, 9, 6, 0, 1, 0], - # [0, 0, 16, 2, 0, 0], - # [0, 0, 0, 13, 0, 0], - # [0, 0, 0, 0, 15, 0], - # [0, 0, 1, 0, 0, 15]] # example - - array[array < 0.005] = np.nan # don't annotate (will appear as 0.0) - df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) - - plt.figure(figsize=(12, 9)) - sn.color_palette("magma", as_cmap=True) - sn.set(font_scale=1.0) # for label size - g = sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, - xticklabels=names + ['background FN'] if names else "auto", - yticklabels=names + ['background FP'] if names else "auto") - g.set_facecolor((1, 1, 1)) - plt.tight_layout() - plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) + def plot(self, save_dir='', names=()): + try: + import seaborn as sn + import pandas as pd + + array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize + # array = [[13, 1, 1, 0, 2, 0], + # [3, 9, 6, 0, 1, 0], + # [0, 0, 16, 2, 0, 0], + # [0, 0, 0, 13, 0, 0], + # [0, 0, 0, 0, 15, 0], + # [0, 0, 1, 0, 0, 15]] # example + + array[array < 0.005] = np.nan # don't annotate (will appear as 0.0) + df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) + + plt.figure(figsize=(12, 9)) + sn.color_palette("magma", as_cmap=True) + sn.set(font_scale=1.0) # for label size + g = sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + xticklabels=names + ['background FN'] if names else "auto", + yticklabels=names + ['background FP'] if names else "auto") + g.set_facecolor((1, 1, 1)) + plt.tight_layout() + plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) + except Exception as e: + pass def print(self): for i in range(self.nc + 1): From df01a8e238e79b9ae16cc87f5524bea9d6c86364 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 18:08:40 +0100 Subject: [PATCH 60/78] cleanup --- utils/metrics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/metrics.py b/utils/metrics.py index 6536e8ce96a8..79406bb84930 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -174,7 +174,6 @@ def plot(self, save_dir='', names=()): df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) plt.figure(figsize=(12, 9)) - sn.color_palette("magma", as_cmap=True) sn.set(font_scale=1.0) # for label size g = sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", From b1f7e70b811e0fc6209403b1d9c07b2ee54f7c86 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 18:25:06 +0100 Subject: [PATCH 61/78] cleanup --- utils/metrics.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 79406bb84930..2e5de1c2b926 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -126,33 +126,33 @@ def process_batch(self, detections, labels): detections = detections[detections[:, 4] > self.conf] gt_classes = labels[:, 0].int() detection_classes = detections[:, 5].int() - - matches = [] iou = general.box_iou(labels[:, 1:], detections[:, :4]) - for g1, g2 in zip(*torch.where(iou > self.iou_thres)): - matches.append([g1, g2, iou[g1, g2]]) - - if matches: - matches = np.array(matches) - if matches.shape[0]: # if there is match - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique(matches[:, 1], return_index=True)[1]] - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + + # matches = [] + # for g1, g2 in zip(*torch.where(iou > self.iou_thres)): + # matches.append([g1, g2, iou[g1, g2]]) + + x = torch.where(iou > self.iou_thres) + if x[0].shape[0]: + matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).numpy() + + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] else: matches = np.zeros((0, 1)) - for i in range(len(labels)): - gti = gt_classes[i] + for i, gc in enumerate(gt_classes): if matches.shape[0] and matches[matches[:, 0] == i].shape[0] == 1: - c = detection_classes[int(matches[matches[:, 0] == i, 1][0])] - self.matrix[gti, c] += 1 + dc = detection_classes[int(matches[matches[:, 0] == i, 1][0])] + self.matrix[gc, dc] += 1 else: - self.matrix[gti, self.nc] += 1 + self.matrix[gc, self.nc] += 1 - for i in range(len(detections)): + for i, dc in enumerate(detection_classes): if matches.shape[0] and matches[matches[:, 1] == i].shape[0] == 0: - self.matrix[self.nc, detection_classes[i]] += 1 + self.matrix[self.nc, dc] += 1 def matrix(self): return self.matrix From 7fa0ac01c5513098d199f7c3c089209ef333793e Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 18:26:26 +0100 Subject: [PATCH 62/78] cleanup --- utils/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/metrics.py b/utils/metrics.py index 2e5de1c2b926..fea4ef7dc655 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -134,7 +134,7 @@ def process_batch(self, detections, labels): x = torch.where(iou > self.iou_thres) if x[0].shape[0]: - matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).numpy() + matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 1], return_index=True)[1]] From 9ae045190da309945936092594f385bd043c98fa Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 18:52:46 +0100 Subject: [PATCH 63/78] cleanup --- utils/metrics.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index fea4ef7dc655..1e0e2a9e6798 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -128,10 +128,6 @@ def process_batch(self, detections, labels): detection_classes = detections[:, 5].int() iou = general.box_iou(labels[:, 1:], detections[:, :4]) - # matches = [] - # for g1, g2 in zip(*torch.where(iou > self.iou_thres)): - # matches.append([g1, g2, iou[g1, g2]]) - x = torch.where(iou > self.iou_thres) if x[0].shape[0]: matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() @@ -148,7 +144,8 @@ def process_batch(self, detections, labels): dc = detection_classes[int(matches[matches[:, 0] == i, 1][0])] self.matrix[gc, dc] += 1 else: - self.matrix[gc, self.nc] += 1 + self.matrix[gc, self.nc] += 1 # background + for i, dc in enumerate(detection_classes): if matches.shape[0] and matches[matches[:, 1] == i].shape[0] == 0: From 40b25bc669f21163591eaf1445320a72514d3f96 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 19:09:53 +0100 Subject: [PATCH 64/78] cleanup --- utils/metrics.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 1e0e2a9e6798..d364a9f2bb5c 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -137,18 +137,20 @@ def process_batch(self, detections, labels): matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 0], return_index=True)[1]] else: - matches = np.zeros((0, 1)) + matches = np.zeros((0, 3)) + n = matches.shape[0] > 0 + m0, m1, _ = matches.transpose().astype(np.uint16) for i, gc in enumerate(gt_classes): - if matches.shape[0] and matches[matches[:, 0] == i].shape[0] == 1: - dc = detection_classes[int(matches[matches[:, 0] == i, 1][0])] + j = m0 == i + if n and sum(j) == 1: + dc = detection_classes[m1[j]] self.matrix[gc, dc] += 1 else: self.matrix[gc, self.nc] += 1 # background - for i, dc in enumerate(detection_classes): - if matches.shape[0] and matches[matches[:, 1] == i].shape[0] == 0: + if n and not any(m1 == i): self.matrix[self.nc, dc] += 1 def matrix(self): From c146030a7257171e3515d674fd88e4afd959a20e Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 19:11:42 +0100 Subject: [PATCH 65/78] cleanup --- utils/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/metrics.py b/utils/metrics.py index d364a9f2bb5c..7bd897179cf1 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -140,7 +140,7 @@ def process_batch(self, detections, labels): matches = np.zeros((0, 3)) n = matches.shape[0] > 0 - m0, m1, _ = matches.transpose().astype(np.uint16) + m0, m1, _ = matches.transpose().astype(np.int16) for i, gc in enumerate(gt_classes): j = m0 == i if n and sum(j) == 1: From d73dcd38ddb1c0c8d83c50a101108e85f0703136 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 19:18:55 +0100 Subject: [PATCH 66/78] cleanup --- utils/metrics.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 7bd897179cf1..59bbfbe4c17b 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -131,11 +131,11 @@ def process_batch(self, detections, labels): x = torch.where(iou > self.iou_thres) if x[0].shape[0]: matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() - - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique(matches[:, 1], return_index=True)[1]] - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + if x[0].shape[0] > 1: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] else: matches = np.zeros((0, 3)) @@ -147,11 +147,12 @@ def process_batch(self, detections, labels): dc = detection_classes[m1[j]] self.matrix[gc, dc] += 1 else: - self.matrix[gc, self.nc] += 1 # background + self.matrix[gc, self.nc] += 1 # background FP - for i, dc in enumerate(detection_classes): - if n and not any(m1 == i): - self.matrix[self.nc, dc] += 1 + if n: + for i, dc in enumerate(detection_classes): + if not any(m1 == i): + self.matrix[self.nc, dc] += 1 # background FN def matrix(self): return self.matrix From 9e2c3f9b59de48e289b96fed000c758424a62f44 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 19:38:12 +0100 Subject: [PATCH 67/78] cleanup --- utils/metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 59bbfbe4c17b..a5eddb9e94c6 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -144,8 +144,7 @@ def process_batch(self, detections, labels): for i, gc in enumerate(gt_classes): j = m0 == i if n and sum(j) == 1: - dc = detection_classes[m1[j]] - self.matrix[gc, dc] += 1 + self.matrix[gc, detection_classes[m1[j]]] += 1 # correct else: self.matrix[gc, self.nc] += 1 # background FP From cdc3ec7692e6c50b0b053ca266e1bd54fdb268bc Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 19:47:30 +0100 Subject: [PATCH 68/78] seaborn pandas to requirements.txt --- requirements.txt | 4 ++-- train.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index fb4f21a3a01b..5e813fb0420c 100755 --- a/requirements.txt +++ b/requirements.txt @@ -17,8 +17,8 @@ tqdm>=4.41.0 # wandb # plotting ------------------------------------ -# seaborn -# pandas +seaborn +pandas # export -------------------------------------- # coremltools==4.0 diff --git a/train.py b/train.py index a2244fcbf395..c4aaa4c34eea 100644 --- a/train.py +++ b/train.py @@ -396,8 +396,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if plots: plot_results(save_dir=save_dir) # save as results.png if wandb: - wandb.log({"Results": [wandb.Image(str(save_dir / x), caption=x) for x in - ['results.png', 'precision_recall_curve.png']]}) + wandb.log({"Results": [wandb.Image(str(save_dir / x), caption=str(x)) for x in save_dir.glob('*.png')]}) logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600)) else: dist.destroy_process_group() From 8c4901ba4382505a04d9b0ef743a5d880a57d394 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:06:35 +0100 Subject: [PATCH 69/78] seaborn pandas to requirements.txt --- train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/train.py b/train.py index c4aaa4c34eea..76e64e597edf 100644 --- a/train.py +++ b/train.py @@ -396,7 +396,8 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): if plots: plot_results(save_dir=save_dir) # save as results.png if wandb: - wandb.log({"Results": [wandb.Image(str(save_dir / x), caption=str(x)) for x in save_dir.glob('*.png')]}) + files = ['results.png', 'precision_recall_curve.png', 'confusion_matrix.png'] + wandb.log({"Results": [wandb.Image(str(save_dir / x), caption=x) for x in files if os.path.exists(x)]}) logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600)) else: dist.destroy_process_group() From dbff8b5be05154fd65f3878024d0e5b54ccc79f9 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:19:04 +0100 Subject: [PATCH 70/78] update wandb plotting --- test.py | 2 +- train.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test.py b/test.py index c3a42f8d7671..1393a5ad8b15 100644 --- a/test.py +++ b/test.py @@ -227,7 +227,7 @@ def test(data, confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) if wandb and wandb.run: wandb.log({"Images": wandb_images}) - wandb.log({"Validation": [wandb.Image(str(x), caption=x.name) for x in sorted(save_dir.glob('test*.jpg'))]}) + wandb.log({"Validation": [wandb.Image(str(f), caption=f.name) for f in sorted(save_dir.glob('test*.jpg'))]}) # Print results pf = '%20s' + '%12.3g' * 6 # print format diff --git a/train.py b/train.py index 76e64e597edf..60da58b91137 100644 --- a/train.py +++ b/train.py @@ -397,7 +397,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): plot_results(save_dir=save_dir) # save as results.png if wandb: files = ['results.png', 'precision_recall_curve.png', 'confusion_matrix.png'] - wandb.log({"Results": [wandb.Image(str(save_dir / x), caption=x) for x in files if os.path.exists(x)]}) + wandb.log({"Results": [wandb.Image(str(save_dir / f), caption=f) for f in files]}) logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600)) else: dist.destroy_process_group() From 8647b21ed89bfcbb9a0c76934b6b9a8fcb4478df Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:23:18 +0100 Subject: [PATCH 71/78] remove pandas --- utils/metrics.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index a5eddb9e94c6..8d3a3dc38e2d 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -159,7 +159,6 @@ def matrix(self): def plot(self, save_dir='', names=()): try: import seaborn as sn - import pandas as pd array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize # array = [[13, 1, 1, 0, 2, 0], @@ -168,13 +167,11 @@ def plot(self, save_dir='', names=()): # [0, 0, 0, 13, 0, 0], # [0, 0, 0, 0, 15, 0], # [0, 0, 1, 0, 0, 15]] # example - array[array < 0.005] = np.nan # don't annotate (will appear as 0.0) - df_cm = pd.DataFrame(array, range(self.nc + 1), range(self.nc + 1)) plt.figure(figsize=(12, 9)) sn.set(font_scale=1.0) # for label size - g = sn.heatmap(df_cm, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + g = sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", yticklabels=names + ['background FP'] if names else "auto") g.set_facecolor((1, 1, 1)) From 658b2ef7d60f37b0757f03475cf9ac0d481bc350 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:27:59 +0100 Subject: [PATCH 72/78] if plots --- test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test.py b/test.py index 1393a5ad8b15..0c06f37e5e4b 100644 --- a/test.py +++ b/test.py @@ -177,9 +177,8 @@ def test(data, # target boxes tbox = xywh2xyxy(labels[:, 1:5]) scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1]) # native-space labels - - # Confusion Matrix - confusion_matrix.process_batch(pred, torch.cat((labels[:, 0:1], tbox), 1)) + if plots: + confusion_matrix.process_batch(pred, torch.cat((labels[:, 0:1], tbox), 1)) # Per target class for cls in torch.unique(tcls_tensor): From fb8dfaa23bd29a3a4e628a0c8c27907c3908be1d Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:33:46 +0100 Subject: [PATCH 73/78] if plots --- utils/metrics.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 8d3a3dc38e2d..f0db5870b130 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -161,22 +161,15 @@ def plot(self, save_dir='', names=()): import seaborn as sn array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize - # array = [[13, 1, 1, 0, 2, 0], - # [3, 9, 6, 0, 1, 0], - # [0, 0, 16, 2, 0, 0], - # [0, 0, 0, 13, 0, 0], - # [0, 0, 0, 0, 15, 0], - # [0, 0, 1, 0, 0, 15]] # example - array[array < 0.005] = np.nan # don't annotate (will appear as 0.0) - - plt.figure(figsize=(12, 9)) + array[array < 0.005] = np.nan # don't annotate (would appear as 0.00) + + fig = plt.figure(figsize=(12, 9)) sn.set(font_scale=1.0) # for label size - g = sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, - xticklabels=names + ['background FN'] if names else "auto", - yticklabels=names + ['background FP'] if names else "auto") - g.set_facecolor((1, 1, 1)) - plt.tight_layout() - plt.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) + sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + xticklabels=names + ['background FN'] if names else "auto", + yticklabels=names + ['background FP'] if names else "auto").set_facecolor((1, 1, 1)) + fig.tight_layout() + fig.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) except Exception as e: pass From 3c3c780146ddad0e7b01b7eb6d16f0807bd9eb1f Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:37:11 +0100 Subject: [PATCH 74/78] if plots --- utils/metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index f0db5870b130..cabb7c64c7b0 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -163,9 +163,9 @@ def plot(self, save_dir='', names=()): array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize array[array < 0.005] = np.nan # don't annotate (would appear as 0.00) - fig = plt.figure(figsize=(12, 9)) - sn.set(font_scale=1.0) # for label size - sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + fig, ax = plt.figure(figsize=(12, 9)) + # sn.set(font_scale=1.0) # for label size + sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, ax=ax, xticklabels=names + ['background FN'] if names else "auto", yticklabels=names + ['background FP'] if names else "auto").set_facecolor((1, 1, 1)) fig.tight_layout() From 9ea0612d22ce5870a424f62a23149dc3843d9ccd Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:45:21 +0100 Subject: [PATCH 75/78] if plots --- test.py | 2 +- train.py | 3 ++- utils/metrics.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test.py b/test.py index 0c06f37e5e4b..2e8bafc55801 100644 --- a/test.py +++ b/test.py @@ -283,7 +283,7 @@ def test(data, parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model.pt path(s)') parser.add_argument('--data', type=str, default='data/coco128.yaml', help='*.data path') parser.add_argument('--batch-size', type=int, default=32, help='size of each image batch') - parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--img-size', type=int, default=256, help='inference size (pixels)') parser.add_argument('--conf-thres', type=float, default=0.001, help='object confidence threshold') parser.add_argument('--iou-thres', type=float, default=0.6, help='IOU threshold for NMS') parser.add_argument('--task', default='val', help="'val', 'test', 'study'") diff --git a/train.py b/train.py index 60da58b91137..ca8148a82176 100644 --- a/train.py +++ b/train.py @@ -397,7 +397,8 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): plot_results(save_dir=save_dir) # save as results.png if wandb: files = ['results.png', 'precision_recall_curve.png', 'confusion_matrix.png'] - wandb.log({"Results": [wandb.Image(str(save_dir / f), caption=f) for f in files]}) + wandb.log({"Results": [wandb.Image(str(save_dir / f), caption=f) for f in files + if (save_dir / f).exists()]}) logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600)) else: dist.destroy_process_group() diff --git a/utils/metrics.py b/utils/metrics.py index cabb7c64c7b0..c5dac2b6103b 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -163,9 +163,9 @@ def plot(self, save_dir='', names=()): array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize array[array < 0.005] = np.nan # don't annotate (would appear as 0.00) - fig, ax = plt.figure(figsize=(12, 9)) + fig = plt.figure(figsize=(12, 9)) # sn.set(font_scale=1.0) # for label size - sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, ax=ax, + sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, xticklabels=names + ['background FN'] if names else "auto", yticklabels=names + ['background FP'] if names else "auto").set_facecolor((1, 1, 1)) fig.tight_layout() From 7ec84d4c050da99deff614e004d24ea5125a27c2 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Nov 2020 20:45:56 +0100 Subject: [PATCH 76/78] if plots --- test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.py b/test.py index 2e8bafc55801..0c06f37e5e4b 100644 --- a/test.py +++ b/test.py @@ -283,7 +283,7 @@ def test(data, parser.add_argument('--weights', nargs='+', type=str, default='yolov5s.pt', help='model.pt path(s)') parser.add_argument('--data', type=str, default='data/coco128.yaml', help='*.data path') parser.add_argument('--batch-size', type=int, default=32, help='size of each image batch') - parser.add_argument('--img-size', type=int, default=256, help='inference size (pixels)') + parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') parser.add_argument('--conf-thres', type=float, default=0.001, help='object confidence threshold') parser.add_argument('--iou-thres', type=float, default=0.6, help='IOU threshold for NMS') parser.add_argument('--task', default='val', help="'val', 'test', 'study'") From 3dab081ad9877af124f6469d4df32f2f172bda49 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 23 Nov 2020 14:06:04 +0100 Subject: [PATCH 77/78] Cat apriori to autolabels --- utils/metrics.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index c5dac2b6103b..9e003b327aa3 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -106,7 +106,7 @@ def compute_ap(recall, precision): class ConfusionMatrix: - # From https://github.com/kaanakan/object_detection_confusion_matrix + # Updated version of https://github.com/kaanakan/object_detection_confusion_matrix def __init__(self, nc, conf=0.25, iou_thres=0.45): self.matrix = np.zeros((nc + 1, nc + 1)) self.nc = nc # number of classes @@ -164,10 +164,13 @@ def plot(self, save_dir='', names=()): array[array < 0.005] = np.nan # don't annotate (would appear as 0.00) fig = plt.figure(figsize=(12, 9)) - # sn.set(font_scale=1.0) # for label size + sn.set(font_scale=1.0 if self.nc < 50 else 0.8) # for label size + ticklabels = (0 < len(names) < 99) and len(names) == self.nc # apply names to ticklabels sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, - xticklabels=names + ['background FN'] if names else "auto", - yticklabels=names + ['background FP'] if names else "auto").set_facecolor((1, 1, 1)) + xticklabels=names + ['background FN'] if ticklabels else "auto", + yticklabels=names + ['background FP'] if ticklabels else "auto").set_facecolor((1, 1, 1)) + fig.axes[0].set_xlabel('True') + fig.axes[0].set_ylabel('Predicted') fig.tight_layout() fig.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) except Exception as e: From 8bbe0b55217e97f279e2d57bca7f6e8f10f95218 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 23 Nov 2020 14:35:05 +0100 Subject: [PATCH 78/78] cleanup --- utils/metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/metrics.py b/utils/metrics.py index 9e003b327aa3..26872f2704c5 100644 --- a/utils/metrics.py +++ b/utils/metrics.py @@ -165,10 +165,10 @@ def plot(self, save_dir='', names=()): fig = plt.figure(figsize=(12, 9)) sn.set(font_scale=1.0 if self.nc < 50 else 0.8) # for label size - ticklabels = (0 < len(names) < 99) and len(names) == self.nc # apply names to ticklabels + labels = (0 < len(names) < 99) and len(names) == self.nc # apply names to ticklabels sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, - xticklabels=names + ['background FN'] if ticklabels else "auto", - yticklabels=names + ['background FP'] if ticklabels else "auto").set_facecolor((1, 1, 1)) + xticklabels=names + ['background FN'] if labels else "auto", + yticklabels=names + ['background FP'] if labels else "auto").set_facecolor((1, 1, 1)) fig.axes[0].set_xlabel('True') fig.axes[0].set_ylabel('Predicted') fig.tight_layout()