From eae25aa4f8cd7703f266f63b006f31ed783ac666 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 16 May 2022 01:28:59 +0200 Subject: [PATCH 01/15] New TensorFlow `TFCrossConv()` module --- models/tf.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/models/tf.py b/models/tf.py index 3a26256ea2b1..d3d52b303aa3 100644 --- a/models/tf.py +++ b/models/tf.py @@ -65,10 +65,8 @@ def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None): # ch_in, ch_out, weights, kernel, stride, padding, groups super().__init__() assert g == 1, "TF v2.2 Conv2D does not support 'groups' argument" - assert isinstance(k, int), "Convolution with multiple kernels are not allowed." # TensorFlow convolution padding is inconsistent with PyTorch (e.g. k=3 s=2 'SAME' padding) # see https://stackoverflow.com/questions/52975843/comparing-conv2d-with-padding-between-tensorflow-and-pytorch - conv = keras.layers.Conv2D( filters=c2, kernel_size=k, @@ -90,8 +88,6 @@ class TFDWConv(keras.layers.Layer): def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None): # ch_in, ch_out, weights, kernel, stride, padding, groups super().__init__() - assert isinstance(k, int), "Convolution with multiple kernels are not allowed." - conv = keras.layers.DepthwiseConv2D( kernel_size=k, strides=s, @@ -133,6 +129,22 @@ def call(self, inputs): return inputs + self.cv2(self.cv1(inputs)) if self.add else self.cv2(self.cv1(inputs)) +class TFCrossConv(keras.layers.Layer): + # Cross Convolution + def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False, w=None): + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = TFConv(c1, c_, (1, k), (1, s), w=w.cv1) + self.cv2 = TFConv(c_, c2, (k, 1), (s, 1), g=g, w=w.cv2) + self.add = shortcut and c1 == c2 + + self.cv1 = Conv(c1, c_, (1, k), (1, s)) + self.cv2 = Conv(c_, c2, (k, 1), (s, 1), g=g) + + def call(self, inputs): + return inputs + self.cv2(self.cv1(inputs)) if self.add else self.cv2(self.cv1(inputs)) + + class TFConv2d(keras.layers.Layer): # Substitution for PyTorch nn.Conv2D def __init__(self, c1, c2, k, s=1, g=1, bias=True, w=None): From 1a4d69316b35d85772d64cd8f710c78cc721ed60 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 16 May 2022 01:37:36 +0200 Subject: [PATCH 02/15] Move from experimental to common --- models/common.py | 14 ++++++++++++++ models/experimental.py | 13 ------------- models/tf.py | 4 ++-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/models/common.py b/models/common.py index 9d181ed7a40a..3c114f6673db 100644 --- a/models/common.py +++ b/models/common.py @@ -124,6 +124,20 @@ def forward(self, x): return self.cv4(self.act(self.bn(torch.cat((y1, y2), 1)))) +class CrossConv(nn.Module): + # Cross Convolution Downsample + def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False): + # ch_in, ch_out, kernel, stride, groups, expansion, shortcut + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, (1, k), (1, s)) + self.cv2 = Conv(c_, c2, (k, 1), (s, 1), g=g) + self.add = shortcut and c1 == c2 + + def forward(self, x): + return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x)) + + class C3(nn.Module): # CSP Bottleneck with 3 convolutions def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion diff --git a/models/experimental.py b/models/experimental.py index 1ffe0fcb5971..16a3f1aa284f 100644 --- a/models/experimental.py +++ b/models/experimental.py @@ -12,19 +12,6 @@ from utils.downloads import attempt_download -class CrossConv(nn.Module): - # Cross Convolution Downsample - def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False): - # ch_in, ch_out, kernel, stride, groups, expansion, shortcut - super().__init__() - c_ = int(c2 * e) # hidden channels - self.cv1 = Conv(c1, c_, (1, k), (1, s)) - self.cv2 = Conv(c_, c2, (k, 1), (s, 1), g=g) - self.add = shortcut and c1 == c2 - - def forward(self, x): - return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x)) - class Sum(nn.Module): # Weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070 diff --git a/models/tf.py b/models/tf.py index d3d52b303aa3..0a655e5c77a5 100644 --- a/models/tf.py +++ b/models/tf.py @@ -27,8 +27,8 @@ import torch.nn as nn from tensorflow import keras -from models.common import C3, SPP, SPPF, Bottleneck, BottleneckCSP, Concat, Conv, DWConv, Focus, autopad -from models.experimental import CrossConv, MixConv2d, attempt_load +from models.common import C3, CrossConv, SPP, SPPF, Bottleneck, BottleneckCSP, Concat, Conv, DWConv, Focus, autopad +from models.experimental import MixConv2d, attempt_load from models.yolo import Detect from utils.activations import SiLU from utils.general import LOGGER, make_divisible, print_args From be84b21a2a711cf5625fd583bb3eea0271545bcc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 15 May 2022 23:37:59 +0000 Subject: [PATCH 03/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- models/experimental.py | 1 - models/tf.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/models/experimental.py b/models/experimental.py index 16a3f1aa284f..7bf249e80984 100644 --- a/models/experimental.py +++ b/models/experimental.py @@ -12,7 +12,6 @@ from utils.downloads import attempt_download - class Sum(nn.Module): # Weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070 def __init__(self, n, weight=False): # n: number of inputs diff --git a/models/tf.py b/models/tf.py index 0a655e5c77a5..05f830568f75 100644 --- a/models/tf.py +++ b/models/tf.py @@ -27,7 +27,7 @@ import torch.nn as nn from tensorflow import keras -from models.common import C3, CrossConv, SPP, SPPF, Bottleneck, BottleneckCSP, Concat, Conv, DWConv, Focus, autopad +from models.common import C3, SPP, SPPF, Bottleneck, BottleneckCSP, Concat, Conv, CrossConv, DWConv, Focus, autopad from models.experimental import MixConv2d, attempt_load from models.yolo import Detect from utils.activations import SiLU From f8fff226d2d5aa3245891a535a45aa6fff7673f2 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 16 May 2022 13:51:45 +0200 Subject: [PATCH 04/15] Add C3x --- models/common.py | 11 +++++++++-- models/tf.py | 22 ++++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/models/common.py b/models/common.py index 3c114f6673db..00641eaa8d15 100644 --- a/models/common.py +++ b/models/common.py @@ -31,7 +31,7 @@ def autopad(k, p=None): # kernel, padding # Pad to 'same' if p is None: - p = k // 2 if isinstance(k, int) else (x // 2 for x in k) # auto-pad + p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad return p @@ -147,12 +147,19 @@ def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, nu self.cv2 = Conv(c1, c_, 1, 1) self.cv3 = Conv(2 * c_, c2, 1) # optional act=FReLU(c2) self.m = nn.Sequential(*(Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n))) - # self.m = nn.Sequential(*(CrossConv(c_, c_, 3, 1, g, 1.0, shortcut) for _ in range(n))) def forward(self, x): return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), 1)) +class C3x(C3): + # C3 module with cross-convolutions + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) + self.m = nn.Sequential(*(CrossConv(c_, c_, 3, 1, g, 1.0, shortcut) for _ in range(n))) + + class C3TR(C3): # C3 module with TransformerBlock() def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): diff --git a/models/tf.py b/models/tf.py index 05f830568f75..a464e29df396 100644 --- a/models/tf.py +++ b/models/tf.py @@ -50,10 +50,12 @@ def call(self, inputs): class TFPad(keras.layers.Layer): - def __init__(self, pad): super().__init__() - self.pad = tf.constant([[0, 0], [pad, pad], [pad, pad], [0, 0]]) + if isinstance(pad, int): + self.pad = tf.constant([[0, 0], [pad, pad], [pad, pad], [0, 0]]) + else: # tuple/list + self.pad = tf.constant([[0, 0], [pad[0], pad[0]], [pad[1], pad[1]], [0, 0]]) def call(self, inputs): return tf.pad(inputs, self.pad, mode='constant', constant_values=0) @@ -199,6 +201,22 @@ def call(self, inputs): return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3)) +class TFC3x(keras.layers.Layer): + # 3 module with cross-convolutions + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None): + # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1) + self.cv2 = TFConv(c1, c_, 1, 1, w=w.cv2) + self.cv3 = TFConv(2 * c_, c2, 1, 1, w=w.cv3) + self.m = keras.Sequential( + [TFCrossConv(c_, c_, k=3, s=1, g=1, e=1.0, shortcut=False, w=w.m[j]) for j in range(n)]) + + def call(self, inputs): + return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3)) + + class TFSPP(keras.layers.Layer): # Spatial pyramid pooling layer used in YOLOv3-SPP def __init__(self, c1, c2, k=(5, 9, 13), w=None): From aa2b45ce81e769e7323a3c44b07484e62b076e5d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 11:52:13 +0000 Subject: [PATCH 05/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- models/tf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/models/tf.py b/models/tf.py index a464e29df396..8d9ae6bb5b53 100644 --- a/models/tf.py +++ b/models/tf.py @@ -50,6 +50,7 @@ def call(self, inputs): class TFPad(keras.layers.Layer): + def __init__(self, pad): super().__init__() if isinstance(pad, int): @@ -210,8 +211,8 @@ def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None): self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1) self.cv2 = TFConv(c1, c_, 1, 1, w=w.cv2) self.cv3 = TFConv(2 * c_, c2, 1, 1, w=w.cv3) - self.m = keras.Sequential( - [TFCrossConv(c_, c_, k=3, s=1, g=1, e=1.0, shortcut=False, w=w.m[j]) for j in range(n)]) + self.m = keras.Sequential([ + TFCrossConv(c_, c_, k=3, s=1, g=1, e=1.0, shortcut=False, w=w.m[j]) for j in range(n)]) def call(self, inputs): return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3)) From aa6a25c3aec6da384e50f628e50f76365256bfb3 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 16 May 2022 13:56:12 +0200 Subject: [PATCH 06/15] Add to C3x to yolo.py --- models/yolo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/yolo.py b/models/yolo.py index 55356a6a9b44..9695ed7ff186 100644 --- a/models/yolo.py +++ b/models/yolo.py @@ -266,13 +266,13 @@ def parse_model(d, ch): # model_dict, input_channels(3) n = n_ = max(round(n * gd), 1) if n > 1 else n # depth gain if m in (Conv, GhostConv, Bottleneck, GhostBottleneck, SPP, SPPF, DWConv, MixConv2d, Focus, CrossConv, - BottleneckCSP, C3, C3TR, C3SPP, C3Ghost): + BottleneckCSP, C3, C3TR, C3SPP, C3Ghost, C3x): c1, c2 = ch[f], args[0] if c2 != no: # if not output c2 = make_divisible(c2 * gw, 8) args = [c1, c2, *args[1:]] - if m in [BottleneckCSP, C3, C3TR, C3Ghost]: + if m in [BottleneckCSP, C3, C3TR, C3Ghost, C3x]: args.insert(2, n) # number of repeats n = 1 elif m is nn.BatchNorm2d: From fe94d7204b357f3bc60ea5596133f8019b2a8639 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 16 May 2022 13:57:40 +0200 Subject: [PATCH 07/15] Add to C3x to tf.py --- models/tf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/tf.py b/models/tf.py index 8d9ae6bb5b53..b03b347e0a09 100644 --- a/models/tf.py +++ b/models/tf.py @@ -27,7 +27,7 @@ import torch.nn as nn from tensorflow import keras -from models.common import C3, SPP, SPPF, Bottleneck, BottleneckCSP, Concat, Conv, CrossConv, DWConv, Focus, autopad +from models.common import C3, C3x, SPP, SPPF, Bottleneck, BottleneckCSP, Concat, Conv, CrossConv, DWConv, Focus, autopad from models.experimental import MixConv2d, attempt_load from models.yolo import Detect from utils.activations import SiLU @@ -341,7 +341,7 @@ def parse_model(d, ch, model, imgsz): # model_dict, input_channels(3) pass n = max(round(n * gd), 1) if n > 1 else n # depth gain - if m in [nn.Conv2d, Conv, Bottleneck, SPP, SPPF, DWConv, MixConv2d, Focus, CrossConv, BottleneckCSP, C3]: + if m in [nn.Conv2d, Conv, Bottleneck, SPP, SPPF, DWConv, MixConv2d, Focus, CrossConv, BottleneckCSP, C3, C3x]: c1, c2 = ch[f], args[0] c2 = make_divisible(c2 * gw, 8) if c2 != no else c2 From 896717b493d52763ec84165fd32fd75fed5982e9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 11:58:03 +0000 Subject: [PATCH 08/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- models/tf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/tf.py b/models/tf.py index b03b347e0a09..a7b822bae2f9 100644 --- a/models/tf.py +++ b/models/tf.py @@ -27,7 +27,7 @@ import torch.nn as nn from tensorflow import keras -from models.common import C3, C3x, SPP, SPPF, Bottleneck, BottleneckCSP, Concat, Conv, CrossConv, DWConv, Focus, autopad +from models.common import C3, SPP, SPPF, Bottleneck, BottleneckCSP, C3x, Concat, Conv, CrossConv, DWConv, Focus, autopad from models.experimental import MixConv2d, attempt_load from models.yolo import Detect from utils.activations import SiLU From 20451acceb4a4c115b6bb773d2fad22e282ae5f4 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 16 May 2022 14:00:31 +0200 Subject: [PATCH 09/15] TFC3x bug fix --- models/tf.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/models/tf.py b/models/tf.py index b03b347e0a09..488a9d3a0ac9 100644 --- a/models/tf.py +++ b/models/tf.py @@ -141,9 +141,6 @@ def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False, w=None): self.cv2 = TFConv(c_, c2, (k, 1), (s, 1), g=g, w=w.cv2) self.add = shortcut and c1 == c2 - self.cv1 = Conv(c1, c_, (1, k), (1, s)) - self.cv2 = Conv(c_, c2, (k, 1), (s, 1), g=g) - def call(self, inputs): return inputs + self.cv2(self.cv1(inputs)) if self.add else self.cv2(self.cv1(inputs)) From f861aad4921328919214ad8df05c86d19b203c86 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 16 May 2022 14:43:19 +0200 Subject: [PATCH 10/15] TFC3x bug fix --- models/tf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/tf.py b/models/tf.py index 1fb74fc94c12..43735d6398a5 100644 --- a/models/tf.py +++ b/models/tf.py @@ -343,7 +343,7 @@ def parse_model(d, ch, model, imgsz): # model_dict, input_channels(3) c2 = make_divisible(c2 * gw, 8) if c2 != no else c2 args = [c1, c2, *args[1:]] - if m in [BottleneckCSP, C3]: + if m in [BottleneckCSP, C3, C3x]: args.insert(2, n) n = 1 elif m is nn.BatchNorm2d: From d2b6405eb8ed2e2988f59222f1af1dbf069fdbe0 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 16 May 2022 14:49:39 +0200 Subject: [PATCH 11/15] TFC3x bug fix --- models/tf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/tf.py b/models/tf.py index 43735d6398a5..b24003a4b8ea 100644 --- a/models/tf.py +++ b/models/tf.py @@ -209,7 +209,7 @@ def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None): self.cv2 = TFConv(c1, c_, 1, 1, w=w.cv2) self.cv3 = TFConv(2 * c_, c2, 1, 1, w=w.cv3) self.m = keras.Sequential([ - TFCrossConv(c_, c_, k=3, s=1, g=1, e=1.0, shortcut=False, w=w.m[j]) for j in range(n)]) + TFCrossConv(c_, c_, k=3, s=1, g=g, e=1.0, shortcut=shortcut, w=w.m[j]) for j in range(n)]) def call(self, inputs): return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3)) From 1b62859fbc7f14602479056f707146aa3a62856c Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 16 May 2022 16:49:44 +0200 Subject: [PATCH 12/15] Add TFDWConv g==c1==c2 check --- models/tf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/models/tf.py b/models/tf.py index b24003a4b8ea..d84b13f98a28 100644 --- a/models/tf.py +++ b/models/tf.py @@ -91,6 +91,7 @@ class TFDWConv(keras.layers.Layer): def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None): # ch_in, ch_out, weights, kernel, stride, padding, groups super().__init__() + assert g == c1 == c2, f'TFDWConv() groups={g} must equal input={c1} and output={c2} channels' conv = keras.layers.DepthwiseConv2D( kernel_size=k, strides=s, From 677b9cc76991d99abe5e3a6ddab3122d51368ac0 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 16 May 2022 16:50:55 +0200 Subject: [PATCH 13/15] Add comment --- models/tf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/tf.py b/models/tf.py index d84b13f98a28..b70e37488009 100644 --- a/models/tf.py +++ b/models/tf.py @@ -50,7 +50,7 @@ def call(self, inputs): class TFPad(keras.layers.Layer): - + # Pad inputs in spatial dimensions 1 and 2 def __init__(self, pad): super().__init__() if isinstance(pad, int): From 591b11e3005ddbef520bab01db5869c07d80ac28 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 16 May 2022 17:01:04 +0200 Subject: [PATCH 14/15] Update tf.py --- models/tf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/tf.py b/models/tf.py index b70e37488009..8151979fbb41 100644 --- a/models/tf.py +++ b/models/tf.py @@ -27,7 +27,7 @@ import torch.nn as nn from tensorflow import keras -from models.common import C3, SPP, SPPF, Bottleneck, BottleneckCSP, C3x, Concat, Conv, CrossConv, DWConv, Focus, autopad +from models.common import Bottleneck, BottleneckCSP, C3, C3x, Concat, Conv, CrossConv, DWConv, Focus, SPP, SPPF, autopad from models.experimental import MixConv2d, attempt_load from models.yolo import Detect from utils.activations import SiLU From c6395dd61f7916fda9e8cddaa56ecbf9c7d533b2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 15:01:21 +0000 Subject: [PATCH 15/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- models/tf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/tf.py b/models/tf.py index 8151979fbb41..b70e37488009 100644 --- a/models/tf.py +++ b/models/tf.py @@ -27,7 +27,7 @@ import torch.nn as nn from tensorflow import keras -from models.common import Bottleneck, BottleneckCSP, C3, C3x, Concat, Conv, CrossConv, DWConv, Focus, SPP, SPPF, autopad +from models.common import C3, SPP, SPPF, Bottleneck, BottleneckCSP, C3x, Concat, Conv, CrossConv, DWConv, Focus, autopad from models.experimental import MixConv2d, attempt_load from models.yolo import Detect from utils.activations import SiLU