Skip to content

Commit

Permalink
Update activations.py
Browse files Browse the repository at this point in the history
Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com>
  • Loading branch information
glenn-jocher committed Feb 25, 2024
1 parent 2a49d14 commit 51140c6
Showing 1 changed file with 5 additions and 8 deletions.
13 changes: 5 additions & 8 deletions utils/activations.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@


class SiLU(nn.Module):
# SiLU activation https://arxiv.org/pdf/1606.08415.pdf
@staticmethod
def forward(x):
"""
Expand All @@ -19,7 +18,6 @@ def forward(x):


class Hardswish(nn.Module):
# Hard-SiLU activation
@staticmethod
def forward(x):
"""
Expand All @@ -40,7 +38,6 @@ def forward(x):


class MemoryEfficientMish(nn.Module):
# Mish activation memory-efficient
class F(torch.autograd.Function):
@staticmethod
def forward(ctx, x):
Expand Down Expand Up @@ -78,9 +75,9 @@ def forward(self, x):


class AconC(nn.Module):
r"""ACON activation (activate or not)
"""ACON activation (activate or not) function.
AconC: (p1*x-p2*x) * sigmoid(beta*(p1*x-p2*x)) + p2*x, beta is a learnable parameter
according to "Activate or Not: Learning Customized Activation" <https://arxiv.org/pdf/2009.04759.pdf>.
See "Activate or Not: Learning Customized Activation" https://arxiv.org/pdf/2009.04759.pdf.
"""

def __init__(self, c1):
Expand All @@ -97,9 +94,9 @@ def forward(self, x):


class MetaAconC(nn.Module):
r"""ACON activation (activate or not)
MetaAconC: (p1*x-p2*x) * sigmoid(beta*(p1*x-p2*x)) + p2*x, beta is generated by a small network
according to "Activate or Not: Learning Customized Activation" <https://arxiv.org/pdf/2009.04759.pdf>.
"""ACON activation (activate or not) function.
AconC: (p1*x-p2*x) * sigmoid(beta*(p1*x-p2*x)) + p2*x, beta is a learnable parameter
See "Activate or Not: Learning Customized Activation" https://arxiv.org/pdf/2009.04759.pdf.
"""

def __init__(self, c1, k=1, s=1, r=16): # ch_in, kernel, stride, r
Expand Down

0 comments on commit 51140c6

Please sign in to comment.