From 7c54aa9763b542886407ddc2c3913b8d36e45407 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 25 Sep 2016 01:08:17 -0700 Subject: [PATCH 01/53] extract tcp transport from go-libp2p-transport --- p2p/transport/tcp/reuseport.go | 65 +++++++ p2p/transport/tcp/reuseport_test.go | 48 ++++++ p2p/transport/tcp/tcp.go | 252 ++++++++++++++++++++++++++++ p2p/transport/tcp/tcp_test.go | 30 ++++ 4 files changed, 395 insertions(+) create mode 100644 p2p/transport/tcp/reuseport.go create mode 100644 p2p/transport/tcp/reuseport_test.go create mode 100644 p2p/transport/tcp/tcp.go create mode 100644 p2p/transport/tcp/tcp_test.go diff --git a/p2p/transport/tcp/reuseport.go b/p2p/transport/tcp/reuseport.go new file mode 100644 index 0000000000..6ead3a74a4 --- /dev/null +++ b/p2p/transport/tcp/reuseport.go @@ -0,0 +1,65 @@ +package tcp + +import ( + "net" + "os" + "strings" + "syscall" + + reuseport "github.com/jbenet/go-reuseport" +) + +// envReuseport is the env variable name used to turn off reuse port. +// It default to true. +const envReuseport = "IPFS_REUSEPORT" + +// envReuseportVal stores the value of envReuseport. defaults to true. +var envReuseportVal = true + +func init() { + v := strings.ToLower(os.Getenv(envReuseport)) + if v == "false" || v == "f" || v == "0" { + envReuseportVal = false + log.Infof("REUSEPORT disabled (IPFS_REUSEPORT=%s)", v) + } +} + +// reuseportIsAvailable returns whether reuseport is available to be used. This +// is here because we want to be able to turn reuseport on and off selectively. +// For now we use an ENV variable, as this handles our pressing need: +// +// IPFS_REUSEPORT=false ipfs daemon +// +// If this becomes a sought after feature, we could add this to the config. +// In the end, reuseport is a stop-gap. +func ReuseportIsAvailable() bool { + return envReuseportVal && reuseport.Available() +} + +// ReuseErrShouldRetry diagnoses whether to retry after a reuse error. +// if we failed to bind, we should retry. if bind worked and this is a +// real dial error (remote end didnt answer) then we should not retry. +func ReuseErrShouldRetry(err error) bool { + if err == nil { + return false // hey, it worked! no need to retry. + } + + // if it's a network timeout error, it's a legitimate failure. + if nerr, ok := err.(net.Error); ok && nerr.Timeout() { + return false + } + + errno, ok := err.(syscall.Errno) + if !ok { // not an errno? who knows what this is. retry. + return true + } + + switch errno { + case syscall.EADDRINUSE, syscall.EADDRNOTAVAIL: + return true // failure to bind. retry. + case syscall.ECONNREFUSED: + return false // real dial error + default: + return true // optimistically default to retry. + } +} diff --git a/p2p/transport/tcp/reuseport_test.go b/p2p/transport/tcp/reuseport_test.go new file mode 100644 index 0000000000..ea746cd5f8 --- /dev/null +++ b/p2p/transport/tcp/reuseport_test.go @@ -0,0 +1,48 @@ +package tcp + +import ( + "net" + "syscall" + "testing" +) + +type netTimeoutErr struct { + timeout bool +} + +func (e netTimeoutErr) Error() string { + return "" +} + +func (e netTimeoutErr) Timeout() bool { + return e.timeout +} + +func (e netTimeoutErr) Temporary() bool { + panic("not checked") +} + +func TestReuseError(t *testing.T) { + var nte1 net.Error = &netTimeoutErr{true} + var nte2 net.Error = &netTimeoutErr{false} + + cases := map[error]bool{ + nil: false, + syscall.EADDRINUSE: true, + syscall.EADDRNOTAVAIL: true, + syscall.ECONNREFUSED: false, + + nte1: false, + nte2: true, // this ones a little weird... we should check neterror.Temporary() too + + // test 'default' to true + syscall.EBUSY: true, + } + + for k, v := range cases { + if ReuseErrShouldRetry(k) != v { + t.Fatalf("expected %b for %#v", v, k) + } + } + +} diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go new file mode 100644 index 0000000000..2285d165dc --- /dev/null +++ b/p2p/transport/tcp/tcp.go @@ -0,0 +1,252 @@ +package tcp + +import ( + "context" + "fmt" + "net" + "sync" + "time" + + lgbl "github.com/ipfs/go-libp2p-loggables" + logging "github.com/ipfs/go-log" + ma "github.com/jbenet/go-multiaddr" + manet "github.com/jbenet/go-multiaddr-net" + reuseport "github.com/jbenet/go-reuseport" + tpt "github.com/libp2p/go-libp2p-transport" + mafmt "github.com/whyrusleeping/mafmt" +) + +var log = logging.Logger("tcp-tpt") + +type TcpTransport struct { + dlock sync.Mutex + dialers map[string]tpt.Dialer + + llock sync.Mutex + listeners map[string]tpt.Listener +} + +// NewTCPTransport creates a tcp transport object that tracks dialers and listeners +// created. It represents an entire tcp stack (though it might not necessarily be) +func NewTCPTransport() *TcpTransport { + return &TcpTransport{ + dialers: make(map[string]tpt.Dialer), + listeners: make(map[string]tpt.Listener), + } +} + +func (t *TcpTransport) Dialer(laddr ma.Multiaddr, opts ...tpt.DialOpt) (tpt.Dialer, error) { + t.dlock.Lock() + defer t.dlock.Unlock() + s := laddr.String() + d, found := t.dialers[s] + if found { + return d, nil + } + var base manet.Dialer + + var doReuse bool + for _, o := range opts { + switch o := o.(type) { + case tpt.TimeoutOpt: + base.Timeout = time.Duration(o) + case tpt.ReuseportOpt: + doReuse = bool(o) + default: + return nil, fmt.Errorf("unrecognized option: %#v", o) + } + } + + tcpd, err := t.newTcpDialer(base, laddr, doReuse) + if err != nil { + return nil, err + } + + t.dialers[s] = tcpd + return tcpd, nil +} + +func (t *TcpTransport) Listen(laddr ma.Multiaddr) (tpt.Listener, error) { + if !t.Matches(laddr) { + return nil, fmt.Errorf("tcp transport cannot listen on %q", laddr) + } + + t.llock.Lock() + defer t.llock.Unlock() + s := laddr.String() + l, found := t.listeners[s] + if found { + return l, nil + } + + list, err := manetListen(laddr) + if err != nil { + return nil, err + } + + tlist := &tcpListener{ + list: list, + transport: t, + } + + t.listeners[s] = tlist + return tlist, nil +} + +func manetListen(addr ma.Multiaddr) (manet.Listener, error) { + network, naddr, err := manet.DialArgs(addr) + if err != nil { + return nil, err + } + + if ReuseportIsAvailable() { + nl, err := reuseport.Listen(network, naddr) + if err == nil { + // hey, it worked! + return manet.WrapNetListener(nl) + } + // reuseport is available, but we failed to listen. log debug, and retry normally. + log.Debugf("reuseport available, but failed to listen: %s %s, %s", network, naddr, err) + } + + // either reuseport not available, or it failed. try normally. + return manet.Listen(addr) +} + +func (t *TcpTransport) Matches(a ma.Multiaddr) bool { + return mafmt.TCP.Matches(a) +} + +type tcpDialer struct { + laddr ma.Multiaddr + + doReuse bool + + rd reuseport.Dialer + madialer manet.Dialer + + transport tpt.Transport +} + +func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReuse bool) (*tcpDialer, error) { + // get the local net.Addr manually + la, err := manet.ToNetAddr(laddr) + if err != nil { + return nil, err // something wrong with laddr. + } + + if doReuse && ReuseportIsAvailable() { + rd := reuseport.Dialer{ + D: net.Dialer{ + LocalAddr: la, + Timeout: base.Timeout, + }, + } + + return &tcpDialer{ + doReuse: true, + laddr: laddr, + rd: rd, + madialer: base, + transport: t, + }, nil + } + + return &tcpDialer{ + doReuse: false, + laddr: laddr, + madialer: base, + transport: t, + }, nil +} + +func (d *tcpDialer) Dial(raddr ma.Multiaddr) (tpt.Conn, error) { + return d.DialContext(context.Background(), raddr) +} + +func (d *tcpDialer) DialContext(ctx context.Context, raddr ma.Multiaddr) (tpt.Conn, error) { + var c manet.Conn + var err error + if d.doReuse { + c, err = d.reuseDial(ctx, raddr) + } else { + c, err = d.madialer.DialContext(ctx, raddr) + } + + if err != nil { + return nil, err + } + + return &tpt.ConnWrap{ + Conn: c, + Tpt: d.transport, + }, nil +} + +func (d *tcpDialer) reuseDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) { + logdial := lgbl.Dial("conn", "", "", d.laddr, raddr) + rpev := log.EventBegin(ctx, "tptDialReusePort", logdial) + + network, netraddr, err := manet.DialArgs(raddr) + if err != nil { + return nil, err + } + + _ = ctx // TODO: implement DialContext in reuseport + con, err := d.rd.Dial(network, netraddr) + if err == nil { + logdial["reuseport"] = "success" + rpev.Done() + return manet.WrapNetConn(con) + } + + if !ReuseErrShouldRetry(err) { + logdial["reuseport"] = "failure" + logdial["error"] = err + rpev.Done() + return nil, err + } + + logdial["reuseport"] = "retry" + logdial["error"] = err + rpev.Done() + + return d.madialer.DialContext(ctx, raddr) +} + +func (d *tcpDialer) Matches(a ma.Multiaddr) bool { + return mafmt.TCP.Matches(a) +} + +type tcpListener struct { + list manet.Listener + transport tpt.Transport +} + +func (d *tcpListener) Accept() (tpt.Conn, error) { + c, err := d.list.Accept() + if err != nil { + return nil, err + } + + return &tpt.ConnWrap{ + Conn: c, + Tpt: d.transport, + }, nil +} + +func (d *tcpListener) Addr() net.Addr { + return d.list.Addr() +} + +func (t *tcpListener) Multiaddr() ma.Multiaddr { + return t.list.Multiaddr() +} + +func (t *tcpListener) NetListener() net.Listener { + return t.list.NetListener() +} + +func (d *tcpListener) Close() error { + return d.list.Close() +} diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go new file mode 100644 index 0000000000..5e1e29087e --- /dev/null +++ b/p2p/transport/tcp/tcp_test.go @@ -0,0 +1,30 @@ +package tcp + +import ( + "testing" + + ma "github.com/jbenet/go-multiaddr" + + utils "github.com/libp2p/go-libp2p-transport/test" +) + +func TestTcpTransport(t *testing.T) { + ta := NewTCPTransport() + tb := NewTCPTransport() + + zero := "/ip4/127.0.0.1/tcp/0" + utils.SubtestTransport(t, ta, tb, zero) +} + +func TestTcpTransportCantListenUtp(t *testing.T) { + utpa, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/0/utp") + if err != nil { + t.Fatal(err) + } + + tpt := NewTCPTransport() + _, err = tpt.Listen(utpa) + if err == nil { + t.Fatal("shouldnt be able to listen on utp addr with tcp transport") + } +} From 8df46c82c359c825c6daecbf4a3251580d656ee8 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 3 Oct 2016 19:47:09 +0200 Subject: [PATCH 02/53] fix: use DialContext of go-reuseport It got skipped when go-reuseport got migrated to context --- p2p/transport/tcp/tcp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 2285d165dc..4def4dfb00 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -193,7 +193,7 @@ func (d *tcpDialer) reuseDial(ctx context.Context, raddr ma.Multiaddr) (manet.Co } _ = ctx // TODO: implement DialContext in reuseport - con, err := d.rd.Dial(network, netraddr) + con, err := d.rd.DialContext(ctx, network, netraddr) if err == nil { logdial["reuseport"] = "success" rpev.Done() From aa9e1d0d7ebe9d30b76edfc3377ea08e79da955c Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 3 Oct 2016 20:04:59 +0200 Subject: [PATCH 03/53] fix(govet): correct fmt for bool is %t --- p2p/transport/tcp/reuseport_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport/tcp/reuseport_test.go b/p2p/transport/tcp/reuseport_test.go index ea746cd5f8..1b03ffd33b 100644 --- a/p2p/transport/tcp/reuseport_test.go +++ b/p2p/transport/tcp/reuseport_test.go @@ -41,7 +41,7 @@ func TestReuseError(t *testing.T) { for k, v := range cases { if ReuseErrShouldRetry(k) != v { - t.Fatalf("expected %b for %#v", v, k) + t.Fatalf("expected %t for %#v", v, k) } } From 9532d3f16a8a68ef7bb5c7f8edd083607df68e89 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 4 Oct 2016 19:38:14 -0700 Subject: [PATCH 04/53] update deps --- p2p/transport/tcp/tcp.go | 6 +++--- p2p/transport/tcp/tcp_test.go | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 4def4dfb00..ae069cce63 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -7,12 +7,12 @@ import ( "sync" "time" - lgbl "github.com/ipfs/go-libp2p-loggables" logging "github.com/ipfs/go-log" - ma "github.com/jbenet/go-multiaddr" - manet "github.com/jbenet/go-multiaddr-net" reuseport "github.com/jbenet/go-reuseport" + lgbl "github.com/libp2p/go-libp2p-loggables" tpt "github.com/libp2p/go-libp2p-transport" + ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr-net" mafmt "github.com/whyrusleeping/mafmt" ) diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go index 5e1e29087e..8eeb5139f3 100644 --- a/p2p/transport/tcp/tcp_test.go +++ b/p2p/transport/tcp/tcp_test.go @@ -3,9 +3,8 @@ package tcp import ( "testing" - ma "github.com/jbenet/go-multiaddr" - utils "github.com/libp2p/go-libp2p-transport/test" + ma "github.com/multiformats/go-multiaddr" ) func TestTcpTransport(t *testing.T) { From e37704103a964f7e68b8680b4a2b4b1c17e22bb9 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 21 Mar 2017 16:59:49 -0700 Subject: [PATCH 05/53] fix ip4 vs ip6 dial failures --- p2p/transport/tcp/tcp.go | 23 ++++++++++++++++++++++- p2p/transport/tcp/tcp_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index ae069cce63..e38c2d1f53 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -124,6 +124,7 @@ type tcpDialer struct { rd reuseport.Dialer madialer manet.Dialer + pattern mafmt.Pattern transport tpt.Transport } @@ -135,6 +136,15 @@ func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReu return nil, err // something wrong with laddr. } + var pattern mafmt.Pattern + if TCP4.Matches(laddr) { + pattern = TCP4 + } else if TCP6.Matches(laddr) { + pattern = TCP6 + } else { + return nil, fmt.Errorf("local addr did not match TCP4 or TCP6: %s", laddr) + } + if doReuse && ReuseportIsAvailable() { rd := reuseport.Dialer{ D: net.Dialer{ @@ -149,6 +159,7 @@ func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReu rd: rd, madialer: base, transport: t, + pattern: pattern, }, nil } @@ -165,6 +176,13 @@ func (d *tcpDialer) Dial(raddr ma.Multiaddr) (tpt.Conn, error) { } func (d *tcpDialer) DialContext(ctx context.Context, raddr ma.Multiaddr) (tpt.Conn, error) { + if raddr == nil { + zaddr, err := ma.NewMultiaddr("/ip4/0.0.0.0/tcp/0") + if err != nil { + return nil, err + } + raddr = zaddr + } var c manet.Conn var err error if d.doReuse { @@ -214,8 +232,11 @@ func (d *tcpDialer) reuseDial(ctx context.Context, raddr ma.Multiaddr) (manet.Co return d.madialer.DialContext(ctx, raddr) } +var TCP4 = mafmt.And(mafmt.Base(ma.P_IP4), mafmt.Base(ma.P_TCP)) +var TCP6 = mafmt.And(mafmt.Base(ma.P_IP6), mafmt.Base(ma.P_TCP)) + func (d *tcpDialer) Matches(a ma.Multiaddr) bool { - return mafmt.TCP.Matches(a) + return d.pattern.Matches(a) } type tcpListener struct { diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go index 8eeb5139f3..2a9a21b49d 100644 --- a/p2p/transport/tcp/tcp_test.go +++ b/p2p/transport/tcp/tcp_test.go @@ -3,6 +3,7 @@ package tcp import ( "testing" + tpt "github.com/libp2p/go-libp2p-transport" utils "github.com/libp2p/go-libp2p-transport/test" ma "github.com/multiformats/go-multiaddr" ) @@ -27,3 +28,34 @@ func TestTcpTransportCantListenUtp(t *testing.T) { t.Fatal("shouldnt be able to listen on utp addr with tcp transport") } } + +func TestCorrectIPVersionMatching(t *testing.T) { + ta := NewTCPTransport() + + addr4, err := ma.NewMultiaddr("/ip4/0.0.0.0/tcp/0") + if err != nil { + t.Fatal(err) + } + addr6, err := ma.NewMultiaddr("/ip6/::1/tcp/0") + if err != nil { + t.Fatal(err) + } + + d4, err := ta.Dialer(addr4, tpt.ReuseportOpt(true)) + if err != nil { + t.Fatal(err) + } + + d6, err := ta.Dialer(addr6, tpt.ReuseportOpt(true)) + if err != nil { + t.Fatal(err) + } + + if d4.Matches(addr6) { + t.Fatal("tcp4 dialer should not match ipv6 address") + } + + if d6.Matches(addr4) { + t.Fatal("tcp4 dialer should not match ipv6 address") + } +} From cfef65cee378689dc823ed537d14ca17459ed356 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 21 Mar 2017 17:01:26 -0700 Subject: [PATCH 06/53] allow nil for dialer laddr --- p2p/transport/tcp/tcp.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index e38c2d1f53..0efa3ffe4e 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -36,6 +36,13 @@ func NewTCPTransport() *TcpTransport { } func (t *TcpTransport) Dialer(laddr ma.Multiaddr, opts ...tpt.DialOpt) (tpt.Dialer, error) { + if laddr == nil { + zaddr, err := ma.NewMultiaddr("/ip4/0.0.0.0/tcp/0") + if err != nil { + return nil, err + } + laddr = zaddr + } t.dlock.Lock() defer t.dlock.Unlock() s := laddr.String() @@ -176,13 +183,6 @@ func (d *tcpDialer) Dial(raddr ma.Multiaddr) (tpt.Conn, error) { } func (d *tcpDialer) DialContext(ctx context.Context, raddr ma.Multiaddr) (tpt.Conn, error) { - if raddr == nil { - zaddr, err := ma.NewMultiaddr("/ip4/0.0.0.0/tcp/0") - if err != nil { - return nil, err - } - raddr = zaddr - } var c manet.Conn var err error if d.doReuse { From 59530f21f48196535bdbd056a81f49f29466e44e Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 21 Mar 2017 17:08:53 -0700 Subject: [PATCH 07/53] clean up reusedial and remove go-libp2p-logging --- p2p/transport/tcp/tcp.go | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 0efa3ffe4e..908229d0bc 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -9,7 +9,6 @@ import ( logging "github.com/ipfs/go-log" reuseport "github.com/jbenet/go-reuseport" - lgbl "github.com/libp2p/go-libp2p-loggables" tpt "github.com/libp2p/go-libp2p-transport" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" @@ -202,33 +201,27 @@ func (d *tcpDialer) DialContext(ctx context.Context, raddr ma.Multiaddr) (tpt.Co } func (d *tcpDialer) reuseDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) { - logdial := lgbl.Dial("conn", "", "", d.laddr, raddr) - rpev := log.EventBegin(ctx, "tptDialReusePort", logdial) - network, netraddr, err := manet.DialArgs(raddr) if err != nil { return nil, err } - _ = ctx // TODO: implement DialContext in reuseport + rpev := log.EventBegin(ctx, "tptDialReusePort", logging.LoggableMap{ + "raddr": raddr, + }) + con, err := d.rd.DialContext(ctx, network, netraddr) if err == nil { - logdial["reuseport"] = "success" rpev.Done() return manet.WrapNetConn(con) } + rpev.SetError(err) + rpev.Done() if !ReuseErrShouldRetry(err) { - logdial["reuseport"] = "failure" - logdial["error"] = err - rpev.Done() return nil, err } - logdial["reuseport"] = "retry" - logdial["error"] = err - rpev.Done() - return d.madialer.DialContext(ctx, raddr) } From 53da0c05ceeb11e00c72639584447eb99b23c8e8 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 23 Mar 2017 22:35:08 -0700 Subject: [PATCH 08/53] fix panic when reuseport was no available --- p2p/transport/tcp/tcp.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 908229d0bc..a30dcbb7b0 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -172,6 +172,7 @@ func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReu return &tcpDialer{ doReuse: false, laddr: laddr, + pattern: pattern, madialer: base, transport: t, }, nil From af1302af3ea4c560159cf4f4ff3379ea6595d7f4 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 26 Jul 2017 16:40:00 +0200 Subject: [PATCH 09/53] fix source address not being set in non REUSEPORT dialer --- p2p/transport/tcp/tcp.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index a30dcbb7b0..e851a2084d 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -51,6 +51,12 @@ func (t *TcpTransport) Dialer(laddr ma.Multiaddr, opts ...tpt.DialOpt) (tpt.Dial } var base manet.Dialer + la, err := manet.ToNetAddr(laddr) + if err != nil { + return nil, err // something wrong with laddr. + } + base.Dialer.LocalAddr = la + var doReuse bool for _, o := range opts { switch o := o.(type) { @@ -136,12 +142,6 @@ type tcpDialer struct { } func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReuse bool) (*tcpDialer, error) { - // get the local net.Addr manually - la, err := manet.ToNetAddr(laddr) - if err != nil { - return nil, err // something wrong with laddr. - } - var pattern mafmt.Pattern if TCP4.Matches(laddr) { pattern = TCP4 @@ -153,10 +153,7 @@ func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReu if doReuse && ReuseportIsAvailable() { rd := reuseport.Dialer{ - D: net.Dialer{ - LocalAddr: la, - Timeout: base.Timeout, - }, + D: base.Dialer, } return &tcpDialer{ From aaf9fcb7135baa542baf6218858ba06d399fc335 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 9 Apr 2017 02:46:53 +0700 Subject: [PATCH 10/53] be explicit about the interfaces implemented --- p2p/transport/tcp/tcp.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index a30dcbb7b0..e4534b7544 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -25,6 +25,8 @@ type TcpTransport struct { listeners map[string]tpt.Listener } +var _ tpt.Transport = &TcpTransport{} + // NewTCPTransport creates a tcp transport object that tracks dialers and listeners // created. It represents an entire tcp stack (though it might not necessarily be) func NewTCPTransport() *TcpTransport { @@ -135,6 +137,8 @@ type tcpDialer struct { transport tpt.Transport } +var _ tpt.Dialer = &tcpDialer{} + func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReuse bool) (*tcpDialer, error) { // get the local net.Addr manually la, err := manet.ToNetAddr(laddr) @@ -238,6 +242,8 @@ type tcpListener struct { transport tpt.Transport } +var _ tpt.Listener = &tcpListener{} + func (d *tcpListener) Accept() (tpt.Conn, error) { c, err := d.list.Accept() if err != nil { From d29927f2f7700141df77a8ebd0be02ca507fb51d Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 9 Apr 2017 02:49:05 +0700 Subject: [PATCH 11/53] replace tpt.ConnWrap by a tcpConn The tpt.ConnWrap is being removed from go-libp2p-transport --- p2p/transport/tcp/tcp.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index e4534b7544..066fffe3df 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -199,9 +199,9 @@ func (d *tcpDialer) DialContext(ctx context.Context, raddr ma.Multiaddr) (tpt.Co return nil, err } - return &tpt.ConnWrap{ + return &tcpConn{ Conn: c, - Tpt: d.transport, + t: d.transport, }, nil } @@ -250,9 +250,9 @@ func (d *tcpListener) Accept() (tpt.Conn, error) { return nil, err } - return &tpt.ConnWrap{ + return &tcpConn{ Conn: c, - Tpt: d.transport, + t: d.transport, }, nil } @@ -271,3 +271,15 @@ func (t *tcpListener) NetListener() net.Listener { func (d *tcpListener) Close() error { return d.list.Close() } + +type tcpConn struct { + manet.Conn + t tpt.Transport +} + +var _ tpt.Conn = &tcpConn{} +var _ tpt.SingleStreamConn = &tcpConn{} + +func (c *tcpConn) Transport() tpt.Transport { + return c.t +} From d7cda66e0f5e84ef598e2283331c2991f35f6c19 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 6 Sep 2017 12:05:11 +0200 Subject: [PATCH 12/53] remove the timeoutOpt Timeout should only be set by using contexts. --- p2p/transport/tcp/tcp.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 066fffe3df..991981ca9c 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -5,7 +5,6 @@ import ( "fmt" "net" "sync" - "time" logging "github.com/ipfs/go-log" reuseport "github.com/jbenet/go-reuseport" @@ -56,8 +55,6 @@ func (t *TcpTransport) Dialer(laddr ma.Multiaddr, opts ...tpt.DialOpt) (tpt.Dial var doReuse bool for _, o := range opts { switch o := o.(type) { - case tpt.TimeoutOpt: - base.Timeout = time.Duration(o) case tpt.ReuseportOpt: doReuse = bool(o) default: From 21fd167df35acbda11a41e4458c3e6a4b558729c Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 6 Sep 2017 12:05:25 +0200 Subject: [PATCH 13/53] use the renamed transport interfaces --- p2p/transport/tcp/tcp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 991981ca9c..87aef6e92c 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -275,7 +275,7 @@ type tcpConn struct { } var _ tpt.Conn = &tcpConn{} -var _ tpt.SingleStreamConn = &tcpConn{} +var _ tpt.DuplexConn = &tcpConn{} func (c *tcpConn) Transport() tpt.Transport { return c.t From a7ae57bbfc7a9094f218a9219d4408fd782cd6f0 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 19 Oct 2017 21:40:57 -0700 Subject: [PATCH 14/53] Revert "use the renamed transport interfaces" This reverts commit 21fd167df35acbda11a41e4458c3e6a4b558729c. --- p2p/transport/tcp/tcp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 87aef6e92c..991981ca9c 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -275,7 +275,7 @@ type tcpConn struct { } var _ tpt.Conn = &tcpConn{} -var _ tpt.DuplexConn = &tcpConn{} +var _ tpt.SingleStreamConn = &tcpConn{} func (c *tcpConn) Transport() tpt.Transport { return c.t From f4deba088c4ceccff9d1761795128ff11f72f3b8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 19 Oct 2017 21:43:38 -0700 Subject: [PATCH 15/53] remove reference to SingleStreamConn --- p2p/transport/tcp/tcp.go | 1 - 1 file changed, 1 deletion(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 991981ca9c..5501ae8a56 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -275,7 +275,6 @@ type tcpConn struct { } var _ tpt.Conn = &tcpConn{} -var _ tpt.SingleStreamConn = &tcpConn{} func (c *tcpConn) Transport() tpt.Transport { return c.t From 055327437924d45ee50403c34528f2f34e006657 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 9 Nov 2017 11:58:38 -0800 Subject: [PATCH 16/53] update go-reuseport 1. Migrate to libp2p repo. 2. Pull in fixes for edge platforms. Related: use `x/sys` instead of `syscall` when possible. 3. Pull in less complicated and slightly faster dial logic. --- p2p/transport/tcp/reuseport.go | 2 +- p2p/transport/tcp/tcp.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/transport/tcp/reuseport.go b/p2p/transport/tcp/reuseport.go index 6ead3a74a4..db2b3da698 100644 --- a/p2p/transport/tcp/reuseport.go +++ b/p2p/transport/tcp/reuseport.go @@ -6,7 +6,7 @@ import ( "strings" "syscall" - reuseport "github.com/jbenet/go-reuseport" + reuseport "github.com/libp2p/go-reuseport" ) // envReuseport is the env variable name used to turn off reuse port. diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 5501ae8a56..442b3d9fc3 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -7,7 +7,7 @@ import ( "sync" logging "github.com/ipfs/go-log" - reuseport "github.com/jbenet/go-reuseport" + reuseport "github.com/libp2p/go-reuseport" tpt "github.com/libp2p/go-libp2p-transport" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" From 5ec6564e7d7201d9c76841e181913791cb5716f0 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sun, 3 Dec 2017 21:11:42 -0800 Subject: [PATCH 17/53] gx publish 1.2.6 --- p2p/transport/tcp/tcp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index edc3b2fc10..bd9b99431e 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -7,8 +7,8 @@ import ( "sync" logging "github.com/ipfs/go-log" - reuseport "github.com/libp2p/go-reuseport" tpt "github.com/libp2p/go-libp2p-transport" + reuseport "github.com/libp2p/go-reuseport" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" mafmt "github.com/whyrusleeping/mafmt" From ec33afbcaae0faabc76544a55fe9578149f59230 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 27 Dec 2017 16:26:37 -0800 Subject: [PATCH 18/53] Revert "fix source address not being set in non REUSEPORT dialer" This reverts commit af1302af3ea4c560159cf4f4ff3379ea6595d7f4. If we do this, we end up using the source *port* for dialing. That's not what we want when not using reuseport. fixes #19 --- p2p/transport/tcp/tcp.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index bd9b99431e..cd8466aae7 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -52,12 +52,6 @@ func (t *TcpTransport) Dialer(laddr ma.Multiaddr, opts ...tpt.DialOpt) (tpt.Dial } var base manet.Dialer - la, err := manet.ToNetAddr(laddr) - if err != nil { - return nil, err // something wrong with laddr. - } - base.Dialer.LocalAddr = la - var doReuse bool for _, o := range opts { switch o := o.(type) { @@ -143,6 +137,12 @@ type tcpDialer struct { var _ tpt.Dialer = &tcpDialer{} func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReuse bool) (*tcpDialer, error) { + // get the local net.Addr manually + la, err := manet.ToNetAddr(laddr) + if err != nil { + return nil, err // something wrong with laddr. + } + var pattern mafmt.Pattern if TCP4.Matches(laddr) { pattern = TCP4 @@ -154,7 +154,10 @@ func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReu if doReuse && ReuseportIsAvailable() { rd := reuseport.Dialer{ - D: base.Dialer, + D: net.Dialer{ + LocalAddr: la, + Timeout: base.Timeout, + }, } return &tcpDialer{ From 69b7572bb71f922970f198b53cb52d6cfe075ca8 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 27 Dec 2017 16:52:04 -0800 Subject: [PATCH 19/53] set the source IP, but not port, when not using the reuseport dialer --- p2p/transport/tcp/tcp.go | 59 +++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index cd8466aae7..0711edebd6 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -50,8 +50,6 @@ func (t *TcpTransport) Dialer(laddr ma.Multiaddr, opts ...tpt.DialOpt) (tpt.Dial if found { return d, nil } - var base manet.Dialer - var doReuse bool for _, o := range opts { switch o := o.(type) { @@ -62,7 +60,7 @@ func (t *TcpTransport) Dialer(laddr ma.Multiaddr, opts ...tpt.DialOpt) (tpt.Dial } } - tcpd, err := t.newTcpDialer(base, laddr, doReuse) + tcpd, err := t.newTcpDialer(laddr, doReuse) if err != nil { return nil, err } @@ -136,11 +134,23 @@ type tcpDialer struct { var _ tpt.Dialer = &tcpDialer{} -func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReuse bool) (*tcpDialer, error) { +func maddrToTcp(addr ma.Multiaddr) (*net.TCPAddr, error) { + la, err := manet.ToNetAddr(addr) + if err != nil { + return nil, err // something wrong with addr. + } + latcp, ok := la.(*net.TCPAddr) + if !ok { + return nil, fmt.Errorf("not a tcp multiaddr: %s", addr) + } + return latcp, nil +} + +func (t *TcpTransport) newTcpDialer(laddr ma.Multiaddr, doReuse bool) (*tcpDialer, error) { // get the local net.Addr manually - la, err := manet.ToNetAddr(laddr) + la, err := maddrToTcp(laddr) if err != nil { - return nil, err // something wrong with laddr. + return nil, err } var pattern mafmt.Pattern @@ -152,31 +162,30 @@ func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReu return nil, fmt.Errorf("local addr did not match TCP4 or TCP6: %s", laddr) } + // Ignore the port when constructing the default (non-reuseport) dialer. + labase := *la + labase.Port = 0 + + dialer := &tcpDialer{ + laddr: laddr, + pattern: pattern, + madialer: manet.Dialer{ + Dialer: net.Dialer{ + LocalAddr: &labase, + }, + }, + transport: t, + } + if doReuse && ReuseportIsAvailable() { - rd := reuseport.Dialer{ + dialer.doReuse = true + dialer.rd = reuseport.Dialer{ D: net.Dialer{ LocalAddr: la, - Timeout: base.Timeout, }, } - - return &tcpDialer{ - doReuse: true, - laddr: laddr, - rd: rd, - madialer: base, - transport: t, - pattern: pattern, - }, nil } - - return &tcpDialer{ - doReuse: false, - laddr: laddr, - pattern: pattern, - madialer: base, - transport: t, - }, nil + return dialer, nil } func (d *tcpDialer) Dial(raddr ma.Multiaddr) (tpt.Conn, error) { From 4784a35f5244d8d365682c465e773cb93e8b6a33 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 18 Jan 2018 22:04:35 -0800 Subject: [PATCH 20/53] refactor for transport interface changes Also, refactor out reuseport logic into a separate package. --- p2p/transport/tcp/reuseport.go | 30 --- p2p/transport/tcp/reuseport_test.go | 48 ----- p2p/transport/tcp/tcp.go | 286 +++++----------------------- p2p/transport/tcp/tcp_test.go | 77 ++++---- 4 files changed, 79 insertions(+), 362 deletions(-) delete mode 100644 p2p/transport/tcp/reuseport_test.go diff --git a/p2p/transport/tcp/reuseport.go b/p2p/transport/tcp/reuseport.go index db2b3da698..58105ce537 100644 --- a/p2p/transport/tcp/reuseport.go +++ b/p2p/transport/tcp/reuseport.go @@ -1,10 +1,8 @@ package tcp import ( - "net" "os" "strings" - "syscall" reuseport "github.com/libp2p/go-reuseport" ) @@ -35,31 +33,3 @@ func init() { func ReuseportIsAvailable() bool { return envReuseportVal && reuseport.Available() } - -// ReuseErrShouldRetry diagnoses whether to retry after a reuse error. -// if we failed to bind, we should retry. if bind worked and this is a -// real dial error (remote end didnt answer) then we should not retry. -func ReuseErrShouldRetry(err error) bool { - if err == nil { - return false // hey, it worked! no need to retry. - } - - // if it's a network timeout error, it's a legitimate failure. - if nerr, ok := err.(net.Error); ok && nerr.Timeout() { - return false - } - - errno, ok := err.(syscall.Errno) - if !ok { // not an errno? who knows what this is. retry. - return true - } - - switch errno { - case syscall.EADDRINUSE, syscall.EADDRNOTAVAIL: - return true // failure to bind. retry. - case syscall.ECONNREFUSED: - return false // real dial error - default: - return true // optimistically default to retry. - } -} diff --git a/p2p/transport/tcp/reuseport_test.go b/p2p/transport/tcp/reuseport_test.go deleted file mode 100644 index 1b03ffd33b..0000000000 --- a/p2p/transport/tcp/reuseport_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package tcp - -import ( - "net" - "syscall" - "testing" -) - -type netTimeoutErr struct { - timeout bool -} - -func (e netTimeoutErr) Error() string { - return "" -} - -func (e netTimeoutErr) Timeout() bool { - return e.timeout -} - -func (e netTimeoutErr) Temporary() bool { - panic("not checked") -} - -func TestReuseError(t *testing.T) { - var nte1 net.Error = &netTimeoutErr{true} - var nte2 net.Error = &netTimeoutErr{false} - - cases := map[error]bool{ - nil: false, - syscall.EADDRINUSE: true, - syscall.EADDRNOTAVAIL: true, - syscall.ECONNREFUSED: false, - - nte1: false, - nte2: true, // this ones a little weird... we should check neterror.Temporary() too - - // test 'default' to true - syscall.EBUSY: true, - } - - for k, v := range cases { - if ReuseErrShouldRetry(k) != v { - t.Fatalf("expected %t for %#v", v, k) - } - } - -} diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 0711edebd6..3ed2ff3f2f 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -2,13 +2,12 @@ package tcp import ( "context" - "fmt" - "net" - "sync" logging "github.com/ipfs/go-log" + peer "github.com/libp2p/go-libp2p-peer" tpt "github.com/libp2p/go-libp2p-transport" - reuseport "github.com/libp2p/go-reuseport" + tptu "github.com/libp2p/go-libp2p-transport-upgrader" + rtpt "github.com/libp2p/go-reuseport-transport" ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" mafmt "github.com/whyrusleeping/mafmt" @@ -16,275 +15,80 @@ import ( var log = logging.Logger("tcp-tpt") +// TcpTransport is the TCP transport. type TcpTransport struct { - dlock sync.Mutex - dialers map[string]tpt.Dialer + // Connection upgrader for upgrading insecure stream connections to + // secure multiplex connections. + Upgrader *tptu.Upgrader - llock sync.Mutex - listeners map[string]tpt.Listener + // Explicitly disable reuseport. + DisableReuseport bool + + reuse rtpt.Transport } var _ tpt.Transport = &TcpTransport{} // NewTCPTransport creates a tcp transport object that tracks dialers and listeners // created. It represents an entire tcp stack (though it might not necessarily be) -func NewTCPTransport() *TcpTransport { - return &TcpTransport{ - dialers: make(map[string]tpt.Dialer), - listeners: make(map[string]tpt.Listener), - } -} - -func (t *TcpTransport) Dialer(laddr ma.Multiaddr, opts ...tpt.DialOpt) (tpt.Dialer, error) { - if laddr == nil { - zaddr, err := ma.NewMultiaddr("/ip4/0.0.0.0/tcp/0") - if err != nil { - return nil, err - } - laddr = zaddr - } - t.dlock.Lock() - defer t.dlock.Unlock() - s := laddr.String() - d, found := t.dialers[s] - if found { - return d, nil - } - var doReuse bool - for _, o := range opts { - switch o := o.(type) { - case tpt.ReuseportOpt: - doReuse = bool(o) - default: - return nil, fmt.Errorf("unrecognized option: %#v", o) - } - } - - tcpd, err := t.newTcpDialer(laddr, doReuse) - if err != nil { - return nil, err - } - - t.dialers[s] = tcpd - return tcpd, nil -} - -func (t *TcpTransport) Listen(laddr ma.Multiaddr) (tpt.Listener, error) { - if !t.Matches(laddr) { - return nil, fmt.Errorf("tcp transport cannot listen on %q", laddr) - } - - t.llock.Lock() - defer t.llock.Unlock() - s := laddr.String() - l, found := t.listeners[s] - if found { - return l, nil - } - - list, err := manetListen(laddr) - if err != nil { - return nil, err - } - - tlist := &tcpListener{ - list: list, - transport: t, - } - - t.listeners[s] = tlist - return tlist, nil -} - -func manetListen(addr ma.Multiaddr) (manet.Listener, error) { - network, naddr, err := manet.DialArgs(addr) - if err != nil { - return nil, err - } - - if ReuseportIsAvailable() { - nl, err := reuseport.Listen(network, naddr) - if err == nil { - // hey, it worked! - return manet.WrapNetListener(nl) - } - // reuseport is available, but we failed to listen. log debug, and retry normally. - log.Debugf("reuseport available, but failed to listen: %s %s, %s", network, naddr, err) - } - - // either reuseport not available, or it failed. try normally. - return manet.Listen(addr) -} - -func (t *TcpTransport) Matches(a ma.Multiaddr) bool { - return mafmt.TCP.Matches(a) +func NewTCPTransport(upgrader *tptu.Upgrader) *TcpTransport { + return &TcpTransport{Upgrader: upgrader} } -type tcpDialer struct { - laddr ma.Multiaddr - - doReuse bool - - rd reuseport.Dialer - madialer manet.Dialer - pattern mafmt.Pattern - - transport tpt.Transport +// CanDial returns true if this transport believes it can dial the given +// multiaddr. +func (t *TcpTransport) CanDial(addr ma.Multiaddr) bool { + return mafmt.TCP.Matches(addr) } -var _ tpt.Dialer = &tcpDialer{} - -func maddrToTcp(addr ma.Multiaddr) (*net.TCPAddr, error) { - la, err := manet.ToNetAddr(addr) - if err != nil { - return nil, err // something wrong with addr. +func (t *TcpTransport) maDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) { + if t.UseReuseport() { + return t.reuse.DialContext(ctx, raddr) } - latcp, ok := la.(*net.TCPAddr) - if !ok { - return nil, fmt.Errorf("not a tcp multiaddr: %s", addr) - } - return latcp, nil + var d manet.Dialer + return d.DialContext(ctx, raddr) } -func (t *TcpTransport) newTcpDialer(laddr ma.Multiaddr, doReuse bool) (*tcpDialer, error) { - // get the local net.Addr manually - la, err := maddrToTcp(laddr) +// Dial dials the peer at the remote address. +func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (tpt.Conn, error) { + conn, err := t.maDial(ctx, raddr) if err != nil { return nil, err } - - var pattern mafmt.Pattern - if TCP4.Matches(laddr) { - pattern = TCP4 - } else if TCP6.Matches(laddr) { - pattern = TCP6 - } else { - return nil, fmt.Errorf("local addr did not match TCP4 or TCP6: %s", laddr) - } - - // Ignore the port when constructing the default (non-reuseport) dialer. - labase := *la - labase.Port = 0 - - dialer := &tcpDialer{ - laddr: laddr, - pattern: pattern, - madialer: manet.Dialer{ - Dialer: net.Dialer{ - LocalAddr: &labase, - }, - }, - transport: t, - } - - if doReuse && ReuseportIsAvailable() { - dialer.doReuse = true - dialer.rd = reuseport.Dialer{ - D: net.Dialer{ - LocalAddr: la, - }, - } - } - return dialer, nil -} - -func (d *tcpDialer) Dial(raddr ma.Multiaddr) (tpt.Conn, error) { - return d.DialContext(context.Background(), raddr) + return t.Upgrader.UpgradeOutbound(ctx, t, conn, p) } -func (d *tcpDialer) DialContext(ctx context.Context, raddr ma.Multiaddr) (tpt.Conn, error) { - var c manet.Conn - var err error - if d.doReuse { - c, err = d.reuseDial(ctx, raddr) - } else { - c, err = d.madialer.DialContext(ctx, raddr) - } - - if err != nil { - return nil, err - } - - return &tcpConn{ - Conn: c, - t: d.transport, - }, nil +// UseReuseport returns true if reuseport is enabled and available. +func (t *TcpTransport) UseReuseport() bool { + return !t.DisableReuseport && ReuseportIsAvailable() } -func (d *tcpDialer) reuseDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) { - network, netraddr, err := manet.DialArgs(raddr) - if err != nil { - return nil, err - } - - rpev := log.EventBegin(ctx, "tptDialReusePort", logging.LoggableMap{ - "raddr": raddr, - }) - - con, err := d.rd.DialContext(ctx, network, netraddr) - if err == nil { - rpev.Done() - return manet.WrapNetConn(con) +func (t *TcpTransport) maListen(laddr ma.Multiaddr) (manet.Listener, error) { + if t.UseReuseport() { + return t.reuse.Listen(laddr) } - rpev.SetError(err) - rpev.Done() - - if !ReuseErrShouldRetry(err) { - return nil, err - } - - return d.madialer.DialContext(ctx, raddr) -} - -var TCP4 = mafmt.And(mafmt.Base(ma.P_IP4), mafmt.Base(ma.P_TCP)) -var TCP6 = mafmt.And(mafmt.Base(ma.P_IP6), mafmt.Base(ma.P_TCP)) - -func (d *tcpDialer) Matches(a ma.Multiaddr) bool { - return d.pattern.Matches(a) -} - -type tcpListener struct { - list manet.Listener - transport tpt.Transport + return manet.Listen(laddr) } -var _ tpt.Listener = &tcpListener{} - -func (d *tcpListener) Accept() (tpt.Conn, error) { - c, err := d.list.Accept() +// Listen listens on the given multiaddr. +func (t *TcpTransport) Listen(laddr ma.Multiaddr) (tpt.Listener, error) { + list, err := t.maListen(laddr) if err != nil { return nil, err } - - return &tcpConn{ - Conn: c, - t: d.transport, - }, nil -} - -func (d *tcpListener) Addr() net.Addr { - return d.list.Addr() + return t.Upgrader.UpgradeListener(t, list), nil } -func (t *tcpListener) Multiaddr() ma.Multiaddr { - return t.list.Multiaddr() +// Protocols returns the list of terminal protocols this transport can dial. +func (t *TcpTransport) Protocols() []int { + return []int{ma.P_TCP} } -func (t *tcpListener) NetListener() net.Listener { - return t.list.NetListener() +// Proxy always returns false for the TCP transport. +func (t *TcpTransport) Proxy() bool { + return false } -func (d *tcpListener) Close() error { - return d.list.Close() -} - -type tcpConn struct { - manet.Conn - t tpt.Transport -} - -var _ tpt.Conn = &tcpConn{} - -func (c *tcpConn) Transport() tpt.Transport { - return c.t +func (t *TcpTransport) String() string { + return "TCP" } diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go index 2a9a21b49d..4467e8a2d7 100644 --- a/p2p/transport/tcp/tcp_test.go +++ b/p2p/transport/tcp/tcp_test.go @@ -3,59 +3,50 @@ package tcp import ( "testing" - tpt "github.com/libp2p/go-libp2p-transport" + insecure "github.com/libp2p/go-conn-security/insecure" + tptu "github.com/libp2p/go-libp2p-transport-upgrader" utils "github.com/libp2p/go-libp2p-transport/test" ma "github.com/multiformats/go-multiaddr" + mplex "github.com/whyrusleeping/go-smux-multiplex" ) func TestTcpTransport(t *testing.T) { - ta := NewTCPTransport() - tb := NewTCPTransport() - - zero := "/ip4/127.0.0.1/tcp/0" - utils.SubtestTransport(t, ta, tb, zero) + for i := 0; i < 2; i++ { + ta := NewTCPTransport(&tptu.Upgrader{ + Secure: insecure.New("peerA"), + Muxer: new(mplex.Transport), + }) + tb := NewTCPTransport(&tptu.Upgrader{ + Secure: insecure.New("peerB"), + Muxer: new(mplex.Transport), + }) + + zero := "/ip4/127.0.0.1/tcp/0" + utils.SubtestTransport(t, ta, tb, zero, "peerA") + + envReuseportVal = false + } + envReuseportVal = true } func TestTcpTransportCantListenUtp(t *testing.T) { - utpa, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/0/utp") - if err != nil { - t.Fatal(err) - } + for i := 0; i < 2; i++ { + utpa, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/0/utp") + if err != nil { + t.Fatal(err) + } - tpt := NewTCPTransport() - _, err = tpt.Listen(utpa) - if err == nil { - t.Fatal("shouldnt be able to listen on utp addr with tcp transport") - } -} + tpt := NewTCPTransport(&tptu.Upgrader{ + Secure: insecure.New("peerB"), + Muxer: new(mplex.Transport), + }) -func TestCorrectIPVersionMatching(t *testing.T) { - ta := NewTCPTransport() - - addr4, err := ma.NewMultiaddr("/ip4/0.0.0.0/tcp/0") - if err != nil { - t.Fatal(err) - } - addr6, err := ma.NewMultiaddr("/ip6/::1/tcp/0") - if err != nil { - t.Fatal(err) - } - - d4, err := ta.Dialer(addr4, tpt.ReuseportOpt(true)) - if err != nil { - t.Fatal(err) - } - - d6, err := ta.Dialer(addr6, tpt.ReuseportOpt(true)) - if err != nil { - t.Fatal(err) - } - - if d4.Matches(addr6) { - t.Fatal("tcp4 dialer should not match ipv6 address") - } + _, err = tpt.Listen(utpa) + if err == nil { + t.Fatal("shouldnt be able to listen on utp addr with tcp transport") + } - if d6.Matches(addr4) { - t.Fatal("tcp4 dialer should not match ipv6 address") + envReuseportVal = false } + envReuseportVal = true } From bd1a6b0bdd25eef53c01870a480e3fa1dcced734 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 8 Jun 2018 10:11:41 -0700 Subject: [PATCH 21/53] add a TCP connect timeout --- p2p/transport/tcp/tcp.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 3ed2ff3f2f..18e119ec71 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -2,6 +2,7 @@ package tcp import ( "context" + "time" logging "github.com/ipfs/go-log" peer "github.com/libp2p/go-libp2p-peer" @@ -13,6 +14,10 @@ import ( mafmt "github.com/whyrusleeping/mafmt" ) +// DefaultConnectTimeout is the (default) maximum amount of time the TCP +// transport will spend on the initial TCP connect before giving up. +var DefaultConnectTimeout = 5 * time.Second + var log = logging.Logger("tcp-tpt") // TcpTransport is the TCP transport. @@ -24,6 +29,9 @@ type TcpTransport struct { // Explicitly disable reuseport. DisableReuseport bool + // TCP connect timeout + ConnectTimeout time.Duration + reuse rtpt.Transport } @@ -32,7 +40,7 @@ var _ tpt.Transport = &TcpTransport{} // NewTCPTransport creates a tcp transport object that tracks dialers and listeners // created. It represents an entire tcp stack (though it might not necessarily be) func NewTCPTransport(upgrader *tptu.Upgrader) *TcpTransport { - return &TcpTransport{Upgrader: upgrader} + return &TcpTransport{Upgrader: upgrader, ConnectTimeout: DefaultConnectTimeout} } // CanDial returns true if this transport believes it can dial the given @@ -42,6 +50,16 @@ func (t *TcpTransport) CanDial(addr ma.Multiaddr) bool { } func (t *TcpTransport) maDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) { + // Apply the deadline iff applicable + if t.ConnectTimeout > 0 { + deadline := time.Now().Add(t.ConnectTimeout) + if d, ok := ctx.Deadline(); !ok || deadline.Before(d) { + var cancel func() + ctx, cancel = context.WithDeadline(ctx, deadline) + defer cancel() + } + } + if t.UseReuseport() { return t.reuse.DialContext(ctx, raddr) } From e8b9e0d5e344d85ae438103834826b95fb6f6532 Mon Sep 17 00:00:00 2001 From: Can ZHANG Date: Tue, 30 Oct 2018 11:24:46 +0800 Subject: [PATCH 22/53] Deprecate IPFS_REUSEPORT, use LIBP2P_TCP_REUSEPORT --- p2p/transport/tcp/reuseport.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/p2p/transport/tcp/reuseport.go b/p2p/transport/tcp/reuseport.go index 58105ce537..04a5da3ed1 100644 --- a/p2p/transport/tcp/reuseport.go +++ b/p2p/transport/tcp/reuseport.go @@ -9,7 +9,8 @@ import ( // envReuseport is the env variable name used to turn off reuse port. // It default to true. -const envReuseport = "IPFS_REUSEPORT" +const envReuseport = "LIBP2P_TCP_REUSEPORT" +const deprecatedEnvReuseport = "IPFS_REUSEPORT" // envReuseportVal stores the value of envReuseport. defaults to true. var envReuseportVal = true @@ -18,7 +19,15 @@ func init() { v := strings.ToLower(os.Getenv(envReuseport)) if v == "false" || v == "f" || v == "0" { envReuseportVal = false - log.Infof("REUSEPORT disabled (IPFS_REUSEPORT=%s)", v) + log.Infof("REUSEPORT disabled (LIBP2P_TCP_REUSEPORT=%s)", v) + } + v, exist := os.LookupEnv(deprecatedEnvReuseport) + if exist { + log.Warning("IPFS_REUSEPORT is deprecated, use LIBP2P_TCP_REUSEPORT instead") + if v == "false" || v == "f" || v == "0" { + envReuseportVal = false + log.Infof("REUSEPORT disabled (IPFS_REUSEPORT=%s)", v) + } } } @@ -26,7 +35,7 @@ func init() { // is here because we want to be able to turn reuseport on and off selectively. // For now we use an ENV variable, as this handles our pressing need: // -// IPFS_REUSEPORT=false ipfs daemon +// LIBP2P_TCP_REUSEPORT=false ipfs daemon // // If this becomes a sought after feature, we could add this to the config. // In the end, reuseport is a stop-gap. From 87f331a7fc4ca5aa71f695190f2e187165d00421 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 3 Apr 2019 19:29:38 -0700 Subject: [PATCH 23/53] set linger to 0 for both inbound and outbound connections This causes us to send RST packets instead of FIN packets when closing connections and means connections immediately enter the "reset" state instead of entering the TIME-WAIT state. Importantly, this means we can immediately reuse the 5-tuple and reconnect. --- p2p/transport/tcp/tcp.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 18e119ec71..5f7fec8a81 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -2,6 +2,7 @@ package tcp import ( "context" + "net" "time" logging "github.com/ipfs/go-log" @@ -20,6 +21,29 @@ var DefaultConnectTimeout = 5 * time.Second var log = logging.Logger("tcp-tpt") +// try to set linger on the connection, if possible. +func tryLinger(conn net.Conn, sec int) { + if lingerConn, ok := conn.(interface { + SetLinger(int) error + }); ok { + _ = lingerConn.SetLinger(sec) + } +} + +type lingerListener struct { + manet.Listener + sec int +} + +func (ll *lingerListener) Accept() (manet.Conn, error) { + c, err := ll.Listener.Accept() + if err != nil { + return nil, err + } + tryLinger(c, ll.sec) + return c, nil +} + // TcpTransport is the TCP transport. type TcpTransport struct { // Connection upgrader for upgrading insecure stream connections to @@ -73,6 +97,10 @@ func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) if err != nil { return nil, err } + // Set linger to 0 so we never get stuck in the TIME-WAIT state. When + // linger is 0, connections are _reset_ instead of closed with a FIN. + // This means we can immediately reuse the 5-tuple and reconnect. + tryLinger(conn, 0) return t.Upgrader.UpgradeOutbound(ctx, t, conn, p) } @@ -94,6 +122,7 @@ func (t *TcpTransport) Listen(laddr ma.Multiaddr) (tpt.Listener, error) { if err != nil { return nil, err } + list = &lingerListener{list, 0} return t.Upgrader.UpgradeListener(t, list), nil } From a517f4d213c8c950b8b1d761b1158aa283306cfc Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 4 Apr 2019 12:59:57 -0700 Subject: [PATCH 24/53] chore: remove inline interface (@vyzo CR) --- p2p/transport/tcp/tcp.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 5f7fec8a81..84a34a4d4f 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -23,9 +23,11 @@ var log = logging.Logger("tcp-tpt") // try to set linger on the connection, if possible. func tryLinger(conn net.Conn, sec int) { - if lingerConn, ok := conn.(interface { + type canLinger interface { SetLinger(int) error - }); ok { + } + + if lingerConn, ok := conn.(canLinger); ok { _ = lingerConn.SetLinger(sec) } } From b06b0e6b216e33f1a39fe984acfe47c1b4a13eaa Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 21 May 2019 19:00:11 -0700 Subject: [PATCH 25/53] dep: import go-libp2p-mplex into the libp2p org --- p2p/transport/tcp/tcp_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go index 4467e8a2d7..4efd1cb6b2 100644 --- a/p2p/transport/tcp/tcp_test.go +++ b/p2p/transport/tcp/tcp_test.go @@ -4,10 +4,10 @@ import ( "testing" insecure "github.com/libp2p/go-conn-security/insecure" + mplex "github.com/libp2p/go-libp2p-mplex" tptu "github.com/libp2p/go-libp2p-transport-upgrader" utils "github.com/libp2p/go-libp2p-transport/test" ma "github.com/multiformats/go-multiaddr" - mplex "github.com/whyrusleeping/go-smux-multiplex" ) func TestTcpTransport(t *testing.T) { From ed7828ceefc655c6120275ce2e70fb368cb04e3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Sun, 26 May 2019 15:11:07 +0100 Subject: [PATCH 26/53] migrate to consolidated types (#40) --- p2p/transport/tcp/reuseport.go | 2 +- p2p/transport/tcp/tcp.go | 13 +++++++------ p2p/transport/tcp/tcp_test.go | 8 +++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/p2p/transport/tcp/reuseport.go b/p2p/transport/tcp/reuseport.go index 04a5da3ed1..e3b7e1250f 100644 --- a/p2p/transport/tcp/reuseport.go +++ b/p2p/transport/tcp/reuseport.go @@ -4,7 +4,7 @@ import ( "os" "strings" - reuseport "github.com/libp2p/go-reuseport" + "github.com/libp2p/go-reuseport" ) // envReuseport is the env variable name used to turn off reuse port. diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 84a34a4d4f..1483d67e69 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -6,13 +6,14 @@ import ( "time" logging "github.com/ipfs/go-log" - peer "github.com/libp2p/go-libp2p-peer" - tpt "github.com/libp2p/go-libp2p-transport" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/transport" tptu "github.com/libp2p/go-libp2p-transport-upgrader" rtpt "github.com/libp2p/go-reuseport-transport" + ma "github.com/multiformats/go-multiaddr" + mafmt "github.com/multiformats/go-multiaddr-fmt" manet "github.com/multiformats/go-multiaddr-net" - mafmt "github.com/whyrusleeping/mafmt" ) // DefaultConnectTimeout is the (default) maximum amount of time the TCP @@ -61,7 +62,7 @@ type TcpTransport struct { reuse rtpt.Transport } -var _ tpt.Transport = &TcpTransport{} +var _ transport.Transport = &TcpTransport{} // NewTCPTransport creates a tcp transport object that tracks dialers and listeners // created. It represents an entire tcp stack (though it might not necessarily be) @@ -94,7 +95,7 @@ func (t *TcpTransport) maDial(ctx context.Context, raddr ma.Multiaddr) (manet.Co } // Dial dials the peer at the remote address. -func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (tpt.Conn, error) { +func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (transport.CapableConn, error) { conn, err := t.maDial(ctx, raddr) if err != nil { return nil, err @@ -119,7 +120,7 @@ func (t *TcpTransport) maListen(laddr ma.Multiaddr) (manet.Listener, error) { } // Listen listens on the given multiaddr. -func (t *TcpTransport) Listen(laddr ma.Multiaddr) (tpt.Listener, error) { +func (t *TcpTransport) Listen(laddr ma.Multiaddr) (transport.Listener, error) { list, err := t.maListen(laddr) if err != nil { return nil, err diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go index 4efd1cb6b2..cfceef50f3 100644 --- a/p2p/transport/tcp/tcp_test.go +++ b/p2p/transport/tcp/tcp_test.go @@ -3,10 +3,12 @@ package tcp import ( "testing" - insecure "github.com/libp2p/go-conn-security/insecure" + "github.com/libp2p/go-libp2p-core/sec/insecure" mplex "github.com/libp2p/go-libp2p-mplex" tptu "github.com/libp2p/go-libp2p-transport-upgrader" - utils "github.com/libp2p/go-libp2p-transport/test" + + ttransport "github.com/libp2p/go-libp2p-testing/suites/transport" + ma "github.com/multiformats/go-multiaddr" ) @@ -22,7 +24,7 @@ func TestTcpTransport(t *testing.T) { }) zero := "/ip4/127.0.0.1/tcp/0" - utils.SubtestTransport(t, ta, tb, zero, "peerA") + ttransport.SubtestTransport(t, ta, tb, zero, "peerA") envReuseportVal = false } From cea811c943b7e6370322d1cf6fa4e0edf5b52f16 Mon Sep 17 00:00:00 2001 From: Yusef Napora Date: Thu, 25 Jul 2019 12:20:15 -0400 Subject: [PATCH 27/53] use new insecure transport constructor --- p2p/transport/tcp/tcp_test.go | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go index cfceef50f3..a4a85bdf04 100644 --- a/p2p/transport/tcp/tcp_test.go +++ b/p2p/transport/tcp/tcp_test.go @@ -1,6 +1,8 @@ package tcp import ( + "github.com/libp2p/go-libp2p-core/crypto" + "github.com/libp2p/go-libp2p-core/peer" "testing" "github.com/libp2p/go-libp2p-core/sec/insecure" @@ -14,17 +16,20 @@ import ( func TestTcpTransport(t *testing.T) { for i := 0; i < 2; i++ { + ia := makeInsecureTransport(t) + ib := makeInsecureTransport(t) + ta := NewTCPTransport(&tptu.Upgrader{ - Secure: insecure.New("peerA"), + Secure: ia, Muxer: new(mplex.Transport), }) tb := NewTCPTransport(&tptu.Upgrader{ - Secure: insecure.New("peerB"), + Secure: ib, Muxer: new(mplex.Transport), }) zero := "/ip4/127.0.0.1/tcp/0" - ttransport.SubtestTransport(t, ta, tb, zero, "peerA") + ttransport.SubtestTransport(t, ta, tb, zero, ia.LocalPeer()) envReuseportVal = false } @@ -39,7 +44,7 @@ func TestTcpTransportCantListenUtp(t *testing.T) { } tpt := NewTCPTransport(&tptu.Upgrader{ - Secure: insecure.New("peerB"), + Secure: makeInsecureTransport(t), Muxer: new(mplex.Transport), }) @@ -52,3 +57,15 @@ func TestTcpTransportCantListenUtp(t *testing.T) { } envReuseportVal = true } + +func makeInsecureTransport(t *testing.T) *insecure.Transport { + priv, pub, err := crypto.GenerateKeyPair(crypto.Ed25519, 256) + if err != nil { + t.Fatal(err) + } + id, err := peer.IDFromPublicKey(pub) + if err != nil { + t.Fatal(err) + } + return insecure.NewWithIdentity(id, priv) +} From 503a78e2800a6a010c40371b1fdffe47e0575a07 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 2 Apr 2020 22:20:21 -0700 Subject: [PATCH 28/53] fix: don't allow dialing DNS addresses See https://github.com/libp2p/go-libp2p/issues/841 --- p2p/transport/tcp/tcp.go | 4 +++- p2p/transport/tcp/tcp_test.go | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 1483d67e69..6c204109f3 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -70,10 +70,12 @@ func NewTCPTransport(upgrader *tptu.Upgrader) *TcpTransport { return &TcpTransport{Upgrader: upgrader, ConnectTimeout: DefaultConnectTimeout} } +var dialMatcher = mafmt.And(mafmt.IP, mafmt.Base(ma.P_TCP)) + // CanDial returns true if this transport believes it can dial the given // multiaddr. func (t *TcpTransport) CanDial(addr ma.Multiaddr) bool { - return mafmt.TCP.Matches(addr) + return dialMatcher.Matches(addr) } func (t *TcpTransport) maDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) { diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go index a4a85bdf04..8dcad45248 100644 --- a/p2p/transport/tcp/tcp_test.go +++ b/p2p/transport/tcp/tcp_test.go @@ -36,6 +36,27 @@ func TestTcpTransport(t *testing.T) { envReuseportVal = true } +func TestTcpTransportCantDialDNS(t *testing.T) { + for i := 0; i < 2; i++ { + dnsa, err := ma.NewMultiaddr("/dns4/example.com/tcp/1234") + if err != nil { + t.Fatal(err) + } + + tpt := NewTCPTransport(&tptu.Upgrader{ + Secure: makeInsecureTransport(t), + Muxer: new(mplex.Transport), + }) + + if tpt.CanDial(dnsa) { + t.Fatal("shouldn't be able to dial dns") + } + + envReuseportVal = false + } + envReuseportVal = true +} + func TestTcpTransportCantListenUtp(t *testing.T) { for i := 0; i < 2; i++ { utpa, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/0/utp") From d9fec436cd1e4c18f82d2c620369036b7a6a42ad Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 16 Feb 2021 13:59:34 +0800 Subject: [PATCH 29/53] stop using the deprecated go-multiaddr-net package --- p2p/transport/tcp/tcp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 6c204109f3..03b2203247 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -13,7 +13,7 @@ import ( ma "github.com/multiformats/go-multiaddr" mafmt "github.com/multiformats/go-multiaddr-fmt" - manet "github.com/multiformats/go-multiaddr-net" + manet "github.com/multiformats/go-multiaddr/net" ) // DefaultConnectTimeout is the (default) maximum amount of time the TCP From 2f74f826074d8d7a1f3d96bf832bed7352cd0a03 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 16 Feb 2021 15:52:48 +0800 Subject: [PATCH 30/53] enable TCP keepalives --- p2p/transport/tcp/tcp.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 03b2203247..bb949974ff 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -22,6 +22,30 @@ var DefaultConnectTimeout = 5 * time.Second var log = logging.Logger("tcp-tpt") +const keepAlivePeriod = 30 * time.Second + +type canKeepAlive interface { + SetKeepAlive(bool) error + SetKeepAlivePeriod(time.Duration) error +} + +var _ canKeepAlive = &net.TCPConn{} + +func tryKeepAlive(conn net.Conn, keepAlive bool) { + keepAliveConn, ok := conn.(canKeepAlive) + if !ok { + log.Errorf("Can't set TCP keepalives.") + return + } + if err := keepAliveConn.SetKeepAlive(keepAlive); err != nil { + log.Errorf("Failed to enable TCP keepalive: %s", err) + return + } + if err := keepAliveConn.SetKeepAlivePeriod(keepAlivePeriod); err != nil { + log.Errorf("Failed set keepalive period: %s", err) + } +} + // try to set linger on the connection, if possible. func tryLinger(conn net.Conn, sec int) { type canLinger interface { @@ -44,6 +68,7 @@ func (ll *lingerListener) Accept() (manet.Conn, error) { return nil, err } tryLinger(c, ll.sec) + tryKeepAlive(c, true) return c, nil } @@ -106,6 +131,7 @@ func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) // linger is 0, connections are _reset_ instead of closed with a FIN. // This means we can immediately reuse the 5-tuple and reconnect. tryLinger(conn, 0) + tryKeepAlive(conn, true) return t.Upgrader.UpgradeOutbound(ctx, t, conn, p) } From c585e14f87eef5064ca1e744d71a383c2d795aaa Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 24 Feb 2021 11:24:22 +0800 Subject: [PATCH 31/53] rename the lingerListener to tcpListener --- p2p/transport/tcp/tcp.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index bb949974ff..95c30dfcd8 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -57,12 +57,12 @@ func tryLinger(conn net.Conn, sec int) { } } -type lingerListener struct { +type tcpListener struct { manet.Listener sec int } -func (ll *lingerListener) Accept() (manet.Conn, error) { +func (ll *tcpListener) Accept() (manet.Conn, error) { c, err := ll.Listener.Accept() if err != nil { return nil, err @@ -153,7 +153,7 @@ func (t *TcpTransport) Listen(laddr ma.Multiaddr) (transport.Listener, error) { if err != nil { return nil, err } - list = &lingerListener{list, 0} + list = &tcpListener{list, 0} return t.Upgrader.UpgradeListener(t, list), nil } From 3cf9d581c9bf5bd144c32e62c0646f728627873e Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 27 Mar 2021 17:06:29 +0700 Subject: [PATCH 32/53] expose some Prometheus metrics --- p2p/transport/tcp/metrics.go | 176 ++++++++++++++++++++++++++++++++++ p2p/transport/tcp/tcp.go | 8 +- p2p/transport/tcp/tcp_conn.go | 39 ++++++++ 3 files changed, 221 insertions(+), 2 deletions(-) create mode 100644 p2p/transport/tcp/metrics.go create mode 100644 p2p/transport/tcp/tcp_conn.go diff --git a/p2p/transport/tcp/metrics.go b/p2p/transport/tcp/metrics.go new file mode 100644 index 0000000000..0150d51104 --- /dev/null +++ b/p2p/transport/tcp/metrics.go @@ -0,0 +1,176 @@ +package tcp + +import ( + "strings" + "sync" + "time" + + "github.com/mikioh/tcp" + "github.com/mikioh/tcpinfo" + manet "github.com/multiformats/go-multiaddr/net" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + newConns *prometheus.CounterVec + closedConns *prometheus.CounterVec +) + +var collector *aggregatingCollector + +func init() { + collector = newAggregatingCollector() + prometheus.MustRegister(collector) + + const direction = "direction" + + newConns = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "tcp_connections_new_total", + Help: "TCP new connections", + }, + []string{direction}, + ) + prometheus.MustRegister(newConns) + closedConns = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "tcp_connections_closed_total", + Help: "TCP connections closed", + }, + []string{direction}, + ) + prometheus.MustRegister(closedConns) +} + +type aggregatingCollector struct { + mutex sync.Mutex + + highestID uint64 + conns map[uint64] /* id */ *tracingConn + rtts prometheus.Histogram + connDurations prometheus.Histogram +} + +var _ prometheus.Collector = &aggregatingCollector{} + +func newAggregatingCollector() *aggregatingCollector { + return &aggregatingCollector{ + conns: make(map[uint64]*tracingConn), + rtts: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "tcp_rtt", + Help: "TCP round trip time", + Buckets: prometheus.ExponentialBuckets(0.001, 1.25, 40), // 1ms to ~6000ms + }), + connDurations: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "tcp_connection_duration", + Help: "TCP Connection Duration", + Buckets: prometheus.ExponentialBuckets(1, 1.5, 40), // 1s to ~12 weeks + }), + } +} + +func (c *aggregatingCollector) AddConn(t *tracingConn) uint64 { + c.mutex.Lock() + defer c.mutex.Unlock() + c.highestID++ + c.conns[c.highestID] = t + return c.highestID +} + +func (c *aggregatingCollector) removeConn(id uint64) { + delete(c.conns, id) +} + +func (c *aggregatingCollector) Describe(descs chan<- *prometheus.Desc) { + descs <- c.rtts.Desc() + descs <- c.connDurations.Desc() +} + +func (c *aggregatingCollector) Collect(metrics chan<- prometheus.Metric) { + now := time.Now() + c.mutex.Lock() + for _, conn := range c.conns { + info, err := conn.getTCPInfo() + if err != nil { + if strings.Contains(err.Error(), "use of closed network connection") { + c.closedConn(conn) + continue + } + log.Errorf("Failed to get TCP info: %s", err) + continue + } + c.rtts.Observe(info.RTT.Seconds()) + c.connDurations.Observe(now.Sub(conn.startTime).Seconds()) + if info.State == tcpinfo.Closed { + c.closedConn(conn) + } + } + c.mutex.Unlock() + metrics <- c.rtts + metrics <- c.connDurations +} + +func (c *aggregatingCollector) closedConn(conn *tracingConn) { + collector.removeConn(conn.id) + closedConns.WithLabelValues(conn.getDirection()).Inc() +} + +type tracingConn struct { + id uint64 + + startTime time.Time + isClient bool + + manet.Conn + tcpConn *tcp.Conn +} + +func newTracingConn(c manet.Conn, isClient bool) (*tracingConn, error) { + conn, err := newTCPConn(c) + if err != nil { + return nil, err + } + tc := &tracingConn{ + startTime: time.Now(), + isClient: isClient, + Conn: c, + tcpConn: conn, + } + tc.id = collector.AddConn(tc) + newConns.WithLabelValues(tc.getDirection()).Inc() + return tc, nil +} + +func (c *tracingConn) getDirection() string { + if c.isClient { + return "outgoing" + } + return "incoming" +} + +func (c *tracingConn) Close() error { + return c.Conn.Close() +} + +func (c *tracingConn) getTCPInfo() (*tcpinfo.Info, error) { + var o tcpinfo.Info + var b [256]byte + i, err := c.tcpConn.Option(o.Level(), o.Name(), b[:]) + if err != nil { + return nil, err + } + info := i.(*tcpinfo.Info) + return info, nil +} + +type tracingListener struct { + manet.Listener +} + +func (l *tracingListener) Accept() (manet.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + return newTracingConn(conn, false) +} diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 95c30dfcd8..3e17663708 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -132,7 +132,11 @@ func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) // This means we can immediately reuse the 5-tuple and reconnect. tryLinger(conn, 0) tryKeepAlive(conn, true) - return t.Upgrader.UpgradeOutbound(ctx, t, conn, p) + c, err := newTracingConn(conn, true) + if err != nil { + return nil, err + } + return t.Upgrader.UpgradeOutbound(ctx, t, c, p) } // UseReuseport returns true if reuseport is enabled and available. @@ -153,7 +157,7 @@ func (t *TcpTransport) Listen(laddr ma.Multiaddr) (transport.Listener, error) { if err != nil { return nil, err } - list = &tcpListener{list, 0} + list = &tracingListener{&tcpListener{list, 0}} return t.Upgrader.UpgradeListener(t, list), nil } diff --git a/p2p/transport/tcp/tcp_conn.go b/p2p/transport/tcp/tcp_conn.go new file mode 100644 index 0000000000..a163c2320c --- /dev/null +++ b/p2p/transport/tcp/tcp_conn.go @@ -0,0 +1,39 @@ +package tcp + +import ( + "errors" + "net" + "syscall" + "unsafe" + + "github.com/mikioh/tcp" +) + +// This is only needed because mikioh/tcp doesn't accept wrapped connections. +// See https://github.com/mikioh/tcp/pull/2. + +type tcpConn struct { + net.Conn + c syscall.RawConn +} + +// newTCPConn returns a new end point. +func newTCPConn(c net.Conn) (*tcp.Conn, error) { + type tcpConnI interface { + SyscallConn() (syscall.RawConn, error) + SetLinger(int) error + } + var _ tcpConnI = &net.TCPConn{} + cc := &tcpConn{Conn: c} + switch c := c.(type) { + case tcpConnI: + var err error + cc.c, err = c.SyscallConn() + if err != nil { + return nil, err + } + return (*tcp.Conn)(unsafe.Pointer(cc)), nil + default: + return nil, errors.New("unknown connection type") + } +} From 0006048d3721716d43fdea395f9550baeae094b3 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 6 Apr 2021 18:22:14 +0700 Subject: [PATCH 33/53] use a fork of mikioh/tcp --- p2p/transport/tcp/metrics.go | 4 ++-- p2p/transport/tcp/tcp_conn.go | 39 ----------------------------------- 2 files changed, 2 insertions(+), 41 deletions(-) delete mode 100644 p2p/transport/tcp/tcp_conn.go diff --git a/p2p/transport/tcp/metrics.go b/p2p/transport/tcp/metrics.go index 0150d51104..595da0cdc4 100644 --- a/p2p/transport/tcp/metrics.go +++ b/p2p/transport/tcp/metrics.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/mikioh/tcp" + "github.com/marten-seemann/tcp" "github.com/mikioh/tcpinfo" manet "github.com/multiformats/go-multiaddr/net" "github.com/prometheus/client_golang/prometheus" @@ -126,7 +126,7 @@ type tracingConn struct { } func newTracingConn(c manet.Conn, isClient bool) (*tracingConn, error) { - conn, err := newTCPConn(c) + conn, err := tcp.NewConn(c) if err != nil { return nil, err } diff --git a/p2p/transport/tcp/tcp_conn.go b/p2p/transport/tcp/tcp_conn.go deleted file mode 100644 index a163c2320c..0000000000 --- a/p2p/transport/tcp/tcp_conn.go +++ /dev/null @@ -1,39 +0,0 @@ -package tcp - -import ( - "errors" - "net" - "syscall" - "unsafe" - - "github.com/mikioh/tcp" -) - -// This is only needed because mikioh/tcp doesn't accept wrapped connections. -// See https://github.com/mikioh/tcp/pull/2. - -type tcpConn struct { - net.Conn - c syscall.RawConn -} - -// newTCPConn returns a new end point. -func newTCPConn(c net.Conn) (*tcp.Conn, error) { - type tcpConnI interface { - SyscallConn() (syscall.RawConn, error) - SetLinger(int) error - } - var _ tcpConnI = &net.TCPConn{} - cc := &tcpConn{Conn: c} - switch c := c.(type) { - case tcpConnI: - var err error - cc.c, err = c.SyscallConn() - if err != nil { - return nil, err - } - return (*tcp.Conn)(unsafe.Pointer(cc)), nil - default: - return nil, errors.New("unknown connection type") - } -} From 7fae0d63c6392b9055bdc8ade4e7fe8fd804a9ef Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 12 Apr 2021 21:43:41 +0700 Subject: [PATCH 34/53] add bandwidth-related metrics (for Linux and OSX) --- p2p/transport/tcp/metrics.go | 59 +++++++++++++++++++++++++++- p2p/transport/tcp/metrics_darwin.go | 15 +++++++ p2p/transport/tcp/metrics_general.go | 15 +++++++ p2p/transport/tcp/metrics_linux.go | 15 +++++++ 4 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 p2p/transport/tcp/metrics_darwin.go create mode 100644 p2p/transport/tcp/metrics_general.go create mode 100644 p2p/transport/tcp/metrics_linux.go diff --git a/p2p/transport/tcp/metrics.go b/p2p/transport/tcp/metrics.go index 595da0cdc4..4db7271c4c 100644 --- a/p2p/transport/tcp/metrics.go +++ b/p2p/transport/tcp/metrics.go @@ -12,13 +12,22 @@ import ( ) var ( - newConns *prometheus.CounterVec - closedConns *prometheus.CounterVec + newConns *prometheus.CounterVec + closedConns *prometheus.CounterVec + segsSentDesc *prometheus.Desc + segsRcvdDesc *prometheus.Desc + bytesSentDesc *prometheus.Desc + bytesRcvdDesc *prometheus.Desc ) var collector *aggregatingCollector func init() { + segsSentDesc = prometheus.NewDesc("tcp_sent_segments_total", "TCP segments sent", nil, nil) + segsRcvdDesc = prometheus.NewDesc("tcp_rcvd_segments_total", "TCP segments received", nil, nil) + bytesSentDesc = prometheus.NewDesc("tcp_sent_bytes", "TCP bytes sent", nil, nil) + bytesRcvdDesc = prometheus.NewDesc("tcp_rcvd_bytes", "TCP bytes received", nil, nil) + collector = newAggregatingCollector() prometheus.MustRegister(collector) @@ -84,11 +93,21 @@ func (c *aggregatingCollector) removeConn(id uint64) { func (c *aggregatingCollector) Describe(descs chan<- *prometheus.Desc) { descs <- c.rtts.Desc() descs <- c.connDurations.Desc() + if hasSegmentCounter { + descs <- segsSentDesc + descs <- segsRcvdDesc + } + if hasByteCounter { + descs <- bytesSentDesc + descs <- bytesRcvdDesc + } } func (c *aggregatingCollector) Collect(metrics chan<- prometheus.Metric) { now := time.Now() c.mutex.Lock() + var segsSent, segsRcvd uint64 + var bytesSent, bytesRcvd uint64 for _, conn := range c.conns { info, err := conn.getTCPInfo() if err != nil { @@ -99,6 +118,14 @@ func (c *aggregatingCollector) Collect(metrics chan<- prometheus.Metric) { log.Errorf("Failed to get TCP info: %s", err) continue } + if hasSegmentCounter { + segsSent += getSegmentsSent(info) + segsRcvd += getSegmentsRcvd(info) + } + if hasByteCounter { + bytesSent += getBytesSent(info) + bytesRcvd += getBytesRcvd(info) + } c.rtts.Observe(info.RTT.Seconds()) c.connDurations.Observe(now.Sub(conn.startTime).Seconds()) if info.State == tcpinfo.Closed { @@ -108,6 +135,34 @@ func (c *aggregatingCollector) Collect(metrics chan<- prometheus.Metric) { c.mutex.Unlock() metrics <- c.rtts metrics <- c.connDurations + if hasSegmentCounter { + segsSentMetric, err := prometheus.NewConstMetric(segsSentDesc, prometheus.CounterValue, float64(segsSent)) + if err != nil { + log.Errorf("creating tcp_sent_segments_total metric failed: %v", err) + return + } + segsRcvdMetric, err := prometheus.NewConstMetric(segsRcvdDesc, prometheus.CounterValue, float64(segsRcvd)) + if err != nil { + log.Errorf("creating tcp_rcvd_segments_total metric failed: %v", err) + return + } + metrics <- segsSentMetric + metrics <- segsRcvdMetric + } + if hasByteCounter { + bytesSentMetric, err := prometheus.NewConstMetric(bytesSentDesc, prometheus.CounterValue, float64(bytesSent)) + if err != nil { + log.Errorf("creating tcp_sent_bytes metric failed: %v", err) + return + } + bytesRcvdMetric, err := prometheus.NewConstMetric(bytesRcvdDesc, prometheus.CounterValue, float64(bytesRcvd)) + if err != nil { + log.Errorf("creating tcp_rcvd_bytes metric failed: %v", err) + return + } + metrics <- bytesSentMetric + metrics <- bytesRcvdMetric + } } func (c *aggregatingCollector) closedConn(conn *tracingConn) { diff --git a/p2p/transport/tcp/metrics_darwin.go b/p2p/transport/tcp/metrics_darwin.go new file mode 100644 index 0000000000..7edf378887 --- /dev/null +++ b/p2p/transport/tcp/metrics_darwin.go @@ -0,0 +1,15 @@ +//+build darwin + +package tcp + +import "github.com/mikioh/tcpinfo" + +const ( + hasSegmentCounter = true + hasByteCounter = true +) + +func getSegmentsSent(info *tcpinfo.Info) uint64 { return info.Sys.SegsSent } +func getSegmentsRcvd(info *tcpinfo.Info) uint64 { return info.Sys.SegsReceived } +func getBytesSent(info *tcpinfo.Info) uint64 { return info.Sys.BytesSent } +func getBytesRcvd(info *tcpinfo.Info) uint64 { return info.Sys.BytesReceived } diff --git a/p2p/transport/tcp/metrics_general.go b/p2p/transport/tcp/metrics_general.go new file mode 100644 index 0000000000..83096c8cca --- /dev/null +++ b/p2p/transport/tcp/metrics_general.go @@ -0,0 +1,15 @@ +//+build !linux,!darwin + +package tcp + +import "github.com/mikioh/tcpinfo" + +const ( + hasSegmentCounter = false + hasByteCounter = false +) + +func getSegmentsSent(info *tcpinfo.Info) uint64 { return 0 } +func getSegmentsRcvd(info *tcpinfo.Info) uint64 { return 0 } +func getBytesSent(info *tcpinfo.Info) uint64 { return 0 } +func getBytesRcvd(info *tcpinfo.Info) uint64 { return 0 } diff --git a/p2p/transport/tcp/metrics_linux.go b/p2p/transport/tcp/metrics_linux.go new file mode 100644 index 0000000000..b5dd11e280 --- /dev/null +++ b/p2p/transport/tcp/metrics_linux.go @@ -0,0 +1,15 @@ +//+build linux + +package tcp + +import "github.com/mikioh/tcpinfo" + +const ( + hasSegmentCounter = true + hasByteCounter = false +) + +func getSegmentsSent(info *tcpinfo.Info) uint64 { return uint64(info.Sys.SegsOut) } +func getSegmentsRcvd(info *tcpinfo.Info) uint64 { return uint64(info.Sys.SegsIn) } +func getBytesSent(info *tcpinfo.Info) uint64 { return 0 } +func getBytesRcvd(info *tcpinfo.Info) uint64 { return 0 } From ff00d9c72b1be0e4fbb39bba7d8d6bad06186adf Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 23 Apr 2021 10:49:32 +0700 Subject: [PATCH 35/53] use log.Warn instead of log.Warning --- p2p/transport/tcp/reuseport.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport/tcp/reuseport.go b/p2p/transport/tcp/reuseport.go index e3b7e1250f..83e13f62b5 100644 --- a/p2p/transport/tcp/reuseport.go +++ b/p2p/transport/tcp/reuseport.go @@ -23,7 +23,7 @@ func init() { } v, exist := os.LookupEnv(deprecatedEnvReuseport) if exist { - log.Warning("IPFS_REUSEPORT is deprecated, use LIBP2P_TCP_REUSEPORT instead") + log.Warn("IPFS_REUSEPORT is deprecated, use LIBP2P_TCP_REUSEPORT instead") if v == "false" || v == "f" || v == "0" { envReuseportVal = false log.Infof("REUSEPORT disabled (IPFS_REUSEPORT=%s)", v) From 4eb467a7243473a69ad8bb95b1812af316198c62 Mon Sep 17 00:00:00 2001 From: Aaron Bieber Date: Wed, 23 Jun 2021 07:31:05 -0600 Subject: [PATCH 36/53] Skip SetKeepAlivePeriod call on OpenBSD OpenBSD does not have a user-settable per-socket TCP keepalives. This changes tryKeepAlive to skip attempting set it when on OpenBSD. https://github.com/golang/go/blob/master/src/net/tcpsockopt_openbsd.go#L13-L14 This prevents thousands of `log.Errorf("Failed set keepalive period: %s", err)`'s from happening. --- p2p/transport/tcp/tcp.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 3e17663708..4dbc7260f2 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -3,6 +3,7 @@ package tcp import ( "context" "net" + "runtime" "time" logging "github.com/ipfs/go-log" @@ -41,8 +42,11 @@ func tryKeepAlive(conn net.Conn, keepAlive bool) { log.Errorf("Failed to enable TCP keepalive: %s", err) return } - if err := keepAliveConn.SetKeepAlivePeriod(keepAlivePeriod); err != nil { - log.Errorf("Failed set keepalive period: %s", err) + + if runtime.GOOS != "openbsd" { + if err := keepAliveConn.SetKeepAlivePeriod(keepAlivePeriod); err != nil { + log.Errorf("Failed set keepalive period: %s", err) + } } } From c4c5e0fcf126e26aafd17958a9955ac789cacde8 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 10 Jul 2021 11:32:51 -0500 Subject: [PATCH 37/53] collect metrics in a separate go routine --- p2p/transport/tcp/metrics.go | 88 ++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 35 deletions(-) diff --git a/p2p/transport/tcp/metrics.go b/p2p/transport/tcp/metrics.go index 4db7271c4c..69a83e6973 100644 --- a/p2p/transport/tcp/metrics.go +++ b/p2p/transport/tcp/metrics.go @@ -20,6 +20,8 @@ var ( bytesRcvdDesc *prometheus.Desc ) +const collectFrequency = 10 * time.Second + var collector *aggregatingCollector func init() { @@ -54,16 +56,18 @@ func init() { type aggregatingCollector struct { mutex sync.Mutex - highestID uint64 - conns map[uint64] /* id */ *tracingConn - rtts prometheus.Histogram - connDurations prometheus.Histogram + highestID uint64 + conns map[uint64] /* id */ *tracingConn + rtts prometheus.Histogram + connDurations prometheus.Histogram + segsSent, segsRcvd uint64 + bytesSent, bytesRcvd uint64 } var _ prometheus.Collector = &aggregatingCollector{} func newAggregatingCollector() *aggregatingCollector { - return &aggregatingCollector{ + c := &aggregatingCollector{ conns: make(map[uint64]*tracingConn), rtts: prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "tcp_rtt", @@ -76,6 +80,8 @@ func newAggregatingCollector() *aggregatingCollector { Buckets: prometheus.ExponentialBuckets(1, 1.5, 40), // 1s to ~12 weeks }), } + go c.cron() + return c } func (c *aggregatingCollector) AddConn(t *tracingConn) uint64 { @@ -103,45 +109,57 @@ func (c *aggregatingCollector) Describe(descs chan<- *prometheus.Desc) { } } -func (c *aggregatingCollector) Collect(metrics chan<- prometheus.Metric) { - now := time.Now() - c.mutex.Lock() - var segsSent, segsRcvd uint64 - var bytesSent, bytesRcvd uint64 - for _, conn := range c.conns { - info, err := conn.getTCPInfo() - if err != nil { - if strings.Contains(err.Error(), "use of closed network connection") { - c.closedConn(conn) +func (c *aggregatingCollector) cron() { + ticker := time.NewTicker(collectFrequency) + defer ticker.Stop() + + for now := range ticker.C { + c.mutex.Lock() + c.segsSent = 0 + c.segsRcvd = 0 + c.bytesSent = 0 + c.bytesRcvd = 0 + for _, conn := range c.conns { + info, err := conn.getTCPInfo() + if err != nil { + if strings.Contains(err.Error(), "use of closed network connection") { + c.closedConn(conn) + continue + } + log.Errorf("Failed to get TCP info: %s", err) continue } - log.Errorf("Failed to get TCP info: %s", err) - continue - } - if hasSegmentCounter { - segsSent += getSegmentsSent(info) - segsRcvd += getSegmentsRcvd(info) - } - if hasByteCounter { - bytesSent += getBytesSent(info) - bytesRcvd += getBytesRcvd(info) - } - c.rtts.Observe(info.RTT.Seconds()) - c.connDurations.Observe(now.Sub(conn.startTime).Seconds()) - if info.State == tcpinfo.Closed { - c.closedConn(conn) + if hasSegmentCounter { + c.segsSent += getSegmentsSent(info) + c.segsRcvd += getSegmentsRcvd(info) + } + if hasByteCounter { + c.bytesSent += getBytesSent(info) + c.bytesRcvd += getBytesRcvd(info) + } + c.rtts.Observe(info.RTT.Seconds()) + c.connDurations.Observe(now.Sub(conn.startTime).Seconds()) + if info.State == tcpinfo.Closed { + c.closedConn(conn) + } } + c.mutex.Unlock() } - c.mutex.Unlock() +} + +func (c *aggregatingCollector) Collect(metrics chan<- prometheus.Metric) { + c.mutex.Lock() + defer c.mutex.Unlock() + metrics <- c.rtts metrics <- c.connDurations if hasSegmentCounter { - segsSentMetric, err := prometheus.NewConstMetric(segsSentDesc, prometheus.CounterValue, float64(segsSent)) + segsSentMetric, err := prometheus.NewConstMetric(segsSentDesc, prometheus.CounterValue, float64(c.segsSent)) if err != nil { log.Errorf("creating tcp_sent_segments_total metric failed: %v", err) return } - segsRcvdMetric, err := prometheus.NewConstMetric(segsRcvdDesc, prometheus.CounterValue, float64(segsRcvd)) + segsRcvdMetric, err := prometheus.NewConstMetric(segsRcvdDesc, prometheus.CounterValue, float64(c.segsRcvd)) if err != nil { log.Errorf("creating tcp_rcvd_segments_total metric failed: %v", err) return @@ -150,12 +168,12 @@ func (c *aggregatingCollector) Collect(metrics chan<- prometheus.Metric) { metrics <- segsRcvdMetric } if hasByteCounter { - bytesSentMetric, err := prometheus.NewConstMetric(bytesSentDesc, prometheus.CounterValue, float64(bytesSent)) + bytesSentMetric, err := prometheus.NewConstMetric(bytesSentDesc, prometheus.CounterValue, float64(c.bytesSent)) if err != nil { log.Errorf("creating tcp_sent_bytes metric failed: %v", err) return } - bytesRcvdMetric, err := prometheus.NewConstMetric(bytesRcvdDesc, prometheus.CounterValue, float64(bytesRcvd)) + bytesRcvdMetric, err := prometheus.NewConstMetric(bytesRcvdDesc, prometheus.CounterValue, float64(c.bytesRcvd)) if err != nil { log.Errorf("creating tcp_rcvd_bytes metric failed: %v", err) return From 15dab89d35a3363cb14d0b8f89c42b14b5e7f3f9 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 14 Jul 2021 11:52:16 -0700 Subject: [PATCH 38/53] fix: avoid logging "invalid argument" errors when setting keepalive This can happen if, e.g., the connection is already dead. I think? See https://discuss.ipfs.io/t/tcp-keepalive-and-setsockopt-invalid-argument/11725 Really, I don't know _what_ was causing this, but there's nothing we can do about it. --- p2p/transport/tcp/tcp.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 4dbc7260f2..b870e9df76 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -2,7 +2,9 @@ package tcp import ( "context" + "errors" "net" + "os" "runtime" "time" @@ -39,13 +41,22 @@ func tryKeepAlive(conn net.Conn, keepAlive bool) { return } if err := keepAliveConn.SetKeepAlive(keepAlive); err != nil { - log.Errorf("Failed to enable TCP keepalive: %s", err) + // Sometimes we seem to get "invalid argument" results from this function on Darwin. + // This might be due to a closed connection, but I can't reproduce that on Linux. + // + // But there's nothing we can do about invalid arguments, so we'll drop this to a + // debug. + if errors.Is(err, os.ErrInvalid) { + log.Debugw("failed to enable TCP keepalive", "error", err) + } else { + log.Errorw("failed to enable TCP keepalive", "error", err) + } return } if runtime.GOOS != "openbsd" { if err := keepAliveConn.SetKeepAlivePeriod(keepAlivePeriod); err != nil { - log.Errorf("Failed set keepalive period: %s", err) + log.Errorw("failed set keepalive period", "error", err) } } } From 40bda4a141b73ff6c0b7e39346e2228f64e10d76 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 14 Jul 2021 19:15:36 -0400 Subject: [PATCH 39/53] use conn.Close() to remove closed connections from tracer --- p2p/transport/tcp/metrics.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/p2p/transport/tcp/metrics.go b/p2p/transport/tcp/metrics.go index 69a83e6973..ec3fbfd730 100644 --- a/p2p/transport/tcp/metrics.go +++ b/p2p/transport/tcp/metrics.go @@ -123,7 +123,6 @@ func (c *aggregatingCollector) cron() { info, err := conn.getTCPInfo() if err != nil { if strings.Contains(err.Error(), "use of closed network connection") { - c.closedConn(conn) continue } log.Errorf("Failed to get TCP info: %s", err) @@ -139,9 +138,6 @@ func (c *aggregatingCollector) cron() { } c.rtts.Observe(info.RTT.Seconds()) c.connDurations.Observe(now.Sub(conn.startTime).Seconds()) - if info.State == tcpinfo.Closed { - c.closedConn(conn) - } } c.mutex.Unlock() } @@ -183,9 +179,11 @@ func (c *aggregatingCollector) Collect(metrics chan<- prometheus.Metric) { } } -func (c *aggregatingCollector) closedConn(conn *tracingConn) { +func (c *aggregatingCollector) ClosedConn(conn *tracingConn, direction string) { + c.mutex.Lock() collector.removeConn(conn.id) - closedConns.WithLabelValues(conn.getDirection()).Inc() + c.mutex.Unlock() + closedConns.WithLabelValues(direction).Inc() } type tracingConn struct { @@ -222,6 +220,7 @@ func (c *tracingConn) getDirection() string { } func (c *tracingConn) Close() error { + collector.ClosedConn(c, c.getDirection()) return c.Conn.Close() } From e520e28c0985db4854aa222bf45954b35a9e6f78 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 14 Jul 2021 19:16:28 -0400 Subject: [PATCH 40/53] start collecting RTT and bandwidth metrics when Collect is calleed --- p2p/transport/tcp/metrics.go | 61 +++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/p2p/transport/tcp/metrics.go b/p2p/transport/tcp/metrics.go index ec3fbfd730..179978f288 100644 --- a/p2p/transport/tcp/metrics.go +++ b/p2p/transport/tcp/metrics.go @@ -54,8 +54,9 @@ func init() { } type aggregatingCollector struct { - mutex sync.Mutex + cronOnce sync.Once + mutex sync.Mutex highestID uint64 conns map[uint64] /* id */ *tracingConn rtts prometheus.Histogram @@ -80,7 +81,6 @@ func newAggregatingCollector() *aggregatingCollector { Buckets: prometheus.ExponentialBuckets(1, 1.5, 40), // 1s to ~12 weeks }), } - go c.cron() return c } @@ -114,36 +114,47 @@ func (c *aggregatingCollector) cron() { defer ticker.Stop() for now := range ticker.C { - c.mutex.Lock() - c.segsSent = 0 - c.segsRcvd = 0 - c.bytesSent = 0 - c.bytesRcvd = 0 - for _, conn := range c.conns { - info, err := conn.getTCPInfo() - if err != nil { - if strings.Contains(err.Error(), "use of closed network connection") { - continue - } - log.Errorf("Failed to get TCP info: %s", err) + c.gatherMetrics(now) + } +} + +func (c *aggregatingCollector) gatherMetrics(now time.Time) { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.segsSent = 0 + c.segsRcvd = 0 + c.bytesSent = 0 + c.bytesRcvd = 0 + for _, conn := range c.conns { + info, err := conn.getTCPInfo() + if err != nil { + if strings.Contains(err.Error(), "use of closed network connection") { continue } - if hasSegmentCounter { - c.segsSent += getSegmentsSent(info) - c.segsRcvd += getSegmentsRcvd(info) - } - if hasByteCounter { - c.bytesSent += getBytesSent(info) - c.bytesRcvd += getBytesRcvd(info) - } - c.rtts.Observe(info.RTT.Seconds()) - c.connDurations.Observe(now.Sub(conn.startTime).Seconds()) + log.Errorf("Failed to get TCP info: %s", err) + continue } - c.mutex.Unlock() + if hasSegmentCounter { + c.segsSent += getSegmentsSent(info) + c.segsRcvd += getSegmentsRcvd(info) + } + if hasByteCounter { + c.bytesSent += getBytesSent(info) + c.bytesRcvd += getBytesRcvd(info) + } + c.rtts.Observe(info.RTT.Seconds()) + c.connDurations.Observe(now.Sub(conn.startTime).Seconds()) } } func (c *aggregatingCollector) Collect(metrics chan<- prometheus.Metric) { + // Start collecting the metrics collection the first time Collect is called. + c.cronOnce.Do(func() { + c.gatherMetrics(time.Now()) + go c.cron() + }) + c.mutex.Lock() defer c.mutex.Unlock() From 302c1b5971a32c819f0ddef0c023a952c2092c74 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 24 Jul 2021 19:06:54 +0200 Subject: [PATCH 41/53] chore: update go-libp2p-transport-upgrader and go-reuseport-transport --- p2p/transport/tcp/tcp_test.go | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go index 8dcad45248..3d9d1ad197 100644 --- a/p2p/transport/tcp/tcp_test.go +++ b/p2p/transport/tcp/tcp_test.go @@ -1,23 +1,24 @@ package tcp import ( - "github.com/libp2p/go-libp2p-core/crypto" - "github.com/libp2p/go-libp2p-core/peer" "testing" + csms "github.com/libp2p/go-conn-security-multistream" + "github.com/libp2p/go-libp2p-core/crypto" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/sec" "github.com/libp2p/go-libp2p-core/sec/insecure" mplex "github.com/libp2p/go-libp2p-mplex" - tptu "github.com/libp2p/go-libp2p-transport-upgrader" - ttransport "github.com/libp2p/go-libp2p-testing/suites/transport" + tptu "github.com/libp2p/go-libp2p-transport-upgrader" ma "github.com/multiformats/go-multiaddr" ) func TestTcpTransport(t *testing.T) { for i := 0; i < 2; i++ { - ia := makeInsecureTransport(t) - ib := makeInsecureTransport(t) + peerA, ia := makeInsecureMuxer(t) + _, ib := makeInsecureMuxer(t) ta := NewTCPTransport(&tptu.Upgrader{ Secure: ia, @@ -29,7 +30,7 @@ func TestTcpTransport(t *testing.T) { }) zero := "/ip4/127.0.0.1/tcp/0" - ttransport.SubtestTransport(t, ta, tb, zero, ia.LocalPeer()) + ttransport.SubtestTransport(t, ta, tb, zero, peerA) envReuseportVal = false } @@ -43,8 +44,9 @@ func TestTcpTransportCantDialDNS(t *testing.T) { t.Fatal(err) } + _, sm := makeInsecureMuxer(t) tpt := NewTCPTransport(&tptu.Upgrader{ - Secure: makeInsecureTransport(t), + Secure: sm, Muxer: new(mplex.Transport), }) @@ -64,8 +66,9 @@ func TestTcpTransportCantListenUtp(t *testing.T) { t.Fatal(err) } + _, sm := makeInsecureMuxer(t) tpt := NewTCPTransport(&tptu.Upgrader{ - Secure: makeInsecureTransport(t), + Secure: sm, Muxer: new(mplex.Transport), }) @@ -79,14 +82,17 @@ func TestTcpTransportCantListenUtp(t *testing.T) { envReuseportVal = true } -func makeInsecureTransport(t *testing.T) *insecure.Transport { - priv, pub, err := crypto.GenerateKeyPair(crypto.Ed25519, 256) +func makeInsecureMuxer(t *testing.T) (peer.ID, sec.SecureMuxer) { + t.Helper() + priv, _, err := crypto.GenerateKeyPair(crypto.Ed25519, 256) if err != nil { t.Fatal(err) } - id, err := peer.IDFromPublicKey(pub) + id, err := peer.IDFromPrivateKey(priv) if err != nil { t.Fatal(err) } - return insecure.NewWithIdentity(id, priv) + var secMuxer csms.SSMuxer + secMuxer.AddTransport(insecure.ID, insecure.NewWithIdentity(id, priv)) + return id, &secMuxer } From 873a3c57c2aad248a7107c48bd2e10aedeb1e88d Mon Sep 17 00:00:00 2001 From: web3-bot <81333946+web3-bot@users.noreply.github.com> Date: Tue, 17 Aug 2021 10:47:00 -0400 Subject: [PATCH 42/53] sync: update CI config files (#90) --- p2p/transport/tcp/metrics_darwin.go | 3 ++- p2p/transport/tcp/metrics_general.go | 3 ++- p2p/transport/tcp/metrics_linux.go | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/p2p/transport/tcp/metrics_darwin.go b/p2p/transport/tcp/metrics_darwin.go index 7edf378887..f0015f4057 100644 --- a/p2p/transport/tcp/metrics_darwin.go +++ b/p2p/transport/tcp/metrics_darwin.go @@ -1,4 +1,5 @@ -//+build darwin +//go:build darwin +// +build darwin package tcp diff --git a/p2p/transport/tcp/metrics_general.go b/p2p/transport/tcp/metrics_general.go index 83096c8cca..bf84346281 100644 --- a/p2p/transport/tcp/metrics_general.go +++ b/p2p/transport/tcp/metrics_general.go @@ -1,4 +1,5 @@ -//+build !linux,!darwin +//go:build !linux && !darwin +// +build !linux,!darwin package tcp diff --git a/p2p/transport/tcp/metrics_linux.go b/p2p/transport/tcp/metrics_linux.go index b5dd11e280..046faa51c6 100644 --- a/p2p/transport/tcp/metrics_linux.go +++ b/p2p/transport/tcp/metrics_linux.go @@ -1,4 +1,5 @@ -//+build linux +//go:build linux +// +build linux package tcp From 8296ce34e4aa1625ac5020f18054c992ec8f85b3 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Tue, 31 Aug 2021 11:38:48 +0100 Subject: [PATCH 43/53] disable metrics collection on Windows --- p2p/transport/tcp/metrics.go | 7 +++++++ p2p/transport/tcp/metrics_general.go | 4 ++-- p2p/transport/tcp/metrics_windows.go | 9 +++++++++ p2p/transport/tcp/tcp.go | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 p2p/transport/tcp/metrics_windows.go diff --git a/p2p/transport/tcp/metrics.go b/p2p/transport/tcp/metrics.go index 179978f288..c3b6c7d9f4 100644 --- a/p2p/transport/tcp/metrics.go +++ b/p2p/transport/tcp/metrics.go @@ -1,3 +1,6 @@ +//go:build !windows +// +build !windows + package tcp import ( @@ -250,6 +253,10 @@ type tracingListener struct { manet.Listener } +func newTracingListener(l manet.Listener) *tracingListener { + return &tracingListener{Listener: l} +} + func (l *tracingListener) Accept() (manet.Conn, error) { conn, err := l.Listener.Accept() if err != nil { diff --git a/p2p/transport/tcp/metrics_general.go b/p2p/transport/tcp/metrics_general.go index bf84346281..36b3d648c8 100644 --- a/p2p/transport/tcp/metrics_general.go +++ b/p2p/transport/tcp/metrics_general.go @@ -1,5 +1,5 @@ -//go:build !linux && !darwin -// +build !linux,!darwin +//go:build !linux && !darwin && !windows +// +build !linux,!darwin,!windows package tcp diff --git a/p2p/transport/tcp/metrics_windows.go b/p2p/transport/tcp/metrics_windows.go new file mode 100644 index 0000000000..c80cf6fae4 --- /dev/null +++ b/p2p/transport/tcp/metrics_windows.go @@ -0,0 +1,9 @@ +//go:build windows +// +build windows + +package tcp + +import manet "github.com/multiformats/go-multiaddr/net" + +func newTracingConn(c manet.Conn, _ bool) (manet.Conn, error) { return c, nil } +func newTracingListener(l manet.Listener) manet.Listener { return l } diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index b870e9df76..558eb20fb1 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -172,7 +172,7 @@ func (t *TcpTransport) Listen(laddr ma.Multiaddr) (transport.Listener, error) { if err != nil { return nil, err } - list = &tracingListener{&tcpListener{list, 0}} + list = newTracingListener(&tcpListener{list, 0}) return t.Upgrader.UpgradeListener(t, list), nil } From a738e019b5109461e370270eee151f7abeff73e0 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 18 Sep 2021 16:36:27 +0200 Subject: [PATCH 44/53] simplify dial timeout context From the documentation: WithDeadline returns a copy of the parent context with the deadline adjusted to be no later than d. If the parent's deadline is already earlier than d, WithDeadline(parent, d) is semantically equivalent to parent. --- p2p/transport/tcp/tcp.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 558eb20fb1..5c9482517b 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -121,12 +121,9 @@ func (t *TcpTransport) CanDial(addr ma.Multiaddr) bool { func (t *TcpTransport) maDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) { // Apply the deadline iff applicable if t.ConnectTimeout > 0 { - deadline := time.Now().Add(t.ConnectTimeout) - if d, ok := ctx.Deadline(); !ok || deadline.Before(d) { - var cancel func() - ctx, cancel = context.WithDeadline(ctx, deadline) - defer cancel() - } + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, t.ConnectTimeout) + defer cancel() } if t.UseReuseport() { From 650b013faddef3754d4c4bc2b9a4830f932c0305 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 20 Sep 2021 15:53:32 +0100 Subject: [PATCH 45/53] chore: update go-log to v2 --- p2p/transport/tcp/tcp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 5c9482517b..2d68879021 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -8,7 +8,7 @@ import ( "runtime" "time" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" tptu "github.com/libp2p/go-libp2p-transport-upgrader" From e9c99b7e62018b80460f3dfa4c95617e53898b65 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 19 Sep 2021 09:13:47 +0100 Subject: [PATCH 46/53] use the assigned role when upgrading a sim open connection --- p2p/transport/tcp/tcp.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 2d68879021..d671e16c1f 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -8,12 +8,14 @@ import ( "runtime" "time" - logging "github.com/ipfs/go-log/v2" + "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" + tptu "github.com/libp2p/go-libp2p-transport-upgrader" rtpt "github.com/libp2p/go-reuseport-transport" + logging "github.com/ipfs/go-log/v2" ma "github.com/multiformats/go-multiaddr" mafmt "github.com/multiformats/go-multiaddr-fmt" manet "github.com/multiformats/go-multiaddr/net" @@ -148,7 +150,11 @@ func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) if err != nil { return nil, err } - return t.Upgrader.UpgradeOutbound(ctx, t, c, p) + direction := network.DirOutbound + if ok, isClient, _ := network.GetSimultaneousConnect(ctx); ok && !isClient { + direction = network.DirInbound + } + return t.Upgrader.Upgrade(ctx, t, c, direction, p) } // UseReuseport returns true if reuseport is enabled and available. From 79d3f044d00a297fd497f92e74ee4c8a01be218e Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 25 Sep 2021 11:20:49 +0100 Subject: [PATCH 47/53] use a config option to disable reuseport --- p2p/transport/tcp/tcp.go | 25 ++++++++++++++++++++----- p2p/transport/tcp/tcp_test.go | 34 +++++++++++++++------------------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index d671e16c1f..a8c6d76bb6 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -89,6 +89,15 @@ func (ll *tcpListener) Accept() (manet.Conn, error) { return c, nil } +type Option func(*TcpTransport) error + +func DisableReuseport() Option { + return func(tr *TcpTransport) error { + tr.disableReuseport = true + return nil + } +} + // TcpTransport is the TCP transport. type TcpTransport struct { // Connection upgrader for upgrading insecure stream connections to @@ -96,7 +105,7 @@ type TcpTransport struct { Upgrader *tptu.Upgrader // Explicitly disable reuseport. - DisableReuseport bool + disableReuseport bool // TCP connect timeout ConnectTimeout time.Duration @@ -107,9 +116,15 @@ type TcpTransport struct { var _ transport.Transport = &TcpTransport{} // NewTCPTransport creates a tcp transport object that tracks dialers and listeners -// created. It represents an entire tcp stack (though it might not necessarily be) -func NewTCPTransport(upgrader *tptu.Upgrader) *TcpTransport { - return &TcpTransport{Upgrader: upgrader, ConnectTimeout: DefaultConnectTimeout} +// created. It represents an entire TCP stack (though it might not necessarily be). +func NewTCPTransport(upgrader *tptu.Upgrader, opts ...Option) (*TcpTransport, error) { + tr := &TcpTransport{Upgrader: upgrader, ConnectTimeout: DefaultConnectTimeout} + for _, o := range opts { + if err := o(tr); err != nil { + return nil, err + } + } + return tr, nil } var dialMatcher = mafmt.And(mafmt.IP, mafmt.Base(ma.P_TCP)) @@ -159,7 +174,7 @@ func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) // UseReuseport returns true if reuseport is enabled and available. func (t *TcpTransport) UseReuseport() bool { - return !t.DisableReuseport && ReuseportIsAvailable() + return !t.disableReuseport && ReuseportIsAvailable() } func (t *TcpTransport) maListen(laddr ma.Multiaddr) (manet.Listener, error) { diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go index 3d9d1ad197..35b0aad7b7 100644 --- a/p2p/transport/tcp/tcp_test.go +++ b/p2p/transport/tcp/tcp_test.go @@ -13,6 +13,8 @@ import ( tptu "github.com/libp2p/go-libp2p-transport-upgrader" ma "github.com/multiformats/go-multiaddr" + + "github.com/stretchr/testify/require" ) func TestTcpTransport(t *testing.T) { @@ -20,14 +22,16 @@ func TestTcpTransport(t *testing.T) { peerA, ia := makeInsecureMuxer(t) _, ib := makeInsecureMuxer(t) - ta := NewTCPTransport(&tptu.Upgrader{ + ta, err := NewTCPTransport(&tptu.Upgrader{ Secure: ia, Muxer: new(mplex.Transport), }) - tb := NewTCPTransport(&tptu.Upgrader{ + require.NoError(t, err) + tb, err := NewTCPTransport(&tptu.Upgrader{ Secure: ib, Muxer: new(mplex.Transport), }) + require.NoError(t, err) zero := "/ip4/127.0.0.1/tcp/0" ttransport.SubtestTransport(t, ta, tb, zero, peerA) @@ -40,15 +44,14 @@ func TestTcpTransport(t *testing.T) { func TestTcpTransportCantDialDNS(t *testing.T) { for i := 0; i < 2; i++ { dnsa, err := ma.NewMultiaddr("/dns4/example.com/tcp/1234") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) _, sm := makeInsecureMuxer(t) - tpt := NewTCPTransport(&tptu.Upgrader{ + tpt, err := NewTCPTransport(&tptu.Upgrader{ Secure: sm, Muxer: new(mplex.Transport), }) + require.NoError(t, err) if tpt.CanDial(dnsa) { t.Fatal("shouldn't be able to dial dns") @@ -62,20 +65,17 @@ func TestTcpTransportCantDialDNS(t *testing.T) { func TestTcpTransportCantListenUtp(t *testing.T) { for i := 0; i < 2; i++ { utpa, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/0/utp") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) _, sm := makeInsecureMuxer(t) - tpt := NewTCPTransport(&tptu.Upgrader{ + tpt, err := NewTCPTransport(&tptu.Upgrader{ Secure: sm, Muxer: new(mplex.Transport), }) + require.NoError(t, err) _, err = tpt.Listen(utpa) - if err == nil { - t.Fatal("shouldnt be able to listen on utp addr with tcp transport") - } + require.Error(t, err, "shouldnt be able to listen on utp addr with tcp transport") envReuseportVal = false } @@ -85,13 +85,9 @@ func TestTcpTransportCantListenUtp(t *testing.T) { func makeInsecureMuxer(t *testing.T) (peer.ID, sec.SecureMuxer) { t.Helper() priv, _, err := crypto.GenerateKeyPair(crypto.Ed25519, 256) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) id, err := peer.IDFromPrivateKey(priv) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) var secMuxer csms.SSMuxer secMuxer.AddTransport(insecure.ID, insecure.NewWithIdentity(id, priv)) return id, &secMuxer From edff8962724f8fb14492f8196efe528083116061 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 25 Sep 2021 11:25:52 +0100 Subject: [PATCH 48/53] add an option for the TCP connection timeout --- p2p/transport/tcp/tcp.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index a8c6d76bb6..1405c6c200 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -21,9 +21,7 @@ import ( manet "github.com/multiformats/go-multiaddr/net" ) -// DefaultConnectTimeout is the (default) maximum amount of time the TCP -// transport will spend on the initial TCP connect before giving up. -var DefaultConnectTimeout = 5 * time.Second +const defaultConnectTimeout = 5 * time.Second var log = logging.Logger("tcp-tpt") @@ -97,6 +95,12 @@ func DisableReuseport() Option { return nil } } +func WithConnectionTimeout(d time.Duration) Option { + return func(tr *TcpTransport) error { + tr.connectTimeout = d + return nil + } +} // TcpTransport is the TCP transport. type TcpTransport struct { @@ -108,7 +112,7 @@ type TcpTransport struct { disableReuseport bool // TCP connect timeout - ConnectTimeout time.Duration + connectTimeout time.Duration reuse rtpt.Transport } @@ -118,7 +122,10 @@ var _ transport.Transport = &TcpTransport{} // NewTCPTransport creates a tcp transport object that tracks dialers and listeners // created. It represents an entire TCP stack (though it might not necessarily be). func NewTCPTransport(upgrader *tptu.Upgrader, opts ...Option) (*TcpTransport, error) { - tr := &TcpTransport{Upgrader: upgrader, ConnectTimeout: DefaultConnectTimeout} + tr := &TcpTransport{ + Upgrader: upgrader, + connectTimeout: defaultConnectTimeout, // can be set by using the WithConnectionTimeout option + } for _, o := range opts { if err := o(tr); err != nil { return nil, err @@ -137,9 +144,9 @@ func (t *TcpTransport) CanDial(addr ma.Multiaddr) bool { func (t *TcpTransport) maDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) { // Apply the deadline iff applicable - if t.ConnectTimeout > 0 { + if t.connectTimeout > 0 { var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, t.ConnectTimeout) + ctx, cancel = context.WithTimeout(ctx, t.connectTimeout) defer cancel() } From 7fa06d3cfe8ff1e858f526e524a0a9520e0fa612 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Mon, 27 Sep 2021 18:56:51 +0100 Subject: [PATCH 49/53] remove the deprecated IPFS_REUSEPORT command line flag --- p2p/transport/tcp/reuseport.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/p2p/transport/tcp/reuseport.go b/p2p/transport/tcp/reuseport.go index 83e13f62b5..c40d2ebe28 100644 --- a/p2p/transport/tcp/reuseport.go +++ b/p2p/transport/tcp/reuseport.go @@ -10,7 +10,6 @@ import ( // envReuseport is the env variable name used to turn off reuse port. // It default to true. const envReuseport = "LIBP2P_TCP_REUSEPORT" -const deprecatedEnvReuseport = "IPFS_REUSEPORT" // envReuseportVal stores the value of envReuseport. defaults to true. var envReuseportVal = true @@ -21,17 +20,9 @@ func init() { envReuseportVal = false log.Infof("REUSEPORT disabled (LIBP2P_TCP_REUSEPORT=%s)", v) } - v, exist := os.LookupEnv(deprecatedEnvReuseport) - if exist { - log.Warn("IPFS_REUSEPORT is deprecated, use LIBP2P_TCP_REUSEPORT instead") - if v == "false" || v == "f" || v == "0" { - envReuseportVal = false - log.Infof("REUSEPORT disabled (IPFS_REUSEPORT=%s)", v) - } - } } -// reuseportIsAvailable returns whether reuseport is available to be used. This +// ReuseportIsAvailable returns whether reuseport is available to be used. This // is here because we want to be able to turn reuseport on and off selectively. // For now we use an ENV variable, as this handles our pressing need: // From c747de37d3c96f7fa50844cc5b6297f88c33fa08 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sun, 2 Jan 2022 15:58:57 +0400 Subject: [PATCH 50/53] use the transport.Upgrader interface --- p2p/transport/tcp/tcp.go | 5 ++--- p2p/transport/tcp/tcp_test.go | 34 ++++++++++++++-------------------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index 1405c6c200..d21dcb0793 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -12,7 +12,6 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" - tptu "github.com/libp2p/go-libp2p-transport-upgrader" rtpt "github.com/libp2p/go-reuseport-transport" logging "github.com/ipfs/go-log/v2" @@ -106,7 +105,7 @@ func WithConnectionTimeout(d time.Duration) Option { type TcpTransport struct { // Connection upgrader for upgrading insecure stream connections to // secure multiplex connections. - Upgrader *tptu.Upgrader + Upgrader transport.Upgrader // Explicitly disable reuseport. disableReuseport bool @@ -121,7 +120,7 @@ var _ transport.Transport = &TcpTransport{} // NewTCPTransport creates a tcp transport object that tracks dialers and listeners // created. It represents an entire TCP stack (though it might not necessarily be). -func NewTCPTransport(upgrader *tptu.Upgrader, opts ...Option) (*TcpTransport, error) { +func NewTCPTransport(upgrader transport.Upgrader, opts ...Option) (*TcpTransport, error) { tr := &TcpTransport{ Upgrader: upgrader, connectTimeout: defaultConnectTimeout, // can be set by using the WithConnectionTimeout option diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go index 35b0aad7b7..21a12bd0fd 100644 --- a/p2p/transport/tcp/tcp_test.go +++ b/p2p/transport/tcp/tcp_test.go @@ -3,11 +3,13 @@ package tcp import ( "testing" - csms "github.com/libp2p/go-conn-security-multistream" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/sec" "github.com/libp2p/go-libp2p-core/sec/insecure" + "github.com/libp2p/go-libp2p-core/transport" + + csms "github.com/libp2p/go-conn-security-multistream" mplex "github.com/libp2p/go-libp2p-mplex" ttransport "github.com/libp2p/go-libp2p-testing/suites/transport" tptu "github.com/libp2p/go-libp2p-transport-upgrader" @@ -22,15 +24,13 @@ func TestTcpTransport(t *testing.T) { peerA, ia := makeInsecureMuxer(t) _, ib := makeInsecureMuxer(t) - ta, err := NewTCPTransport(&tptu.Upgrader{ - Secure: ia, - Muxer: new(mplex.Transport), - }) + ua, err := tptu.New(ia, new(mplex.Transport)) + require.NoError(t, err) + ta, err := NewTCPTransport(ua) + require.NoError(t, err) + ub, err := tptu.New(ib, new(mplex.Transport)) require.NoError(t, err) - tb, err := NewTCPTransport(&tptu.Upgrader{ - Secure: ib, - Muxer: new(mplex.Transport), - }) + tb, err := NewTCPTransport(ub) require.NoError(t, err) zero := "/ip4/127.0.0.1/tcp/0" @@ -46,11 +46,8 @@ func TestTcpTransportCantDialDNS(t *testing.T) { dnsa, err := ma.NewMultiaddr("/dns4/example.com/tcp/1234") require.NoError(t, err) - _, sm := makeInsecureMuxer(t) - tpt, err := NewTCPTransport(&tptu.Upgrader{ - Secure: sm, - Muxer: new(mplex.Transport), - }) + var u transport.Upgrader + tpt, err := NewTCPTransport(u) require.NoError(t, err) if tpt.CanDial(dnsa) { @@ -67,15 +64,12 @@ func TestTcpTransportCantListenUtp(t *testing.T) { utpa, err := ma.NewMultiaddr("/ip4/127.0.0.1/udp/0/utp") require.NoError(t, err) - _, sm := makeInsecureMuxer(t) - tpt, err := NewTCPTransport(&tptu.Upgrader{ - Secure: sm, - Muxer: new(mplex.Transport), - }) + var u transport.Upgrader + tpt, err := NewTCPTransport(u) require.NoError(t, err) _, err = tpt.Listen(utpa) - require.Error(t, err, "shouldnt be able to listen on utp addr with tcp transport") + require.Error(t, err, "shouldn't be able to listen on utp addr with tcp transport") envReuseportVal = false } From 7d5a5f83570faf2439a9db672e50d2150a5b0e39 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Wed, 22 Dec 2021 15:11:32 +0400 Subject: [PATCH 51/53] use the ResourceManager --- p2p/transport/tcp/tcp.go | 27 +++++++++++++-- p2p/transport/tcp/tcp_test.go | 64 ++++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index d21dcb0793..e9de3b345e 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -83,6 +83,10 @@ func (ll *tcpListener) Accept() (manet.Conn, error) { } tryLinger(c, ll.sec) tryKeepAlive(c, true) + // We're not calling OpenConnection in the resource manager here, + // since the manet.Conn doesn't allow us to save the scope. + // It's the caller's (usually the go-libp2p-transport-upgrader) responsibility + // to call the resource manager. return c, nil } @@ -94,6 +98,7 @@ func DisableReuseport() Option { return nil } } + func WithConnectionTimeout(d time.Duration) Option { return func(tr *TcpTransport) error { tr.connectTimeout = d @@ -113,6 +118,8 @@ type TcpTransport struct { // TCP connect timeout connectTimeout time.Duration + rcmgr network.ResourceManager + reuse rtpt.Transport } @@ -120,10 +127,14 @@ var _ transport.Transport = &TcpTransport{} // NewTCPTransport creates a tcp transport object that tracks dialers and listeners // created. It represents an entire TCP stack (though it might not necessarily be). -func NewTCPTransport(upgrader transport.Upgrader, opts ...Option) (*TcpTransport, error) { +func NewTCPTransport(upgrader transport.Upgrader, rcmgr network.ResourceManager, opts ...Option) (*TcpTransport, error) { + if rcmgr == nil { + rcmgr = network.NullResourceManager + } tr := &TcpTransport{ Upgrader: upgrader, connectTimeout: defaultConnectTimeout, // can be set by using the WithConnectionTimeout option + rcmgr: rcmgr, } for _, o := range opts { if err := o(tr); err != nil { @@ -158,8 +169,19 @@ func (t *TcpTransport) maDial(ctx context.Context, raddr ma.Multiaddr) (manet.Co // Dial dials the peer at the remote address. func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) (transport.CapableConn, error) { + connScope, err := t.rcmgr.OpenConnection(network.DirOutbound, true) + if err != nil { + log.Debugw("resource manager blocked outgoing connection", "peer", p, "addr", raddr, "error", err) + return nil, err + } + if err := connScope.SetPeer(p); err != nil { + log.Debugw("resource manager blocked outgoing connection for peer", "peer", p, "addr", raddr, "error", err) + connScope.Done() + return nil, err + } conn, err := t.maDial(ctx, raddr) if err != nil { + connScope.Done() return nil, err } // Set linger to 0 so we never get stuck in the TIME-WAIT state. When @@ -169,13 +191,14 @@ func (t *TcpTransport) Dial(ctx context.Context, raddr ma.Multiaddr, p peer.ID) tryKeepAlive(conn, true) c, err := newTracingConn(conn, true) if err != nil { + connScope.Done() return nil, err } direction := network.DirOutbound if ok, isClient, _ := network.GetSimultaneousConnect(ctx); ok && !isClient { direction = network.DirInbound } - return t.Upgrader.Upgrade(ctx, t, c, direction, p) + return t.Upgrader.Upgrade(ctx, t, c, direction, p, connScope) } // UseReuseport returns true if reuseport is enabled and available. diff --git a/p2p/transport/tcp/tcp_test.go b/p2p/transport/tcp/tcp_test.go index 21a12bd0fd..b650e9cbfa 100644 --- a/p2p/transport/tcp/tcp_test.go +++ b/p2p/transport/tcp/tcp_test.go @@ -1,8 +1,12 @@ package tcp import ( + "context" + "errors" "testing" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/sec" @@ -11,11 +15,13 @@ import ( csms "github.com/libp2p/go-conn-security-multistream" mplex "github.com/libp2p/go-libp2p-mplex" + mocknetwork "github.com/libp2p/go-libp2p-testing/mocks/network" ttransport "github.com/libp2p/go-libp2p-testing/suites/transport" tptu "github.com/libp2p/go-libp2p-transport-upgrader" ma "github.com/multiformats/go-multiaddr" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" ) @@ -26,11 +32,11 @@ func TestTcpTransport(t *testing.T) { ua, err := tptu.New(ia, new(mplex.Transport)) require.NoError(t, err) - ta, err := NewTCPTransport(ua) + ta, err := NewTCPTransport(ua, nil) require.NoError(t, err) ub, err := tptu.New(ib, new(mplex.Transport)) require.NoError(t, err) - tb, err := NewTCPTransport(ub) + tb, err := NewTCPTransport(ub, nil) require.NoError(t, err) zero := "/ip4/127.0.0.1/tcp/0" @@ -41,13 +47,63 @@ func TestTcpTransport(t *testing.T) { envReuseportVal = true } +func TestResourceManager(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + peerA, ia := makeInsecureMuxer(t) + _, ib := makeInsecureMuxer(t) + + ua, err := tptu.New(ia, new(mplex.Transport)) + require.NoError(t, err) + ta, err := NewTCPTransport(ua, nil) + require.NoError(t, err) + ln, err := ta.Listen(ma.StringCast("/ip4/127.0.0.1/tcp/0")) + require.NoError(t, err) + defer ln.Close() + + ub, err := tptu.New(ib, new(mplex.Transport)) + require.NoError(t, err) + rcmgr := mocknetwork.NewMockResourceManager(ctrl) + tb, err := NewTCPTransport(ub, rcmgr) + require.NoError(t, err) + + t.Run("success", func(t *testing.T) { + scope := mocknetwork.NewMockConnManagementScope(ctrl) + rcmgr.EXPECT().OpenConnection(network.DirOutbound, true).Return(scope, nil) + scope.EXPECT().SetPeer(peerA) + scope.EXPECT().PeerScope().Return(network.NullScope).AnyTimes() // called by the upgrader + conn, err := tb.Dial(context.Background(), ln.Multiaddr(), peerA) + require.NoError(t, err) + scope.EXPECT().Done() + defer conn.Close() + }) + + t.Run("connection denied", func(t *testing.T) { + rerr := errors.New("nope") + rcmgr.EXPECT().OpenConnection(network.DirOutbound, true).Return(nil, rerr) + _, err = tb.Dial(context.Background(), ln.Multiaddr(), peerA) + require.ErrorIs(t, err, rerr) + }) + + t.Run("peer denied", func(t *testing.T) { + scope := mocknetwork.NewMockConnManagementScope(ctrl) + rcmgr.EXPECT().OpenConnection(network.DirOutbound, true).Return(scope, nil) + rerr := errors.New("nope") + scope.EXPECT().SetPeer(peerA).Return(rerr) + scope.EXPECT().Done() + _, err = tb.Dial(context.Background(), ln.Multiaddr(), peerA) + require.ErrorIs(t, err, rerr) + }) +} + func TestTcpTransportCantDialDNS(t *testing.T) { for i := 0; i < 2; i++ { dnsa, err := ma.NewMultiaddr("/dns4/example.com/tcp/1234") require.NoError(t, err) var u transport.Upgrader - tpt, err := NewTCPTransport(u) + tpt, err := NewTCPTransport(u, nil) require.NoError(t, err) if tpt.CanDial(dnsa) { @@ -65,7 +121,7 @@ func TestTcpTransportCantListenUtp(t *testing.T) { require.NoError(t, err) var u transport.Upgrader - tpt, err := NewTCPTransport(u) + tpt, err := NewTCPTransport(u, nil) require.NoError(t, err) _, err = tpt.Listen(utpa) From 9d3f10b9c97ca89b1a04b1027f84c0b886d79bb3 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sat, 5 Feb 2022 22:37:18 -0800 Subject: [PATCH 52/53] fix: drop raw EINVAL (from keepalives) errors as well (#115) It _looks_ like the standard library doesn't always wrap this error. fixes #113 --- p2p/transport/tcp/tcp.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p2p/transport/tcp/tcp.go b/p2p/transport/tcp/tcp.go index e9de3b345e..494d955e6c 100644 --- a/p2p/transport/tcp/tcp.go +++ b/p2p/transport/tcp/tcp.go @@ -6,6 +6,7 @@ import ( "net" "os" "runtime" + "syscall" "time" "github.com/libp2p/go-libp2p-core/network" @@ -45,7 +46,7 @@ func tryKeepAlive(conn net.Conn, keepAlive bool) { // // But there's nothing we can do about invalid arguments, so we'll drop this to a // debug. - if errors.Is(err, os.ErrInvalid) { + if errors.Is(err, os.ErrInvalid) || errors.Is(err, syscall.EINVAL) { log.Debugw("failed to enable TCP keepalive", "error", err) } else { log.Errorw("failed to enable TCP keepalive", "error", err) From dfc3a3f24a69e9b4276c3e30b2ea6c7d28d9ba5c Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 22 Apr 2022 14:43:07 +0100 Subject: [PATCH 53/53] switch from github.com/libp2p/go-tcp-transport to p2p/transport/tcp --- config/transport_test.go | 3 ++- defaults.go | 6 ++++-- go.mod | 14 ++++++++------ libp2p_test.go | 6 ++++-- p2p/net/mock/log2.txt | 0 p2p/net/swarm/dial_worker_test.go | 8 +++++--- p2p/net/swarm/testing/testing.go | 2 +- p2p/protocol/circuitv2/relay/relay_test.go | 2 +- 8 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 p2p/net/mock/log2.txt diff --git a/config/transport_test.go b/config/transport_test.go index 1a226074c7..9cb5c9a3e0 100644 --- a/config/transport_test.go +++ b/config/transport_test.go @@ -3,9 +3,10 @@ package config import ( "testing" + "github.com/libp2p/go-libp2p/p2p/transport/tcp" + "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" - "github.com/libp2p/go-tcp-transport" "github.com/stretchr/testify/require" ) diff --git a/defaults.go b/defaults.go index 039cef4856..a3a4c13236 100644 --- a/defaults.go +++ b/defaults.go @@ -5,15 +5,17 @@ package libp2p import ( "crypto/rand" + "github.com/libp2p/go-libp2p/p2p/net/connmgr" + "github.com/libp2p/go-libp2p/p2p/transport/tcp" + "github.com/libp2p/go-libp2p-core/crypto" + noise "github.com/libp2p/go-libp2p-noise" "github.com/libp2p/go-libp2p-peerstore/pstoremem" quic "github.com/libp2p/go-libp2p-quic-transport" rcmgr "github.com/libp2p/go-libp2p-resource-manager" tls "github.com/libp2p/go-libp2p-tls" yamux "github.com/libp2p/go-libp2p-yamux" - "github.com/libp2p/go-libp2p/p2p/net/connmgr" - "github.com/libp2p/go-tcp-transport" ws "github.com/libp2p/go-ws-transport" "github.com/multiformats/go-multiaddr" ) diff --git a/go.mod b/go.mod index 4dcf718c66..b1f33b45fb 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/libp2p/go-libp2p-asn-util v0.1.0 github.com/libp2p/go-libp2p-circuit v0.6.0 github.com/libp2p/go-libp2p-core v0.15.1 + github.com/libp2p/go-libp2p-mplex v0.5.0 github.com/libp2p/go-libp2p-nat v0.1.0 github.com/libp2p/go-libp2p-noise v0.4.0 github.com/libp2p/go-libp2p-peerstore v0.6.0 @@ -28,16 +29,20 @@ require ( github.com/libp2p/go-libp2p-yamux v0.9.1 github.com/libp2p/go-msgio v0.2.0 github.com/libp2p/go-netroute v0.2.0 + github.com/libp2p/go-reuseport v0.1.0 + github.com/libp2p/go-reuseport-transport v0.1.0 github.com/libp2p/go-stream-muxer-multistream v0.4.0 - github.com/libp2p/go-tcp-transport v0.5.1 github.com/libp2p/go-ws-transport v0.6.0 github.com/libp2p/zeroconf/v2 v2.1.1 + github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd + github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b github.com/multiformats/go-multiaddr v0.5.0 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multiaddr-fmt v0.1.0 github.com/multiformats/go-multihash v0.1.0 github.com/multiformats/go-multistream v0.3.0 github.com/multiformats/go-varint v0.0.6 + github.com/prometheus/client_golang v1.12.1 github.com/raulk/go-watchdog v1.2.0 github.com/stretchr/testify v1.7.0 github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 @@ -78,20 +83,18 @@ require ( github.com/libp2p/go-libp2p-blankhost v0.3.0 // indirect github.com/libp2p/go-libp2p-pnet v0.2.0 // indirect github.com/libp2p/go-libp2p-swarm v0.10.2 // indirect + github.com/libp2p/go-mplex v0.4.0 // indirect github.com/libp2p/go-nat v0.1.0 // indirect github.com/libp2p/go-openssl v0.0.7 // indirect - github.com/libp2p/go-reuseport v0.1.0 // indirect - github.com/libp2p/go-reuseport-transport v0.1.0 // indirect + github.com/libp2p/go-tcp-transport v0.5.1 // indirect github.com/libp2p/go-yamux/v3 v3.1.1 // indirect github.com/lucas-clemente/quic-go v0.27.0 // indirect github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect - github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/miekg/dns v1.1.48 // indirect - github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/minio/sha256-simd v1.0.0 // indirect @@ -106,7 +109,6 @@ require ( github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.12.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.33.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect diff --git a/libp2p_test.go b/libp2p_test.go index a40360e6d0..78a6e5685c 100644 --- a/libp2p_test.go +++ b/libp2p_test.go @@ -8,15 +8,17 @@ import ( "sync" "testing" + "github.com/libp2p/go-libp2p/p2p/transport/tcp" + "github.com/libp2p/go-libp2p-core/connmgr" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/transport" + noise "github.com/libp2p/go-libp2p-noise" - "github.com/libp2p/go-tcp-transport" - ma "github.com/multiformats/go-multiaddr" + ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" ) diff --git a/p2p/net/mock/log2.txt b/p2p/net/mock/log2.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/p2p/net/swarm/dial_worker_test.go b/p2p/net/swarm/dial_worker_test.go index d9aa115e2f..3d74d62687 100644 --- a/p2p/net/swarm/dial_worker_test.go +++ b/p2p/net/swarm/dial_worker_test.go @@ -8,20 +8,22 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" + "github.com/libp2p/go-libp2p/p2p/transport/tcp" - csms "github.com/libp2p/go-conn-security-multistream" "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/sec/insecure" "github.com/libp2p/go-libp2p-core/transport" + + csms "github.com/libp2p/go-conn-security-multistream" "github.com/libp2p/go-libp2p-peerstore/pstoremem" quic "github.com/libp2p/go-libp2p-quic-transport" tnet "github.com/libp2p/go-libp2p-testing/net" tptu "github.com/libp2p/go-libp2p-transport-upgrader" yamux "github.com/libp2p/go-libp2p-yamux" msmux "github.com/libp2p/go-stream-muxer-multistream" - tcp "github.com/libp2p/go-tcp-transport" ma "github.com/multiformats/go-multiaddr" + + "github.com/stretchr/testify/require" ) func makeSwarm(t *testing.T) *Swarm { diff --git a/p2p/net/swarm/testing/testing.go b/p2p/net/swarm/testing/testing.go index 30d8f22055..b6289182c3 100644 --- a/p2p/net/swarm/testing/testing.go +++ b/p2p/net/swarm/testing/testing.go @@ -5,6 +5,7 @@ import ( "time" "github.com/libp2p/go-libp2p/p2p/net/swarm" + "github.com/libp2p/go-libp2p/p2p/transport/tcp" "github.com/libp2p/go-libp2p-core/connmgr" "github.com/libp2p/go-libp2p-core/control" @@ -15,7 +16,6 @@ import ( "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/sec/insecure" "github.com/libp2p/go-libp2p-core/transport" - "github.com/libp2p/go-tcp-transport" csms "github.com/libp2p/go-conn-security-multistream" "github.com/libp2p/go-libp2p-peerstore/pstoremem" diff --git a/p2p/protocol/circuitv2/relay/relay_test.go b/p2p/protocol/circuitv2/relay/relay_test.go index 5bd840406c..b14271e556 100644 --- a/p2p/protocol/circuitv2/relay/relay_test.go +++ b/p2p/protocol/circuitv2/relay/relay_test.go @@ -14,6 +14,7 @@ import ( swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/client" "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay" + "github.com/libp2p/go-libp2p/p2p/transport/tcp" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" @@ -23,7 +24,6 @@ import ( "github.com/libp2p/go-libp2p-core/transport" "github.com/libp2p/go-libp2p-peerstore/pstoremem" - "github.com/libp2p/go-tcp-transport" ma "github.com/multiformats/go-multiaddr" )