Merge branch 'master' into marco/transport-test-increase-linearly

README.md

@@ -18,6 +18,7 @@
 - [Roadmap](#roadmap)
 - [Usage](#usage)
   - [Examples](#examples)
+  - [Dashboards](#dashboards)
 - [Contribute](#contribute)
   - [Supported Go Versions](#supported-go-versions)
 - [Notable Users](#notable-users)
@@ -54,6 +55,13 @@ import "github.com/libp2p/go-libp2p"
 
 Examples can be found in the [examples folder](examples).
 
+## Dashboards
+
+We provide prebuilt Grafana dashboards so that applications can better monitor libp2p in production.
+You can find the [dashboard JSON files here](https://github.com/libp2p/go-libp2p/tree/master/dashboards).
+
+We also have live [Public Dashboards](https://github.com/libp2p/go-libp2p/tree/master/dashboards/README.md#public-dashboards) that you can check out to see real time monitoring in action.
+
 
 # Contribute
 

dashboards/README.md

@@ -3,6 +3,17 @@
 This directory contains prebuilt dashboards (provided as JSON files) for various components.
 For steps on how to import and use them [please read the official Grafana documentation.](https://grafana.com/docs/grafana/latest/dashboards/export-import/#import-dashboard)
 
+## Public dashboards
+
+You can check the following prebuilt dashboards in action:
+
+1. [AutoNAT](https://protocollabs.grafana.net/public-dashboards/fce8fdeb629742c89bd70f0ce38dfd97)
+2. [Auto Relay](https://protocollabs.grafana.net/public-dashboards/380d52aded12404e9cf6ceccb824b7f9)
+3. [Eventbus](https://protocollabs.grafana.net/public-dashboards/048029ac2d7e4a71b281ffea3535026e)
+4. [Identify](https://protocollabs.grafana.net/public-dashboards/96b70328253d47c0b352dfae06f12a1b)
+5. [Relay Service](https://protocollabs.grafana.net/public-dashboards/4a8cb5d245294893874ed65279b049be)
+6. [Swarm](https://protocollabs.grafana.net/public-dashboards/2bd3f1bee9964d40b6786fbe3eafd9fc)
+
 ## Using locally
 
 For local development and debugging, it can be useful to spin up a local Prometheus and Grafana instance.

defaults.go

@@ -79,11 +79,9 @@ var RandomIdentity = func(cfg *Config) error {
 var DefaultListenAddrs = func(cfg *Config) error {
 	addrs := []string{
 		"/ip4/0.0.0.0/tcp/0",
-		"/ip4/0.0.0.0/udp/0/quic",
 		"/ip4/0.0.0.0/udp/0/quic-v1",
 		"/ip4/0.0.0.0/udp/0/quic-v1/webtransport",
 		"/ip6/::/tcp/0",
-		"/ip6/::/udp/0/quic",
 		"/ip6/::/udp/0/quic-v1",
 		"/ip6/::/udp/0/quic-v1/webtransport",
 	}

go.mod

@@ -12,7 +12,8 @@ require (
 	github.com/golang/mock v1.6.0
 	github.com/google/gopacket v1.1.19
 	github.com/gorilla/websocket v1.5.0
-	github.com/hashicorp/golang-lru/v2 v2.0.2
+	github.com/hashicorp/golang-lru/arc/v2 v2.0.4
+	github.com/hashicorp/golang-lru/v2 v2.0.4
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-datastore v0.6.0
 	github.com/ipfs/go-ds-badger v0.3.0
@@ -36,7 +37,7 @@ require (
 	github.com/minio/sha256-simd v1.0.1
 	github.com/mr-tron/base58 v1.2.0
 	github.com/multiformats/go-base32 v0.1.0
-	github.com/multiformats/go-multiaddr v0.10.1
+	github.com/multiformats/go-multiaddr v0.11.0
 	github.com/multiformats/go-multiaddr-dns v0.3.1
 	github.com/multiformats/go-multiaddr-fmt v0.1.0
 	github.com/multiformats/go-multibase v0.2.0
@@ -54,7 +55,7 @@ require (
 	go.uber.org/fx v1.20.0
 	go.uber.org/goleak v1.1.12
 	golang.org/x/crypto v0.11.0
-	golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1
+	golang.org/x/exp v0.0.0-20230725012225-302865e7556b
 	golang.org/x/sync v0.3.0
 	golang.org/x/sys v0.10.0
 	golang.org/x/tools v0.11.0

go.sum

@@ -227,8 +227,10 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU=
-github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/hashicorp/golang-lru/arc/v2 v2.0.4 h1:+tHnVSaabYlClRqUq4/+xzeyy9nAf8ju/JJsb4KTNBc=
+github.com/hashicorp/golang-lru/arc/v2 v2.0.4/go.mod h1:rbQ1sKlUmbE1QbWxZbqtbpw8frA8ecNEhI0cQBxYtaU=
+github.com/hashicorp/golang-lru/v2 v2.0.4 h1:7GHuZcgid37q8o5i3QI9KMT4nCWQQ3Kx3Ov6bb9MfK0=
+github.com/hashicorp/golang-lru/v2 v2.0.4/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY=
@@ -353,8 +355,8 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9
 github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
 github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
 github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4=
-github.com/multiformats/go-multiaddr v0.10.1 h1:HghtFrWyZEPrpTvgAMFJi6gFdgHfs2cb0pyfDsk+lqU=
-github.com/multiformats/go-multiaddr v0.10.1/go.mod h1:jLEZsA61rwWNZQTHHnqq2HNa+4os/Hz54eqiRnsRqYQ=
+github.com/multiformats/go-multiaddr v0.11.0 h1:XqGyJ8ufbCE0HmTDwx2kPdsrQ36AGPZNZX6s6xfJH10=
+github.com/multiformats/go-multiaddr v0.11.0/go.mod h1:gWUm0QLR4thQ6+ZF6SXUw8YjtwQSPapICM+NmCkxHSM=
 github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A=
 github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk=
 github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E=
@@ -559,8 +561,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw=
-golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/exp v0.0.0-20230725012225-302865e7556b h1:tK7yjGqVRzYdXsBcfD2MLhFAhHfDgGLm2rY1ub7FA9k=
+golang.org/x/exp v0.0.0-20230725012225-302865e7556b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

libp2p_test.go

@@ -99,7 +99,7 @@ func TestAutoNATService(t *testing.T) {
 
 func TestDefaultListenAddrs(t *testing.T) {
 	reTCP := regexp.MustCompile("/(ip)[4|6]/((0.0.0.0)|(::))/tcp/")
-	reQUIC := regexp.MustCompile("/(ip)[4|6]/((0.0.0.0)|(::))/udp/([0-9]*)/quic")
+	reQUIC := regexp.MustCompile("/(ip)[4|6]/((0.0.0.0)|(::))/udp/([0-9]*)/quic-v1")
 	reCircuit := regexp.MustCompile("/p2p-circuit")
 
 	// Test 1: Setting the correct listen addresses if userDefined.Transport == nil && userDefined.ListenAddrs == nil
@@ -180,7 +180,7 @@ func TestTransportConstructorTCP(t *testing.T) {
 	require.NoError(t, err)
 	defer h.Close()
 	require.NoError(t, h.Network().Listen(ma.StringCast("/ip4/127.0.0.1/tcp/0")))
-	err = h.Network().Listen(ma.StringCast("/ip4/127.0.0.1/udp/0/quic"))
+	err = h.Network().Listen(ma.StringCast("/ip4/127.0.0.1/udp/0/quic-v1"))
 	require.Error(t, err)
 	require.Contains(t, err.Error(), swarm.ErrNoTransport.Error())
 }
@@ -192,7 +192,7 @@ func TestTransportConstructorQUIC(t *testing.T) {
 	)
 	require.NoError(t, err)
 	defer h.Close()
-	require.NoError(t, h.Network().Listen(ma.StringCast("/ip4/127.0.0.1/udp/0/quic")))
+	require.NoError(t, h.Network().Listen(ma.StringCast("/ip4/127.0.0.1/udp/0/quic-v1")))
 	err = h.Network().Listen(ma.StringCast("/ip4/127.0.0.1/tcp/0"))
 	require.Error(t, err)
 	require.Contains(t, err.Error(), swarm.ErrNoTransport.Error())

p2p/host/autorelay/addrsplosion_test.go

@@ -11,14 +11,14 @@ func TestCleanupAddrs(t *testing.T) {
 	t.Run("with no addrplosion", func(t *testing.T) {
 		addrs := makeAddrList(
 			"/ip4/127.0.0.1/tcp/4001",
-			"/ip4/127.0.0.1/udp/4002/quic",
+			"/ip4/127.0.0.1/udp/4002/quic-v1",
 			"/ip4/1.2.3.4/tcp/4001",
-			"/ip4/1.2.3.4/udp/4002/quic",
+			"/ip4/1.2.3.4/udp/4002/quic-v1",
 			"/dnsaddr/somedomain.com/tcp/4002/ws",
 		)
 		clean := makeAddrList(
 			"/ip4/1.2.3.4/tcp/4001",
-			"/ip4/1.2.3.4/udp/4002/quic",
+			"/ip4/1.2.3.4/udp/4002/quic-v1",
 			"/dnsaddr/somedomain.com/tcp/4002/ws",
 		)
 		require.ElementsMatch(t, clean, cleanupAddressSet(addrs), "cleaned up set doesn't match expected")
@@ -32,11 +32,11 @@ func TestCleanupAddrs(t *testing.T) {
 			"/ip4/1.2.3.4/tcp/33333",
 			"/ip4/1.2.3.4/tcp/33334",
 			"/ip4/1.2.3.4/tcp/33335",
-			"/ip4/1.2.3.4/udp/4002/quic",
+			"/ip4/1.2.3.4/udp/4002/quic-v1",
 		)
 		clean := makeAddrList(
 			"/ip4/1.2.3.4/tcp/4001",
-			"/ip4/1.2.3.4/udp/4002/quic",
+			"/ip4/1.2.3.4/udp/4002/quic-v1",
 		)
 		require.ElementsMatch(t, clean, cleanupAddressSet(addrs), "cleaned up set doesn't match expected")
 	})
@@ -48,11 +48,11 @@ func TestCleanupAddrs(t *testing.T) {
 			"/ip4/1.2.3.4/tcp/33333",
 			"/ip4/1.2.3.4/tcp/33334",
 			"/ip4/1.2.3.4/tcp/33335",
-			"/ip4/1.2.3.4/udp/4002/quic",
+			"/ip4/1.2.3.4/udp/4002/quic-v1",
 		)
 		clean := makeAddrList(
 			"/ip4/1.2.3.4/tcp/4001",
-			"/ip4/1.2.3.4/udp/4002/quic",
+			"/ip4/1.2.3.4/udp/4002/quic-v1",
 		)
 		require.ElementsMatch(t, clean, cleanupAddressSet(addrs), "cleaned up set doesn't match expected")
 	})
@@ -75,7 +75,7 @@ func TestCleanupAddrs(t *testing.T) {
 		// test with a squeaky clean address set
 		addrs := makeAddrList(
 			"/ip4/1.2.3.4/tcp/4001",
-			"/ip4/1.2.3.4/udp/4001/quic",
+			"/ip4/1.2.3.4/udp/4001/quic-v1",
 		)
 		require.ElementsMatch(t, addrs, cleanupAddressSet(addrs), "cleaned up set doesn't match expected")
 	})

p2p/host/basic/basic_host.go

@@ -792,10 +792,11 @@ func (h *BasicHost) Addrs() []ma.Multiaddr {
 				continue
 			}
 			addrWithCerthash, added := tpt.AddCertHashes(addr)
-			addrs[i] = addrWithCerthash
 			if !added {
 				log.Debug("Couldn't add certhashes to webtransport multiaddr because we aren't listening on webtransport")
+				continue
 			}
+			addrs[i] = addrWithCerthash
 		}
 	}
 	return addrs
@@ -942,17 +943,17 @@ func inferWebtransportAddrsFromQuic(in []ma.Multiaddr) []ma.Multiaddr {
 				// Remove certhashes
 				addr, _ = ma.SplitLast(addr)
 			}
-			webtransportAddrs[addr.String()] = struct{}{}
+			webtransportAddrs[string(addr.Bytes())] = struct{}{}
 			// Remove webtransport component, now it's a multiaddr that ends in /quic-v1
 			addr, _ = ma.SplitLast(addr)
 		}
 
 		if _, lastComponent := ma.SplitLast(addr); lastComponent.Protocol().Code == ma.P_QUIC_V1 {
-			addrStr := addr.String()
-			if _, ok := quicOrWebtransportAddrs[addrStr]; ok {
+			bytes := addr.Bytes()
+			if _, ok := quicOrWebtransportAddrs[string(bytes)]; ok {
 				foundSameListeningAddr = true
 			} else {
-				quicOrWebtransportAddrs[addrStr] = struct{}{}
+				quicOrWebtransportAddrs[string(bytes)] = struct{}{}
 			}
 		}
 	}
@@ -974,7 +975,7 @@ func inferWebtransportAddrsFromQuic(in []ma.Multiaddr) []ma.Multiaddr {
 		if _, lastComponent := ma.SplitLast(addr); lastComponent.Protocol().Code == ma.P_QUIC_V1 {
 			// Convert quic to webtransport
 			addr = addr.Encapsulate(wtComponent)
-			if _, ok := webtransportAddrs[addr.String()]; ok {
+			if _, ok := webtransportAddrs[string(addr.Bytes())]; ok {
 				// We already have this address
 				continue
 			}

p2p/host/basic/basic_host_test.go

@@ -838,6 +838,11 @@ func TestInferWebtransportAddrsFromQuic(t *testing.T) {
 			in:   []string{"/ip4/0.0.0.0/udp/9999/quic-v1", "/ip4/0.0.0.0/udp/9999/quic-v1/webtransport", "/ip4/1.2.3.4/udp/9999/quic-v1"},
 			out:  []string{"/ip4/0.0.0.0/udp/9999/quic-v1", "/ip4/0.0.0.0/udp/9999/quic-v1/webtransport", "/ip4/1.2.3.4/udp/9999/quic-v1", "/ip4/1.2.3.4/udp/9999/quic-v1/webtransport"},
 		},
+		{
+			name: "Happy Path With CertHashes",
+			in:   []string{"/ip4/0.0.0.0/udp/9999/quic-v1", "/ip4/0.0.0.0/udp/9999/quic-v1/webtransport/certhash/uEgNmb28/certhash/uEgNmb28", "/ip4/1.2.3.4/udp/9999/quic-v1"},
+			out:  []string{"/ip4/0.0.0.0/udp/9999/quic-v1", "/ip4/0.0.0.0/udp/9999/quic-v1/webtransport/certhash/uEgNmb28/certhash/uEgNmb28", "/ip4/1.2.3.4/udp/9999/quic-v1", "/ip4/1.2.3.4/udp/9999/quic-v1/webtransport"},
+		},
 		{
 			name: "Already discovered",
 			in:   []string{"/ip4/0.0.0.0/udp/9999/quic-v1", "/ip4/0.0.0.0/udp/9999/quic-v1/webtransport", "/ip4/1.2.3.4/udp/9999/quic-v1", "/ip4/1.2.3.4/udp/9999/quic-v1/webtransport"},
@@ -877,9 +882,6 @@ func TestInferWebtransportAddrsFromQuic(t *testing.T) {
 			sort.StringSlice(tc.in).Sort()
 			sort.StringSlice(tc.out).Sort()
 			min := make([]ma.Multiaddr, 0, len(tc.in))
-			sort.Slice(tc.in, func(i, j int) bool {
-				return tc.in[i] < tc.in[j]
-			})
 			for _, addr := range tc.in {
 				min = append(min, ma.StringCast(addr))
 			}

p2p/host/peerstore/pstoreds/addr_book.go

@@ -14,7 +14,7 @@ import (
 	"github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoreds/pb"
 	"github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem"
 
-	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/hashicorp/golang-lru/arc/v2"
 	ds "github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/query"
 	logging "github.com/ipfs/go-log/v2"
@@ -202,7 +202,7 @@ func NewAddrBook(ctx context.Context, store ds.Batching, opts Options) (ab *dsAd
 	}
 
 	if opts.CacheSize > 0 {
-		if ab.cache, err = lru.NewARC[peer.ID, *addrsRecord](int(opts.CacheSize)); err != nil {
+		if ab.cache, err = arc.NewARC[peer.ID, *addrsRecord](int(opts.CacheSize)); err != nil {
 			return nil, err
 		}
 	} else {

p2p/host/peerstore/pstoreds/deprecate.go

@@ -0,0 +1,5 @@
+// Deprecated: The database-backed peerstore will be removed from go-libp2p in the future.
+// Use the memory peerstore (pstoremem) instead.
+// For more details see https://github.com/libp2p/go-libp2p/issues/2329
+// and https://github.com/libp2p/go-libp2p/issues/2355.
+package pstoreds