diff --git a/p2p/net/swarm/limiter.go b/p2p/net/swarm/limiter.go
index f6cf6dda26..c3d8951894 100644
--- a/p2p/net/swarm/limiter.go
+++ b/p2p/net/swarm/limiter.go
@@ -144,6 +144,15 @@ func (dl *dialLimiter) AddDialJob(dj *dialJob) {
 	go dl.executeDial(dj)
 }
 
+func (dl *dialLimiter) clearAllPeerDials(p peer.ID) {
+	dl.rllock.Lock()
+	defer dl.rllock.Unlock()
+	delete(dl.waitingOnPeerLimit, p)
+	// NB: the waitingOnFd list doesnt need to be cleaned out here, we will
+	// remove them as we encounter them because they are 'cancelled' at this
+	// point
+}
+
 // executeDial calls the dialFunc, and reports the result through the response
 // channel when finished. Once the response is sent it also releases all tokens
 // it held during the dial.
diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go
index 1cc03580fd..1531bb2ed9 100644
--- a/p2p/net/swarm/swarm_dial.go
+++ b/p2p/net/swarm/swarm_dial.go
@@ -293,6 +293,8 @@ func (s *Swarm) dialAddrs(ctx context.Context, p peer.ID, remoteAddrs <-chan ma.
 	defaultDialFail := fmt.Errorf("failed to dial %s (default failure)", p)
 	exitErr := defaultDialFail
 
+	defer s.limiter.clearAllPeerDials(p)
+
 	var active int
 	for {
 		select {