From bfc6fc30de7ff85264974508a2fb218732bb8f43 Mon Sep 17 00:00:00 2001 From: Sukun Date: Tue, 7 Mar 2023 14:56:37 +0530 Subject: [PATCH] relay service: add metrics (#2154) * relay service: add metrics Metrics Added: ReservationRequest: Opened, Closed, Renewed ReservationRequestResponseStatus ReservationRejectionReason ConnectionRequest: Opened, Closed ConnectionRequestResponseStatus ConnectionRejectionReason ConnectionDuration BytesTransferred RelayStatus * add dashboard and changelog * bugfix * add metrics tests * address review comments * rename dashboard * address review comments * change label to rolling avg --- CHANGELOG.md | 10 + dashboards/relaysvc/relaysvc.json | 1204 ++++++++++++++++++ p2p/host/basic/basic_host.go | 7 + p2p/protocol/circuitv2/relay/metrics.go | 268 ++++ p2p/protocol/circuitv2/relay/metrics_test.go | 37 + p2p/protocol/circuitv2/relay/options.go | 8 + p2p/protocol/circuitv2/relay/relay.go | 144 ++- 7 files changed, 1645 insertions(+), 33 deletions(-) create mode 100644 dashboards/relaysvc/relaysvc.json create mode 100644 p2p/protocol/circuitv2/relay/metrics.go create mode 100644 p2p/protocol/circuitv2/relay/metrics_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 012815d332..2a45657279 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,16 @@ This patch release fixes two bugs: **Full Changelog**: https://github.com/libp2p/go-libp2p/compare/v0.26.0...v0.26.1 +# [v0.27.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.27.0) + +## 🔦 Highlights + +### Additional metrics +Since the last release, we've added metrics for: +* [Relay Service](https://github.com/libp2p/go-libp2p/pull/2154): RequestStatus, RequestCounts, RejectionReasons for Reservation and Connection Requests, +ConnectionDuration, BytesTransferred, Relay Service Status. + + # [v0.26.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.26.0) ## 🔦 Highlights diff --git a/dashboards/relaysvc/relaysvc.json b/dashboards/relaysvc/relaysvc.json new file mode 100644 index 0000000000..47dacc7c82 --- /dev/null +++ b/dashboards/relaysvc/relaysvc.json @@ -0,0 +1,1204 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.3.6" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "state-timeline", + "name": "State timeline", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 20, + "panels": [], + "title": "Relay Service", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "purple", + "index": 0, + "text": "no" + }, + "1": { + "color": "green", + "index": 1, + "text": "yes" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "alignValue": "center", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "libp2p_relaysvc_status", + "legendFormat": "active", + "range": true, + "refId": "A" + } + ], + "title": "Status", + "type": "state-timeline" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 22, + "panels": [], + "title": "Reservations", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 9 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "libp2p_relaysvc_reservations_total{type=\"opened\"} - ignoring(type) libp2p_relaysvc_reservations_total{type=\"closed\"}", + "legendFormat": "active reservations", + "range": true, + "refId": "A" + } + ], + "title": "Active Reservations", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ok" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 9 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(libp2p_relaysvc_reservation_request_response_status_total[$__rate_interval])", + "legendFormat": "{{status}}", + "range": true, + "refId": "A" + } + ], + "title": "Reservation Request Response Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(libp2p_relaysvc_reservations_total{type=\"opened\"}[$__rate_interval])", + "legendFormat": "new", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(libp2p_relaysvc_reservations_total{type=\"renewed\"}[$__rate_interval])", + "hide": false, + "legendFormat": "renewed", + "range": true, + "refId": "B" + } + ], + "title": "Reservation Requests: New vs Renewal", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(libp2p_relaysvc_reservation_rejections_total[$__rate_interval])", + "legendFormat": "{{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Reservation Request Rejected", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 24, + "panels": [], + "title": "Connections", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 26 + }, + "id": 28, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(libp2p_relaysvc_data_transferred_bytes_total[$__range])", + "legendFormat": "data transferred", + "range": true, + "refId": "A" + } + ], + "title": "Total Data Transferred", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 26 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "libp2p_relaysvc_connections_total{type=\"opened\"} - ignoring(type) libp2p_relaysvc_connections_total{type=\"closed\"}", + "legendFormat": "active connections", + "range": true, + "refId": "A" + } + ], + "title": "Active Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "error" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ok" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(libp2p_relaysvc_connection_request_response_status_total[$__rate_interval])", + "legendFormat": "{{status}}", + "range": true, + "refId": "A" + } + ], + "title": "Connection Request Response Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "bandwidth" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "bandwidth" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "2 * rate(libp2p_relaysvc_data_transferred_bytes_total[$__rate_interval])", + "legendFormat": "bandwidth", + "range": true, + "refId": "A" + } + ], + "title": "Bandwidth Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(libp2p_relaysvc_connection_rejections_total[$__rate_interval])", + "legendFormat": "{{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Connection Request Rejected", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(libp2p_relaysvc_connection_duration_seconds_sum[$__range])/rate(libp2p_relaysvc_connection_duration_seconds_count[$__range])\n", + "hide": false, + "legendFormat": "rolling average", + "range": true, + "refId": "A" + } + ], + "title": "Connection Duration", + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "libp2p Relay Service", + "uid": "C6RUfAx4z", + "version": 5, + "weekStart": "" +} \ No newline at end of file diff --git a/p2p/host/basic/basic_host.go b/p2p/host/basic/basic_host.go index 63a8f4eed7..c11447974b 100644 --- a/p2p/host/basic/basic_host.go +++ b/p2p/host/basic/basic_host.go @@ -280,6 +280,13 @@ func NewHost(n network.Network, opts *HostOpts) (*BasicHost, error) { } if opts.EnableRelayService { + if opts.EnableMetrics { + // Prefer explicitly provided metrics tracer + metricsOpt := []relayv2.Option{ + relayv2.WithMetricsTracer( + relayv2.NewMetricsTracer(relayv2.WithRegisterer(opts.PrometheusRegisterer)))} + opts.RelayServiceOpts = append(metricsOpt, opts.RelayServiceOpts...) + } h.relayManager = relaysvc.NewRelayManager(h, opts.RelayServiceOpts...) } diff --git a/p2p/protocol/circuitv2/relay/metrics.go b/p2p/protocol/circuitv2/relay/metrics.go new file mode 100644 index 0000000000..7786459133 --- /dev/null +++ b/p2p/protocol/circuitv2/relay/metrics.go @@ -0,0 +1,268 @@ +package relay + +import ( + "time" + + "github.com/libp2p/go-libp2p/p2p/metricshelper" + pbv2 "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/pb" + "github.com/prometheus/client_golang/prometheus" +) + +const metricNamespace = "libp2p_relaysvc" + +var ( + status = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "status", + Help: "Relay Status", + }, + ) + + reservationsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "reservations_total", + Help: "Relay Reservation Request", + }, + []string{"type"}, + ) + reservationRequestResponseStatusTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "reservation_request_response_status_total", + Help: "Relay Reservation Request Response Status", + }, + []string{"status"}, + ) + reservationRejectionsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "reservation_rejections_total", + Help: "Relay Reservation Rejected Reason", + }, + []string{"reason"}, + ) + + connectionsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "connections_total", + Help: "Relay Connection Total", + }, + []string{"type"}, + ) + connectionRequestResponseStatusTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "connection_request_response_status_total", + Help: "Relay Connection Request Status", + }, + []string{"status"}, + ) + connectionRejectionsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "connection_rejections_total", + Help: "Relay Connection Rejected Reason", + }, + []string{"reason"}, + ) + connectionDurationSeconds = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: metricNamespace, + Name: "connection_duration_seconds", + Help: "Relay Connection Duration", + }, + ) + + dataTransferredBytesTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "data_transferred_bytes_total", + Help: "Bytes Transferred Total", + }, + ) + + collectors = []prometheus.Collector{ + status, + reservationsTotal, + reservationRequestResponseStatusTotal, + reservationRejectionsTotal, + connectionsTotal, + connectionRequestResponseStatusTotal, + connectionRejectionsTotal, + connectionDurationSeconds, + dataTransferredBytesTotal, + } +) + +const ( + requestStatusOK = "ok" + requestStatusRejected = "rejected" + requestStatusError = "error" +) + +// MetricsTracer is the interface for tracking metrics for relay service +type MetricsTracer interface { + // RelayStatus tracks whether the service is currently active + RelayStatus(enabled bool) + + // ConnectionOpened tracks metrics on opening a relay connection + ConnectionOpened() + // ConnectionClosed tracks metrics on closing a relay connection + ConnectionClosed(d time.Duration) + // ConnectionRequestHandled tracks metrics on handling a relay connection request + ConnectionRequestHandled(status pbv2.Status) + + // ReservationAllowed tracks metrics on opening or renewing a relay reservation + ReservationAllowed(isRenewal bool) + // ReservationRequestClosed tracks metrics on closing a relay reservation + ReservationClosed(cnt int) + // ReservationRequestHandled tracks metrics on handling a relay reservation request + ReservationRequestHandled(status pbv2.Status) + + // BytesTransferred tracks the total bytes transferred by the relay service + BytesTransferred(cnt int) +} + +type metricsTracer struct{} + +var _ MetricsTracer = &metricsTracer{} + +type metricsTracerSetting struct { + reg prometheus.Registerer +} + +type MetricsTracerOption func(*metricsTracerSetting) + +func WithRegisterer(reg prometheus.Registerer) MetricsTracerOption { + return func(s *metricsTracerSetting) { + if reg != nil { + s.reg = reg + } + } +} + +func NewMetricsTracer(opts ...MetricsTracerOption) MetricsTracer { + setting := &metricsTracerSetting{reg: prometheus.DefaultRegisterer} + for _, opt := range opts { + opt(setting) + } + metricshelper.RegisterCollectors(setting.reg, collectors...) + return &metricsTracer{} +} + +func (mt *metricsTracer) RelayStatus(enabled bool) { + if enabled { + status.Set(1) + } else { + status.Set(0) + } +} + +func (mt *metricsTracer) ConnectionOpened() { + tags := metricshelper.GetStringSlice() + defer metricshelper.PutStringSlice(tags) + *tags = append(*tags, "opened") + + connectionsTotal.WithLabelValues(*tags...).Add(1) +} + +func (mt *metricsTracer) ConnectionClosed(d time.Duration) { + tags := metricshelper.GetStringSlice() + defer metricshelper.PutStringSlice(tags) + *tags = append(*tags, "closed") + + connectionsTotal.WithLabelValues(*tags...).Add(1) + connectionDurationSeconds.Observe(d.Seconds()) +} + +func (mt *metricsTracer) ConnectionRequestHandled(status pbv2.Status) { + tags := metricshelper.GetStringSlice() + defer metricshelper.PutStringSlice(tags) + + respStatus := getResponseStatus(status) + + *tags = append(*tags, respStatus) + connectionRequestResponseStatusTotal.WithLabelValues(*tags...).Add(1) + if respStatus == requestStatusRejected { + *tags = (*tags)[:0] + *tags = append(*tags, getRejectionReason(status)) + connectionRejectionsTotal.WithLabelValues(*tags...).Add(1) + } +} + +func (mt *metricsTracer) ReservationAllowed(isRenewal bool) { + tags := metricshelper.GetStringSlice() + defer metricshelper.PutStringSlice(tags) + if isRenewal { + *tags = append(*tags, "renewed") + } else { + *tags = append(*tags, "opened") + } + + reservationsTotal.WithLabelValues(*tags...).Add(1) +} + +func (mt *metricsTracer) ReservationClosed(cnt int) { + tags := metricshelper.GetStringSlice() + defer metricshelper.PutStringSlice(tags) + *tags = append(*tags, "closed") + + reservationsTotal.WithLabelValues(*tags...).Add(float64(cnt)) +} + +func (mt *metricsTracer) ReservationRequestHandled(status pbv2.Status) { + tags := metricshelper.GetStringSlice() + defer metricshelper.PutStringSlice(tags) + + respStatus := getResponseStatus(status) + + *tags = append(*tags, respStatus) + reservationRequestResponseStatusTotal.WithLabelValues(*tags...).Add(1) + if respStatus == requestStatusRejected { + *tags = (*tags)[:0] + *tags = append(*tags, getRejectionReason(status)) + reservationRejectionsTotal.WithLabelValues(*tags...).Add(1) + } +} + +func (mt *metricsTracer) BytesTransferred(cnt int) { + dataTransferredBytesTotal.Add(float64(cnt)) +} + +func getResponseStatus(status pbv2.Status) string { + responseStatus := "unknown" + switch status { + case pbv2.Status_RESERVATION_REFUSED, + pbv2.Status_RESOURCE_LIMIT_EXCEEDED, + pbv2.Status_PERMISSION_DENIED, + pbv2.Status_NO_RESERVATION, + pbv2.Status_MALFORMED_MESSAGE: + + responseStatus = requestStatusRejected + case pbv2.Status_UNEXPECTED_MESSAGE, pbv2.Status_CONNECTION_FAILED: + responseStatus = requestStatusError + case pbv2.Status_OK: + responseStatus = requestStatusOK + } + return responseStatus +} + +func getRejectionReason(status pbv2.Status) string { + reason := "unknown" + switch status { + case pbv2.Status_RESERVATION_REFUSED: + reason = "ip constraint violation" + case pbv2.Status_RESOURCE_LIMIT_EXCEEDED: + reason = "resource limit exceeded" + case pbv2.Status_PERMISSION_DENIED: + reason = "permission denied" + case pbv2.Status_NO_RESERVATION: + reason = "no reservation" + case pbv2.Status_MALFORMED_MESSAGE: + reason = "malformed message" + } + return reason +} diff --git a/p2p/protocol/circuitv2/relay/metrics_test.go b/p2p/protocol/circuitv2/relay/metrics_test.go new file mode 100644 index 0000000000..9af23fec75 --- /dev/null +++ b/p2p/protocol/circuitv2/relay/metrics_test.go @@ -0,0 +1,37 @@ +//go:build nocover + +package relay + +import ( + "math/rand" + "testing" + "time" + + pbv2 "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/pb" +) + +func TestNoCoverNoAlloc(t *testing.T) { + statuses := []pbv2.Status{ + pbv2.Status_OK, + pbv2.Status_NO_RESERVATION, + pbv2.Status_RESOURCE_LIMIT_EXCEEDED, + pbv2.Status_PERMISSION_DENIED, + } + mt := NewMetricsTracer() + tests := map[string]func(){ + "RelayStatus": func() { mt.RelayStatus(rand.Intn(2) == 1) }, + "ConnectionOpened": func() { mt.ConnectionOpened() }, + "ConnectionClosed": func() { mt.ConnectionClosed(time.Duration(rand.Intn(10)) * time.Second) }, + "ConnectionRequestHandled": func() { mt.ConnectionRequestHandled(statuses[rand.Intn(len(statuses))]) }, + "ReservationAllowed": func() { mt.ReservationAllowed(rand.Intn(2) == 1) }, + "ReservationClosed": func() { mt.ReservationClosed(rand.Intn(10)) }, + "ReservationRequestHandled": func() { mt.ReservationRequestHandled(statuses[rand.Intn(len(statuses))]) }, + "BytesTransferred": func() { mt.BytesTransferred(rand.Intn(1000)) }, + } + for method, f := range tests { + allocs := testing.AllocsPerRun(1000, f) + if allocs > 0 { + t.Fatalf("Alloc Test: %s, got: %0.2f, expected: 0 allocs", method, allocs) + } + } +} diff --git a/p2p/protocol/circuitv2/relay/options.go b/p2p/protocol/circuitv2/relay/options.go index a69b658ec1..3b50ec385f 100644 --- a/p2p/protocol/circuitv2/relay/options.go +++ b/p2p/protocol/circuitv2/relay/options.go @@ -33,3 +33,11 @@ func WithACL(acl ACLFilter) Option { return nil } } + +// WithMetricsTracer is a Relay option that supplies a MetricsTracer for metrics +func WithMetricsTracer(mt MetricsTracer) Option { + return func(r *Relay) error { + r.metricsTracer = mt + return nil + } +} diff --git a/p2p/protocol/circuitv2/relay/relay.go b/p2p/protocol/circuitv2/relay/relay.go index 77c592e8b7..81857827bf 100644 --- a/p2p/protocol/circuitv2/relay/relay.go +++ b/p2p/protocol/circuitv2/relay/relay.go @@ -2,6 +2,7 @@ package relay import ( "context" + "errors" "fmt" "io" "sync" @@ -59,6 +60,8 @@ type Relay struct { closed bool selfAddr ma.Multiaddr + + metricsTracer MetricsTracer } // New constructs a new limited relay that can provide relay services in the given host. @@ -100,6 +103,9 @@ func New(h host.Host, opts ...Option) (*Relay, error) { r.notifiee = &network.NotifyBundle{DisconnectedF: r.disconnected} h.Network().Notify(r.notifiee) + if r.metricsTracer != nil { + r.metricsTracer.RelayStatus(true) + } r.wg.Add(1) go r.background() @@ -116,6 +122,9 @@ func (r *Relay) Close() error { r.host.Network().StopNotify(r.notifiee) r.scope.Done() r.cancel() + if r.metricsTracer != nil { + r.metricsTracer.RelayStatus(false) + } r.wg.Wait() return nil } @@ -153,35 +162,37 @@ func (r *Relay) handleStream(s network.Stream) { } // reset stream deadline as message has been read s.SetReadDeadline(time.Time{}) - switch msg.GetType() { case pbv2.HopMessage_RESERVE: - r.handleReserve(s) - + status := r.handleReserve(s) + if r.metricsTracer != nil { + r.metricsTracer.ReservationRequestHandled(status) + } case pbv2.HopMessage_CONNECT: - r.handleConnect(s, &msg) - + status := r.handleConnect(s, &msg) + if r.metricsTracer != nil { + r.metricsTracer.ConnectionRequestHandled(status) + } default: r.handleError(s, pbv2.Status_MALFORMED_MESSAGE) } } -func (r *Relay) handleReserve(s network.Stream) { +func (r *Relay) handleReserve(s network.Stream) pbv2.Status { defer s.Close() - p := s.Conn().RemotePeer() a := s.Conn().RemoteMultiaddr() if isRelayAddr(a) { log.Debugf("refusing relay reservation for %s; reservation attempt over relay connection") r.handleError(s, pbv2.Status_PERMISSION_DENIED) - return + return pbv2.Status_PERMISSION_DENIED } if r.acl != nil && !r.acl.AllowReserve(p, a) { log.Debugf("refusing relay reservation for %s; permission denied", p) r.handleError(s, pbv2.Status_PERMISSION_DENIED) - return + return pbv2.Status_PERMISSION_DENIED } r.mx.Lock() @@ -191,6 +202,7 @@ func (r *Relay) handleReserve(s network.Stream) { r.mx.Unlock() log.Debugf("refusing relay reservation for %s; relay closed", p) r.handleError(s, pbv2.Status_PERMISSION_DENIED) + return pbv2.Status_PERMISSION_DENIED } now := time.Now() @@ -200,7 +212,7 @@ func (r *Relay) handleReserve(s network.Stream) { r.mx.Unlock() log.Debugf("refusing relay reservation for %s; IP constraint violation: %s", p, err) r.handleError(s, pbv2.Status_RESERVATION_REFUSED) - return + return pbv2.Status_RESERVATION_REFUSED } } @@ -208,6 +220,9 @@ func (r *Relay) handleReserve(s network.Stream) { r.rsvp[p] = expire r.host.ConnManager().TagPeer(p, "relay-reservation", ReservationTagWeight) r.mx.Unlock() + if r.metricsTracer != nil { + r.metricsTracer.ReservationAllowed(exists) + } log.Debugf("reserving relay slot for %s", p) @@ -217,10 +232,12 @@ func (r *Relay) handleReserve(s network.Stream) { if err := r.writeResponse(s, pbv2.Status_OK, r.makeReservationMsg(p, expire), r.makeLimitMsg(p)); err != nil { log.Debugf("error writing reservation response; retracting reservation for %s", p) s.Reset() + return pbv2.Status_CONNECTION_FAILED } + return pbv2.Status_OK } -func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) { +func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) pbv2.Status { src := s.Conn().RemotePeer() a := s.Conn().RemoteMultiaddr() @@ -228,7 +245,7 @@ func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) { if err != nil { log.Debugf("failed to begin relay transaction: %s", err) r.handleError(s, pbv2.Status_RESOURCE_LIMIT_EXCEEDED) - return + return pbv2.Status_RESOURCE_LIMIT_EXCEEDED } fail := func(status pbv2.Status) { @@ -240,25 +257,25 @@ func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) { if err := span.ReserveMemory(2*r.rc.BufferSize, network.ReservationPriorityHigh); err != nil { log.Debugf("error reserving memory for relay: %s", err) fail(pbv2.Status_RESOURCE_LIMIT_EXCEEDED) - return + return pbv2.Status_RESOURCE_LIMIT_EXCEEDED } if isRelayAddr(a) { log.Debugf("refusing connection from %s; connection attempt over relay connection") fail(pbv2.Status_PERMISSION_DENIED) - return + return pbv2.Status_PERMISSION_DENIED } dest, err := util.PeerToPeerInfoV2(msg.GetPeer()) if err != nil { fail(pbv2.Status_MALFORMED_MESSAGE) - return + return pbv2.Status_MALFORMED_MESSAGE } if r.acl != nil && !r.acl.AllowConnect(src, s.Conn().RemoteMultiaddr(), dest.ID) { log.Debugf("refusing connection from %s to %s; permission denied", src, dest.ID) fail(pbv2.Status_PERMISSION_DENIED) - return + return pbv2.Status_PERMISSION_DENIED } r.mx.Lock() @@ -267,7 +284,7 @@ func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) { r.mx.Unlock() log.Debugf("refusing connection from %s to %s; no reservation", src, dest.ID) fail(pbv2.Status_NO_RESERVATION) - return + return pbv2.Status_NO_RESERVATION } srcConns := r.conns[src] @@ -275,7 +292,7 @@ func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) { r.mx.Unlock() log.Debugf("refusing connection from %s to %s; too many connections from %s", src, dest.ID, src) fail(pbv2.Status_RESOURCE_LIMIT_EXCEEDED) - return + return pbv2.Status_RESOURCE_LIMIT_EXCEEDED } destConns := r.conns[dest.ID] @@ -283,19 +300,27 @@ func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) { r.mx.Unlock() log.Debugf("refusing connection from %s to %s; too many connecitons to %s", src, dest.ID, dest.ID) fail(pbv2.Status_RESOURCE_LIMIT_EXCEEDED) - return + return pbv2.Status_RESOURCE_LIMIT_EXCEEDED } r.addConn(src) r.addConn(dest.ID) r.mx.Unlock() + if r.metricsTracer != nil { + r.metricsTracer.ConnectionOpened() + } + connStTime := time.Now() + cleanup := func() { span.Done() r.mx.Lock() r.rmConn(src) r.rmConn(dest.ID) r.mx.Unlock() + if r.metricsTracer != nil { + r.metricsTracer.ConnectionClosed(time.Since(connStTime)) + } } ctx, cancel := context.WithTimeout(r.ctx, ConnectTimeout) @@ -308,7 +333,7 @@ func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) { log.Debugf("error opening relay stream to %s: %s", dest.ID, err) cleanup() r.handleError(s, pbv2.Status_CONNECTION_FAILED) - return + return pbv2.Status_CONNECTION_FAILED } fail = func(status pbv2.Status) { @@ -320,14 +345,14 @@ func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) { if err := bs.Scope().SetService(ServiceName); err != nil { log.Debugf("error attaching stream to relay service: %s", err) fail(pbv2.Status_RESOURCE_LIMIT_EXCEEDED) - return + return pbv2.Status_RESOURCE_LIMIT_EXCEEDED } // handshake if err := bs.Scope().ReserveMemory(maxMessageSize, network.ReservationPriorityAlways); err != nil { - log.Debugf("erro reserving memory for stream: %s", err) + log.Debugf("error reserving memory for stream: %s", err) fail(pbv2.Status_RESOURCE_LIMIT_EXCEEDED) - return + return pbv2.Status_RESOURCE_LIMIT_EXCEEDED } defer bs.Scope().ReleaseMemory(maxMessageSize) @@ -346,7 +371,7 @@ func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) { if err != nil { log.Debugf("error writing stop handshake") fail(pbv2.Status_CONNECTION_FAILED) - return + return pbv2.Status_CONNECTION_FAILED } stopmsg.Reset() @@ -355,19 +380,19 @@ func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) { if err != nil { log.Debugf("error reading stop response: %s", err.Error()) fail(pbv2.Status_CONNECTION_FAILED) - return + return pbv2.Status_CONNECTION_FAILED } if t := stopmsg.GetType(); t != pbv2.StopMessage_STATUS { log.Debugf("unexpected stop response; not a status message (%d)", t) fail(pbv2.Status_CONNECTION_FAILED) - return + return pbv2.Status_CONNECTION_FAILED } if status := stopmsg.GetStatus(); status != pbv2.Status_OK { log.Debugf("relay stop failure: %d", status) fail(pbv2.Status_CONNECTION_FAILED) - return + return pbv2.Status_CONNECTION_FAILED } var response pbv2.HopMessage @@ -382,7 +407,7 @@ func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) { bs.Reset() s.Reset() cleanup() - return + return pbv2.Status_CONNECTION_FAILED } // reset deadline @@ -411,6 +436,8 @@ func (r *Relay) handleConnect(s network.Stream, msg *pbv2.HopMessage) { go r.relayUnlimited(s, bs, src, dest.ID, done) go r.relayUnlimited(bs, s, dest.ID, src, done) } + + return pbv2.Status_OK } func (r *Relay) addConn(p peer.ID) { @@ -441,7 +468,7 @@ func (r *Relay) relayLimited(src, dest network.Stream, srcID, destID peer.ID, li limitedSrc := io.LimitReader(src, limit) - count, err := io.CopyBuffer(dest, limitedSrc, buf) + count, err := r.copyWithBuffer(dest, limitedSrc, buf) if err != nil { log.Debugf("relay copy error: %s", err) // Reset both. @@ -465,7 +492,7 @@ func (r *Relay) relayUnlimited(src, dest network.Stream, srcID, destID peer.ID, buf := pool.Get(r.rc.BufferSize) defer pool.Put(buf) - count, err := io.CopyBuffer(dest, src, buf) + count, err := r.copyWithBuffer(dest, src, buf) if err != nil { log.Debugf("relay copy error: %s", err) // Reset both. @@ -479,6 +506,47 @@ func (r *Relay) relayUnlimited(src, dest network.Stream, srcID, destID peer.ID, log.Debugf("relayed %d bytes from %s to %s", count, srcID, destID) } +// errInvalidWrite means that a write returned an impossible count. +// copied from io.errInvalidWrite +var errInvalidWrite = errors.New("invalid write result") + +// copyWithBuffer copies from src to dst using the provided buf until either EOF is reached +// on src or an error occurs. It reports the number of bytes transferred to metricsTracer. +// The implementation is a modified form of io.CopyBuffer to support metrics tracking. +func (r *Relay) copyWithBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) { + for { + nr, er := src.Read(buf) + if nr > 0 { + nw, ew := dst.Write(buf[0:nr]) + if nw < 0 || nr < nw { + nw = 0 + if ew == nil { + ew = errInvalidWrite + } + } + written += int64(nw) + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + if r.metricsTracer != nil { + r.metricsTracer.BytesTransferred(nw) + } + } + if er != nil { + if er != io.EOF { + err = er + } + break + } + } + return written, err +} + func (r *Relay) handleError(s network.Stream, status pbv2.Status) { log.Debugf("relay error: %s (%d)", pbv2.Status_name[int32(status)], status) err := r.writeResponse(s, status, nil, nil) @@ -580,13 +648,17 @@ func (r *Relay) gc() { defer r.mx.Unlock() now := time.Now() - + cnt := 0 for p, expire := range r.rsvp { if r.closed || expire.Before(now) { delete(r.rsvp, p) r.host.ConnManager().UntagPeer(p, "relay-reservation") + cnt++ } } + if r.metricsTracer != nil { + r.metricsTracer.ReservationClosed(cnt) + } for p, count := range r.conns { if count == 0 { @@ -602,9 +674,15 @@ func (r *Relay) disconnected(n network.Network, c network.Conn) { } r.mx.Lock() - defer r.mx.Unlock() + _, ok := r.rsvp[p] + if ok { + delete(r.rsvp, p) + } + r.mx.Unlock() - delete(r.rsvp, p) + if ok && r.metricsTracer != nil { + r.metricsTracer.ReservationClosed(1) + } } func isRelayAddr(a ma.Multiaddr) bool {