Add more fine grained metrics to trie leafs request server (#765)

ava-labs · May 20, 2022 · 3bd3b56 · 3bd3b56
1 parent 100f520
commit 3bd3b56
Show file tree

Hide file tree

Showing 3 changed files with 69 additions and 14 deletions.
diff --git a/sync/handlers/leafs_request.go b/sync/handlers/leafs_request.go
@@ -118,6 +118,8 @@ func (lrh *LeafsRequestHandler) OnLeafsRequest(ctx context.Context, nodeID ids.N
 		leafsResponse.Keys = append(leafsResponse.Keys, it.Key)
 		leafsResponse.Vals = append(leafsResponse.Vals, it.Value)
 	}
+	// Update read leafs time here, so that we include the case that an error occurred.
+	lrh.stats.UpdateReadLeafsTime(time.Since(startTime))
 
 	if it.Err != nil {
 		log.Debug("failed to iterate trie, dropping request", "nodeID", nodeID, "requestID", requestID, "request", leafsRequest, "err", it.Err)
@@ -147,7 +149,10 @@ func (lrh *LeafsRequestHandler) OnLeafsRequest(ctx context.Context, nodeID ids.N
 		if len(leafsResponse.Keys) > 0 {
 			end = leafsResponse.Keys[len(leafsResponse.Keys)-1]
 		}
+		rangeProofStart := time.Now()
 		leafsResponse.ProofKeys, leafsResponse.ProofVals, err = GenerateRangeProof(t, start, end)
+		lrh.stats.UpdateGenerateRangeProofTime(time.Since(rangeProofStart))
+		lrh.stats.UpdateRangeProofKeysReturned(int64(len(leafsResponse.Keys)))
 		// Generate the proof and add it to the response.
 		if err != nil {
 			log.Debug("failed to create valid proof serving leafs request", "nodeID", nodeID, "requestID", requestID, "request", leafsRequest, "err", err)

diff --git a/sync/handlers/stats/mock_stats.go b/sync/handlers/stats/mock_stats.go
@@ -8,6 +8,8 @@ import (
 	"time"
 )
 
+var _ HandlerStats = &MockHandlerStats{}
+
 // MockHandlerStats is mock for capturing and asserting on handler metrics in test
 type MockHandlerStats struct {
 	lock sync.Mutex
@@ -27,6 +29,9 @@ type MockHandlerStats struct {
 	LeafsReturnedSum,
 	MissingRootCount,
 	TrieErrorCount uint32
+	ProofKeysReturned int64
+	LeafsReadTime,
+	GenerateRangeProofTime,
 	LeafRequestProcessingTimeSum time.Duration
 }
 
@@ -47,6 +52,9 @@ func (m *MockHandlerStats) Reset() {
 	m.MissingRootCount = 0
 	m.TrieErrorCount = 0
 	m.LeafRequestProcessingTimeSum = 0
+	m.ProofKeysReturned = 0
+	m.GenerateRangeProofTime = 0
+	m.LeafsReadTime = 0
 }
 
 func (m *MockHandlerStats) IncBlockRequest() {
@@ -121,6 +129,24 @@ func (m *MockHandlerStats) UpdateLeafsRequestProcessingTime(duration time.Durati
 	m.LeafRequestProcessingTimeSum += duration
 }
 
+func (m *MockHandlerStats) UpdateReadLeafsTime(duration time.Duration) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.LeafsReadTime += duration
+}
+
+func (m *MockHandlerStats) UpdateGenerateRangeProofTime(duration time.Duration) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.GenerateRangeProofTime += duration
+}
+
+func (m *MockHandlerStats) UpdateRangeProofKeysReturned(numProofKeys int64) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.ProofKeysReturned += numProofKeys
+}
+
 func (m *MockHandlerStats) IncMissingRoot() {
 	m.lock.Lock()
 	defer m.lock.Unlock()

diff --git a/sync/handlers/stats/stats.go b/sync/handlers/stats/stats.go
@@ -35,6 +35,9 @@ type LeafsRequestHandlerStats interface {
 	IncInvalidLeafsRequest()
 	UpdateLeafsReturned(numLeafs uint16)
 	UpdateLeafsRequestProcessingTime(duration time.Duration)
+	UpdateReadLeafsTime(duration time.Duration)
+	UpdateGenerateRangeProofTime(duration time.Duration)
+	UpdateRangeProofKeysReturned(numProofKeys int64)
 	IncMissingRoot()
 	IncTrieError()
 }
@@ -57,6 +60,9 @@ type handlerStats struct {
 	invalidLeafsRequest        metrics.Counter
 	leafsReturned              metrics.Histogram
 	leafsRequestProcessingTime metrics.Timer
+	leafsReadTime              metrics.Timer
+	generateRangeProofTime     metrics.Timer
+	proofKeysReturned          metrics.Histogram
 	missingRoot                metrics.Counter
 	trieError                  metrics.Counter
 }
@@ -109,6 +115,18 @@ func (h *handlerStats) UpdateLeafsReturned(numLeafs uint16) {
 	h.leafsReturned.Update(int64(numLeafs))
 }
 
+func (h *handlerStats) UpdateReadLeafsTime(duration time.Duration) {
+	h.leafsReadTime.Update(duration)
+}
+
+func (h *handlerStats) UpdateGenerateRangeProofTime(duration time.Duration) {
+	h.generateRangeProofTime.Update(duration)
+}
+
+func (h *handlerStats) UpdateRangeProofKeysReturned(numProofKeys int64) {
+	h.proofKeysReturned.Update(numProofKeys)
+}
+
 func (h *handlerStats) IncMissingRoot() {
 	h.missingRoot.Inc(1)
 }
@@ -139,6 +157,9 @@ func NewHandlerStats(enabled bool) HandlerStats {
 		invalidLeafsRequest:        metrics.GetOrRegisterCounter("leafs_request_invalid", nil),
 		leafsRequestProcessingTime: metrics.GetOrRegisterTimer("leafs_request_processing_time", nil),
 		leafsReturned:              metrics.GetOrRegisterHistogram("leafs_request_total_leafs", nil, metrics.NewExpDecaySample(1028, 0.015)),
+		leafsReadTime:              metrics.GetOrRegisterTimer("leafs_read_time", nil),
+		generateRangeProofTime:     metrics.GetOrRegisterTimer("generate_range_proof_time", nil),
+		proofKeysReturned:          metrics.GetOrRegisterHistogram("proof_keys_returned", nil, metrics.NewExpDecaySample(1028, 0.015)),
 		missingRoot:                metrics.GetOrRegisterCounter("leafs_request_missing_root", nil),
 		trieError:                  metrics.GetOrRegisterCounter("leafs_request_trie_error", nil),
 	}
@@ -152,17 +173,20 @@ func NewNoopHandlerStats() HandlerStats {
 }
 
 // all operations are no-ops
-func (n *noopHandlerStats) IncBlockRequest()                               {}
-func (n *noopHandlerStats) IncMissingBlockHash()                           {}
-func (n *noopHandlerStats) UpdateBlocksReturned(uint16)                    {}
-func (n *noopHandlerStats) UpdateBlockRequestProcessingTime(time.Duration) {}
-func (n *noopHandlerStats) IncCodeRequest()                                {}
-func (n *noopHandlerStats) IncMissingCodeHash()                            {}
-func (n *noopHandlerStats) UpdateCodeReadTime(time.Duration)               {}
-func (n *noopHandlerStats) UpdateCodeBytesReturned(uint32)                 {}
-func (n *noopHandlerStats) IncLeafsRequest()                               {}
-func (n *noopHandlerStats) IncInvalidLeafsRequest()                        {}
-func (n *noopHandlerStats) UpdateLeafsRequestProcessingTime(time.Duration) {}
-func (n *noopHandlerStats) UpdateLeafsReturned(uint16)                     {}
-func (n *noopHandlerStats) IncMissingRoot()                                {}
-func (n *noopHandlerStats) IncTrieError()                                  {}
+func (n *noopHandlerStats) IncBlockRequest()                                    {}
+func (n *noopHandlerStats) IncMissingBlockHash()                                {}
+func (n *noopHandlerStats) UpdateBlocksReturned(uint16)                         {}
+func (n *noopHandlerStats) UpdateBlockRequestProcessingTime(time.Duration)      {}
+func (n *noopHandlerStats) IncCodeRequest()                                     {}
+func (n *noopHandlerStats) IncMissingCodeHash()                                 {}
+func (n *noopHandlerStats) UpdateCodeReadTime(time.Duration)                    {}
+func (n *noopHandlerStats) UpdateCodeBytesReturned(uint32)                      {}
+func (n *noopHandlerStats) IncLeafsRequest()                                    {}
+func (n *noopHandlerStats) IncInvalidLeafsRequest()                             {}
+func (n *noopHandlerStats) UpdateLeafsRequestProcessingTime(time.Duration)      {}
+func (n *noopHandlerStats) UpdateLeafsReturned(uint16)                          {}
+func (n *noopHandlerStats) UpdateReadLeafsTime(duration time.Duration)          {}
+func (n *noopHandlerStats) UpdateGenerateRangeProofTime(duration time.Duration) {}
+func (n *noopHandlerStats) UpdateRangeProofKeysReturned(numProofKeys int64)     {}
+func (n *noopHandlerStats) IncMissingRoot()                                     {}
+func (n *noopHandlerStats) IncTrieError()                                       {}