Merge branch 'master' into fe/selector-component

.github/workflows/go.yml

@@ -40,10 +40,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          # TODO: this can be shorter, maybe 500-1,000? This may be slower (see: https://github.com/CocoaPods/CocoaPods/issues/4989#issuecomment-193772935)
           fetch-depth: 0
-          # TODO: is this neccesary?
-          submodules: 'recursive'
 
       # note: after this action is pushed, whatever this ends up being should go back to latest.
       - uses: docker://ghcr.io/synapsecns/sanguine/git-changes-action:07a4696bbbafbba2d3ad7ee828b4c8adb21b4d20

.github/workflows/lint.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
         with:
-          ignore_paths: ./contrib/terraform-provider-iap/scripts/add-tfmac.sh ./contrib/terraform-provider-helmproxy/scripts/add-tfmac.sh ./contrib/terraform-provider-kubeproxy/scripts/add-tfmac.sh
+          ignore_paths: ./contrib/terraform-provider-iap/scripts/add-tfmac.sh ./contrib/terraform-provider-helmproxy/scripts/add-tfmac.sh ./contrib/terraform-provider-kubeproxy/scripts/add-tfmac.sh ./contrib/scripts/txdecoder.sh
 
       - name: Validate renovate
         uses: rinchsan/renovate-config-validator@v0.0.12

.nvmrc

@@ -1 +1 @@
-v16.14.0
+v18.17.0

agents/go.mod

@@ -80,6 +80,7 @@ require (
 	github.com/status-im/keycard-go v0.2.0 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otelutil v0.2.3 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otelzap v0.2.3 // indirect
+	github.com/valyala/fastjson v1.6.4 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240108191215-35c7eff3a6b1 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect

agents/go.sum

@@ -1147,6 +1147,8 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn
 github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0=
 github.com/valyala/fasthttp v1.41.0 h1:zeR0Z1my1wDHTRiamBCXVglQdbUwgb9uWG3k1HQz6jY=
 github.com/valyala/fasthttp v1.41.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY=
+github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
+github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
 github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
 github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=

committee/go.mod

@@ -316,6 +316,7 @@ require (
 	github.com/uptrace/opentelemetry-go-extra/otelzap v0.2.3 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.41.0 // indirect
+	github.com/valyala/fastjson v1.6.4 // indirect
 	github.com/valyala/fastrand v1.1.0 // indirect
 	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
 	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect

committee/go.sum

@@ -1296,6 +1296,8 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn
 github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0=
 github.com/valyala/fasthttp v1.41.0 h1:zeR0Z1my1wDHTRiamBCXVglQdbUwgb9uWG3k1HQz6jY=
 github.com/valyala/fasthttp v1.41.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY=
+github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
+github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
 github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
 github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=

contrib/promexporter/go.mod

@@ -200,6 +200,7 @@ require (
 	github.com/uptrace/opentelemetry-go-extra/otelsql v0.2.2 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otelutil v0.2.3 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otelzap v0.2.3 // indirect
+	github.com/valyala/fastjson v1.6.4 // indirect
 	github.com/vektah/gqlparser/v2 v2.5.8 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect

contrib/promexporter/go.sum

@@ -1614,6 +1614,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
 github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
 github.com/valyala/fasthttp v1.41.0 h1:zeR0Z1my1wDHTRiamBCXVglQdbUwgb9uWG3k1HQz6jY=
 github.com/valyala/fasthttp v1.41.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY=
+github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
+github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
 github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=

contrib/scripts/txdecoder.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Takes the raw transaction as input
+rawTx=$1
+
+# Decode the transaction using cast dt and store the result in a variable
+decodedTx=$(cast dt $rawTx)
+
+# Use jq to pretty print the decoded transaction
+echo "$decodedTx" | jq '.'
+
+# Extract the type of the transaction
+type=$(echo "$decodedTx" | jq -r '.type')
+
+# Convert the type from hexadecimal to decimal
+txType=$(cast --to-dec $type)
+
+# Extract common fields
+gas=$(echo "$decodedTx" | jq -r '.gas')
+value=$(echo "$decodedTx" | jq -r '.value')
+nonce=$(echo "$decodedTx" | jq -r '.nonce')
+
+echo "Converted Values:"
+echo "Type: $txType"
+echo "Gas: $(cast --to-dec $gas)"
+echo "Value: $(cast --to-unit $value ether)"
+echo "Nonce: $(cast --to-dec $nonce)"
+
+# Conditional logic based on transaction type
+if [[ "$txType" == "0" ]]; then
+    # Transaction type 0 specific field
+    gasPrice=$(echo "$decodedTx" | jq -r '.gasPrice')
+    echo "GasPrice: $(cast --to-unit $gasPrice gwei)"
+elif [[ "$txType" == "2" ]]; then
+    # Transaction type 2 specific fields
+    maxPriorityFeePerGas=$(echo "$decodedTx" | jq -r '.maxPriorityFeePerGas')
+    maxFeePerGas=$(echo "$decodedTx" | jq -r '.maxFeePerGas')
+    echo "MaxPriorityFeePerGas: $(cast --to-unit $maxPriorityFeePerGas gwei)"
+    echo "MaxFeePerGas: $(cast --to-unit $maxFeePerGas gwei)"
+fi

core/README.md

@@ -7,6 +7,7 @@ Core contains common libraries used across the synapse Go repositories.
 
 ## Directory Structure
 
+
 <pre>
 root
 ├── <a href="./bytemap">bytemap</a>: Implements a map using `[]rune` or `[]byte` instead of `string`

core/go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/uptrace/opentelemetry-go-extra/otelgorm v0.1.21
 	github.com/uptrace/opentelemetry-go-extra/otelzap v0.2.3
 	github.com/urfave/cli/v2 v2.27.1
+	github.com/valyala/fastjson v1.6.4
 	go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.42.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0
 	go.opentelemetry.io/contrib/propagators/b3 v1.21.0

core/go.sum

@@ -485,6 +485,8 @@ github.com/uptrace/opentelemetry-go-extra/otelzap v0.2.3/go.mod h1:9IVEh9mPv3NwF
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
 github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
+github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
+github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=

core/metrics/README.md

@@ -27,11 +27,12 @@ Pass in the `PYROSCOPE_ENDPOINT` environment variable
 
 The metrics endpoint is exposed on `/metrics` on port `8080` by default and is compatible with prometheus. The following options control the metrics endpoint:
 
-| Enviornment Variable   | Description                                   | Default    |
-|------------------------|-----------------------------------------------|------------|
-| `METRICS_PORT_ENABLED` | Wether or not to enable the metrics endpoint. | `true`     |
-| `METRICS_PORT`         | Port to serve metrics on.                     | `8080`     |
-| `METRICS_PATH`         | Path to serve metrics on                      | `/metrics` |
+| Enviornment Variable   | Description                                                                 | Default    |
+|------------------------|-----------------------------------------------------------------------------|------------|
+| `METRICS_PORT_ENABLED` | Wether or not to enable the metrics endpoint.                               | `true`     |
+| `METRICS_PORT`         | Port to serve metrics on.                                                   | `8080`     |
+| `METRICS_PATH`         | Path to serve metrics on                                                    | `/metrics` |
+| `METRICS_ALWAYS_SYNC`  | Wether or not to wait until metrics have been exported before ending a span | `false`    |
 
 **Note: this server failing to bind to `METRICS_PORT` will not cause the application to fail to start. The error will be logged.**
 

core/metrics/base.go

@@ -194,6 +194,10 @@ func newBaseHandler(buildInfo config.BuildInfo, extraOpts ...tracesdk.TracerProv
 		logger.Warn("could not merge resources", "error", err)
 	}
 
+	if core.GetEnvBool("METRICS_ALWAYS_SYNC", false) {
+		extraOpts = append(extraOpts, tracesdk.WithSampler(tracesdk.AlwaysSample()))
+	}
+
 	opts := append([]tracesdk.TracerProviderOption{tracesdk.WithResource(rsr)}, extraOpts...)
 
 	// TODO: add a way for users to pass in extra pyroscope options

core/metrics/consistentnames.go

@@ -15,6 +15,12 @@ const (
 	Nonce = "nonce"
 	// TxHash is the metric name for the transaction hash.
 	TxHash = "tx_hash"
+	// BlockHash is the metric name for the block hash.
+	BlockHash = "block_hash"
+	// FromBlock is the metric name for the block number.
+	FromBlock = "from_block"
+	// ToBlock is the metric name for the to block.
+	ToBlock = "to_block"
 	// Page is the metric name for the page.
 	Page = "page"
 	// Contract is the metric name for the contract.

core/metrics/instrumentation/http.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/synapsecns/sanguine/core/metrics"
+	"github.com/valyala/fastjson"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 	"io"
@@ -32,6 +33,8 @@ const (
 	RequestEventName = "http.request.body"
 	// ResponseEventName is the name of the event created for each response body.
 	ResponseEventName = "http.response.body"
+	// IDSpanName is the name of the span created for each request.
+	IDSpanName = "rpc.id"
 )
 
 // nolint: cyclop
@@ -97,7 +100,15 @@ func (t *captureTransport) RoundTrip(req *http.Request) (_ *http.Response, err e
 
 	// Add the request/response body as events to the span
 	if requestBody.Len() > 0 {
-		span.AddEvent(RequestEventName, trace.WithAttributes(attribute.String("body", requestBody.String())))
+		requestBytes := requestBody.Bytes()
+		// now track the id in a way we can search in signoz.
+		// will not work on batch requests.
+		id := fastjson.GetInt(requestBytes, "id")
+		if id != 0 {
+			span.SetAttributes(attribute.Int(IDSpanName, id))
+		}
+
+		span.AddEvent(RequestEventName, trace.WithAttributes(attribute.String("body", string(requestBytes))))
 	}
 	if len(response) > 0 {
 		span.AddEvent(ResponseEventName, trace.WithAttributes(attribute.String("body", response)))

ethergo/client/client.go

@@ -15,6 +15,7 @@ import (
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 	"math/big"
+	"reflect"
 )
 
 // EVM is the set of functions that the scribe needs from a client.
@@ -121,8 +122,8 @@ func (c *clientImpl) BatchCallContext(ctx context.Context, b []rpc.BatchElem) (e
 	return c.captureClient.rpcClient.BatchCallContext(requestCtx, b)
 }
 
-func (c *clientImpl) startSpan(parentCtx context.Context, method RPCMethod) (context.Context, trace.Span) {
-	ctx, span := c.tracing.Tracer().Start(parentCtx, method.String())
+func (c *clientImpl) startSpan(parentCtx context.Context, method RPCMethod, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
+	ctx, span := c.tracing.Tracer().Start(parentCtx, method.String(), opts...)
 	span.SetAttributes(attribute.String("endpoint", c.endpoint))
 
 	return ctx, span
@@ -132,19 +133,41 @@ func (c *clientImpl) startSpan(parentCtx context.Context, method RPCMethod) (con
 //
 //nolint:wrapcheck
 func (c *clientImpl) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) (contractResponse []byte, err error) {
-	requestCtx, span := c.startSpan(ctx, CallMethod)
+	requestCtx, span := c.startSpan(ctx, CallMethod, trace.WithAttributes(attribute.String(metrics.ContractAddress, nillableToString(call.To)), attribute.String("data", common.Bytes2Hex(call.Data)), attribute.Bool("pending", false)))
 	defer func() {
 		metrics.EndSpanWithErr(span, err)
 	}()
 
 	return c.getEthClient().CallContract(requestCtx, call, blockNumber)
 }
 
+// toStrings converts a slice of any type that satisfies the Stringer interface into a slice of strings.
+func toStrings[T fmt.Stringer](items []T) (res []string) {
+	for _, item := range items {
+		res = append(res, item.String())
+	}
+	return res
+}
+
+func nillableToString(nillable fmt.Stringer) string {
+	if nillable == nil {
+		return ""
+	}
+
+	// Use reflection to check if the interface's underlying value is nil.
+	val := reflect.ValueOf(nillable)
+	if val.Kind() == reflect.Ptr && val.IsNil() {
+		return ""
+	}
+
+	return nillable.String()
+}
+
 // PendingCallContract calls contract on the underlying client
 //
 //nolint:wrapcheck
 func (c *clientImpl) PendingCallContract(ctx context.Context, call ethereum.CallMsg) (contractResponse []byte, err error) {
-	requestCtx, span := c.startSpan(ctx, CallMethod)
+	requestCtx, span := c.startSpan(ctx, CallMethod, trace.WithAttributes(attribute.String(metrics.ContractAddress, nillableToString(call.To)), attribute.String("data", common.Bytes2Hex(call.Data)), attribute.Bool("pending", true)))
 	defer func() {
 		metrics.EndSpanWithErr(span, err)
 	}()
@@ -271,11 +294,21 @@ func (c *clientImpl) SendTransaction(ctx context.Context, tx *types.Transaction)
 	return c.getEthClient().SendTransaction(requestCtx, tx)
 }
 
+func queryToTraceParams(query ethereum.FilterQuery) (res []attribute.KeyValue) {
+	res = append(res, attribute.String(metrics.BlockHash, nillableToString(query.BlockHash)))
+	res = append(res, attribute.String(metrics.FromBlock, nillableToString(query.FromBlock)))
+	res = append(res, attribute.String(metrics.ToBlock, nillableToString(query.ToBlock)))
+	res = append(res, attribute.StringSlice("addresses", toStrings(query.Addresses)))
+	// TODO: add topics
+
+	return res
+}
+
 // FilterLogs calls FilterLogs on the underlying client
 //
 //nolint:wrapcheck
 func (c *clientImpl) FilterLogs(ctx context.Context, query ethereum.FilterQuery) (logs []types.Log, err error) {
-	requestCtx, span := c.startSpan(ctx, GetLogsMethod)
+	requestCtx, span := c.startSpan(ctx, GetLogsMethod, trace.WithAttributes(queryToTraceParams(query)...))
 	defer func() {
 		metrics.EndSpanWithErr(span, err)
 	}()

ethergo/go.mod

@@ -232,6 +232,7 @@ require (
 	github.com/uptrace/opentelemetry-go-extra/otelutil v0.2.3 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otelzap v0.2.3 // indirect
 	github.com/urfave/cli/v2 v2.27.1 // indirect
+	github.com/valyala/fastjson v1.6.4 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect

ethergo/go.sum

@@ -1013,6 +1013,8 @@ github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6S
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
+github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
+github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=