Skip to content

Commit

Permalink
Merge da1678e into 741fc06
Browse files Browse the repository at this point in the history
  • Loading branch information
abtestingalpha authored Sep 16, 2024
2 parents 741fc06 + da1678e commit 77859d5
Show file tree
Hide file tree
Showing 29 changed files with 1,196 additions and 31 deletions.
2 changes: 2 additions & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,8 @@ linters:
- perfsprint
# malfunctions on embedded structs
- typecheck
# magic numbers
- mnd
fast: false

issues:
Expand Down
2 changes: 1 addition & 1 deletion core/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ require (
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.27.0
golang.org/x/sync v0.8.0
google.golang.org/grpc v1.64.0
gorm.io/driver/sqlite v1.5.6
gorm.io/gorm v1.25.10
k8s.io/apimachinery v0.29.3
Expand Down Expand Up @@ -189,7 +190,6 @@ require (
golang.org/x/tools v0.24.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
google.golang.org/grpc v1.64.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
Expand Down
22 changes: 21 additions & 1 deletion core/metrics/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,27 @@ There's also a `NAME_PREFIX` environment variable that will prefix all the metri

## OTLP

We do our best to support enviornment variables specified in the [Otel Spec](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) and have added a few of our own. Key ones to note are:
We do our best to support enviornment variables specified in the [Otel Spec](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) and the [OTLP Spec](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/) and have added a few of our own. This was to allow for multiple exporter backends for traces, as otel clients only allow for one URL. The relevant multi exporter code is in `multiexporter.go`, and simply wraps multiple otel clients.

The additional environment variables to note are:
| Enviornment Variable | Description | Default |
|------------------------------------------|-------------------------------------------|---------|
| `OTEL_EXPORTER_OTLP_ENDPOINT` | The endpoint for the primary OTLP exporter | None |
| `OTEL_EXPORTER_OTLP_ENDPOINT_1` | The endpoint for the first additional OTLP exporter | None |
| `OTEL_EXPORTER_OTLP_ENDPOINT_2` | The endpoint for the second additional OTLP exporter | None |
| `OTEL_EXPORTER_OTLP_ENDPOINT_3` | The endpoint for the third additional OTLP exporter | None |
| ... | Additional endpoints can be added by incrementing the number | None |
| `OTEL_EXPORTER_OTLP_TRANSPORT` | The transport protocol for the primary OTLP exporter | `http` |
| `OTEL_EXPORTER_OTLP_TRANSPORT_1` | The transport protocol for the first additional OTLP exporter | `http` |
| `OTEL_EXPORTER_OTLP_TRANSPORT_2` | The transport protocol for the second additional OTLP exporter | `http` |
| `OTEL_EXPORTER_OTLP_TRANSPORT_3` | The transport protocol for the third additional OTLP exporter | `http` |
| ... | Additional transports can be specified by incrementing the number | `http` |

You can do the same thing for `OTEL_EXPORTER_OTLP_SECURE_MODE` and `OTEL_EXPORTER_OTLP_HEADERS`

<!-- TODO: fully document these optins-->

Note: The OTLP exporter endpoints and transports can be specified for multiple exporters by using incrementing numbers (1, 2, 3, etc.) in the environment variable names. This allows for configuration of multiple OTLP exporters. The primary exporter uses the base names without numbers.


## Jaeger
Expand Down
6 changes: 6 additions & 0 deletions core/metrics/export_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package metrics

// HeadersToMap converts a string of headers to a map.
func HeadersToMap(val string) map[string]string {
return headersToMap(val)
}
83 changes: 83 additions & 0 deletions core/metrics/multiexporter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package metrics

import (
"context"
"fmt"
"go.uber.org/multierr"
"sync"
"time"

"go.opentelemetry.io/otel/sdk/trace"
)

// MultiExporter is an interface that allows exporting spans to multiple OTLP trace exporters.
type MultiExporter interface {
trace.SpanExporter
AddExporter(exporter trace.SpanExporter)
}

type multiExporter struct {
exporters []trace.SpanExporter
}

// NewMultiExporter creates a new multi exporter that forwards spans to multiple OTLP trace exporters.
// It takes in one or more trace.SpanExporter instances and ensures that spans are sent to all of them.
// This is useful when you need to send trace data to multiple backends or endpoints.
func NewMultiExporter(exporters ...trace.SpanExporter) MultiExporter {
return &multiExporter{
exporters: exporters,
}
}

const defaultTimeout = 30 * time.Second

// ExportSpans exports a batch of spans.
func (m *multiExporter) ExportSpans(parentCtx context.Context, ss []trace.ReadOnlySpan) error {
return m.doParallel(parentCtx, func(ctx context.Context, exporter trace.SpanExporter) error {
return exporter.ExportSpans(ctx, ss)
})
}

func (m *multiExporter) doParallel(parentCtx context.Context, fn func(context.Context, trace.SpanExporter) error) error {
ctx, cancel := context.WithTimeout(parentCtx, defaultTimeout)
defer cancel()

var wg sync.WaitGroup
var errors []error
var mu sync.Mutex

wg.Add(len(m.exporters))
for _, exporter := range m.exporters {
go func(exporter trace.SpanExporter) {
defer wg.Done()
err := fn(ctx, exporter)
if err != nil {
mu.Lock()
errors = append(errors, fmt.Errorf("error in doMultiple: %w", err))
mu.Unlock()
}
}(exporter)
}

wg.Wait()
if len(errors) > 0 {
// nolint: wrapcheck
return multierr.Combine(errors...)
}

return nil
}

// Shutdown notifies the exporter of a pending halt to operations.
func (m *multiExporter) Shutdown(ctx context.Context) error {
return m.doParallel(ctx, func(ctx context.Context, exporter trace.SpanExporter) error {
return exporter.Shutdown(ctx)
})
}

// AddExporter adds an exporter to the multi exporter.
func (m *multiExporter) AddExporter(exporter trace.SpanExporter) {
m.exporters = append(m.exporters, exporter)
}

var _ trace.SpanExporter = &multiExporter{}
45 changes: 45 additions & 0 deletions core/metrics/multiexporter_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package metrics_test

import (
"context"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/synapsecns/sanguine/core/metrics"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
)

func TestMultiExporter(t *testing.T) {
// Create in-memory exporters
exporter1 := tracetest.NewInMemoryExporter()
exporter2 := tracetest.NewInMemoryExporter()

// Create multi-exporter
multiExporter := metrics.NewMultiExporter(exporter1, exporter2)

// Create test spans
spans := []sdktrace.ReadOnlySpan{
tracetest.SpanStub{}.Snapshot(),
tracetest.SpanStub{}.Snapshot(),
}

// Test ExportSpans
err := multiExporter.ExportSpans(context.Background(), spans)
require.NoError(t, err)

// Verify that spans were exported to both exporters
assert.Equal(t, 2, len(exporter1.GetSpans()))
assert.Equal(t, 2, len(exporter2.GetSpans()))

// Test Shutdown
err = multiExporter.Shutdown(context.Background())
require.NoError(t, err)

// Verify that both exporters were shut down
// Note: InMemoryExporter doesn't have a Stopped() method, so we can't check this directly
// Instead, we can try to export spans again and check for an error
err = multiExporter.ExportSpans(context.Background(), spans)
assert.NoError(t, err, "Expected no error after shutdown")
}
Loading

0 comments on commit 77859d5

Please sign in to comment.