Skip to content

Commit

Permalink
feat: Send plugin logs via OTEL
Browse files Browse the repository at this point in the history
  • Loading branch information
@erezrokah
erezrokah committed Jul 19, 2024
1 parent 14c373f commit c214c24
Show file tree
Hide file tree
Showing 4 changed files with 180 additions and 29 deletions.
19 changes: 11 additions & 8 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,17 @@ require (
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
github.com/thoas/go-funk v0.9.3
go.opentelemetry.io/otel v1.27.0
go.opentelemetry.io/otel v1.28.0
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.4.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0
go.opentelemetry.io/otel/metric v1.27.0
go.opentelemetry.io/otel/sdk v1.27.0
go.opentelemetry.io/otel/log v0.4.0
go.opentelemetry.io/otel/metric v1.28.0
go.opentelemetry.io/otel/sdk v1.28.0
go.opentelemetry.io/otel/sdk/log v0.4.0
go.opentelemetry.io/otel/sdk/metric v1.27.0
go.opentelemetry.io/otel/trace v1.27.0
go.opentelemetry.io/otel/trace v1.28.0
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8
golang.org/x/sync v0.7.0
golang.org/x/text v0.16.0
Expand All @@ -42,7 +45,7 @@ require (
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/flatbuffers v24.3.25+incompatible // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
Expand All @@ -60,14 +63,14 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opentelemetry.io/proto/otlp v1.2.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/tools v0.22.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
38 changes: 22 additions & 16 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4Nij
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
Expand Down Expand Up @@ -115,24 +115,30 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg=
go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ=
go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.4.0 h1:zBPZAISA9NOc5cE8zydqDiS0itvg/P/0Hn9m72a5gvM=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.4.0/go.mod h1:gcj2fFjEsqpV3fXuzAA+0Ze1p2/4MJ4T7d77AmkvueQ=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 h1:CIHWikMsN3wO+wq1Tp5VGdVRTcON+DmOJSfDjXypKOc=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0/go.mod h1:TNupZ6cxqyFEpLXAZW7On+mLFL0/g0TE3unIYL91xWc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY=
go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik=
go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak=
go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI=
go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A=
go.opentelemetry.io/otel/log v0.4.0 h1:/vZ+3Utqh18e8TPjuc3ecg284078KWrR8BRz+PQAj3o=
go.opentelemetry.io/otel/log v0.4.0/go.mod h1:DhGnQvky7pHy82MIRV43iXh3FlKN8UUKftn0KbLOq6I=
go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE=
go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
go.opentelemetry.io/otel/sdk/log v0.4.0 h1:1mMI22L82zLqf6KtkjrRy5BbagOTWdJsqMY/HSqILAA=
go.opentelemetry.io/otel/sdk/log v0.4.0/go.mod h1:AYJ9FVF0hNOgAVzUG/ybg/QttnXhUePWAupmCqtdESo=
go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI=
go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw=
go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw=
go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4=
go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94=
go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A=
go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY=
golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
Expand All @@ -156,10 +162,10 @@ golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSm
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ=
gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo=
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw=
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0=
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
Expand Down
147 changes: 143 additions & 4 deletions serve/opentelemetry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,27 @@ package serve

import (
"context"
"encoding/json"
"fmt"
"reflect"
"strings"
"time"

"github.com/cloudquery/plugin-sdk/v4/plugin"
"github.com/rs/zerolog"

"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
otellog "go.opentelemetry.io/otel/log"
logglobal "go.opentelemetry.io/otel/log/global"
"go.opentelemetry.io/otel/sdk/log"
"go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
"go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.25.0"
semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
)

// newResource returns a resource describing this application.
Expand Down Expand Up @@ -116,9 +122,41 @@ func getMetricReader(ctx context.Context, opts otelConfig) (*metric.PeriodicRead
return reader, nil
}

func getLogsProcessor(ctx context.Context, opts otelConfig) (*log.BatchProcessor, error) {
if opts.endpoint == "" {
return nil, nil
}

logOptions := []otlploghttp.Option{
otlploghttp.WithEndpoint(opts.endpoint),
otlploghttp.WithCompression(otlploghttp.GzipCompression),
}

if opts.insecure {
logOptions = append(logOptions, otlploghttp.WithInsecure())
}

if len(opts.headers) > 0 {
headers := parseOtelHeaders(opts.headers)
logOptions = append(logOptions, otlploghttp.WithHeaders(headers))
}

if opts.urlPath != "" {
logOptions = append(logOptions, otlploghttp.WithURLPath(opts.urlPath))
}

exporter, err := otlploghttp.New(ctx, logOptions...)
if err != nil {
return nil, fmt.Errorf("creating OTLP log exporter: %w", err)
}

processor := log.NewBatchProcessor(exporter)
return processor, nil
}

func setupOtel(ctx context.Context, logger zerolog.Logger, p *plugin.Plugin, otelEndpoint string, otelEndpointInsecure bool, otelEndpointHeaders []string, otelEndpointURLPath string) (shutdown func(), err error) {
if otelEndpoint == "" {
return func() {}, nil
return nil, nil
}
opts := otelConfig{
endpoint: otelEndpoint,
Expand All @@ -136,6 +174,11 @@ func setupOtel(ctx context.Context, logger zerolog.Logger, p *plugin.Plugin, ote
return nil, err
}

logsProcessor, err := getLogsProcessor(ctx, opts)
if err != nil {
return nil, err
}

pluginResource := newResource(p)
tp := trace.NewTracerProvider(
trace.WithBatcher(traceExporter),
Expand All @@ -146,20 +189,116 @@ func setupOtel(ctx context.Context, logger zerolog.Logger, p *plugin.Plugin, ote
metric.WithReader(metricReader),
metric.WithResource(pluginResource),
)

lp := log.NewLoggerProvider(
log.WithProcessor(logsProcessor),
log.WithResource(newResource(p)),
)

otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
logger.Warn().Err(err).Msg("otel error")
}))
otel.SetTracerProvider(tp)
otel.SetMeterProvider(mt)
logglobal.SetLoggerProvider(lp)

shutdown = func() {
if err := tp.Shutdown(context.Background()); err != nil {
logger.Error().Err(err).Msg("failed to shutdown OTLP trace exporter")
logger.Error().Err(err).Msg("failed to shutdown OTLP trace provider")
}
if err := mt.Shutdown(context.Background()); err != nil {
logger.Error().Err(err).Msg("failed to shutdown OTLP metric exporter")
logger.Error().Err(err).Msg("failed to shutdown OTLP metric provider")
}
if err := lp.Shutdown(context.Background()); err != nil {
logger.Error().Err(err).Msg("failed to shutdown OTLP logger provider")
}
}

return shutdown, nil
}

// Similar to https://github.com/AkhigbeEromo/opentelemetry-go-contrib/blob/dedcf91a55a36a5a8589c56f2e43c188eb42f4f2/bridges/otelzerolog/hook.go
// but with `TraceLevel` and attributes support
type otelLoggerHook struct {
otellog.Logger
ctx context.Context
}

func (h *otelLoggerHook) Run(e *zerolog.Event, level zerolog.Level, message string) {
record := otellog.Record{}
record.SetTimestamp(time.Now().UTC())
record.SetSeverity(otellogSeverity(level))
record.SetBody(otellog.StringValue(message))
// See https://github.com/rs/zerolog/issues/493, this is ugly but it works
logData := make(map[string]any)
eventBuffer := fmt.Sprintf("%s}", reflect.ValueOf(e).Elem().FieldByName("buf"))
err := json.Unmarshal([]byte(eventBuffer), &logData)
if err == nil {
recordAttributes := make([]otellog.KeyValue, 0, len(logData))
for k, v := range logData {
if k == "level" {
continue
}
if k == "time" {
eventTimestamp, ok := v.(string)
if !ok {
continue
}
t, err := time.Parse(time.RFC3339Nano, eventTimestamp)
if err == nil {
record.SetTimestamp(t)
continue
}
}
var attributeValue otellog.Value
switch v := v.(type) {
case string:
attributeValue = otellog.StringValue(v)
case int:
attributeValue = otellog.IntValue(v)
case int64:
attributeValue = otellog.Int64Value(v)
case float64:
attributeValue = otellog.Float64Value(v)
case bool:
attributeValue = otellog.BoolValue(v)
case []byte:
attributeValue = otellog.BytesValue(v)
default:
attributeValue = otellog.StringValue(fmt.Sprintf("%v", v))
}
recordAttributes = append(recordAttributes, otellog.KeyValue{
Key: k,
Value: attributeValue,
})
}
record.AddAttributes(recordAttributes...)
}

h.Emit(h.ctx, record)
}

func otellogSeverity(level zerolog.Level) otellog.Severity {
switch level {
case zerolog.DebugLevel:
return otellog.SeverityDebug
case zerolog.InfoLevel:
return otellog.SeverityInfo
case zerolog.WarnLevel:
return otellog.SeverityWarn
case zerolog.ErrorLevel:
return otellog.SeverityError
case zerolog.FatalLevel:
return otellog.SeverityFatal2
case zerolog.PanicLevel:
return otellog.SeverityFatal1
case zerolog.TraceLevel:
return otellog.SeverityTrace
default:
return otellog.SeverityUndefined
}
}

func newOTELLoggerHook() zerolog.Hook {
return &otelLoggerHook{logglobal.GetLoggerProvider().Logger("cloudquery"), context.Background()}
}
5 changes: 4 additions & 1 deletion serve/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,10 @@ func (s *PluginServe) newCmdPluginServe() *cobra.Command {
if err != nil {
return fmt.Errorf("failed to setup OpenTelemetry: %w", err)
}
defer shutdown()
if shutdown != nil {
logger = logger.Hook(newOTELLoggerHook())
defer shutdown()
}

if licenseFile != "" {
switch err := premium.ValidateLicense(logger, s.plugin.Meta(), licenseFile); err {
Expand Down

0 comments on commit c214c24

Please sign in to comment.