Skip to content

Commit

Permalink
add open telemetry framework, and using it add 1 new metric (#296)
Browse files Browse the repository at this point in the history
  • Loading branch information
nirvanagit committed Jun 7, 2024
1 parent 6f1cd09 commit 74cd140
Show file tree
Hide file tree
Showing 8 changed files with 215 additions and 56 deletions.
4 changes: 2 additions & 2 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ orbs:

jobs:
build:
working_directory: /go/pkg/mod/github.com/admiral
working_directory: admiral
docker:
- image: circleci/golang:1.17
- image: cimg/go:1.21
steps:
- checkout
- run:
Expand Down
12 changes: 12 additions & 0 deletions admiral/pkg/clusters/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package clusters

import "github.com/istio-ecosystem/admiral/admiral/pkg/monitoring"

// List all metrics part of the clusters package here
var (
configWriterMeter = monitoring.NewMeter("config_writer")
totalConfigWriterEvents = monitoring.NewCounter(
"total_config_write_invocations",
"total number of times config writer was invoked",
monitoring.WithMeter(configWriterMeter))
)
31 changes: 18 additions & 13 deletions admiral/pkg/clusters/serviceentry.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"errors"
"fmt"
"go.opentelemetry.io/otel/attribute"
api "go.opentelemetry.io/otel/metric"
"math"
"math/rand"
"reflect"
Expand Down Expand Up @@ -81,7 +83,10 @@ func modifyServiceEntryForNewServiceOrPod(
log.Infof(LogFormat, event, env, sourceIdentity, "", "Processing skipped during cache warm up state")
return nil
}

totalConfigWriterEvents.Increment(api.WithAttributes(
attribute.Key("identity").String(sourceIdentity),
attribute.Key("environment").String(env),
))
var (
cname string
namespace string
Expand Down Expand Up @@ -340,9 +345,9 @@ func getAdmiralGeneratedVirtualService(ctx context.Context, remoteController *Re
return result, nil
}

//Does two things;
//i) Picks the GTP that was created most recently from the passed in GTP list based on GTP priority label (GTPs from all clusters)
//ii) Updates the global GTP cache with the selected GTP in i)
// Does two things;
// i) Picks the GTP that was created most recently from the passed in GTP list based on GTP priority label (GTPs from all clusters)
// ii) Updates the global GTP cache with the selected GTP in i)
func updateGlobalGtpCache(cache *AdmiralCache, identity, env string, gtps map[string][]*v1.GlobalTrafficPolicy) {
defer util.LogElapsedTime("updateGlobalGtpCache", identity, env, "")()
gtpsOrdered := make([]*v1.GlobalTrafficPolicy, 0)
Expand Down Expand Up @@ -414,7 +419,7 @@ func updateEndpointsForBlueGreen(rollout *argo.Rollout, weightedServices map[str
}
}

//update endpoints for Argo rollouts specific Service Entries to account for traffic splitting (Canary strategy)
// update endpoints for Argo rollouts specific Service Entries to account for traffic splitting (Canary strategy)
func updateEndpointsForWeightedServices(serviceEntry *networking.ServiceEntry, weightedServices map[string]*WeightedService, clusterIngress string, meshPorts map[string]uint32) {
var endpoints = make([]*networking.WorkloadEntry, 0)
var endpointToReplace *networking.WorkloadEntry
Expand Down Expand Up @@ -518,7 +523,7 @@ func copySidecar(sidecar *v1alpha3.Sidecar) *v1alpha3.Sidecar {
return newSidecarObj
}

//AddServiceEntriesWithDr will create the default service entries and also additional ones specified in GTP
// AddServiceEntriesWithDr will create the default service entries and also additional ones specified in GTP
func AddServiceEntriesWithDr(ctx context.Context, rr *RemoteRegistry, sourceClusters map[string]string,
serviceEntries map[string]*networking.ServiceEntry, isAdditionalEndpointsEnabled bool) {

Expand Down Expand Up @@ -659,9 +664,9 @@ func AddServiceEntriesWithDr(ctx context.Context, rr *RemoteRegistry, sourceClus

// This func returns a bool to indicate if additional endpoints generation is needed
// based on the following conditions.
// 1. Additional endpoint suffixes have been configured in the admiral params
// 2. The rollout/deployment labels passed contains any of the allowed labels
// configured in the admiral params 'additional_endpoint_label_filters'
// 1. Additional endpoint suffixes have been configured in the admiral params
// 2. The rollout/deployment labels passed contains any of the allowed labels
// configured in the admiral params 'additional_endpoint_label_filters'
func doGenerateAdditionalEndpoints(labels map[string]string) bool {
additionalEndpointSuffixes := common.GetAdditionalEndpointSuffixes()
if len(additionalEndpointSuffixes) <= 0 {
Expand Down Expand Up @@ -885,8 +890,8 @@ func loadServiceEntryCacheData(ctx context.Context, c admiral.ConfigMapControlle

}

//GetLocalAddressForSe gets a guarenteed unique local address for a serviceentry. Returns the address, True iff the configmap was updated false otherwise, and an error if any
//Any error coupled with an empty string address means the method should be retried
// GetLocalAddressForSe gets a guarenteed unique local address for a serviceentry. Returns the address, True iff the configmap was updated false otherwise, and an error if any
// Any error coupled with an empty string address means the method should be retried
func GetLocalAddressForSe(ctx context.Context, seName string, seAddressCache *ServiceEntryAddressStore, configMapController admiral.ConfigMapControllerInterface) (string, bool, error) {
var address = seAddressCache.EntryAddresses[seName]
if len(address) == 0 {
Expand Down Expand Up @@ -914,7 +919,7 @@ func GetServiceEntriesByCluster(ctx context.Context, clusterID string, remoteReg
}
}

//GenerateNewAddressAndAddToConfigMap an atomic fetch and update operation against the configmap (using K8s built in optimistic consistency mechanism via resource version)
// GenerateNewAddressAndAddToConfigMap an atomic fetch and update operation against the configmap (using K8s built in optimistic consistency mechanism via resource version)
func GenerateNewAddressAndAddToConfigMap(ctx context.Context, seName string, configMapController admiral.ConfigMapControllerInterface) (string, error) {
//1. get cm, see if there. 2. gen new uq address. 3. put configmap. RETURN SUCCESSFULLY IFF CONFIGMAP PUT SUCCEEDS
cm, err := configMapController.GetConfigMap(ctx)
Expand Down Expand Up @@ -956,7 +961,7 @@ func GenerateNewAddressAndAddToConfigMap(ctx context.Context, seName string, con
return address, nil
}

//puts new data into an existing configmap. Providing the original is necessary to prevent fetch and update race conditions
// puts new data into an existing configmap. Providing the original is necessary to prevent fetch and update race conditions
func putServiceEntryStateFromConfigmap(ctx context.Context, c admiral.ConfigMapControllerInterface, originalConfigmap *k8sV1.ConfigMap, data *ServiceEntryAddressStore) error {
if originalConfigmap == nil {
return errors.New("configmap must not be nil")
Expand Down
5 changes: 5 additions & 0 deletions admiral/pkg/controller/common/config.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package common

import (
"github.com/istio-ecosystem/admiral/admiral/pkg/monitoring"
"time"

"github.com/matryer/resync"
Expand All @@ -23,6 +24,10 @@ func InitializeConfig(params AdmiralParams) {
admiralParams = params
initHappened = true
InitializeMetrics()
err := monitoring.InitializeMonitoring()
if err != nil {
log.Errorf("failed to setup monitoring: %v", err)
}
})
if initHappened {
log.Info("InitializeConfig was called.")
Expand Down
61 changes: 61 additions & 0 deletions admiral/pkg/monitoring/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package monitoring

import (
"context"
"log"
"reflect"

api "go.opentelemetry.io/otel/metric"
)

// Metric interface for abstracting the different operations
// for the various metric types provided by open telemetry
type Metric interface {
Increment(attributes api.MeasurementOption)
Name() string
}

// NewCounter returns a new counter
func NewCounter(name, description string, opts ...Options) Metric {
o := createOptions(opts...)
return newFloat64Counter(name, description, o)
}

type counter struct {
name string
description string
ctx context.Context
int64Counter api.Int64Counter
}

// Increment increases the value of the counter by 1, and adds the provided attributes
func (c *counter) Increment(attributes api.MeasurementOption) {
c.int64Counter.Add(c.ctx, 1, attributes)
}

// Name returns the name of the metric
func (c *counter) Name() string {
return c.name
}

func newFloat64Counter(name, description string, opts *options) *counter {
ctx := context.TODO()
meter := defaultMeter
if reflect.ValueOf(opts.meter).IsValid() {
meter = opts.meter
}
int64Counter, err := meter.Int64Counter(
name,
api.WithUnit("1"),
api.WithDescription(description),
)
if err != nil {
log.Fatalf("error creating int64 counter: %v", err)
}
return &counter{
name: name,
description: description,
ctx: ctx,
int64Counter: int64Counter,
}
}
47 changes: 47 additions & 0 deletions admiral/pkg/monitoring/monitoring.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package monitoring

import (
"go.opentelemetry.io/otel/exporters/prometheus"
api "go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/sdk/metric"
)

var (
meterName = "admiral_monitoring"
exporter, _ = prometheus.New()
provider = metric.NewMeterProvider(metric.WithReader(exporter))
defaultMeter = provider.Meter(meterName)
)

func InitializeMonitoring() error {
return nil
}

// Options accepts a pointer to options. It is used
// to update the options by calling an array of functions
type Options func(*options)

// NewMeter creates a new meter which defines the metric scope
func NewMeter(meterName string) api.Meter {
return provider.Meter(meterName)
}

// WithMeter configures the given Meter
func WithMeter(meter api.Meter) Options {
return func(opts *options) {
opts.meter = meter
}
}

type options struct {
meter api.Meter
unit string
}

func createOptions(opts ...Options) *options {
o := &options{}
for _, opt := range opts {
opt(o)
}
return o
}
44 changes: 26 additions & 18 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
module github.com/istio-ecosystem/admiral

go 1.18
go 1.21

require (
github.com/argoproj/argo-rollouts v1.2.1
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/go-openapi/swag v0.19.15 // indirect
github.com/golang/protobuf v1.5.2
github.com/google/go-cmp v0.5.6
github.com/golang/protobuf v1.5.3
github.com/google/go-cmp v0.6.0
github.com/gorilla/mux v1.8.0
github.com/imdario/mergo v0.3.12 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/onsi/gomega v1.19.0
github.com/prometheus/client_golang v1.12.2
github.com/prometheus/client_model v0.2.0
github.com/prometheus/client_golang v1.19.1
github.com/prometheus/client_model v0.6.1
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.5.0
github.com/stretchr/testify v1.7.0
golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
github.com/stretchr/testify v1.9.0
golang.org/x/net v0.20.0 // indirect
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
google.golang.org/genproto v0.0.0-20220531134929-86cf59382f1b // indirect
gopkg.in/yaml.v2 v2.4.0
Expand All @@ -30,22 +30,31 @@ require (
)

require (
github.com/prometheus/common v0.32.1
google.golang.org/protobuf v1.28.1
github.com/prometheus/common v0.53.0
go.opentelemetry.io/otel v1.27.0
go.opentelemetry.io/otel/exporters/prometheus v0.49.0
go.opentelemetry.io/otel/metric v1.27.0
go.opentelemetry.io/otel/sdk/metric v1.27.0
google.golang.org/protobuf v1.34.1
)

require github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
require (
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
go.opentelemetry.io/otel/sdk v1.27.0 // indirect
go.opentelemetry.io/otel/trace v1.27.0 // indirect
)

require (
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.8.0 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-logr/logr v1.2.0 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand All @@ -63,13 +72,12 @@ require (
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/prometheus/procfs v0.15.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
golang.org/x/oauth2 v0.16.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/term v0.16.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
Expand Down
Loading

0 comments on commit 74cd140

Please sign in to comment.