add open telemetry framework, and using it add 1 new metric

.circleci/config.yml

@@ -5,9 +5,9 @@ orbs:
 
 jobs:
   build:
-    working_directory: /go/pkg/mod/github.com/admiral
+    working_directory: admiral
     docker:
-    - image: circleci/golang:1.17
+    - image: cimg/go:1.21
     steps:
     - checkout
     - run:

admiral/pkg/clusters/metrics.go

@@ -0,0 +1,12 @@
+package clusters
+
+import "github.com/istio-ecosystem/admiral/admiral/pkg/monitoring"
+
+// List all metrics part of the clusters package here
+var (
+	configWriterMeter       = monitoring.NewMeter("config_writer")
+	totalConfigWriterEvents = monitoring.NewCounter(
+		"total_config_write_invocations",
+		"total number of times config writer was invoked",
+		monitoring.WithMeter(configWriterMeter))
+)

admiral/pkg/clusters/serviceentry.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"go.opentelemetry.io/otel/attribute"
+	api "go.opentelemetry.io/otel/metric"
 	"math"
 	"math/rand"
 	"reflect"
@@ -81,7 +83,10 @@ func modifyServiceEntryForNewServiceOrPod(
 		log.Infof(LogFormat, event, env, sourceIdentity, "", "Processing skipped during cache warm up state")
 		return nil
 	}
-
+	totalConfigWriterEvents.Increment(api.WithAttributes(
+		attribute.Key("identity").String(sourceIdentity),
+		attribute.Key("environment").String(env),
+	))
 	var (
 		cname                                 string
 		namespace                             string
@@ -340,9 +345,9 @@ func getAdmiralGeneratedVirtualService(ctx context.Context, remoteController *Re
 	return result, nil
 }
 
-//Does two things;
-//i)  Picks the GTP that was created most recently from the passed in GTP list based on GTP priority label (GTPs from all clusters)
-//ii) Updates the global GTP cache with the selected GTP in i)
+// Does two things;
+// i)  Picks the GTP that was created most recently from the passed in GTP list based on GTP priority label (GTPs from all clusters)
+// ii) Updates the global GTP cache with the selected GTP in i)
 func updateGlobalGtpCache(cache *AdmiralCache, identity, env string, gtps map[string][]*v1.GlobalTrafficPolicy) {
 	defer util.LogElapsedTime("updateGlobalGtpCache", identity, env, "")()
 	gtpsOrdered := make([]*v1.GlobalTrafficPolicy, 0)
@@ -414,7 +419,7 @@ func updateEndpointsForBlueGreen(rollout *argo.Rollout, weightedServices map[str
 	}
 }
 
-//update endpoints for Argo rollouts specific Service Entries to account for traffic splitting (Canary strategy)
+// update endpoints for Argo rollouts specific Service Entries to account for traffic splitting (Canary strategy)
 func updateEndpointsForWeightedServices(serviceEntry *networking.ServiceEntry, weightedServices map[string]*WeightedService, clusterIngress string, meshPorts map[string]uint32) {
 	var endpoints = make([]*networking.WorkloadEntry, 0)
 	var endpointToReplace *networking.WorkloadEntry
@@ -518,7 +523,7 @@ func copySidecar(sidecar *v1alpha3.Sidecar) *v1alpha3.Sidecar {
 	return newSidecarObj
 }
 
-//AddServiceEntriesWithDr will create the default service entries and also additional ones specified in GTP
+// AddServiceEntriesWithDr will create the default service entries and also additional ones specified in GTP
 func AddServiceEntriesWithDr(ctx context.Context, rr *RemoteRegistry, sourceClusters map[string]string,
 	serviceEntries map[string]*networking.ServiceEntry, isAdditionalEndpointsEnabled bool) {
 
@@ -659,9 +664,9 @@ func AddServiceEntriesWithDr(ctx context.Context, rr *RemoteRegistry, sourceClus
 
 // This func returns a bool to indicate if additional endpoints generation is needed
 // based on the following conditions.
-// 1. Additional endpoint suffixes have been configured in the admiral params
-// 2. The rollout/deployment labels passed contains any of the allowed labels
-//    configured in the admiral params 'additional_endpoint_label_filters'
+//  1. Additional endpoint suffixes have been configured in the admiral params
+//  2. The rollout/deployment labels passed contains any of the allowed labels
+//     configured in the admiral params 'additional_endpoint_label_filters'
 func doGenerateAdditionalEndpoints(labels map[string]string) bool {
 	additionalEndpointSuffixes := common.GetAdditionalEndpointSuffixes()
 	if len(additionalEndpointSuffixes) <= 0 {
@@ -885,8 +890,8 @@ func loadServiceEntryCacheData(ctx context.Context, c admiral.ConfigMapControlle
 
 }
 
-//GetLocalAddressForSe gets a guarenteed unique local address for a serviceentry. Returns the address, True iff the configmap was updated false otherwise, and an error if any
-//Any error coupled with an empty string address means the method should be retried
+// GetLocalAddressForSe gets a guarenteed unique local address for a serviceentry. Returns the address, True iff the configmap was updated false otherwise, and an error if any
+// Any error coupled with an empty string address means the method should be retried
 func GetLocalAddressForSe(ctx context.Context, seName string, seAddressCache *ServiceEntryAddressStore, configMapController admiral.ConfigMapControllerInterface) (string, bool, error) {
 	var address = seAddressCache.EntryAddresses[seName]
 	if len(address) == 0 {
@@ -914,7 +919,7 @@ func GetServiceEntriesByCluster(ctx context.Context, clusterID string, remoteReg
 	}
 }
 
-//GenerateNewAddressAndAddToConfigMap an atomic fetch and update operation against the configmap (using K8s built in optimistic consistency mechanism via resource version)
+// GenerateNewAddressAndAddToConfigMap an atomic fetch and update operation against the configmap (using K8s built in optimistic consistency mechanism via resource version)
 func GenerateNewAddressAndAddToConfigMap(ctx context.Context, seName string, configMapController admiral.ConfigMapControllerInterface) (string, error) {
 	//1. get cm, see if there. 2. gen new uq address. 3. put configmap. RETURN SUCCESSFULLY IFF CONFIGMAP PUT SUCCEEDS
 	cm, err := configMapController.GetConfigMap(ctx)
@@ -956,7 +961,7 @@ func GenerateNewAddressAndAddToConfigMap(ctx context.Context, seName string, con
 	return address, nil
 }
 
-//puts new data into an existing configmap. Providing the original is necessary to prevent fetch and update race conditions
+// puts new data into an existing configmap. Providing the original is necessary to prevent fetch and update race conditions
 func putServiceEntryStateFromConfigmap(ctx context.Context, c admiral.ConfigMapControllerInterface, originalConfigmap *k8sV1.ConfigMap, data *ServiceEntryAddressStore) error {
 	if originalConfigmap == nil {
 		return errors.New("configmap must not be nil")

admiral/pkg/controller/common/config.go

@@ -1,6 +1,7 @@
 package common
 
 import (
+	"github.com/istio-ecosystem/admiral/admiral/pkg/monitoring"
 	"time"
 
 	"github.com/matryer/resync"
@@ -23,6 +24,10 @@ func InitializeConfig(params AdmiralParams) {
 		admiralParams = params
 		initHappened = true
 		InitializeMetrics()
+		err := monitoring.InitializeMonitoring()
+		if err != nil {
+			log.Errorf("failed to setup monitoring: %v", err)
+		}
 	})
 	if initHappened {
 		log.Info("InitializeConfig was called.")

admiral/pkg/monitoring/metrics.go

@@ -0,0 +1,61 @@
+package monitoring
+
+import (
+	"context"
+	"log"
+	"reflect"
+
+	api "go.opentelemetry.io/otel/metric"
+)
+
+// Metric interface for abstracting the different operations
+// for the various metric types provided by open telemetry
+type Metric interface {
+	Increment(attributes api.MeasurementOption)
+	Name() string
+}
+
+// NewCounter returns a new counter
+func NewCounter(name, description string, opts ...Options) Metric {
+	o := createOptions(opts...)
+	return newFloat64Counter(name, description, o)
+}
+
+type counter struct {
+	name         string
+	description  string
+	ctx          context.Context
+	int64Counter api.Int64Counter
+}
+
+// Increment increases the value of the counter by 1, and adds the provided attributes
+func (c *counter) Increment(attributes api.MeasurementOption) {
+	c.int64Counter.Add(c.ctx, 1, attributes)
+}
+
+// Name returns the name of the metric
+func (c *counter) Name() string {
+	return c.name
+}
+
+func newFloat64Counter(name, description string, opts *options) *counter {
+	ctx := context.TODO()
+	meter := defaultMeter
+	if reflect.ValueOf(opts.meter).IsValid() {
+		meter = opts.meter
+	}
+	int64Counter, err := meter.Int64Counter(
+		name,
+		api.WithUnit("1"),
+		api.WithDescription(description),
+	)
+	if err != nil {
+		log.Fatalf("error creating int64 counter: %v", err)
+	}
+	return &counter{
+		name:         name,
+		description:  description,
+		ctx:          ctx,
+		int64Counter: int64Counter,
+	}
+}

admiral/pkg/monitoring/monitoring.go

@@ -0,0 +1,47 @@
+package monitoring
+
+import (
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	api "go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/sdk/metric"
+)
+
+var (
+	meterName    = "admiral_monitoring"
+	exporter, _  = prometheus.New()
+	provider     = metric.NewMeterProvider(metric.WithReader(exporter))
+	defaultMeter = provider.Meter(meterName)
+)
+
+func InitializeMonitoring() error {
+	return nil
+}
+
+// Options accepts a pointer to options. It is used
+// to update the options by calling an array of functions
+type Options func(*options)
+
+// NewMeter creates a new meter which defines the metric scope
+func NewMeter(meterName string) api.Meter {
+	return provider.Meter(meterName)
+}
+
+// WithMeter configures the given Meter
+func WithMeter(meter api.Meter) Options {
+	return func(opts *options) {
+		opts.meter = meter
+	}
+}
+
+type options struct {
+	meter api.Meter
+	unit  string
+}
+
+func createOptions(opts ...Options) *options {
+	o := &options{}
+	for _, opt := range opts {
+		opt(o)
+	}
+	return o
+}

go.mod

@@ -1,23 +1,23 @@
 module github.com/istio-ecosystem/admiral
 
-go 1.18
+go 1.21
 
 require (
 	github.com/argoproj/argo-rollouts v1.2.1
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/go-openapi/swag v0.19.15 // indirect
-	github.com/golang/protobuf v1.5.2
-	github.com/google/go-cmp v0.5.6
+	github.com/golang/protobuf v1.5.3
+	github.com/google/go-cmp v0.6.0
 	github.com/gorilla/mux v1.8.0
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/onsi/gomega v1.19.0
-	github.com/prometheus/client_golang v1.12.2
-	github.com/prometheus/client_model v0.2.0
+	github.com/prometheus/client_golang v1.19.1
+	github.com/prometheus/client_model v0.6.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.5.0
-	github.com/stretchr/testify v1.7.0
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
+	github.com/stretchr/testify v1.9.0
+	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/genproto v0.0.0-20220531134929-86cf59382f1b // indirect
 	gopkg.in/yaml.v2 v2.4.0
@@ -30,22 +30,31 @@ require (
 )
 
 require (
-	github.com/prometheus/common v0.32.1
-	google.golang.org/protobuf v1.28.1
+	github.com/prometheus/common v0.53.0
+	go.opentelemetry.io/otel v1.27.0
+	go.opentelemetry.io/otel/exporters/prometheus v0.49.0
+	go.opentelemetry.io/otel/metric v1.27.0
+	go.opentelemetry.io/otel/sdk/metric v1.27.0
+	google.golang.org/protobuf v1.34.1
 )
 
-require github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
+require (
+	github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	go.opentelemetry.io/otel/sdk v1.27.0 // indirect
+	go.opentelemetry.io/otel/trace v1.27.0 // indirect
+)
 
 require (
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
-	github.com/go-logr/logr v1.2.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -63,13 +72,12 @@ require (
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/procfs v0.15.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/oauth2 v0.16.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/term v0.16.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect