From 3cc793a5dd1a50a14ad05bf531b7eeba7923cbc6 Mon Sep 17 00:00:00 2001 From: Adil Fulara Date: Thu, 28 Apr 2022 11:10:08 -0700 Subject: [PATCH] Add Prometheus metric for number of clusters monitored (#208) - Add cli flag to enable metrics collection via prometheus. - Add endpoint to expose metrics in prometheus format when enabled and noop when disabled. - Add gauge to capture clusters monitored metric. --- admiral/cmd/admiral/cmd/root.go | 19 ++- .../pkg/apis/admiral/routes/handler_test.go | 124 ++++++++++-------- admiral/pkg/apis/admiral/routes/routes.go | 29 ++++ admiral/pkg/apis/admiral/server/server.go | 2 +- admiral/pkg/clusters/registry_test.go | 26 ++-- admiral/pkg/clusters/types.go | 4 +- admiral/pkg/controller/common/common_test.go | 2 +- admiral/pkg/controller/common/config.go | 9 ++ admiral/pkg/controller/common/config_test.go | 4 + admiral/pkg/controller/common/metrics.go | 31 +++++ admiral/pkg/controller/common/metrics_test.go | 39 ++++++ admiral/pkg/controller/common/types.go | 1 + .../pkg/controller/secret/secretcontroller.go | 5 + .../secret/secretcontroller_test.go | 31 ++++- go.mod | 2 + go.sum | 6 + 16 files changed, 253 insertions(+), 81 deletions(-) create mode 100644 admiral/pkg/controller/common/metrics.go create mode 100644 admiral/pkg/controller/common/metrics_test.go diff --git a/admiral/cmd/admiral/cmd/root.go b/admiral/cmd/admiral/cmd/root.go index 1e925405..d34462a5 100644 --- a/admiral/cmd/admiral/cmd/root.go +++ b/admiral/cmd/admiral/cmd/root.go @@ -11,6 +11,7 @@ import ( log "github.com/sirupsen/logrus" "os" "os/signal" + "sync" "syscall" "time" @@ -50,14 +51,27 @@ func GetRootCmd(args []string) *cobra.Command { } service := server.Service{} + metricsService := server.Service{} opts.RemoteRegistry = remoteRegistry - ret_routes := routes.NewAdmiralAPIServer(&opts) + + mainRoutes := routes.NewAdmiralAPIServer(&opts) + metricRoutes := routes.NewMetricsServer() if err != nil { log.Error("Error setting up server:", err.Error()) } - service.Start(ctx, 8080, ret_routes, routes.Filter, remoteRegistry) + wg := new(sync.WaitGroup) + wg.Add(2) + go func() { + metricsService.Start(ctx, 6900, metricRoutes, routes.Filter, remoteRegistry) + wg.Done() + }() + go func() { + service.Start(ctx, 8080, mainRoutes, routes.Filter, remoteRegistry) + wg.Done() + }() + wg.Wait() log.WithFields(log.Fields{ "error": err.Error(), @@ -116,6 +130,7 @@ func GetRootCmd(args []string) *cobra.Command { "The order would be to use annotation specified as `env_key`, followed by label specified as `env_key` and then fallback to the label `env`") rootCmd.PersistentFlags().StringVar(¶ms.LabelSet.GatewayApp, "gateway_app", "istio-ingressgateway", "The the value of the `app` label to use to match and find the service that represents the ingress for cross cluster traffic (AUTO_PASSTHROUGH mode)") + rootCmd.PersistentFlags().BoolVar(¶ms.MetricsEnabled, "metrics", true, "Enable prometheus metrics collections") return rootCmd } diff --git a/admiral/pkg/apis/admiral/routes/handler_test.go b/admiral/pkg/apis/admiral/routes/handler_test.go index 17cb9b64..f008004b 100644 --- a/admiral/pkg/apis/admiral/routes/handler_test.go +++ b/admiral/pkg/apis/admiral/routes/handler_test.go @@ -17,7 +17,7 @@ import ( "testing" ) -func TestReturnSuccessGET (t *testing.T) { +func TestReturnSuccessGET(t *testing.T) { url := "https://admiral.com/health" opts := RouteOpts{} r := httptest.NewRequest("GET", url, strings.NewReader("")) @@ -28,7 +28,18 @@ func TestReturnSuccessGET (t *testing.T) { assert.Equal(t, 200, resp.StatusCode) } -func TestGetClusters (t *testing.T) { +func TestReturnSuccessMetrics(t *testing.T) { + url := "https://admiral.com/metrics" + r := httptest.NewRequest("GET", url, strings.NewReader("")) + w := httptest.NewRecorder() + + Noop(w, r) + resp := w.Result() + + assert.Equal(t, 200, resp.StatusCode) +} + +func TestGetClusters(t *testing.T) { url := "https://admiral.com/clusters" opts := RouteOpts{ RemoteRegistry: &clusters.RemoteRegistry{ @@ -40,30 +51,30 @@ func TestGetClusters (t *testing.T) { }, } testCases := []struct { - name string + name string remoteCluster map[string]*secret.RemoteCluster - expectedErr interface{} - statusCode int + expectedErr interface{} + statusCode int }{ { - name: "success with two clusters case", - remoteCluster: map[string]*secret.RemoteCluster{ + name: "success with two clusters case", + remoteCluster: map[string]*secret.RemoteCluster{ "cluster1": {}, }, - expectedErr: []string{"cluster1"}, - statusCode: 200, + expectedErr: []string{"cluster1"}, + statusCode: 200, }, { - name: "success with no cluster case", - remoteCluster: map[string]*secret.RemoteCluster{}, - expectedErr: "No cluster is monitored by admiral", - statusCode: 200, + name: "success with no cluster case", + remoteCluster: map[string]*secret.RemoteCluster{}, + expectedErr: "No cluster is monitored by admiral", + statusCode: 200, }, } //Run the test for every provided case for _, c := range testCases { t.Run(c.name, func(t *testing.T) { - r:= httptest.NewRequest("GET", url, strings.NewReader("")) + r := httptest.NewRequest("GET", url, strings.NewReader("")) w := httptest.NewRecorder() opts.RemoteRegistry.SecretController.Cs.RemoteClusters = c.remoteCluster opts.GetClusters(w, r) @@ -72,7 +83,7 @@ func TestGetClusters (t *testing.T) { expectedOutput, _ := json.Marshal(c.expectedErr) if bytes.Compare(body, expectedOutput) != 0 { t.Errorf("Error mismatch. Got %v, want %v", string(body), c.expectedErr) - t.Errorf("%d",bytes.Compare(body, expectedOutput)) + t.Errorf("%d", bytes.Compare(body, expectedOutput)) } if c.statusCode != 200 && resp.StatusCode != c.statusCode { t.Errorf("Status code mismatch. Got %v, want %v", resp.StatusCode, c.statusCode) @@ -81,64 +92,64 @@ func TestGetClusters (t *testing.T) { } } -func TestGetServiceEntriesByCluster (t *testing.T) { +func TestGetServiceEntriesByCluster(t *testing.T) { url := "https://admiral.com/cluster/cluster1/serviceentries" opts := RouteOpts{ RemoteRegistry: &clusters.RemoteRegistry{}, } fakeIstioClient := istiofake.NewSimpleClientset() testCases := []struct { - name string - clusterName string + name string + clusterName string remoteControllers map[string]*clusters.RemoteController - expectedErr string - statusCode int + expectedErr string + statusCode int }{ { - name: "failure with admiral not monitored cluster", - clusterName: "bar", + name: "failure with admiral not monitored cluster", + clusterName: "bar", remoteControllers: nil, - expectedErr: "Admiral is not monitoring cluster bar\n", + expectedErr: "Admiral is not monitoring cluster bar\n", statusCode: 404, }, { - name: "failure with cluster not provided request", - clusterName: "", + name: "failure with cluster not provided request", + clusterName: "", remoteControllers: nil, - expectedErr: "Cluster name not provided as part of the request\n", + expectedErr: "Cluster name not provided as part of the request\n", statusCode: 400, }, { - name: "success with no service entry for cluster", - clusterName: "cluster1", - remoteControllers:map[string]*clusters.RemoteController{ + name: "success with no service entry for cluster", + clusterName: "cluster1", + remoteControllers: map[string]*clusters.RemoteController{ "cluster1": &clusters.RemoteController{ - ServiceEntryController: &istio.ServiceEntryController{ + ServiceEntryController: &istio.ServiceEntryController{ IstioClient: fakeIstioClient, }, }, }, - expectedErr: "No service entries configured for cluster - cluster1", - statusCode: 200, + expectedErr: "No service entries configured for cluster - cluster1", + statusCode: 200, }, { - name: "success with service entry for cluster", - clusterName: "cluster1", - remoteControllers: map[string]*clusters.RemoteController{ + name: "success with service entry for cluster", + clusterName: "cluster1", + remoteControllers: map[string]*clusters.RemoteController{ "cluster1": &clusters.RemoteController{ - ServiceEntryController: &istio.ServiceEntryController{ + ServiceEntryController: &istio.ServiceEntryController{ IstioClient: fakeIstioClient, }, }, }, - expectedErr: "", - statusCode: 200, + expectedErr: "", + statusCode: 200, }, } //Run the test for every provided case for _, c := range testCases { t.Run(c.name, func(t *testing.T) { - r:= httptest.NewRequest("GET", url, nil) + r := httptest.NewRequest("GET", url, nil) r = mux.SetURLVars(r, map[string]string{"clustername": c.clusterName}) w := httptest.NewRecorder() opts.RemoteRegistry.RemoteControllers = c.remoteControllers @@ -158,41 +169,41 @@ func TestGetServiceEntriesByCluster (t *testing.T) { } } -func TestGetServiceEntriesByIdentity (t *testing.T) { +func TestGetServiceEntriesByIdentity(t *testing.T) { url := "https://admiral.com/identity/service1/serviceentries" opts := RouteOpts{ RemoteRegistry: &clusters.RemoteRegistry{ AdmiralCache: &clusters.AdmiralCache{ - SeClusterCache: common.NewMapOfMaps() , + SeClusterCache: common.NewMapOfMaps(), }, }, } testCases := []struct { - name string - identity string - host string + name string + identity string + host string expectedErr string - statusCode int + statusCode int }{ { - name: "failure with identity not provided request", - identity: "", - host: "", - expectedErr: "Identity not provided as part of the request\n", - statusCode: 400, + name: "failure with identity not provided request", + identity: "", + host: "", + expectedErr: "Identity not provided as part of the request\n", + statusCode: 400, }, { - name: "success with service entry for service", - identity: "meshhealthcheck", - host: "anil-test-bdds-10-k8s-e2e.intuit.services.mesh.meshhealthcheck.mesh", - expectedErr: "Identity not provided as part of the request\n", - statusCode: 200, + name: "success with service entry for service", + identity: "meshhealthcheck", + host: "anil-test-bdds-10-k8s-e2e.intuit.services.mesh.meshhealthcheck.mesh", + expectedErr: "Identity not provided as part of the request\n", + statusCode: 200, }, } //Run the test for every provided case for _, c := range testCases { t.Run(c.name, func(t *testing.T) { - r:= httptest.NewRequest("GET", url, nil) + r := httptest.NewRequest("GET", url, nil) r = mux.SetURLVars(r, map[string]string{"identity": c.identity}) w := httptest.NewRecorder() if c.host != "" { @@ -210,4 +221,3 @@ func TestGetServiceEntriesByIdentity (t *testing.T) { }) } } - diff --git a/admiral/pkg/apis/admiral/routes/routes.go b/admiral/pkg/apis/admiral/routes/routes.go index 44b564a5..660b6d0f 100644 --- a/admiral/pkg/apis/admiral/routes/routes.go +++ b/admiral/pkg/apis/admiral/routes/routes.go @@ -3,8 +3,11 @@ package routes import ( "github.com/istio-ecosystem/admiral/admiral/pkg/apis/admiral/filters" "github.com/istio-ecosystem/admiral/admiral/pkg/apis/admiral/server" + "github.com/istio-ecosystem/admiral/admiral/pkg/controller/common" + "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/client-go/tools/clientcmd" "log" + "net/http" ) var Filter = server.Filters{ @@ -47,3 +50,29 @@ func NewAdmiralAPIServer(opts *RouteOpts) server.Routes { }, } } + +func NewMetricsServer() server.Routes { + + if common.GetMetricsEnabled() { + return server.Routes{ + server.Route{ + Name: "Get metrics in prometheus format", + Method: "GET", + Pattern: "/metrics", + HandlerFunc: promhttp.Handler().ServeHTTP, + }, + } + } + return server.Routes{ + server.Route{ + Name: "Noop metrics", + Method: "GET", + Pattern: "/metrics", + HandlerFunc: Noop, + }, + } +} + +func Noop(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} diff --git a/admiral/pkg/apis/admiral/server/server.go b/admiral/pkg/apis/admiral/server/server.go index 03619012..02a07115 100644 --- a/admiral/pkg/apis/admiral/server/server.go +++ b/admiral/pkg/apis/admiral/server/server.go @@ -50,7 +50,7 @@ func (s *Service) Start(ctx context.Context, port int, routes Routes, filter []F s.server = http.Server{Addr: ":" + strconv.Itoa(port), Handler: router} - log.Printf("Starting admiral api server on port=%d", port) + log.Printf("Starting server on port=%d", port) log.Fatalln(s.server.ListenAndServe()) } diff --git a/admiral/pkg/clusters/registry_test.go b/admiral/pkg/clusters/registry_test.go index 56810b52..cac474b4 100644 --- a/admiral/pkg/clusters/registry_test.go +++ b/admiral/pkg/clusters/registry_test.go @@ -449,24 +449,24 @@ func TestUpdateCacheController(t *testing.T) { //Struct of test case info. Name is required. testCases := []struct { - name string - oldConfig *rest.Config - newConfig *rest.Config - clusterId string + name string + oldConfig *rest.Config + newConfig *rest.Config + clusterId string shouldRefresh bool }{ { - name: "Should update controller when kubeconfig changes", - oldConfig: originalConfig, - newConfig: changedConfig, - clusterId: "test.cluster", + name: "Should update controller when kubeconfig changes", + oldConfig: originalConfig, + newConfig: changedConfig, + clusterId: "test.cluster", shouldRefresh: true, }, { - name: "Should not update controller when kubeconfig doesn't change", - oldConfig: originalConfig, - newConfig: originalConfig, - clusterId: "test.cluster", + name: "Should not update controller when kubeconfig doesn't change", + oldConfig: originalConfig, + newConfig: originalConfig, + clusterId: "test.cluster", shouldRefresh: false, }, } @@ -476,7 +476,7 @@ func TestUpdateCacheController(t *testing.T) { t.Run(c.name, func(t *testing.T) { hook := logTest.NewGlobal() rr.RemoteControllers[c.clusterId].ApiServer = c.oldConfig.Host - d, err := admiral.NewDeploymentController(make(chan struct{}), &test.MockDeploymentHandler{}, c.oldConfig, time.Second*time.Duration(300)) + d, err := admiral.NewDeploymentController(make(chan struct{}), &test.MockDeploymentHandler{}, c.oldConfig, time.Second*time.Duration(300)) if err != nil { t.Fatalf("Unexpected error creating controller %v", err) } diff --git a/admiral/pkg/clusters/types.go b/admiral/pkg/clusters/types.go index 6ccc75c4..41fef53f 100644 --- a/admiral/pkg/clusters/types.go +++ b/admiral/pkg/clusters/types.go @@ -21,7 +21,7 @@ import ( type RemoteController struct { ClusterID string ApiServer string - StartTime time.Time + StartTime time.Time GlobalTraffic *admiral.GlobalTrafficController DeploymentController *admiral.DeploymentController ServiceController *admiral.ServiceController @@ -432,7 +432,7 @@ func (rh *RolloutHandler) Deleted(obj *argo.Rollout) { // helper function to handle add and delete for RolloutHandler func HandleEventForRollout(event admiral.EventType, obj *argo.Rollout, remoteRegistry *RemoteRegistry, clusterName string) { - log.Infof(LogFormat, event, "rollout", obj.Name, clusterName, "Received") + log.Infof(LogFormat, event, "rollout", obj.Name, clusterName, "Received") globalIdentifier := common.GetRolloutGlobalIdentifier(obj) if len(globalIdentifier) == 0 { diff --git a/admiral/pkg/controller/common/common_test.go b/admiral/pkg/controller/common/common_test.go index 299196a1..dd0f94e9 100644 --- a/admiral/pkg/controller/common/common_test.go +++ b/admiral/pkg/controller/common/common_test.go @@ -29,12 +29,12 @@ func init() { SecretResolver: "", WorkloadSidecarName: "default", WorkloadSidecarUpdate: "disabled", + MetricsEnabled: true, } p.LabelSet.WorkloadIdentityKey = "identity" p.LabelSet.GlobalTrafficDeploymentLabel = "identity" p.LabelSet.EnvKey = "admiral.io/env" - InitializeConfig(p) } diff --git a/admiral/pkg/controller/common/config.go b/admiral/pkg/controller/common/config.go index 8c7f0189..94d22459 100644 --- a/admiral/pkg/controller/common/config.go +++ b/admiral/pkg/controller/common/config.go @@ -91,8 +91,17 @@ func GetEnvKey() string { return admiralParams.LabelSet.EnvKey } +func GetMetricsEnabled() bool { + return admiralParams.MetricsEnabled +} + ///Setters - be careful func SetKubeconfigPath(path string) { admiralParams.KubeconfigPath = path } + +// for unit test only +func SetEnablePrometheus(value bool) { + admiralParams.MetricsEnabled = value +} diff --git a/admiral/pkg/controller/common/config_test.go b/admiral/pkg/controller/common/config_test.go index 0543105a..23a11be2 100644 --- a/admiral/pkg/controller/common/config_test.go +++ b/admiral/pkg/controller/common/config_test.go @@ -87,4 +87,8 @@ func TestConfigManagement(t *testing.T) { t.Fatalf("Workload Sidecar Name mismatch. Expected /mypath/custom/kubeconfig, got %v", GetKubeconfigPath()) } + if GetMetricsEnabled() != true { + t.Errorf("Enable Prometheus mismatch, expected false, got %v", GetMetricsEnabled()) + } + } diff --git a/admiral/pkg/controller/common/metrics.go b/admiral/pkg/controller/common/metrics.go new file mode 100644 index 00000000..213ad99c --- /dev/null +++ b/admiral/pkg/controller/common/metrics.go @@ -0,0 +1,31 @@ +package common + +import "github.com/prometheus/client_golang/prometheus" + +const ClustersMonitoredMetricName = "clusters_monitored" + +type Gauge interface { + Set(value float64) +} + +func NewGaugeFrom(name string, help string) Gauge { + if !GetMetricsEnabled() { + return &Noop{} + } + opts := prometheus.GaugeOpts{Name: name, Help: help} + g := prometheus.NewGauge(opts) + prometheus.MustRegister(g) + return &PromGauge{g} +} + +type Noop struct{} + +type PromGauge struct { + g prometheus.Gauge +} + +func (g *PromGauge) Set(value float64) { + g.g.Set(value) +} + +func (g *Noop) Set(value float64) {} diff --git a/admiral/pkg/controller/common/metrics_test.go b/admiral/pkg/controller/common/metrics_test.go new file mode 100644 index 00000000..081adc42 --- /dev/null +++ b/admiral/pkg/controller/common/metrics_test.go @@ -0,0 +1,39 @@ +package common + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewGaugeFrom(t *testing.T) { + type args struct { + prom bool + Name string + Help string + } + tc := []struct { + name string + args args + want Gauge + }{ + { + "Should return a Prometheus gauge", + args{true, "gauge", ""}, + &PromGauge{prometheus.NewGauge(prometheus.GaugeOpts{Name: "gauge", Help: ""})}, + }, + { + "Should return a Noop gauge", + args{false, "gauge", ""}, + &Noop{}, + }, + } + + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + SetEnablePrometheus(tt.args.prom) + actual := NewGaugeFrom(tt.args.Name, tt.args.Help) + assert.Equal(t, tt.want, actual, "want: %#v, got: %#v", tt.want, actual) + }) + } +} diff --git a/admiral/pkg/controller/common/types.go b/admiral/pkg/controller/common/types.go index df3960b8..7a51f7ab 100644 --- a/admiral/pkg/controller/common/types.go +++ b/admiral/pkg/controller/common/types.go @@ -42,6 +42,7 @@ type AdmiralParams struct { LogLevel int HostnameSuffix string PreviewHostnamePrefix string + MetricsEnabled bool WorkloadSidecarUpdate string WorkloadSidecarName string } diff --git a/admiral/pkg/controller/secret/secretcontroller.go b/admiral/pkg/controller/secret/secretcontroller.go index 27662cdb..a07a5f17 100644 --- a/admiral/pkg/controller/secret/secretcontroller.go +++ b/admiral/pkg/controller/secret/secretcontroller.go @@ -45,6 +45,8 @@ const ( // DO NOT USE - TEST ONLY. var LoadKubeConfig = clientcmd.Load +var remoteClustersMetric common.Gauge + // addSecretCallback prototype for the add secret callback function. type addSecretCallback func(config *rest.Config, dataKey string, resyncPeriod time.Duration) error @@ -162,6 +164,7 @@ func NewController( }, }) + remoteClustersMetric = common.NewGaugeFrom(common.ClustersMonitoredMetricName, "Gauge for the clusters monitored by Admiral") return controller } @@ -334,6 +337,7 @@ func (c *Controller) addMemberCluster(secretName string, s *corev1.Secret) { } } + remoteClustersMetric.Set(float64(len(c.Cs.RemoteClusters))) log.Infof("Number of remote clusters: %d", len(c.Cs.RemoteClusters)) } @@ -348,5 +352,6 @@ func (c *Controller) deleteMemberCluster(secretName string) { delete(c.Cs.RemoteClusters, clusterID) } } + remoteClustersMetric.Set(float64(len(c.Cs.RemoteClusters))) log.Infof("Number of remote clusters: %d", len(c.Cs.RemoteClusters)) } diff --git a/admiral/pkg/controller/secret/secretcontroller_test.go b/admiral/pkg/controller/secret/secretcontroller_test.go index c6ae4030..577fdfd1 100644 --- a/admiral/pkg/controller/secret/secretcontroller_test.go +++ b/admiral/pkg/controller/secret/secretcontroller_test.go @@ -17,6 +17,9 @@ package secret import ( "context" "fmt" + "github.com/istio-ecosystem/admiral/admiral/pkg/controller/common" + "github.com/prometheus/client_golang/prometheus" + io_prometheus_client "github.com/prometheus/client_model/go" "k8s.io/client-go/rest" "sync" "testing" @@ -203,6 +206,9 @@ func Test_SecretController(t *testing.T) { secret1 = makeSecret("s1", "c1", []byte("kubeconfig1-0")) ) + p := common.AdmiralParams{MetricsEnabled: true} + common.InitializeConfig(p) + steps := []struct { // only set one of these per step. The others should be nil. add *v1.Secret @@ -213,16 +219,20 @@ func Test_SecretController(t *testing.T) { wantAdded string wantUpdated string wantDeleted string + + // clusters-monitored metric + clustersMonitored float64 }{ - {add: secret0, wantAdded: "c0"}, - {update: secret0UpdateKubeconfigChanged, wantUpdated: "c0"}, - {add: secret1, wantAdded: "c1"}, - {delete: secret0, wantDeleted: "c0"}, - {delete: secret1, wantDeleted: "c1"}, + {add: secret0, wantAdded: "c0", clustersMonitored: 1}, + {update: secret0UpdateKubeconfigChanged, wantUpdated: "c0", clustersMonitored: 1}, + {add: secret1, wantAdded: "c1", clustersMonitored: 2}, + {delete: secret0, wantDeleted: "c0", clustersMonitored: 1}, + {delete: secret1, wantDeleted: "c1", clustersMonitored: 0}, } // Start the secret controller and sleep to allow secret process to start. // The assertion ShouldNot(BeNil()) make sure that start secret controller return a not nil controller and nil error + registry := prometheus.DefaultGatherer g.Expect( StartSecretController(clientset, addCallback, updateCallback, deleteCallback, secretNameSpace, context.TODO(), "")). ShouldNot(BeNil()) @@ -271,6 +281,17 @@ func Test_SecretController(t *testing.T) { return added == "" && updated == "" && deleted == "" }).Should(Equal(true)) } + + g.Eventually(func() float64 { + mf, _ := registry.Gather() + var clustersMonitored *io_prometheus_client.MetricFamily + for _, m := range mf { + if *m.Name == "clusters_monitored" { + clustersMonitored = m + } + } + return *clustersMonitored.Metric[0].Gauge.Value + }).Should(Equal(step.clustersMonitored)) }) } } diff --git a/go.mod b/go.mod index ae100d95..4b05581d 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,8 @@ require ( github.com/mailru/easyjson v0.7.1 // indirect github.com/onsi/ginkgo v1.10.2 // indirect github.com/onsi/gomega v1.7.0 + github.com/prometheus/client_golang v1.5.0 + github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.9.1 github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.5 diff --git a/go.sum b/go.sum index b6c73113..fe461bac 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,7 @@ github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89/go.mod h1:5J github.com/bazelbuild/rules_go v0.0.0-20190719190356-6dae44dc5cab/go.mod h1:MC23Dc/wkXEyk3Wpq6lCqz0ZAYOZDw2DR5y3N1q2i7M= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU= @@ -73,6 +74,7 @@ github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEe github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/checkpoint-restore/go-criu v0.0.0-20190109184317-bdb7599cd87b/go.mod h1:TrMrLQfeENAPYPRsJuq3jsqdlRh3lvi6trTZJG8+tho= @@ -392,6 +394,7 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.5/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mesos/mesos-go v0.0.9/go.mod h1:kPYCMQ9gsOXVAle1OsoY4I1+9kPu8GHkf88aV59fDr4= github.com/mholt/certmagic v0.6.2-0.20190624175158-6a42ef9fe8c2/go.mod h1:g4cOPxcjV0oFq3qwpjSA30LReKD8AoIfwAY9VvG35NY= @@ -457,16 +460,19 @@ github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prY github.com/pquerna/ffjson v0.0.0-20180717144149-af8b230fcd20/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/quobyte/api v0.1.2/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=