Read metrics from file

changelog/unreleased/read-metrics-from-file.md

@@ -0,0 +1,5 @@
+Enhancement: Metrics module can be configured to retrieve metrics data from file.
+
+- Export site metrics in Prometheus #698
+
+https://github.com/cs3org/reva/pull/973

examples/metrics/metrics.toml

@@ -2,6 +2,9 @@
 jwt_secret = "Pive-Fumkiu4"
 
 [http.services.prometheus]
+# metrics_data_driver_type, one of: dummy, json
+metrics_data_driver_type = "dummy"
+metrics_data_location = ""
 
 [http]
 address = "0.0.0.0:5550"

internal/http/services/prometheus/prometheus.go

@@ -21,15 +21,16 @@ package prometheus
 import (
 	"net/http"
 
+	"github.com/cs3org/reva/pkg/appctx"
+	"github.com/cs3org/reva/pkg/metrics"
+	"github.com/cs3org/reva/pkg/metrics/config"
+
 	"contrib.go.opencensus.io/exporter/prometheus"
 	"github.com/cs3org/reva/pkg/rhttp/global"
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"go.opencensus.io/stats/view"
-
-	// Initializes goroutines which periodically update stats
-	_ "github.com/cs3org/reva/pkg/metrics/reader/dummy"
 )
 
 func init() {
@@ -38,32 +39,57 @@ func init() {
 
 // New returns a new prometheus service
 func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) {
-	conf := &config{}
+	conf := &config.Config{}
 	if err := mapstructure.Decode(m, conf); err != nil {
 		return nil, err
 	}
 
-	conf.init()
+	conf.Init()
+
+	metrics, err := metrics.New(conf)
+	if err != nil {
+		return nil, errors.Wrap(err, "prometheus: error creating metrics")
+	}
 
+	// prometheus handler
 	pe, err := prometheus.NewExporter(prometheus.Options{
 		Namespace: "revad",
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "prometheus: error creating exporter")
 	}
+	// metricsHandler wraps the prometheus handler
+	metricsHandler := &MetricsHandler{
+		pe:      pe,
+		metrics: metrics,
+	}
+	view.RegisterExporter(metricsHandler)
 
-	view.RegisterExporter(pe)
-	return &svc{prefix: conf.Prefix, h: pe}, nil
+	return &svc{prefix: conf.Prefix, h: metricsHandler}, nil
 }
 
-type config struct {
-	Prefix string `mapstructure:"prefix"`
+// MetricsHandler struct and methods (ServeHTTP, ExportView) is a wrapper for prometheus Exporter
+// so we can override (execute our own logic) before forwarding to the prometheus Exporter: see overriding method MetricsHandler.ServeHTTP()
+type MetricsHandler struct {
+	pe      *prometheus.Exporter
+	metrics *metrics.Metrics
 }
 
-func (c *config) init() {
-	if c.Prefix == "" {
-		c.Prefix = "metrics"
+// ServeHTTP override and forward to prometheus.Exporter ServeHTTP()
+func (h *MetricsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	log := appctx.GetLogger(r.Context())
+	// make sure the latest metrics data are recorded
+	if err := h.metrics.RecordMetrics(); err != nil {
+		log.Err(err).Msg("Unable to record metrics")
 	}
+	// proceed with regular flow
+	h.pe.ServeHTTP(w, r)
+}
+
+// ExportView must only be implemented to adhere to prometheus.Exporter signature; we simply forward to prometheus ExportView
+func (h *MetricsHandler) ExportView(vd *view.Data) {
+	// just proceed with regular flow
+	h.pe.ExportView(vd)
 }
 
 type svc struct {

pkg/metrics/config/config.go

@@ -0,0 +1,39 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package config
+
+// Config holds the config options that need to be passed down to the metrics reader
+type Config struct {
+	Prefix                string `mapstructure:"prefix"`
+	MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"`
+	MetricsDataLocation   string `mapstructure:"metrics_data_location"`
+}
+
+// Init sets sane defaults
+func (c *Config) Init() {
+	if c.Prefix == "" {
+		c.Prefix = "metrics"
+	}
+	if c.MetricsDataDriverType == "json" {
+		// default values
+		if c.MetricsDataLocation == "" {
+			c.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json"
+		}
+	}
+}

pkg/metrics/driver/dummy/dummy.go

@@ -0,0 +1,54 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package dummy
+
+import (
+	"math/rand"
+
+	"github.com/cs3org/reva/pkg/metrics/config"
+)
+
+// New returns a new MetricsDummyDriver object.
+func New(config *config.Config) (*MetricsDummyDriver, error) {
+	driver := &MetricsDummyDriver{
+		config: config,
+	}
+
+	return driver, nil
+}
+
+// MetricsDummyDriver the MetricsDummyDriver struct
+type MetricsDummyDriver struct {
+	config *config.Config
+}
+
+// GetNumUsers returns the number of site users, it's a dummy number
+func (d *MetricsDummyDriver) GetNumUsers() int64 {
+	return int64(rand.Intn(30000))
+}
+
+// GetNumGroups returns the number of site groups, it's a dummy number
+func (d *MetricsDummyDriver) GetNumGroups() int64 {
+	return int64(rand.Intn(200))
+}
+
+// GetAmountStorage returns the amount of site storage used, it's a dummy amount
+func (d *MetricsDummyDriver) GetAmountStorage() int64 {
+	return int64(rand.Intn(70000000000))
+}

pkg/metrics/driver/json/json.go

@@ -0,0 +1,92 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package json
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+
+	"github.com/cs3org/reva/pkg/metrics/config"
+)
+
+// New returns a new MetricsJSONDriver object.
+// It reads the data file from the specified config.MetricsDataLocation upon initializing.
+// It does not reload the data file for each metric.
+func New(config *config.Config) (*MetricsJSONDriver, error) {
+	// the json driver reads the data metrics file upon initializing
+	metricsData, err := readJSON(config)
+	if err != nil {
+		return nil, err
+	}
+
+	driver := &MetricsJSONDriver{
+		config: config,
+		data:   metricsData,
+	}
+
+	return driver, nil
+}
+
+func readJSON(config *config.Config) (*data, error) {
+	if config.MetricsDataLocation == "" {
+		err := errors.New("Unable to initialize a metrics data driver, has the data location (metrics_data_location) been configured?")
+		return nil, err
+	}
+
+	file, err := ioutil.ReadFile(config.MetricsDataLocation)
+	if err != nil {
+		return nil, err
+	}
+
+	data := &data{}
+	err = json.Unmarshal(file, data)
+	if err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}
+
+type data struct {
+	NumUsers      int64 `json:"cs3_org_sciencemesh_site_total_num_users"`
+	NumGroups     int64 `json:"cs3_org_sciencemesh_site_total_num_groups"`
+	AmountStorage int64 `json:"cs3_org_sciencemesh_site_total_amount_storage"`
+}
+
+// MetricsJSONDriver the JsonDriver struct that also holds the data
+type MetricsJSONDriver struct {
+	config *config.Config
+	data   *data
+}
+
+// GetNumUsers returns the number of site users
+func (d *MetricsJSONDriver) GetNumUsers() int64 {
+	return d.data.NumUsers
+}
+
+// GetNumGroups returns the number of site groups
+func (d *MetricsJSONDriver) GetNumGroups() int64 {
+	return d.data.NumGroups
+}
+
+// GetAmountStorage returns the amount of site storage used
+func (d *MetricsJSONDriver) GetAmountStorage() int64 {
+	return d.data.AmountStorage
+}