diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index c57ebeac69f..57ddbf5e0d9 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -119,12 +119,13 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add graphite protocol metricbeat module. {pull}4734[4734] - Add http server metricset to support push metrics via http. {pull}4770[4770] - Make config object public for graphite and http server {pull}4820[4820] -- Add system uptime metricset. {issue}[4848[4848] +- Add system uptime metricset. {issue}4848[4848] - Add experimental `queue` metricset to RabbitMQ module. {pull}4788[4788] - Add additional php-fpm pool status kpis for Metricbeat module {pull}5287[5287] - Add etcd module. {issue}4970[4970] - Add ip address of docker containers to event. {pull}5379[5379] - Add ceph osd tree infomation to metricbeat {pull}5498[5498] +- Add ceph osd_df to metricbeat {pull}5606[5606] - Add basic Logstash module. {pull}5540[5540] - Add dashboard for Windows service metricset. {pull}5603[5603] - Add experimental Docker autodiscover functionality. {pull}5245[5245] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d8ed956b833..4d2cf6a419a 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1191,6 +1191,85 @@ type: long Last updated +[float] +== osd_df fields + +ceph osd disk usage information + + + +[float] +=== `ceph.osd_df.id` + +type: long + +osd node id + + +[float] +=== `ceph.osd_df.name` + +type: text + +osd node name + + +[float] +=== `ceph.osd_df.device_class` + +type: keyword + +osd node type, illegal type include hdd, ssd etc. + + +[float] +=== `ceph.osd_df.total.byte` + +type: long + +format: bytes + +osd disk total volume + + +[float] +=== `ceph.osd_df.used.byte` + +type: long + +format: bytes + +osd disk usage volume + + +[float] +=== `ceph.osd_df.available.bytes` + +type: long + +format: bytes + +osd disk available volume + + +[float] +=== `ceph.osd_df.pg_num` + +type: long + +shows how many pg located on this osd + + +[float] +=== `ceph.osd_df.used.pct` + +type: scaled_float + +format: percent + +osd disk usage percentage + + [float] == osd_tree fields diff --git a/metricbeat/docs/modules/ceph.asciidoc b/metricbeat/docs/modules/ceph.asciidoc index 273c346e50d..962ce16f10a 100644 --- a/metricbeat/docs/modules/ceph.asciidoc +++ b/metricbeat/docs/modules/ceph.asciidoc @@ -39,6 +39,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -51,6 +53,8 @@ include::ceph/cluster_status.asciidoc[] include::ceph/monitor_health.asciidoc[] +include::ceph/osd_df.asciidoc[] + include::ceph/osd_tree.asciidoc[] include::ceph/pool_disk.asciidoc[] diff --git a/metricbeat/docs/modules/ceph/osd_df.asciidoc b/metricbeat/docs/modules/ceph/osd_df.asciidoc new file mode 100644 index 00000000000..882faebf2ef --- /dev/null +++ b/metricbeat/docs/modules/ceph/osd_df.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-ceph-osd_df]] +=== Ceph osd_df metricset + +experimental[] + +include::../../../module/ceph/osd_df/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/osd_df/_meta/data.json[] +---- diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 59bc16ae58a..c6403c5258e 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -17,6 +17,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_health" _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_status" _ "github.com/elastic/beats/metricbeat/module/ceph/monitor_health" + _ "github.com/elastic/beats/metricbeat/module/ceph/osd_df" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_tree" _ "github.com/elastic/beats/metricbeat/module/ceph/pool_disk" _ "github.com/elastic/beats/metricbeat/module/couchbase" diff --git a/metricbeat/module/ceph/_meta/testdata/osd_df_sample_response.json b/metricbeat/module/ceph/_meta/testdata/osd_df_sample_response.json new file mode 100644 index 00000000000..8871bbb58c1 --- /dev/null +++ b/metricbeat/module/ceph/_meta/testdata/osd_df_sample_response.json @@ -0,0 +1,51 @@ +{ + "output": { + "nodes": [ + { + "crush_weight": 0.048691, + "depth": 2, + "device_class": "hdd", + "id": 0, + "kb": 52325356, + "kb_avail": 51245860, + "kb_used": 1079496, + "name": "osd.0", + "pgs": 0, + "pool_weights": {}, + "reweight": 1.0, + "type": "osd", + "type_id": 0, + "utilization": 2.063046, + "var": 1.0 + }, + { + "crush_weight": 0.048691, + "depth": 2, + "device_class": "hdd", + "id": 1, + "kb": 52325356, + "kb_avail": 51245860, + "kb_used": 1079496, + "name": "osd.1", + "pgs": 0, + "pool_weights": {}, + "reweight": 1.0, + "type": "osd", + "type_id": 0, + "utilization": 2.063046, + "var": 1.0 + } + ], + "stray": [], + "summary": { + "average_utilization": 2.063046, + "dev": 0.0, + "max_var": 1.0, + "min_var": 1.0, + "total_kb": 104650712, + "total_kb_avail": 102491720, + "total_kb_used": 2158992 + } + }, + "status": "OK" +} \ No newline at end of file diff --git a/metricbeat/module/ceph/osd_df/_meta/data.json b/metricbeat/module/ceph/osd_df/_meta/data.json new file mode 100644 index 00000000000..2e3bc2921ab --- /dev/null +++ b/metricbeat/module/ceph/osd_df/_meta/data.json @@ -0,0 +1,31 @@ +{ + "@timestamp": "2017-11-16T02:30:32.779Z", + "@metadata": { + "beat": "metricbeat", + "type": "doc", + "version": "7.0.0-alpha1" + }, + "ceph": { + "osd_df": { + "device_class": "hdd", + "name": "osd.1", + "total_byte": 52325356, + "used_byte": 1079496, + "avail_byte": 51245860, + "pg_num": 0, + "used_pct": 0.020630456866839092, + "id": 1 + } + }, + "metricset": { + "module": "ceph", + "name": "osd_df", + "host": "192.168.56.131:5000", + "rtt": 5452 + }, + "beat": { + "name": "centos7gui", + "hostname": "centos7gui", + "version": "7.0.0-alpha1" + } +} \ No newline at end of file diff --git a/metricbeat/module/ceph/osd_df/_meta/docs.asciidoc b/metricbeat/module/ceph/osd_df/_meta/docs.asciidoc new file mode 100644 index 00000000000..7064e3d983e --- /dev/null +++ b/metricbeat/module/ceph/osd_df/_meta/docs.asciidoc @@ -0,0 +1,2 @@ + +This is the `osd_df` metricset of the Ceph module. diff --git a/metricbeat/module/ceph/osd_df/_meta/fields.yml b/metricbeat/module/ceph/osd_df/_meta/fields.yml new file mode 100644 index 00000000000..d544adeb1c0 --- /dev/null +++ b/metricbeat/module/ceph/osd_df/_meta/fields.yml @@ -0,0 +1,42 @@ +- name: osd_df + type: group + release: experimental + description: > + ceph osd disk usage information + fields: + - name: id + type: long + description: > + osd node id + - name: name + type: text + description: > + osd node name + - name: device_class + type: keyword + description: > + osd node type, illegal type include hdd, ssd etc. + - name: total.byte + type: long + format: bytes + description: > + osd disk total volume + - name: used.byte + type: long + format: bytes + description: > + osd disk usage volume + - name: available.bytes + type: long + format: bytes + description: > + osd disk available volume + - name: pg_num + type: long + description: > + shows how many pg located on this osd + - name: used.pct + type: scaled_float + format: percent + description: > + osd disk usage percentage diff --git a/metricbeat/module/ceph/osd_df/data.go b/metricbeat/module/ceph/osd_df/data.go new file mode 100644 index 00000000000..e85fce9842b --- /dev/null +++ b/metricbeat/module/ceph/osd_df/data.go @@ -0,0 +1,62 @@ +package osd_df + +import ( + "encoding/json" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" +) + +type Node struct { + ID int64 `json:"id"` + Name string `json:"name"` + Used int64 `json:"kb_used"` + Available int64 `json:"kb_avail"` + Total int64 `json:"kb"` + PgNum int64 `json:"pgs"` + DeviceClass string `json:"device_class"` +} + +type Output struct { + Nodes []Node `json:"nodes"` +} + +type OsdDfRequest struct { + Status string `json:"status"` + Output Output `json:"output"` +} + +func eventsMapping(content []byte) ([]common.MapStr, error) { + var d OsdDfRequest + err := json.Unmarshal(content, &d) + if err != nil { + logp.Err("Error: ", err) + return nil, err + } + + nodeList := d.Output.Nodes + + //osd node list + events := []common.MapStr{} + for _, node := range nodeList { + nodeInfo := common.MapStr{ + "id": node.ID, + "name": node.Name, + "total.byte": node.Total, + "used.byte": node.Used, + "available.byte": node.Available, + "device_class": node.DeviceClass, + "pg_num": node.PgNum, + } + + if 0 != node.Total { + var usedPct float64 + usedPct = float64(node.Used) / float64(node.Total) + nodeInfo["used.pct"] = usedPct + } + + events = append(events, nodeInfo) + } + + return events, nil +} diff --git a/metricbeat/module/ceph/osd_df/osd_df.go b/metricbeat/module/ceph/osd_df/osd_df.go new file mode 100644 index 00000000000..7f9dbb7f4fb --- /dev/null +++ b/metricbeat/module/ceph/osd_df/osd_df.go @@ -0,0 +1,58 @@ +package osd_df + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "http" + defaultPath = "/api/v0.1/osd/df" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +func init() { + if err := mb.Registry.AddMetricSet("ceph", "osd_df", New, hostParser); err != nil { + panic(err) + } +} + +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The ceph osd_df metricset is experimental") + + http := helper.NewHTTP(base) + http.SetHeader("Accept", "application/json") + + return &MetricSet{ + base, + http, + }, nil +} + +func (m *MetricSet) Fetch() ([]common.MapStr, error) { + content, err := m.HTTP.FetchContent() + if err != nil { + return nil, err + } + + events, err := eventsMapping(content) + if err != nil { + return nil, err + } + + return events, nil +} diff --git a/metricbeat/module/ceph/osd_df/osd_df_integration_test.go b/metricbeat/module/ceph/osd_df/osd_df_integration_test.go new file mode 100644 index 00000000000..a0d7ce7da8e --- /dev/null +++ b/metricbeat/module/ceph/osd_df/osd_df_integration_test.go @@ -0,0 +1,48 @@ +package osd_df + +import ( + "fmt" + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewEventsFetcher(t, getConfig()) + err := mbtest.WriteEvents(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"osd_df"}, + "hosts": getTestCephHost(), + } +} + +const ( + cephDefaultHost = "127.0.0.1" + cephDefaultPort = "5000" +) + +func getTestCephHost() string { + return fmt.Sprintf("%v:%v", + getenv("CEPH_HOST", cephDefaultHost), + getenv("CEPH_PORT", cephDefaultPort), + ) +} + +func getenv(name, defaultValue string) string { + return strDefault(os.Getenv(name), defaultValue) +} + +func strDefault(a, defaults string) string { + if len(a) == 0 { + return defaults + } + return a +} diff --git a/metricbeat/module/ceph/osd_df/osd_df_test.go b/metricbeat/module/ceph/osd_df/osd_df_test.go new file mode 100644 index 00000000000..7476f623fac --- /dev/null +++ b/metricbeat/module/ceph/osd_df/osd_df_test.go @@ -0,0 +1,63 @@ +package osd_df + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestFetchEventContents(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/osd_df_sample_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"osd_df"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewEventsFetcher(t, config) + events, err := f.Fetch() + event := events[0] + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) + + //check osd0 df info + nodeInfo := events[0] + assert.EqualValues(t, 0, nodeInfo["pg_num"]) + assert.EqualValues(t, 52325356, nodeInfo["total.byte"]) + assert.EqualValues(t, 1079496, nodeInfo["used.byte"]) + assert.EqualValues(t, 51245860, nodeInfo["available.byte"]) + assert.EqualValues(t, "hdd", nodeInfo["device_class"]) + assert.EqualValues(t, 0.020630456866839092, nodeInfo["used.pct"]) + assert.EqualValues(t, 0, nodeInfo["id"]) + assert.EqualValues(t, "osd.0", nodeInfo["name"]) + + //check osd1 df info + nodeInfo = events[1] + assert.EqualValues(t, 0, nodeInfo["pg_num"]) + assert.EqualValues(t, 52325356, nodeInfo["total.byte"]) + assert.EqualValues(t, 1079496, nodeInfo["used.byte"]) + assert.EqualValues(t, 51245860, nodeInfo["available.byte"]) + assert.EqualValues(t, "hdd", nodeInfo["device_class"]) + assert.EqualValues(t, 0.020630456866839092, nodeInfo["used.pct"]) + assert.EqualValues(t, 1, nodeInfo["id"]) + assert.EqualValues(t, "osd.1", nodeInfo["name"]) + +}