Collect total_data_set_size_in_bytes from elasticsearch module (#37457)

* Collect total_data_set_size_in_bytes from elasticsearch module Some APIs return total_data_set_size_in_bytes, which is usually the same as size_in_bytes, except when using partially mounted indices/searchable snapshots. For those the size_in_bytes returns zero, while total_data_set_size_in_bytes is the total shard size of the backing index on object storage. We want to collect this to see how much storage each index is using on object storage. Marking it as optional because older versions of ES don't return the field.
elastic · Jan 16, 2024 · fb96791 · fb96791
1 parent 1554d7d
commit fb96791
Show file tree

Hide file tree

Showing 12 changed files with 104 additions and 10 deletions.
diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -30458,6 +30458,13 @@ type: long
 
 --
 
+*`elasticsearch.cluster.stats.indices.store.total_data_set_size.bytes`*::
++
+--
+type: long
+
+--
+
 *`elasticsearch.cluster.stats.indices.total`*::
 +
 --
@@ -30790,6 +30797,13 @@ type: long
 
 --
 
+*`elasticsearch.index.primaries.store.total_data_set_size_in_bytes`*::
++
+--
+type: long
+
+--
+
 *`elasticsearch.index.primaries.docs.count`*::
 +
 --
@@ -30952,6 +30966,18 @@ type: long
 Total size of the index in bytes.
 
 
+type: long
+
+format: bytes
+
+--
+
+*`elasticsearch.index.total.store.total_data_set_size_in_bytes`*::
++
+--
+Total size of the index in bytes including backing data for partially mounted indices.
+
+
 type: long
 
 format: bytes
@@ -31482,6 +31508,18 @@ type: long
 Total size of the index in bytes.
 
 
+type: long
+
+format: bytes
+
+--
+
+*`elasticsearch.index.summary.primaries.store.total_data_set_size.bytes`*::
++
+--
+Total size of the index in bytes including backing data for partially mounted indices.
+
+
 type: long
 
 format: bytes
@@ -31605,6 +31643,18 @@ type: long
 Total size of the index in bytes.
 
 
+type: long
+
+format: bytes
+
+--
+
+*`elasticsearch.index.summary.total.store.total_data_set_size.bytes`*::
++
+--
+Total size of the index in bytes including backing data for partially mounted indices.
+
+
 type: long
 
 format: bytes
@@ -32104,7 +32154,17 @@ format: bytes
 *`elasticsearch.node.stats.indices.store.size.bytes`*::
 +
 --
-Total size of the store in bytes.
+Total size of all shards assigned to this node in bytes.
+
+
+type: long
+
+--
+
+*`elasticsearch.node.stats.indices.store.total_data_set_size.bytes`*::
++
+--
+Total size of shards in bytes assigned to this node including backing data for partially mounted indices.
 
 
 type: long

diff --git a/metricbeat/module/elasticsearch/cluster_stats/_meta/data.json b/metricbeat/module/elasticsearch/cluster_stats/_meta/data.json
@@ -143,4 +143,4 @@
     "host": {
       "name": "host.example.com"
     }
-  }
+  }
diff --git a/metricbeat/module/elasticsearch/cluster_stats/_meta/fields.yml b/metricbeat/module/elasticsearch/cluster_stats/_meta/fields.yml
@@ -55,6 +55,8 @@
       fields:
       - name: store.size.bytes
         type: long
+      - name: store.total_data_set_size.bytes
+        type: long
       - name: total
         type: long
         description: >

diff --git a/metricbeat/module/elasticsearch/cluster_stats/data.go b/metricbeat/module/elasticsearch/cluster_stats/data.go
@@ -76,7 +76,8 @@ var (
 				"primaries": c.Int("primaries"),
 			}),
 			"store": c.Dict("store", s.Schema{
-				"size": s.Object{"bytes": c.Int("size_in_bytes")},
+				"size":                s.Object{"bytes": c.Int("size_in_bytes")},
+				"total_data_set_size": s.Object{"bytes": c.Int("total_data_set_size_in_bytes", s.Optional)},
 			}),
 			"fielddata": c.Dict("fielddata", s.Schema{
 				"memory": s.Object{

diff --git a/metricbeat/module/elasticsearch/fields.go b/metricbeat/module/elasticsearch/fields.go
diff --git a/metricbeat/module/elasticsearch/index/_meta/data.json b/metricbeat/module/elasticsearch/index/_meta/data.json
@@ -50,7 +50,8 @@
             "query_time_in_millis": 2456214
           },
           "store": {
-            "size_in_bytes": 17759832
+            "size_in_bytes": 17759832,
+            "total_data_set_size_in_bytes": 17759832
           },
           "query_cache": {
             "memory_size_in_bytes": 21120,
@@ -89,7 +90,8 @@
             "total_size_in_bytes": 2199683211
           },
           "store": {
-            "size_in_bytes": 17759832
+            "size_in_bytes": 17759832,
+            "total_data_set_size_in_bytes": 17759832
           },
           "indexing": {
             "index_time_in_millis": 117187,
@@ -152,4 +154,4 @@
     "host": {
       "name": "host.example.com"
     }
-  }
+  }
diff --git a/metricbeat/module/elasticsearch/index/_meta/fields.yml b/metricbeat/module/elasticsearch/index/_meta/fields.yml
@@ -53,6 +53,8 @@
               type: long
         - name: store.size_in_bytes
           type: long
+        - name: store.total_data_set_size_in_bytes
+          type: long
         - name: docs.count
           type: long
         - name: docs.deleted
@@ -113,6 +115,11 @@
           type: long
           description: >
               Total size of the index in bytes.
+        - name: store.total_data_set_size_in_bytes
+          format: bytes
+          type: long
+          description: >
+            Total size of the index in bytes including backing data for partially mounted indices.
         - name: query_cache
           type: group
           fields:

diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go
@@ -73,7 +73,8 @@ type primaries struct {
 		FixedBitSetMemoryInBytes  int `json:"fixed_bit_set_memory_in_bytes"`
 	} `json:"segments"`
 	Store struct {
-		SizeInBytes int `json:"size_in_bytes"`
+		SizeInBytes             int `json:"size_in_bytes"`
+		TotalDataSetSizeInBytes int `json:"total_data_set_size_in_bytes"`
 	} `json:"store"`
 	Refresh struct {
 		TotalTimeInMillis         int `json:"total_time_in_millis"`
@@ -132,7 +133,8 @@ type total struct {
 		FixedBitSetMemoryInBytes  int `json:"fixed_bit_set_memory_in_bytes"`
 	} `json:"segments"`
 	Store struct {
-		SizeInBytes int `json:"size_in_bytes"`
+		SizeInBytes             int `json:"size_in_bytes"`
+		TotalDataSetSizeInBytes int `json:"total_data_set_size_in_bytes"`
 	} `json:"store"`
 	Refresh struct {
 		TotalTimeInMillis         int `json:"total_time_in_millis"`

diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/fields.yml b/metricbeat/module/elasticsearch/index_summary/_meta/fields.yml
@@ -20,6 +20,11 @@
           format: bytes
           description: >
             Total size of the index in bytes.
+        - name: store.total_data_set_size.bytes
+          type: long
+          format: bytes
+          description: >
+            Total size of the index in bytes including backing data for partially mounted indices.
         - name: segments.count
           type: long
           description: >
@@ -78,6 +83,11 @@
           format: bytes
           description: >
             Total size of the index in bytes.
+        - name: store.total_data_set_size.bytes
+          type: long
+          format: bytes
+          description: >
+            Total size of the index in bytes including backing data for partially mounted indices.
         - name: segments.count
           type: long
           description: >

diff --git a/metricbeat/module/elasticsearch/index_summary/data.go b/metricbeat/module/elasticsearch/index_summary/data.go
@@ -46,6 +46,9 @@ var indexSummaryDict = s.Schema{
 		"size": s.Object{
 			"bytes": c.Int("size_in_bytes"),
 		},
+		"total_data_set_size": s.Object{
+			"bytes": c.Int("total_data_set_size_in_bytes", s.Optional),
+		},
 	}),
 	"segments": c.Dict("segments", s.Schema{
 		"count": c.Int("count"),

diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/fields.yml b/metricbeat/module/elasticsearch/node_stats/_meta/fields.yml
@@ -66,7 +66,11 @@
         - name: store.size.bytes
           type: long
           description: >
-            Total size of the store in bytes.
+            Total size of all shards assigned to this node in bytes.
+        - name: store.total_data_set_size.bytes
+          type: long
+          description: >
+            Total size of shards in bytes assigned to this node including backing data for partially mounted indices.
         - name: fielddata
           type: group
           fields:

diff --git a/metricbeat/module/elasticsearch/node_stats/data.go b/metricbeat/module/elasticsearch/node_stats/data.go
@@ -116,6 +116,9 @@ var (
 				"size": s.Object{
 					"bytes": c.Int("size_in_bytes"),
 				},
+				"total_data_set_size": s.Object{
+					"bytes": c.Int("total_data_set_size_in_bytes", s.Optional),
+				},
 			}),
 			"segments": c.Dict("segments", s.Schema{
 				"count": c.Int("count"),