Fix metrics appending in prometheus collector (elastic#11398)

Update method of common.MapStr doesn't update nested maps
so we were overwriting existing metrics.

Fix fields definition to add wildcard so fields match.

Add testdata.

metricbeat/docs/fields.asciidoc

@@ -20707,7 +20707,7 @@ Stats scraped from a Prometheus endpoint.
 
 
 
-*`prometheus.labels`*::
+*`prometheus.labels.*`*::
 +
 --
 type: object
@@ -20717,7 +20717,7 @@ Prometheus metric labels
 
 --
 
-*`prometheus.metrics`*::
+*`prometheus.metrics.*`*::
 +
 --
 type: object

metricbeat/module/prometheus/_meta/fields.yml

@@ -7,14 +7,14 @@
   settings: ["ssl", "http"]
   fields:
     # Order is important here, labels will match first, the rest are double
-    - name: prometheus.labels
+    - name: prometheus.labels.*
       type: object
       object_type: keyword
       description: >
         Prometheus metric labels
-    - name: prometheus.metrics
+    - name: prometheus.metrics.*
       type: object
       object_type: double
       object_type_mapping_type: "*"
       description: >
-        Prometheus metric
+        Prometheus metric

metricbeat/module/prometheus/collector/_meta/data.json

@@ -1,9 +1,5 @@
 {
-    "@timestamp": "2017-10-12T08:05:34.853Z",
-    "agent": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
-    },
+    "@timestamp": "2019-03-01T08:05:34.853Z",
     "event": {
         "dataset": "prometheus.collector",
         "duration": 115000,
@@ -14,15 +10,15 @@
     },
     "prometheus": {
         "labels": {
-            "interval": "15s"
+            "listener_name": "http"
         },
         "metrics": {
-            "prometheus_target_interval_length_seconds_count": 1,
-            "prometheus_target_interval_length_seconds_sum": 15.000226176
+            "net_conntrack_listener_conn_accepted_total": 3,
+            "net_conntrack_listener_conn_closed_total": 0
         }
     },
     "service": {
-        "address": "172.29.0.2:9090",
+        "address": "127.0.0.1:55555",
         "type": "prometheus"
     }
 }

metricbeat/module/prometheus/collector/_meta/testdata/config.yml

@@ -0,0 +1,3 @@
+type: http
+url: "/metrics"
+suffix: plain

metricbeat/module/prometheus/collector/_meta/testdata/docs.plain

@@ -0,0 +1,6 @@
+# HELP net_conntrack_listener_conn_accepted_total Total number of connections opened to the listener of a given name.
+# TYPE net_conntrack_listener_conn_accepted_total counter
+net_conntrack_listener_conn_accepted_total{listener_name="http"} 3
+# HELP net_conntrack_listener_conn_closed_total Total number of connections closed that were made to the listener of a given name.
+# TYPE net_conntrack_listener_conn_closed_total counter
+net_conntrack_listener_conn_closed_total{listener_name="http"} 0

metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json

@@ -0,0 +1,25 @@
+[
+    {
+        "event": {
+            "dataset": "prometheus.collector",
+            "duration": 115000,
+            "module": "prometheus"
+        },
+        "metricset": {
+            "name": "collector"
+        },
+        "prometheus": {
+            "labels": {
+                "listener_name": "http"
+            },
+            "metrics": {
+                "net_conntrack_listener_conn_accepted_total": 3,
+                "net_conntrack_listener_conn_closed_total": 0
+            }
+        },
+        "service": {
+            "address": "127.0.0.1:55555",
+            "type": "prometheus"
+        }
+    }
+]

metricbeat/module/prometheus/collector/_meta/testdata/etcd-3.3.10-partial.plain

@@ -0,0 +1,214 @@
+# HELP etcd_debugging_mvcc_db_compaction_keys_total Total number of db keys compacted.
+# TYPE etcd_debugging_mvcc_db_compaction_keys_total counter
+etcd_debugging_mvcc_db_compaction_keys_total 0
+# HELP etcd_debugging_mvcc_db_total_size_in_bytes Total size of the underlying database physically allocated in bytes. Use etcd_mvcc_db_total_size_in_bytes
+# TYPE etcd_debugging_mvcc_db_total_size_in_bytes gauge
+etcd_debugging_mvcc_db_total_size_in_bytes 20480
+# HELP etcd_debugging_mvcc_delete_total Total number of deletes seen by this member.
+# TYPE etcd_debugging_mvcc_delete_total counter
+etcd_debugging_mvcc_delete_total 0
+# HELP etcd_debugging_mvcc_events_total Total number of events sent by this member.
+# TYPE etcd_debugging_mvcc_events_total counter
+etcd_debugging_mvcc_events_total 0
+# HELP etcd_debugging_mvcc_keys_total Total number of keys.
+# TYPE etcd_debugging_mvcc_keys_total gauge
+etcd_debugging_mvcc_keys_total 0
+# HELP etcd_debugging_mvcc_pending_events_total Total number of pending events to be sent.
+# TYPE etcd_debugging_mvcc_pending_events_total gauge
+etcd_debugging_mvcc_pending_events_total 0
+# HELP etcd_debugging_mvcc_put_total Total number of puts seen by this member.
+# TYPE etcd_debugging_mvcc_put_total counter
+etcd_debugging_mvcc_put_total 0
+# HELP etcd_debugging_mvcc_range_total Total number of ranges seen by this member.
+# TYPE etcd_debugging_mvcc_range_total counter
+etcd_debugging_mvcc_range_total 0
+# HELP etcd_debugging_mvcc_slow_watcher_total Total number of unsynced slow watchers.
+# TYPE etcd_debugging_mvcc_slow_watcher_total gauge
+etcd_debugging_mvcc_slow_watcher_total 0
+# HELP etcd_debugging_mvcc_txn_total Total number of txns seen by this member.
+# TYPE etcd_debugging_mvcc_txn_total counter
+etcd_debugging_mvcc_txn_total 0
+# HELP etcd_debugging_mvcc_watch_stream_total Total number of watch streams.
+# TYPE etcd_debugging_mvcc_watch_stream_total gauge
+etcd_debugging_mvcc_watch_stream_total 0
+# HELP etcd_debugging_mvcc_watcher_total Total number of watchers.
+# TYPE etcd_debugging_mvcc_watcher_total gauge
+etcd_debugging_mvcc_watcher_total 0
+# HELP etcd_debugging_server_lease_expired_total The total number of expired leases.
+# TYPE etcd_debugging_server_lease_expired_total counter
+etcd_debugging_server_lease_expired_total 0
+# HELP etcd_debugging_store_expires_total Total number of expired keys.
+# TYPE etcd_debugging_store_expires_total counter
+etcd_debugging_store_expires_total 0
+# HELP etcd_debugging_store_reads_total Total number of reads action by (get/getRecursive), local to this member.
+# TYPE etcd_debugging_store_reads_total counter
+etcd_debugging_store_reads_total{action="getRecursive"} 1
+# HELP etcd_debugging_store_watch_requests_total Total number of incoming watch requests (new or reestablished).
+# TYPE etcd_debugging_store_watch_requests_total counter
+etcd_debugging_store_watch_requests_total 0
+# HELP etcd_debugging_store_watchers Count of currently active watchers.
+# TYPE etcd_debugging_store_watchers gauge
+etcd_debugging_store_watchers 0
+# HELP etcd_debugging_store_writes_total Total number of writes (e.g. set/compareAndDelete) seen by this member.
+# TYPE etcd_debugging_store_writes_total counter
+etcd_debugging_store_writes_total{action="create"} 1
+etcd_debugging_store_writes_total{action="set"} 2
+# HELP etcd_grpc_proxy_cache_hits_total Total number of cache hits
+# TYPE etcd_grpc_proxy_cache_hits_total gauge
+etcd_grpc_proxy_cache_hits_total 0
+# HELP etcd_grpc_proxy_cache_keys_total Total number of keys/ranges cached
+# TYPE etcd_grpc_proxy_cache_keys_total gauge
+etcd_grpc_proxy_cache_keys_total 0
+# HELP etcd_grpc_proxy_cache_misses_total Total number of cache misses
+# TYPE etcd_grpc_proxy_cache_misses_total gauge
+etcd_grpc_proxy_cache_misses_total 0
+# HELP etcd_grpc_proxy_events_coalescing_total Total number of events coalescing
+# TYPE etcd_grpc_proxy_events_coalescing_total counter
+etcd_grpc_proxy_events_coalescing_total 0
+# HELP etcd_grpc_proxy_watchers_coalescing_total Total number of current watchers coalescing
+# TYPE etcd_grpc_proxy_watchers_coalescing_total gauge
+etcd_grpc_proxy_watchers_coalescing_total 0
+# HELP etcd_mvcc_db_total_size_in_bytes Total size of the underlying database physically allocated in bytes.
+# TYPE etcd_mvcc_db_total_size_in_bytes gauge
+etcd_mvcc_db_total_size_in_bytes 20480
+# HELP etcd_mvcc_db_total_size_in_use_in_bytes Total size of the underlying database logically in use in bytes.
+# TYPE etcd_mvcc_db_total_size_in_use_in_bytes gauge
+etcd_mvcc_db_total_size_in_use_in_bytes 16384
+# HELP etcd_network_client_grpc_received_bytes_total The total number of bytes received from grpc clients.
+# TYPE etcd_network_client_grpc_received_bytes_total counter
+etcd_network_client_grpc_received_bytes_total 0
+# HELP etcd_network_client_grpc_sent_bytes_total The total number of bytes sent to grpc clients.
+# TYPE etcd_network_client_grpc_sent_bytes_total counter
+etcd_network_client_grpc_sent_bytes_total 0
+# HELP etcd_server_go_version Which Go version server is running with. 1 for 'server_go_version' label with current version.
+# TYPE etcd_server_go_version gauge
+etcd_server_go_version{server_go_version="go1.10.4"} 1
+# HELP etcd_server_has_leader Whether or not a leader exists. 1 is existence, 0 is not.
+# TYPE etcd_server_has_leader gauge
+etcd_server_has_leader 1
+# HELP etcd_server_health_failures The total number of failed health checks
+# TYPE etcd_server_health_failures counter
+etcd_server_health_failures 0
+# HELP etcd_server_health_success The total number of successful health checks
+# TYPE etcd_server_health_success counter
+etcd_server_health_success 0
+# HELP etcd_server_heartbeat_send_failures_total The total number of leader heartbeat send failures (likely overloaded from slow disk).
+# TYPE etcd_server_heartbeat_send_failures_total counter
+etcd_server_heartbeat_send_failures_total 0
+# HELP etcd_server_id Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID.
+# TYPE etcd_server_id gauge
+etcd_server_id{server_id="8e9e05c52164694d"} 1
+# HELP etcd_server_is_leader Whether or not this member is a leader. 1 if is, 0 otherwise.
+# TYPE etcd_server_is_leader gauge
+etcd_server_is_leader 1
+# HELP etcd_server_leader_changes_seen_total The number of leader changes seen.
+# TYPE etcd_server_leader_changes_seen_total counter
+etcd_server_leader_changes_seen_total 1
+# HELP etcd_server_proposals_applied_total The total number of consensus proposals applied.
+# TYPE etcd_server_proposals_applied_total gauge
+etcd_server_proposals_applied_total 4
+# HELP etcd_server_proposals_committed_total The total number of consensus proposals committed.
+# TYPE etcd_server_proposals_committed_total gauge
+etcd_server_proposals_committed_total 4
+# HELP etcd_server_proposals_failed_total The total number of failed proposals seen.
+# TYPE etcd_server_proposals_failed_total counter
+etcd_server_proposals_failed_total 0
+# HELP etcd_server_proposals_pending The current number of pending proposals to commit.
+# TYPE etcd_server_proposals_pending gauge
+etcd_server_proposals_pending 0
+# HELP etcd_server_quota_backend_bytes Current backend storage quota size in bytes.
+# TYPE etcd_server_quota_backend_bytes gauge
+etcd_server_quota_backend_bytes 2.147483648e+09
+# HELP etcd_server_read_indexes_failed_total The total number of failed read indexes seen.
+# TYPE etcd_server_read_indexes_failed_total counter
+etcd_server_read_indexes_failed_total 0
+# HELP etcd_server_slow_apply_total The total number of slow apply requests (likely overloaded from slow disk).
+# TYPE etcd_server_slow_apply_total counter
+etcd_server_slow_apply_total 0
+# HELP etcd_server_slow_read_indexes_total The total number of pending read indexes not in sync with leader's or timed out read index requests.
+# TYPE etcd_server_slow_read_indexes_total counter
+etcd_server_slow_read_indexes_total 0
+# HELP etcd_server_version Which version is running. 1 for 'server_version' label with current version.
+# TYPE etcd_server_version gauge
+etcd_server_version{server_version="3.3.10"} 1
+# HELP go_goroutines Number of goroutines that currently exist.
+# TYPE go_goroutines gauge
+go_goroutines 76
+# HELP go_info Information about the Go environment.
+# TYPE go_info gauge
+go_info{version="go1.10.4"} 1
+# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
+# TYPE go_memstats_alloc_bytes gauge
+go_memstats_alloc_bytes 5.187216e+06
+# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
+# TYPE go_memstats_alloc_bytes_total counter
+go_memstats_alloc_bytes_total 5.818832e+06
+# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
+# TYPE go_memstats_buck_hash_sys_bytes gauge
+go_memstats_buck_hash_sys_bytes 1.444537e+06
+# HELP go_memstats_frees_total Total number of frees.
+# TYPE go_memstats_frees_total counter
+go_memstats_frees_total 5341
+# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started.
+# TYPE go_memstats_gc_cpu_fraction gauge
+go_memstats_gc_cpu_fraction 0.06982921260067133
+# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
+# TYPE go_memstats_gc_sys_bytes gauge
+go_memstats_gc_sys_bytes 413696
+# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
+# TYPE go_memstats_heap_alloc_bytes gauge
+go_memstats_heap_alloc_bytes 5.187216e+06
+# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
+# TYPE go_memstats_heap_idle_bytes gauge
+go_memstats_heap_idle_bytes 811008
+# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
+# TYPE go_memstats_heap_inuse_bytes gauge
+go_memstats_heap_inuse_bytes 6.660096e+06
+# HELP go_memstats_heap_objects Number of allocated objects.
+# TYPE go_memstats_heap_objects gauge
+go_memstats_heap_objects 27762
+# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS.
+# TYPE go_memstats_heap_released_bytes gauge
+go_memstats_heap_released_bytes 0
+# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
+# TYPE go_memstats_heap_sys_bytes gauge
+go_memstats_heap_sys_bytes 7.471104e+06
+# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
+# TYPE go_memstats_last_gc_time_seconds gauge
+go_memstats_last_gc_time_seconds 1.5534302814931297e+09
+# HELP go_memstats_lookups_total Total number of pointer lookups.
+# TYPE go_memstats_lookups_total counter
+go_memstats_lookups_total 55
+# HELP go_memstats_mallocs_total Total number of mallocs.
+# TYPE go_memstats_mallocs_total counter
+go_memstats_mallocs_total 33103
+# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
+# TYPE go_memstats_mcache_inuse_bytes gauge
+go_memstats_mcache_inuse_bytes 6944
+# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
+# TYPE go_memstats_mcache_sys_bytes gauge
+go_memstats_mcache_sys_bytes 16384
+# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
+# TYPE go_memstats_mspan_inuse_bytes gauge
+go_memstats_mspan_inuse_bytes 58520
+# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
+# TYPE go_memstats_mspan_sys_bytes gauge
+go_memstats_mspan_sys_bytes 65536
+# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
+# TYPE go_memstats_next_gc_bytes gauge
+go_memstats_next_gc_bytes 8.4788e+06
+# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
+# TYPE go_memstats_other_sys_bytes gauge
+go_memstats_other_sys_bytes 1.289791e+06
+# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
+# TYPE go_memstats_stack_inuse_bytes gauge
+go_memstats_stack_inuse_bytes 1.114112e+06
+# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
+# TYPE go_memstats_stack_sys_bytes gauge
+go_memstats_stack_sys_bytes 1.114112e+06
+# HELP go_memstats_sys_bytes Number of bytes obtained from system.
+# TYPE go_memstats_sys_bytes gauge
+go_memstats_sys_bytes 1.181516e+07
+# HELP go_threads Number of OS threads created.
+# TYPE go_threads gauge
+go_threads 13