diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3fa49552947..15653f1e5f6 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -20707,7 +20707,7 @@ Stats scraped from a Prometheus endpoint. -*`prometheus.labels`*:: +*`prometheus.labels.*`*:: + -- type: object @@ -20717,7 +20717,7 @@ Prometheus metric labels -- -*`prometheus.metrics`*:: +*`prometheus.metrics.*`*:: + -- type: object diff --git a/metricbeat/module/prometheus/_meta/fields.yml b/metricbeat/module/prometheus/_meta/fields.yml index d0f8a48c2fa..c6d1f9a1746 100644 --- a/metricbeat/module/prometheus/_meta/fields.yml +++ b/metricbeat/module/prometheus/_meta/fields.yml @@ -7,14 +7,14 @@ settings: ["ssl", "http"] fields: # Order is important here, labels will match first, the rest are double - - name: prometheus.labels + - name: prometheus.labels.* type: object object_type: keyword description: > Prometheus metric labels - - name: prometheus.metrics + - name: prometheus.metrics.* type: object object_type: double object_type_mapping_type: "*" description: > - Prometheus metric \ No newline at end of file + Prometheus metric diff --git a/metricbeat/module/prometheus/collector/_meta/data.json b/metricbeat/module/prometheus/collector/_meta/data.json index 6f63e95343c..75b369afa69 100644 --- a/metricbeat/module/prometheus/collector/_meta/data.json +++ b/metricbeat/module/prometheus/collector/_meta/data.json @@ -1,9 +1,5 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, + "@timestamp": "2019-03-01T08:05:34.853Z", "event": { "dataset": "prometheus.collector", "duration": 115000, @@ -14,15 +10,15 @@ }, "prometheus": { "labels": { - "interval": "15s" + "listener_name": "http" }, "metrics": { - "prometheus_target_interval_length_seconds_count": 1, - "prometheus_target_interval_length_seconds_sum": 15.000226176 + "net_conntrack_listener_conn_accepted_total": 3, + "net_conntrack_listener_conn_closed_total": 0 } }, "service": { - "address": "172.29.0.2:9090", + "address": "127.0.0.1:55555", "type": "prometheus" } } \ No newline at end of file diff --git a/metricbeat/module/prometheus/collector/_meta/testdata/config.yml b/metricbeat/module/prometheus/collector/_meta/testdata/config.yml new file mode 100644 index 00000000000..ab6bf241654 --- /dev/null +++ b/metricbeat/module/prometheus/collector/_meta/testdata/config.yml @@ -0,0 +1,3 @@ +type: http +url: "/metrics" +suffix: plain diff --git a/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain b/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain new file mode 100644 index 00000000000..e1639901d26 --- /dev/null +++ b/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain @@ -0,0 +1,6 @@ +# HELP net_conntrack_listener_conn_accepted_total Total number of connections opened to the listener of a given name. +# TYPE net_conntrack_listener_conn_accepted_total counter +net_conntrack_listener_conn_accepted_total{listener_name="http"} 3 +# HELP net_conntrack_listener_conn_closed_total Total number of connections closed that were made to the listener of a given name. +# TYPE net_conntrack_listener_conn_closed_total counter +net_conntrack_listener_conn_closed_total{listener_name="http"} 0 diff --git a/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json b/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json new file mode 100644 index 00000000000..60ee88f5f0e --- /dev/null +++ b/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json @@ -0,0 +1,25 @@ +[ + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "listener_name": "http" + }, + "metrics": { + "net_conntrack_listener_conn_accepted_total": 3, + "net_conntrack_listener_conn_closed_total": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/prometheus/collector/_meta/testdata/etcd-3.3.10-partial.plain b/metricbeat/module/prometheus/collector/_meta/testdata/etcd-3.3.10-partial.plain new file mode 100644 index 00000000000..eae96e07c3a --- /dev/null +++ b/metricbeat/module/prometheus/collector/_meta/testdata/etcd-3.3.10-partial.plain @@ -0,0 +1,214 @@ +# HELP etcd_debugging_mvcc_db_compaction_keys_total Total number of db keys compacted. +# TYPE etcd_debugging_mvcc_db_compaction_keys_total counter +etcd_debugging_mvcc_db_compaction_keys_total 0 +# HELP etcd_debugging_mvcc_db_total_size_in_bytes Total size of the underlying database physically allocated in bytes. Use etcd_mvcc_db_total_size_in_bytes +# TYPE etcd_debugging_mvcc_db_total_size_in_bytes gauge +etcd_debugging_mvcc_db_total_size_in_bytes 20480 +# HELP etcd_debugging_mvcc_delete_total Total number of deletes seen by this member. +# TYPE etcd_debugging_mvcc_delete_total counter +etcd_debugging_mvcc_delete_total 0 +# HELP etcd_debugging_mvcc_events_total Total number of events sent by this member. +# TYPE etcd_debugging_mvcc_events_total counter +etcd_debugging_mvcc_events_total 0 +# HELP etcd_debugging_mvcc_keys_total Total number of keys. +# TYPE etcd_debugging_mvcc_keys_total gauge +etcd_debugging_mvcc_keys_total 0 +# HELP etcd_debugging_mvcc_pending_events_total Total number of pending events to be sent. +# TYPE etcd_debugging_mvcc_pending_events_total gauge +etcd_debugging_mvcc_pending_events_total 0 +# HELP etcd_debugging_mvcc_put_total Total number of puts seen by this member. +# TYPE etcd_debugging_mvcc_put_total counter +etcd_debugging_mvcc_put_total 0 +# HELP etcd_debugging_mvcc_range_total Total number of ranges seen by this member. +# TYPE etcd_debugging_mvcc_range_total counter +etcd_debugging_mvcc_range_total 0 +# HELP etcd_debugging_mvcc_slow_watcher_total Total number of unsynced slow watchers. +# TYPE etcd_debugging_mvcc_slow_watcher_total gauge +etcd_debugging_mvcc_slow_watcher_total 0 +# HELP etcd_debugging_mvcc_txn_total Total number of txns seen by this member. +# TYPE etcd_debugging_mvcc_txn_total counter +etcd_debugging_mvcc_txn_total 0 +# HELP etcd_debugging_mvcc_watch_stream_total Total number of watch streams. +# TYPE etcd_debugging_mvcc_watch_stream_total gauge +etcd_debugging_mvcc_watch_stream_total 0 +# HELP etcd_debugging_mvcc_watcher_total Total number of watchers. +# TYPE etcd_debugging_mvcc_watcher_total gauge +etcd_debugging_mvcc_watcher_total 0 +# HELP etcd_debugging_server_lease_expired_total The total number of expired leases. +# TYPE etcd_debugging_server_lease_expired_total counter +etcd_debugging_server_lease_expired_total 0 +# HELP etcd_debugging_store_expires_total Total number of expired keys. +# TYPE etcd_debugging_store_expires_total counter +etcd_debugging_store_expires_total 0 +# HELP etcd_debugging_store_reads_total Total number of reads action by (get/getRecursive), local to this member. +# TYPE etcd_debugging_store_reads_total counter +etcd_debugging_store_reads_total{action="getRecursive"} 1 +# HELP etcd_debugging_store_watch_requests_total Total number of incoming watch requests (new or reestablished). +# TYPE etcd_debugging_store_watch_requests_total counter +etcd_debugging_store_watch_requests_total 0 +# HELP etcd_debugging_store_watchers Count of currently active watchers. +# TYPE etcd_debugging_store_watchers gauge +etcd_debugging_store_watchers 0 +# HELP etcd_debugging_store_writes_total Total number of writes (e.g. set/compareAndDelete) seen by this member. +# TYPE etcd_debugging_store_writes_total counter +etcd_debugging_store_writes_total{action="create"} 1 +etcd_debugging_store_writes_total{action="set"} 2 +# HELP etcd_grpc_proxy_cache_hits_total Total number of cache hits +# TYPE etcd_grpc_proxy_cache_hits_total gauge +etcd_grpc_proxy_cache_hits_total 0 +# HELP etcd_grpc_proxy_cache_keys_total Total number of keys/ranges cached +# TYPE etcd_grpc_proxy_cache_keys_total gauge +etcd_grpc_proxy_cache_keys_total 0 +# HELP etcd_grpc_proxy_cache_misses_total Total number of cache misses +# TYPE etcd_grpc_proxy_cache_misses_total gauge +etcd_grpc_proxy_cache_misses_total 0 +# HELP etcd_grpc_proxy_events_coalescing_total Total number of events coalescing +# TYPE etcd_grpc_proxy_events_coalescing_total counter +etcd_grpc_proxy_events_coalescing_total 0 +# HELP etcd_grpc_proxy_watchers_coalescing_total Total number of current watchers coalescing +# TYPE etcd_grpc_proxy_watchers_coalescing_total gauge +etcd_grpc_proxy_watchers_coalescing_total 0 +# HELP etcd_mvcc_db_total_size_in_bytes Total size of the underlying database physically allocated in bytes. +# TYPE etcd_mvcc_db_total_size_in_bytes gauge +etcd_mvcc_db_total_size_in_bytes 20480 +# HELP etcd_mvcc_db_total_size_in_use_in_bytes Total size of the underlying database logically in use in bytes. +# TYPE etcd_mvcc_db_total_size_in_use_in_bytes gauge +etcd_mvcc_db_total_size_in_use_in_bytes 16384 +# HELP etcd_network_client_grpc_received_bytes_total The total number of bytes received from grpc clients. +# TYPE etcd_network_client_grpc_received_bytes_total counter +etcd_network_client_grpc_received_bytes_total 0 +# HELP etcd_network_client_grpc_sent_bytes_total The total number of bytes sent to grpc clients. +# TYPE etcd_network_client_grpc_sent_bytes_total counter +etcd_network_client_grpc_sent_bytes_total 0 +# HELP etcd_server_go_version Which Go version server is running with. 1 for 'server_go_version' label with current version. +# TYPE etcd_server_go_version gauge +etcd_server_go_version{server_go_version="go1.10.4"} 1 +# HELP etcd_server_has_leader Whether or not a leader exists. 1 is existence, 0 is not. +# TYPE etcd_server_has_leader gauge +etcd_server_has_leader 1 +# HELP etcd_server_health_failures The total number of failed health checks +# TYPE etcd_server_health_failures counter +etcd_server_health_failures 0 +# HELP etcd_server_health_success The total number of successful health checks +# TYPE etcd_server_health_success counter +etcd_server_health_success 0 +# HELP etcd_server_heartbeat_send_failures_total The total number of leader heartbeat send failures (likely overloaded from slow disk). +# TYPE etcd_server_heartbeat_send_failures_total counter +etcd_server_heartbeat_send_failures_total 0 +# HELP etcd_server_id Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID. +# TYPE etcd_server_id gauge +etcd_server_id{server_id="8e9e05c52164694d"} 1 +# HELP etcd_server_is_leader Whether or not this member is a leader. 1 if is, 0 otherwise. +# TYPE etcd_server_is_leader gauge +etcd_server_is_leader 1 +# HELP etcd_server_leader_changes_seen_total The number of leader changes seen. +# TYPE etcd_server_leader_changes_seen_total counter +etcd_server_leader_changes_seen_total 1 +# HELP etcd_server_proposals_applied_total The total number of consensus proposals applied. +# TYPE etcd_server_proposals_applied_total gauge +etcd_server_proposals_applied_total 4 +# HELP etcd_server_proposals_committed_total The total number of consensus proposals committed. +# TYPE etcd_server_proposals_committed_total gauge +etcd_server_proposals_committed_total 4 +# HELP etcd_server_proposals_failed_total The total number of failed proposals seen. +# TYPE etcd_server_proposals_failed_total counter +etcd_server_proposals_failed_total 0 +# HELP etcd_server_proposals_pending The current number of pending proposals to commit. +# TYPE etcd_server_proposals_pending gauge +etcd_server_proposals_pending 0 +# HELP etcd_server_quota_backend_bytes Current backend storage quota size in bytes. +# TYPE etcd_server_quota_backend_bytes gauge +etcd_server_quota_backend_bytes 2.147483648e+09 +# HELP etcd_server_read_indexes_failed_total The total number of failed read indexes seen. +# TYPE etcd_server_read_indexes_failed_total counter +etcd_server_read_indexes_failed_total 0 +# HELP etcd_server_slow_apply_total The total number of slow apply requests (likely overloaded from slow disk). +# TYPE etcd_server_slow_apply_total counter +etcd_server_slow_apply_total 0 +# HELP etcd_server_slow_read_indexes_total The total number of pending read indexes not in sync with leader's or timed out read index requests. +# TYPE etcd_server_slow_read_indexes_total counter +etcd_server_slow_read_indexes_total 0 +# HELP etcd_server_version Which version is running. 1 for 'server_version' label with current version. +# TYPE etcd_server_version gauge +etcd_server_version{server_version="3.3.10"} 1 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 76 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.10.4"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 5.187216e+06 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 5.818832e+06 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 1.444537e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 5341 +# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started. +# TYPE go_memstats_gc_cpu_fraction gauge +go_memstats_gc_cpu_fraction 0.06982921260067133 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 413696 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 5.187216e+06 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 811008 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 6.660096e+06 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 27762 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 0 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 7.471104e+06 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.5534302814931297e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total 55 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 33103 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 6944 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 16384 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 58520 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 65536 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 8.4788e+06 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 1.289791e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 1.114112e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 1.114112e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 1.181516e+07 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 13 diff --git a/metricbeat/module/prometheus/collector/_meta/testdata/etcd-3.3.10-partial.plain-expected.json b/metricbeat/module/prometheus/collector/_meta/testdata/etcd-3.3.10-partial.plain-expected.json new file mode 100644 index 00000000000..74706ab9b39 --- /dev/null +++ b/metricbeat/module/prometheus/collector/_meta/testdata/etcd-3.3.10-partial.plain-expected.json @@ -0,0 +1,239 @@ +[ + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "action": "create" + }, + "metrics": { + "etcd_debugging_store_writes_total": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "action": "set" + }, + "metrics": { + "etcd_debugging_store_writes_total": 2 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "server_id": "8e9e05c52164694d" + }, + "metrics": { + "etcd_server_id": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "server_version": "3.3.10" + }, + "metrics": { + "etcd_server_version": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "action": "getRecursive" + }, + "metrics": { + "etcd_debugging_store_reads_total": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "metrics": { + "etcd_debugging_mvcc_db_compaction_keys_total": 0, + "etcd_debugging_mvcc_db_total_size_in_bytes": 20480, + "etcd_debugging_mvcc_delete_total": 0, + "etcd_debugging_mvcc_events_total": 0, + "etcd_debugging_mvcc_keys_total": 0, + "etcd_debugging_mvcc_pending_events_total": 0, + "etcd_debugging_mvcc_put_total": 0, + "etcd_debugging_mvcc_range_total": 0, + "etcd_debugging_mvcc_slow_watcher_total": 0, + "etcd_debugging_mvcc_txn_total": 0, + "etcd_debugging_mvcc_watch_stream_total": 0, + "etcd_debugging_mvcc_watcher_total": 0, + "etcd_debugging_server_lease_expired_total": 0, + "etcd_debugging_store_expires_total": 0, + "etcd_debugging_store_watch_requests_total": 0, + "etcd_debugging_store_watchers": 0, + "etcd_grpc_proxy_cache_hits_total": 0, + "etcd_grpc_proxy_cache_keys_total": 0, + "etcd_grpc_proxy_cache_misses_total": 0, + "etcd_grpc_proxy_events_coalescing_total": 0, + "etcd_grpc_proxy_watchers_coalescing_total": 0, + "etcd_mvcc_db_total_size_in_bytes": 20480, + "etcd_mvcc_db_total_size_in_use_in_bytes": 16384, + "etcd_network_client_grpc_received_bytes_total": 0, + "etcd_network_client_grpc_sent_bytes_total": 0, + "etcd_server_has_leader": 1, + "etcd_server_health_failures": 0, + "etcd_server_health_success": 0, + "etcd_server_heartbeat_send_failures_total": 0, + "etcd_server_is_leader": 1, + "etcd_server_leader_changes_seen_total": 1, + "etcd_server_proposals_applied_total": 4, + "etcd_server_proposals_committed_total": 4, + "etcd_server_proposals_failed_total": 0, + "etcd_server_proposals_pending": 0, + "etcd_server_quota_backend_bytes": 2147483648, + "etcd_server_read_indexes_failed_total": 0, + "etcd_server_slow_apply_total": 0, + "etcd_server_slow_read_indexes_total": 0, + "go_goroutines": 76, + "go_memstats_alloc_bytes": 5187216, + "go_memstats_alloc_bytes_total": 5818832, + "go_memstats_buck_hash_sys_bytes": 1444537, + "go_memstats_frees_total": 5341, + "go_memstats_gc_cpu_fraction": 0.06982921260067133, + "go_memstats_gc_sys_bytes": 413696, + "go_memstats_heap_alloc_bytes": 5187216, + "go_memstats_heap_idle_bytes": 811008, + "go_memstats_heap_inuse_bytes": 6660096, + "go_memstats_heap_objects": 27762, + "go_memstats_heap_released_bytes": 0, + "go_memstats_heap_sys_bytes": 7471104, + "go_memstats_last_gc_time_seconds": 1553430281.4931297, + "go_memstats_lookups_total": 55, + "go_memstats_mallocs_total": 33103, + "go_memstats_mcache_inuse_bytes": 6944, + "go_memstats_mcache_sys_bytes": 16384, + "go_memstats_mspan_inuse_bytes": 58520, + "go_memstats_mspan_sys_bytes": 65536, + "go_memstats_next_gc_bytes": 8478800, + "go_memstats_other_sys_bytes": 1289791, + "go_memstats_stack_inuse_bytes": 1114112, + "go_memstats_stack_sys_bytes": 1114112, + "go_memstats_sys_bytes": 11815160, + "go_threads": 13 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "version": "go1.10.4" + }, + "metrics": { + "go_info": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "server_go_version": "go1.10.4" + }, + "metrics": { + "etcd_server_go_version": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain b/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain new file mode 100644 index 00000000000..136196b72c2 --- /dev/null +++ b/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain @@ -0,0 +1,135 @@ +# HELP go_gc_duration_seconds A summary of the GC invocation durations. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 3.8386e-05 +go_gc_duration_seconds{quantile="0.25"} 4.2803e-05 +go_gc_duration_seconds{quantile="0.5"} 6.0618e-05 +go_gc_duration_seconds{quantile="0.75"} 0.004392391 +go_gc_duration_seconds{quantile="1"} 0.004392391 +go_gc_duration_seconds_sum 0.004534198 +go_gc_duration_seconds_count 4 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 35 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.11.3"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 1.0558112e+07 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 1.408776e+07 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 1.447018e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 15673 +# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started. +# TYPE go_memstats_gc_cpu_fraction gauge +go_memstats_gc_cpu_fraction 0.0008429952574435172 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 2.379776e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 1.0558112e+07 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 5.4042624e+07 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 1.2214272e+07 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 61771 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 0 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 6.6256896e+07 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.5534303161488917e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 77444 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 6912 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 16384 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 127984 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 131072 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 1.8390112e+07 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 1.201294e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 851968 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 851968 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 7.2284408e+07 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 14 +# HELP net_conntrack_dialer_conn_attempted_total Total number of connections attempted by the given dialer a given name. +# TYPE net_conntrack_dialer_conn_attempted_total counter +net_conntrack_dialer_conn_attempted_total{dialer_name="alertmanager"} 0 +net_conntrack_dialer_conn_attempted_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_attempted_total{dialer_name="prometheus"} 1 +# HELP net_conntrack_dialer_conn_closed_total Total number of connections closed which originated from the dialer of a given name. +# TYPE net_conntrack_dialer_conn_closed_total counter +net_conntrack_dialer_conn_closed_total{dialer_name="alertmanager"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_closed_total{dialer_name="prometheus"} 0 +# HELP net_conntrack_dialer_conn_established_total Total number of connections successfully established by the given dialer a given name. +# TYPE net_conntrack_dialer_conn_established_total counter +net_conntrack_dialer_conn_established_total{dialer_name="alertmanager"} 0 +net_conntrack_dialer_conn_established_total{dialer_name="default"} 0 +net_conntrack_dialer_conn_established_total{dialer_name="prometheus"} 1 +# HELP net_conntrack_listener_conn_accepted_total Total number of connections opened to the listener of a given name. +# TYPE net_conntrack_listener_conn_accepted_total counter +net_conntrack_listener_conn_accepted_total{listener_name="http"} 3 +# HELP net_conntrack_listener_conn_closed_total Total number of connections closed that were made to the listener of a given name. +# TYPE net_conntrack_listener_conn_closed_total counter +net_conntrack_listener_conn_closed_total{listener_name="http"} 0 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 0.14 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 1.048576e+06 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 13 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 3.5934208e+07 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.5534303054e+09 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 1.50646784e+08 +# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. +# TYPE process_virtual_memory_max_bytes gauge +process_virtual_memory_max_bytes -1 +# HELP prometheus_api_remote_read_queries The current number of remote read queries being executed or waiting. +# TYPE prometheus_api_remote_read_queries gauge +prometheus_api_remote_read_queries 0 diff --git a/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json b/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json new file mode 100644 index 00000000000..664c058add5 --- /dev/null +++ b/metricbeat/module/prometheus/collector/_meta/testdata/prometheus-2.6.0-partial.plain-expected.json @@ -0,0 +1,283 @@ +[ + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "quantile": "0.25" + }, + "metrics": { + "go_gc_duration_seconds": 0.000042803 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "quantile": "0" + }, + "metrics": { + "go_gc_duration_seconds": 0.000038386 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "quantile": "1" + }, + "metrics": { + "go_gc_duration_seconds": 0.004392391 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "dialer_name": "default" + }, + "metrics": { + "net_conntrack_dialer_conn_attempted_total": 0, + "net_conntrack_dialer_conn_closed_total": 0, + "net_conntrack_dialer_conn_established_total": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "dialer_name": "prometheus" + }, + "metrics": { + "net_conntrack_dialer_conn_attempted_total": 1, + "net_conntrack_dialer_conn_closed_total": 0, + "net_conntrack_dialer_conn_established_total": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "dialer_name": "alertmanager" + }, + "metrics": { + "net_conntrack_dialer_conn_attempted_total": 0, + "net_conntrack_dialer_conn_closed_total": 0, + "net_conntrack_dialer_conn_established_total": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "quantile": "0.5" + }, + "metrics": { + "go_gc_duration_seconds": 0.000060618 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "quantile": "0.75" + }, + "metrics": { + "go_gc_duration_seconds": 0.004392391 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "listener_name": "http" + }, + "metrics": { + "net_conntrack_listener_conn_accepted_total": 3, + "net_conntrack_listener_conn_closed_total": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "labels": { + "version": "go1.11.3" + }, + "metrics": { + "go_info": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + }, + { + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "collector" + }, + "prometheus": { + "metrics": { + "go_gc_duration_seconds_count": 4, + "go_gc_duration_seconds_sum": 0.004534198, + "go_goroutines": 35, + "go_memstats_alloc_bytes": 10558112, + "go_memstats_alloc_bytes_total": 14087760, + "go_memstats_buck_hash_sys_bytes": 1447018, + "go_memstats_frees_total": 15673, + "go_memstats_gc_cpu_fraction": 0.0008429952574435172, + "go_memstats_gc_sys_bytes": 2379776, + "go_memstats_heap_alloc_bytes": 10558112, + "go_memstats_heap_idle_bytes": 54042624, + "go_memstats_heap_inuse_bytes": 12214272, + "go_memstats_heap_objects": 61771, + "go_memstats_heap_released_bytes": 0, + "go_memstats_heap_sys_bytes": 66256896, + "go_memstats_last_gc_time_seconds": 1553430316.1488917, + "go_memstats_lookups_total": 0, + "go_memstats_mallocs_total": 77444, + "go_memstats_mcache_inuse_bytes": 6912, + "go_memstats_mcache_sys_bytes": 16384, + "go_memstats_mspan_inuse_bytes": 127984, + "go_memstats_mspan_sys_bytes": 131072, + "go_memstats_next_gc_bytes": 18390112, + "go_memstats_other_sys_bytes": 1201294, + "go_memstats_stack_inuse_bytes": 851968, + "go_memstats_stack_sys_bytes": 851968, + "go_memstats_sys_bytes": 72284408, + "go_threads": 14, + "process_cpu_seconds_total": 0.14, + "process_max_fds": 1048576, + "process_open_fds": 13, + "process_resident_memory_bytes": 35934208, + "process_start_time_seconds": 1553430305.4, + "process_virtual_memory_bytes": 150646784, + "process_virtual_memory_max_bytes": -1, + "prometheus_api_remote_read_queries": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index e16bdf6cc53..51cbfa1ee86 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -79,7 +79,9 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) { for _, promEvent := range promEvents { labelsHash := promEvent.LabelsHash() if _, ok := eventList[labelsHash]; !ok { - eventList[labelsHash] = common.MapStr{} + eventList[labelsHash] = common.MapStr{ + "metrics": common.MapStr{}, + } // Add labels if len(promEvent.labels) > 0 { @@ -87,9 +89,9 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) { } } - eventList[labelsHash].Update(common.MapStr{ - "metrics": promEvent.data, - }) + // Not checking anything here because we create these maps some lines before + metrics := eventList[labelsHash]["metrics"].(common.MapStr) + metrics.Update(promEvent.data) } } diff --git a/metricbeat/module/prometheus/collector/collector_integration_test.go b/metricbeat/module/prometheus/collector/collector_integration_test.go deleted file mode 100644 index 98d9169ae1d..00000000000 --- a/metricbeat/module/prometheus/collector/collector_integration_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build integration - -package collector - -import ( - "os" - "testing" - - "github.com/elastic/beats/libbeat/tests/compose" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" -) - -func TestData(t *testing.T) { - compose.EnsureUp(t, "prometheus") - - ms := mbtest.NewReportingMetricSetV2(t, getConfig()) - err := mbtest.WriteEventsReporterV2(ms, t, "") - if err != nil { - t.Fatal(err) - } -} - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "prometheus", - "metricsets": []string{"collector"}, - "hosts": []string{getPrometheusEnvHost() + ":" + getPrometheusEnvPort()}, - "namespace": "collector", - } -} - -func getPrometheusEnvHost() string { - host := os.Getenv("PROMETHEUS_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func getPrometheusEnvPort() string { - port := os.Getenv("PROMETHEUS_PORT") - - if len(port) == 0 { - port = "9090" - } - return port -} diff --git a/metricbeat/module/prometheus/fields.go b/metricbeat/module/prometheus/fields.go index d4c3f017f04..23d94ffe4c5 100644 --- a/metricbeat/module/prometheus/fields.go +++ b/metricbeat/module/prometheus/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPrometheus returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/prometheus. func AssetPrometheus() string { - return "eJyUkU1u6zAMhPc+xUBv95DkAFr0Ci3QZVEEikXbbPQHkkGQ2xeJjdQBsmi1E78RORpucaSLR5OaySY6aQcYWyIP93Yvug6IpL1wM67F46UDgHcLptBeQqOIQWpGwM8rUImtcrFdB+hUxfZ9LQOPHkNISh0glCgoeYzhqiEzLqN6fDjV5DZwk1lznx0wMKWo/jb3H14lkoAVnFsVC8UwkdAGKRwoKc6cEnKwfsLAoraBTQQhNQQhxHo6JLr12qKETOsEdnOPGwXs0sijHr6ot6U0X/YzOdLlXCUu6ElI17PKJJMJ91jNeOJgFv3Wwuo3D2SfQ2tcxkXm/rs/uryT7cOivgMAAP//FvGzhQ==" + return "eJyUkc1qwzAQhO9+ikG9hSQPoENfoYUeSwmKtba30R+7G0LeviQ2aQIttLppv9FoNNrgQGePJjWTTXTUDjC2RB7u9TZ0HRBJe+FmXIvHcwcAbxZMob2ERhGD1IyA71OgElvlYtsO0KmK7fpaBh49hpCUOkAoUVDyGMNFQ2ZcRvV4d6rJreEms+Y+OmBgSlH99d4nvEgkASs4tyoWimEioTVS2FNSnDgl5GD9hIFFbQ2bCEJqCEKI9bhPdPXaoIRM9w1sZ4/t6soBOzfyqPtP6m0ZzZvdTA50PlWJC/qhpsu6ayWTCfdL0t8yzKK/h7h70QPZ5dAal3GRuZX7Z84b2Tx81lcAAAD//3hXtDU=" }