diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index a7910878cd5..295dd71b002 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -51,6 +51,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Heartbeat* +- Made monitors.d configuration part of the default config. {pull}9004[9004] + *Journalbeat* *Metricbeat* diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index 392e86155d7..6802be3c2ed 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -97,8 +97,9 @@ func (b packageBuilder) Build() error { } type testPackagesParams struct { - HasModules bool - HasModulesD bool + HasModules bool + HasMonitorsD bool + HasModulesD bool } // TestPackagesOption defines a option to the TestPackages target. @@ -111,6 +112,13 @@ func WithModules() func(params *testPackagesParams) { } } +// WithMonitorsD enables monitors folder contents testing. +func WithMonitorsD() func(params *testPackagesParams) { + return func(params *testPackagesParams) { + params.HasMonitorsD = true + } +} + // WithModulesD enables modules.d folder contents testing func WithModulesD() func(params *testPackagesParams) { return func(params *testPackagesParams) { @@ -140,6 +148,10 @@ func TestPackages(options ...TestPackagesOption) error { args = append(args, "--modules") } + if params.HasMonitorsD { + args = append(args, "--monitors.d") + } + if params.HasModulesD { args = append(args, "--modules.d") } diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index cea1e9e9b69..bc29e4dfb33 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -55,13 +55,15 @@ var ( modulesDirPattern = regexp.MustCompile(`module/.+`) modulesDDirPattern = regexp.MustCompile(`modules.d/$`) modulesDFilePattern = regexp.MustCompile(`modules.d/.+`) + monitorsDFilePattern = regexp.MustCompile(`monitors.d/.+`) systemdUnitFilePattern = regexp.MustCompile(`/lib/systemd/system/.*\.service`) ) var ( - files = flag.String("files", "../build/distributions/*/*", "filepath glob containing package files") - modules = flag.Bool("modules", false, "check modules folder contents") - modulesd = flag.Bool("modules.d", false, "check modules.d folder contents") + files = flag.String("files", "../build/distributions/*/*", "filepath glob containing package files") + modules = flag.Bool("modules", false, "check modules folder contents") + modulesd = flag.Bool("modules.d", false, "check modules.d folder contents") + monitorsd = flag.Bool("monitors.d", false, "check monitors.d folder contents") ) func TestRPM(t *testing.T) { @@ -117,6 +119,7 @@ func checkRPM(t *testing.T, file string) { checkModulesPermissions(t, p) checkModulesPresent(t, "/usr/share", p) checkModulesDPresent(t, "/etc/", p) + checkMonitorsDPresent(t, "/etc", p) checkModulesOwner(t, p) checkSystemdUnitPermissions(t, p) } @@ -134,6 +137,7 @@ func checkDeb(t *testing.T, file string, buf *bytes.Buffer) { checkManifestOwner(t, p) checkModulesPresent(t, "./usr/share", p) checkModulesDPresent(t, "./etc/", p) + checkMonitorsDPresent(t, "./etc/", p) checkModulesPermissions(t, p) checkModulesOwner(t, p) checkSystemdUnitPermissions(t, p) @@ -316,6 +320,12 @@ func checkModulesDPresent(t *testing.T, prefix string, p *packageFile) { } } +func checkMonitorsDPresent(t *testing.T, prefix string, p *packageFile) { + if *monitorsd { + checkMonitors(t, "monitors.d", prefix, monitorsDFilePattern, p) + } +} + func checkModules(t *testing.T, name, prefix string, r *regexp.Regexp, p *packageFile) { t.Run(fmt.Sprintf("%s %s contents", p.Name, name), func(t *testing.T) { minExpectedModules := 4 @@ -333,6 +343,23 @@ func checkModules(t *testing.T, name, prefix string, r *regexp.Regexp, p *packag }) } +func checkMonitors(t *testing.T, name, prefix string, r *regexp.Regexp, p *packageFile) { + t.Run(fmt.Sprintf("%s %s contents", p.Name, name), func(t *testing.T) { + minExpectedModules := 1 + total := 0 + for _, entry := range p.Contents { + if strings.HasPrefix(entry.File, prefix) && r.MatchString(entry.File) { + total++ + } + } + + if total < minExpectedModules { + t.Errorf("not enough monitors found under %s: actual=%d, expected>=%d", + name, total, minExpectedModules) + } + }) +} + func checkDockerEntryPoint(t *testing.T, p *packageFile, info *dockerInfo) { expectedMode := os.FileMode(0755) diff --git a/heartbeat/_meta/beat.docker.yml b/heartbeat/_meta/beat.docker.yml index 496602310d1..f845c4a47a1 100644 --- a/heartbeat/_meta/beat.docker.yml +++ b/heartbeat/_meta/beat.docker.yml @@ -1,3 +1,14 @@ +# Define a directory to load monitor definitions from. Definitions take the form +# of individual yaml files. +heartbeat.config.monitors: + # Directory + glob pattern to search for configuration files + path: ${path.config}/monitors.d/*.yml + # If enabled, heartbeat will periodically check the config.monitors path for changes + reload.enabled: false + # How often to check for changes + reload.period: 5s + + heartbeat.monitors: - type: http schedule: '@every 5s' diff --git a/heartbeat/_meta/beat.reference.yml b/heartbeat/_meta/beat.reference.yml index 86e2cfbd3a9..9e163c45101 100644 --- a/heartbeat/_meta/beat.reference.yml +++ b/heartbeat/_meta/beat.reference.yml @@ -9,6 +9,17 @@ ############################# Heartbeat ###################################### + +# Define a directory to load monitor definitions from. Definitions take the form +# of individual yaml files. +heartbeat.config.monitors: + # Directory + glob pattern to search for configuration files + path: ${path.config}/monitors.d/*.yml + # If enabled, heartbeat will periodically check the config.monitors path for changes + reload.enabled: false + # How often to check for changes + reload.period: 5s + # Configure monitors heartbeat.monitors: - type: icmp # monitor type `icmp` (requires root) uses ICMP Echo Request to ping diff --git a/heartbeat/_meta/beat.yml b/heartbeat/_meta/beat.yml index 76c35e2690e..0e30f34f239 100644 --- a/heartbeat/_meta/beat.yml +++ b/heartbeat/_meta/beat.yml @@ -9,7 +9,17 @@ ############################# Heartbeat ###################################### -# Configure monitors +# Define a directory to load monitor definitions from. Definitions take the form +# of individual yaml files. +heartbeat.config.monitors: + # Directory + glob pattern to search for configuration files + path: ${path.config}/monitors.d/*.yml + # If enabled, heartbeat will periodically check the config.monitors path for changes + reload.enabled: false + # How often to check for changes + reload.period: 5s + +# Configure monitors inline heartbeat.monitors: - type: http diff --git a/heartbeat/heartbeat.docker.yml b/heartbeat/heartbeat.docker.yml index 9a344a539e1..2302283f674 100644 --- a/heartbeat/heartbeat.docker.yml +++ b/heartbeat/heartbeat.docker.yml @@ -1,3 +1,14 @@ +# Define a directory to load monitor definitions from. Definitions take the form +# of individual yaml files. +heartbeat.config.monitors: + # Directory + glob pattern to search for configuration files + path: ${path.config}/monitors.d/*.yml + # If enabled, heartbeat will periodically check the config.monitors path for changes + reload.enabled: false + # How often to check for changes + reload.period: 5s + + heartbeat.monitors: - type: http schedule: '@every 5s' diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index fd3d8bb8af9..99ccee877a8 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -9,6 +9,17 @@ ############################# Heartbeat ###################################### + +# Define a directory to load monitor definitions from. Definitions take the form +# of individual yaml files. +heartbeat.config.monitors: + # Directory + glob pattern to search for configuration files + path: ${path.config}/monitors.d/*.yml + # If enabled, heartbeat will periodically check the config.monitors path for changes + reload.enabled: false + # How often to check for changes + reload.period: 5s + # Configure monitors heartbeat.monitors: - type: icmp # monitor type `icmp` (requires root) uses ICMP Echo Request to ping diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml index 1e20e85e6b3..eeb6a95475f 100644 --- a/heartbeat/heartbeat.yml +++ b/heartbeat/heartbeat.yml @@ -9,7 +9,17 @@ ############################# Heartbeat ###################################### -# Configure monitors +# Define a directory to load monitor definitions from. Definitions take the form +# of individual yaml files. +heartbeat.config.monitors: + # Directory + glob pattern to search for configuration files + path: ${path.config}/monitors.d/*.yml + # If enabled, heartbeat will periodically check the config.monitors path for changes + reload.enabled: false + # How often to check for changes + reload.period: 5s + +# Configure monitors inline heartbeat.monitors: - type: http diff --git a/heartbeat/magefile.go b/heartbeat/magefile.go index 78233688487..9dab7cec2c5 100644 --- a/heartbeat/magefile.go +++ b/heartbeat/magefile.go @@ -90,7 +90,7 @@ func Package() { // TestPackages tests the generated packages (i.e. file modes, owners, groups). func TestPackages() error { - return mage.TestPackages() + return mage.TestPackages(mage.WithMonitorsD()) } // Update updates the generated files (aka make update). @@ -118,11 +118,23 @@ func GoTestIntegration(ctx context.Context) error { } func customizePackaging() { + monitorsDTarget := "monitors.d" + unixMonitorsDir := "/etc/{{.BeatName}}/monitors.d" + monitorsD := mage.PackageFile{ + Mode: 0644, + Source: "monitors.d", + } + for _, args := range mage.Packages { pkgType := args.Types[0] switch pkgType { case mage.Docker: args.Spec.ExtraVar("linux_capabilities", "cap_net_raw=eip") + args.Spec.Files[monitorsDTarget] = monitorsD + case mage.TarGz, mage.Zip: + args.Spec.Files[monitorsDTarget] = monitorsD + case mage.Deb, mage.RPM, mage.DMG: + args.Spec.Files[unixMonitorsDir] = monitorsD } } } diff --git a/heartbeat/monitors.d/sample.http.yml.disabled b/heartbeat/monitors.d/sample.http.yml.disabled new file mode 100644 index 00000000000..5001f41b5ae --- /dev/null +++ b/heartbeat/monitors.d/sample.http.yml.disabled @@ -0,0 +1,91 @@ +# These files contain a list of monitor configurations identical +# to the heartbeat.monitors section in heartbeat.yml +# The .example extension on this file must be removed for it to +# be loaded. + +- type: http # monitor type `http`. Connect via HTTP an optionally verify response + + # Monitor name used for job name and document type + #name: http + + # Enable/Disable monitor + #enabled: true + + # Configure task schedule + schedule: '@every 5s' # every 5 seconds from start of beat + + # Configure URLs to ping + urls: ["http://localhost:9200"] + + # Configure IP protocol types to ping on if hostnames are configured. + # Ping all resolvable IPs if `mode` is `all`, or only one IP if `mode` is `any`. + ipv4: true + ipv6: true + mode: any + + # Configure file json file to be watched for changes to the monitor: + #watch.poll_file: + # Path to check for updates. + #path: + + # Interval between file file changed checks. + #interval: 5s + + # Optional HTTP proxy url. + #proxy_url: '' + + # Total test connection and data exchange timeout + #timeout: 16s + + # Optional Authentication Credentials + #username: '' + #password: '' + + # TLS/SSL connection settings for use with HTTPS endpoint. If not configured + # system defaults will be used. + #ssl: + # Certificate Authorities + #certificate_authorities: [''] + + # Required TLS protocols + #supported_protocols: ["TLSv1.0", "TLSv1.1", "TLSv1.2"] + + # Request settings: + #check.request: + # Configure HTTP method to use. Only 'HEAD', 'GET' and 'POST' methods are allowed. + #method: "GET" + + # Dictionary of additional HTTP headers to send: + #headers: + + # Optional request body content + #body: + + # Expected response settings + #check.response: + # Expected status code. If not configured or set to 0 any status code not + # being 404 is accepted. + #status: 0 + + # Required response headers. + #headers: + + # Required response contents. + #body: + + # Parses the body as JSON, then checks against the given condition expression + #json: + #- description: Explanation of what the check does + # condition: + # equals: + # myField: expectedValue + + + # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE + # Configure file json file to be watched for changes to the monitor: + #watch.poll_file: + # Path to check for updates. + #path: + + # Interval between file file changed checks. + #interval: 5s diff --git a/heartbeat/monitors.d/sample.icmp.yml.disabled b/heartbeat/monitors.d/sample.icmp.yml.disabled new file mode 100644 index 00000000000..7e58047b230 --- /dev/null +++ b/heartbeat/monitors.d/sample.icmp.yml.disabled @@ -0,0 +1,67 @@ +# These files contain a list of monitor configurations identical +# to the heartbeat.monitors section in heartbeat.yml +# The .example extension on this file must be removed for it to +# be loaded. + +- type: icmp # monitor type `icmp` (requires root) uses ICMP Echo Request to ping + # configured hosts + + # Monitor name used for job name and document type. + #name: icmp + + # Enable/Disable monitor + #enabled: true + + # Configure task schedule using cron-like syntax + schedule: '*/5 * * * * * *' # exactly every 5 seconds like 10:00:00, 10:00:05, ... + + # List of hosts to ping + hosts: ["localhost"] + + # Configure IP protocol types to ping on if hostnames are configured. + # Ping all resolvable IPs if `mode` is `all`, or only one IP if `mode` is `any`. + ipv4: true + ipv6: true + mode: any + + # Total running time per ping test. + timeout: 16s + + # Waiting duration until another ICMP Echo Request is emitted. + wait: 1s + + # The tags of the monitors are included in their own field with each + # transaction published. Tags make it easy to group servers by different + # logical properties. + #tags: ["service-X", "web-tier"] + + # Optional fields that you can specify to add additional information to the + # monitor output. Fields can be scalar values, arrays, dictionaries, or any nested + # combination of these. + #fields: + # env: staging + + # If this option is set to true, the custom fields are stored as top-level + # fields in the output document instead of being grouped under a fields + # sub-dictionary. Default is false. + #fields_under_root: false + + # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE + # Configure file json file to be watched for changes to the monitor: + #watch.poll_file: + # Path to check for updates. + #path: + + # Interval between file file changed checks. + #interval: 5s + + # Define a directory to load monitor definitions from. Definitions take the form + # of individual yaml files. + # heartbeat.config.monitors: + # Directory + glob pattern to search for configuration files + #path: /path/to/my/monitors.d/*.yml + # If enabled, heartbeat will periodically check the config.monitors path for changes + #reload.enabled: true + # How often to check for changes + #reload.period: 1s + diff --git a/heartbeat/monitors.d/sample.tcp.yml.disabled b/heartbeat/monitors.d/sample.tcp.yml.disabled new file mode 100644 index 00000000000..01e0d13ae07 --- /dev/null +++ b/heartbeat/monitors.d/sample.tcp.yml.disabled @@ -0,0 +1,78 @@ +# These files contain a list of monitor configurations identical +# to the heartbeat.monitors section in heartbeat.yml +# The .example extension on this file must be removed for it to +# be loaded. + +- type: tcp # monitor type `tcp`. Connect via TCP and optionally verify endpoint + # by sending/receiving a custom payload + + # Monitor name used for job name and document type + #name: tcp + + # Enable/Disable monitor + #enabled: true + + # Configure task schedule + schedule: '@every 5s' # every 5 seconds from start of beat + + # configure hosts to ping. + # Entries can be: + # - plain host name or IP like `localhost`: + # Requires ports configs to be checked. If ssl is configured, + # a SSL/TLS based connection will be established. Otherwise plain tcp connection + # will be established + # - hostname + port like `localhost:12345`: + # Connect to port on given host. If ssl is configured, + # a SSL/TLS based connection will be established. Otherwise plain tcp connection + # will be established + # - full url syntax. `scheme://:[port]`. The `` can be one of + # `tcp`, `plain`, `ssl` and `tls`. If `tcp`, `plain` is configured, a plain + # tcp connection will be established, even if ssl is configured. + # Using `tls`/`ssl`, an SSL connection is established. If no ssl is configured, + # system defaults will be used (not supported on windows). + # If `port` is missing in url, the ports setting is required. + hosts: ["localhost:9200"] + + # Configure IP protocol types to ping on if hostnames are configured. + # Ping all resolvable IPs if `mode` is `all`, or only one IP if `mode` is `any`. + ipv4: true + ipv6: true + mode: any + + # List of ports to ping if host does not contain a port number + # ports: [80, 9200, 5044] + + # Total test connection and data exchange timeout + #timeout: 16s + + # Optional payload string to send to remote and expected answer. If none is + # configured, the endpoint is expected to be up if connection attempt was + # successful. If only `send_string` is configured, any response will be + # accepted as ok. If only `receive_string` is configured, no payload will be + # send, but client expects to receive expected payload on connect. + #check: + #send: '' + #receive: '' + + # SOCKS5 proxy url + # proxy_url: '' + + # Resolve hostnames locally instead on SOCKS5 server: + #proxy_use_local_resolver: false + + # TLS/SSL connection settings: + #ssl: + # Certificate Authorities + #certificate_authorities: [''] + + # Required TLS protocols + #supported_protocols: ["TLSv1.0", "TLSv1.1", "TLSv1.2"] + + # NOTE: THIS FEATURE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE + # Configure file json file to be watched for changes to the monitor: + #watch.poll_file: + # Path to check for updates. + #path: + + # Interval between file file changed checks. + #interval: 5s