Skip to content

Commit

Permalink
WildFly Monitoring (#224)
Browse files Browse the repository at this point in the history
* Added wildfly monitoring capabilities

* Updated wildfly metrics

* Modified integration tests to support custom scraper builds

* Updated documentation for test

* Fixed mbeans in wildfly script

* Updated description and removed unnecessary properties

* Updated wildfly docs to include jmx url
  • Loading branch information
jmwilliams89 committed Feb 3, 2022
1 parent ed5c91e commit 49729b3
Show file tree
Hide file tree
Showing 9 changed files with 316 additions and 32 deletions.
1 change: 1 addition & 0 deletions jmx-metrics/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ mutually exclusive with `otel.jmx.groovy.script`. The currently supported target
| [`kafka-producer`](./docs/target-systems/kafka-producer.md) |
| [`solr`](./docs/target-systems/solr.md) |
| [`tomcat`](./docs/target-systems/tomcat.md) |
| [`wildfly`](./docs/target-systems/wildfly.md) |


### JMX Query Helpers
Expand Down
77 changes: 77 additions & 0 deletions jmx-metrics/docs/target-systems/wildfly.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# WildFly Metrics

The JMX Metric Gatherer provides built in WildFly metric gathering capabilities.
Details about using JMX with WildFly can be found here: https://docs.jboss.org/author/display/WFLY/JMX%20subsystem%20configuration.html

### Deployment Metrics
* Name: `wildfly.session.count`
* Description: The number of sessions created.
* Unit: `{sessions}`
* Labels: `deployment`
* Instrument Type: LongCounterCallback

* Name: `wildfly.session.active`
* Description: The number of currently active sessions.
* Unit: `{sessions}`
* Labels: `deployment`
* Instrument Type: LongUpDownCounterCallback

* Name: `wildfly.session.expired`
* Description: The number of sessions that have expired.
* Unit: `{sessions}`
* Labels: `deployment`
* Instrument Type: LongCounterCallback

* Name: `wildfly.session.rejected`
* Description: The number of sessions that have been rejected.
* Unit: `{sessions}`
* Labels: `deployment`
* Instrument Type: LongCounterCallback

### Listener Metrics
* Name: `wildfly.request.count`
* Description: The number of requests received.
* Unit: `{requests}`
* Labels: `server`, `listener`
* Instrument Type: LongCounterCallback

* Name: `wildfly.request.time`
* Description: The total amount of time spent on requests.
* Unit: `ns`
* Labels: `server`, `listener`
* Instrument Type: LongCounterCallback

* Name: `wildfly.request.server_error`
* Description: The number of requests that have resulted in a 5xx response.
* Unit: `{requests}`
* Labels: `server`, `listener`
* Instrument Type: LongCounterCallback

* Name: `wildfly.network.io`
* Description: The number of bytes transmitted.
* Unit: `by`
* Labels: `server`, `listener`, `state`
* Instrument Type: LongCounterCallback

### Data Source Metrics
* Name: `wildfly.jdbc.connection.open`
* Description: The number of open jdbc connections.
* Unit: `{connections}`
* Labels: `data_source`, `state`
* Instrument Type: LongUpDownCounterCallback

* Name: `wildfly.jdbc.request.wait`
* Description: The number of jdbc connections that had to wait before opening.
* Unit: `{requests}`
* Labels: `data_source`
* Instrument Type: LongCounterCallback

* Name: `wildfly.jdbc.transaction.count`
* Description: The number of transactions created.
* Unit: `{transactions}`
* Instrument Type: LongCounterCallback

* Name: `wildfly.jdbc.rollback.count`
* Description: The number of transactions rolled back.
* Unit: `{transactions}`
* Instrument Type: LongCounterCallback
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,26 @@ void beforeAll() {
otlpServer = new OtlpGrpcServer();
otlpServer.start();
exposeHostPorts(otlpServer.httpPort());
String otlpEndpoint = "http://host.testcontainers.internal:" + otlpServer.httpPort();

scraper = buildScraper(otlpEndpoint);
scraper.start();
}

@AfterAll
// No need to block other tests waiting on return.
@SuppressWarnings("FutureReturnValueIgnored")
void afterAll() {
otlpServer.stop();
scraper.stop();
}

@BeforeEach
void beforeEach() {
otlpServer.reset();
}

protected GenericContainer<?> buildScraper(String otlpEndpoint) {
String scraperJarPath = System.getProperty("shadow.jar.path");

List<String> scraperCommand = new ArrayList<>();
Expand All @@ -71,9 +90,7 @@ void beforeAll() {
scraperCommand.add("/app/OpenTelemetryJava.jar");
scraperCommand.add("-Dotel.jmx.username=cassandra");
scraperCommand.add("-Dotel.jmx.password=cassandra");
scraperCommand.add(
"-Dotel.exporter.otlp.endpoint=http://host.testcontainers.internal:"
+ otlpServer.httpPort());
scraperCommand.add("-Dotel.exporter.otlp.endpoint=" + otlpEndpoint);
scraperCommand.add("io.opentelemetry.contrib.jmxmetrics.JmxMetrics");
scraperCommand.add("-config");

Expand All @@ -84,32 +101,17 @@ void beforeAll() {
scraperCommand.add("/app/" + configName);
}

scraper =
new GenericContainer<>("openjdk:8u272-jre-slim")
.withNetwork(Network.SHARED)
.withCopyFileToContainer(
MountableFile.forHostPath(scraperJarPath), "/app/OpenTelemetryJava.jar")
.withCopyFileToContainer(
MountableFile.forClasspathResource("script.groovy"), "/app/script.groovy")
.withCopyFileToContainer(
MountableFile.forClasspathResource(configName), "/app/" + configName)
.withCommand(scraperCommand.toArray(new String[0]))
.withStartupTimeout(Duration.ofMinutes(2))
.waitingFor(Wait.forLogMessage(".*Started GroovyRunner.*", 1));
scraper.start();
}

@AfterAll
// No need to block other tests waiting on return.
@SuppressWarnings("FutureReturnValueIgnored")
void afterAll() {
otlpServer.stop();
scraper.stop();
}

@BeforeEach
void beforeEach() {
otlpServer.reset();
return new GenericContainer<>("openjdk:8u272-jre-slim")
.withNetwork(Network.SHARED)
.withCopyFileToContainer(
MountableFile.forHostPath(scraperJarPath), "/app/OpenTelemetryJava.jar")
.withCopyFileToContainer(
MountableFile.forClasspathResource("script.groovy"), "/app/script.groovy")
.withCopyFileToContainer(
MountableFile.forClasspathResource(configName), "/app/" + configName)
.withCommand(scraperCommand.toArray(new String[0]))
.withStartupTimeout(Duration.ofMinutes(2))
.waitingFor(Wait.forLogMessage(".*Started GroovyRunner.*", 1));
}

protected static GenericContainer<?> cassandraContainer() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
/*
* Copyright The OpenTelemetry Authors
* SPDX-License-Identifier: Apache-2.0
*/

package io.opentelemetry.contrib.jmxmetrics.target_systems;

import static org.assertj.core.api.Assertions.entry;

import io.opentelemetry.contrib.jmxmetrics.AbstractIntegrationTest;
import java.time.Duration;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.Network;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.MountableFile;

class WildflyIntegrationTest extends AbstractIntegrationTest {

WildflyIntegrationTest() {
super(/* configFromStdin= */ false, "target-systems/wildfly.properties");
}

/* In order to create a JMX connection to WildFLy, the scraper requires an additional
client jar in the classpath. To facilitate this, the scraper runs in the same container,
which gives it access to the client jar packaged with WildFly. */
@Override
protected GenericContainer<?> buildScraper(String otlpEndpoint) {
String scraperJarPath = System.getProperty("shadow.jar.path");

return new GenericContainer<>("jboss/wildfly:23.0.1.Final")
.withNetwork(Network.SHARED)
.withNetworkAliases("wildfly")
.withCopyFileToContainer(
MountableFile.forHostPath(scraperJarPath), "/app/OpenTelemetryJMXMetrics.jar")
.withCopyFileToContainer(
MountableFile.forClasspathResource("script.groovy"), "/app/script.groovy")
.withCopyFileToContainer(
MountableFile.forClasspathResource("target-systems/wildfly.properties"),
"/app/target-systems/wildfly.properties")
.withCopyFileToContainer(
MountableFile.forClasspathResource("wildfly/start.sh", 0007), "/app/start.sh")
.withEnv("OTLP_ENDPOINT", otlpEndpoint)
.withCommand("/app/start.sh")
.withStartupTimeout(Duration.ofMinutes(2))
.waitingFor(Wait.forLogMessage(".*Started GroovyRunner.*", 1));
}

@Test
void endToEnd() {
waitAndAssertMetrics(
metric ->
assertSumWithAttributes(
metric,
"wildfly.request.count",
"The number of requests received.",
"{requests}",
attrs ->
attrs.containsOnly(
entry("server", "default-server"), entry("listener", "default"))),
metric ->
assertSumWithAttributes(
metric,
"wildfly.request.time",
"The total amount of time spent on requests.",
"ns",
attrs ->
attrs.containsOnly(
entry("server", "default-server"), entry("listener", "default"))),
metric ->
assertSumWithAttributes(
metric,
"wildfly.request.server_error",
"The number of requests that have resulted in a 5xx response.",
"{requests}",
attrs ->
attrs.containsOnly(
entry("server", "default-server"), entry("listener", "default"))),
metric ->
assertSumWithAttributes(
metric,
"wildfly.network.io",
"The number of bytes transmitted.",
"by",
attrs ->
attrs.containsOnly(
entry("server", "default-server"),
entry("listener", "default"),
entry("state", "in")),
attrs ->
attrs.containsOnly(
entry("server", "default-server"),
entry("listener", "default"),
entry("state", "out"))),
metric ->
assertSumWithAttributes(
metric,
"wildfly.jdbc.connection.open",
"The number of open jdbc connections.",
"{connections}",
attrs ->
attrs.containsOnly(entry("data_source", "ExampleDS"), entry("state", "active")),
attrs ->
attrs.containsOnly(entry("data_source", "ExampleDS"), entry("state", "idle"))),
metric ->
assertSumWithAttributes(
metric,
"wildfly.jdbc.request.wait",
"The number of jdbc connections that had to wait before opening.",
"{requests}",
attrs -> attrs.containsOnly(entry("data_source", "ExampleDS"))),
metric ->
assertSum(
metric,
"wildfly.jdbc.transaction.count",
"The number of transactions created.",
"{transactions}"),
metric ->
assertSumWithAttributes(
metric,
"wildfly.jdbc.rollback.count",
"The number of transactions rolled back.",
"{transactions}",
attrs -> attrs.containsOnly(entry("cause", "system")),
attrs -> attrs.containsOnly(entry("cause", "resource")),
attrs -> attrs.containsOnly(entry("cause", "application"))));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
otel.jmx.interval.milliseconds = 3000
otel.metrics.exporter = otlp
otel.jmx.service.url = service:jmx:remote+http://wildfly:9990
otel.jmx.target.system = wildfly
10 changes: 10 additions & 0 deletions jmx-metrics/src/integrationTest/resources/wildfly/start.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/bin/sh

/opt/jboss/wildfly/bin/standalone.sh -b 0.0.0.0 -bmanagement 0.0.0.0 & \

/opt/jboss/wildfly/bin/add-user.sh user password --silent & \

java -cp /app/OpenTelemetryJMXMetrics.jar:/opt/jboss/wildfly/bin/client/jboss-client.jar \
-Dotel.jmx.username=user -Dotel.jmx.password=password \
-Dotel.exporter.otlp.endpoint=$OTLP_ENDPOINT \
io.opentelemetry.contrib.jmxmetrics.JmxMetrics -config /app/target-systems/wildfly.properties
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ class JmxConfig {
"kafka-consumer",
"kafka-producer",
"solr",
"tomcat");
"tomcat",
"wildfly");

final String serviceUrl;
final String groovyScript;
Expand Down
60 changes: 60 additions & 0 deletions jmx-metrics/src/main/resources/target-systems/wildfly.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
* Copyright The OpenTelemetry Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

def beanWildflyDeployment = otel.mbeans("jboss.as.expr:deployment=*,subsystem=undertow")
otel.instrument(beanWildflyDeployment, "wildfly.session.count", "The number of sessions created.", "{sessions}",
["deployment": { mbean -> mbean.name().getKeyProperty("deployment")}],
"sessionsCreated", otel.&longCounterCallback)
otel.instrument(beanWildflyDeployment, "wildfly.session.active", "The number of currently active sessions.", "{sessions}",
["deployment": { mbean -> mbean.name().getKeyProperty("deployment")}],
"activeSessions", otel.&longUpDownCounterCallback)
otel.instrument(beanWildflyDeployment, "wildfly.session.expired", "The number of sessions that have expired.", "{sessions}",
["deployment": { mbean -> mbean.name().getKeyProperty("deployment")}],
"expiredSessions", otel.&longCounterCallback)
otel.instrument(beanWildflyDeployment, "wildfly.session.rejected", "The number of sessions that have been rejected.", "{sessions}",
["deployment": { mbean -> mbean.name().getKeyProperty("deployment")}],
"rejectedSessions", otel.&longCounterCallback)

def beanWildflyHttpListener = otel.mbeans("jboss.as:subsystem=undertow,server=*,http-listener=*")
otel.instrument(beanWildflyHttpListener, "wildfly.request.count", "The number of requests received.", "{requests}",
["server": { mbean -> mbean.name().getKeyProperty("server")}, "listener": { mbean -> mbean.name().getKeyProperty("http-listener")}],
"requestCount", otel.&longCounterCallback)
otel.instrument(beanWildflyHttpListener, "wildfly.request.time", "The total amount of time spent on requests.", "ns",
["server": { mbean -> mbean.name().getKeyProperty("server")}, "listener": { mbean -> mbean.name().getKeyProperty("http-listener")}],
"processingTime", otel.&longCounterCallback)
otel.instrument(beanWildflyHttpListener, "wildfly.request.server_error", "The number of requests that have resulted in a 5xx response.", "{requests}",
["server": { mbean -> mbean.name().getKeyProperty("server")}, "listener": { mbean -> mbean.name().getKeyProperty("http-listener")}],
"errorCount", otel.&longCounterCallback)
otel.instrument(beanWildflyHttpListener, "wildfly.network.io", "The number of bytes transmitted.", "by",
["server": { mbean -> mbean.name().getKeyProperty("server")}, "listener": { mbean -> mbean.name().getKeyProperty("http-listener")}],
["bytesSent":["state":{"out"}], "bytesReceived":["state":{"in"}]],
otel.&longCounterCallback)

def beanWildflyDataSource = otel.mbeans("jboss.as:subsystem=datasources,data-source=*,statistics=pool")
otel.instrument(beanWildflyDataSource, "wildfly.jdbc.connection.open", "The number of open jdbc connections.", "{connections}",
["data_source": { mbean -> mbean.name().getKeyProperty("data-source")}],
["ActiveCount":["state":{"active"}], "IdleCount":["state":{"idle"}]],
otel.&longUpDownCounterCallback)
otel.instrument(beanWildflyDataSource, "wildfly.jdbc.request.wait", "The number of jdbc connections that had to wait before opening.", "{requests}",
["data_source": { mbean -> mbean.name().getKeyProperty("data-source")}],
"WaitCount", otel.&longCounterCallback)

def beanWildflyTransaction = otel.mbean("jboss.as:subsystem=transactions")
otel.instrument(beanWildflyTransaction, "wildfly.jdbc.transaction.count", "The number of transactions created.", "{transactions}",
"numberOfTransactions", otel.&longCounterCallback)
otel.instrument(beanWildflyTransaction, "wildfly.jdbc.rollback.count", "The number of transactions rolled back.", "{transactions}",
["numberOfSystemRollbacks":["cause":{"system"}], "numberOfResourceRollbacks":["cause":{"resource"}], "numberOfApplicationRollbacks":["cause":{"application"}]],
otel.&longCounterCallback)
Loading

0 comments on commit 49729b3

Please sign in to comment.