Skip to content

Commit

Permalink
Merge pull request #534 from jenkinsci/refactoring
Browse files Browse the repository at this point in the history 
Refactoring code to make it more testable
  • Loading branch information
Waschndolos committed Jun 21, 2023
2 parents 0075069 + 8b6edb8 commit 3586406
Show file tree
Hide file tree
Showing 89 changed files with 3,166 additions and 665 deletions.
140 changes: 58 additions & 82 deletions src/main/java/org/jenkinsci/plugins/prometheus/DiskUsageCollector.java
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,19 @@
package org.jenkinsci.plugins.prometheus;

import com.cloudbees.simplediskusage.DiskItem;
import com.cloudbees.simplediskusage.JobDiskItem;
import hudson.model.LoadStatistics;
import io.prometheus.client.Collector;
import io.prometheus.client.Gauge;
import jenkins.model.Jenkins;
import org.jenkinsci.plugins.prometheus.collectors.CollectorFactory;
import org.jenkinsci.plugins.prometheus.collectors.CollectorType;
import org.jenkinsci.plugins.prometheus.collectors.MetricCollector;
import org.jenkinsci.plugins.prometheus.collectors.disk.DiskUsageBytesGauge;
import org.jenkinsci.plugins.prometheus.collectors.disk.FileStoreAvailableGauge;
import org.jenkinsci.plugins.prometheus.collectors.disk.FileStoreCapacityGauge;
import org.jenkinsci.plugins.prometheus.collectors.disk.JobUsageBytesGauge;
import org.jenkinsci.plugins.prometheus.config.PrometheusConfiguration;
import org.jenkinsci.plugins.prometheus.util.ConfigurationUtils;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -14,122 +22,90 @@
import java.io.IOException;
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class DiskUsageCollector extends Collector {

private static final Logger logger = LoggerFactory.getLogger(DiskUsageCollector.class);

private static Gauge newDirectoryUsageGauge() {
return newGaugeBuilder()
.name("disk_usage_bytes")
.labelNames("file_store", "directory")
.help("Disk usage of first level folder in JENKINS_HOME in bytes")
.create();
}

private static Gauge newJobUsageGauge() {
return newGaugeBuilder()
.name("job_usage_bytes")
.labelNames("file_store", "jobName", "url")
.help("Amount of disk usage (bytes) for each job in Jenkins")
.create();
}

private static Gauge newFileStoreCapacityGauge() {
return newGaugeBuilder()
.name("file_store_capacity_bytes")
.labelNames("file_store")
.help("Total size in bytes of the file stores used by Jenkins")
.create();
}

private static Gauge newFileStoreAvailableGauge() {
return newGaugeBuilder()
.name("file_store_available_bytes")
.labelNames("file_store")
.help("Estimated available space on the file stores used by Jenkins")
.create();
}

private static Gauge.Builder newGaugeBuilder() {
return Gauge.build()
.namespace(ConfigurationUtils.getNamespace())
.subsystem(ConfigurationUtils.getSubSystem());
}
private static final Logger LOGGER = LoggerFactory.getLogger(DiskUsageCollector.class);

@Override
@Nonnull
public List<MetricFamilySamples> collect() {

if (!PrometheusConfiguration.get().getCollectDiskUsage()) {
return Collections.emptyList();
}

try {
return collectDiskUsage();
} catch (final IOException | RuntimeException e) {
logger.warn("Failed to get disk usage data due to an unexpected error.", e);
LOGGER.warn("Failed to get disk usage data due to an unexpected error.", e);

Check warning on line 44 in src/main/java/org/jenkinsci/plugins/prometheus/DiskUsageCollector.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered line 

Line 44 is not covered by tests
return Collections.emptyList();
} catch (final java.lang.NoClassDefFoundError e) {
logger.warn("Cannot collect disk usage data because plugin CloudBees Disk Usage Simple is not installed: {}", e.toString());
LOGGER.warn("Cannot collect disk usage data because plugin CloudBees Disk Usage Simple is not installed: {}", e.toString());

Check warning on line 47 in src/main/java/org/jenkinsci/plugins/prometheus/DiskUsageCollector.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered line 

Line 47 is not covered by tests
return Collections.emptyList();
}
}

@Nonnull
private static List<MetricFamilySamples> collectDiskUsage() throws IOException {
final com.cloudbees.simplediskusage.QuickDiskUsagePlugin diskUsagePlugin = Jenkins.get()
.getPlugin(com.cloudbees.simplediskusage.QuickDiskUsagePlugin.class);
.getPlugin(com.cloudbees.simplediskusage.QuickDiskUsagePlugin.class);
if (diskUsagePlugin == null) {
logger.warn("Cannot collect disk usage data because plugin CloudBees Disk Usage Simple is not loaded.");
LOGGER.warn("Cannot collect disk usage data because plugin CloudBees Disk Usage Simple is not loaded.");

Check warning on line 57 in src/main/java/org/jenkinsci/plugins/prometheus/DiskUsageCollector.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered line 

Line 57 is not covered by tests
return Collections.emptyList();
}

final List<MetricFamilySamples> samples = new ArrayList<>();
CollectorFactory factory = new CollectorFactory();
final Set<FileStore> usedFileStores = new HashSet<>();
List<MetricCollector<DiskItem, ? extends Collector>> diskItemCollectors = new ArrayList<>();
diskItemCollectors.add(factory.createDiskItemCollector(CollectorType.DISK_USAGE_BYTES_GAUGE, new String[]{"file_store", "directory"}));

final Gauge directoryUsageGauge = newDirectoryUsageGauge();
diskUsagePlugin.getDirectoriesUsages().forEach(i -> {
final Optional<FileStore> fileStore = getFileStore(i.getPath());
fileStore.ifPresent(usedFileStores::add);
directoryUsageGauge.labels(toLabelValue(fileStore), i.getDisplayName()).set(i.getUsage() * 1024);
final Optional<FileStore> fileStore = getFileStore(i.getPath());
fileStore.ifPresent(usedFileStores::add);
diskItemCollectors.forEach(c -> c.calculateMetric(i, new String[]{toLabelValue(fileStore), i.getDisplayName()}));
});
samples.addAll(directoryUsageGauge.collect());

final Gauge jobUsageGauge = newJobUsageGauge();
List<MetricCollector<JobDiskItem, ? extends Collector>> jobDiskItemCollectors = new ArrayList<>();
jobDiskItemCollectors.add(factory.createJobDiskItemCollector(CollectorType.JOB_USAGE_BYTES_GAUGE, new String[]{"file_store", "jobName", "url"}));

diskUsagePlugin.getJobsUsages().forEach(i -> {
final Optional<FileStore> fileStore = getFileStore(i.getPath());
fileStore.ifPresent(usedFileStores::add);
jobUsageGauge.labels(toLabelValue(fileStore), i.getFullName(), i.getUrl()).set(i.getUsage() * 1024);
final Optional<FileStore> fileStore = getFileStore(i.getPath());
fileStore.ifPresent(usedFileStores::add);
jobDiskItemCollectors.forEach(c -> c.calculateMetric(i, new String[]{toLabelValue(fileStore), i.getFullName(), i.getUrl()}));
});
samples.addAll(jobUsageGauge.collect());

final Gauge fileStoreCapacityGauge = newFileStoreCapacityGauge();
final Gauge fileStoreAvailableGauge = newFileStoreAvailableGauge();
List<MetricCollector<FileStore, ? extends Collector>> fileStoreCollectors = new ArrayList<>();
fileStoreCollectors.add(factory.createFileStoreCollector(CollectorType.FILE_STORE_CAPACITY_GAUGE,new String[]{"file_store"} ));
fileStoreCollectors.add(factory.createFileStoreCollector(CollectorType.FILE_STORE_AVAILABLE_GAUGE ,new String[]{"file_store"} ));

usedFileStores.forEach(store -> {
final String labelValue = toLabelValue(Optional.of(store));

try {
fileStoreCapacityGauge.labels(labelValue).set(store.getTotalSpace());
} catch (final IOException | RuntimeException e) {
logger.debug("Failed to get total space of {}", store, e);
fileStoreCapacityGauge.labels(labelValue).set(Double.NaN);
}

try {
fileStoreAvailableGauge.labels(labelValue).set(store.getUsableSpace());
} catch (final IOException | RuntimeException e) {
logger.debug("Failed to get usable space of {}", store, e);
fileStoreAvailableGauge.labels(labelValue).set(Double.NaN);
}
final String labelValue = toLabelValue(Optional.of(store));
fileStoreCollectors.forEach(c -> c.calculateMetric(store, new String[]{labelValue}));
});
samples.addAll(fileStoreCapacityGauge.collect());
samples.addAll(fileStoreAvailableGauge.collect());

List<MetricFamilySamples> samples = new ArrayList<>();

samples.addAll(Stream.of(diskItemCollectors)
.flatMap(Collection::stream)
.map(MetricCollector::collect)
.flatMap(Collection::stream)
.collect(Collectors.toList()));

samples.addAll(Stream.of(jobDiskItemCollectors)
.flatMap(Collection::stream)
.map(MetricCollector::collect)
.flatMap(Collection::stream)
.collect(Collectors.toList()));

samples.addAll(Stream.of(fileStoreCollectors)
.flatMap(Collection::stream)
.map(MetricCollector::collect)
.flatMap(Collection::stream)
.collect(Collectors.toList()));

return samples;
}
Expand All @@ -145,7 +121,7 @@ private static Optional<FileStore> getFileStore(File file) {
try {
return Optional.of(Files.getFileStore(file.toPath().toRealPath()));
} catch (IOException | RuntimeException e) {
logger.debug("Failed to get file store for {}", file, e);
LOGGER.debug("Failed to get file store for {}", file, e);

Check warning on line 124 in src/main/java/org/jenkinsci/plugins/prometheus/DiskUsageCollector.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered line 

Line 124 is not covered by tests
return Optional.empty();
}
}
Expand Down
118 changes: 23 additions & 95 deletions src/main/java/org/jenkinsci/plugins/prometheus/ExecutorCollector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,123 +3,51 @@
import hudson.model.Label;
import hudson.model.LoadStatistics;
import io.prometheus.client.Collector;
import io.prometheus.client.Gauge;
import jenkins.model.Jenkins;
import org.jenkinsci.plugins.prometheus.util.ConfigurationUtils;
import org.jenkinsci.plugins.prometheus.collectors.CollectorFactory;
import org.jenkinsci.plugins.prometheus.collectors.CollectorType;
import org.jenkinsci.plugins.prometheus.collectors.MetricCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;

public class ExecutorCollector extends Collector {

private static final Logger logger = LoggerFactory.getLogger(ExecutorCollector.class);

private Gauge executorsAvailable;
private Gauge executorsBusy;
private Gauge executorsConnecting;
private Gauge executorsDefined;
private Gauge executorsIdle;
private Gauge executorsOnline;
private Gauge queueLength;


public ExecutorCollector() {
}

private static final Logger LOGGER = LoggerFactory.getLogger(ExecutorCollector.class);
@Override
public List<MetricFamilySamples> collect() {
logger.debug("Collecting executor metrics for prometheus");
LOGGER.debug("Collecting executor metrics for prometheus");

CollectorFactory factory = new CollectorFactory();

String namespace = ConfigurationUtils.getNamespace();
List<MetricFamilySamples> samples = new ArrayList<>();
String prefix = "executors";
String subsystem = ConfigurationUtils.getSubSystem();
String[] labelNameArray = {"label"};

// Number of executors (among the online executors) that are available to carry out builds.
executorsAvailable = Gauge.build()
.name(prefix + "_available")
.subsystem(subsystem).namespace(namespace)
.labelNames(labelNameArray)
.help("Executors Available")
.create();

// Number of executors (among the online executors) that are carrying out builds.
executorsBusy = Gauge.build()
.name(prefix + "_busy")
.subsystem(subsystem).namespace(namespace)
.labelNames(labelNameArray)
.help("Executors Busy")
.create();
List<MetricCollector<LoadStatistics.LoadStatisticsSnapshot, ? extends Collector>> collectors = new ArrayList<>();

// Number of executors that are currently in the process of connecting to Jenkins.
executorsConnecting = Gauge.build()
.name(prefix + "_connecting")
.subsystem(subsystem).namespace(namespace)
.labelNames(labelNameArray)
.help("Executors Connecting")
.create();
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_AVAILABLE_GAUGE, labelNameArray, prefix));
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_BUSY_GAUGE, labelNameArray, prefix));
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_CONNECTING_GAUGE, labelNameArray, prefix));
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_DEFINED_GAUGE, labelNameArray, prefix));
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_IDLE_GAUGE, labelNameArray, prefix));
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_ONLINE_GAUGE, labelNameArray, prefix));
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_QUEUE_LENGTH_GAUGE, labelNameArray, prefix));

// Number of executors that Jenkins currently knows, this includes all off-line agents.
executorsDefined = Gauge.build()
.name(prefix + "_defined")
.subsystem(subsystem).namespace(namespace)
.labelNames(labelNameArray)
.help("Executors Defined")
.create();

// Number of executors that are currently on-line and idle.
executorsIdle = Gauge.build()
.name(prefix + "_idle")
.subsystem(subsystem).namespace(namespace)
.labelNames(labelNameArray)
.help("Executors Idle")
.create();

// Sum of all executors across all online computers in this label.
executorsOnline = Gauge.build()
.name(prefix + "_online")
.subsystem(subsystem).namespace(namespace)
.labelNames(labelNameArray)
.help("Executors Online")
.create();

// Number of jobs that are in the build queue, waiting for an available executor of this label.
queueLength = Gauge.build()
.name(prefix + "_queue_length")
.subsystem(subsystem).namespace(namespace)
.labelNames(labelNameArray)
.help("Executors Queue Length")
.create();

logger.debug("getting load statistics for Executors");
LOGGER.debug("getting load statistics for Executors");

Label[] labels = Jenkins.get().getLabels().toArray(new Label[0]);
for (Label l : labels) {
appendExecutorMetrics(l.getDisplayName(), l.loadStatistics.computeSnapshot());
collectors.forEach(c -> c.calculateMetric(l.loadStatistics.computeSnapshot(), new String[] {l.getDisplayName()}));
}

samples.addAll(executorsAvailable.collect());
samples.addAll(executorsBusy.collect());
samples.addAll(executorsConnecting.collect());
samples.addAll(executorsDefined.collect());
samples.addAll(executorsIdle.collect());
samples.addAll(executorsOnline.collect());
samples.addAll(queueLength.collect());

return samples;
return collectors.stream()
.map(MetricCollector::collect)
.flatMap(Collection::stream)
.collect(Collectors.toList());
}

protected void appendExecutorMetrics(String labelDisplayName, LoadStatistics.LoadStatisticsSnapshot computeSnapshot) {
String[] labelValueArray = {labelDisplayName};
executorsAvailable.labels(labelValueArray).set(computeSnapshot.getAvailableExecutors());
executorsBusy.labels(labelValueArray).set(computeSnapshot.getBusyExecutors());
executorsConnecting.labels(labelValueArray).set(computeSnapshot.getConnectingExecutors());
executorsDefined.labels(labelValueArray).set(computeSnapshot.getDefinedExecutors());
executorsIdle.labels(labelValueArray).set(computeSnapshot.getIdleExecutors());
executorsOnline.labels(labelValueArray).set(computeSnapshot.getOnlineExecutors());
queueLength.labels(labelValueArray).set(computeSnapshot.getQueueLength());
}
}
Loading

0 comments on commit 3586406

Please sign in to comment.