From 39512870360fdc7aa1e58795f7609058b1cab169 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 2 Mar 2023 14:51:55 +0100 Subject: [PATCH] Use singleton class to manage system data (#5202) * Use singleton class to manage system data * Use type instead of interface for property definitions * Ensure that there will always only be one instance of System class * Simplify system class implementation - remove getters and make all properties public instead (https://github.com/ChainSafe/lodestar/pull/5202#discussion_r1121806852) - remove custom constructor to ensure singleton (https://github.com/ChainSafe/lodestar/pull/5202#discussion_r1118639955) --- .../beacon-node/src/monitoring/clientStats.ts | 40 +- .../beacon-node/src/monitoring/properties.ts | 16 +- .../beacon-node/src/monitoring/service.ts | 4 +- packages/beacon-node/src/monitoring/system.ts | 360 ++++++------------ 4 files changed, 148 insertions(+), 272 deletions(-) diff --git a/packages/beacon-node/src/monitoring/clientStats.ts b/packages/beacon-node/src/monitoring/clientStats.ts index 430b71025374..185d03428c8c 100644 --- a/packages/beacon-node/src/monitoring/clientStats.ts +++ b/packages/beacon-node/src/monitoring/clientStats.ts @@ -1,6 +1,6 @@ import {DynamicProperty, MetricProperty, StaticProperty} from "./properties.js"; import {Client} from "./service.js"; -import * as system from "./system.js"; +import system from "./system.js"; import {ClientStats, JsonType, ProcessType} from "./types.js"; // Definition of client stats based on specification @@ -200,97 +200,97 @@ function createSystemStats(): ClientStats { ...createCommonStats(ProcessType.System), cpuCores: new DynamicProperty({ jsonKey: "cpu_cores", - provider: system.getCpuCores, + provider: () => system.cpuCores, description: "Number of CPU cores available", }), cpuThreads: new DynamicProperty({ jsonKey: "cpu_threads", - provider: system.getCpuThreads, + provider: () => system.cpuThreads, description: "Number of CPU threads available", }), cpuNodeSystemSecondsTotal: new DynamicProperty({ jsonKey: "cpu_node_system_seconds_total", - provider: system.getCpuNodeSystemSecondsTotal, + provider: () => system.cpuNodeSystemSecondsTotal, description: "CPU seconds consumed by all processes", }), cpuNodeUserSecondsTotal: new DynamicProperty({ jsonKey: "cpu_node_user_seconds_total", - provider: system.getCpuNodeUserSecondsTotal, + provider: () => system.cpuNodeUserSecondsTotal, description: "CPU seconds consumed by user processes", }), cpuNodeIOWaitSecondsTotal: new DynamicProperty({ jsonKey: "cpu_node_iowait_seconds_total", - provider: system.getCpuNodeIOWaitSecondsTotal, + provider: () => system.cpuNodeIOWaitSecondsTotal, description: "CPU seconds spent in I/O wait state", }), cpuNodeIdleSecondsTotal: new DynamicProperty({ jsonKey: "cpu_node_idle_seconds_total", - provider: system.getCpuNodeIdleSecondsTotal, + provider: () => system.cpuNodeIdleSecondsTotal, description: "CPU seconds spent in idle state", }), memoryNodeBytesTotal: new DynamicProperty({ jsonKey: "memory_node_bytes_total", - provider: system.getMemoryNodeBytesTotal, + provider: () => system.memoryNodeBytesTotal, description: "Total amount of memory in bytes available", }), memoryNodeBytesFree: new DynamicProperty({ jsonKey: "memory_node_bytes_free", - provider: system.getMemoryNodeBytesFree, + provider: () => system.memoryNodeBytesFree, description: "Amount of free memory in bytes", }), memoryNodeBytesCached: new DynamicProperty({ jsonKey: "memory_node_bytes_cached", - provider: system.getMemoryNodeBytesCached, + provider: () => system.memoryNodeBytesCached, description: "Amount of memory in bytes used by cache", }), memoryNodeBytesBuffers: new DynamicProperty({ jsonKey: "memory_node_bytes_buffers", - provider: system.getMemoryNodeBytesBuffers, + provider: () => system.memoryNodeBytesBuffers, description: "Amount of memory in bytes used by buffers", }), diskNodeBytesTotal: new DynamicProperty({ jsonKey: "disk_node_bytes_total", - provider: system.getDiskNodeBytesTotal, + provider: () => system.diskNodeBytesTotal, description: "Total amount of available disk space in bytes", }), diskNodeBytesFree: new DynamicProperty({ jsonKey: "disk_node_bytes_free", - provider: system.getDiskNodeBytesFree, + provider: () => system.diskNodeBytesFree, description: "Amount of free disk space in bytes", }), diskNodeIOSeconds: new DynamicProperty({ jsonKey: "disk_node_io_seconds", - provider: system.getDiskNodeIOSeconds, + provider: () => system.diskNodeIOSeconds, description: "Total time spent in seconds on disk I/O operations", }), diskNodeReadsTotal: new DynamicProperty({ jsonKey: "disk_node_reads_total", - provider: system.getDiskNodeReadsTotal, + provider: () => system.diskNodeReadsTotal, description: "Total number of disk read I/O operations", }), diskNodeWritesTotal: new DynamicProperty({ jsonKey: "disk_node_writes_total", - provider: system.getDiskNodeWritesTotal, + provider: () => system.diskNodeWritesTotal, description: "Total number of disk write I/O operations", }), networkNodeBytesTotalReceive: new DynamicProperty({ jsonKey: "network_node_bytes_total_receive", - provider: system.getNetworkNodeBytesTotalReceive, + provider: () => system.networkNodeBytesTotalReceive, description: "Total amount of bytes received over the network", }), networkNodeBytesTotalTransmit: new DynamicProperty({ jsonKey: "network_node_bytes_total_transmit", - provider: system.getNetworkNodeBytesTotalTransmit, + provider: () => system.networkNodeBytesTotalTransmit, description: "Total amount of bytes transmitted over the network", }), miscNodeBootTsSeconds: new DynamicProperty({ jsonKey: "misc_node_boot_ts_seconds", - provider: system.getMiscNodeBootTsSeconds, + provider: () => system.miscNodeBootTsSeconds, description: "Unix timestamp in seconds of boot time", }), miscOs: new DynamicProperty({ jsonKey: "misc_os", - provider: system.getMiscOs, + provider: () => system.miscOs, description: "Operating system, can be one of: lin, win, mac, unk for unknown", }), }; diff --git a/packages/beacon-node/src/monitoring/properties.ts b/packages/beacon-node/src/monitoring/properties.ts index 8d0f6e13f25e..0cc80b407fbe 100644 --- a/packages/beacon-node/src/monitoring/properties.ts +++ b/packages/beacon-node/src/monitoring/properties.ts @@ -1,24 +1,24 @@ import {Registry} from "prom-client"; import {JsonRecord, JsonType, MetricObject, MetricValue, MetricWithGetter, RecordValue} from "./types.js"; -interface PropertyDefinition { +type PropertyDefinition = { /** Key of value to be sent to remote service */ jsonKey: string; /** Description of the property */ description?: string; -} +}; -interface StaticPropertyDefinition extends PropertyDefinition { +type StaticPropertyDefinition = PropertyDefinition & { /** Static value */ value: T; -} +}; -interface DynamicPropertyDefinition extends PropertyDefinition { +type DynamicPropertyDefinition = PropertyDefinition & { /** Value provider function */ provider: () => T; -} +}; -interface MetricPropertyDefinition extends PropertyDefinition { +type MetricPropertyDefinition = PropertyDefinition & { /** Type of value to be sent to remote service */ jsonType: JsonType; /** Name of the metric */ @@ -37,7 +37,7 @@ interface MetricPropertyDefinition extends PropertyDefini cacheResult?: boolean; /** Default value if metric does not exist */ defaultValue: T; -} +}; /** * Interface to be implemented by client stats properties diff --git a/packages/beacon-node/src/monitoring/service.ts b/packages/beacon-node/src/monitoring/service.ts index 74bbc175c12f..510cda79d048 100644 --- a/packages/beacon-node/src/monitoring/service.ts +++ b/packages/beacon-node/src/monitoring/service.ts @@ -6,7 +6,7 @@ import {HistogramExtra} from "../metrics/utils/histogram.js"; import {defaultMonitoringOptions, MonitoringOptions} from "./options.js"; import {createClientStats} from "./clientStats.js"; import {ClientStats} from "./types.js"; -import {collectSystemData} from "./system.js"; +import system from "./system.js"; type MonitoringData = Record; @@ -147,7 +147,7 @@ export class MonitoringService { const recordPromises = []; if (this.options.collectSystemStats) { - await collectSystemData(this.logger); + await system.collectData(this.logger); } for (const [i, s] of this.clientStats.entries()) { diff --git a/packages/beacon-node/src/monitoring/system.ts b/packages/beacon-node/src/monitoring/system.ts index cefa0fb24d4d..ee85e9ae5088 100644 --- a/packages/beacon-node/src/monitoring/system.ts +++ b/packages/beacon-node/src/monitoring/system.ts @@ -5,264 +5,140 @@ import {Logger} from "@lodestar/utils"; type MiscOs = "lin" | "win" | "mac" | "unk"; -// static data only needs to be collected once -let staticDataCollected = false; -// disk I/O is not measurable in some environments -let diskIOMeasurable = true; - -let cpuCores = 0; -let cpuThreads = 0; -let cpuNodeSystemSecondsTotal = 0; -let cpuNodeUserSecondsTotal = 0; -let cpuNodeIdleSecondsTotal = 0; -let memoryNodeBytesTotal = 0; -let memoryNodeBytesFree = 0; -let memoryNodeBytesCached = 0; -let memoryNodeBytesBuffers = 0; -let diskNodeBytesTotal = 0; -let diskNodeBytesFree = 0; -let diskNodeReadsTotal = 0; -let diskNodeWritesTotal = 0; -let networkNodeBytesTotalReceive = 0; -let networkNodeBytesTotalTransmit = 0; -let miscNodeBootTsSeconds = 0; -let miscOs: MiscOs = "unk"; - /** - * Collect system data and update cached values - */ -export async function collectSystemData(logger: Logger): Promise { - const debug = (dataType: string, e: Error): void => logger.debug(`Failed to collect ${dataType} data`, {}, e); - - await Promise.all([ - collectStaticData().catch((e) => debug("static system", e)), - collectCpuData().catch((e) => debug("CPU", e)), - collectMemoryData().catch((e) => debug("memory", e)), - collectDiskData().catch((e) => debug("disk", e)), - collectNetworkData().catch((e) => debug("network", e)), - ]); - - miscNodeBootTsSeconds = getSystemBootTime(); -} + * Singleton class to collect and provide system information + */ +class System { + // static data only needs to be collected once + private staticDataCollected = false; + // disk I/O is not measurable in some environments + private diskIOMeasurable = true; + + cpuCores = 0; + cpuThreads = 0; + cpuNodeSystemSecondsTotal = 0; + cpuNodeUserSecondsTotal = 0; + // Note: CPU I/O wait is not measured by os.cpus() + cpuNodeIOWaitSecondsTotal = 0; + cpuNodeIdleSecondsTotal = 0; + memoryNodeBytesTotal = 0; + memoryNodeBytesFree = 0; + memoryNodeBytesCached = 0; + memoryNodeBytesBuffers = 0; + diskNodeBytesTotal = 0; + diskNodeBytesFree = 0; + // Note: disk I/O seconds is currently unused by beaconcha.in + diskNodeIOSeconds = 0; + diskNodeReadsTotal = 0; + diskNodeWritesTotal = 0; + networkNodeBytesTotalReceive = 0; + networkNodeBytesTotalTransmit = 0; + miscNodeBootTsSeconds = 0; + miscOs: MiscOs = "unk"; + + /** + * Collect system data and update cached values + */ + async collectData(logger: Logger): Promise { + const debug = (dataType: string, e: Error): void => logger.debug(`Failed to collect ${dataType} data`, {}, e); + + await Promise.all([ + this.collectStaticData().catch((e) => debug("static system", e)), + this.collectCpuData().catch((e) => debug("CPU", e)), + this.collectMemoryData().catch((e) => debug("memory", e)), + this.collectDiskData().catch((e) => debug("disk", e)), + this.collectNetworkData().catch((e) => debug("network", e)), + ]); + + this.miscNodeBootTsSeconds = this.getSystemBootTime(); + } -async function collectStaticData(): Promise { - if (staticDataCollected) return; + private async collectStaticData(): Promise { + if (this.staticDataCollected) return; - const cpu = await system.cpu(); - // Note: inside container this might be inaccurate as - // physicalCores in some cases is the count of logical CPU cores - cpuCores = cpu.physicalCores; - cpuThreads = cpu.cores; + const cpu = await system.cpu(); + // Note: inside container this might be inaccurate as + // physicalCores in some cases is the count of logical CPU cores + this.cpuCores = cpu.physicalCores; + this.cpuThreads = cpu.cores; - miscOs = getNormalizedOsVersion(); + this.miscOs = this.getNormalizedOsVersion(); - staticDataCollected = true; -} + this.staticDataCollected = true; + } -async function collectCpuData(): Promise { - const cpuTimes: Record = {}; + private async collectCpuData(): Promise { + const cpuTimes: Record = {}; - os.cpus().forEach((cpu) => { - // sum up CPU times per mode and convert to seconds - for (const [mode, time] of Object.entries(cpu.times)) { - if (cpuTimes[mode] == null) cpuTimes[mode] = 0; - cpuTimes[mode] += Math.floor(time / 1000); + for (const cpu of os.cpus()) { + // sum up CPU times per mode and convert to seconds + for (const [mode, time] of Object.entries(cpu.times)) { + if (cpuTimes[mode] == null) cpuTimes[mode] = 0; + cpuTimes[mode] += Math.floor(time / 1000); + } } - }); - // Note: currently beaconcha.in expects system CPU seconds to be everything - cpuNodeSystemSecondsTotal = Object.values(cpuTimes).reduce((total, time) => total + time, 0); - cpuNodeUserSecondsTotal = cpuTimes.user; - cpuNodeIdleSecondsTotal = cpuTimes.idle; -} - -async function collectMemoryData(): Promise { - const memory = await system.mem(); - memoryNodeBytesTotal = memory.total; - memoryNodeBytesFree = memory.free; - memoryNodeBytesCached = memory.cached; - memoryNodeBytesBuffers = memory.buffers; -} + // Note: currently beaconcha.in expects system CPU seconds to be everything + this.cpuNodeSystemSecondsTotal = Object.values(cpuTimes).reduce((total, time) => total + time, 0); + this.cpuNodeUserSecondsTotal = cpuTimes.user; + this.cpuNodeIdleSecondsTotal = cpuTimes.idle; + } -async function collectDiskData(): Promise { - const fileSystems = await system.fsSize(); - // get file system root, on windows this is the name of the hard disk partition - const rootFs = process.platform === "win32" ? process.cwd().split(path.sep)[0] : "/"; - // only consider root file system, if it does not exist use first entry in the list - const fileSystem = fileSystems.find((fs) => fs.mount === rootFs) ?? fileSystems[0]; - diskNodeBytesTotal = fileSystem.size; - diskNodeBytesFree = fileSystem.available; + private async collectMemoryData(): Promise { + const memory = await system.mem(); + this.memoryNodeBytesTotal = memory.total; + this.memoryNodeBytesFree = memory.free; + this.memoryNodeBytesCached = memory.cached; + this.memoryNodeBytesBuffers = memory.buffers; + } - if (diskIOMeasurable) { - const disk = await system.disksIO(); - if (disk != null && disk.rIO !== 0) { - // Note: rIO and wIO might not be available inside container - // see https://github.com/sebhildebrandt/systeminformation/issues/777 - diskNodeReadsTotal = disk.rIO; - diskNodeWritesTotal = disk.wIO; - } else { - diskIOMeasurable = false; + private async collectDiskData(): Promise { + const fileSystems = await system.fsSize(); + // get file system root, on windows this is the name of the hard disk partition + const rootFs = process.platform === "win32" ? process.cwd().split(path.sep)[0] : "/"; + // only consider root file system, if it does not exist use first entry in the list + const fileSystem = fileSystems.find((fs) => fs.mount === rootFs) ?? fileSystems[0]; + this.diskNodeBytesTotal = fileSystem.size; + this.diskNodeBytesFree = fileSystem.available; + + if (this.diskIOMeasurable) { + const disk = await system.disksIO(); + if (disk != null && disk.rIO !== 0) { + // Note: rIO and wIO might not be available inside container + // see https://github.com/sebhildebrandt/systeminformation/issues/777 + this.diskNodeReadsTotal = disk.rIO; + this.diskNodeWritesTotal = disk.wIO; + } else { + this.diskIOMeasurable = false; + } } } -} -async function collectNetworkData(): Promise { - // defaults to first external network interface - const [network] = await system.networkStats(); - // Note: rx_bytes and tx_bytes will be inaccurate if process - // runs inside container as it only captures local network traffic - networkNodeBytesTotalReceive = network.rx_bytes; - networkNodeBytesTotalTransmit = network.tx_bytes; -} - -function getNormalizedOsVersion(): MiscOs { - switch (process.platform) { - case "linux": - return "lin"; - case "darwin": - return "mac"; - case "win32": - return "win"; - default: - return "unk"; + private async collectNetworkData(): Promise { + // defaults to first external network interface + const [network] = await system.networkStats(); + // Note: rx_bytes and tx_bytes will be inaccurate if process + // runs inside container as it only captures local network traffic + this.networkNodeBytesTotalReceive = network.rx_bytes; + this.networkNodeBytesTotalTransmit = network.tx_bytes; } -} - -function getSystemBootTime(): number { - return Math.floor(Date.now() / 1000 - os.uptime()); -} - -/** - * Number of CPU cores available - */ -export function getCpuCores(): number { - return cpuCores; -} - -/** - * Number of CPU threads available - */ -export function getCpuThreads(): number { - return cpuThreads; -} - -/** - * CPU seconds consumed by all processes - */ -export function getCpuNodeSystemSecondsTotal(): number { - return cpuNodeSystemSecondsTotal; -} - -/** - * CPU seconds consumed by user processes - */ -export function getCpuNodeUserSecondsTotal(): number { - return cpuNodeUserSecondsTotal; -} - -/** - * CPU seconds spent in I/O wait state - */ -export function getCpuNodeIOWaitSecondsTotal(): number { - // Note: not measured by os.cpus() - return 0; -} - -/** - * CPU seconds spent in idle state - */ -export function getCpuNodeIdleSecondsTotal(): number { - return cpuNodeIdleSecondsTotal; -} - -/** - * Total amount of memory in bytes available - */ -export function getMemoryNodeBytesTotal(): number { - return memoryNodeBytesTotal; -} -/** - * Amount of free memory in bytes - */ -export function getMemoryNodeBytesFree(): number { - return memoryNodeBytesFree; -} - -/** - * Amount of memory in bytes used by cache - */ -export function getMemoryNodeBytesCached(): number { - return memoryNodeBytesCached; -} - -/** - * Amount of memory in bytes used by buffers - */ -export function getMemoryNodeBytesBuffers(): number { - return memoryNodeBytesBuffers; -} - -/** - * Total amount of available disk space in bytes - */ -export function getDiskNodeBytesTotal(): number { - return diskNodeBytesTotal; -} - -/** - * Amount of free disk space in bytes - */ -export function getDiskNodeBytesFree(): number { - return diskNodeBytesFree; -} - -/** - * Total time spent in seconds on disk I/O operations - */ -export function getDiskNodeIOSeconds(): number { - // Note: currently unused by beaconcha.in - return 0; -} - -/** - * Total number of disk read I/O operations - */ -export function getDiskNodeReadsTotal(): number { - return diskNodeReadsTotal; -} - -/** - * Total number of disk write I/O operations - */ -export function getDiskNodeWritesTotal(): number { - return diskNodeWritesTotal; -} - -/** - * Total amount of bytes received over the network - */ -export function getNetworkNodeBytesTotalReceive(): number { - return networkNodeBytesTotalReceive; -} - -/** - * Total amount of bytes transmitted over the network - */ -export function getNetworkNodeBytesTotalTransmit(): number { - return networkNodeBytesTotalTransmit; -} + private getNormalizedOsVersion(): MiscOs { + switch (process.platform) { + case "linux": + return "lin"; + case "darwin": + return "mac"; + case "win32": + return "win"; + default: + return "unk"; + } + } -/** - * Unix timestamp in seconds of boot time - */ -export function getMiscNodeBootTsSeconds(): number { - return miscNodeBootTsSeconds; + private getSystemBootTime(): number { + return Math.floor(Date.now() / 1000 - os.uptime()); + } } -/** - * Operating system, can be one of: lin, win, mac, unk for unknown - */ -export function getMiscOs(): MiscOs { - return miscOs; -} +export default new System();