Skip to content

Commit

Permalink
Show multiple alert types that are over the threshold
Browse files Browse the repository at this point in the history
  • Loading branch information
chrisronline committed Jun 23, 2021
1 parent 5ecf7e4 commit 2ccf1b6
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 9 deletions.
50 changes: 48 additions & 2 deletions x-pack/plugins/task_manager/server/lib/log_health_metrics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ describe('logHealthMetrics', () => {
expect(logMessage).toMatchObject(health);
});

it('should log as warn if drift exceeds the threshold', () => {
it('should log as warn if drift exceeds the threshold for a single alert type', () => {
const logger = loggingSystemMock.create().get();
const config = getTaskManagerConfig({
monitored_stats_health_verbose_log: {
Expand All @@ -199,6 +199,52 @@ describe('logHealthMetrics', () => {
'taskType:test': {
p99: 60000,
},
'taskType:test2': {
p99: 60000 - 1,
},
},
drift: {
p99: 60000,
},
},
},
},
});

logHealthMetrics(health, logger, config);

expect((logger as jest.Mocked<Logger>).warn.mock.calls[0][0] as string).toBe(
`Detected delay task start of 60s for task(s) \"taskType:test\" (which exceeds configured value of 60s)`
);

const secondMessage = JSON.parse(
((logger as jest.Mocked<Logger>).warn.mock.calls[1][0] as string).replace(
`Latest Monitored Stats: `,
''
)
);
expect(secondMessage).toMatchObject(health);
});

it('should log as warn if drift exceeds the threshold for multiple alert types', () => {
const logger = loggingSystemMock.create().get();
const config = getTaskManagerConfig({
monitored_stats_health_verbose_log: {
enabled: true,
warn_delayed_task_start_in_seconds: 60,
},
});
const health = getMockMonitoredHealth({
stats: {
runtime: {
value: {
drift_by_type: {
'taskType:test': {
p99: 60000,
},
'taskType:test2': {
p99: 60000,
},
},
drift: {
p99: 60000,
Expand All @@ -211,7 +257,7 @@ describe('logHealthMetrics', () => {
logHealthMetrics(health, logger, config);

expect((logger as jest.Mocked<Logger>).warn.mock.calls[0][0] as string).toBe(
`Detected delay task start of 60s for task \"taskType:test\" (which exceeds configured value of 60s)`
`Detected delay task start of 60s for task(s) \"taskType:test, taskType:test2\" (which exceeds configured value of 60s)`
);

const secondMessage = JSON.parse(
Expand Down
14 changes: 7 additions & 7 deletions x-pack/plugins/task_manager/server/lib/log_health_metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,20 +49,20 @@ export function logHealthMetrics(
if (
driftInSeconds >= config.monitored_stats_health_verbose_log.warn_delayed_task_start_in_seconds
) {
const taskType = Object.keys(monitoredHealth.stats.runtime?.value.drift_by_type ?? {}).reduce(
(accum: string, typeName) => {
const taskTypes = Object.keys(monitoredHealth.stats.runtime?.value.drift_by_type ?? {})
.reduce((accum: string[], typeName) => {
if (
monitoredHealth.stats.runtime?.value.drift_by_type[typeName].p99 ===
monitoredHealth.stats.runtime?.value.drift.p99
) {
accum = typeName;
accum.push(typeName);
}
return accum;
},
'unknown'
);
}, [])
.join(', ');

logger.warn(
`Detected delay task start of ${driftInSeconds}s for task "${taskType}" (which exceeds configured value of ${config.monitored_stats_health_verbose_log.warn_delayed_task_start_in_seconds}s)`
`Detected delay task start of ${driftInSeconds}s for task(s) "${taskTypes}" (which exceeds configured value of ${config.monitored_stats_health_verbose_log.warn_delayed_task_start_in_seconds}s)`
);
logLevel = LogLevel.Warn;
}
Expand Down

0 comments on commit 2ccf1b6

Please sign in to comment.