From 0576431612c668b178847591e75f6d100c748af3 Mon Sep 17 00:00:00 2001 From: Avishag Israeli Date: Tue, 9 Feb 2021 11:42:32 +0200 Subject: [PATCH] test: add pagerduty, check if test is running before attemmpting rerun --- test/cli-alert/package-lock.json | 28 ++++++++++++ test/cli-alert/package.json | 1 + test/cli-alert/src/index.ts | 75 ++++++++++++++++++++++---------- 3 files changed, 81 insertions(+), 23 deletions(-) diff --git a/test/cli-alert/package-lock.json b/test/cli-alert/package-lock.json index 9a7c6819f8..72ebc51d1e 100644 --- a/test/cli-alert/package-lock.json +++ b/test/cli-alert/package-lock.json @@ -111,6 +111,16 @@ "@types/node": ">= 8" } }, + "@pagerduty/pdjs": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@pagerduty/pdjs/-/pdjs-2.2.0.tgz", + "integrity": "sha512-VH4MkjMeKjj0RmZXoz5HFDxdBflwhhdOww92U1XGLfUpMLZ7SRcJB0hGM7NnWkQLY8aFOSJCfcxMrOTHpBTJaA==", + "requires": { + "abortcontroller-polyfill": "^1.5.0", + "browser-or-node": "^1.3.0", + "cross-fetch": "^3.0.6" + } + }, "@slack/types": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/@slack/types/-/types-1.10.0.tgz", @@ -131,6 +141,11 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.2.tgz", "integrity": "sha512-jeYJU2kl7hL9U5xuI/BhKPZ4vqGM/OmK6whiFAXVhlstzZhVamWhDSmHyGLIp+RVyuF9/d0dqr2P85aFj4BvJg==" }, + "abortcontroller-polyfill": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.1.tgz", + "integrity": "sha512-yml9NiDEH4M4p0G4AcPkg8AAa4mF3nfYF28VQxaokpO67j9H7gWgmsVWJ/f1Rn+PzsnDYvzJzWIQzCqDKRvWlA==" + }, "axios": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", @@ -144,6 +159,19 @@ "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.1.0.tgz", "integrity": "sha512-IWIbu7pMqyw3EAJHzzHbWa85b6oud/yfKYg5rqB5hNE8CeMi3nX+2C2sj0HswfblST86hpVEOAb9x34NZd6P7A==" }, + "browser-or-node": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-1.3.0.tgz", + "integrity": "sha512-0F2z/VSnLbmEeBcUrSuDH5l0HxTXdQQzLjkmBR4cYfvg1zJrKSlmIZFqyFR8oX0NrwPhy3c3HQ6i3OxMbew4Tg==" + }, + "cross-fetch": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.0.6.tgz", + "integrity": "sha512-KBPUbqgFjzWlVcURG+Svp9TlhA5uliYtiNx/0r8nv0pdypeQCRJ9IaSIc3q/x3q8t3F75cHuwxVql1HFGHCNJQ==", + "requires": { + "node-fetch": "2.6.1" + } + }, "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", diff --git a/test/cli-alert/package.json b/test/cli-alert/package.json index 7c5c4d3f5e..523015ad1d 100644 --- a/test/cli-alert/package.json +++ b/test/cli-alert/package.json @@ -14,6 +14,7 @@ "license": "Apache-2.0", "dependencies": { "@octokit/rest": "^18.0.5", + "@pagerduty/pdjs": "^2.2.0", "@slack/webhook": "^5.0.3", "typescript": "^4.0.2" }, diff --git a/test/cli-alert/src/index.ts b/test/cli-alert/src/index.ts index b6186cb4fe..d9493ee7ff 100644 --- a/test/cli-alert/src/index.ts +++ b/test/cli-alert/src/index.ts @@ -1,14 +1,22 @@ import { Octokit } from '@octokit/rest'; import { IncomingWebhook } from '@slack/webhook'; import { IncomingWebhookDefaultArguments } from '@slack/webhook'; - -if (!process.env.USER_GITHUB_TOKEN || !process.env.SLACK_WEBHOOK_URL) { - console.error('Missing USER_GITHUB_TOKEN or SLACK_WEBHOOK_URL'); +import { event } from '@pagerduty/pdjs'; + +if ( + !process.env.USER_GITHUB_TOKEN || + !process.env.SLACK_WEBHOOK_URL || + !process.env.PD_ROUTING_KEY +) { + console.error( + 'Missing USER_GITHUB_TOKEN, SLACK_WEBHOOK_URL or PD_ROUTING_KEY', + ); process.exit(1); } const GITHUB_TOKEN = process.env.USER_GITHUB_TOKEN; const SLACK_WEBHOOK_URL = process.env.SLACK_WEBHOOK_URL; +const PD_ROUTING_KEY = process.env.PD_ROUTING_KEY; const octokit = new Octokit({ auth: GITHUB_TOKEN, @@ -54,7 +62,7 @@ async function discoverConsecutiveFailures( process.exit(1); } else if ( 'failure' === firstJob.conclusion && - firstJob?.conclusion === secondJob.conclusion + firstJob.conclusion === secondJob.conclusion ) { console.log(`Found a job that failed 2 times in a row: ${jobName}`); failedJobs.push(jobName); @@ -64,6 +72,24 @@ async function discoverConsecutiveFailures( return failedJobs; } +async function sendPagerDuty() { + await event({ + data: { + // eslint-disable-next-line @typescript-eslint/camelcase + routing_key: PD_ROUTING_KEY, + // eslint-disable-next-line @typescript-eslint/camelcase + event_action: 'trigger', + payload: { + summary: 'CLI Alert. Smoke tests failing', + source: 'Snyk CLI Smoke tests', + severity: 'warning', + }, + }, + }) + .then(console.log) + .catch(console.error); +} + async function sendSlackAlert(failedJobs: string[]) { console.log('Jobs failed again. Sending Slack alert...'); const args: IncomingWebhookDefaultArguments = { @@ -170,21 +196,6 @@ async function run() { // Check the latest 2 smoke tests for tests that had the same job fail 2 times in a row. const latestWorkflowRuns = workflowRuns.workflow_runs.slice(0, 2); - - console.log('Checking smoke tests jobs...'); - const failedWorkflows = await discoverConsecutiveFailures( - latestWorkflowRuns[0].id, - latestWorkflowRuns[1].id, - ); - - if (!failedWorkflows.length || failedWorkflows.length < 1) { - console.log( - 'There were no 2 consecutive fails on a job. No need to alert.', - ); - return; - } - - console.log('Trying to re-run smoke test...'); const id = latestWorkflowRuns[0].id; // Check current status of smoke test workflow and wait if it's still running @@ -202,6 +213,21 @@ async function run() { await waitForConclusion(id); } + console.log('Checking smoke tests jobs...'); + const failedWorkflows = await discoverConsecutiveFailures( + latestWorkflowRuns[0].id, + latestWorkflowRuns[1].id, + ); + + if (!failedWorkflows.length || failedWorkflows.length < 1) { + console.log( + 'There were no 2 consecutive fails on a job. No need to alert.', + ); + return; + } + + console.log('Trying to re-run smoke test...'); + // After making sure smoke test isn't currently running - try to re-run console.log(`Starting re-run of Smoke Test. ID number: ${id}...`); await octokit.actions.reRunWorkflow({ @@ -216,10 +242,13 @@ async function run() { const failedAgainJobs = await checkJobConclusion(id, failedWorkflows); console.log('Re-run completed.'); - // If run failed again, send Slack alert - failedAgainJobs.length > 0 - ? await sendSlackAlert(failedAgainJobs) - : console.log('Jobs succeeded after re-run. Do not alert.'); + // If run failed again, send Slack alert and PagerDuty + if (failedAgainJobs.length > 0) { + await sendSlackAlert(failedAgainJobs); + await sendPagerDuty(); + } else { + console.log('Jobs succeeded after re-run. Do not alert.'); + } } catch (error) { console.error(error); process.exit(1);