tests(smoke): check runner result is sensible before using

GoogleChrome · Sep 1, 2022 · 7decdca · 7decdca
1 parent bb2bed6
commit 7decdca
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/cli/test/smokehouse/report-assert.js b/cli/test/smokehouse/report-assert.js
@@ -259,10 +259,6 @@ function pruneExpectations(localConsole, lhr, expected, reportOptions) {
    * @param {*} obj
    */
   function failsChromeVersionCheck(obj) {
-    // LHR never actually run, so we can't know. Better to not prune
-    // things than to fail loudly when accessing lhr.environment.
-    if (!lhr.environment) return false;
-
     return !chromiumVersionCheck({
       version: getChromeVersionString(),
       min: obj._minChromiumVersion,

diff --git a/cli/test/smokehouse/smokehouse.js b/cli/test/smokehouse/smokehouse.js
@@ -195,6 +195,13 @@ async function runSmokeTest(smokeTestDefn, testOptions) {
         ...await lighthouseRunner(requestedUrl, configJson, {isDebug, useLegacyNavigation}),
         networkRequests: takeNetworkRequestUrls ? takeNetworkRequestUrls() : undefined,
       };
+
+      if (!result || !result.lhr?.audits || !result.artifacts) {
+        // Don't try again–something went really wrong! ex: the config could not be resolved
+        i = retries + 1;
+        throw new Error('lighthouse runner returned a bad result. got lhr:\n' +
+          JSON.stringify(result.lhr, null, 2));
+      }
     } catch (e) {
       // Clear the network requests so that when we retry, we don't see duplicates.
       if (takeNetworkRequestUrls) takeNetworkRequestUrls();