test: ignore windowHeight in partialRun tests (#4571)

This should fix the issue with the tests failing on`windowHeight` being different between two consecutive runs. I also added a deep axe-core results compare function that can do two significant things for us when comparing result objects. First it can ignore keys (like `timestamp`) so we don't have to always set it equal to each other in the tests. Second is that instead of dumping the entire result objects when a value is different this will output they key path and the values that are different which make debugging a lot easier. ``` Expected result.testEnvironment.windowHeight to equal "948" but got "896" ```
dequelabs · Sep 3, 2024 · bdd80c2 · bdd80c2
1 parent 1a03dc6
commit bdd80c2
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 15 deletions.
diff --git a/test/integration/full/run-partial/after-method.js b/test/integration/full/run-partial/after-method.js
@@ -22,9 +22,9 @@ describe('run-partial, after-method', function () {
         var axeRunPartialResult = results[0];
         var axeRunResult = results[1];
         assert.lengthOf(axeRunPartialResult.violations, 0);
-        axeRunPartialResult.testEnvironment = axeRunResult.testEnvironment;
-        axeRunPartialResult.timestamp = axeRunResult.timestamp;
-        assert.deepEqual(axeRunPartialResult, axeRunResult);
+        axe.testUtils.resultsDeepEqual(axeRunPartialResult, axeRunResult, [
+          'testEnvironment'
+        ]);
         done();
       })
       .catch(done);
@@ -51,8 +51,9 @@ describe('run-partial, after-method', function () {
         assert.lengthOf(nodes, 1);
         assert.deepEqual(nodes[0].target, ['#frame1', 'h3']);
 
-        axeRunPartialResult.timestamp = axeRunResult.timestamp;
-        assert.deepEqual(axeRunPartialResult, axeRunResult);
+        axe.testUtils.resultsDeepEqual(axeRunPartialResult, axeRunResult, [
+          'testEnvironment'
+        ]);
         done();
       })
       .catch(done);

diff --git a/test/integration/full/run-partial/context-size-focusable.js b/test/integration/full/run-partial/context-size-focusable.js
@@ -25,9 +25,9 @@ describe('run-partial, context-size-focusable', function () {
         var axeRunPartialResult = results[0];
         var axeRunResult = results[1];
         assert.lengthOf(axeRunPartialResult.violations, 0);
-        axeRunPartialResult.testEnvironment = axeRunResult.testEnvironment;
-        axeRunPartialResult.timestamp = axeRunResult.timestamp;
-        assert.deepEqual(axeRunPartialResult, axeRunResult);
+        axe.testUtils.resultsDeepEqual(axeRunPartialResult, axeRunResult, [
+          'testEnvironment'
+        ]);
         done();
       })
       .catch(done);
@@ -54,8 +54,9 @@ describe('run-partial, context-size-focusable', function () {
         assert.deepEqual(nodes[0].target, ['#fail1', 'html']);
         assert.deepEqual(nodes[1].target, ['#fail2', 'iframe', 'html']);
 
-        axeRunPartialResult.timestamp = axeRunResult.timestamp;
-        assert.deepEqual(axeRunPartialResult, axeRunResult);
+        axe.testUtils.resultsDeepEqual(axeRunPartialResult, axeRunResult, [
+          'testEnvironment'
+        ]);
         done();
       })
       .catch(done);

diff --git a/test/integration/full/run-partial/page-level.js b/test/integration/full/run-partial/page-level.js
@@ -31,9 +31,9 @@ describe('run-partial, page-level', function () {
         var axeRunResult = results[1];
         assert.lengthOf(axeRunPartialResult.incomplete, 0);
         assert.lengthOf(axeRunPartialResult.passes, 0);
-        axeRunPartialResult.timestamp = axeRunResult.timestamp;
-        axeRunPartialResult.testEnvironment = axeRunResult.testEnvironment;
-        assert.deepEqual(axeRunPartialResult, axeRunResult);
+        axe.testUtils.resultsDeepEqual(axeRunPartialResult, axeRunResult, [
+          'testEnvironment'
+        ]);
         done();
       })
       .catch(done);
@@ -60,8 +60,9 @@ describe('run-partial, page-level', function () {
         assert.lengthOf(nodes, 1);
         assert.deepEqual(nodes[0].target, ['html']);
 
-        axeRunPartialResult.timestamp = axeRunResult.timestamp;
-        assert.deepEqual(axeRunPartialResult, axeRunResult);
+        axe.testUtils.resultsDeepEqual(axeRunPartialResult, axeRunResult, [
+          'testEnvironment'
+        ]);
         done();
       })
       .catch(done);

diff --git a/test/testutils.js b/test/testutils.js
@@ -708,4 +708,85 @@ var commons;
       fixtureNode.appendChild(child.cloneNode(true));
     }
   }
+
+  testUtils.resultsDeepEqual = (
+    obj1,
+    obj2,
+    ignoredPaths = [],
+    keyPath = 'result'
+  ) => {
+    // timestamp should always be ignored
+    if (keyPath === 'result') {
+      ignoredPaths.push('timestamp');
+    }
+
+    const typeObj1 = getType(obj1);
+    const typeObj2 = getType(obj2);
+
+    axe.utils.assert(
+      typeObj1 === typeObj2,
+      `Expected type of ${keyPath} to equal ${typeObj1} but got ${typeObj2}`
+    );
+
+    if (typeObj1 === 'object') {
+      const res1Keys = Object.keys(obj1);
+      const res2Keys = Object.keys(obj2);
+
+      axe.utils.assert(
+        res1Keys.length === res2Keys.length &&
+          res1Keys.every(key => res2Keys.includes(key)),
+        `Expected ${keyPath} to have keys "${JSON.stringify(res1Keys)}" but got "${JSON.stringify(res2Keys)}"`
+      );
+
+      for (const key of res1Keys) {
+        if (ignoredPaths.includes(key)) {
+          continue;
+        }
+
+        testUtils.resultsDeepEqual(
+          obj1[key],
+          obj2[key],
+          ignoredPaths,
+          `${keyPath}.${key}`
+        );
+      }
+    } else if (typeObj1 === 'array') {
+      axe.utils.assert(
+        obj1.length === obj2.length,
+        `Expected ${keyPath} to have length of "${obj1.length}" but got "${obj2.length}"`
+      );
+
+      for (let i = 0; i < obj1.length; i++) {
+        testUtils.resultsDeepEqual(
+          obj1[i],
+          obj2[i],
+          ignoredPaths,
+          `${keyPath}[${i}]`
+        );
+      }
+    } else {
+      axe.utils.assert(
+        obj1 === obj2,
+        `Expected ${keyPath} to equal "${obj1}" but got "${obj2}"`
+      );
+    }
+  };
+
+  function isObject(obj) {
+    return obj && typeof obj === 'object' && !Array.isArray(obj);
+  }
+
+  function getType(obj) {
+    if (isObject(obj)) {
+      return 'object';
+    }
+    if (Array.isArray(obj)) {
+      return 'array';
+    }
+    if (obj === null) {
+      return 'null';
+    }
+
+    return typeof obj;
+  }
 })();