Add randomization support

Fixes #902.

bin/_mocha

@@ -68,6 +68,7 @@ program
   .option('-O, --reporter-options <k=v,k2=v2,...>', 'reporter-specific options')
   .option('-R, --reporter <name>', 'specify the reporter to use', 'spec')
   .option('-S, --sort', 'sort test files')
+  .option('-a, --randomize <mode>[:seed]', "randomize tests")
   .option('-b, --bail', 'bail after first test failure')
   .option('-d, --debug', "enable node's debugger, synonym for node --debug")
   .option('-g, --grep <pattern>', 'only run tests matching <pattern>')
@@ -275,6 +276,13 @@ if (program.fgrep) {
   mocha.fgrep(program.fgrep);
 }
 
+// --randomize
+
+if (program.randomize) {
+  var matches = program.randomize.split(':');
+  mocha.randomize(matches[0], matches[1]);
+}
+
 // --invert
 
 if (program.invert) {

lib/interfaces/bdd.js

@@ -58,6 +58,20 @@ module.exports = function(suite) {
       });
     };
 
+    /**
+     * Do not randomize.
+     */
+    context.describe.noRandom = function func(title, fn) {
+      var suite = context.describe(title, fn);
+      suite.enableRandomize(false);
+      return suite;
+    };
+    // these three are disabled anyway, so don't actually implement noRandom
+    context.describe.noRandom.skip =
+    context.describe.skip.noRandom =
+    context.xdescribe.noRandom     =
+    context.describe.skip;
+
     /**
      * Exclusive suite.
      */
@@ -69,6 +83,12 @@ module.exports = function(suite) {
         fn: fn
       });
     };
+    context.describe.only.noRandom =
+    context.describe.noRandom.only = function(title, fn){
+      var suite = context.describe.noRandom(title, fn);
+      mocha.grep(suite.fullTitle());
+      return suite;
+    };
 
     /**
      * Describe a specification or test-case

lib/interfaces/qunit.js

@@ -54,6 +54,28 @@ module.exports = function(suite) {
       });
     };
 
+    /**
+     * Do not randomize.
+     */
+    context.suite.noRandom = function(title) {
+      var suite = common.suite.create({
+        title: title,
+        file: file,
+      });
+      suite.enableRandomize(false);
+      return suite;
+    };
+
+    context.suite.only.noRandom =
+    context.suite.noRandom.only = function(title) {
+      var suite = common.suite.only({
+        title: title,
+        file: file,
+      });
+      suite.enableRandomize(false);
+      return suite;
+    };
+
     /**
      * Exclusive Suite.
      */

lib/interfaces/tdd.js

@@ -64,6 +64,16 @@ module.exports = function(suite) {
       });
     };
 
+    /**
+     * Do not randomize.
+     */
+    context.suite.noRandom = function(title, fn) {
+      var suite = context.suite(title, fn);
+      suite.enableRandomize(false);
+      return suite;
+    };
+    context.suite.noRandom.skip = context.suite.skip;
+
     /**
      * Exclusive test-case.
      */
@@ -74,6 +84,11 @@ module.exports = function(suite) {
         fn: fn
       });
     };
+    context.suite.only.noRandom =
+    context.suite.noRandom.only = function(title, fn){
+      var suite = context.suite.noRandom(title, fn);
+      mocha.grep(suite.fullTitle());
+    };
 
     /**
      * Describe a specification or test-case with the given `title` and

lib/mocha.js

@@ -68,6 +68,8 @@ function image(name) {
  *   - `ignoreLeaks` ignore global leaks
  *   - `fullTrace` display the full stack-trace on failing
  *   - `grep` string or regexp to filter tests with
+ *   - `randomMode` mode for randomization of tests
+ *   - `randomSeed` seed for randomization of tests
  *
  * @param {Object} options
  * @api public
@@ -288,19 +290,42 @@ Mocha.prototype.invert = function() {
 };
 
 /**
- * Ignore global leaks.
+ * Apply randomization to the tests
  *
- * @param {Boolean} ignore
+ * @param {String} mode
+ * @param          seed
  * @return {Mocha}
  * @api public
  * @param {boolean} ignore
  * @return {Mocha}
  */
 Mocha.prototype.ignoreLeaks = function(ignore) {
+  if (mode !== 'tests' /* && mode !== 'suites' && mode !== 'both' */) {
+    throw new Error('unrecognized randomization mode "' + mode + '"');
+  }
+  this.options.randomizeMode = mode;
   this.options.ignoreLeaks = Boolean(ignore);
   return this;
 };
 
+/**
+ * Apply randomization to the tests
+ *
+ * @param {String} mode
+ * @param          seed
+ * @return {Mocha}
+ * @api public
+ */
+
+Mocha.prototype.randomize = function(mode, seed){
+  if (mode !== 'tests' /* && mode !== 'suites' && mode !== 'both' */) {
+    throw new Error('unrecognized randomization mode "' + mode + '"');
+  }
+  this.options.randomizeMode = mode;
+  this.options.randomizeSeed = seed != null ? seed : Math.random();
+  return this;
+};
+
 /**
  * Enable global leak checking.
  *
@@ -506,6 +531,10 @@ Mocha.prototype.run = function(fn) {
   if (options.useColors !== undefined) {
     exports.reporters.Base.useColors = options.useColors;
   }
+  if (options.randomizeMode !== undefined) {
+    runner.randomize(options.randomizeMode, options.randomizeSeed);
+  }
+
   exports.reporters.Base.inlineDiffs = options.useInlineDiffs;
 
   function done(failures) {

lib/runner.js

@@ -212,6 +212,25 @@ Runner.prototype.checkGlobals = function(test) {
   }
 };
 
+/**
+ * Randomize tests in a suite.
+ *
+ * @param seed
+ * @return {Runner} for chaining
+ * @api public
+ */
+
+Runner.prototype.randomize = function(mode, seed){
+  debug('randomize mode: "' + mode + '"; seed: ' + seed);
+  if (mode !== 'tests') return this;
+  this._randomize = function(suite){
+    debug('randomize: ', suite.title, suite._enableRandomize);
+    if (!suite._enableRandomize) return suite.tests;
+    return randomize(suite.tests, seed);
+  }
+  return this;
+};
+
 /**
  * Fail the given `test`.
  *
@@ -448,6 +467,7 @@ Runner.prototype.runTest = function(fn) {
 Runner.prototype.runTests = function(suite, fn) {
   var self = this;
   var tests = suite.tests.slice();
+  if (self._randomize) self._randomize(self.suite);
   var test;
 
   function hookErr(_, errSuite, after) {

lib/suite.js

@@ -61,6 +61,7 @@ function Suite(title, parentContext) {
   this._slow = 75;
   this._bail = false;
   this._retries = -1;
+  this._enableRandomize = false;
   this._onlyTests = [];
   this._onlySuites = [];
   this.delayed = false;
@@ -86,6 +87,7 @@ Suite.prototype.clone = function() {
   suite.enableTimeouts(this.enableTimeouts());
   suite.slow(this.slow());
   suite.bail(this.bail());
+  suite.enableRandomize(this.enableRandomize());
   return suite;
 };
 
@@ -187,6 +189,24 @@ Suite.prototype.isPending = function() {
   return this.pending || (this.parent && this.parent.isPending());
 };
 
+/**
+  * Set randomization `enabled`.
+  *
+  * Do NOT use `this.enableRandomize()` in your test; use `describe.noRandom`
+  * instead.
+  *
+  * @param {Boolean} enabled
+  * @return {Suite|Boolean} self or enabled
+  * @api private
+  */
+
+Suite.prototype.enableRandomize = function(enabled){
+  if (arguments.length === 0) return this._enableRandomize;
+  debug('enableRandomize %s', enabled);
+  this._enableRandomize = enabled;
+  return this;
+};
+
 /**
  * Run `fn(test[, done])` before running tests.
  *

test/hook-async.spec.js

@@ -1,4 +1,4 @@
-describe('async', function(){
+describe.noRandom('async', function(){
   var calls = [];
 
   before(function(){
@@ -43,7 +43,7 @@ describe('async', function(){
     calls.push('parent after' );
   })
 
-  describe('hooks', function(){
+  describe.noRandom('hooks', function(){
     before(function(){
       calls.push('before all');
     });

test/hook-sync-nested.spec.js

@@ -1,5 +1,5 @@
-describe('serial', function(){
-  describe('nested', function(){
+describe.noRandom('serial', function(){
+  describe.noRandom('nested', function(){
     var calls = [];
 
     beforeEach(function(){
@@ -34,7 +34,7 @@ describe('serial', function(){
         , 'parent before test bar']);
     })
 
-    describe('hooks', function(){
+    describe.noRandom('hooks', function(){
       beforeEach(function(){
         calls.push('before');
         if (this.currentTest) {

test/hook-sync.spec.js

@@ -1,4 +1,4 @@
-describe('serial', function(){
+describe.noRandom('serial', function(){
   var calls = [];
 
   beforeEach(function(){
@@ -9,7 +9,7 @@ describe('serial', function(){
     calls.push('parent after');
   })
 
-  describe('hooks', function(){
+  describe.noRandom('hooks', function(){
     beforeEach(function(){
       calls.push('before');
       if (this.currentTest) {