util: allow returning this from custom inspect

If a custom inspection function returned `this`, use that value
for further formatting instead of going into infinite recursion.

This is particularly useful when combined with `util.inspect.custom`
because returning `this` from such a method makes it easy to
have an `inspect()` function that is ignored by `util.inspect` without
actually having to provide an alternative for custom inspection.

lib/util.js

@@ -362,10 +362,15 @@ function formatValue(ctx, value, recurseTimes) {
         // Also filter out any prototype objects using the circular check.
         !(value.constructor && value.constructor.prototype === value)) {
       let ret = maybeCustomInspect.call(value, recurseTimes, ctx);
-      if (typeof ret !== 'string') {
-        ret = formatValue(ctx, ret, recurseTimes);
+
+      // If the custom inspection method returned `this`, don't go into
+      // infinite recursion.
+      if (ret !== value) {
+        if (typeof ret !== 'string') {
+          ret = formatValue(ctx, ret, recurseTimes);
+        }
+        return ret;
       }
-      return ret;
     }
   }
 

test/parallel/test-util-inspect.js

@@ -567,6 +567,16 @@ assert.doesNotThrow(function() {
   );
 }
 
+{
+  // Returning `this` from a custom inspection function works.
+  assert.strictEqual(util.inspect({ a: 123, inspect() { return this; } }),
+                     '{ a: 123, inspect: [Function: inspect] }');
+
+  const subject = { a: 123, [util.inspect.custom]() { return this; } };
+  assert.strictEqual(util.inspect(subject),
+                     '{ a: 123 }');
+}
+
 // util.inspect with "colors" option should produce as many lines as without it
 function test_lines(input) {
   var count_lines = function(str) {