Skip to content

Commit

Permalink
async_hooks: use typed array stack as fast path
Browse files Browse the repository at this point in the history
- Communicate the current async stack length through a
  typed array field rather than a native binding method
- Add a new fixed-size `async_ids_fast_stack` typed array
  that contains the async ID stack up to a fixed limit.
  This increases performance noticeably, since most of the time
  the async ID stack will not be more than a handful of
  levels deep.
- Make the JS `pushAsyncIds()` and `popAsyncIds()` functions
  do the same thing as the native ones if the fast path
  is applicable.

Benchmarks:

    $ ./node benchmark/compare.js --new ./node --old ./node-master --runs 10 --filter next-tick process | Rscript benchmark/compare.R
    [00:03:25|% 100| 6/6 files | 20/20 runs | 1/1 configs]: Done
                                                   improvement confidence      p.value
     process/next-tick-breadth-args.js millions=4     19.72 %        *** 3.013913e-06
     process/next-tick-breadth.js millions=4          27.33 %        *** 5.847983e-11
     process/next-tick-depth-args.js millions=12      40.08 %        *** 1.237127e-13
     process/next-tick-depth.js millions=12           77.27 %        *** 1.413290e-11
     process/next-tick-exec-args.js millions=5        13.58 %        *** 1.245180e-07
     process/next-tick-exec.js millions=5             16.80 %        *** 2.961386e-07

Backport-PR-URL: #18290
PR-URL: #17780
Reviewed-By: James M Snell <jasnell@gmail.com>
  • Loading branch information
addaleax authored and evanlucas committed Jan 24, 2018
1 parent 722fe46 commit dd56bd1
Show file tree
Hide file tree
Showing 9 changed files with 164 additions and 54 deletions.
44 changes: 41 additions & 3 deletions lib/internal/async_hooks.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,20 @@ const async_wrap = process.binding('async_wrap');
* retrieving the triggerAsyncId value is passing directly to the
* constructor -> value set in kDefaultTriggerAsyncId -> executionAsyncId of
* the current resource.
*
* async_ids_fast_stack is a Float64Array that contains part of the async ID
* stack. Each pushAsyncIds() call adds two doubles to it, and each
* popAsyncIds() call removes two doubles from it.
* It has a fixed size, so if that is exceeded, calls to the native
* side are used instead in pushAsyncIds() and popAsyncIds().
*/
const { async_hook_fields, async_id_fields } = async_wrap;
// Store the pair executionAsyncId and triggerAsyncId in a std::stack on
// Environment::AsyncHooks::ids_stack_ tracks the resource responsible for the
// current execution stack. This is unwound as each resource exits. In the case
// of a fatal exception this stack is emptied after calling each hook's after()
// callback.
const { pushAsyncIds, popAsyncIds } = async_wrap;
const { pushAsyncIds: pushAsyncIds_, popAsyncIds: popAsyncIds_ } = async_wrap;
// For performance reasons, only track Proimses when a hook is enabled.
const { enablePromiseHook, disablePromiseHook } = async_wrap;
// Properties in active_hooks are used to keep track of the set of hooks being
Expand Down Expand Up @@ -60,8 +66,8 @@ const active_hooks = {
// async execution. These are tracked so if the user didn't include callbacks
// for a given step, that step can bail out early.
const { kInit, kBefore, kAfter, kDestroy, kPromiseResolve,
kCheck, kExecutionAsyncId, kAsyncIdCounter,
kDefaultTriggerAsyncId } = async_wrap.constants;
kCheck, kExecutionAsyncId, kAsyncIdCounter, kTriggerAsyncId,
kDefaultTriggerAsyncId, kStackLength } = async_wrap.constants;

// Used in AsyncHook and AsyncResource.
const init_symbol = Symbol('init');
Expand Down Expand Up @@ -329,6 +335,38 @@ function emitDestroyScript(asyncId) {
}


// This is the equivalent of the native push_async_ids() call.
function pushAsyncIds(asyncId, triggerAsyncId) {
const offset = async_hook_fields[kStackLength];
if (offset * 2 >= async_wrap.async_ids_stack.length)
return pushAsyncIds_(asyncId, triggerAsyncId);
async_wrap.async_ids_stack[offset * 2] = async_id_fields[kExecutionAsyncId];
async_wrap.async_ids_stack[offset * 2 + 1] = async_id_fields[kTriggerAsyncId];
async_hook_fields[kStackLength]++;
async_id_fields[kExecutionAsyncId] = asyncId;
async_id_fields[kTriggerAsyncId] = triggerAsyncId;
}


// This is the equivalent of the native pop_async_ids() call.
function popAsyncIds(asyncId) {
if (async_hook_fields[kStackLength] === 0) return false;
const stackLength = async_hook_fields[kStackLength];

if (async_hook_fields[kCheck] > 0 &&
async_id_fields[kExecutionAsyncId] !== asyncId) {
// Do the same thing as the native code (i.e. crash hard).
return popAsyncIds_(asyncId);
}

const offset = stackLength - 1;
async_id_fields[kExecutionAsyncId] = async_wrap.async_ids_stack[2 * offset];
async_id_fields[kTriggerAsyncId] = async_wrap.async_ids_stack[2 * offset + 1];
async_hook_fields[kStackLength] = offset;
return offset > 0;
}


module.exports = {
// Private API
getHookArrays,
Expand Down
6 changes: 3 additions & 3 deletions lib/internal/bootstrap_node.js
Original file line number Diff line number Diff line change
Expand Up @@ -366,9 +366,9 @@
// Arrays containing hook flags and ids for async_hook calls.
const { async_hook_fields, async_id_fields } = async_wrap;
// Internal functions needed to manipulate the stack.
const { clearAsyncIdStack, asyncIdStackSize } = async_wrap;
const { clearAsyncIdStack } = async_wrap;
const { kAfter, kExecutionAsyncId,
kDefaultTriggerAsyncId } = async_wrap.constants;
kDefaultTriggerAsyncId, kStackLength } = async_wrap.constants;

process._fatalException = function(er) {
var caught;
Expand Down Expand Up @@ -406,7 +406,7 @@
do {
NativeModule.require('internal/async_hooks').emitAfter(
async_id_fields[kExecutionAsyncId]);
} while (asyncIdStackSize() > 0);
} while (async_hook_fields[kStackLength] > 0);
// Or completely empty the id stack.
} else {
clearAsyncIdStack();
Expand Down
29 changes: 27 additions & 2 deletions src/aliased_buffer.h
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,21 @@ class AliasedBuffer {
js_array_.Reset();
}

AliasedBuffer& operator=(AliasedBuffer&& that) {
this->~AliasedBuffer();
isolate_ = that.isolate_;
count_ = that.count_;
byte_offset_ = that.byte_offset_;
buffer_ = that.buffer_;
free_buffer_ = that.free_buffer_;

js_array_.Reset(isolate_, that.js_array_.Get(isolate_));

that.buffer_ = nullptr;
that.js_array_.Reset();
return *this;
}

/**
* Helper class that is returned from operator[] to support assignment into
* a specified location.
Expand All @@ -111,11 +126,17 @@ class AliasedBuffer {
index_(that.index_) {
}

inline Reference& operator=(const NativeT &val) {
template <typename T>
inline Reference& operator=(const T& val) {
aliased_buffer_->SetValue(index_, val);
return *this;
}

// This is not caught by the template operator= above.
inline Reference& operator=(const Reference& val) {
return *this = static_cast<NativeT>(val);
}

operator NativeT() const {
return aliased_buffer_->GetValue(index_);
}
Expand Down Expand Up @@ -186,8 +207,12 @@ class AliasedBuffer {
return GetValue(index);
}

size_t Length() const {
return count_;
}

private:
v8::Isolate* const isolate_;
v8::Isolate* isolate_;
size_t count_;
size_t byte_offset_;
NativeT* buffer_;
Expand Down
14 changes: 6 additions & 8 deletions src/async_wrap.cc
Original file line number Diff line number Diff line change
Expand Up @@ -467,13 +467,6 @@ void AsyncWrap::PopAsyncIds(const FunctionCallbackInfo<Value>& args) {
}


void AsyncWrap::AsyncIdStackSize(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
args.GetReturnValue().Set(
static_cast<double>(env->async_hooks()->stack_size()));
}


void AsyncWrap::ClearAsyncIdStack(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
env->async_hooks()->clear_async_id_stack();
Expand Down Expand Up @@ -512,7 +505,6 @@ void AsyncWrap::Initialize(Local<Object> target,
env->SetMethod(target, "setupHooks", SetupHooks);
env->SetMethod(target, "pushAsyncIds", PushAsyncIds);
env->SetMethod(target, "popAsyncIds", PopAsyncIds);
env->SetMethod(target, "asyncIdStackSize", AsyncIdStackSize);
env->SetMethod(target, "clearAsyncIdStack", ClearAsyncIdStack);
env->SetMethod(target, "queueDestroyAsyncId", QueueDestroyAsyncId);
env->SetMethod(target, "enablePromiseHook", EnablePromiseHook);
Expand Down Expand Up @@ -550,6 +542,10 @@ void AsyncWrap::Initialize(Local<Object> target,
"async_id_fields",
env->async_hooks()->async_id_fields().GetJSArray());

target->Set(context,
env->async_ids_stack_string(),
env->async_hooks()->async_ids_stack().GetJSArray()).FromJust();

Local<Object> constants = Object::New(isolate);
#define SET_HOOKS_CONSTANT(name) \
FORCE_SET_TARGET_FIELD( \
Expand All @@ -566,6 +562,7 @@ void AsyncWrap::Initialize(Local<Object> target,
SET_HOOKS_CONSTANT(kTriggerAsyncId);
SET_HOOKS_CONSTANT(kAsyncIdCounter);
SET_HOOKS_CONSTANT(kDefaultTriggerAsyncId);
SET_HOOKS_CONSTANT(kStackLength);
#undef SET_HOOKS_CONSTANT
FORCE_SET_TARGET_FIELD(target, "constants", constants);

Expand Down Expand Up @@ -595,6 +592,7 @@ void AsyncWrap::Initialize(Local<Object> target,
env->set_async_hooks_after_function(Local<Function>());
env->set_async_hooks_destroy_function(Local<Function>());
env->set_async_hooks_promise_resolve_function(Local<Function>());
env->set_async_hooks_binding(target);
}


Expand Down
1 change: 0 additions & 1 deletion src/async_wrap.h
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,6 @@ class AsyncWrap : public BaseObject {
static void GetAsyncId(const v8::FunctionCallbackInfo<v8::Value>& args);
static void PushAsyncIds(const v8::FunctionCallbackInfo<v8::Value>& args);
static void PopAsyncIds(const v8::FunctionCallbackInfo<v8::Value>& args);
static void AsyncIdStackSize(const v8::FunctionCallbackInfo<v8::Value>& args);
static void ClearAsyncIdStack(
const v8::FunctionCallbackInfo<v8::Value>& args);
static void AsyncReset(const v8::FunctionCallbackInfo<v8::Value>& args);
Expand Down
62 changes: 38 additions & 24 deletions src/env-inl.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@ inline MultiIsolatePlatform* IsolateData::platform() const {
return platform_;
}

inline Environment::AsyncHooks::AsyncHooks(v8::Isolate* isolate)
: isolate_(isolate),
fields_(isolate, kFieldsCount),
async_id_fields_(isolate, kUidFieldsCount) {
v8::HandleScope handle_scope(isolate_);
inline Environment::AsyncHooks::AsyncHooks()
: async_ids_stack_(env()->isolate(), 16 * 2),
fields_(env()->isolate(), kFieldsCount),
async_id_fields_(env()->isolate(), kUidFieldsCount) {
v8::HandleScope handle_scope(env()->isolate());

// Always perform async_hooks checks, not just when async_hooks is enabled.
// TODO(AndreasMadsen): Consider removing this for LTS releases.
Expand All @@ -81,9 +81,9 @@ inline Environment::AsyncHooks::AsyncHooks(v8::Isolate* isolate)
// strings can be retrieved quickly.
#define V(Provider) \
providers_[AsyncWrap::PROVIDER_ ## Provider].Set( \
isolate_, \
env()->isolate(), \
v8::String::NewFromOneByte( \
isolate_, \
env()->isolate(), \
reinterpret_cast<const uint8_t*>(#Provider), \
v8::NewStringType::kInternalized, \
sizeof(#Provider) - 1).ToLocalChecked());
Expand All @@ -101,15 +101,25 @@ Environment::AsyncHooks::async_id_fields() {
return async_id_fields_;
}

inline AliasedBuffer<double, v8::Float64Array>&
Environment::AsyncHooks::async_ids_stack() {
return async_ids_stack_;
}

inline v8::Local<v8::String> Environment::AsyncHooks::provider_string(int idx) {
return providers_[idx].Get(isolate_);
return providers_[idx].Get(env()->isolate());
}

inline void Environment::AsyncHooks::no_force_checks() {
// fields_ does not have the -= operator defined
fields_[kCheck] = fields_[kCheck] - 1;
}

inline Environment* Environment::AsyncHooks::env() {
return Environment::ForAsyncHooks(this);
}

// Remember to keep this code aligned with pushAsyncIds() in JS.
inline void Environment::AsyncHooks::push_async_ids(double async_id,
double trigger_async_id) {
// Since async_hooks is experimental, do only perform the check
Expand All @@ -119,16 +129,21 @@ inline void Environment::AsyncHooks::push_async_ids(double async_id,
CHECK_GE(trigger_async_id, -1);
}

async_ids_stack_.push({ async_id_fields_[kExecutionAsyncId],
async_id_fields_[kTriggerAsyncId] });
uint32_t offset = fields_[kStackLength];
if (offset * 2 >= async_ids_stack_.Length())
grow_async_ids_stack();
async_ids_stack_[2 * offset] = async_id_fields_[kExecutionAsyncId];
async_ids_stack_[2 * offset + 1] = async_id_fields_[kTriggerAsyncId];
fields_[kStackLength] = fields_[kStackLength] + 1;
async_id_fields_[kExecutionAsyncId] = async_id;
async_id_fields_[kTriggerAsyncId] = trigger_async_id;
}

// Remember to keep this code aligned with popAsyncIds() in JS.
inline bool Environment::AsyncHooks::pop_async_id(double async_id) {
// In case of an exception then this may have already been reset, if the
// stack was multiple MakeCallback()'s deep.
if (async_ids_stack_.empty()) return false;
if (fields_[kStackLength] == 0) return false;

// Ask for the async_id to be restored as a check that the stack
// hasn't been corrupted.
Expand All @@ -140,32 +155,27 @@ inline bool Environment::AsyncHooks::pop_async_id(double async_id) {
"actual: %.f, expected: %.f)\n",
async_id_fields_.GetValue(kExecutionAsyncId),
async_id);
Environment* env = Environment::GetCurrent(isolate_);
DumpBacktrace(stderr);
fflush(stderr);
if (!env->abort_on_uncaught_exception())
if (!env()->abort_on_uncaught_exception())
exit(1);
fprintf(stderr, "\n");
fflush(stderr);
ABORT_NO_BACKTRACE();
}

auto async_ids = async_ids_stack_.top();
async_ids_stack_.pop();
async_id_fields_[kExecutionAsyncId] = async_ids.async_id;
async_id_fields_[kTriggerAsyncId] = async_ids.trigger_async_id;
return !async_ids_stack_.empty();
}
uint32_t offset = fields_[kStackLength] - 1;
async_id_fields_[kExecutionAsyncId] = async_ids_stack_[2 * offset];
async_id_fields_[kTriggerAsyncId] = async_ids_stack_[2 * offset + 1];
fields_[kStackLength] = offset;

inline size_t Environment::AsyncHooks::stack_size() {
return async_ids_stack_.size();
return fields_[kStackLength] > 0;
}

inline void Environment::AsyncHooks::clear_async_id_stack() {
while (!async_ids_stack_.empty())
async_ids_stack_.pop();
async_id_fields_[kExecutionAsyncId] = 0;
async_id_fields_[kTriggerAsyncId] = 0;
fields_[kStackLength] = 0;
}

inline Environment::AsyncHooks::DefaultTriggerAsyncIdScope
Expand All @@ -189,6 +199,11 @@ inline Environment::AsyncHooks::DefaultTriggerAsyncIdScope
}


Environment* Environment::ForAsyncHooks(AsyncHooks* hooks) {
return ContainerOf(&Environment::async_hooks_, hooks);
}


inline Environment::AsyncCallbackScope::AsyncCallbackScope(Environment* env)
: env_(env) {
env_->makecallback_cntr_++;
Expand Down Expand Up @@ -262,7 +277,6 @@ inline Environment::Environment(IsolateData* isolate_data,
v8::Local<v8::Context> context)
: isolate_(context->GetIsolate()),
isolate_data_(isolate_data),
async_hooks_(context->GetIsolate()),
timer_base_(uv_now(isolate_data->event_loop())),
using_domains_(false),
printed_error_(false),
Expand Down
16 changes: 16 additions & 0 deletions src/env.cc
Original file line number Diff line number Diff line change
Expand Up @@ -323,4 +323,20 @@ void Environment::ActivateImmediateCheck() {
uv_idle_start(&immediate_idle_handle_, [](uv_idle_t*){ });
}

void Environment::AsyncHooks::grow_async_ids_stack() {
const uint32_t old_capacity = async_ids_stack_.Length() / 2;
const uint32_t new_capacity = old_capacity * 1.5;
AliasedBuffer<double, v8::Float64Array> new_buffer(
env()->isolate(), new_capacity * 2);

for (uint32_t i = 0; i < old_capacity * 2; ++i)
new_buffer[i] = async_ids_stack_[i];
async_ids_stack_ = std::move(new_buffer);

env()->async_hooks_binding()->Set(
env()->context(),
env()->async_ids_stack_string(),
async_ids_stack_.GetJSArray()).FromJust();
}

} // namespace node
Loading

0 comments on commit dd56bd1

Please sign in to comment.