From 7202f7ae822e26fe5bcdcce0d56b962b923b35fc Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Fri, 20 Apr 2018 21:22:29 +0800 Subject: [PATCH] [beta] Backports (#8450) * Use forked app_dirs crate for reverted Windows dir behavior (#8438) * Remove unused appdirs dependency in CLI * Use forked app_dirs crate for reverted Windows dir behavior * remove Tendermint extra_info due to seal inconsistencies (#8367) * handle queue import errors a bit more gracefully (#8385) * Improve VM executor stack size estimation rules (#8439) * Improve VM executor stack size estimation rules * typo: docs add "(Debug build)" comment * Fix an off by one typo and set minimal stack size This avoids the case if `depth_threshold == max_depth`. Usually setting stack size to zero will just rebound it to platform minimal stack size, but we set it here just in case. * Use saturating_sub to avoid potential overflow --- Cargo.lock | 7 +++--- Cargo.toml | 1 - ethcore/src/engines/tendermint/mod.rs | 13 +---------- ethcore/src/executive.rs | 33 +++++++++++++++++---------- parity/main.rs | 1 - sync/src/light_sync/mod.rs | 19 +++++++++++---- util/dir/Cargo.toml | 2 +- 7 files changed, 41 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 512e7ed21d2..c935c2daa1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "app_dirs" version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" +source = "git+https://github.com/paritytech/app-dirs-rs#0b37f9481ce29e9d5174ad185bca695b206368eb" dependencies = [ "ole32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "shell32-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -319,7 +319,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "dir" version = "0.1.0" dependencies = [ - "app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "app_dirs 1.2.1 (git+https://github.com/paritytech/app-dirs-rs)", "ethereum-types 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "journaldb 0.1.0", ] @@ -1865,7 +1865,6 @@ name = "parity" version = "1.10.1" dependencies = [ "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", - "app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.29.1 (registry+https://github.com/rust-lang/crates.io-index)", "ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)", "daemonize 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3556,7 +3555,7 @@ dependencies = [ [metadata] "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" "checksum ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6b3568b48b7cefa6b8ce125f9bb4989e52fbcc29ebea88df04cc7c5f12f70455" -"checksum app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e73a24bad9bd6a94d6395382a6c69fe071708ae4409f763c5475e14ee896313d" +"checksum app_dirs 1.2.1 (git+https://github.com/paritytech/app-dirs-rs)" = "" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" "checksum aster 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ccfdf7355d9db158df68f976ed030ab0f6578af811f5a7bb6dcf221ec24e0e0" "checksum atty 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d912da0db7fa85514874458ca3651fe2cddace8d0b0505571dbdcd41ab490159" diff --git a/Cargo.toml b/Cargo.toml index 7ff7903fe8d..cfb8d32204b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,6 @@ toml = "0.4" serde = "1.0" serde_json = "1.0" serde_derive = "1.0" -app_dirs = "1.1.1" futures = "0.1" futures-cpupool = "0.1" fdlimit = "0.1" diff --git a/ethcore/src/engines/tendermint/mod.rs b/ethcore/src/engines/tendermint/mod.rs index dfac00bea32..0ecbfa648d5 100644 --- a/ethcore/src/engines/tendermint/mod.rs +++ b/ethcore/src/engines/tendermint/mod.rs @@ -27,7 +27,7 @@ mod params; use std::sync::{Weak, Arc}; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; -use std::collections::{HashSet, BTreeMap}; +use std::collections::HashSet; use hash::keccak; use ethereum_types::{H256, H520, U128, U256, Address}; use parking_lot::RwLock; @@ -449,17 +449,6 @@ impl Engine for Tendermint { fn maximum_uncle_age(&self) -> usize { 0 } - /// Additional engine-specific information for the user/developer concerning `header`. - fn extra_info(&self, header: &Header) -> BTreeMap { - let message = ConsensusMessage::new_proposal(header).expect("Invalid header."); - map![ - "signature".into() => message.signature.to_string(), - "height".into() => message.vote_step.height.to_string(), - "view".into() => message.vote_step.view.to_string(), - "block_hash".into() => message.block_hash.as_ref().map(ToString::to_string).unwrap_or("".into()) - ] - } - fn populate_from_parent(&self, header: &mut Header, parent: &Header) { // Chain scoring: total weight is sqrt(U256::max_value())*height - view let new_difficulty = U256::from(U128::max_value()) diff --git a/ethcore/src/executive.rs b/ethcore/src/executive.rs index c8b27576f24..2c3be92fd27 100644 --- a/ethcore/src/executive.rs +++ b/ethcore/src/executive.rs @@ -34,10 +34,21 @@ use transaction::{Action, SignedTransaction}; use crossbeam; pub use executed::{Executed, ExecutionResult}; -/// Roughly estimate what stack size each level of evm depth will use -/// TODO [todr] We probably need some more sophisticated calculations here (limit on my machine 132) -/// Maybe something like here: `https://github.com/ethereum/libethereum/blob/4db169b8504f2b87f7d5a481819cfb959fc65f6c/libethereum/ExtVM.cpp` -const STACK_SIZE_PER_DEPTH: usize = 24*1024; +#[cfg(debug_assertions)] +/// Roughly estimate what stack size each level of evm depth will use. (Debug build) +const STACK_SIZE_PER_DEPTH: usize = 128 * 1024; + +#[cfg(not(debug_assertions))] +/// Roughly estimate what stack size each level of evm depth will use. +const STACK_SIZE_PER_DEPTH: usize = 24 * 1024; + +#[cfg(debug_assertions)] +/// Entry stack overhead prior to execution. (Debug build) +const STACK_SIZE_ENTRY_OVERHEAD: usize = 100 * 1024; + +#[cfg(not(debug_assertions))] +/// Entry stack overhead prior to execution. +const STACK_SIZE_ENTRY_OVERHEAD: usize = 20 * 1024; /// Returns new address created from address, nonce, and code hash pub fn contract_address(address_scheme: CreateContractAddress, sender: &Address, nonce: &U256, code: &[u8]) -> (Address, Option) { @@ -332,12 +343,12 @@ impl<'a, B: 'a + StateBackend> Executive<'a, B> { tracer: &mut T, vm_tracer: &mut V ) -> vm::Result where T: Tracer, V: VMTracer { - - let depth_threshold = ::io::LOCAL_STACK_SIZE.with(|sz| sz.get() / STACK_SIZE_PER_DEPTH); + let local_stack_size = ::io::LOCAL_STACK_SIZE.with(|sz| sz.get()); + let depth_threshold = local_stack_size.saturating_sub(STACK_SIZE_ENTRY_OVERHEAD) / STACK_SIZE_PER_DEPTH; let static_call = params.call_type == CallType::StaticCall; // Ordinary execution - keep VM in same thread - if (self.depth + 1) % depth_threshold != 0 { + if self.depth != depth_threshold { let vm_factory = self.state.vm_factory(); let mut ext = self.as_externalities(OriginInfo::from(¶ms), unconfirmed_substate, output_policy, tracer, vm_tracer, static_call); trace!(target: "executive", "ext.schedule.have_delegate_call: {}", ext.schedule().have_delegate_call); @@ -345,17 +356,15 @@ impl<'a, B: 'a + StateBackend> Executive<'a, B> { return vm.exec(params, &mut ext).finalize(ext); } - // Start in new thread to reset stack - // TODO [todr] No thread builder yet, so we need to reset once for a while - // https://github.com/aturon/crossbeam/issues/16 + // Start in new thread with stack size needed up to max depth crossbeam::scope(|scope| { let vm_factory = self.state.vm_factory(); let mut ext = self.as_externalities(OriginInfo::from(¶ms), unconfirmed_substate, output_policy, tracer, vm_tracer, static_call); - scope.spawn(move || { + scope.builder().stack_size(::std::cmp::max(schedule.max_depth.saturating_sub(depth_threshold) * STACK_SIZE_PER_DEPTH, local_stack_size)).spawn(move || { let mut vm = vm_factory.create(¶ms, &schedule); vm.exec(params, &mut ext).finalize(ext) - }) + }).expect("Sub-thread creation cannot fail; the host might run out of resources; qed") }).join() } diff --git a/parity/main.rs b/parity/main.rs index 1a1eccf120a..97e107f1814 100644 --- a/parity/main.rs +++ b/parity/main.rs @@ -19,7 +19,6 @@ #![warn(missing_docs)] extern crate ansi_term; -extern crate app_dirs; extern crate ctrlc; extern crate docopt; #[macro_use] diff --git a/sync/src/light_sync/mod.rs b/sync/src/light_sync/mod.rs index ef1bd8742f3..df76d25ccca 100644 --- a/sync/src/light_sync/mod.rs +++ b/sync/src/light_sync/mod.rs @@ -427,6 +427,8 @@ impl LightSync { // handles request dispatch, block import, state machine transitions, and timeouts. fn maintain_sync(&self, ctx: &BasicContext) { + use ethcore::error::{BlockImportError, ImportError}; + const DRAIN_AMOUNT: usize = 128; let client = self.client.as_light_client(); @@ -453,11 +455,20 @@ impl LightSync { trace!(target: "sync", "Drained {} headers to import", sink.len()); for header in sink.drain(..) { - if let Err(e) = client.queue_header(header) { - debug!(target: "sync", "Found bad header ({:?}). Reset to search state.", e); + match client.queue_header(header) { + Ok(_) => {} + Err(BlockImportError::Import(ImportError::AlreadyInChain)) => { + trace!(target: "sync", "Block already in chain. Continuing."); + }, + Err(BlockImportError::Import(ImportError::AlreadyQueued)) => { + trace!(target: "sync", "Block already queued. Continuing."); + }, + Err(e) => { + debug!(target: "sync", "Found bad header ({:?}). Reset to search state.", e); - self.begin_search(&mut state); - break 'a; + self.begin_search(&mut state); + break 'a; + } } } } diff --git a/util/dir/Cargo.toml b/util/dir/Cargo.toml index c94086386c9..62c61af54da 100644 --- a/util/dir/Cargo.toml +++ b/util/dir/Cargo.toml @@ -6,4 +6,4 @@ authors = ["Parity Technologies "] [dependencies] ethereum-types = "0.2" journaldb = { path = "../journaldb" } -app_dirs = "1.1.1" +app_dirs = { git = "https://github.com/paritytech/app-dirs-rs" }