Merge branch 'origin/develop' into parallel_block_download

.travis.yml

@@ -8,7 +8,7 @@ before_install:
 - sudo apt-get update
 - sudo apt-get install -y solc
 install:
-- pip install -r requirements.txt
+- USE_PYETHEREUM_DEVELOP=1 python setup.py install
 - pip install coveralls readme_renderer
 script:
 - coverage run --source pyethapp setup.py test

README.rst

@@ -35,7 +35,7 @@ pyethapp leverages two ethereum core components to implement the client:
 * pydevp2p_ - the p2p networking library, featuring node discovery for and transport of multiple services over multiplexed and encrypted connections
 
 
-.. _Ethereum: http://ethereum.org/
+.. _Ethereum: https://ethereum.org/
 .. _pyethereum: https://github.com/ethereum/pyethereum
 .. _pydevp2p: https://github.com/ethereum/pydevp2p
 
@@ -57,7 +57,7 @@ advised to install system-dependecies with the help of a package manager
 Please install a *virtualenv* environment for a comfortable Pyethapp
 installation. Also, it is always recommended to use it in combination
 with the
-`virtualenvwrapper <http://virtualenvwrapper.readthedocs.org/en/latest/>`__
+`virtualenvwrapper <https://virtualenvwrapper.readthedocs.org/en/latest/>`__
 extension.
 
 The
@@ -118,7 +118,7 @@ above, then:
     ($ mkvirtualenv pyethapp)
     $ git clone https://github.com/ethereum/pyethapp
     $ cd pyethapp
-    $ python setup.py develop
+    $ USE_PYETHEREUM_DEVELOP=1 python setup.py develop
 
 This has the advantage that inside of Python's
 ``lib/python2.7/site-packages`` there is a direct link to your directory

bumpversion.txt

@@ -0,0 +1,15 @@
+With the config in setup.cfg it's possible to use bump2version to generate the
+usual <major>.<minor>.<patch> (e.g. 1.5.0) releases as well as alpha and rc
+ones (e.g. 1.6.0a1, 1.6.0rc2, etc). For the latter, one just has to tell
+bump2version to update either the "release" or "num" parts. For example,
+assuming a current version of 1.5.0:
+
+  $ bump2version minor    # will update the current version to 1.6.0a0
+
+  $ bump2version num      # will update the current version to 1.6.0a1
+
+  $ bump2version release  # will update the current version to 1.6.0rc0
+
+  $ bump2version num      # will update the current version to 1.6.0rc1
+
+  $ bump2version release  # will update the current version to 1.6.0

dev_requirements.txt

@@ -0,0 +1,6 @@
+pytest==2.9.1
+mock==2.0.0
+pytest-mock==1.6.0
+ethereum-serpent>=1.8.1
+pytest==2.9.1
+coverage==4.0.3

examples/export.py

@@ -35,7 +35,7 @@ def export_blocks(chain, head_number=None):
     head_number = head_number or chain.head.header.number
     block_number = 0
     while block_number < head_number:
-        h = chain.index.get_block_by_number(block_number)
+        h = chain.get_blockhash_by_number(block_number)
         raw = chain.blockchain.get(h)
         print raw.encode('hex')
         _progress(block_number)

examples/importblock.py

@@ -27,7 +27,7 @@ def import_block(chain, rlp_data):
 if __name__ == '__main__':
     chain = get_chain()
     print '\nIMPORTING BLOCK'
-    # h = chain.index.get_block_by_number(447360)
+    # h = chain.get_blockhash_by_number(447360)
     # b = chain.get(h)
     # rlp_data = rlp.encode(b)
     import_block(chain, rlp_data)

pyethapp/accounts.py

@@ -1,17 +1,19 @@
 import json
 import os
-import random
+from random import SystemRandom
 import shutil
 from uuid import UUID
 from devp2p.service import BaseService
-from ethereum import keys
+from ethereum.tools import keys
 from ethereum.slogging import get_logger
 from ethereum.utils import privtopub  # this is different  than the one used in devp2p.crypto
 from ethereum.utils import sha3, is_string, decode_hex, remove_0x_head
 log = get_logger('accounts')
 
 DEFAULT_COINBASE = 'de0b295669a9fd93d5f28d9ec85e40f4cb697bae'.decode('hex')
 
+random = SystemRandom()
+
 
 def mk_privkey(seed):
     return sha3(seed)
@@ -478,7 +480,7 @@ def get_by_address(self, address):
         assert len(address) == 20
         accounts = [account for account in self.accounts if account.address == address]
         if len(accounts) == 0:
-            raise KeyError('account not found by address', address=address.encode('hex'))
+            raise KeyError('account with address {} not found'.format(address.encode('hex')))
         elif len(accounts) > 1:
             log.warning('multiple accounts with same address found', address=address.encode('hex'))
         return accounts[0]

pyethapp/app.py

@@ -16,11 +16,12 @@
 from devp2p.discovery import NodeDiscovery
 from devp2p.peermanager import PeerManager
 from devp2p.service import BaseService
-from ethereum import blocks
-from ethereum.blocks import Block
+from ethereum import config as eth_config
+from ethereum.block import Block
+from ethereum.snapshot import create_snapshot, load_snapshot as _load_snapshot
 from gevent.event import Event
 
-import config as konfig
+import config as app_config
 import eth_protocol
 import utils
 from accounts import AccountsService, Account
@@ -36,8 +37,8 @@
 
 log = slogging.get_logger('app')
 
-services = [DBService, AccountsService, NodeDiscovery, PeerManager, ChainService, PoWService,
-            JSONRPCServer, IPCRPCServer, Console]
+services = [DBService, AccountsService, NodeDiscovery, PeerManager, ChainService,
+            PoWService, JSONRPCServer, IPCRPCServer, Console]
 
 
 class EthApp(BaseApp):
@@ -62,12 +63,12 @@ class EthApp(BaseApp):
                   "'livenet' and 'testnet'. The previous values 'frontier' and "
                   "'morden' will be removed in a future update."),
               default=DEFAULT_PROFILE, help="Configuration profile.", show_default=True)
-@click.option('alt_config', '--Config', '-C', type=str, callback=konfig.validate_alt_config_file,
+@click.option('alt_config', '--Config', '-C', type=str, callback=app_config.validate_alt_config_file,
               help='Alternative config file')
 @click.option('config_values', '-c', multiple=True, type=str,
               help='Single configuration parameters (<param>=<value>)')
 @click.option('alt_data_dir', '-d', '--data-dir', multiple=False, type=str,
-              help='data directory', default=konfig.default_data_dir, show_default=True)
+              help='data directory', default=app_config.default_data_dir, show_default=True)
 @click.option('-l', '--log_config', multiple=False, type=str, default=":info",
               help='log_config string: e.g. ":info,eth:debug', show_default=True)
 @click.option('--log-json/--log-no-json', default=False,
@@ -89,18 +90,18 @@ def app(ctx, profile, alt_config, config_values, alt_data_dir, log_config, boots
 
     # data dir default or from cli option
     alt_data_dir = os.path.expanduser(alt_data_dir)
-    data_dir = alt_data_dir or konfig.default_data_dir
-    konfig.setup_data_dir(data_dir)  # if not available, sets up data_dir and required config
+    data_dir = alt_data_dir or app_config.default_data_dir
+    app_config.setup_data_dir(data_dir)  # if not available, sets up data_dir and required config
     log.info('using data in', path=data_dir)
 
     # prepare configuration
     # config files only contain required config (privkeys) and config different from the default
     if alt_config:  # specified config file
-        config = konfig.load_config(alt_config)
+        config = app_config.load_config(alt_config)
         if not config:
             log.warning('empty config given. default config values will be used')
     else:  # load config from default or set data_dir
-        config = konfig.load_config(data_dir)
+        config = app_config.load_config(data_dir)
 
     config['data_dir'] = data_dir
 
@@ -111,9 +112,8 @@ def app(ctx, profile, alt_config, config_values, alt_data_dir, log_config, boots
     bootstrap_nodes_from_config_file = config.get('discovery', {}).get('bootstrap_nodes')
 
     # add default config
-    konfig.update_config_with_defaults(config, konfig.get_default_config([EthApp] + services))
-
-    konfig.update_config_with_defaults(config, {'eth': {'block': blocks.default_config}})
+    app_config.update_config_with_defaults(config, app_config.get_default_config([EthApp] + services))
+    app_config.update_config_with_defaults(config, {'eth': {'block': eth_config.default_config}})
 
     # Set config values based on profile selection
     merge_dict(config, PROFILES[profile])
@@ -132,7 +132,7 @@ def app(ctx, profile, alt_config, config_values, alt_data_dir, log_config, boots
     # override values with values from cmd line
     for config_value in config_values:
         try:
-            konfig.set_config_param(config, config_value)
+            app_config.set_config_param(config, config_value)
         except ValueError:
             raise BadParameter('Config parameter must be of the form "a.b.c=d" where "a.b.c" '
                                'specifies the parameter to set and d is a valid yaml value '
@@ -144,7 +144,7 @@ def app(ctx, profile, alt_config, config_values, alt_data_dir, log_config, boots
             del config['eth']['genesis_hash']
 
     # Load genesis config
-    konfig.update_config_from_genesis_json(config,
+    app_config.update_config_from_genesis_json(config,
                                            genesis_json_filename_or_dict=config['eth']['genesis'])
     if bootstrap_node:
         config['discovery']['bootstrap_nodes'] = [bytes(bootstrap_node)]
@@ -181,10 +181,6 @@ def run(ctx, dev, nodial, fake, console):
         config['p2p']['min_peers'] = 0
 
     if fake:
-        from ethereum import blocks
-        blocks.GENESIS_DIFFICULTY = 1024
-        blocks.BLOCK_DIFF_FACTOR = 16
-        blocks.MIN_GAS_LIMIT = blocks.default_config['GENESIS_GAS_LIMIT'] / 2
         config['eth']['block']['GENESIS_DIFFICULTY'] = 1024
         config['eth']['block']['BLOCK_DIFF_FACTOR'] = 16
 
@@ -201,7 +197,8 @@ def run(ctx, dev, nodial, fake, console):
             pass
 
     # dump config
-    dump_config(config)
+    if log.is_active('debug'):
+        dump_config(config)
 
     # init and unlock accounts first to check coinbase
     if AccountsService in services:
@@ -256,9 +253,9 @@ def dump_config(config):
     cfg = copy.deepcopy(config)
     alloc = cfg.get('eth', {}).get('block', {}).get('GENESIS_INITIAL_ALLOC', {})
     if len(alloc) > 100:
-        log.info('omitting reporting of %d accounts in genesis' % len(alloc))
+        log.debug('omitting reporting of %d accounts in genesis' % len(alloc))
         del cfg['eth']['block']['GENESIS_INITIAL_ALLOC']
-    konfig.dump_config(cfg)
+    app_config.dump_config(cfg)
 
 
 @app.command()
@@ -335,6 +332,53 @@ def blocktest(ctx, file, name):
     app.stop()
 
 
+@app.command('snapshot')
+@click.option('-r', '--recent', type=int, default=1024,
+              help='Number of recent blocks. State before these blocks and these blocks will be dumped. On recover these blocks will be applied on restored state. (default: 1024)')
+@click.option('-f', '--filename', type=str, default=None,
+              help='Output file name. (default: auto-gen file prefixed by snapshot-')
+@click.pass_context
+def snapshot(ctx, recent, filename):
+    """Take a snapshot of current world state.
+
+    The snapshot will be saved in JSON format, including data like chain configurations and accounts.
+
+    It will overwrite exiting file if it already exists.
+    """
+    app = EthApp(ctx.obj['config'])
+    DBService.register_with_app(app)
+    AccountsService.register_with_app(app)
+    ChainService.register_with_app(app)
+
+    if not filename:
+        import time
+        filename = 'snapshot-%d.json' % int(time.time()*1000)
+
+    s = create_snapshot(app.services.chain.chain, recent)
+    with open(filename, 'w') as f:
+        json.dump(s, f, sort_keys=False, indent=4, separators=(',', ': '), encoding='ascii')
+        print 'snapshot saved to %s' % filename
+
+
+@app.command('load_snapshot')
+@click.argument('filename', type=str)
+@click.pass_context
+def load_snapshot(ctx, filename):
+    """Load snapshot FILE into local node database.
+
+    This process will OVERWRITE data in current database!!!
+    """
+    app = EthApp(ctx.obj['config'])
+    DBService.register_with_app(app)
+    AccountsService.register_with_app(app)
+    ChainService.register_with_app(app)
+
+    with open(filename, 'r') as f:
+        s = json.load(f, encoding='ascii')
+        _load_snapshot(app.services.chain.chain, s)
+        print 'snapshot %s loaded.' % filename
+
+
 @app.command('export')
 @click.option('--from', 'from_', type=int, help='Number of the first block (default: genesis)')
 @click.option('--to', type=int, help='Number of the last block (default: latest)')
@@ -376,7 +420,7 @@ def export_blocks(ctx, from_, to, file):
         log.debug('Exporting block {}'.format(n))
         if (n - from_) % 50000 == 0:
             log.info('Exporting block {} to {}'.format(n, min(n + 50000, to)))
-        block_hash = app.services.chain.chain.index.get_block_by_number(n)
+        block_hash = app.services.chain.chain.get_blockhash_by_number(n)
         # bypass slow block decoding by directly accessing db
         block_rlp = app.services.db.get(block_hash)
         file.write(block_rlp)

pyethapp/config.py

@@ -26,7 +26,7 @@
 from devp2p.service import BaseService
 from devp2p.app import BaseApp
 from accounts import mk_random_privkey
-from ethereum.keys import decode_hex
+from ethereum.tools.keys import decode_hex
 from ethereum.utils import parse_int_or_hex, remove_0x_head
 
 
@@ -206,6 +206,9 @@ def update_config_from_genesis_json(config, genesis_json_filename_or_dict):
     if unknown_keys:
         raise ValueError('genesis_dict contains invalid keys.')
 
+    # TODO: we should only keep raw genesis data to have better interoperability?
+    config['eth']['genesis_data'] = genesis_dict
+
     config.setdefault('eth', {}).setdefault('block', {})
     ethblock_config = config['eth']['block']