Merge branch 'master' into feat/docs-reorg

.circleci/config.yml

@@ -225,12 +225,17 @@ jobs:
                   cd extern/test-vectors
                   git fetch
                   git checkout origin/<< parameters.vectors-branch >>
-            - run:
-                name: go get vectors branch
-                command: go get github.com/filecoin-project/test-vectors@<< parameters.vectors-branch >>
       - go/install-gotestsum:
           gobin: $HOME/.local/bin
           version: 0.5.2
+      - run:
+          name: install statediff globally
+          command: |
+            ## statediff is optional; we succeed even if compilation fails.
+            mkdir -p /tmp/statediff
+            git clone https://github.com/filecoin-project/statediff.git /tmp/statediff
+            cd /tmp/statediff
+            go install ./cmd/statediff || exit 0
       - run:
           name: go test
           environment:
@@ -249,6 +254,25 @@ jobs:
           path: /tmp/test-reports
       - store_artifacts:
           path: /tmp/test-artifacts/conformance-coverage.html
+  build-lotus-soup:
+    description: |
+      Compile `lotus-soup` Testground test plan using the current version of Lotus.
+    parameters:
+      <<: *test-params
+    executor: << parameters.executor >>
+    steps:
+      - install-deps
+      - prepare
+      - run: cd extern/oni && git submodule sync
+      - run: cd extern/oni && git submodule update --init
+      - run: cd extern/filecoin-ffi && make
+      - run:
+          name: "replace lotus, filecoin-ffi, blst and fil-blst deps"
+          command: cd extern/oni/lotus-soup && go mod edit -replace github.com/filecoin-project/lotus=../../../ && go mod edit -replace github.com/filecoin-project/filecoin-ffi=../../filecoin-ffi && go mod edit -replace github.com/supranational/blst=../../fil-blst/blst && go mod edit -replace github.com/filecoin-project/fil-blst=../../fil-blst
+      - run:
+          name: "build lotus-soup testplan"
+          command: pushd extern/oni/lotus-soup && go build -tags=testground .
+
 
   build-macos:
     description: build darwin lotus binary
@@ -395,7 +419,8 @@ workflows:
   version: 2.1
   ci:
     jobs:
-      - lint-all
+      - lint-all:
+          concurrency: "16"   # expend all docker 2xlarge CPUs.
       - mod-tidy-check
       - gofmt
       - cbor-gen-check
@@ -422,6 +447,7 @@ workflows:
           test-suite-name: conformance-bleeding-edge
           packages: "./conformance"
           vectors-branch: master
+      - build-lotus-soup
       - build-debug
       - build-all:
           requires:

.github/CODEOWNERS

@@ -0,0 +1,15 @@
+## filecoin-project/lotus CODEOWNERS
+## Refer to https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners.
+##
+## These users or groups will be automatically assigned as reviewers every time
+## a PR is submitted that modifies code in the specified locations.
+##
+## The Lotus repo configuration requires that at least ONE codeowner approves
+## the PR before merging.
+
+### Global owners.
+*                       @magik6k @whyrusleeping @Kubuxu
+
+### Conformance testing.
+conformance/            @raulk
+extern/test-vectors     @raulk

.gitignore

@@ -10,6 +10,8 @@
 /lotus-fountain
 /lotus-stats
 /lotus-bench
+/lotus-gateway
+/lotus-pcr
 /bench.json
 /lotuspond/front/node_modules
 /lotuspond/front/build

.gitmodules

@@ -8,3 +8,9 @@
 [submodule "extern/test-vectors"]
 	path = extern/test-vectors
 	url = https://github.com/filecoin-project/test-vectors.git
+[submodule "extern/fil-blst"]
+	path = extern/fil-blst
+	url = https://github.com/filecoin-project/fil-blst.git
+[submodule "extern/oni"]
+	path = extern/oni
+	url = https://github.com/filecoin-project/oni

CHANGELOG.md

@@ -1,5 +1,194 @@
 # Lotus changelog
 
+# 0.7.0 / 2020-09-10
+
+This consensus-breaking release of Lotus is designed to test a network upgrade on the space race testnet. The changes that break consensus are:
+
+- Upgrading the Drand network used from the test Drand network to the League of Entropy main drand network. This is the same Drand network that will be used in the Filecoin mainnet.
+- Upgrading to specs-actors v0.9.8, which adds a new method to the Multisig actor.
+
+## Changes
+
+#### Core Lotus
+
+- Fix IsAncestorOf (https://github.com/filecoin-project/lotus/pull/3717)
+- Update to specs-actors v0.9.8 (https://github.com/filecoin-project/lotus/pull/3725)
+- Increase chain throughput by 20% (https://github.com/filecoin-project/lotus/pull/3732)
+- Updare to go-libp2p-pubsub `master` (https://github.com/filecoin-project/lotus/pull/3735)
+- Drand upgrade (https://github.com/filecoin-project/lotus/pull/3670)
+- Multisig API additions (https://github.com/filecoin-project/lotus/pull/3590)
+
+#### Storage Miner 
+
+- Increase the number of times precommit2 is attempted before moving back to precommit1 (https://github.com/filecoin-project/lotus/pull/3720)
+
+#### Message pool
+
+- Relax mpool add strictness checks for local pushes (https://github.com/filecoin-project/lotus/pull/3724)
+
+
+#### Maintenance
+
+- Fix devnets (https://github.com/filecoin-project/lotus/pull/3712)
+- Fix(chainwatch): compare prev miner with cur miner (https://github.com/filecoin-project/lotus/pull/3715)
+- CI: fix statediff build; make optional (https://github.com/filecoin-project/lotus/pull/3729)
+- Feat: Chaos abort (https://github.com/filecoin-project/lotus/pull/3733)
+
+## Contributors
+
+The following contributors had commits go into this release.
+We are grateful for every contribution!
+
+| Contributor        | Commits | Lines ±       |
+|--------------------|---------|---------------|
+| arajasek           | 28      | +1144/-239    |
+| Kubuxu             | 19      | +452/-261     |
+| whyrusleeping      | 13      | +456/-87      |
+| vyzo               | 11      | +318/-20      |
+| raulk              | 10      | +1289/-350    |
+| magik6k            | 6       | +188/-55      |
+| dirkmc             | 3       | +31/-8        |
+| alanshaw           | 3       | +176/-37      |
+| Stebalien          | 2       | +9/-12        |
+| lanzafame          | 1       | +1/-1         |
+| frrist             | 1       | +1/-1         |
+| mishmosh           | 1       | +1/-1         |
+| nonsense           | 1       | +1/-0         |
+
+# 0.6.2 / 2020-09-09
+
+This release introduces some critical fixes to message selection and gas estimation logic. It also adds the ability for nodes to mark a certain tipset as checkpointed, as well as various minor improvements and bugfixes.
+
+## Changes
+
+#### Messagepool 
+
+- Warn when optimal selection fails to pack a block and we fall back to random selection (https://github.com/filecoin-project/lotus/pull/3708)
+- Add basic command for printing gas performance of messages in the mpool (https://github.com/filecoin-project/lotus/pull/3701)
+- Adjust optimal selection to always try to fill blocks (https://github.com/filecoin-project/lotus/pull/3685)
+- Fix very minor bug in repub baseFeeLowerBound (https://github.com/filecoin-project/lotus/pull/3663)
+- Add an auto flag to mpool replace (https://github.com/filecoin-project/lotus/pull/3676)
+- Fix mpool optimal selection packing failure (https://github.com/filecoin-project/lotus/pull/3698)
+
+#### Core Lotus
+
+- Don't use latency as initital estimate for blocksync (https://github.com/filecoin-project/lotus/pull/3648)
+- Add niceSleep 1 second when drand errors (https://github.com/filecoin-project/lotus/pull/3664)
+- Fix isChainNearSync check in block validator (https://github.com/filecoin-project/lotus/pull/3650)
+- Add peer to peer manager before fetching the tipset (https://github.com/filecoin-project/lotus/pull/3667)
+- Add StageFetchingMessages to sync status (https://github.com/filecoin-project/lotus/pull/3668)
+- Pass tipset through upgrade logic (https://github.com/filecoin-project/lotus/pull/3673)
+- Allow nodes to mark tipsets as checkpointed (https://github.com/filecoin-project/lotus/pull/3680)
+- Remove hard-coded late-fee in window PoSt (https://github.com/filecoin-project/lotus/pull/3702)
+- Gas: Fix median calc (https://github.com/filecoin-project/lotus/pull/3686)
+
+#### Storage
+
+- Storage manager: bail out with an error if unsealed cid is undefined (https://github.com/filecoin-project/lotus/pull/3655)
+- Storage: return true from Sealer.ReadPiece() on success (https://github.com/filecoin-project/lotus/pull/3657)
+
+#### Maintenance
+
+- Resolve lotus, test-vectors, statediff dependency cycle (https://github.com/filecoin-project/lotus/pull/3688)
+- Paych: add docs on how to use paych status (https://github.com/filecoin-project/lotus/pull/3690)
+- Initial CODEOWNERS (https://github.com/filecoin-project/lotus/pull/3691)
+
+# 0.6.1 / 2020-09-08
+
+This optional release introduces a minor improvement to the sync process, ensuring nodes don't fall behind and then resync.
+
+## Changes
+
+- Update `test-vectors` (https://github.com/filecoin-project/lotus/pull/3645)
+- Revert "only subscribe to pubsub topics once we are synced" (https://github.com/filecoin-project/lotus/pull/3643)
+
+# 0.6.0 / 2020-09-07
+
+This consensus-breaking release of Lotus is designed to test a network upgrade on the space race testnet. The changes that break consensus are:
+
+- Tweaking of some cryptoecon parameters in specs-actors 0.9.7 (https://github.com/filecoin-project/specs-actors/releases/tag/v0.9.7)
+- Rebalancing FIL distribution to make testnet FIL scarce, which prevents base fee spikes and sets better expectations for mainnet
+
+This release also introduces many improvements to Lotus! Among them are a new version of go-fil-markets that supports non-blocking retrieval, various spam reduction measures in the messagepool and p2p logic, and UX improvements to payment channels, dealmaking, and state inspection.
+
+## Changes
+
+#### Core Lotus and dependencies
+
+- Implement faucet funds reallocation logic (https://github.com/filecoin-project/lotus/pull/3632)
+- Network upgrade: Upgrade to correct fork threshold (https://github.com/filecoin-project/lotus/pull/3628)
+- Update to specs 0.9.7 and markets 0.6.0 (https://github.com/filecoin-project/lotus/pull/3627)
+- Network upgrade: Perform base fee tamping (https://github.com/filecoin-project/lotus/pull/3623)
+- Chain events: if cache best() is nil, return chain head (https://github.com/filecoin-project/lotus/pull/3611)
+- Update to specs actors v0.9.6 (https://github.com/filecoin-project/lotus/pull/3603)
+
+#### Messagepool
+
+- Temporarily allow negative chains (https://github.com/filecoin-project/lotus/pull/3625)
+- Improve publish/republish logic (https://github.com/filecoin-project/lotus/pull/3592)
+- Fix selection bug; priority messages were not included if other chains were negative (https://github.com/filecoin-project/lotus/pull/3580)
+- Add defensive check for minimum GasFeeCap for inclusion within the next 20 blocks (https://github.com/filecoin-project/lotus/pull/3579)
+- Add additional info about gas premium (https://github.com/filecoin-project/lotus/pull/3578)
+- Fix GasPremium capping logic  (https://github.com/filecoin-project/lotus/pull/3552)
+
+#### Payment channels 
+
+- Get available funds by address or by from/to (https://github.com/filecoin-project/lotus/pull/3547)
+- Create `lotus paych status` command (https://github.com/filecoin-project/lotus/pull/3523)
+- Rename CLI command from "paych get" to "paych add-funds" (https://github.com/filecoin-project/lotus/pull/3520)
+
+#### Peer-to-peer
+
+- Only subscribe to pubsub topics once we are synced (https://github.com/filecoin-project/lotus/pull/3602)
+- Reduce mpool add failure log spam (https://github.com/filecoin-project/lotus/pull/3562)
+- Republish messages even if the chains have negative performance(https://github.com/filecoin-project/lotus/pull/3557)
+- Adjust gossipsub gossip factor (https://github.com/filecoin-project/lotus/pull/3556)
+- Integrate pubsub Random Early Drop (https://github.com/filecoin-project/lotus/pull/3518)
+
+#### Miscellaneous
+
+- Fix panic in OnDealExpiredSlashed (https://github.com/filecoin-project/lotus/pull/3553)
+- Robustify state manager against holes in actor method numbers (https://github.com/filecoin-project/lotus/pull/3538)
+
+#### UX
+
+- VM: Fix an error message (https://github.com/filecoin-project/lotus/pull/3608)
+- Documentation: Batch replacement,update lotus-storage-miner to lotus-miner (https://github.com/filecoin-project/lotus/pull/3571)
+- CLI: Robust actor lookup (https://github.com/filecoin-project/lotus/pull/3535)
+- Add agent flag to net peers (https://github.com/filecoin-project/lotus/pull/3534)
+- Add watch option to storage-deals list (https://github.com/filecoin-project/lotus/pull/3527)
+
+#### Testing & tooling
+
+- Decommission chain-validation (https://github.com/filecoin-project/lotus/pull/3606)
+- Metrics: add expected height metric (https://github.com/filecoin-project/lotus/pull/3586)
+- PCR: Use current tipset during refund (https://github.com/filecoin-project/lotus/pull/3570)
+- Lotus-shed: Add math command (https://github.com/filecoin-project/lotus/pull/3568)
+- PCR: Add tipset aggergation (https://github.com/filecoin-project/lotus/pull/3565)- Fix broken paych tests (https://github.com/filecoin-project/lotus/pull/3551)
+- Make chain export ~1000x times faster (https://github.com/filecoin-project/lotus/pull/3533)
+- Chainwatch: Stop SyncIncomingBlocks from leaking into chainwatch processing; No panics during processing (https://github.com/filecoin-project/lotus/pull/3526)
+- Conformance: various changes (https://github.com/filecoin-project/lotus/pull/3521)
+
+# 0.5.10 / 2020-09-03
+
+This patch includes a crucial fix to the message pool selection logic, strongly disfavouring messages that might cause a miner penalty.
+
+## Changes
+
+- Fix calculation of GasReward in messagepool (https://github.com/filecoin-project/lotus/pull/3528)
+
+# 0.5.9 / 2020-09-03
+
+This patch includes a hotfix to the `GasEstimateFeeCap` method, capping the estimated fee to a reasonable level by default.
+
+## Changes 
+
+- Added target height to sync wait (https://github.com/filecoin-project/lotus/pull/3502)
+- Disable codecov annotations (https://github.com/filecoin-project/lotus/pull/3514)
+- Cap fees to reasonable level by default (https://github.com/filecoin-project/lotus/pull/3516)
+- Add APIs and command to inspect bandwidth usage (https://github.com/filecoin-project/lotus/pull/3497)
+- Track expected nonce in mpool, ignore messages with large nonce gaps (https://github.com/filecoin-project/lotus/pull/3450)
+
 # 0.5.8 / 2020-09-02
 
 This patch includes some bugfixes to the sector sealing process, and updates go-fil-markets. It also improves the performance of blocksync, adds a method to export chain state trees, and improves chainwatch.

Makefile

@@ -92,6 +92,12 @@ lotus-shed: $(BUILD_DEPS)
 .PHONY: lotus-shed
 BINS+=lotus-shed
 
+lotus-gateway: $(BUILD_DEPS)
+	rm -f lotus-gateway
+	go build $(GOFLAGS) -o lotus-gateway ./cmd/lotus-gateway
+.PHONY: lotus-gateway
+BINS+=lotus-gateway
+
 build: lotus lotus-miner lotus-worker
 	@[[ $$(type -P "lotus") ]] && echo "Caution: you have \
 an existing lotus binary in your PATH. This may cause problems if you don't run 'sudo make install'" || true

api/api_common.go

@@ -30,6 +30,7 @@ type Common interface {
 	NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)
 	NetPubsubScores(context.Context) ([]PubsubScore, error)
 	NetAutoNatStatus(context.Context) (NatInfo, error)
+	NetAgentVersion(ctx context.Context, p peer.ID) (string, error)
 
 	// NetBandwidthStats returns statistics about the nodes total bandwidth
 	// usage and current rate across all peers and protocols.