Commit batcher #584

Markuu-s · 2022-02-08T11:37:51Z

Description of the Change

Benefits

Possible Drawbacks

Usage Examples or Tests [optional]

Alternate Designs [optional]

Signed-off-by: Markuu-s <qwert11223381@gmail.com>

wer1st · 2022-02-08T11:54:29Z

test/core/miner/commit_batcher_test.cpp

+  using fc::mining::types::FeeConfig;
+  using fc::mining::types::PaddedPieceSize;
+  using fc::mining::types::Piece;
+  using fc::mining::types::PieceInfo;


Suggested change

using fc::mining::types::FeeConfig;

using fc::mining::types::PaddedPieceSize;

using fc::mining::types::Piece;

using fc::mining::types::PieceInfo;

using types::FeeConfig;

using types::PaddedPieceSize;

using types::Piece;

using types::PieceInfo;

wer1st · 2022-02-08T11:55:33Z

core/miner/storage_fsm/commit_batcher.hpp

+#pragma once
+#include "api/full_node/node_api.hpp"


Suggested change

#pragma once

#include "api/full_node/node_api.hpp"

#pragma once

#include "api/full_node/node_api.hpp"

wer1st · 2022-02-08T11:56:26Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+#pragma once
+#include "commit_batcher_impl.hpp"


Suggested change

#pragma once

#include "commit_batcher_impl.hpp"

#include "commit_batcher_impl.hpp"

wer1st · 2022-02-08T11:56:59Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+  using fc::BytesIn;
+  using fc::proofs::ProofEngine;


Suggested change

using fc::BytesIn;

using fc::proofs::ProofEngine;

using proofs::ProofEngine;

wer1st · 2022-02-08T11:57:18Z

core/miner/storage_fsm/impl/commit_batcher_impl.hpp

+#pragma once
+#include <libp2p/basic/scheduler.hpp>


Suggested change

#pragma once

#include <libp2p/basic/scheduler.hpp>

#pragma once

#include <libp2p/basic/scheduler.hpp>

wer1st · 2022-02-08T14:19:32Z

core/miner/storage_fsm/impl/commit_batcher_impl.hpp

+        const SectorNumber &sector_number, const TipsetKey &tip_set_key);
+  };
+
+}  // namespace fc::mining


missed empty line

wer1st · 2022-02-08T14:21:44Z

core/miner/storage_fsm/impl/commit_batcher_impl.hpp

+
+    void forceSend() override;
+
+    void setCommitCutoff(const ChainEpoch &current_epoch,


Probably we want to make it private

wer1st · 2022-02-08T14:24:45Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+    OUTCOME_TRY(tipset, api_->ChainGetTipSet(head->key));
+    const BigInt base_fee = tipset->blks[0].parent_base_fee;
+
+    TokenAmount agg_fee_raw = AggregateProveCommitNetworkFee(


Suggested change

TokenAmount agg_fee_raw = AggregateProveCommitNetworkFee(

const TokenAmount agg_fee_raw = AggregateProveCommitNetworkFee(

wer1st · 2022-02-08T14:25:21Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+    TokenAmount agg_fee = bigdiv(agg_fee_raw * agg_fee_num_, agg_fee_den_);
+    TokenAmount need_funds = collateral + agg_fee;
+    TokenAmount good_funds = max_fee + need_funds;


Suggested change

TokenAmount agg_fee = bigdiv(agg_fee_raw * agg_fee_num_, agg_fee_den_);

TokenAmount need_funds = collateral + agg_fee;

TokenAmount good_funds = max_fee + need_funds;

const TokenAmount agg_fee = bigdiv(agg_fee_raw * agg_fee_num_, agg_fee_den_);

const TokenAmount need_funds = collateral + agg_fee;

const TokenAmount good_funds = max_fee + need_funds;

wer1st · 2022-02-08T14:27:20Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+                                          const SectorInfo &sector_info) {
+    ChainEpoch cutoff_epoch =
+        sector_info.ticket_epoch
+        + static_cast<int64_t>(kEpochsInDay + kChainFinality);


Suggested change

+ static_cast<int64_t>(kEpochsInDay + kChainFinality);

+ static_cast<ChainEpoch>(kEpochsInDay + kChainFinality);

Alexey-N-Chernyshov · 2022-02-09T09:41:30Z

core/miner/storage_fsm/commit_batcher.hpp

+  struct AggregateInput {
+    Proof proof;
+    AggregateSealVerifyInfo info;
+    RegisteredSealProof spt;


Suggested change

RegisteredSealProof spt;

RegisteredSealProof seal_proof_type{RegisteredSealProof::kUndefined};

Alexey-N-Chernyshov · 2022-02-09T09:42:13Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once


Suggested change

#pragma once

Not in .cpp file.

Alexey-N-Chernyshov · 2022-02-09T09:44:26Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+    collateral = collateral + pci.precommit_deposit;
+    collateral = std::max(BigInt(0), collateral);
+
+    return collateral;


Suggested change

return collateral;

return std::move(collateral);

Alexey-N-Chernyshov · 2022-02-09T09:44:42Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+    const BigInt base_fee = tipset->blks[0].parent_base_fee;
+
+    TokenAmount agg_fee_raw = AggregateProveCommitNetworkFee(
+        infos.size(), base_fee);  // TODO change to aggregateNetworkFee


Suggested change

infos.size(), base_fee); // TODO change to aggregateNetworkFee

infos.size(), base_fee); // TODO((Markuuu-s) change to aggregateNetworkFee

Alexey-N-Chernyshov · 2022-02-09T09:44:50Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+    }
+
+    const ActorId mid = miner_address_.getId();
+    // TODO maybe long (AggregateSealProofs)


Suggested change

// TODO maybe long (AggregateSealProofs)

// TODO(Markuuu-s) maybe long (AggregateSealProofs)

Alexey-N-Chernyshov · 2022-02-09T09:51:45Z

core/miner/storage_fsm/types.hpp

-    return lhs.publish_cid == rhs.publish_cid && lhs.deal_id == rhs.deal_id
-           && lhs.deal_schedule == rhs.deal_schedule
-           && lhs.is_keep_unsealed == rhs.is_keep_unsealed;
+    if (lhs.is_keep_unsealed == rhs.is_keep_unsealed) {


What is the difference? Please, revert.

Alexey-N-Chernyshov · 2022-02-09T10:03:20Z

core/miner/storage_fsm/impl/commit_batcher_impl.hpp

+    const BigInt agg_fee_num_ = BigInt(110);
+    const BigInt agg_fee_den_ = BigInt(100);
+    const RegisteredAggregationProof arp_ = RegisteredAggregationProof(0);


Please, use const naming for constants.

Alexey-N-Chernyshov · 2022-02-09T10:18:49Z

test/core/miner/commit_batcher_test.cpp

+  using BatcherCallbackMock =
+      std::function<void(const outcome::result<CID> &cid)>;


Suggested change

using BatcherCallbackMock =

std::function<void(const outcome::result<CID> &cid)>;

It is a fc::mining::CommitCallback

Alexey-N-Chernyshov · 2022-02-09T10:19:07Z

test/core/miner/commit_batcher_test.cpp

+    ActorId miner_id_;
+    std::shared_ptr<ProofEngineMock> proof_;
+    std::shared_ptr<FeeConfig> fee_config_;
+    BatcherCallbackMock callback_mock_;


Suggested change

BatcherCallbackMock callback_mock_;

CommitCallback callback_mock_;

wer1st · 2022-03-14T13:29:18Z

core/vm/actor/builtin/v6/monies.hpp

+#pragma once
+#include "const.hpp"


Suggested change

#pragma once

#include "const.hpp"

#pragma once

#include "const.hpp"

ortyomka · 2022-03-24T08:49:02Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+ */
+
+#pragma once
+#include "commit_batcher_impl.hpp"


Please use full path

ortyomka · 2022-03-24T09:06:42Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+
+  void CommitBatcherImpl::reschedule(std::chrono::milliseconds time) {
+    handle_ = scheduler_->scheduleWithHandle(
+        [&]() {


Could be segfault

ortyomka · 2022-03-24T09:22:07Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+  outcome::result<CID> CommitBatcherImpl::sendBatch(
+      const MapPairStorage &pair_storage_for_send) {
+    if (pair_storage_for_send.empty()) {
+      cutoff_start_ = std::chrono::system_clock::now();


We can remove it, because after sendBatch call we set it to now

ortyomka · 2022-03-24T09:22:35Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+      const MapPairStorage &pair_storage_for_send) {
+    if (pair_storage_for_send.empty()) {
+      cutoff_start_ = std::chrono::system_clock::now();
+      return ERROR_TEXT("Empty Batcher");


Does it error, maybe just outcome::success()?

ortyomka · 2022-03-24T09:23:48Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+      proofsSpan.push_back(gsl::make_span(proof));
+    }
+
+    OUTCOME_TRY(proof_->aggregateSealProofs(aggregate_seal, proofsSpan));


Suggested change

OUTCOME_TRY(proof_->aggregateSealProofs(aggregate_seal, proofsSpan));

// TODO(Markuuu-s) maybe long (AggregateSealProofs)

OUTCOME_TRY(proof_->aggregateSealProofs(aggregate_seal, proofsSpan));

ortyomka · 2022-03-24T09:23:59Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+    }
+
+    const ActorId mid = miner_address_.getId();
+    // TODO maybe long (AggregateSealProofs)


Suggested change

// TODO maybe long (AggregateSealProofs)

ortyomka · 2022-03-24T09:26:40Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+    for (const Proof &proof : proofs) {
+      proofsSpan.push_back(gsl::make_span(proof));
+    }


proofs is redundant. You could fill proofsSpan directly

ortyomka · 2022-03-24T09:27:26Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+                                                 MethodParams{encode}),
+                    kPushNoSpec));
+
+    cutoff_start_ = std::chrono::system_clock::now();


Suggested change

cutoff_start_ = std::chrono::system_clock::now();

core/vm/actor/builtin/v6/monies.hpp

turuslan · 2022-03-25T05:46:26Z

core/vm/actor/builtin/v6/monies.hpp

+    const TokenAmount effectiveGasFee = std::max(base_fee, kBatchBalancer);
+    const TokenAmount networkFeeNum =
+        effectiveGasFee * kEstimatedSinglePreCommitGasUsage * aggregate_size
+        * kBatchDiscountNumerator;
+    return bigdiv(networkFeeNum, kBatchDiscountDenominator);


Suggested change

const TokenAmount effectiveGasFee = std::max(base_fee, kBatchBalancer);

const TokenAmount networkFeeNum =

effectiveGasFee * kEstimatedSinglePreCommitGasUsage * aggregate_size

* kBatchDiscountNumerator;

return bigdiv(networkFeeNum, kBatchDiscountDenominator);

return aggregateNetworkFee(aggregate_size, kEstimatedSinglePreCommitGasUsage, base_fee);

turuslan · 2022-03-25T06:23:29Z

test/core/miner/commit_batcher_test.cpp

+    ActorId miner_id_;
+    std::shared_ptr<ProofEngineMock> proof_;
+    std::shared_ptr<FeeConfig> fee_config_;
+    BatcherCallbackMock callback_mock_;


Suggested change

BatcherCallbackMock callback_mock_;

MockStdFunction<CommitCallback> callback_mock_;

example

turuslan · 2022-03-25T06:25:51Z

test/core/miner/commit_batcher_test.cpp

+  /**
+   * @given 2 commits and max_size_callback is 2
+   * @when send the 2 commits
+   * @then the result should be 2 messages in message pool with pair of commits


Suggested change

* @then the result should be 2 messages in message pool with pair of commits

* @then the result should be 1 message in message pool with 2 commits

turuslan · 2022-03-25T06:30:42Z

test/core/miner/commit_batcher_test.cpp

+  }
+
+  /**
+   * @given 4 commits


There are 3 commits currently: 1 before first timeout, and two after

turuslan · 2022-03-25T06:34:34Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+        fee_config_(std::move(fee_config)),
+        proof_(std::move(proof)),
+        address_selector_(std::move(address_selector)) {
+    cutoff_start_ = std::chrono::system_clock::now();


maybe use scheduler

Suggested change

cutoff_start_ = std::chrono::system_clock::now();

cutoff_start_ = scheduler_->now();

turuslan · 2022-03-25T07:06:22Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+                                          const SectorInfo &sector_info) {
+    ChainEpoch cutoff_epoch =
+        sector_info.ticket_epoch
+        + static_cast<int64_t>(kEpochsInDay + kChainFinality);


it maybe 30*.
may move to miner policy header

https://github.com/filecoin-project/specs-actors/blob/v7.0.0/actors/builtin/miner/policy.go#L152-L164

turuslan · 2022-03-25T07:08:32Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+      forceSend();
+    } else {
+      const auto temp_cutoff = std::chrono::milliseconds(
+          (cutoff_epoch - current_epoch) * kEpochDurationSeconds);


Suggested change

(cutoff_epoch - current_epoch) * kEpochDurationSeconds);

(cutoff_epoch - current_epoch) * kBlockDelaySecs);

turuslan · 2022-03-25T07:09:50Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+                api_->StateMinerInitialPledgeCollateral(
+                    miner_address_, pci.info, tip_set_key));
+
+    collateral = collateral + pci.precommit_deposit;


Suggested change

collateral = collateral + pci.precommit_deposit;

collateral = collateral - pci.precommit_deposit;

turuslan · 2022-03-25T07:13:32Z

core/miner/storage_fsm/impl/commit_batcher_impl.cpp

+    cutoff_start_ = std::chrono::system_clock::now();
+    closest_cutoff_ = max_delay_;
+    reschedule(max_delay_);


maybe better to store queued commits with cutoff, sorted by cutoff, and reschedule by minimal cutoff from queue

turuslan · 2022-03-25T07:24:55Z

core/miner/storage_fsm/impl/commit_batcher_impl.hpp

+
+    const BigInt agg_fee_num_ = BigInt(110);
+    const BigInt agg_fee_den_ = BigInt(100);
+    const RegisteredAggregationProof arp_ = RegisteredAggregationProof(0);


Suggested change

const RegisteredAggregationProof arp_ = RegisteredAggregationProof(0);

const RegisteredAggregationProof arp_ = RegisteredAggregationProof::SnarkPackV1;

Markuu-s added 10 commits January 17, 2022 13:12

Add CommitBatcher.

d52eb62

Added setCommitCuttof.

cf975e7

Added sendBatch.

e7ad727

Signed-off-by: Markuu-s <qwert11223381@gmail.com>

Change unionStorage.

5cb3422

Signed-off-by: Markuu-s <qwert11223381@gmail.com>

Need save change.

2abda47

Signed-off-by: Markuu-s <qwert11223381@gmail.com>

Add completely sendBatch.

cbf7438

Signed-off-by: Markuu-s <qwert11223381@gmail.com>

Add tests

ed1d438

Signed-off-by: Markuu-s <qwert11223381@gmail.com>

Fix sendBatch.

fe60a37

Signed-off-by: Markuu-s <qwert11223381@gmail.com>

Merge branch 'master' into feature/commit_batcher

ff742e9

Small fixes.

6d0d80f

Signed-off-by: Markuu-s <qwert11223381@gmail.com>

Markuu-s requested review from Alexey-N-Chernyshov, wer1st, turuslan, ortyomka and Elestrias February 8, 2022 11:37

Fix aggregate.

d64c1d0

Signed-off-by: Markuu-s <qwert11223381@gmail.com>

wer1st reviewed Feb 8, 2022

View reviewed changes

Alexey-N-Chernyshov reviewed Feb 9, 2022

View reviewed changes

wer1st reviewed Mar 14, 2022

View reviewed changes

ortyomka reviewed Mar 24, 2022

View reviewed changes

turuslan reviewed Mar 25, 2022

View reviewed changes

	#pragma once
	#include "commit_batcher_impl.hpp"
	#include "commit_batcher_impl.hpp"

	using fc::BytesIn;
	using fc::proofs::ProofEngine;
	using proofs::ProofEngine;


		void forceSend() override;

		void setCommitCutoff(const ChainEpoch &current_epoch,

	TokenAmount agg_fee_raw = AggregateProveCommitNetworkFee(
	const TokenAmount agg_fee_raw = AggregateProveCommitNetworkFee(

	+ static_cast<int64_t>(kEpochsInDay + kChainFinality);
	+ static_cast<ChainEpoch>(kEpochsInDay + kChainFinality);

	RegisteredSealProof spt;
	RegisteredSealProof seal_proof_type{RegisteredSealProof::kUndefined};

	infos.size(), base_fee); // TODO change to aggregateNetworkFee
	infos.size(), base_fee); // TODO((Markuuu-s) change to aggregateNetworkFee

	// TODO maybe long (AggregateSealProofs)
	// TODO(Markuuu-s) maybe long (AggregateSealProofs)

		using BatcherCallbackMock =
		std::function<void(const outcome::result<CID> &cid)>;

	BatcherCallbackMock callback_mock_;
	CommitCallback callback_mock_;

	OUTCOME_TRY(proof_->aggregateSealProofs(aggregate_seal, proofsSpan));
	// TODO(Markuuu-s) maybe long (AggregateSealProofs)
	OUTCOME_TRY(proof_->aggregateSealProofs(aggregate_seal, proofsSpan));

	BatcherCallbackMock callback_mock_;
	MockStdFunction<CommitCallback> callback_mock_;

	* @then the result should be 2 messages in message pool with pair of commits
	* @then the result should be 1 message in message pool with 2 commits

	cutoff_start_ = std::chrono::system_clock::now();
	cutoff_start_ = scheduler_->now();

	(cutoff_epoch - current_epoch) * kEpochDurationSeconds);
	(cutoff_epoch - current_epoch) * kBlockDelaySecs);

	collateral = collateral + pci.precommit_deposit;
	collateral = collateral - pci.precommit_deposit;

	const RegisteredAggregationProof arp_ = RegisteredAggregationProof(0);
	const RegisteredAggregationProof arp_ = RegisteredAggregationProof::SnarkPackV1;

Commit batcher #584

Are you sure you want to change the base?

Commit batcher #584

Conversation

Markuu-s commented Feb 8, 2022

Description of the Change

Benefits

Possible Drawbacks

Usage Examples or Tests [optional]

Alternate Designs [optional]

Choose a reason for hiding this comment

Choose a reason for hiding this comment

wer1st Feb 8, 2022 • edited Loading

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

turuslan Mar 25, 2022 • edited Loading

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

wer1st Feb 8, 2022 •

edited

Loading

turuslan Mar 25, 2022 •

edited

Loading