Skip to content

Commit

Permalink
Merge "[FAB-2897] make txn index within block starting at 0"
Browse files Browse the repository at this point in the history
  • Loading branch information
@binhn
binhn authored and Gerrit Code Review committed Apr 19, 2017
2 parents 360d415 + 47f4ee3 commit 03fe133
Show file tree
Hide file tree
Showing 8 changed files with 77 additions and 74 deletions.
4 changes: 2 additions & 2 deletions common/ledger/blkstorage/fsblkstorage/blockfile_mgr_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,8 +165,8 @@ func TestBlockfileMgrGetTxByBlockNumTranNum(t *testing.T) {
blkfileMgrWrapper.addBlocks(blocks)
for blockIndex, blk := range blocks {
for tranIndex, txEnvelopeBytes := range blk.Data.Data {
// blockNum starts with 1, tranNum starts with 1
txEnvelopeFromFileMgr, err := blkfileMgrWrapper.blockfileMgr.retrieveTransactionByBlockNumTranNum(uint64(blockIndex), uint64(tranIndex+1))
// blockNum and tranNum both start with 0
txEnvelopeFromFileMgr, err := blkfileMgrWrapper.blockfileMgr.retrieveTransactionByBlockNumTranNum(uint64(blockIndex), uint64(tranIndex))
testutil.AssertNoError(t, err, "Error while retrieving tx from blkfileMgr")
txEnvelope, err := putil.GetEnvelopeFromBlock(txEnvelopeBytes)
testutil.AssertNoError(t, err, "Error while unmarshalling tx")
Expand Down
4 changes: 2 additions & 2 deletions common/ledger/blkstorage/fsblkstorage/blockindex.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,12 +132,12 @@ func (index *blockIndex) indexBlock(blockIdxInfo *blockIdxInfo) error {
if _, ok := index.indexItemsMap[blkstorage.IndexableAttrBlockNumTranNum]; ok {
for txIterator, txoffset := range txOffsets {
txFlp := newFileLocationPointer(flp.fileSuffixNum, flp.offset, txoffset.loc)
logger.Debugf("Adding txLoc [%s] for tx number:[%d] ID: [%s] to blockNumTranNum index", txFlp, txIterator+1, txoffset.txID)
logger.Debugf("Adding txLoc [%s] for tx number:[%d] ID: [%s] to blockNumTranNum index", txFlp, txIterator, txoffset.txID)
txFlpBytes, marshalErr := txFlp.marshal()
if marshalErr != nil {
return marshalErr
}
batch.Put(constructBlockNumTranNumKey(blockIdxInfo.blockNum, uint64(txIterator+1)), txFlpBytes)
batch.Put(constructBlockNumTranNumKey(blockIdxInfo.blockNum, uint64(txIterator)), txFlpBytes)
}
}

Expand Down
2 changes: 1 addition & 1 deletion common/ledger/blkstorage/fsblkstorage/blockindex_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ func testBlockIndexSelectiveIndexing(t *testing.T, indexItems []blkstorage.Index
}

//test 'retrieveTrasnactionsByBlockNumTranNum
txEnvelope2, err := blockfileMgr.retrieveTransactionByBlockNumTranNum(0, 1)
txEnvelope2, err := blockfileMgr.retrieveTransactionByBlockNumTranNum(0, 0)
if testutil.Contains(indexItems, blkstorage.IndexableAttrBlockNumTranNum) {
testutil.AssertNoError(t, err, "Error while retrieving tx by blockNum and tranNum")
txEnvelopeBytes2 := blocks[0].Data.Data[0]
Expand Down
6 changes: 3 additions & 3 deletions core/ledger/kvledger/history/historydb/historyleveldb/historyleveldb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,13 +103,12 @@ func (historyDB *historyDB) Commit(block *common.Block) error {

// write each tran's write set to history db
for _, envBytes := range block.Data.Data {
tranNo++

// If the tran is marked as invalid, skip it
// Note, tranNo starts at 1 for height, while tranIndex starts at 0 for invalid array
if txsFilter.IsInvalid(int(tranNo) - 1) {
if txsFilter.IsInvalid(int(tranNo)) {
logger.Debugf("Channel [%s]: Skipping history write for invalid transaction number %d",
historyDB.dbName, tranNo)
tranNo++
continue
}

Expand Down Expand Up @@ -163,6 +162,7 @@ func (historyDB *historyDB) Commit(block *common.Block) error {
} else {
logger.Debugf("Skipping transaction [%d] since it is not an endorsement transaction\n", tranNo)
}
tranNo++
}

// add savepoint for recovery purpose
Expand Down
7 changes: 5 additions & 2 deletions core/ledger/kvledger/txmgmt/txmgr/commontests/txmgr_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (
"fmt"
"testing"

"os"

"github.com/hyperledger/fabric/common/ledger/testutil"
"github.com/hyperledger/fabric/core/ledger"
"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/version"
Expand All @@ -29,6 +31,7 @@ import (

func TestMain(m *testing.M) {
ledgertestutil.SetupCoreYAMLConfig("./../../../../../../peer")
os.Exit(m.Run())
}

func TestTxSimulatorWithNoExistingData(t *testing.T) {
Expand Down Expand Up @@ -106,9 +109,9 @@ func testTxSimulatorWithExistingData(t *testing.T, env testEnv) {

// verify the versions of keys in persistence
vv, _ := env.getVDB().GetState("ns1", "key1")
testutil.AssertEquals(t, vv.Version, version.NewHeight(1, 1))
testutil.AssertEquals(t, vv.Version, version.NewHeight(1, 0))
vv, _ = env.getVDB().GetState("ns1", "key2")
testutil.AssertEquals(t, vv.Version, version.NewHeight(0, 1))
testutil.AssertEquals(t, vv.Version, version.NewHeight(0, 0))
}

func TestTxValidation(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion core/ledger/kvledger/txmgmt/txmgr/lockbasedtxmgr/lockbased_txmgr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ func (txmgr *LockBasedTxMgr) Commit() error {
}
defer func() { txmgr.batch = nil }()
if err := txmgr.db.ApplyUpdates(txmgr.batch,
version.NewHeight(txmgr.currentBlock.Header.Number, uint64(len(txmgr.currentBlock.Data.Data)))); err != nil {
version.NewHeight(txmgr.currentBlock.Header.Number, uint64(len(txmgr.currentBlock.Data.Data)-1))); err != nil {
return err
}
logger.Debugf("Updates committed to state database")
Expand Down
2 changes: 1 addition & 1 deletion core/ledger/kvledger/txmgmt/validator/statebasedval/state_based_validator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func (v *Validator) ValidateAndPrepareBatch(block *common.Block, doMVCCValidatio

//txRWSet != nil => t is valid
if txRWSet != nil {
committingTxHeight := version.NewHeight(block.Header.Number, uint64(txIndex+1))
committingTxHeight := version.NewHeight(block.Header.Number, uint64(txIndex))
addWriteSetToBatch(txRWSet, committingTxHeight, updates)
txsFilter.SetFlag(txIndex, peer.TxValidationCode_VALID)
}
Expand Down
124 changes: 62 additions & 62 deletions core/ledger/kvledger/txmgmt/validator/statebasedval/state_based_validator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,24 +45,24 @@ func TestValidator(t *testing.T) {

//populate db with initial data
batch := statedb.NewUpdateBatch()
batch.Put("ns1", "key1", []byte("value1"), version.NewHeight(1, 1))
batch.Put("ns1", "key2", []byte("value2"), version.NewHeight(1, 2))
batch.Put("ns1", "key3", []byte("value3"), version.NewHeight(1, 3))
batch.Put("ns1", "key4", []byte("value4"), version.NewHeight(1, 4))
batch.Put("ns1", "key5", []byte("value5"), version.NewHeight(1, 5))
db.ApplyUpdates(batch, version.NewHeight(1, 5))
batch.Put("ns1", "key1", []byte("value1"), version.NewHeight(1, 0))
batch.Put("ns1", "key2", []byte("value2"), version.NewHeight(1, 1))
batch.Put("ns1", "key3", []byte("value3"), version.NewHeight(1, 2))
batch.Put("ns1", "key4", []byte("value4"), version.NewHeight(1, 3))
batch.Put("ns1", "key5", []byte("value5"), version.NewHeight(1, 4))
db.ApplyUpdates(batch, version.NewHeight(1, 4))

validator := NewValidator(db)

//rwset1 should be valid
rwsetBuilder1 := rwsetutil.NewRWSetBuilder()
rwsetBuilder1.AddToReadSet("ns1", "key1", version.NewHeight(1, 1))
rwsetBuilder1.AddToReadSet("ns1", "key1", version.NewHeight(1, 0))
rwsetBuilder1.AddToReadSet("ns2", "key2", nil)
checkValidation(t, validator, []*rwsetutil.TxRwSet{rwsetBuilder1.GetTxReadWriteSet()}, []int{})

//rwset2 should not be valid
rwsetBuilder2 := rwsetutil.NewRWSetBuilder()
rwsetBuilder2.AddToReadSet("ns1", "key1", version.NewHeight(1, 2))
rwsetBuilder2.AddToReadSet("ns1", "key1", version.NewHeight(1, 1))
checkValidation(t, validator, []*rwsetutil.TxRwSet{rwsetBuilder2.GetTxReadWriteSet()}, []int{0})

//rwset3 should not be valid
Expand All @@ -72,11 +72,11 @@ func TestValidator(t *testing.T) {

// rwset4 and rwset5 within same block - rwset4 should be valid and makes rwset5 as invalid
rwsetBuilder4 := rwsetutil.NewRWSetBuilder()
rwsetBuilder4.AddToReadSet("ns1", "key1", version.NewHeight(1, 1))
rwsetBuilder4.AddToReadSet("ns1", "key1", version.NewHeight(1, 0))
rwsetBuilder4.AddToWriteSet("ns1", "key1", []byte("value1_new"))

rwsetBuilder5 := rwsetutil.NewRWSetBuilder()
rwsetBuilder5.AddToReadSet("ns1", "key1", version.NewHeight(1, 1))
rwsetBuilder5.AddToReadSet("ns1", "key1", version.NewHeight(1, 0))
checkValidation(t, validator,
[]*rwsetutil.TxRwSet{rwsetBuilder4.GetTxReadWriteSet(), rwsetBuilder5.GetTxReadWriteSet()}, []int{1})
}
Expand All @@ -90,52 +90,52 @@ func TestPhantomValidation(t *testing.T) {

//populate db with initial data
batch := statedb.NewUpdateBatch()
batch.Put("ns1", "key1", []byte("value1"), version.NewHeight(1, 1))
batch.Put("ns1", "key2", []byte("value2"), version.NewHeight(1, 2))
batch.Put("ns1", "key3", []byte("value3"), version.NewHeight(1, 3))
batch.Put("ns1", "key4", []byte("value4"), version.NewHeight(1, 4))
batch.Put("ns1", "key5", []byte("value5"), version.NewHeight(1, 5))
db.ApplyUpdates(batch, version.NewHeight(1, 5))
batch.Put("ns1", "key1", []byte("value1"), version.NewHeight(1, 0))
batch.Put("ns1", "key2", []byte("value2"), version.NewHeight(1, 1))
batch.Put("ns1", "key3", []byte("value3"), version.NewHeight(1, 2))
batch.Put("ns1", "key4", []byte("value4"), version.NewHeight(1, 3))
batch.Put("ns1", "key5", []byte("value5"), version.NewHeight(1, 4))
db.ApplyUpdates(batch, version.NewHeight(1, 4))

validator := NewValidator(db)

//rwset1 should be valid
rwsetBuilder1 := rwsetutil.NewRWSetBuilder()
rqi1 := &kvrwset.RangeQueryInfo{StartKey: "key2", EndKey: "key4", ItrExhausted: true}
rqi1.SetRawReads([]*kvrwset.KVRead{
rwsetutil.NewKVRead("key2", version.NewHeight(1, 2)),
rwsetutil.NewKVRead("key3", version.NewHeight(1, 3))})
rwsetutil.NewKVRead("key2", version.NewHeight(1, 1)),
rwsetutil.NewKVRead("key3", version.NewHeight(1, 2))})
rwsetBuilder1.AddToRangeQuerySet("ns1", rqi1)
checkValidation(t, validator, []*rwsetutil.TxRwSet{rwsetBuilder1.GetTxReadWriteSet()}, []int{})

//rwset2 should not be valid - Version of key4 changed
rwsetBuilder2 := rwsetutil.NewRWSetBuilder()
rqi2 := &kvrwset.RangeQueryInfo{StartKey: "key2", EndKey: "key4", ItrExhausted: false}
rqi2.SetRawReads([]*kvrwset.KVRead{
rwsetutil.NewKVRead("key2", version.NewHeight(1, 2)),
rwsetutil.NewKVRead("key3", version.NewHeight(1, 3)),
rwsetutil.NewKVRead("key4", version.NewHeight(1, 3))})
rwsetutil.NewKVRead("key2", version.NewHeight(1, 1)),
rwsetutil.NewKVRead("key3", version.NewHeight(1, 2)),
rwsetutil.NewKVRead("key4", version.NewHeight(1, 2))})
rwsetBuilder2.AddToRangeQuerySet("ns1", rqi2)
checkValidation(t, validator, []*rwsetutil.TxRwSet{rwsetBuilder2.GetTxReadWriteSet()}, []int{1})
checkValidation(t, validator, []*rwsetutil.TxRwSet{rwsetBuilder2.GetTxReadWriteSet()}, []int{0})

//rwset3 should not be valid - simulate key3 got commited to db
rwsetBuilder3 := rwsetutil.NewRWSetBuilder()
rqi3 := &kvrwset.RangeQueryInfo{StartKey: "key2", EndKey: "key4", ItrExhausted: false}
rqi3.SetRawReads([]*kvrwset.KVRead{
rwsetutil.NewKVRead("key2", version.NewHeight(1, 2)),
rwsetutil.NewKVRead("key4", version.NewHeight(1, 4))})
rwsetutil.NewKVRead("key2", version.NewHeight(1, 1)),
rwsetutil.NewKVRead("key4", version.NewHeight(1, 3))})
rwsetBuilder3.AddToRangeQuerySet("ns1", rqi3)
checkValidation(t, validator, []*rwsetutil.TxRwSet{rwsetBuilder3.GetTxReadWriteSet()}, []int{1})
checkValidation(t, validator, []*rwsetutil.TxRwSet{rwsetBuilder3.GetTxReadWriteSet()}, []int{0})

// //Remove a key in rwset4 and rwset5 should become invalid
rwsetBuilder4 := rwsetutil.NewRWSetBuilder()
rwsetBuilder4.AddToWriteSet("ns1", "key3", nil)
rwsetBuilder5 := rwsetutil.NewRWSetBuilder()
rqi5 := &kvrwset.RangeQueryInfo{StartKey: "key2", EndKey: "key4", ItrExhausted: false}
rqi5.SetRawReads([]*kvrwset.KVRead{
rwsetutil.NewKVRead("key2", version.NewHeight(1, 2)),
rwsetutil.NewKVRead("key3", version.NewHeight(1, 3)),
rwsetutil.NewKVRead("key4", version.NewHeight(1, 4))})
rwsetutil.NewKVRead("key2", version.NewHeight(1, 1)),
rwsetutil.NewKVRead("key3", version.NewHeight(1, 2)),
rwsetutil.NewKVRead("key4", version.NewHeight(1, 3))})
rwsetBuilder5.AddToRangeQuerySet("ns1", rqi5)
checkValidation(t, validator, []*rwsetutil.TxRwSet{
rwsetBuilder4.GetTxReadWriteSet(), rwsetBuilder5.GetTxReadWriteSet()}, []int{1})
Expand All @@ -147,9 +147,9 @@ func TestPhantomValidation(t *testing.T) {
rwsetBuilder7 := rwsetutil.NewRWSetBuilder()
rqi7 := &kvrwset.RangeQueryInfo{StartKey: "key2", EndKey: "key4", ItrExhausted: false}
rqi7.SetRawReads([]*kvrwset.KVRead{
rwsetutil.NewKVRead("key2", version.NewHeight(1, 2)),
rwsetutil.NewKVRead("key3", version.NewHeight(1, 3)),
rwsetutil.NewKVRead("key4", version.NewHeight(1, 4))})
rwsetutil.NewKVRead("key2", version.NewHeight(1, 1)),
rwsetutil.NewKVRead("key3", version.NewHeight(1, 2)),
rwsetutil.NewKVRead("key4", version.NewHeight(1, 3))})
rwsetBuilder7.AddToRangeQuerySet("ns1", rqi7)
checkValidation(t, validator, []*rwsetutil.TxRwSet{
rwsetBuilder6.GetTxReadWriteSet(), rwsetBuilder7.GetTxReadWriteSet()}, []int{1})
Expand All @@ -164,29 +164,29 @@ func TestPhantomHashBasedValidation(t *testing.T) {

//populate db with initial data
batch := statedb.NewUpdateBatch()
batch.Put("ns1", "key1", []byte("value1"), version.NewHeight(1, 1))
batch.Put("ns1", "key2", []byte("value2"), version.NewHeight(1, 2))
batch.Put("ns1", "key3", []byte("value3"), version.NewHeight(1, 3))
batch.Put("ns1", "key4", []byte("value4"), version.NewHeight(1, 4))
batch.Put("ns1", "key5", []byte("value5"), version.NewHeight(1, 5))
batch.Put("ns1", "key6", []byte("value6"), version.NewHeight(1, 6))
batch.Put("ns1", "key7", []byte("value7"), version.NewHeight(1, 7))
batch.Put("ns1", "key8", []byte("value8"), version.NewHeight(1, 8))
batch.Put("ns1", "key9", []byte("value9"), version.NewHeight(1, 9))
db.ApplyUpdates(batch, version.NewHeight(1, 9))
batch.Put("ns1", "key1", []byte("value1"), version.NewHeight(1, 0))
batch.Put("ns1", "key2", []byte("value2"), version.NewHeight(1, 1))
batch.Put("ns1", "key3", []byte("value3"), version.NewHeight(1, 2))
batch.Put("ns1", "key4", []byte("value4"), version.NewHeight(1, 3))
batch.Put("ns1", "key5", []byte("value5"), version.NewHeight(1, 4))
batch.Put("ns1", "key6", []byte("value6"), version.NewHeight(1, 5))
batch.Put("ns1", "key7", []byte("value7"), version.NewHeight(1, 6))
batch.Put("ns1", "key8", []byte("value8"), version.NewHeight(1, 7))
batch.Put("ns1", "key9", []byte("value9"), version.NewHeight(1, 8))
db.ApplyUpdates(batch, version.NewHeight(1, 8))

validator := NewValidator(db)

rwsetBuilder1 := rwsetutil.NewRWSetBuilder()
rqi1 := &kvrwset.RangeQueryInfo{StartKey: "key2", EndKey: "key9", ItrExhausted: true}
kvReadsDuringSimulation1 := []*kvrwset.KVRead{
rwsetutil.NewKVRead("key2", version.NewHeight(1, 2)),
rwsetutil.NewKVRead("key3", version.NewHeight(1, 3)),
rwsetutil.NewKVRead("key4", version.NewHeight(1, 4)),
rwsetutil.NewKVRead("key5", version.NewHeight(1, 5)),
rwsetutil.NewKVRead("key6", version.NewHeight(1, 6)),
rwsetutil.NewKVRead("key7", version.NewHeight(1, 7)),
rwsetutil.NewKVRead("key8", version.NewHeight(1, 8)),
rwsetutil.NewKVRead("key2", version.NewHeight(1, 1)),
rwsetutil.NewKVRead("key3", version.NewHeight(1, 2)),
rwsetutil.NewKVRead("key4", version.NewHeight(1, 3)),
rwsetutil.NewKVRead("key5", version.NewHeight(1, 4)),
rwsetutil.NewKVRead("key6", version.NewHeight(1, 5)),
rwsetutil.NewKVRead("key7", version.NewHeight(1, 6)),
rwsetutil.NewKVRead("key8", version.NewHeight(1, 7)),
}
rqi1.SetMerkelSummary(buildTestHashResults(t, 2, kvReadsDuringSimulation1))
rwsetBuilder1.AddToRangeQuerySet("ns1", rqi1)
Expand All @@ -195,19 +195,19 @@ func TestPhantomHashBasedValidation(t *testing.T) {
rwsetBuilder2 := rwsetutil.NewRWSetBuilder()
rqi2 := &kvrwset.RangeQueryInfo{StartKey: "key1", EndKey: "key9", ItrExhausted: false}
kvReadsDuringSimulation2 := []*kvrwset.KVRead{
rwsetutil.NewKVRead("key1", version.NewHeight(1, 1)),
rwsetutil.NewKVRead("key2", version.NewHeight(1, 2)),
rwsetutil.NewKVRead("key3", version.NewHeight(1, 2)),
rwsetutil.NewKVRead("key4", version.NewHeight(1, 4)),
rwsetutil.NewKVRead("key5", version.NewHeight(1, 5)),
rwsetutil.NewKVRead("key6", version.NewHeight(1, 6)),
rwsetutil.NewKVRead("key7", version.NewHeight(1, 7)),
rwsetutil.NewKVRead("key8", version.NewHeight(1, 8)),
rwsetutil.NewKVRead("key9", version.NewHeight(1, 9)),
rwsetutil.NewKVRead("key1", version.NewHeight(1, 0)),
rwsetutil.NewKVRead("key2", version.NewHeight(1, 1)),
rwsetutil.NewKVRead("key3", version.NewHeight(1, 1)),
rwsetutil.NewKVRead("key4", version.NewHeight(1, 3)),
rwsetutil.NewKVRead("key5", version.NewHeight(1, 4)),
rwsetutil.NewKVRead("key6", version.NewHeight(1, 5)),
rwsetutil.NewKVRead("key7", version.NewHeight(1, 6)),
rwsetutil.NewKVRead("key8", version.NewHeight(1, 7)),
rwsetutil.NewKVRead("key9", version.NewHeight(1, 8)),
}
rqi2.SetMerkelSummary(buildTestHashResults(t, 2, kvReadsDuringSimulation2))
rwsetBuilder2.AddToRangeQuerySet("ns1", rqi2)
checkValidation(t, validator, []*rwsetutil.TxRwSet{rwsetBuilder2.GetTxReadWriteSet()}, []int{1})
checkValidation(t, validator, []*rwsetutil.TxRwSet{rwsetBuilder2.GetTxReadWriteSet()}, []int{0})
}

func checkValidation(t *testing.T, validator *Validator, rwsets []*rwsetutil.TxRwSet, invalidTxIndexes []int) {
Expand All @@ -221,15 +221,15 @@ func checkValidation(t *testing.T, validator *Validator, rwsets []*rwsetutil.TxR
block.Metadata.Metadata[common.BlockMetadataIndex_TRANSACTIONS_FILTER] = util.NewTxValidationFlags(len(block.Data.Data))
_, err := validator.ValidateAndPrepareBatch(block, true)
txsFltr := util.TxValidationFlags(block.Metadata.Metadata[common.BlockMetadataIndex_TRANSACTIONS_FILTER])
invalidTxNum := 0
invalidTxs := make([]int, 0)
for i := 0; i < len(block.Data.Data); i++ {
if txsFltr.IsInvalid(i) {
invalidTxNum++
invalidTxs = append(invalidTxs, i)
}
}
testutil.AssertNoError(t, err, "")
testutil.AssertEquals(t, invalidTxNum, len(invalidTxIndexes))
//TODO Add the check for exact txnum that is marked invlid when bitarray is in place
testutil.AssertEquals(t, len(invalidTxs), len(invalidTxIndexes))
testutil.AssertContainsAll(t, invalidTxs, invalidTxIndexes)
}

func buildTestHashResults(t *testing.T, maxDegree int, kvReads []*kvrwset.KVRead) *kvrwset.QueryReadsMerkleSummary {
Expand Down

0 comments on commit 03fe133

Please sign in to comment.