Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

testing(dot/core): rewrite dot/core service tests (part 2) #2308

Merged
merged 75 commits into from
Mar 23, 2022
Merged

testing(dot/core): rewrite dot/core service tests (part 2) #2308

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
75 commits
Select commit Hold shift + click to select a range
635a950
add wasm file generation to core service tests
jimjbrettj Jan 25, 2022
cad1117
WIP/codeSubstituteTest
jimjbrettj Jan 25, 2022
f1b4543
code substitution test
jimjbrettj Jan 25, 2022
b556f79
test handle block
jimjbrettj Jan 25, 2022
5709451
handle BlockProduced test
jimjbrettj Jan 26, 2022
996e4ce
clean up and test maintainTxnPool
jimjbrettj Jan 26, 2022
88ac199
test handleBlockAsync
jimjbrettj Jan 26, 2022
b82d0a5
part1 of service tests
jimjbrettj Jan 26, 2022
43d7ba2
clean up
jimjbrettj Jan 26, 2022
ccbba02
fix test naming
jimjbrettj Feb 10, 2022
86c9152
wip/cr feedback
jimjbrettj Feb 10, 2022
772147b
define storage root tests in subtests
jimjbrettj Feb 10, 2022
b478bf5
parallel storage root tests
jimjbrettj Feb 10, 2022
ba87d17
remove gomack.Any() for getRuntime calls
jimjbrettj Feb 10, 2022
1469c1f
restructure handleCodeSub unit tests
jimjbrettj Feb 10, 2022
1419f73
restructure handle block tests
jimjbrettj Feb 10, 2022
8f0e431
restructure tests and make subtests parallel
jimjbrettj Feb 10, 2022
c25e8dd
reduce integration test diff
jimjbrettj Feb 10, 2022
910984a
CR feedback TODO rewrite code sub
jimjbrettj Feb 16, 2022
b376056
remove wasm files from core service tests
jimjbrettj Feb 16, 2022
a84a7ae
reduce code duplication in tests
jimjbrettj Feb 17, 2022
028ab93
CR feedback
jimjbrettj Feb 17, 2022
c2191bd
wip/reorg test
jimjbrettj Jan 31, 2022
88758d0
maybe finished chainreorg test
jimjbrettj Jan 31, 2022
505f55a
temp comment out reorg test to work on others. TODO/fix extrinsic sig
jimjbrettj Jan 31, 2022
f1386cc
test insert and has key
jimjbrettj Jan 31, 2022
03f287d
test decode session keys
jimjbrettj Jan 31, 2022
7ed1e87
fix core reorg logic and remove wrapper for centrifuge extrinsics
jimjbrettj Feb 1, 2022
902d7ab
test getRuntimeVersion
jimjbrettj Feb 1, 2022
ff2cab4
wip/fix core integration tests
jimjbrettj Feb 1, 2022
77cb3fe
wip/fix core integration tests
jimjbrettj Feb 1, 2022
ef67b6e
wip/fix core integration tests
jimjbrettj Feb 2, 2022
23fb2a6
test hadnle submitted extrinsic
jimjbrettj Feb 2, 2022
72bf8f0
test get metadata
jimjbrettj Feb 3, 2022
c8581d8
test try query storage
jimjbrettj Feb 3, 2022
de52abd
test query storage
jimjbrettj Feb 3, 2022
a6599cf
finish dot core unit tests
jimjbrettj Feb 3, 2022
968fd1b
wip
jimjbrettj Feb 8, 2022
b9fa310
wip temp
jimjbrettj Feb 8, 2022
63b824b
wip/fix unit tests post rebase
jimjbrettj Feb 10, 2022
4ee6cf7
fix first broken core integration test
jimjbrettj Feb 15, 2022
63e67cf
clean up code before fix other tests
jimjbrettj Feb 15, 2022
28cba7c
wip/fix tests
jimjbrettj Feb 15, 2022
7cc72b9
fix core unit tests TODO fix final integration test
jimjbrettj Feb 15, 2022
b61dd70
fix core service integration and unit tests
jimjbrettj Feb 15, 2022
289709b
wip/restructure tests
jimjbrettj Feb 16, 2022
666e407
rebase
jimjbrettj Feb 17, 2022
7fbf8d8
wip/refactor tests
jimjbrettj Feb 17, 2022
439cb3f
wip/test refactor
jimjbrettj Feb 17, 2022
b454d70
finish tests
jimjbrettj Feb 17, 2022
00575be
merge dev
jimjbrettj Feb 28, 2022
5d9021e
update github workflows to test core integration tests
jimjbrettj Feb 28, 2022
f48bae4
fix integration tests
jimjbrettj Mar 1, 2022
efd729c
wip/remove hardcoded test vals
jimjbrettj Mar 1, 2022
7a0f337
wip/CE feedback
jimjbrettj Mar 1, 2022
7fa3495
wip
jimjbrettj Mar 1, 2022
b97ea9d
remove mocks from helper func
jimjbrettj Mar 1, 2022
c4bf88e
remove comment
jimjbrettj Mar 1, 2022
a8e48bc
remove hardcoded babeDigests
jimjbrettj Mar 1, 2022
f161f18
PR feedback wip
jimjbrettj Mar 1, 2022
57f3718
CR feedback
jimjbrettj Mar 2, 2022
43b465a
use helper to get root path
jimjbrettj Mar 7, 2022
61496f3
wip/CR feedback
jimjbrettj Mar 7, 2022
9b899ea
wip
jimjbrettj Mar 7, 2022
4013485
respond to feedback
jimjbrettj Mar 7, 2022
ca65fb6
linting
jimjbrettj Mar 8, 2022
96f7448
fix linting
jimjbrettj Mar 8, 2022
daa077b
CR feedback resolved
jimjbrettj Mar 14, 2022
625bb39
clean up
jimjbrettj Mar 14, 2022
619975f
wip/CR feedback
jimjbrettj Mar 21, 2022
17978df
wip/feedback
jimjbrettj Mar 21, 2022
d2d0c4e
resolve feedback
jimjbrettj Mar 21, 2022
bebac17
merge dev
jimjbrettj Mar 21, 2022
b04b414
remove anon struct from accountInfo
jimjbrettj Mar 22, 2022
efcef54
Merge branch 'development' into jimmy/coreServiceTestsP2
jimjbrettj Mar 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions dot/core/helpers_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"path/filepath"
"testing"

"github.com/ChainSafe/gossamer/dot/digest"
"github.com/ChainSafe/gossamer/dot/network"
"github.com/ChainSafe/gossamer/dot/state"
"github.com/ChainSafe/gossamer/internal/log"
"github.com/ChainSafe/gossamer/lib/common"
Expand All @@ -31,7 +33,7 @@ func NewTestService(t *testing.T, cfg *Config) *Service {
}

if cfg.DigestHandler == nil {
cfg.DigestHandler = NewMockDigestHandler(ctrl)
cfg.DigestHandler = &digest.Handler{} // only for nil check in NewService
}

if cfg.Keystore == nil {
Expand Down Expand Up @@ -120,7 +122,7 @@ func NewTestService(t *testing.T, cfg *Config) *Service {
cfg.BlockState.StoreRuntime(cfg.BlockState.BestBlockHash(), cfg.Runtime)

if cfg.Network == nil {
cfg.Network = NewMockNetwork(ctrl)
cfg.Network = new(network.Service) // only for nil check in NewService
}

if cfg.CodeSubstitutes == nil {
Expand Down
4 changes: 2 additions & 2 deletions dot/core/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,8 +385,8 @@ func (s *Service) handleChainReorg(prev, curr common.Hash) error {
continue
}

externalExt := make(types.Extrinsic, 1, 1+len(ext))
externalExt[0] = byte(types.TxnExternal)
externalExt := make(types.Extrinsic, 0, 1+len(ext))
externalExt = append(externalExt, byte(types.TxnExternal))
externalExt = append(externalExt, ext...)
txv, err := rt.ValidateTransaction(externalExt)
if err != nil {
Expand Down
115 changes: 34 additions & 81 deletions dot/core/service_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
package core

import (
"errors"
"fmt"
"math/big"
"os"
"path/filepath"
"testing"
Expand All @@ -23,31 +23,23 @@ import (
"github.com/ChainSafe/gossamer/lib/genesis"
"github.com/ChainSafe/gossamer/lib/keystore"
"github.com/ChainSafe/gossamer/lib/runtime"
runtimemocks "github.com/ChainSafe/gossamer/lib/runtime/mocks"
"github.com/ChainSafe/gossamer/lib/runtime/storage"
rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage"
"github.com/ChainSafe/gossamer/lib/runtime/wasmer"
"github.com/ChainSafe/gossamer/lib/transaction"
"github.com/ChainSafe/gossamer/lib/trie"
"github.com/ChainSafe/gossamer/lib/utils"
"github.com/ChainSafe/gossamer/pkg/scale"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)

//go:generate mockgen -destination=mock_telemetry_test.go -package $GOPACKAGE github.com/ChainSafe/gossamer/dot/telemetry Client

//nolint
type testAccountData struct {
Free *scale.Uint128
Reserved *scale.Uint128
MiscFrozen *scale.Uint128
FreeFrozen *scale.Uint128
}
const testSlotNumber = 21

func balanceKey(t *testing.T, pub []byte) (bKey []byte) {
t.Helper()

h0, err := common.Twox128Hash([]byte("System"))
qdm12 marked this conversation as resolved.
Show resolved Hide resolved
require.NoError(t, err)
bKey = append(bKey, h0...)
Expand Down Expand Up @@ -99,7 +91,7 @@ func generateTestValidRemarkTxns(t *testing.T, pubKey []byte, accInfo types.Acco
genTrie, err := genesis.NewTrieFromGenesis(gen)
require.NoError(t, err)

genState, err := storage.NewTrieState(genTrie)
genState, err := rtstorage.NewTrieState(genTrie)
require.NoError(t, err)

nodeStorage := runtime.NodeStorage{
Expand All @@ -126,7 +118,7 @@ func generateTestValidRemarkTxns(t *testing.T, pubKey []byte, accInfo types.Acco
rt.(*wasmer.Instance).GetContext().Storage.Set(common.UpgradedToDualRefKey, []byte{1})

genesisHeader := &types.Header{
Number: big.NewInt(0),
Number: 0,
StateRoot: genTrie.MustHash(),
}

Expand Down Expand Up @@ -472,9 +464,9 @@ func TestHandleChainReorg_WithReorg_Transactions(t *testing.T) {
}

func TestMaintainTransactionPool_EmptyBlock(t *testing.T) {
accInfo := types.AccountInfo{
accountInfo := types.AccountInfo{
Nonce: 0,
Data: testAccountData{
Data: types.AccountData{
Free: scale.MustNewUint128(big.NewInt(1152921504606846976)),
Reserved: scale.MustNewUint128(big.NewInt(0)),
MiscFrozen: scale.MustNewUint128(big.NewInt(0)),
Expand All @@ -484,52 +476,53 @@ func TestMaintainTransactionPool_EmptyBlock(t *testing.T) {
keyring, err := keystore.NewSr25519Keyring()
require.NoError(t, err)
alicePub := common.MustHexToBytes(keyring.Alice().Public().Hex())
encExt, rt := generateTestValidRemarkTxns(t, alicePub, accInfo)
encExt, runtimeInstance := generateTestValidRemarkTxns(t, alicePub, accountInfo)
cfg := &Config{
Runtime: rt,
Runtime: runtimeInstance,
}

ctrl := gomock.NewController(t)
telemetryMock := NewMockClient(ctrl)
telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes()

ts := state.NewTransactionState(telemetryMock)
transactionState := state.NewTransactionState(telemetryMock)
tx := &transaction.ValidTransaction{
Extrinsic: types.Extrinsic(encExt),
Validity: &transaction.Validity{Priority: 1},
}
_ = ts.AddToPool(tx)
_ = transactionState.AddToPool(tx)

s := NewTestService(t, cfg)
s.transactionState = ts
service := NewTestService(t, cfg)
service.transactionState = transactionState

// provides is a list of transaction hashes that depend on this tx, see:
// https://github.com/paritytech/substrate/blob/5420de3face1349a97eb954ae71c5b0b940c31de/core/sr-primitives/src/transaction_validity.rs#L195
provides := common.MustHexToBytes("0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d00000000")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what is this? the previous tx hash?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, it is a list of tags that this txn provides (https://github.com/paritytech/substrate/blob/5420de3face1349a97eb954ae71c5b0b940c31de/core/sr-primitives/src/transaction_validity.rs#L195).

I got this from

gossamer/dot/rpc/modules/author_integration_test.go

Line 263 in 4617175

common.MustHexToBytes("0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d00000000"),
and honestly not sure where that is from/how to generate it

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so you are expecting this Provides tags to be added via the runtime? I would put a comment about it. It would be nice to have a link to the runtime code that appends these tags, and what these tags are.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe so? Haven't been able to find much on it. In substrate the provides is a vector of transaction tags, which is just a vector of bytes. Haven't been able to find that the possible values. My understanding is that the provides tags are added by the runtime to deal with transaction dependencies. So maybe it is the previous txn hash @noot? Or at least A previous transaction of which the current txn depends?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah Provides is supposed to be other transaction hashes that depend on this tx, can you remove it and see if it still works? feel like it's not necessary here

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jimjbrettj did you try?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will try this now, just getting to this round of feedback

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems that this test does require the Provides param to pass. @noot correct me if im wrong but it seems to be added by the runtime. When i remove it a check fails because its getting set somewhere, but I haven't been able to find where its set.

Unless told otherwise, thinking I will leave as is and add a comment

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

weird, ok feel free to leave it and just document it then

val := &transaction.Validity{
txnValidity := &transaction.Validity{
Priority: 39325240425794630,
Provides: [][]byte{provides},
Longevity: 18446744073709551614,
Propagate: true,
}

expTxn := transaction.NewValidTransaction(tx.Extrinsic, val)
expectedTx := transaction.NewValidTransaction(tx.Extrinsic, txnValidity)

s.maintainTransactionPool(&types.Block{
service.maintainTransactionPool(&types.Block{
Body: *types.NewBody([]types.Extrinsic{}),
})

res := ts.Pop()
require.Equal(t, expTxn, res)
resultTx := transactionState.Pop()
require.Equal(t, expectedTx, resultTx)

ts.RemoveExtrinsic(tx.Extrinsic)
head := ts.Pop()
transactionState.RemoveExtrinsic(tx.Extrinsic)
head := transactionState.Pop()
require.Nil(t, head)
}

func TestMaintainTransactionPool_BlockWithExtrinsics(t *testing.T) {
accInfo := types.AccountInfo{
accountInfo := types.AccountInfo{
Nonce: 0,
Data: testAccountData{
Data: types.AccountData{
Free: scale.MustNewUint128(big.NewInt(1152921504606846976)),
Reserved: scale.MustNewUint128(big.NewInt(0)),
MiscFrozen: scale.MustNewUint128(big.NewInt(0)),
Expand All @@ -539,7 +532,7 @@ func TestMaintainTransactionPool_BlockWithExtrinsics(t *testing.T) {
keyring, err := keystore.NewSr25519Keyring()
require.NoError(t, err)
alicePub := common.MustHexToBytes(keyring.Alice().Public().Hex())
encExt, _ := generateTestValidRemarkTxns(t, alicePub, accInfo)
extrinsicBytes, _ := generateTestValidRemarkTxns(t, alicePub, accountInfo)

ctrl := gomock.NewController(t)
telemetryMock := NewMockClient(ctrl)
Expand All @@ -549,7 +542,7 @@ func TestMaintainTransactionPool_BlockWithExtrinsics(t *testing.T) {

// Maybe replace validity
tx := &transaction.ValidTransaction{
Extrinsic: types.Extrinsic(encExt),
Extrinsic: types.Extrinsic(extrinsicBytes),
Validity: &transaction.Validity{Priority: 1},
}

Expand All @@ -560,7 +553,7 @@ func TestMaintainTransactionPool_BlockWithExtrinsics(t *testing.T) {
}

s.maintainTransactionPool(&types.Block{
Body: types.Body([]types.Extrinsic{encExt}),
Body: types.Body([]types.Extrinsic{extrinsicBytes}),
})

res := []*transaction.ValidTransaction{}
Expand Down Expand Up @@ -792,9 +785,9 @@ func TestTryQueryStore_WhenThereIsDataToRetrieve(t *testing.T) {
storageStateTrie.Set(testKey, testValue)
require.NoError(t, err)

digest := newTestDigest(t, 420)
header, err := types.NewHeader(s.blockState.GenesisHash(), storageStateTrie.MustRoot(),
common.Hash{}, big.NewInt(1), digest)
digest := newTestDigest(t, testSlotNumber)
header, err := types.NewHeader(s.blockState.GenesisHash(), storageStateTrie.MustRoot(), common.Hash{}, 1, digest)

require.NoError(t, err)

err = s.storageState.StoreTrie(storageStateTrie, header)
Expand Down Expand Up @@ -823,9 +816,8 @@ func TestTryQueryStore_WhenDoesNotHaveDataToRetrieve(t *testing.T) {
storageStateTrie, err := rtstorage.NewTrieState(trie.NewTrie(nil))
require.NoError(t, err)

digest := newTestDigest(t, 420)
header, err := types.NewHeader(s.blockState.GenesisHash(), storageStateTrie.MustRoot(),
common.Hash{}, big.NewInt(1), digest)
digest := newTestDigest(t, testSlotNumber)
header, err := types.NewHeader(s.blockState.GenesisHash(), storageStateTrie.MustRoot(), common.Hash{}, 1, digest)
require.NoError(t, err)

err = s.storageState.StoreTrie(storageStateTrie, header)
Expand Down Expand Up @@ -853,11 +845,10 @@ func TestTryQueryStore_WhenDoesNotHaveDataToRetrieve(t *testing.T) {
func TestTryQueryState_WhenDoesNotHaveStateRoot(t *testing.T) {
s := NewTestService(t, nil)

digest := newTestDigest(t, 420)
digest := newTestDigest(t, testSlotNumber)
header, err := types.NewHeader(
s.blockState.GenesisHash(),
common.Hash{}, common.Hash{},
big.NewInt(1), digest)
common.Hash{}, common.Hash{}, 1, digest)
require.NoError(t, err)

testBlock := &types.Block{
Expand Down Expand Up @@ -942,8 +933,7 @@ func createNewBlockAndStoreDataAtBlock(t *testing.T, s *Service,
require.NoError(t, err)

digest := newTestDigest(t, 421)
header, err := types.NewHeader(parentHash, storageStateTrie.MustRoot(),
common.Hash{}, big.NewInt(number), digest)
header, err := types.NewHeader(parentHash, storageStateTrie.MustRoot(), common.Hash{}, number, digest)
require.NoError(t, err)

err = s.storageState.StoreTrie(storageStateTrie, header)
Expand All @@ -959,40 +949,3 @@ func createNewBlockAndStoreDataAtBlock(t *testing.T, s *Service,

return testBlock
}

func TestDecodeSessionKeys(t *testing.T) {
ctrl := gomock.NewController(t)

mockInstance := new(runtimemocks.Instance)
mockInstance.On("DecodeSessionKeys", mock.AnythingOfType("[]uint8")).Return([]byte{}, nil).Once()

mockBlockState := NewMockBlockState(ctrl)
mockBlockState.EXPECT().GetRuntime(gomock.AssignableToTypeOf(new(common.Hash))).
Return(mockInstance, nil)

coreservice := new(Service)
coreservice.blockState = mockBlockState

b, err := coreservice.DecodeSessionKeys([]byte{})

mockInstance.AssertCalled(t, "DecodeSessionKeys", []uint8{})

require.NoError(t, err)
require.Equal(t, b, []byte{})
}

func TestDecodeSessionKeys_WhenGetRuntimeReturnError(t *testing.T) {
ctrl := gomock.NewController(t)

mockBlockState := NewMockBlockState(ctrl)
mockBlockState.EXPECT().GetRuntime(gomock.AssignableToTypeOf(new(common.Hash))).
Return(nil, errors.New("problems"))

coreservice := new(Service)
coreservice.blockState = mockBlockState

b, err := coreservice.DecodeSessionKeys([]byte{})

require.Error(t, err, "problems")
require.Nil(t, b)
}
Loading