Merge branch 'dev' into network

config/config.go

@@ -928,7 +928,8 @@ func getDatabaseConfig(v *viper.Viper, networkID uint32) (node.DatabaseConfig, e
 	}
 
 	return node.DatabaseConfig{
-		Name: v.GetString(DBTypeKey),
+		Name:     v.GetString(DBTypeKey),
+		ReadOnly: v.GetBool(DBReadOnlyKey),
 		Path: filepath.Join(
 			GetExpandedArg(v, DBPathKey),
 			constants.NetworkName(networkID),

config/flags.go

@@ -105,6 +105,7 @@ func addNodeFlags(fs *pflag.FlagSet) {
 
 	// Database
 	fs.String(DBTypeKey, leveldb.Name, fmt.Sprintf("Database type to use. Must be one of {%s, %s, %s}", leveldb.Name, memdb.Name, pebble.Name))
+	fs.Bool(DBReadOnlyKey, false, "If true, database writes are to memory and never persisted. May still initialize database directory/files on disk if they don't exist")
 	fs.String(DBPathKey, defaultDBDir, "Path to database directory")
 	fs.String(DBConfigFileKey, "", fmt.Sprintf("Path to database config file. Ignored if %s is specified", DBConfigContentKey))
 	fs.String(DBConfigContentKey, "", "Specifies base64 encoded database config content")

config/keys.go

@@ -34,6 +34,7 @@ const (
 	StakeMintingPeriodKey                              = "stake-minting-period"
 	StakeSupplyCapKey                                  = "stake-supply-cap"
 	DBTypeKey                                          = "db-type"
+	DBReadOnlyKey                                      = "db-read-only"
 	DBPathKey                                          = "db-dir"
 	DBConfigFileKey                                    = "db-config-file"
 	DBConfigContentKey                                 = "db-config-file-content"

database/corruptabledb/db_test.go

@@ -26,16 +26,21 @@ func TestInterface(t *testing.T) {
 	}
 }
 
-func FuzzKeyValue(f *testing.F) {
+func newDB() *Database {
 	baseDB := memdb.New()
-	db := New(baseDB)
-	database.FuzzKeyValue(f, db)
+	return New(baseDB)
+}
+
+func FuzzKeyValue(f *testing.F) {
+	database.FuzzKeyValue(f, newDB())
 }
 
 func FuzzNewIteratorWithPrefix(f *testing.F) {
-	baseDB := memdb.New()
-	db := New(baseDB)
-	database.FuzzNewIteratorWithPrefix(f, db)
+	database.FuzzNewIteratorWithPrefix(f, newDB())
+}
+
+func FuzzNewIteratorWithStartAndPrefix(f *testing.F) {
+	database.FuzzNewIteratorWithStartAndPrefix(f, newDB())
 }
 
 // TestCorruption tests to make sure corruptabledb wrapper works as expected.
@@ -70,9 +75,7 @@ func TestCorruption(t *testing.T) {
 			return err
 		},
 	}
-	baseDB := memdb.New()
-	// wrap this db
-	corruptableDB := New(baseDB)
+	corruptableDB := newDB()
 	_ = corruptableDB.handleError(errTest)
 	for name, testFn := range tests {
 		t.Run(name, func(tt *testing.T) {
@@ -176,9 +179,7 @@ func TestIterator(t *testing.T) {
 			ctrl := gomock.NewController(t)
 
 			// Make a database
-			baseDB := memdb.New()
-			corruptableDB := New(baseDB)
-
+			corruptableDB := newDB()
 			// Put a key-value pair in the database.
 			require.NoError(corruptableDB.Put([]byte{0}, []byte{1}))
 

database/encdb/db_test.go

@@ -24,28 +24,30 @@ func TestInterface(t *testing.T) {
 	}
 }
 
-func FuzzKeyValue(f *testing.F) {
+func newDB(t testing.TB) database.Database {
 	unencryptedDB := memdb.New()
 	db, err := New([]byte(testPassword), unencryptedDB)
-	require.NoError(f, err)
-	database.FuzzKeyValue(f, db)
+	require.NoError(t, err)
+	return db
+}
+
+func FuzzKeyValue(f *testing.F) {
+	database.FuzzKeyValue(f, newDB(f))
 }
 
 func FuzzNewIteratorWithPrefix(f *testing.F) {
-	unencryptedDB := memdb.New()
-	db, err := New([]byte(testPassword), unencryptedDB)
-	require.NoError(f, err)
-	database.FuzzNewIteratorWithPrefix(f, db)
+	database.FuzzNewIteratorWithPrefix(f, newDB(f))
+}
+
+func FuzzNewIteratorWithStartAndPrefix(f *testing.F) {
+	database.FuzzNewIteratorWithStartAndPrefix(f, newDB(f))
 }
 
 func BenchmarkInterface(b *testing.B) {
 	for _, size := range database.BenchmarkSizes {
 		keys, values := database.SetupBenchmark(b, size[0], size[1], size[2])
 		for _, bench := range database.Benchmarks {
-			unencryptedDB := memdb.New()
-			db, err := New([]byte(testPassword), unencryptedDB)
-			require.NoError(b, err)
-			bench(b, db, "encdb", keys, values)
+			bench(b, newDB(b), "encdb", keys, values)
 		}
 	}
 }

database/leveldb/db_test.go

@@ -26,34 +26,39 @@ func TestInterface(t *testing.T) {
 	}
 }
 
-func FuzzKeyValue(f *testing.F) {
-	folder := f.TempDir()
+func newDB(t testing.TB) database.Database {
+	folder := t.TempDir()
 	db, err := New(folder, nil, logging.NoLog{}, "", prometheus.NewRegistry())
-	require.NoError(f, err)
+	require.NoError(t, err)
+	return db
+}
 
+func FuzzKeyValue(f *testing.F) {
+	db := newDB(f)
 	defer db.Close()
 
 	database.FuzzKeyValue(f, db)
 }
 
 func FuzzNewIteratorWithPrefix(f *testing.F) {
-	folder := f.TempDir()
-	db, err := New(folder, nil, logging.NoLog{}, "", prometheus.NewRegistry())
-	require.NoError(f, err)
-
+	db := newDB(f)
 	defer db.Close()
 
 	database.FuzzNewIteratorWithPrefix(f, db)
 }
 
+func FuzzNewIteratorWithStartAndPrefix(f *testing.F) {
+	db := newDB(f)
+	defer db.Close()
+
+	database.FuzzNewIteratorWithStartAndPrefix(f, db)
+}
+
 func BenchmarkInterface(b *testing.B) {
 	for _, size := range database.BenchmarkSizes {
 		keys, values := database.SetupBenchmark(b, size[0], size[1], size[2])
 		for _, bench := range database.Benchmarks {
-			folder := b.TempDir()
-
-			db, err := New(folder, nil, logging.NoLog{}, "", prometheus.NewRegistry())
-			require.NoError(b, err)
+			db := newDB(b)
 
 			bench(b, db, "leveldb", keys, values)
 

database/memdb/db_test.go

@@ -23,6 +23,10 @@ func FuzzNewIteratorWithPrefix(f *testing.F) {
 	database.FuzzNewIteratorWithPrefix(f, New())
 }
 
+func FuzzNewIteratorWithStartAndPrefix(f *testing.F) {
+	database.FuzzNewIteratorWithStartAndPrefix(f, New())
+}
+
 func BenchmarkInterface(b *testing.B) {
 	for _, size := range database.BenchmarkSizes {
 		keys, values := database.SetupBenchmark(b, size[0], size[1], size[2])

database/meterdb/db_test.go

@@ -24,28 +24,30 @@ func TestInterface(t *testing.T) {
 	}
 }
 
-func FuzzKeyValue(f *testing.F) {
+func newDB(t testing.TB) database.Database {
 	baseDB := memdb.New()
 	db, err := New("", prometheus.NewRegistry(), baseDB)
-	require.NoError(f, err)
-	database.FuzzKeyValue(f, db)
+	require.NoError(t, err)
+	return db
+}
+
+func FuzzKeyValue(f *testing.F) {
+	database.FuzzKeyValue(f, newDB(f))
 }
 
 func FuzzNewIteratorWithPrefix(f *testing.F) {
-	baseDB := memdb.New()
-	db, err := New("", prometheus.NewRegistry(), baseDB)
-	require.NoError(f, err)
-	database.FuzzNewIteratorWithPrefix(f, db)
+	database.FuzzNewIteratorWithPrefix(f, newDB(f))
+}
+
+func FuzzNewIteratorWithStartAndPrefix(f *testing.F) {
+	database.FuzzNewIteratorWithStartAndPrefix(f, newDB(f))
 }
 
 func BenchmarkInterface(b *testing.B) {
 	for _, size := range database.BenchmarkSizes {
 		keys, values := database.SetupBenchmark(b, size[0], size[1], size[2])
 		for _, bench := range database.Benchmarks {
-			baseDB := memdb.New()
-			db, err := New("", prometheus.NewRegistry(), baseDB)
-			require.NoError(b, err)
-			bench(b, db, "meterdb", keys, values)
+			bench(b, newDB(b), "meterdb", keys, values)
 		}
 	}
 }

database/pebble/db_test.go

@@ -41,6 +41,12 @@ func FuzzNewIteratorWithPrefix(f *testing.F) {
 	_ = db.Close()
 }
 
+func FuzzNewIteratorWithStartAndPrefix(f *testing.F) {
+	db := newDB(f)
+	database.FuzzNewIteratorWithStartAndPrefix(f, db)
+	_ = db.Close()
+}
+
 func BenchmarkInterface(b *testing.B) {
 	for _, size := range database.BenchmarkSizes {
 		keys, values := database.SetupBenchmark(b, size[0], size[1], size[2])

database/prefixdb/db_test.go

@@ -30,6 +30,10 @@ func FuzzNewIteratorWithPrefix(f *testing.F) {
 	database.FuzzNewIteratorWithPrefix(f, New([]byte(""), memdb.New()))
 }
 
+func FuzzNewIteratorWithStartAndPrefix(f *testing.F) {
+	database.FuzzNewIteratorWithStartAndPrefix(f, New([]byte(""), memdb.New()))
+}
+
 func BenchmarkInterface(b *testing.B) {
 	for _, size := range database.BenchmarkSizes {
 		keys, values := database.SetupBenchmark(b, size[0], size[1], size[2])

database/rpcdb/db_test.go

@@ -75,6 +75,13 @@ func FuzzNewIteratorWithPrefix(f *testing.F) {
 	db.closeFn()
 }
 
+func FuzzNewIteratorWithStartAndPrefix(f *testing.F) {
+	db := setupDB(f)
+	database.FuzzNewIteratorWithStartAndPrefix(f, db.client)
+
+	db.closeFn()
+}
+
 func BenchmarkInterface(b *testing.B) {
 	for _, size := range database.BenchmarkSizes {
 		keys, values := database.SetupBenchmark(b, size[0], size[1], size[2])

database/test_database.go

@@ -1266,7 +1266,74 @@ func FuzzNewIteratorWithPrefix(f *testing.F, db Database) {
 			require.Equal(expected[string(iter.Key())], val)
 			numIterElts++
 		}
-		require.Equal(len(expectedList), numIterElts)
+		require.Len(expectedList, numIterElts)
+
+		// Clear the database for the next fuzz iteration.
+		require.NoError(AtomicClear(db, db))
+	})
+}
+
+func FuzzNewIteratorWithStartAndPrefix(f *testing.F, db Database) {
+	const (
+		maxKeyLen   = 32
+		maxValueLen = 32
+	)
+
+	f.Fuzz(func(
+		t *testing.T,
+		randSeed int64,
+		start []byte,
+		prefix []byte,
+		numKeyValues uint,
+	) {
+		require := require.New(t)
+		r := rand.New(rand.NewSource(randSeed)) // #nosec G404
+
+		expected := map[string][]byte{}
+
+		// Put a bunch of key-values
+		for i := 0; i < int(numKeyValues); i++ {
+			key := make([]byte, r.Intn(maxKeyLen))
+			_, _ = r.Read(key) // #nosec G404
+
+			value := make([]byte, r.Intn(maxValueLen))
+			_, _ = r.Read(value) // #nosec G404
+
+			if len(value) == 0 {
+				// Consistently treat zero length values as nil
+				// so that we can compare [expected] and [got] with
+				// require.Equal, which treats nil and empty byte
+				// as being unequal, whereas the database treats
+				// them as being equal.
+				value = nil
+			}
+
+			if bytes.HasPrefix(key, prefix) && bytes.Compare(key, start) >= 0 {
+				expected[string(key)] = value
+			}
+
+			require.NoError(db.Put(key, value))
+		}
+
+		expectedList := maps.Keys(expected)
+		slices.Sort(expectedList)
+
+		iter := db.NewIteratorWithStartAndPrefix(start, prefix)
+		defer iter.Release()
+
+		// Assert the iterator returns the expected key-values.
+		numIterElts := 0
+		for iter.Next() {
+			val := iter.Value()
+			if len(val) == 0 {
+				val = nil
+			}
+			keyStr := string(iter.Key())
+			require.Equal(expectedList[numIterElts], keyStr)
+			require.Equal(expected[keyStr], val)
+			numIterElts++
+		}
+		require.Len(expectedList, numIterElts)
 
 		// Clear the database for the next fuzz iteration.
 		require.NoError(AtomicClear(db, db))

database/versiondb/db_test.go

@@ -27,6 +27,10 @@ func FuzzNewIteratorWithPrefix(f *testing.F) {
 	database.FuzzNewIteratorWithPrefix(f, New(memdb.New()))
 }
 
+func FuzzNewIteratorWithStartAndPrefix(f *testing.F) {
+	database.FuzzNewIteratorWithStartAndPrefix(f, New(memdb.New()))
+}
+
 func TestIterate(t *testing.T) {
 	require := require.New(t)
 

ids/test_generator.go

@@ -23,3 +23,12 @@ func GenerateTestShortID() ShortID {
 func GenerateTestNodeID() NodeID {
 	return NodeID(GenerateTestShortID())
 }
+
+// BuildTestNodeID is an utility to build NodeID from bytes in UTs
+// It must not be used in production code. In production code we should
+// use ToNodeID, which performs proper length checking.
+func BuildTestNodeID(src []byte) NodeID {
+	res := NodeID{}
+	copy(res[:], src)
+	return res
+}