Merge branch 'main' into alex/21429_build_systemtest

* main:
  docs: amend docs for 52 changes  (#21992)
  test: migrate e2e/authz to system tests (#21819)
  refactor(runtime/v2): use StoreBuilder (#21989)
  feat(schema): add API descriptors, struct, oneof & list types, and wire encoding spec (#21482)
  docs: add instructions to change DefaultGenesis (#21680)
  feat(x/staking)!: Add metadata field to validator info (#21315)
  chore(x/authz)!: Remove account keeper dependency (#21632)

docs/build/building-apps/06-app-go-genesis.md

@@ -0,0 +1,47 @@
+---
+sidebar_position: 1
+---
+
+### Modifying the `DefaultGenesis`
+
+It is possible to modify the DefaultGenesis parameters for modules by wrapping the module, providing it to the `*module.Manager` and injecting it with `depinject`.
+
+Example ( staking ) :
+
+```go
+type CustomStakingModule struct {
+    staking.AppModule
+    cdc codec.Codec
+}
+
+// DefaultGenesis will override the Staking module DefaultGenesis AppModuleBasic method.
+func (cm CustomStakingModule) DefaultGenesis() json.RawMessage {
+    params := stakingtypes.DefaultParams()
+    params.BondDenom = "mydenom"
+
+    return cm.cdc.MustMarshalJSON(&stakingtypes.GenesisState{
+        Params: params,
+    })
+}
+
+// option 1 ( for depinject users ): override previous module manager
+depinject.Inject(
+// ... provider/invoker/supplier
+&moduleManager,
+)
+
+oldStakingModule,_ := moduleManager.Modules()[stakingtypes.ModuleName].(staking.AppModule)
+moduleManager.Modules()[stakingtypes.ModuleName] = CustomStakingModule{
+	AppModule: oldStakingModule,
+	cdc: appCodec,
+}
+
+// option 2 ( for non depinject users ): use new module manager
+moduleManager := module.NewManagerFromMap(map[string]appmodule.AppModule{
+stakingtypes.ModuleName: CustomStakingModule{cdc: appCodec, AppModule: staking.NewAppModule(...)},
+// other modules ...
+})
+
+// set the module manager
+app.ModuleManager = moduleManager
+```

docs/build/building-modules/06-beginblock-endblock.md

@@ -24,7 +24,7 @@ When needed, `BeginBlocker` and `EndBlocker` are implemented as part of the [`Ha
 
 The actual implementation of `BeginBlocker` and `EndBlocker` in `abci.go` are very similar to that of a [`Msg` service](./03-msg-services.md):
 
-* They generally use the [`keeper`](./06-keeper.md) and [`ctx`](../../learn/advanced/02-context.md) to retrieve information about the latest state.
+* They generally use the [`keeper`](./06-keeper.md) and [`ctx`](https://pkg.go.dev/context) to retrieve information about the latest state.
 * If needed, they use the `keeper` and `ctx` to trigger state-transitions.
 * If needed, they can emit [`events`](../../learn/advanced/08-events.md) via the `environments`'s `EventManager`.
 
@@ -35,13 +35,20 @@ It is possible for developers to define the order of execution between the `Begi
 See an example implementation of `BeginBlocker` from the `distribution` module:
 
 ```go reference
-https://github.com/cosmos/cosmos-sdk/blob/v0.50.0-alpha.0/x/distribution/abci.go#L14-L38
+https://github.com/cosmos/cosmos-sdk/blob/v0.52.0-beta.1/x/distribution/keeper/abci.go#L13-L40
 ```
 
-and an example implementation of `EndBlocker` from the `staking` module:
+and an example of `EndBlocker` from the `gov` module:
 
 ```go reference
-https://github.com/cosmos/cosmos-sdk/blob/v0.50.0-alpha.0/x/staking/keeper/abci.go#L22-L27
+https://github.com/cosmos/cosmos-sdk/blob/v0.52.0-beta.1/x/gov/keeper/abci.go#L22
 ```
 
+and an example implementation of `EndBlocker` with validator updates from the `staking` module:
+
+```go reference
+https://github.com/cosmos/cosmos-sdk/blob/v0.52.0-beta.1/x/staking/keeper/abci.go#L12-L17
+```
+
+
 <!-- TODO: leaving this here to update docs with core api changes  -->

docs/build/building-modules/06-keeper.md

@@ -41,14 +41,14 @@ type Keeper struct {
 For example, here is the type definition of the `keeper` from the `staking` module:
 
 ```go reference
-https://github.com/cosmos/cosmos-sdk/blob/v0.50.0-alpha.0/x/staking/keeper/keeper.go#L23-L31
+https://github.com/cosmos/cosmos-sdk/blob/v0.52.0-beta.1/x/staking/keeper/keeper.go#L54-L115 
 ```
 
 Let us go through the different parameters:
 
 * An expected `keeper` is a `keeper` external to a module that is required by the internal `keeper` of said module. External `keeper`s are listed in the internal `keeper`'s type definition as interfaces. These interfaces are themselves defined in an `expected_keepers.go` file in the root of the module's folder. In this context, interfaces are used to reduce the number of dependencies, as well as to facilitate the maintenance of the module itself.
 * `KVStoreService`s grant access to the store(s) of the [multistore](../../learn/advanced/04-store.md) managed by the module. They should always remain unexposed to external modules.
-* `cdc` is the [codec](../../learn/advanced/05-encoding.md) used to marshall and unmarshall structs to/from `[]byte`. The `cdc` can be any of `codec.BinaryCodec`, `codec.JSONCodec` or `codec.Codec` based on your requirements. It can be either a proto or amino codec as long as they implement these interfaces.
+* `cdc` is the [codec](../../learn/advanced/05-encoding.md) used to marshal and unmarshal structs to/from `[]byte`. The `cdc` can be any of `codec.BinaryCodec`, `codec.JSONCodec` or `codec.Codec` based on your requirements. It can be either a proto or amino codec as long as they implement these interfaces.
 * The authority listed is a module account or user account that has the right to change module level parameters. Previously this was handled by the param module, which has been deprecated.
 
 Of course, it is possible to define different types of internal `keeper`s for the same module (e.g. a read-only `keeper`). Each type of `keeper` comes with its own constructor function, which is called from the [application's constructor function](../../learn/beginner/00-app-anatomy.md). This is where `keeper`s are instantiated, and where developers make sure to pass correct instances of modules' `keeper`s to other modules that require them.
@@ -60,3 +60,10 @@ Of course, it is possible to define different types of internal `keeper`s for th
 <!-- markdown-link-check-disable -->
 State management is recommended to be done via [Collections](../packages/collections) 
 <!-- The above link is created via the script to generate docs  -->
+
+## State Management
+
+In the Cosmos SDK, it is crucial to be methodical and selective when managing state within a module, as improper state management can lead to inefficiency, security risks, and scalability issues. Not all data belongs in the on-chain state; it's important to store only essential blockchain data that needs to be verified by consensus. Storing unnecessary information, especially client-side data, can bloat the state and slow down performance. Instead, developers should focus on using an off-chain database to handle supplementary data, extending the API as needed. This approach minimizes on-chain complexity, optimizes resource usage, and keeps the blockchain state lean and efficient, ensuring scalability and smooth operations.
+
+
+The Cosmos SDK leverages Protocol Buffers (protobuf) for efficient state management, providing a well-structured, binary encoding format that ensures compatibility and performance across different modules. The SDK’s recommended approach for managing state is through the [collections package](../pacakges/02-collections.md), which simplifies state handling by offering predefined data structures like maps and indexed sets, reducing the complexity of managing raw state data. While users can opt for custom encoding schemes if they need more flexibility or have specialized requirements, they should be aware that such custom implementations may not integrate seamlessly with indexers that decode state data on the fly. This could lead to challenges in data retrieval, querying, and interoperability, making protobuf a safer and more future-proof choice for most use cases.

docs/build/building-modules/12-errors.md

@@ -14,6 +14,10 @@ common or general errors which can be further wrapped to provide additional spec
 
 There are two ways to return errors. You can register custom errors with a codespace that is meant to provide more information to clients and normal go errors. The Cosmos SDK uses a mixture of both. 
 
+:::Note
+Errors v2 has been created as a zero dependency errors package. GRPC errors and tracing support is removed natively from the errors package. Users are required to wrap stack traces and add tracing information to their errors.
+:::
+
 :::Warning
 If errors are registered they are part of consensus and cannot be changed in a minor release
 :::

docs/learn/advanced/05-encoding.md

@@ -47,7 +47,7 @@ via Protobuf. This means that modules may use Protobuf encoding, but the types m
 implement `ProtoMarshaler`. If modules wish to avoid implementing this interface
 for their types, this is autogenerated via [buf](https://buf.build/)
 
-If modules use [Collections](../../build/packages/02-collections.md) or [ORM](../../build/packages/03-orm.md), encoding and decoding are handled, marshal and unmarshal should not be handled manually unless for specific cases identified by the developer.
+Modules are recommended to use [collections](../../build/packages/02-collections.md) for handling encoding and decoding of state. Usage of collections handles marshal and unmarshal for you. By default protobuf is used but other encodings can be used if preferred. 
 
 ### Gogoproto
 
@@ -78,15 +78,15 @@ the consensus engine accepts only transactions in the form of raw bytes.
 * The `TxDecoder` object performs the decoding.
 
 ```go reference
-https://github.com/cosmos/cosmos-sdk/blob/v0.50.0-alpha.0/types/tx_msg.go#L91-L95
+https://github.com/cosmos/cosmos-sdk/blob/v0.52.0-beta.1/types/tx_msg.go#L91-L95
 ```
 
 A standard implementation of both these objects can be found in the [`auth/tx` module](https://docs.cosmos.network/main/build/modules/auth#transactions):
 
-https://github.com/cosmos/cosmos-sdk/blob/v0.50.0-alpha.0/x/auth/tx/decoder.go
+https://github.com/cosmos/cosmos-sdk/blob/v0.52.0-beta.1/x/auth/tx/decoder.go
 
 ```go reference
-https://github.com/cosmos/cosmos-sdk/blob/v0.50.0-alpha.0/x/auth/tx/encoder.go
+https://github.com/cosmos/cosmos-sdk/blob/v0.52.0-beta.1/x/auth/tx/encoder.go
 ```
 
 See [ADR-020](../../architecture/adr-020-protobuf-transaction-encoding.md) for details of how a transaction is encoded.
@@ -245,39 +245,3 @@ Protobuf types can be defined to encode:
 
 We encourage developers to follow industry guidelines: [Protocol Buffers style guide](https://developers.google.com/protocol-buffers/docs/style)
 and [Buf](https://buf.build/docs/style-guide), see more details in [ADR 023](../../architecture/adr-023-protobuf-naming.md)
-
-### How to update modules to protobuf encoding
-
-If modules do not contain any interfaces (e.g. `Account` or `Content`), then they
-may simply migrate any existing types that
-are encoded and persisted via their concrete Amino codec to Protobuf (see 1. for further guidelines) and accept a `Marshaler` as the codec which is implemented via the `ProtoCodec`
-without any further customization.
-
-However, if a module type composes an interface, it must wrap it in the `sdk.Any` (from `/types` package) type. To do that, a module-level .proto file must use [`google.protobuf.Any`](https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/any.proto) for respective message type interface types.
-
-For example, in the `x/evidence` module defines an `Evidence` interface, which is used by the `MsgSubmitEvidence`. The structure definition must use `sdk.Any` to wrap the evidence file. In the proto file we define it as follows:
-
-```protobuf
-// proto/cosmos/evidence/v1beta1/tx.proto
-
-message MsgSubmitEvidence {
-  string              submitter = 1;
-  google.protobuf.Any evidence  = 2 [(cosmos_proto.accepts_interface) = "cosmos.evidence.v1beta1.Evidence"];
-}
-```
-
-The Cosmos SDK `codec.Codec` interface provides support methods `MarshalInterface` and `UnmarshalInterface` to easy encoding of state to `Any`.
-
-Module should register interfaces using `InterfaceRegistry` which provides a mechanism for registering interfaces: `RegisterInterface(protoName string, iface interface{}, impls ...proto.Message)` and implementations: `RegisterImplementations(iface interface{}, impls ...proto.Message)` that can be safely unpacked from Any, similarly to type registration with Amino:
-
-```go reference
-https://github.com/cosmos/cosmos-sdk/blob/v0.50.0-alpha.0/codec/types/interface_registry.go#L28-L75
-```
-
-In addition, an `UnpackInterfaces` phase should be introduced to deserialization to unpack interfaces before they're needed. Protobuf types that contain a protobuf `Any` either directly or via one of their members should implement the `UnpackInterfacesMessage` interface:
-
-```go
-type UnpackInterfacesMessage interface {
-  UnpackInterfaces(InterfaceUnpacker) error
-}
-```

runtime/v2/builder.go

@@ -6,28 +6,22 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"path/filepath"
 
 	"cosmossdk.io/core/appmodule"
 	appmodulev2 "cosmossdk.io/core/appmodule/v2"
-	"cosmossdk.io/core/server"
 	"cosmossdk.io/core/store"
 	"cosmossdk.io/core/transaction"
 	"cosmossdk.io/runtime/v2/services"
 	"cosmossdk.io/server/v2/appmanager"
 	"cosmossdk.io/server/v2/stf"
 	"cosmossdk.io/server/v2/stf/branch"
-	"cosmossdk.io/store/v2/db"
-	rootstore "cosmossdk.io/store/v2/root"
 )
 
 // AppBuilder is a type that is injected into a container by the runtime/v2 module
 // (as *AppBuilder) which can be used to create an app which is compatible with
 // the existing app.go initialization conventions.
 type AppBuilder[T transaction.Tx] struct {
-	app          *App[T]
-	config       server.DynamicConfig
-	storeOptions *rootstore.Options
+	app *App[T]
 
 	// the following fields are used to overwrite the default
 	branch      func(state store.ReaderMap) store.WriterMap
@@ -99,6 +93,10 @@ func (a *AppBuilder[T]) Build(opts ...AppBuilderOption[T]) (*App[T], error) {
 		}
 	}
 
+	if a.app.db == nil {
+		return nil, fmt.Errorf("app.db is not set, it is required to build the app")
+	}
+
 	if err := a.app.moduleManager.RegisterServices(a.app); err != nil {
 		return nil, err
 	}
@@ -122,37 +120,6 @@ func (a *AppBuilder[T]) Build(opts ...AppBuilderOption[T]) (*App[T], error) {
 	}
 	a.app.stf = stf
 
-	home := a.config.GetString(FlagHome)
-	scRawDb, err := db.NewDB(
-		db.DBType(a.config.GetString("store.app-db-backend")),
-		"application",
-		filepath.Join(home, "data"),
-		nil,
-	)
-	if err != nil {
-		panic(err)
-	}
-
-	var storeOptions rootstore.Options
-	if a.storeOptions != nil {
-		storeOptions = *a.storeOptions
-	} else {
-		storeOptions = rootstore.DefaultStoreOptions()
-	}
-	factoryOptions := &rootstore.FactoryOptions{
-		Logger:    a.app.logger,
-		RootDir:   home,
-		Options:   storeOptions,
-		StoreKeys: append(a.app.storeKeys, "stf"),
-		SCRawDB:   scRawDb,
-	}
-
-	rs, err := rootstore.CreateRootStore(factoryOptions)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create root store: %w", err)
-	}
-	a.app.db = rs
-
 	appManagerBuilder := appmanager.Builder[T]{
 		STF:                a.app.stf,
 		DB:                 a.app.db,
@@ -251,9 +218,3 @@ func AppBuilderWithPostTxExec[T transaction.Tx](postTxExec func(ctx context.Cont
 		a.postTxExec = postTxExec
 	}
 }
-
-func AppBuilderWithStoreOptions[T transaction.Tx](opts *rootstore.Options) AppBuilderOption[T] {
-	return func(a *AppBuilder[T]) {
-		a.storeOptions = opts
-	}
-}

runtime/v2/module.go

@@ -16,6 +16,7 @@ import (
 	reflectionv1 "cosmossdk.io/api/cosmos/reflection/v1"
 	appmodulev2 "cosmossdk.io/core/appmodule/v2"
 	"cosmossdk.io/core/comet"
+	"cosmossdk.io/core/event"
 	"cosmossdk.io/core/header"
 	"cosmossdk.io/core/registry"
 	"cosmossdk.io/core/server"
@@ -26,6 +27,7 @@ import (
 	"cosmossdk.io/log"
 	"cosmossdk.io/runtime/v2/services"
 	"cosmossdk.io/server/v2/stf"
+	rootstore "cosmossdk.io/store/v2/root"
 )
 
 var (
@@ -40,19 +42,19 @@ type appModule[T transaction.Tx] struct {
 func (m appModule[T]) IsOnePerModuleType() {}
 func (m appModule[T]) IsAppModule()        {}
 
-func (m appModule[T]) RegisterServices(registar grpc.ServiceRegistrar) error {
+func (m appModule[T]) RegisterServices(registrar grpc.ServiceRegistrar) error {
 	autoCliQueryService, err := services.NewAutoCLIQueryService(m.app.moduleManager.modules)
 	if err != nil {
 		return err
 	}
 
-	autocliv1.RegisterQueryServer(registar, autoCliQueryService)
+	autocliv1.RegisterQueryServer(registrar, autoCliQueryService)
 
 	reflectionSvc, err := services.NewReflectionService()
 	if err != nil {
 		return err
 	}
-	reflectionv1.RegisterReflectionServiceServer(registar, reflectionSvc)
+	reflectionv1.RegisterReflectionServiceServer(registrar, reflectionSvc)
 
 	return nil
 }
@@ -97,6 +99,7 @@ func init() {
 			ProvideAppBuilder[transaction.Tx],
 			ProvideEnvironment[transaction.Tx],
 			ProvideModuleManager[transaction.Tx],
+			ProvideStoreBuilder,
 		),
 		appconfig.Invoke(SetupAppBuilder),
 	)
@@ -146,7 +149,12 @@ type AppInputs struct {
 	InterfaceRegistrar registry.InterfaceRegistrar
 	LegacyAmino        registry.AminoRegistrar
 	Logger             log.Logger
-	DynamicConfig      server.DynamicConfig `optional:"true"` // can be nil in client wiring
+	// StoreBuilder is a builder for a store/v2 RootStore satisfying the Store interface
+	StoreBuilder *StoreBuilder
+	// StoreOptions are required as input for the StoreBuilder. If not provided, the default options are used.
+	StoreOptions *rootstore.Options `optional:"true"`
+	// DynamicConfig can be nil in client wiring, but is required in server wiring.
+	DynamicConfig server.DynamicConfig `optional:"true"`
 }
 
 func SetupAppBuilder(inputs AppInputs) {
@@ -157,8 +165,22 @@ func SetupAppBuilder(inputs AppInputs) {
 	app.moduleManager.RegisterInterfaces(inputs.InterfaceRegistrar)
 	app.moduleManager.RegisterLegacyAminoCodec(inputs.LegacyAmino)
 
-	if inputs.DynamicConfig != nil {
-		inputs.AppBuilder.config = inputs.DynamicConfig
+	if inputs.DynamicConfig == nil {
+		return
+	}
+	storeOptions := rootstore.DefaultStoreOptions()
+	if inputs.StoreOptions != nil {
+		storeOptions = *inputs.StoreOptions
+	}
+	var err error
+	app.db, err = inputs.StoreBuilder.Build(
+		inputs.Logger,
+		app.storeKeys,
+		inputs.DynamicConfig,
+		storeOptions,
+	)
+	if err != nil {
+		panic(err)
 	}
 }
 
@@ -178,6 +200,7 @@ func ProvideEnvironment[T transaction.Tx](
 	appBuilder *AppBuilder[T],
 	kvFactory store.KVStoreServiceFactory,
 	headerService header.Service,
+	eventService event.Service,
 ) (
 	appmodulev2.Environment,
 	store.KVStoreService,
@@ -209,7 +232,7 @@ func ProvideEnvironment[T transaction.Tx](
 	env := appmodulev2.Environment{
 		Logger:             logger,
 		BranchService:      stf.BranchService{},
-		EventService:       stf.NewEventService(),
+		EventService:       eventService,
 		GasService:         stf.NewGasMeterService(),
 		HeaderService:      headerService,
 		QueryRouterService: stf.NewQueryRouterService(),
@@ -254,10 +277,12 @@ func DefaultServiceBindings() depinject.Config {
 		}
 		headerService header.Service = services.NewGenesisHeaderService(stf.HeaderService{})
 		cometService  comet.Service  = &services.ContextAwareCometInfoService{}
+		eventService                 = stf.NewEventService()
 	)
 	return depinject.Supply(
 		kvServiceFactory,
 		headerService,
 		cometService,
+		eventService,
 	)
 }