Replace runsim with Go stdlib testing

cosmos · May 30, 2024 · 53e82b1 · 53e82b1
1 parent ac49374
commit 53e82b1
Show file tree

Hide file tree

Showing 15 changed files with 653 additions and 593 deletions.
diff --git a/.github/workflows/sims.yml b/.github/workflows/sims.yml
@@ -2,10 +2,13 @@ name: Sims
 # Sims workflow runs multiple types of simulations (nondeterminism, import-export, after-import, multi-seed-short)
 # This workflow will run on all Pull Requests, if a .go, .mod or .sum file have been changed
 on:
-  schedule:
-    - cron: "0 */2 * * *"
-  release:
-    types: [published]
+  push:
+    branches:
+      - main
+      - release/**
+  pull_request:
+permissions:
+  contents: read
 
 concurrency:
   group: ci-${{ github.ref }}-sims
@@ -15,7 +18,7 @@ jobs:
   build:
     permissions:
       contents: read # for actions/checkout to fetch code
-    runs-on: ubuntu-latest
+    runs-on: large-sdk-runner
     if: "!contains(github.event.head_commit.message, 'skip-sims')"
     steps:
       - uses: actions/checkout@v4
@@ -24,15 +27,9 @@ jobs:
           go-version: "1.22"
           check-latest: true
       - run: make build
-      - name: Install runsim
-        run: go install github.com/cosmos/tools/cmd/runsim@v1.0.0
-      - uses: actions/cache@v4
-        with:
-          path: ~/go/bin
-          key: ${{ runner.os }}-go-runsim-binary
 
   test-sim-import-export:
-    runs-on: ubuntu-latest
+    runs-on: large-sdk-runner
     needs: [build]
     timeout-minutes: 60
     steps:
@@ -41,33 +38,38 @@ jobs:
         with:
           go-version: "1.22"
           check-latest: true
-      - uses: actions/cache@v4
-        with:
-          path: ~/go/bin
-          key: ${{ runner.os }}-go-runsim-binary
       - name: test-sim-import-export
         run: |
           make test-sim-import-export
 
   test-sim-after-import:
-    runs-on: ubuntu-latest
+    runs-on: large-sdk-runner
     needs: [build]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
           go-version: "1.22"
           check-latest: true
-      - uses: actions/cache@v4
-        with:
-          path: ~/go/bin
-          key: ${{ runner.os }}-go-runsim-binary
       - name: test-sim-after-import
         run: |
           make test-sim-after-import
 
+  test-sim-deterministic:
+    runs-on: large-sdk-runner
+    needs: [build]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+          check-latest: true
+      - name: test-sim-nondeterminism-streaming
+        run: |
+          make test-sim-nondeterminism-streaming
+
   test-sim-multi-seed-short:
-    runs-on: ubuntu-latest
+    runs-on: large-sdk-runner
     needs: [build]
     timeout-minutes: 60
     steps:
@@ -76,55 +78,65 @@ jobs:
         with:
           go-version: "1.22"
           check-latest: true
-      - uses: actions/cache@v4
-        with:
-          path: ~/go/bin
-          key: ${{ runner.os }}-go-runsim-binary
       - name: test-sim-multi-seed-short
         run: |
           make test-sim-multi-seed-short
 
-  sims-notify-success:
-    needs:
-      [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export]
-    runs-on: ubuntu-latest
-    if: ${{ success() }}
+  test-sim-fuzz:
+    runs-on: large-sdk-runner
+    needs: [build]
+    timeout-minutes: 60
     steps:
-      - name: Check out repository
-        uses: actions/checkout@v4
-      - name: Get previous workflow status
-        uses: ./.github/actions/last-workflow-status
-        id: last_status
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Notify Slack on success
-        if: ${{ steps.last_status.outputs.last_status == 'failure' }}
-        uses: rtCamp/action-slack-notify@v2.3.0
-        env:
-          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-          SLACK_CHANNEL: sdk-sims
-          SLACK_USERNAME: Sim Tests
-          SLACK_ICON_EMOJI: ":white_check_mark:"
-          SLACK_COLOR: good
-          SLACK_MESSAGE: Sims are passing
-          SLACK_FOOTER: ""
+          go-version: "1.22"
+          check-latest: true
+      - name: test-sim-fuzz
+        run: |
+          make test-sim-fuzz
 
-  sims-notify-failure:
-    permissions:
-      contents: none
-    needs:
-      [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export]
-    runs-on: ubuntu-latest
-    if: ${{ failure() }}
-    steps:
-      - name: Notify Slack on failure
-        uses: rtCamp/action-slack-notify@v2.3.0
-        env:
-          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-          SLACK_CHANNEL: sdk-sims
-          SLACK_USERNAME: Sim Tests
-          SLACK_ICON_EMOJI: ":skull:"
-          SLACK_COLOR: danger
-          SLACK_MESSAGE: Sims are failing
-          SLACK_FOOTER: ""
+#  sims-notify-success:
+#    needs:
+#      [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export]
+#    runs-on: large-sdk-runner
+#    if: ${{ success() }}
+#    steps:
+#      - name: Check out repository
+#        uses: actions/checkout@v4
+#      - name: Get previous workflow status
+#        uses: ./.github/actions/last-workflow-status
+#        id: last_status
+#        with:
+#          github_token: ${{ secrets.GITHUB_TOKEN }}
+#
+#      - name: Notify Slack on success
+#        if: ${{ steps.last_status.outputs.last_status == 'failure' }}
+#        uses: rtCamp/action-slack-notify@v2.3.0
+#        env:
+#          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+#          SLACK_CHANNEL: sdk-sims
+#          SLACK_USERNAME: Sim Tests
+#          SLACK_ICON_EMOJI: ":white_check_mark:"
+#          SLACK_COLOR: good
+#          SLACK_MESSAGE: Sims are passing
+#          SLACK_FOOTER: ""
+#
+#  sims-notify-failure:
+#    permissions:
+#      contents: none
+#    needs:
+#      [test-sim-multi-seed-short, test-sim-after-import, test-sim-import-export]
+#    runs-on: large-sdk-runner
+#    if: ${{ failure() }}
+#    steps:
+#      - name: Notify Slack on failure
+#        uses: rtCamp/action-slack-notify@v2.3.0
+#        env:
+#          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+#          SLACK_CHANNEL: sdk-sims
+#          SLACK_USERNAME: Sim Tests
+#          SLACK_ICON_EMOJI: ":skull:"
+#          SLACK_COLOR: danger
+#          SLACK_MESSAGE: Sims are failing
+#          SLACK_FOOTER: ""
diff --git a/Makefile b/Makefile
@@ -103,9 +103,6 @@ endif
 #? all: Run tools build 
 all: build
 
-# The below include contains the tools and runsim targets.
-include contrib/devtools/Makefile
-
 ###############################################################################
 ###                                  Build                                  ###
 ###############################################################################
@@ -158,7 +155,7 @@ $(MOCKS_DIR):
 	mkdir -p $(MOCKS_DIR)
 
 #? distclean: Run `make clean` and `make tools-clean`
-distclean: clean tools-clean
+distclean: clean
 
 #? clean: Clean some auto generated directory
 clean:
@@ -280,8 +277,8 @@ endif
 #? test-sim-nondeterminism: Run non-determinism test for simapp
 test-sim-nondeterminism:
 	@echo "Running non-determinism test..."
-	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -run TestAppStateDeterminism -Enabled=true \
-		-NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -run TestAppStateDeterminism \
+		-NumBlocks=100 -BlockSize=200 -Period=0
 
 # Requires an exported plugin. See store/streaming/README.md for documentation.
 #
@@ -294,35 +291,41 @@ test-sim-nondeterminism:
 #   make test-sim-nondeterminism-streaming
 test-sim-nondeterminism-streaming:
 	@echo "Running non-determinism-streaming test..."
-	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -run TestAppStateDeterminism -Enabled=true \
-		-NumBlocks=100 -BlockSize=200 -Commit=true -Period=0 -v -timeout 24h -EnableStreaming=true
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -run TestAppStateDeterminism \
+		-NumBlocks=100 -BlockSize=200 -Period=0 -EnableStreaming=true
 
 test-sim-custom-genesis-fast:
 	@echo "Running custom genesis simulation..."
 	@echo "By default, ${HOME}/.simapp/config/genesis.json will be used."
-	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
-		-Enabled=true -NumBlocks=100 -BlockSize=200 -Commit=true -Seed=99 -Period=5 -SigverifyTx=false -v -timeout 24h
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=30m -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
+		-NumBlocks=100 -BlockSize=200 -Seed=99 -Period=5 -SigverifyTx=false
 
-test-sim-import-export: runsim
+test-sim-import-export:
 	@echo "Running application import/export simulation. This may take several minutes..."
-	@cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 5 TestAppImportExport
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 20m -run TestAppImportExport \
+		-NumBlocks=50 -Period=5
 
-test-sim-after-import: runsim
+test-sim-after-import:
 	@echo "Running application simulation-after-import. This may take several minutes..."
-	@cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 5 TestAppSimulationAfterImport
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -run TestAppSimulationAfterImport \
+		-NumBlocks=50 -Period=5
+
 
-test-sim-custom-genesis-multi-seed: runsim
+test-sim-custom-genesis-multi-seed:
 	@echo "Running multi-seed custom genesis simulation..."
 	@echo "By default, ${HOME}/.simapp/config/genesis.json will be used."
-	@cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Genesis=${HOME}/.simapp/config/genesis.json -SigverifyTx=false -SimAppPkg=. -ExitOnFail 400 5 TestFullAppSimulation
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m  -run TestFullAppSimulation -Genesis=${HOME}/.simapp/config/genesis.json \
+		-NumBlocks=400 -Period=5
 
-test-sim-multi-seed-long: runsim
+test-sim-multi-seed-long:
 	@echo "Running long multi-seed application simulation. This may take awhile!"
-	@cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 500 50 TestFullAppSimulation
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout=1h -run TestFullAppSimulation \
+		-NumBlocks=500 -Period=50
 
-test-sim-multi-seed-short: runsim
+test-sim-multi-seed-short:
 	@echo "Running short multi-seed application simulation. This may take awhile!"
-	@cd ${CURRENT_DIR}/simapp && $(BINDIR)/runsim -Jobs=4 -SimAppPkg=. -ExitOnFail 50 10 TestFullAppSimulation
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -timeout 30m -run TestFullAppSimulation \
+		-NumBlocks=50 -Period=10
 
 test-sim-benchmark-invariants:
 	@echo "Running simulation invariant benchmarks..."
@@ -345,6 +348,12 @@ SIM_NUM_BLOCKS ?= 500
 SIM_BLOCK_SIZE ?= 200
 SIM_COMMIT ?= true
 
+#? test-sim-fuzz: Run fuzz test for simapp
+test-sim-fuzz:
+	@echo "Running application fuzz for numBlocks=2, blockSize=20. This may take awhile!"
+#ld flags are a quick fix to make it work on current osx
+	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -json -ldflags="-extldflags=-Wl,-ld_classic" -timeout=60m -fuzztime=60m -run=^$$ -fuzz=FuzzFullAppSimulation -GenesisTime=1714720615 -NumBlocks=2 -BlockSize=20
+
 #? test-sim-benchmark: Run benchmark test for simapp
 test-sim-benchmark:
 	@echo "Running application benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
@@ -384,7 +393,7 @@ test-sim-profile-streaming:
 	@cd ${CURRENT_DIR}/simapp && go test -mod=readonly -benchmem -run=^$$ $(.) -bench ^BenchmarkFullAppSimulation$$ \
 		-Enabled=true -NumBlocks=$(SIM_NUM_BLOCKS) -BlockSize=$(SIM_BLOCK_SIZE) -Commit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out -EnableStreaming=true
 
-.PHONY: test-sim-profile test-sim-benchmark
+.PHONY: test-sim-profile test-sim-benchmark test-sim-fuzz
 
 #? benchmark: Run benchmark tests
 benchmark:

diff --git a/baseapp/baseapp.go b/baseapp/baseapp.go
@@ -1165,3 +1165,8 @@ func (app *BaseApp) Close() error {
 
 	return errors.Join(errs...)
 }
+
+// GetBaseApp returns the pointer to itself.
+func (app *BaseApp) GetBaseApp() *BaseApp {
+	return app
+}
diff --git a/contrib/devtools/Makefile b/contrib/devtools/Makefile