tests: add robustness test for issue 17780

Signed-off-by: Wei Fu <fuweid89@gmail.com>
fuweid · Jun 12, 2024 · 69152bf · 69152bf
1 parent 5790774
commit 69152bf
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 0 deletions.
diff --git a/tests/robustness/failpoint/gofail.go b/tests/robustness/failpoint/gofail.go
@@ -59,6 +59,15 @@ var (
 	BeforeApplyOneConfChangeSleep            Failpoint = killAndGofailSleep{"beforeApplyOneConfChange", time.Second}
 	RaftBeforeSaveSleep                      Failpoint = gofailSleepAndDeactivate{"raftBeforeSave", time.Second}
 	RaftAfterSaveSleep                       Failpoint = gofailSleepAndDeactivate{"raftAfterSave", time.Second}
+
+	// AllowBatchCompactBeforeSetFinishedCompactPanic is used to trigger
+	// that compactBeforeSetFinishedCompact failpoint only if the current
+	// revision number is higher than that batch limit.
+	AllowBatchCompactBeforeSetFinishedCompactPanic Failpoint = goPanicFailpoint{
+		failpoint: "compactBeforeSetFinishedCompact",
+		trigger:   triggerCompact{multiBatchCompaction: true},
+		target:    AnyMember,
+	}
 )
 
 type goPanicFailpoint struct {

diff --git a/tests/robustness/makefile.mk b/tests/robustness/makefile.mk
@@ -39,6 +39,11 @@ test-robustness-issue15271: /tmp/etcd-v3.5.7-failpoints/bin
 	GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue15271 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.7-failpoints/bin' make test-robustness && \
 	 echo "Failed to reproduce" || echo "Successful reproduction"
 
+.PHONY: test-robustness-issue17780
+test-robustness-issue17780: /tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/bin
+	GO_TEST_FLAGS='-v --run=TestRobustnessRegression/Issue17780 --count 100 --failfast --bin-dir=/tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/bin' make test-robustness && \
+	  echo "Failed to reproduce" || echo "Successful reproduction"
+
 # Failpoints
 
 GOPATH = $(shell go env GOPATH)
@@ -84,6 +89,19 @@ $(GOPATH)/bin/gofail: tools/mod/go.mod tools/mod/go.sum
 	  make gofail-enable; \
 	  make build;
 
+# REF: https://github.com/etcd-io/etcd/commit/6b034466aa0ac2b46fe01fb5bd2233946f46d453
+/tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/bin: $(GOPATH)/bin/gofail
+	rm -rf /tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/
+	mkdir -p /tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/
+	cd /tmp/etcd-3.5.6b034466aa0ac2b46fe01fb5bd2233946f46d453-failpoints/; \
+		git clone --branch release-3.5 https://github.com/etcd-io/etcd.git .; \
+		git checkout 6b034466aa0ac2b46fe01fb5bd2233946f46d453; \
+		go get go.etcd.io/gofail@${GOFAIL_VERSION}; \
+		(cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+		(cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+		(cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+		FAILPOINTS=true ./build;
+
 /tmp/etcd-v3.5.2-failpoints/bin:
 /tmp/etcd-v3.5.4-failpoints/bin:
 /tmp/etcd-v3.5.5-failpoints/bin:

diff --git a/tests/robustness/scenarios.go b/tests/robustness/scenarios.go
@@ -199,5 +199,17 @@ func regressionScenarios(t *testing.T) []testScenario {
 			cluster:   *e2e.NewConfig(opts...),
 		})
 	}
+	scenarios = append(scenarios, testScenario{
+		name:      "Issue17780",
+		failpoint: failpoint.AllowBatchCompactBeforeSetFinishedCompactPanic,
+		profile:   traffic.LowTraffic,
+		traffic:   traffic.Issue17780EtcdPutDelete,
+		cluster: *e2e.NewConfig(
+			e2e.WithClusterSize(1),
+			e2e.WithCompactionBatchLimit(100),
+			e2e.WithSnapshotCount(1000),
+			e2e.WithGoFailEnabled(true),
+		),
+	})
 	return scenarios
 }
diff --git a/tests/robustness/traffic/etcd.go b/tests/robustness/traffic/etcd.go
@@ -63,6 +63,18 @@ var (
 			{choice: Compact, weight: 5},
 		},
 	}
+	// Issue17780EtcdPutDelete is to create high chance to have more delete
+	// requests so that it's likely to compact that revision which is tombstone.
+	Issue17780EtcdPutDelete = etcdTraffic{
+		keyCount:     20,
+		leaseTTL:     DefaultLeaseTTL,
+		largePutSize: 32769,
+		requests: []choiceWeight[etcdRequestType]{
+			{choice: List, weight: 50},
+			{choice: Delete, weight: 20},
+			{choice: Put, weight: 10},
+		},
+	}
 )
 
 type etcdTraffic struct {