Add an end-to-end test for updating corpus database.

.github/workflows/bazel_test.yml

@@ -63,7 +63,15 @@ jobs:
         if: matrix.config == 'fuzztest'
         run: |
           bazel test --build_tests_only --test_output=errors \
-            -c ${{ matrix.compilation_mode }} --config=fuzztest //e2e_tests:all
+            -c ${{ matrix.compilation_mode }} --config=fuzztest //e2e_tests:functional_test
+      - name: Run end-to-end tests with --config=fuzztest-experimental
+        if: matrix.config == 'fuzztest'
+        run: |
+          bazel test --build_tests_only --test_output=errors \
+            -c ${{ matrix.compilation_mode }} \
+            --config=fuzztest-experimental --config=asan \
+            --platform_suffix=fuzztest-experimental-asan \
+            //e2e_tests:corpus_database_test
       - name: Save new cache based on main
         if: github.ref == 'refs/heads/main'
         uses: actions/cache/save@v4

bazel/setup_configs.sh

@@ -4,7 +4,8 @@
 
 set -euf -o pipefail
 
-echo "### DO NOT EDIT. Generated file.
+cat <<EOF
+### DO NOT EDIT. Generated file.
 #
 # To regenerate, run the following from your project's workspace:
 #
@@ -13,16 +14,16 @@ echo "### DO NOT EDIT. Generated file.
 # And don't forget to add the following to your project's .bazelrc:
 #
 #  try-import %workspace%/fuzztest.bazelrc
-"
+EOF
 
-echo "
+cat <<EOF
 ### Common options.
 #
 # Do not use directly.
 
 # Compile and link with Address Sanitizer (ASAN).
-build:fuzztest-common --linkopt=-fsanitize=address
-build:fuzztest-common --copt=-fsanitize=address
+build:asan-common --linkopt=-fsanitize=address
+build:asan-common --copt=-fsanitize=address
 
 # Standard define for \"ifdef-ing\" any fuzz test specific code.
 build:fuzztest-common --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
@@ -33,13 +34,16 @@ build:fuzztest-common --copt=-UNDEBUG
 # Enable libc++ assertions.
 # See https://libcxx.llvm.org/UsingLibcxx.html#enabling-the-safe-libc-mode
 build:fuzztest-common --copt=-D_LIBCPP_ENABLE_ASSERTIONS=1
-"
+EOF
 
-echo "
+cat <<EOF
 ### FuzzTest build configuration.
 #
 # Use with: --config=fuzztest
+#
+# Note that this configuration includes the ASan configuration (defined below).
 
+build:fuzztest --config=asan-common
 build:fuzztest --config=fuzztest-common
 
 # Link statically.
@@ -49,31 +53,75 @@ build:fuzztest --dynamic_mode=off
 # __has_feature(address_sanitizer) to know that we have an ASAN build even in
 # the uninstrumented runtime.
 build:fuzztest --copt=-DADDRESS_SANITIZER
-"
+EOF
 
 REPO_NAME="${1}"
 # When used in the fuzztest repo itself.
 if [[ ${REPO_NAME} == "@" ]]; then
+  COMMON_FILTER="//common:"
   FUZZTEST_FILTER="//fuzztest:"
-  CENTIPEDE_FILTER="//centipede:"
+  CENTIPEDE_FILTER="//centipede:,-//centipede/.*fuzz_target"
 # When used in client repo. This matches both `WORKSPACE` usage and
 # `MODULE.bazel` usage which will prepend information to the repo name to form
 # a canonical repo name.
 #
 # TODO: This will need to be adjusted when making `fuzztest` a native Bazel
 # module.
 elif [[ ${REPO_NAME} =~ ^@.*com_google_fuzztest$ ]]; then
+  COMMON_FILTER="common/.*"
   FUZZTEST_FILTER="fuzztest/.*"
-  CENTIPEDE_FILTER="centipede/.*"
+  CENTIPEDE_FILTER="centipede/.*,-centipede/.*fuzz_target"
 else
   echo "Unexpected repo name: ${REPO_NAME}"
   exit 1
 fi
 
-echo "# We apply coverage tracking instrumentation to everything but the
+cat <<EOF
+# We apply coverage tracking instrumentation to everything but Centipede and the
+# FuzzTest framework itself (including GoogleTest and GoogleMock).
+build:fuzztest --copt=-fsanitize-coverage=inline-8bit-counters,trace-cmp,pc-table
+build:fuzztest --per_file_copt=${COMMON_FILTER},${FUZZTEST_FILTER},${CENTIPEDE_FILTER},googletest/.*,googlemock/.*@-fsanitize-coverage=0
+EOF
+
+cat <<EOF
+### ASan build configuration.
+#
+# Use with: --config=asan
+
+build:asan --config=asan-common
+EOF
+
+cat <<EOF
+### Experimental FuzzTest build configuration.
+#
+# Use with: --config=fuzztest-experimental
+#
+# Use this instead of --config=fuzztest when building test binaries to run with
+# Centipede. Eventually, this will be consolidated with --config=fuzztest.
+# Note that this configuration doesn't include the ASan configuration. If you
+# want to use both, you can use --config=fuzztest-experimental --config=asan.
+
+build:fuzztest-experimental --config=fuzztest-common
+build:fuzztest-experimental --@com_google_fuzztest//fuzztest:centipede_integration
+
+# Generate line tables for debugging.
+build:fuzztest-experimental --copt=-gline-tables-only
+build:fuzztest-experimental --strip=never
+
+# Prevent memcmp & co from being inlined.
+build:fuzztest-experimental --copt=-fno-builtin
+
+# Disable heap checking.
+build:fuzztest-experimental --copt=-DHEAPCHECK_DISABLE
+
+# Link statically.
+build:fuzztest-experimental --dynamic_mode=off
+
+# We apply coverage tracking instrumentation to everything but Centipede and the
 # FuzzTest framework itself (including GoogleTest and GoogleMock).
-build:fuzztest --per_file_copt=+//,-${FUZZTEST_FILTER},-${CENTIPEDE_FILTER},-googletest/.*,-googlemock/.*@-fsanitize-coverage=inline-8bit-counters,-fsanitize-coverage=trace-cmp,-fsanitize-coverage=pc-table
-"
+build:fuzztest-experimental --copt=-fsanitize-coverage=trace-pc-guard,pc-table,trace-loads,trace-cmp,control-flow
+build:fuzztest-experimental --per_file_copt=${COMMON_FILTER},${FUZZTEST_FILTER},${CENTIPEDE_FILTER},googletest/.*,googlemock/.*@-fsanitize-coverage=0
+EOF
 
 # Do not use the extra configurations below, unless you know what you're doing.
 
@@ -98,31 +146,32 @@ if [[ -z "${LLVM_CONFIG}" ]]; then
   exit 1
 fi
 
-echo "
+cat <<EOF
 ### libFuzzer compatibility mode.
 #
 # Use with: --config=libfuzzer
 
+build:libfuzzer --config=asan-common
 build:libfuzzer --config=fuzztest-common
 build:libfuzzer --copt=-DFUZZTEST_COMPATIBILITY_MODE
 build:libfuzzer --copt=-fsanitize=fuzzer-no-link
 build:libfuzzer --linkopt=$(find $(${LLVM_CONFIG} --libdir) -name libclang_rt.fuzzer_no_main-x86_64.a | head -1)
-"
+EOF
 
 fi # libFuzzer
 
 
 # OSS-Fuzz
 if [[ -n ${FUZZING_ENGINE:-} && -n ${SANITIZER:-} ]]; then
-echo "
+cat <<EOF
 ### OSS-Fuzz compatibility mode.
 #
 # Use with: --config=oss-fuzz
 build:oss-fuzz --copt=-DFUZZTEST_COMPATIBILITY_MODE
 build:oss-fuzz --dynamic_mode=off
 build:oss-fuzz --action_env=CC=${CC}
 build:oss-fuzz --action_env=CXX=${CXX}
-"
+EOF
 
 ossfuz_flag_to_bazel_config_flag()
 {

e2e_tests/BUILD

@@ -60,6 +60,7 @@ cc_test(
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/time",
+        "@com_google_fuzztest//centipede:weak_sancov_stubs",
         "@com_google_fuzztest//domain_tests:domain_testing",
         "@com_google_fuzztest//fuzztest:io",
         "@com_google_fuzztest//fuzztest:logging",
@@ -90,3 +91,24 @@ cc_binary(
         "@com_googlesource_code_re2//:re2",
     ],
 )
+
+# Must be run with `--config=fuzztest-experimental --config=asan`.
+cc_test(
+    name = "corpus_database_test",
+    srcs = ["corpus_database_test.cc"],
+    data = [
+        "@com_google_fuzztest//centipede:centipede_uninstrumented",
+        "@com_google_fuzztest//e2e_tests/testdata:fuzz_tests_for_corpus_database_testing.stripped",
+    ],
+    local_defines = select({
+        "@com_google_fuzztest//fuzztest:use_centipede": ["FUZZTEST_USE_CENTIPEDE"],
+        "//conditions:default": [],
+    }),
+    deps = [
+        ":test_binary_util",
+        "@com_google_absl//absl/log:check",
+        "@com_google_absl//absl/strings",
+        "@com_google_fuzztest//centipede:weak_sancov_stubs",
+        "@com_google_googletest//:gtest_main",
+    ],
+)

e2e_tests/corpus_database_test.cc

@@ -0,0 +1,103 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <filesystem>  // NOLINT
+#include <string>
+#include <utility>
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "absl/log/check.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
+#include "./e2e_tests/test_binary_util.h"
+
+namespace fuzztest::internal {
+namespace {
+
+using ::testing::HasSubstr;
+
+class UpdateCorpusDatabaseTest : public testing::Test {
+ protected:
+  static void SetUpTestSuite() {
+#if defined(__has_feature)
+#if !__has_feature(address_sanitizer)
+    GTEST_SKIP() << "Skipping the tests because the test binary is not built "
+                    "with ASAN. Please run with --config=asan.";
+#elif !__has_feature(coverage_sanitizer) || !defined(FUZZTEST_USE_CENTIPEDE)
+    GTEST_SKIP() << "Skipping the tests because the test binary is not built "
+                    "with coverage instrumentation for Centipede. "
+    "Please run with --config=fuzztest-experimental.";
+#endif
+#endif
+
+    temp_dir_ = new TempDir();
+
+    auto [status, std_out, std_err] = RunBinary(
+        CentipedePath(),
+        {.flags = {
+             {"binary",
+              absl::StrCat(BinaryPath((std::filesystem::path("testdata") /
+                                       "fuzz_tests_for_corpus_database_testing")
+                                          .c_str()),
+                           " ",
+                           CreateFuzzTestFlag("corpus_database",
+                                              GetCorpusDatabasePath()),
+                           " ", CreateFuzzTestFlag("fuzz_for", "30s"))}}});
+
+    centipede_std_out_ = new std::string(std::move(std_out));
+    centipede_std_err_ = new std::string(std::move(std_err));
+  }
+
+  static void TearDownTestSuite() {
+    delete temp_dir_;
+    temp_dir_ = nullptr;
+    delete centipede_std_out_;
+    centipede_std_out_ = nullptr;
+    delete centipede_std_err_;
+    centipede_std_err_ = nullptr;
+  }
+
+  static std::string GetCorpusDatabasePath() {
+    CHECK(temp_dir_ != nullptr);
+    return std::filesystem::path(temp_dir_->dirname()) / "corpus_database";
+  }
+
+  static absl::string_view GetCentipedeStdOut() {
+    CHECK(centipede_std_out_ != nullptr);
+    return *centipede_std_out_;
+  }
+
+  static absl::string_view GetCentipedeStdErr() {
+    CHECK(centipede_std_err_ != nullptr);
+    return *centipede_std_err_;
+  }
+
+ private:
+  static TempDir *temp_dir_;
+  static std::string *centipede_std_out_;
+  static std::string *centipede_std_err_;
+};
+
+TempDir *UpdateCorpusDatabaseTest::temp_dir_ = nullptr;
+std::string *UpdateCorpusDatabaseTest::centipede_std_out_ = nullptr;
+std::string *UpdateCorpusDatabaseTest::centipede_std_err_ = nullptr;
+
+TEST_F(UpdateCorpusDatabaseTest, RunsFuzzTests) {
+  EXPECT_THAT(GetCentipedeStdErr(),
+              HasSubstr("Fuzzing FuzzTest.FailsInTwoWays"));
+}
+
+}  // namespace
+}  // namespace fuzztest::internal

e2e_tests/testdata/BUILD

@@ -142,3 +142,13 @@ cc_binary(
         "@com_google_fuzztest//fuzztest:fuzztest_gtest_main",
     ],
 )
+
+cc_binary(
+    name = "fuzz_tests_for_corpus_database_testing",
+    testonly = 1,
+    srcs = ["fuzz_tests_for_corpus_database_testing.cc"],
+    deps = [
+        "@com_google_fuzztest//fuzztest",
+        "@com_google_fuzztest//fuzztest:fuzztest_gtest_main",
+    ],
+)

e2e_tests/testdata/CMakeLists.txt

@@ -95,4 +95,15 @@ set_target_properties(
   unit_test_and_fuzz_tests.stripped
   PROPERTIES RUNTIME_OUTPUT_DIRECTORY
   "${CMAKE_BINARY_DIR}/_main/e2e_tests/testdata"
-)
+)
+
+add_executable(
+  fuzz_tests_for_corpus_database_testing.stripped
+  fuzz_tests_for_corpus_database_testing.cc
+)
+link_fuzztest(fuzz_tests_for_corpus_database_testing.stripped)
+set_target_properties(
+  fuzz_tests_for_corpus_database_testing.stripped
+  PROPERTIES RUNTIME_OUTPUT_DIRECTORY
+  "${CMAKE_BINARY_DIR}/_main/e2e_tests/testdata"
+)

e2e_tests/testdata/fuzz_tests_for_corpus_database_testing.cc

@@ -0,0 +1,29 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "./fuzztest/fuzztest.h"
+
+namespace {
+
+volatile int force_write = 0;
+
+void FailsInTwoWays(const std::vector<int>& v) {
+  if (v.size() % 7 != 1) return;
+  ASSERT_NE(v[0], 2025);
+  if (v[0] == 2 * 2025) force_write = v.data()[v.size()];
+}
+FUZZ_TEST(FuzzTest, FailsInTwoWays);
+
+}  // namespace

fuzztest/BUILD

@@ -206,11 +206,6 @@ cc_library(
     srcs = ["internal/centipede_adaptor.cc"],
     hdrs = ["internal/centipede_adaptor.h"],
     defines = ["FUZZTEST_USE_CENTIPEDE"],
-    linkopts = [
-        # Needed for linking the Centipede engine with the runner, due to
-        # the common source code built separately for the engine and runner.
-        "-Wl,--warn-backrefs-exclude=*/centipede/*",
-    ],
     deps = [
         ":any",
         ":configuration",