Skip to content

Commit

Permalink
Extract utils for running test binaries from fuctional_test to separa…
Browse files Browse the repository at this point in the history
…te library.

PiperOrigin-RevId: 687437701
  • Loading branch information
fniksic authored and copybara-github committed Oct 18, 2024
1 parent 44f5a57 commit 8ca03f0
Show file tree
Hide file tree
Showing 6 changed files with 260 additions and 97 deletions.
17 changes: 17 additions & 0 deletions e2e_tests/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,22 @@ package(default_visibility = ["//visibility:private"])

licenses(["notice"])

cc_library(
name = "test_binary_util",
testonly = 1,
srcs = ["test_binary_util.cc"],
hdrs = ["test_binary_util.h"],
deps = [
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:string_view",
"@com_google_absl//absl/time",
"@com_google_fuzztest//fuzztest:flag_name",
"@com_google_fuzztest//fuzztest:subprocess",
],
)

# Can be run with or without `--config=fuzztest`. When running without, the
# fuzzing mode tests are skipped.
cc_test(
Expand All @@ -39,6 +55,7 @@ cc_test(
}),
shard_count = 50,
deps = [
":test_binary_util",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
Expand Down
19 changes: 19 additions & 0 deletions e2e_tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,24 @@ include_directories(${PARENT_DIR}/domain_tests)

add_subdirectory(testdata)

fuzztest_cc_library(
NAME
test_binary_util
HDRS
test_binary_util.h
SRCS
test_binary_util.cc
DEPS
absl::check
absl::flat_hash_map
absl::strings
absl::string_view
absl::time
fuzztest_flag_name
fuzztest_subprocess
TESTONLY
)

add_executable(
functional_test
functional_test.cc
Expand All @@ -17,6 +35,7 @@ target_link_libraries(
fuzztest_printer
fuzztest_serialization
fuzztest_subprocess
fuzztest_test_binary_util
fuzztest_type_support
re2
absl::flat_hash_map
Expand Down
141 changes: 44 additions & 97 deletions e2e_tests/functional_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,14 @@
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "absl/container/flat_hash_map.h"
#include "absl/strings/match.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/string_view.h"
#include "absl/strings/substitute.h"
#include "absl/time/clock.h"
#include "absl/time/time.h"
#include "./domain_tests/domain_testing.h"
#include "./e2e_tests/test_binary_util.h"
#include "./fuzztest/internal/io.h"
#include "./fuzztest/internal/logging.h"
#include "./fuzztest/internal/printer.h"
Expand All @@ -46,8 +46,6 @@
namespace fuzztest::internal {
namespace {

#define FUZZTEST_FLAG_PREFIX_ ""

using ::fuzztest::domain_implementor::PrintMode;
using ::testing::_;
using ::testing::AllOf;
Expand All @@ -68,26 +66,6 @@ using ::testing::StartsWith;
constexpr absl::string_view kDefaultTargetBinary =
"testdata/fuzz_tests_for_functional_testing";

std::string CreateFuzzTestFlag(absl::string_view flag_name,
absl::string_view flag_value) {
return absl::StrCat("--", FUZZTEST_FLAG_PREFIX_, flag_name, "=", flag_value);
}

std::string BinaryPath(const absl::string_view name) {
const auto test_srcdir = absl::NullSafeStringView(getenv("TEST_SRCDIR"));
FUZZTEST_INTERNAL_CHECK_PRECONDITION(
!test_srcdir.empty(),
"Please set TEST_SRCDIR to non-empty value or use bazel to run the "
"test.");
const std::string binary_path = absl::StrCat(
test_srcdir, "/_main/e2e_tests/", name,
absl::EndsWith(name, ".stripped") ? "" : ".stripped");

FUZZTEST_INTERNAL_CHECK(std::filesystem::exists(binary_path),
absl::StrCat("Can't find ", binary_path));
return binary_path;
}

absl::flat_hash_map<std::string, std::string> WithTestSanitizerOptions(
absl::flat_hash_map<std::string, std::string> env) {
if (!env.contains("ASAN_OPTIONS"))
Expand All @@ -97,38 +75,18 @@ absl::flat_hash_map<std::string, std::string> WithTestSanitizerOptions(
return env;
}

class TempDir {
public:
TempDir() {
dirname_ = "/tmp/replay_test_XXXXXX";
dirname_ = mkdtemp(dirname_.data());
EXPECT_TRUE(std::filesystem::is_directory(dirname_));
}

const std::string& dirname() const { return dirname_; }

~TempDir() { std::filesystem::remove_all(dirname_); }

private:
std::string dirname_;
};

class UnitTestModeTest : public ::testing::Test {
protected:
RunResults Run(
absl::string_view test_filter,
absl::string_view target_binary = kDefaultTargetBinary,
const absl::flat_hash_map<std::string, std::string>& env = {},
const absl::flat_hash_map<std::string, std::string>& fuzzer_flags = {}) {
std::vector<std::string> commandline = {
return RunBinary(
BinaryPath(target_binary),
absl::StrCat("--", GTEST_FLAG_PREFIX_, "filter=", test_filter)};

for (const auto& flag : fuzzer_flags) {
commandline.push_back(CreateFuzzTestFlag(flag.first, flag.second));
}
return RunCommand(commandline, WithTestSanitizerOptions(env),
absl::Minutes(10));
{.flags = {{GTEST_FLAG_PREFIX_ "filter", std::string(test_filter)}},
.fuzztest_flags = fuzzer_flags,
.env = WithTestSanitizerOptions(env)});
}
};

Expand Down Expand Up @@ -673,21 +631,6 @@ TEST_F(GetRandomValueTest, SettingPrngSeedReproducesValue) {
EXPECT_EQ(val, other_val);
}

std::string CentipedePath() {
const auto test_srcdir = absl::NullSafeStringView(getenv("TEST_SRCDIR"));
FUZZTEST_INTERNAL_CHECK_PRECONDITION(
!test_srcdir.empty(),
"Please set TEST_SRCDIR to non-empty value or use bazel to run the "
"test.");
const std::string binary_path = absl::StrCat(
test_srcdir,
"/_main/centipede/centipede_uninstrumented");

FUZZTEST_INTERNAL_CHECK(std::filesystem::exists(binary_path),
absl::StrCat("Can't find ", binary_path));
return binary_path;
}

// Tests for the FuzzTest command line interface.
class GenericCommandLineInterfaceTest : public ::testing::Test {
protected:
Expand All @@ -698,14 +641,11 @@ class GenericCommandLineInterfaceTest : public ::testing::Test {
absl::string_view binary = kDefaultTargetBinary,
const absl::flat_hash_map<std::string, std::string>& non_fuzztest_flags =
{}) {
std::vector<std::string> args = {BinaryPath(binary)};
for (const auto& [key, value] : flags) {
args.push_back(CreateFuzzTestFlag(key, value));
}
for (const auto& [key, value] : non_fuzztest_flags) {
args.push_back(absl::StrCat("--", key, "=", value));
}
return RunCommand(args, WithTestSanitizerOptions(env), timeout);
return RunBinary(BinaryPath(binary),
RunOptions{.flags = non_fuzztest_flags,
.fuzztest_flags = flags,
.env = WithTestSanitizerOptions(env),
.timeout = timeout});
}
};

Expand Down Expand Up @@ -1132,10 +1072,10 @@ TEST_F(FuzzingModeCommandLineInterfaceTest,
{{"FUZZTEST_STACK_LIMIT", "512000"}});
EXPECT_THAT(std_err, HasSubstr("argument 0: "));
EXPECT_THAT(std_err,
HasSubstr("Stack limit is set by FUZZTEST_STACK_LIMIT env var "
"- this is going to be deprecated soon. Consider "
"switching to --" FUZZTEST_FLAG_PREFIX_
"stack_limit_kb flag."));
HasSubstr(absl::StrCat(
"Stack limit is set by FUZZTEST_STACK_LIMIT env var - this "
"is going to be deprecated soon. Consider switching to ",
CreateFuzzTestFlag("stack_limit_kb", ""), " flag.")));
ExpectStackLimitExceededMessage(std_err, 512000);
EXPECT_THAT(status, Eq(Signal(SIGABRT)));
}
Expand Down Expand Up @@ -1279,20 +1219,22 @@ class FuzzingModeFixtureTest
RunResults Run(absl::string_view test_name, int iterations) {
if (GetParam().multi_process) {
TempDir workdir;
return RunCommand(
{CentipedePath(), "--print_runner_log", "--exit_on_crash",
absl::StrCat("--workdir=", workdir.dirname()),
absl::StrCat("--binary=", BinaryPath(kDefaultTargetBinary), " ",
CreateFuzzTestFlag("fuzz", test_name)),
absl::StrCat("--num_runs=", iterations)},
/*environment=*/{},
/*timeout=*/absl::InfiniteDuration());
return RunBinary(
CentipedePath(),
{.flags = {{"print_runner_log", "true"},
{"exit_on_crash", "true"},
{"workdir", workdir.dirname()},
{"binary",
absl::StrCat(BinaryPath(kDefaultTargetBinary), " ",
CreateFuzzTestFlag("fuzz", test_name))},
{"num_runs", absl::StrCat(iterations)}},
.timeout = absl::InfiniteDuration()});
} else {
return RunCommand(
{BinaryPath(kDefaultTargetBinary),
CreateFuzzTestFlag("fuzz", test_name)},
{{"FUZZTEST_MAX_FUZZING_RUNS", absl::StrCat(iterations)}},
/*timeout=*/absl::InfiniteDuration());
return RunBinary(
BinaryPath(kDefaultTargetBinary),
{.fuzztest_flags = {{"fuzz", std::string(test_name)}},
.env = {{"FUZZTEST_MAX_FUZZING_RUNS", absl::StrCat(iterations)}},
.timeout = absl::InfiniteDuration()});
}
}

Expand Down Expand Up @@ -1423,17 +1365,22 @@ class FuzzingModeCrashFindingTest
env = WithTestSanitizerOptions(std::move(env));
if (GetParam().multi_process) {
TempDir workdir;
return RunCommand(
{CentipedePath(), "--exit_on_crash", "--timeout_per_input=0",
absl::StrCat("--stop_at=", absl::Now() + timeout),
absl::StrCat("--workdir=", workdir.dirname()),
absl::StrCat("--binary=", BinaryPath(target_binary), " ",
CreateFuzzTestFlag("fuzz", test_name))},
env, timeout + absl::Seconds(10));
return RunBinary(
CentipedePath(),
{.flags = {{"exit_on_crash", "true"},
{"timeout_per_input", "0"},
{"stop_at", absl::StrCat(absl::Now() + timeout)},
{"workdir", workdir.dirname()},
{"binary",
absl::StrCat(BinaryPath(target_binary), " ",
CreateFuzzTestFlag("fuzz", test_name))}},
.env = std::move(env),
.timeout = timeout + absl::Seconds(10)});
} else {
return RunCommand(
{BinaryPath(target_binary), CreateFuzzTestFlag("fuzz", test_name)},
env, timeout);
return RunBinary(BinaryPath(target_binary),
{.fuzztest_flags = {{"fuzz", std::string(test_name)}},
.env = std::move(env),
.timeout = timeout});
}
}

Expand Down
90 changes: 90 additions & 0 deletions e2e_tests/test_binary_util.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "./e2e_tests/test_binary_util.h"

#include <cstdlib>
#include <filesystem> // NOLINT
#include <string>
#include <system_error> // NOLINT
#include <vector>

#include "absl/log/check.h"
#include "absl/strings/match.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "./fuzztest/internal/flag_name.h"
#include "./fuzztest/internal/subprocess.h"

namespace fuzztest::internal {
namespace {

// Returns the full path for `relative_path` given relative to the FuzzTest
// root.
std::string GetFullPath(const std::filesystem::path& relative_path) {
const auto test_srcdir = absl::NullSafeStringView(std::getenv("TEST_SRCDIR"));
CHECK(!test_srcdir.empty()) << "Please set TEST_SRCDIR to non-empty value or "
"use bazel to run the test.";
const std::string full_path =
std::filesystem::path(test_srcdir) / "_main"
/ relative_path;
CHECK(std::filesystem::exists(full_path)) << "Can't find " << full_path;
return full_path;
}

} // namespace

TempDir::TempDir() {
std::error_code error;
dirname_ = std::filesystem::temp_directory_path(error) / "temp_dir_XXXXXX";
CHECK(!error) << "Failed to get the root temp directory path: " << error;
dirname_ = mkdtemp(dirname_.data());
CHECK(std::filesystem::is_directory(dirname_));
}

TempDir::~TempDir() { std::filesystem::remove_all(dirname_); }

std::string CreateFuzzTestFlag(absl::string_view flag_name,
absl::string_view flag_value) {
return absl::StrCat("--", FUZZTEST_FLAG_PREFIX, flag_name,
(flag_value.empty() ? "" : "="), flag_value);
}

std::string BinaryPath(const absl::string_view relative_path) {
return GetFullPath(
std::filesystem::path("e2e_tests") /
absl::StrCat(relative_path, absl::EndsWith(relative_path, ".stripped")
? ""
: ".stripped"));
}

std::string CentipedePath() {
return GetFullPath(std::filesystem::path("centipede") /
"centipede_uninstrumented");
}

RunResults RunBinary(absl::string_view binary_path, const RunOptions& options) {
std::vector<std::string> args;
args.reserve(1 + options.fuzztest_flags.size() + options.flags.size());
args.push_back(std::string(binary_path));
for (const auto& [key, value] : options.fuzztest_flags) {
args.push_back(CreateFuzzTestFlag(key, value));
}
for (const auto& [key, value] : options.flags) {
args.push_back(absl::StrCat("--", key, "=", value));
}
return RunCommand(args, options.env, options.timeout);
}

} // namespace fuzztest::internal
Loading

0 comments on commit 8ca03f0

Please sign in to comment.