Add Injection and NoSyncFS (#380)

Use a NoSyncFileSystem to skip Sync/FSync to reduce test times
speedb-io · Apr 30, 2023 · ab5e8d5 · ab5e8d5
1 parent da4c0d7
commit ab5e8d5
Show file tree

Hide file tree

Showing 17 changed files with 1,081 additions and 468 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -945,6 +945,7 @@ set(SOURCES
         utilities/fault_injection_env.cc
         utilities/fault_injection_fs.cc
         utilities/fault_injection_secondary_cache.cc
+        utilities/injection_fs.cc
         utilities/leveldb_options/leveldb_options.cc
         utilities/memory/memory_util.cc
         utilities/merge_operators.cc
@@ -955,6 +956,7 @@ set(SOURCES
         utilities/merge_operators/string_append/stringappend.cc
         utilities/merge_operators/string_append/stringappend2.cc
         utilities/merge_operators/uint64add.cc
+        utilities/nosync_fs.cc
         utilities/object_registry.cc
         utilities/option_change_migration/option_change_migration.cc
         utilities/options/options_util.cc

diff --git a/TARGETS b/TARGETS
@@ -289,6 +289,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "utilities/fault_injection_env.cc",
         "utilities/fault_injection_fs.cc",
         "utilities/fault_injection_secondary_cache.cc",
+        "utilities/injection_fs.cc",
         "utilities/leveldb_options/leveldb_options.cc",
         "utilities/memory/memory_util.cc",
         "utilities/merge_operators.cc",
@@ -299,6 +300,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "utilities/merge_operators/string_append/stringappend.cc",
         "utilities/merge_operators/string_append/stringappend2.cc",
         "utilities/merge_operators/uint64add.cc",
+        "utilities/nosync_fs.cc",
         "utilities/object_registry.cc",
         "utilities/option_change_migration/option_change_migration.cc",
         "utilities/options/options_util.cc",
@@ -635,6 +637,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "utilities/fault_injection_env.cc",
         "utilities/fault_injection_fs.cc",
         "utilities/fault_injection_secondary_cache.cc",
+        "utilities/injection_fs.cc",
         "utilities/leveldb_options/leveldb_options.cc",
         "utilities/memory/memory_util.cc",
         "utilities/merge_operators.cc",
@@ -645,6 +648,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "utilities/merge_operators/string_append/stringappend.cc",
         "utilities/merge_operators/string_append/stringappend2.cc",
         "utilities/merge_operators/uint64add.cc",
+        "utilities/nosync_fs.cc",
         "utilities/object_registry.cc",
         "utilities/option_change_migration/option_change_migration.cc",
         "utilities/options/options_util.cc",

diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc
@@ -76,7 +76,7 @@ class CompactionStatsCollector : public EventListener {
 class DBCompactionTest : public DBTestBase {
  public:
   DBCompactionTest()
-      : DBTestBase("db_compaction_test", /*env_do_fsync=*/true) {}
+      : DBTestBase("db_compaction_test", /*env_do_fsync=*/false) {}
 
  protected:
   /*

diff --git a/db/db_with_timestamp_test_util.h b/db/db_with_timestamp_test_util.h
@@ -17,7 +17,7 @@ namespace ROCKSDB_NAMESPACE {
 class DBBasicTestWithTimestampBase : public DBTestBase {
  public:
   explicit DBBasicTestWithTimestampBase(const std::string& dbname)
-      : DBTestBase(dbname, /*env_do_fsync=*/true) {}
+      : DBTestBase(dbname, /*env_do_fsync=*/false) {}
 
  protected:
   static std::string Key1(uint64_t k);

diff --git a/env/env_test.cc b/env/env_test.cc
@@ -67,6 +67,7 @@
 #include "utilities/env_timed.h"
 #include "utilities/fault_injection_env.h"
 #include "utilities/fault_injection_fs.h"
+#include "utilities/nosync_fs.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -3339,6 +3340,37 @@ TEST_F(CreateEnvTest, CreateCompositeEnv) {
   ASSERT_OK(ValidateOptions(db_opts, cf_opts));
 }
 
+TEST_F(CreateEnvTest, CreateNoSyncFileSystem) {
+  std::shared_ptr<FileSystem> fs, copy;
+  auto lib = config_options_.registry->AddLibrary("test");
+  test::RegisterTestObjects(*(lib.get()), "");
+  ASSERT_OK(FileSystem::CreateFromString(config_options_,
+                                         NoSyncFileSystem::kClassName(), &fs));
+  ASSERT_NE(fs, nullptr);
+  ASSERT_STREQ(fs->Name(), NoSyncFileSystem::kClassName());
+  ASSERT_EQ(fs->Inner(), FileSystem::Default().get());
+
+  std::string opts_str = fs->ToString(config_options_);
+  std::string mismatch;
+
+  ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, &copy));
+  ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch));
+
+  ASSERT_OK(FileSystem::CreateFromString(
+      config_options_,
+      std::string("id=") + NoSyncFileSystem::kClassName() +
+          "; target=" + TimedFileSystem::kClassName(),
+      &fs));
+  ASSERT_NE(fs, nullptr);
+  opts_str = fs->ToString(config_options_);
+  ASSERT_STREQ(fs->Name(), NoSyncFileSystem::kClassName());
+  ASSERT_NE(fs->Inner(), nullptr);
+  ASSERT_STREQ(fs->Inner()->Name(), TimedFileSystem::kClassName());
+  ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get());
+  ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, &copy));
+  ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch));
+}
+
 // Forward declaration
 class ReadAsyncFS;
 

diff --git a/include/rocksdb/file_system.h b/include/rocksdb/file_system.h
@@ -1830,7 +1830,7 @@ class FSDirectoryWrapper : public FSDirectory {
     return target_->GetUniqueId(id, max_size);
   }
 
- private:
+ protected:
   std::unique_ptr<FSDirectory> guard_;
   FSDirectory* target_;
 };

diff --git a/src.mk b/src.mk
@@ -275,6 +275,7 @@ LIB_SOURCES =                                                   \
   utilities/fault_injection_env.cc                              \
   utilities/fault_injection_fs.cc                               \
   utilities/fault_injection_secondary_cache.cc                  \
+  utilities/injection_fs.cc                                     \
   utilities/leveldb_options/leveldb_options.cc                  \
   utilities/memory/memory_util.cc                               \
   utilities/merge_operators.cc                                  \
@@ -285,6 +286,7 @@ LIB_SOURCES =                                                   \
   utilities/merge_operators/string_append/stringappend2.cc      \
   utilities/merge_operators/uint64add.cc                        \
   utilities/merge_operators/bytesxor.cc                         \
+  utilities/nosync_fs.cc                                        \
   utilities/object_registry.cc                                  \
   utilities/option_change_migration/option_change_migration.cc  \
   utilities/options/options_util.cc                             \

diff --git a/table/table_test.cc b/table/table_test.cc
@@ -75,6 +75,7 @@
 #include "util/string_util.h"
 #include "utilities/memory_allocators.h"
 #include "utilities/merge_operators.h"
+#include "utilities/nosync_fs.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -571,6 +572,8 @@ class DBConstructor : public Constructor {
   explicit DBConstructor(const Comparator* cmp)
       : Constructor(cmp), comparator_(cmp) {
     db_ = nullptr;
+    std::shared_ptr<FileSystem> fs(new NoSyncFileSystem(FileSystem::Default()));
+    env_ = NewCompositeEnv(fs);
     NewDB();
   }
   ~DBConstructor() override { delete db_; }
@@ -604,6 +607,7 @@ class DBConstructor : public Constructor {
 
     Options options;
     options.comparator = comparator_;
+    options.env = env_.get();
     Status status = DestroyDB(name, options);
     ASSERT_TRUE(status.ok()) << status.ToString();
 
@@ -616,6 +620,7 @@ class DBConstructor : public Constructor {
 
   const Comparator* comparator_;
   DB* db_;
+  std::unique_ptr<Env> env_;
 };
 
 enum TestType {

diff --git a/test_util/testutil.cc b/test_util/testutil.cc
@@ -29,6 +29,7 @@
 #include "test_util/mock_time_env.h"
 #include "test_util/sync_point.h"
 #include "util/random.h"
+#include "utilities/nosync_fs.h"
 
 #ifndef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
 void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
@@ -707,6 +708,13 @@ int RegisterTestObjects(ObjectLibrary& library, const std::string& arg) {
         guard->reset(new MockSystemClock(SystemClock::Default()));
         return guard->get();
       });
+  library.AddFactory<FileSystem>(
+      NoSyncFileSystem::kClassName(),
+      [](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
+         std::string* /* errmsg */) {
+        guard->reset(new NoSyncFileSystem(FileSystem::Default()));
+        return guard->get();
+      });
   return static_cast<int>(library.GetFactoryCount(&num_types));
 }