nbd: unmap wait process exit

nbd/src/NBDTool.cpp

@@ -110,9 +110,27 @@ int NBDTool::Connect(NBDConfig *cfg) {
     return 0;
 }
 
-int NBDTool::Disconnect(const std::string& devpath) {
+int NBDTool::Disconnect(const NBDConfig* config) {
+    pid_t devpid = -1;
+    std::vector<DeviceInfo> devices;
+
+    int ret = List(&devices);
+    for (const auto& device : devices) {
+        if (device.config.devpath == config->devpath) {
+            devpid = device.pid;
+            break;
+        }
+    }
+
     NBDControllerPtr nbdCtrl = GetController(false);
-    return nbdCtrl->DisconnectByPath(devpath);
+    ret = nbdCtrl->DisconnectByPath(config->devpath);
+    if (ret != 0) {
+        return ret;
+    }
+
+    ret = WaitForTerminate(devpid, config);
+
+    return 0;
 }
 
 int NBDTool::List(std::vector<DeviceInfo>* infos) {
@@ -137,6 +155,8 @@ void NBDTool::RunServerUntilQuit() {
     } else {
         ctrl->RunUntilQuit();
     }
+
+    nbdWatchCtx_->StopWatch();
 }
 
 ImagePtr g_test_image = nullptr;
@@ -150,5 +170,29 @@ ImagePtr NBDTool::GenerateImage(const std::string& imageName) {
     return result;
 }
 
+int NBDTool::WaitForTerminate(pid_t pid, const NBDConfig* config) {
+    if (pid < 0) {
+        return 0;
+    }
+
+    int times = config->retry_times;
+    while (times-- > 0) {
+        if (kill(pid, 0) == -1) {
+            if (errno == ESRCH) {
+                return 0;
+            }
+            std::cerr << "curve-nbd test device failed, dev: "
+                      << config->devpath << ", err = " << cpp_strerror(-errno)
+                      << std::endl;
+            return -errno;
+        }
+
+        std::this_thread::sleep_for(
+            std::chrono::milliseconds(config->sleep_ms));
+    }
+
+    return -ETIMEDOUT;
+}
+
 }  // namespace nbd
 }  // namespace curve

nbd/src/NBDTool.h

@@ -52,8 +52,8 @@ class NBDTool {
 
     // 与nbd内核建立通信，将文件映射到本地
     int Connect(NBDConfig *cfg);
-    // 根据设备路径卸载已经映射的文件
-    int Disconnect(const std::string& devpath);
+    // 根据配置卸载已经映射的文件
+    int Disconnect(const NBDConfig* config);
     // 获取已经映射文件的映射信息，包括进程pid、文件名、设备路径
     int List(std::vector<DeviceInfo>* infos);
     // 阻塞直到nbd server退出
@@ -68,6 +68,9 @@ class NBDTool {
     // 生成image instance
     ImagePtr GenerateImage(const std::string& imageName);
 
+    // wait curve-nbd process to exit
+    int WaitForTerminate(pid_t pid, const NBDConfig* config);
+
  private:
     class NBDSocketPair {
      public:

nbd/src/define.h

@@ -81,6 +81,10 @@ struct NBDConfig {
     std::string imgname;
     // 指定需要映射的nbd设备路径
     std::string devpath;
+    // unmap等待进程退出的重试次数
+    int retry_times = 25;
+    // unmap重试之间的睡眠间隔
+    int sleep_ms = 200;
 };
 
 // 用户命令类型

nbd/src/main.cpp

@@ -66,7 +66,7 @@ static void HandleSignal(int signum) {
     std::cout << "Got signal " << sys_siglist[signum] << "\n"
               << ", disconnect now" << std::endl;
 
-    ret = nbdTool->Disconnect(nbdConfig->devpath);
+    ret = nbdTool->Disconnect(nbdConfig.get());
     if (ret != 0) {
         std::cout << "curve-nbd: disconnect failed. Error: " << ret
                   << std::endl;
@@ -77,19 +77,21 @@ static void HandleSignal(int signum) {
 
 static void Usage() {
     std::cout
-        << "Usage: curve-nbd [options] map <image>  Map an image "
-           "to nbd device\n"  // NOLINT
-        << "                 unmap <device|image>   Unmap nbd "
-           "device\n"  // NOLINT
-        << "                 [options] list-mapped  List mapped "
-           "nbd devices\n"  // NOLINT
+        << "Usage: curve-nbd [options] map <image>  Map an image to nbd device\n"  // NOLINT
+        << "                 [options] unmap <device|image>   Unmap nbd device\n"  // NOLINT
+        << "                 [options] list-mapped  List mapped nbd devices\n"     // NOLINT
         << "Map options:\n"
         << "  --device <device path>  Specify nbd device path (/dev/nbd{num})\n"
         << "  --read-only             Map read-only\n"
         << "  --nbds_max <limit>      Override for module param nbds_max\n"
         << "  --max_part <limit>      Override for module param max_part\n"
         << "  --timeout <seconds>     Set nbd request timeout\n"
         << "  --try-netlink           Use the nbd netlink interface\n"
+        << "Unmap options:\n"
+        << "  --retry_times <limit>       The number of retries waiting for the process to exit\n"  // NOLINT
+        << "                              (default: " << nbdConfig->retry_times << ")\n"            // NOLINT
+        << "  --sleep_ms <milliseconds>   Retry interval in milliseconds\n"                         // NOLINT
+        << "                              (default: " << nbdConfig->sleep_ms << ")\n"               // NOLINT
         << std::endl;
 }
 
@@ -185,7 +187,7 @@ static int CurveNbdMain(int argc, const char* argv[]) {
             break;
         }
         case Command::Disconnect: {
-            r = nbdTool->Disconnect(nbdConfig->devpath);
+            r = nbdTool->Disconnect(nbdConfig.get());
             if (r < 0) {
                 return -EINVAL;
             }

nbd/src/util.cpp

@@ -201,6 +201,24 @@ int parse_args(std::vector<const char*>& args, std::ostream *err_msg,   // NOLIN
             }
         } else if (argparse_flag(args, i, "--try-netlink", (char *)NULL)) { // NOLINT
             cfg->try_netlink = true;
+        } else if (argparse_witharg(args, i, &cfg->retry_times, err, "--retry_times", (char*)(NULL))) {  // NOLINT
+            if (!err.str().empty()) {
+                *err_msg << "curve-nbd: " << err.str();
+                return -EINVAL;
+            }
+            if (cfg->retry_times < 0) {
+                *err_msg << "curve-nbd: Invalid argument for retry_times!";
+                return -EINVAL;
+            }
+        } else if (argparse_witharg(args, i, &cfg->sleep_ms, err, "--sleep_ms", (char*)(NULL))) {  // NOLINT
+            if (!err.str().empty()) {
+                *err_msg << "curve-nbd: " << err.str();
+                return -EINVAL;
+            }
+            if (cfg->sleep_ms < 0) {
+                *err_msg << "curve-nbd: Invalid argument for sleep_ms!";
+                return -EINVAL;
+            }
         } else {
             ++i;
         }

nbd/test/nbd_tool_test.cpp

@@ -147,7 +147,7 @@ TEST_F(NBDToolTest, ioctl_connect_test) {
     StartInAnotherThread(&config);
     ASSERT_TRUE(isRunning_);
     AssertWriteSuccess(config.devpath);
-    ASSERT_EQ(0, tool_.Disconnect(config.devpath));
+    ASSERT_EQ(0, tool_.Disconnect(&config));
     sleep(1);
     ASSERT_FALSE(isRunning_);
 }
@@ -158,7 +158,7 @@ TEST_F(NBDToolTest, netlink_connect_test) {
     config.try_netlink = true;
     StartInAnotherThread(&config);
     ASSERT_TRUE(isRunning_);
-    ASSERT_EQ(0, tool_.Disconnect(config.devpath));
+    ASSERT_EQ(0, tool_.Disconnect(&config));
     sleep(1);
     ASSERT_FALSE(isRunning_);
 }
@@ -170,7 +170,7 @@ TEST_F(NBDToolTest, readonly_test) {
     StartInAnotherThread(&config);
     ASSERT_TRUE(isRunning_);
     AssertWriteFailed(config.devpath);
-    ASSERT_EQ(0, tool_.Disconnect(config.devpath));
+    ASSERT_EQ(0, tool_.Disconnect(&config));
     sleep(1);
     ASSERT_FALSE(isRunning_);
 }
@@ -183,7 +183,7 @@ TEST_F(NBDToolTest, timeout_test) {
     ASSERT_TRUE(isRunning_);
     AssertWriteTimeout(config.devpath, 5);
     // io timeout 情况下，nbd_do_it这边会退出，所以这里disconnect会失败
-    ASSERT_EQ(-1, tool_.Disconnect(config.devpath));
+    ASSERT_EQ(-1, tool_.Disconnect(&config));
 }
 
 }  // namespace nbd

robot/Resources/keywords/test_curve_stability_nbd.py

@@ -705,6 +705,7 @@ def test_snapshot_all(vol_uuid):
     test_clone_iovol_consistency(lazy)
 #    test_clone_vol_same_uuid(lazy)
     test_recover_snapshot(lazy)
+    config.snapshot_thrash.nbd_unmap()
     config.snapshot_thrash.nbd_delete()
     return "finally"