From 7baa919888aa232333d3eddb74ceaeb6a5ebd7b2 Mon Sep 17 00:00:00 2001 From: Hanqing Wu Date: Tue, 19 Jan 2021 15:16:51 +0800 Subject: [PATCH] nbd: unmap wait process exit --- nbd/src/NBDTool.cpp | 48 ++++++++++++++++++- nbd/src/NBDTool.h | 7 ++- nbd/src/define.h | 4 ++ nbd/src/main.cpp | 18 +++---- nbd/src/util.cpp | 18 +++++++ nbd/test/nbd_tool_test.cpp | 8 ++-- .../keywords/test_curve_stability_nbd.py | 1 + 7 files changed, 88 insertions(+), 16 deletions(-) diff --git a/nbd/src/NBDTool.cpp b/nbd/src/NBDTool.cpp index 14ffc7fb8e..368c5d63aa 100644 --- a/nbd/src/NBDTool.cpp +++ b/nbd/src/NBDTool.cpp @@ -110,9 +110,27 @@ int NBDTool::Connect(NBDConfig *cfg) { return 0; } -int NBDTool::Disconnect(const std::string& devpath) { +int NBDTool::Disconnect(const NBDConfig* config) { + pid_t devpid = -1; + std::vector devices; + + int ret = List(&devices); + for (const auto& device : devices) { + if (device.config.devpath == config->devpath) { + devpid = device.pid; + break; + } + } + NBDControllerPtr nbdCtrl = GetController(false); - return nbdCtrl->DisconnectByPath(devpath); + ret = nbdCtrl->DisconnectByPath(config->devpath); + if (ret != 0) { + return ret; + } + + ret = WaitForTerminate(devpid, config); + + return 0; } int NBDTool::List(std::vector* infos) { @@ -137,6 +155,8 @@ void NBDTool::RunServerUntilQuit() { } else { ctrl->RunUntilQuit(); } + + nbdWatchCtx_->StopWatch(); } ImagePtr g_test_image = nullptr; @@ -150,5 +170,29 @@ ImagePtr NBDTool::GenerateImage(const std::string& imageName) { return result; } +int NBDTool::WaitForTerminate(pid_t pid, const NBDConfig* config) { + if (pid < 0) { + return 0; + } + + int times = config->retry_times; + while (times-- > 0) { + if (kill(pid, 0) == -1) { + if (errno == ESRCH) { + return 0; + } + std::cerr << "curve-nbd test device failed, dev: " + << config->devpath << ", err = " << cpp_strerror(-errno) + << std::endl; + return -errno; + } + + std::this_thread::sleep_for( + std::chrono::milliseconds(config->sleep_ms)); + } + + return -ETIMEDOUT; +} + } // namespace nbd } // namespace curve diff --git a/nbd/src/NBDTool.h b/nbd/src/NBDTool.h index 9769b8d4f4..3c7d9480fb 100644 --- a/nbd/src/NBDTool.h +++ b/nbd/src/NBDTool.h @@ -52,8 +52,8 @@ class NBDTool { // 与nbd内核建立通信，将文件映射到本地 int Connect(NBDConfig *cfg); - // 根据设备路径卸载已经映射的文件 - int Disconnect(const std::string& devpath); + // 根据配置卸载已经映射的文件 + int Disconnect(const NBDConfig* config); // 获取已经映射文件的映射信息，包括进程pid、文件名、设备路径 int List(std::vector* infos); // 阻塞直到nbd server退出 @@ -68,6 +68,9 @@ class NBDTool { // 生成image instance ImagePtr GenerateImage(const std::string& imageName); + // wait curve-nbd process to exit + int WaitForTerminate(pid_t pid, const NBDConfig* config); + private: class NBDSocketPair { public: diff --git a/nbd/src/define.h b/nbd/src/define.h index 9cb7de01de..114f4ad22e 100644 --- a/nbd/src/define.h +++ b/nbd/src/define.h @@ -81,6 +81,10 @@ struct NBDConfig { std::string imgname; // 指定需要映射的nbd设备路径 std::string devpath; + // unmap等待进程退出的重试次数 + int retry_times = 25; + // unmap重试之间的睡眠间隔 + int sleep_ms = 200; }; // 用户命令类型 diff --git a/nbd/src/main.cpp b/nbd/src/main.cpp index e81bb41184..41eb9fb93f 100644 --- a/nbd/src/main.cpp +++ b/nbd/src/main.cpp @@ -66,7 +66,7 @@ static void HandleSignal(int signum) { std::cout << "Got signal " << sys_siglist[signum] << "\n" << ", disconnect now" << std::endl; - ret = nbdTool->Disconnect(nbdConfig->devpath); + ret = nbdTool->Disconnect(nbdConfig.get()); if (ret != 0) { std::cout << "curve-nbd: disconnect failed. Error: " << ret << std::endl; @@ -77,12 +77,9 @@ static void HandleSignal(int signum) { static void Usage() { std::cout - << "Usage: curve-nbd [options] map Map an image " - "to nbd device\n" // NOLINT - << " unmap Unmap nbd " - "device\n" // NOLINT - << " [options] list-mapped List mapped " - "nbd devices\n" // NOLINT + << "Usage: curve-nbd [options] map Map an image to nbd device\n" // NOLINT + << " [options] unmap Unmap nbd device\n" // NOLINT + << " [options] list-mapped List mapped nbd devices\n" // NOLINT << "Map options:\n" << " --device Specify nbd device path (/dev/nbd{num})\n" << " --read-only Map read-only\n" @@ -90,6 +87,11 @@ static void Usage() { << " --max_part Override for module param max_part\n" << " --timeout Set nbd request timeout\n" << " --try-netlink Use the nbd netlink interface\n" + << "Unmap options:\n" + << " --retry_times The number of retries waiting for the process to exit\n" // NOLINT + << " (default: " << nbdConfig->retry_times << ")\n" // NOLINT + << " --sleep_ms Retry interval in milliseconds\n" // NOLINT + << " (default: " << nbdConfig->sleep_ms << ")\n" // NOLINT << std::endl; } @@ -185,7 +187,7 @@ static int CurveNbdMain(int argc, const char* argv[]) { break; } case Command::Disconnect: { - r = nbdTool->Disconnect(nbdConfig->devpath); + r = nbdTool->Disconnect(nbdConfig.get()); if (r < 0) { return -EINVAL; } diff --git a/nbd/src/util.cpp b/nbd/src/util.cpp index 9715616b62..4ba8eb020d 100644 --- a/nbd/src/util.cpp +++ b/nbd/src/util.cpp @@ -201,6 +201,24 @@ int parse_args(std::vector& args, std::ostream *err_msg, // NOLIN } } else if (argparse_flag(args, i, "--try-netlink", (char *)NULL)) { // NOLINT cfg->try_netlink = true; + } else if (argparse_witharg(args, i, &cfg->retry_times, err, "--retry_times", (char*)(NULL))) { // NOLINT + if (!err.str().empty()) { + *err_msg << "curve-nbd: " << err.str(); + return -EINVAL; + } + if (cfg->retry_times < 0) { + *err_msg << "curve-nbd: Invalid argument for retry_times!"; + return -EINVAL; + } + } else if (argparse_witharg(args, i, &cfg->sleep_ms, err, "--sleep_ms", (char*)(NULL))) { // NOLINT + if (!err.str().empty()) { + *err_msg << "curve-nbd: " << err.str(); + return -EINVAL; + } + if (cfg->sleep_ms < 0) { + *err_msg << "curve-nbd: Invalid argument for sleep_ms!"; + return -EINVAL; + } } else { ++i; } diff --git a/nbd/test/nbd_tool_test.cpp b/nbd/test/nbd_tool_test.cpp index f19aaf5526..d2660c4f6c 100644 --- a/nbd/test/nbd_tool_test.cpp +++ b/nbd/test/nbd_tool_test.cpp @@ -147,7 +147,7 @@ TEST_F(NBDToolTest, ioctl_connect_test) { StartInAnotherThread(&config); ASSERT_TRUE(isRunning_); AssertWriteSuccess(config.devpath); - ASSERT_EQ(0, tool_.Disconnect(config.devpath)); + ASSERT_EQ(0, tool_.Disconnect(&config)); sleep(1); ASSERT_FALSE(isRunning_); } @@ -158,7 +158,7 @@ TEST_F(NBDToolTest, netlink_connect_test) { config.try_netlink = true; StartInAnotherThread(&config); ASSERT_TRUE(isRunning_); - ASSERT_EQ(0, tool_.Disconnect(config.devpath)); + ASSERT_EQ(0, tool_.Disconnect(&config)); sleep(1); ASSERT_FALSE(isRunning_); } @@ -170,7 +170,7 @@ TEST_F(NBDToolTest, readonly_test) { StartInAnotherThread(&config); ASSERT_TRUE(isRunning_); AssertWriteFailed(config.devpath); - ASSERT_EQ(0, tool_.Disconnect(config.devpath)); + ASSERT_EQ(0, tool_.Disconnect(&config)); sleep(1); ASSERT_FALSE(isRunning_); } @@ -183,7 +183,7 @@ TEST_F(NBDToolTest, timeout_test) { ASSERT_TRUE(isRunning_); AssertWriteTimeout(config.devpath, 5); // io timeout 情况下，nbd_do_it这边会退出，所以这里disconnect会失败 - ASSERT_EQ(-1, tool_.Disconnect(config.devpath)); + ASSERT_EQ(-1, tool_.Disconnect(&config)); } } // namespace nbd diff --git a/robot/Resources/keywords/test_curve_stability_nbd.py b/robot/Resources/keywords/test_curve_stability_nbd.py index bcb86ed6a6..e6ab1f52c5 100644 --- a/robot/Resources/keywords/test_curve_stability_nbd.py +++ b/robot/Resources/keywords/test_curve_stability_nbd.py @@ -705,6 +705,7 @@ def test_snapshot_all(vol_uuid): test_clone_iovol_consistency(lazy) # test_clone_vol_same_uuid(lazy) test_recover_snapshot(lazy) + config.snapshot_thrash.nbd_unmap() config.snapshot_thrash.nbd_delete() return "finally"