快照克隆版本镜像及快照删除2

conf/mds.conf

@@ -180,6 +180,12 @@ mds.chunkserverclient.updateLeaderRetryTimes=5
 #  从copyset的每个chunkserver getleader的每一轮的间隔，需大于raft选主的时间
 mds.chunkserverclient.updateLeaderRetryIntervalMs=5000
 
+#
+# snapshotclone config
+#
+# snapshot clone server 地址
+mds.snapshotcloneclient.addr=127.0.0.1:5555
+
 #
 # common options
 #

curve-ansible/roles/generate_config/defaults/main.yml

@@ -70,6 +70,7 @@ mds_chunkserverclient_rpc_retry_times: 5
 mds_chunkserverclient_rpc_retry_interval_ms: 500
 mds_chunkserverclient_update_leader_retry_times: 5
 mds_chunkserverclient_update_leader_retry_interval_ms: 5000
+mds_snapshotcloneclient_addr: 127.0.0.1:5555
 mds_common_log_dir: ./
 
 # chunkserver配置默认值

curve-ansible/roles/generate_config/templates/mds.conf.j2

@@ -188,6 +188,11 @@ mds.chunkserverclient.updateLeaderRetryTimes={{ mds_chunkserverclient_update_lea
 #  从copyset的每个chunkserver getleader的每一轮的间隔，需大于raft选主的时间
 mds.chunkserverclient.updateLeaderRetryIntervalMs={{ mds_chunkserverclient_update_leader_retry_interval_ms }}
 
+# snapshotclone config
+#
+# snapshot clone server 地址
+mds.snapshotcloneclient.addr={{ mds_snapshotcloneclient_addr }}
+
 #
 # common options
 #

proto/nameserver2.proto

@@ -130,6 +130,10 @@ enum  StatusCode {
     kClientVersionNotMatch = 134;
     // snapshot功能禁用中
     kSnapshotFrozen = 135;
+    // 快照克隆服务连不上
+    kSnapshotCloneConnectFail = 136;
+    // 快照克隆服务未初始化
+    kSnapshotCloneServerNotInit = 137;
 
     // 元数据存储错误
     kStorageError = 501;

src/common/BUILD

@@ -38,7 +38,9 @@ cc_library(
     srcs = glob([
         "*.h",
         "*.cpp",
-        ],exclude = ["authenticator.*", "s3_adapter.*"]
+        ],exclude = ["authenticator.*",
+                      "s3_adapter.*",
+                      "snapshotclone_define.*"]
     ),
     copts = COPTS,
     visibility = ["//visibility:public"],
@@ -91,3 +93,15 @@ cc_library(
         "@aws",
     ],
 )
+
+cc_library(
+    name = "curve_snapshotclone",
+    srcs = glob([
+        "snapshotclone_define.*",
+    ]),
+    copts = COPTS,
+    visibility = ["//visibility:public"],
+    deps = [
+        "//external:json"
+    ],
+)

src/snapshotcloneserver/common/define.h → src/common/define.h

@@ -20,8 +20,8 @@
  * Author: xuchaojie
  */
 
-#ifndef SRC_SNAPSHOTCLONESERVER_COMMON_DEFINE_H_
-#define SRC_SNAPSHOTCLONESERVER_COMMON_DEFINE_H_
+#ifndef SRC_COMMON_DEFINE_H_
+#define SRC_COMMON_DEFINE_H_
 
 #include <string>
 #include <map>
@@ -149,4 +149,4 @@ constexpr uint32_t kProgressCloneComplete = 100;
 }  // namespace snapshotcloneserver
 }  // namespace curve
 
-#endif  // SRC_SNAPSHOTCLONESERVER_COMMON_DEFINE_H_
+#endif  // SRC_COMMON_DEFINE_H_

src/snapshotcloneserver/common/define.cpp → src/common/snapshotclone_define.cpp

@@ -22,7 +22,7 @@
 
 #include <json/json.h>
 
-#include "src/snapshotcloneserver/common/define.h"
+#include "src/common/snapshotclone_define.h"
 
 namespace curve {
 namespace snapshotcloneserver {
@@ -40,6 +40,7 @@ const char* kCleanCloneTaskAction = "CleanCloneTask";
 const char* kFlattenAction = "Flatten";
 const char* kGetFileSnapshotListAction = "GetFileSnapshotList";
 const char* kGetCloneTaskListAction = "GetCloneTaskList";
+const char* kGetCloneRefStatusAction = "GetCloneRefStatus";
 
 const char* kActionStr = "Action";
 const char* kVersionStr = "Version";
@@ -54,13 +55,16 @@ const char* kDestinationStr = "Destination";
 const char* kLazyStr = "Lazy";
 const char* kStatusStr = "Status";
 const char* kTypeStr = "Type";
+const char* kInodeStr = "Inode";
 
 const char* kCodeStr = "Code";
 const char* kMessageStr = "Message";
 const char* kRequestIdStr = "RequestId";
 const char* kTotalCountStr = "TotalCount";
 const char* kSnapshotsStr = "Snapshots";
 const char* kTaskInfosStr = "TaskInfos";
+const char* kRefStatusStr = "RefStatus";
+const char* kCloneFileInfoStr = "CloneFileInfo";
 
 std::map<int, std::string> code2Msg = {
     {kErrCodeSuccess, "Exec success."},

src/common/snapshotclone_define.h

@@ -0,0 +1,160 @@
+/*
+ *  Copyright (c) 2020 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Created Date: Mon Dec 24 2018
+ * Author: xuchaojie
+ */
+
+#ifndef SRC_COMMON_SNAPSHOTCLONE_DEFINE_H_
+#define SRC_COMMON_SNAPSHOTCLONE_DEFINE_H_
+
+#include <string>
+#include <map>
+
+namespace curve {
+namespace snapshotcloneserver {
+
+// snapshotcloneservice字符串常量定义
+extern const char* kServiceName;
+// action
+extern const char* kCreateSnapshotAction;
+extern const char* kDeleteSnapshotAction;
+extern const char* kCancelSnapshotAction;
+extern const char* kGetFileSnapshotInfoAction;
+extern const char* kCloneAction;
+extern const char* kRecoverAction;
+extern const char* kGetCloneTasksAction;
+extern const char* kCleanCloneTaskAction;
+extern const char* kFlattenAction;
+extern const char* kGetFileSnapshotListAction;
+extern const char* kGetCloneTaskListAction;
+extern const char* kGetCloneRefStatusAction;
+// param
+extern const char* kActionStr;
+extern const char* kVersionStr;
+extern const char* kUserStr;
+extern const char* kFileStr;
+extern const char* kNameStr;
+extern const char* kUUIDStr;
+extern const char* kLimitStr;
+extern const char* kOffsetStr;
+extern const char* kSourceStr;
+extern const char* kDestinationStr;
+extern const char* kLazyStr;
+extern const char* kStatusStr;
+extern const char* kTypeStr;
+extern const char* kInodeStr;
+
+// json key
+extern const char* kCodeStr;
+extern const char* kMessageStr;
+extern const char* kRequestIdStr;
+extern const char* kTotalCountStr;
+extern const char* kSnapshotsStr;
+extern const char* kTaskInfosStr;
+extern const char* kRefStatusStr;
+extern const char* kCloneFileInfoStr;
+
+typedef std::string UUID;
+using TaskIdType = UUID;
+
+enum class CloneTaskType {
+    kClone = 0,
+    kRecover
+};
+
+enum class CloneRefStatus {
+    kNoRef = 0,
+    kHasRef = 1,
+    kNeedCheck = 2
+};
+
+// 未初始序列号
+const uint64_t kUnInitializeSeqNum = 0;
+// 初始序列号
+const uint64_t kInitializeSeqNum = 1;
+
+// 错误码：执行成功
+const int kErrCodeSuccess = 0;
+// 错误码: 内部错误
+const int kErrCodeInternalError = -1;
+// 错误码：服务器初始化失败
+const int kErrCodeServerInitFail = -2;
+// 错误码：服务器启动失败
+const int kErrCodeServerStartFail = -3;
+// 错误码：服务已停止
+const int kErrCodeServiceIsStop = -4;
+// 错误码：非法请求
+const int kErrCodeInvalidRequest = -5;
+// 错误码：任务已存在
+const int kErrCodeTaskExist = -6;
+// 错误码：非法的用户
+const int kErrCodeInvalidUser = -7;
+// 错误码：文件不存在
+const int kErrCodeFileNotExist = -8;
+// 错误码：文件状态异常
+const int kErrCodeFileStatusInvalid = -9;
+// 错误码：chunk大小未按chunk分片大小对齐
+const int kErrCodeChunkSizeNotAligned = -10;
+// 错误码：文件名不匹配
+const int kErrCodeFileNameNotMatch = -11;
+// 错误码： 不能删除未完成的快照
+const int kErrCodeSnapshotCannotDeleteUnfinished = -12;
+// 错误码: 不能对存在异常快照的文件打快照，或不能对存在错误的目标文件克隆/恢复
+const int kErrCodeSnapshotCannotCreateWhenError = -13;
+// 错误码：取消的快照已完成
+const int kErrCodeCannotCancelFinished = -14;
+// 错误码：不能从未完成或存在错误的快照克隆
+const int kErrCodeInvalidSnapshot = -15;
+// 错误码：不能删除正在克隆的快照
+const int kErrCodeSnapshotCannotDeleteCloning = -16;
+// 错误码：不能清理未完成的克隆
+const int kErrCodeCannotCleanCloneUnfinished = -17;
+// 错误码：快照到达上限
+const int kErrCodeSnapshotCountReachLimit = -18;
+// 错误码：文件已存在
+const int kErrCodeFileExist = -19;
+// 错误码：克隆任务已满
+const int kErrCodeTaskIsFull = -20;
+// 错误码：不支持
+const int kErrCodeNotSupport = -21;
+
+extern std::map<int, std::string> code2Msg;
+
+std::string BuildErrorMessage(
+    int errCode,
+    const std::string &requestId,
+    const std::string &uuid = "");
+
+
+// clone progress
+constexpr uint32_t kProgressCloneStart = 0;
+constexpr uint32_t kProgressCloneError = kProgressCloneStart;
+constexpr uint32_t kProgressCreateCloneFile = 1;
+constexpr uint32_t kProgressCreateCloneMeta = 2;
+constexpr uint32_t kProgressMetaInstalled = 5;
+constexpr uint32_t kProgressRecoverChunkBegin = kProgressMetaInstalled;
+constexpr uint32_t kProgressRecoverChunkEnd = 95;
+constexpr uint32_t kProgressCloneComplete = 100;
+
+
+
+}  // namespace snapshotcloneserver
+}  // namespace curve
+
+#endif  // SRC_COMMON_SNAPSHOTCLONE_DEFINE_H_

src/mds/nameserver2/BUILD

@@ -29,6 +29,7 @@ cc_library(
             "//src/mds/topology:topology",
             "//src/mds/nameserver2/allocstatistic:alloc_statistic",
             "//src/mds/chunkserverclient:chunkserverclient",
+            "//src/mds/snapshotcloneclient:snapshotcloneclient",
             "//src/mds/common:mds_common",
             "//src/mds/nameserver2/helper:helper",
             "//src/mds/nameserver2/idgenerator:idgenerator",

src/mds/nameserver2/curvefs.cpp

@@ -23,7 +23,7 @@
 #include "src/mds/nameserver2/curvefs.h"
 #include <glog/logging.h>
 #include <memory>
-#include <chrono>
+#include <chrono>    //NOLINT
 #include <set>
 #include <utility>
 #include "src/common/string_util.h"
@@ -89,7 +89,8 @@ bool CurveFS::Init(std::shared_ptr<NameServerStorage> storage,
                 std::shared_ptr<FileRecordManager> fileRecordManager,
                 std::shared_ptr<AllocStatistic> allocStatistic,
                 const struct CurveFSOption &curveFSOptions,
-                std::shared_ptr<Topology> topology) {
+                std::shared_ptr<Topology> topology,
+                std::shared_ptr<SnapshotCloneClient> snapshotCloneClient) {
     startTime_ = steady_clock::now();
     storage_ = storage;
     InodeIDGenerator_ = InodeIDGenerator;
@@ -101,6 +102,7 @@ bool CurveFS::Init(std::shared_ptr<NameServerStorage> storage,
 
     defaultChunkSize_ = curveFSOptions.defaultChunkSize;
     topology_ = topology;
+    snapshotCloneClient_ = snapshotCloneClient;
 
     InitRootFile();
     bool ret = InitRecycleBinDir();
@@ -126,6 +128,7 @@ void CurveFS::Uninit() {
     cleanManager_ = nullptr;
     allocStatistic_ = nullptr;
     fileRecordManager_ = nullptr;
+    snapshotCloneClient_ = nullptr;
 }
 
 void CurveFS::InitRootFile(void) {
@@ -567,7 +570,54 @@ StatusCode CurveFS::DeleteFile(const std::string & filename, uint64_t fileId,
         return StatusCode::kOK;
     } else if (fileInfo.filetype() == FileType::INODE_PAGEFILE) {
         StatusCode ret = CheckFileCanChange(filename, fileInfo);
-        if (ret != StatusCode::kOK) {
+        if (ret == StatusCode::kDeleteFileBeingCloned) {
+            CloneRefStatus refStatus;
+            std::vector<snapshotcloneclient::DestFileInfo> fileCheckList;
+            StatusCode ret1 = snapshotCloneClient_->GetCloneRefStatus(filename,
+                                    fileInfo.owner(),
+                                    &refStatus, &fileCheckList);
+            if (ret1 != StatusCode::kOK) {
+                LOG(ERROR) << "delete file, check file clone ref fail,"
+                           << "filename = " << filename
+                           << ", ret = " << ret1;
+                return ret1;
+            }
+            bool hasCloneRef = false;
+            if (refStatus == CloneRefStatus::kHasRef) {
+                hasCloneRef = true;
+            } else if (refStatus == CloneRefStatus::kNoRef) {
+                hasCloneRef = false;
+            } else {
+                int recordNum = fileCheckList.size();
+                for (int i = 0; i < recordNum; i++) {
+                    FileInfo  destFileInfo;
+                    StatusCode ret2 = GetFileInfo(fileCheckList[i].filename,
+                                                    &destFileInfo);
+                    if (ret2 == StatusCode::kFileNotExists) {
+                        continue;
+                    }
+                    if (ret2 == StatusCode::kOK) {
+                        if (destFileInfo.id() == fileCheckList[i].inodeid) {
+                            hasCloneRef = true;
+                            break;
+                        } else {
+                            continue;
+                        }
+                    }
+                    LOG(ERROR) << "delete file, check clonefile exist fail"
+                               << ", filename = " << filename
+                               << ", clonefile = " << fileCheckList[i].filename
+                               << ", ret = " << ret2;
+                    return ret2;
+                }
+            }
+            if (hasCloneRef == true) {
+                LOG(WARNING) << "delete file, can not delete file"
+                       << ", filename = " << filename
+                       << ", ret = " << ret;
+                return ret;
+            }
+        } else if (ret != StatusCode::kOK) {
             LOG(ERROR) << "delete file, can not delete file"
                        << ", filename = " << filename
                        << ", ret = " << ret;
@@ -693,8 +743,8 @@ StatusCode CurveFS::CheckFileCanChange(const std::string &fileName,
     }
 
     if (fileInfo.filestatus() == FileStatus::kFileBeingCloned) {
-        LOG(ERROR) << "CheckFileCanChange, file is being Cloned, "
-                   << "cannot delete or rename, fileName = " << fileName;
+        LOG(WARNING) << "CheckFileCanChange, file is being Cloned, "
+                   << "need check first, fileName = " << fileName;
         return StatusCode::kDeleteFileBeingCloned;
     }