Pick "[feature](Resource) Support to specify the root path for hdfs r…

…esource apache#32632" (apache#35848) same as apache#32632
TangSiyang2001 · Jun 5, 2024 · 630fd06 · 630fd06
1 parent c2b830e
commit 630fd06
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 2 deletions.
diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
@@ -1279,9 +1279,13 @@ void push_storage_policy_callback(StorageEngine& engine, const TAgentTaskRequest
         } else if (resource.__isset.hdfs_storage_param) {
             Status st;
             std::shared_ptr<io::HdfsFileSystem> fs;
+            std::string root_path = resource.hdfs_storage_param.__isset.root_path
+                                            ? resource.hdfs_storage_param.root_path
+                                            : "";
             if (existed_resource.fs == nullptr) {
                 st = io::HdfsFileSystem::create(resource.hdfs_storage_param,
-                                                std::to_string(resource.id), "", nullptr, &fs);
+                                                std::to_string(resource.id), root_path, nullptr,
+                                                &fs);
             } else {
                 fs = std::static_pointer_cast<io::HdfsFileSystem>(existed_resource.fs);
             }
@@ -1290,7 +1294,8 @@ void push_storage_policy_callback(StorageEngine& engine, const TAgentTaskRequest
             } else {
                 LOG_INFO("successfully update hdfs resource")
                         .tag("resource_id", resource.id)
-                        .tag("resource_name", resource.name);
+                        .tag("resource_name", resource.name)
+                        .tag("root_path", fs->root_path().string());
                 put_storage_resource(resource.id, {std::move(fs), resource.version});
             }
         } else {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
@@ -45,6 +45,7 @@
 public class HdfsResource extends Resource {
     public static final String HADOOP_FS_PREFIX = "dfs.";
     public static String HADOOP_FS_NAME = "fs.defaultFS";
+    public static String HADOOP_FS_ROOT_PATH = "root_path";
     public static String HADOOP_SHORT_CIRCUIT = "dfs.client.read.shortcircuit";
     public static String HADOOP_SOCKET_PATH = "dfs.domain.socket.path";
     public static String DSF_NAMESERVICES = "dfs.nameservices";
@@ -106,6 +107,8 @@ public static THdfsParams generateHdfsParam(Map<String, String> properties) {
         for (Map.Entry<String, String> property : properties.entrySet()) {
             if (property.getKey().equalsIgnoreCase(HADOOP_FS_NAME)) {
                 tHdfsParams.setFsName(property.getValue());
+            } else if (property.getKey().equalsIgnoreCase(HADOOP_FS_ROOT_PATH)) {
+                tHdfsParams.setRootPath(property.getValue());
             } else if (property.getKey().equalsIgnoreCase(AuthenticationConfig.HADOOP_USER_NAME)) {
                 tHdfsParams.setUser(property.getValue());
             } else if (property.getKey().equalsIgnoreCase(AuthenticationConfig.HADOOP_KERBEROS_PRINCIPAL)) {

diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
@@ -154,6 +154,8 @@ struct THdfsParams {
     3: optional string hdfs_kerberos_principal
     4: optional string hdfs_kerberos_keytab
     5: optional list<THdfsConf> hdfs_conf
+    // Used for Cold Heat Separation to specify the root path
+    6: optional string root_path
 }
 
 // One broker range information.