add curve segment storage

opencurve · Sep 21, 2020 · 66964ff · 66964ff
1 parent d86ccb4
commit 66964ff
Show file tree

Hide file tree

Showing 102 changed files with 4,356 additions and 870 deletions.
diff --git a/conf/chunkserver.conf.example b/conf/chunkserver.conf.example
@@ -161,6 +161,24 @@ chunkfilepool.cpmeta_file_size=4096
 # chunkfilepool get chunk最大重试次数
 chunkfilepool.retry_times=5
 
+#
+# WAL file pool
+#
+# 是否开启从walfilepool获取chunk，一般是true
+walfilepool.enable_get_segment_from_pool=true
+# walpool目录
+walfilepool.file_pool_dir=./0/
+# walpool meta文件路径
+walfilepool.meta_path=./walfilepool.meta
+# walpool meta文件大小
+walfilepool.segment_size=8388608
+# WAL metapage大小
+walfilepool.metapage_size=4096
+# WAL filepool 元数据文件大小
+walfilepool.meta_file_size=4096
+# WAL filepool get chunk最大重试次数
+walfilepool.retry_times=5
+
 #
 # trash settings
 #

diff --git a/curve-ansible/deploy_chunkserver.yml b/curve-ansible/deploy_chunkserver.yml
@@ -0,0 +1,17 @@
+
+- name: prepare chunkserver
+  hosts: chunkservers
+  any_errors_fatal: true
+  gather_facts: no
+  become: yes
+  become_user: "{{ sudo_user }}"
+  become_flags: -iu {{ sudo_user }}
+  tasks:
+    - name: get disk list
+      shell: lsblk | grep chunkserver | awk -F' ' '{print $1 }'
+      register: result
+    - name: free space
+      shell: sudo tune2fs -m 0 /dev/{{ item }}
+      with_items:
+        "{{ result.stdout.split("\n") }}"
+
diff --git a/curve-ansible/roles/format_chunkserver/defaults/main.yml b/curve-ansible/roles/format_chunkserver/defaults/main.yml
@@ -16,3 +16,4 @@
 #
 
 chunk_alloc_percent: 80
+wal_segment_alloc_percent: 10
diff --git a/curve-ansible/roles/generate_config/defaults/main.yml b/curve-ansible/roles/generate_config/defaults/main.yml
@@ -116,10 +116,15 @@ chunkserver_metric_onoff: true
 chunkserver_storeng_sync_write: false
 chunkserver_concurrentapply_size: 10
 chunkserver_concurrentapply_queuedepth: 1
-chunkserver_chunkfilepool_enable_get_chunk_from_pool: true
 chunkserver_chunkfilepool_chunk_file_pool_dir: ./0/
 chunkserver_chunkfilepool_cpmeta_file_size: 4096
 chunkserver_chunkfilepool_retry_times: 5
+chunkserver_walfilepool_file_pool_dir: ./0/
+chunkserver_walfilepool_meta_path: ./walfilepool.meta
+chunkserver_walfilepool_segment_size: 8388608
+chunkserver_walfilepool_metapage_size: 4096
+chunkserver_walfilepool_meta_file_size: 4096
+chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/

diff --git a/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2 b/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2
@@ -157,15 +157,33 @@ concurrentapply.queuedepth={{ chunkserver_concurrentapply_queuedepth }}
 # Chunkfile pool
 #
 # 是否开启从chunkfilepool获取chunk，一般是true
-chunkfilepool.enable_get_chunk_from_pool={{ chunkserver_chunkfilepool_enable_get_chunk_from_pool }}
+chunkfilepool.enable_get_chunk_from_pool={{ chunkserver_format_disk }}
 # chunkfilepool目录
 chunkfilepool.chunk_file_pool_dir={{ chunkserver_chunkfilepool_chunk_file_pool_dir }}
 # chunkfilepool meta文件路径
 #chunkfilepool.meta_path=./chunkfilepool.meta
 # chunkfilepool meta文件大小
 chunkfilepool.cpmeta_file_size={{ chunkserver_chunkfilepool_cpmeta_file_size }}
 # chunkfilepool get chunk最大重试次数
-chunkfilepool.retry_times={{ chunkserver_chunkfilepool_retry_times }}
+chunkfilepool.retry_times=5
+
+#
+# WAL file pool
+#
+# 是否开启从walfilepool获取chunk，一般是true
+walfilepool.enable_get_segment_from_pool={{ chunkserver_format_disk }}
+# walpool目录
+walfilepool.file_pool_dir={{ chunkserver_walfilepool_file_pool_dir }}
+# walpool meta文件路径
+walfilepool.meta_path={{ chunkserver_walfilepool_meta_path }}
+# walpool meta文件大小
+walfilepool.segment_size={{ chunkserver_walfilepool_segment_size }}
+# WAL metapage大小
+walfilepool.metapage_size={{ chunkserver_walfilepool_metapage_size }}
+# WAL filepool 元数据文件大小
+walfilepool.meta_file_size={{ chunkserver_walfilepool_meta_file_size }}
+# WAL filepool get chunk最大重试次数
+walfilepool.retry_times={{ chunkserver_walfilepool_retry_times }}
 
 #
 # trash settings

diff --git a/curve-ansible/roles/install_package/tasks/include/copy_file_to_remote.yml b/curve-ansible/roles/install_package/tasks/include/copy_file_to_remote.yml
@@ -29,15 +29,14 @@
 - name: copy file with sudo
   block:
     - name: clear tmp dir
-      become: false
-      shell: rm -rf {{ tmp_dir }}
+      shell: sudo rm -rf {{ tmp_dir }}
     - name: create tmp dir
-      become: false
       shell: mkdir -p {{ tmp_dir }}
+      become: false
     - name: copy files to tempory directory
       become: false
       copy: src={{ local_file_path }} dest={{ tmp_dir }} mode={{ file_mode }}
     - name: mv file
       shell: sudo cp -rf {{ tmp_dir }}/* {{ remote_dir_name }}
-  when: copy_file_res.failed
+  when: copy_file_res.failedss
 
diff --git a/curve-ansible/roles/install_package/tasks/include/install_curve-chunkserver.yml b/curve-ansible/roles/install_package/tasks/include/install_curve-chunkserver.yml
@@ -49,7 +49,7 @@
 
 - name: generate chunkserver_ctl.sh
   vars:
-    enable_chunkfilepool: "{{ chunkserver_format_disk }}"
+    enable_FilePool: "{{ chunkserver_format_disk }}"
     jemalloc_path: "{{ lib_install_prefix }}/lib/libjemalloc.so.1"
   template: src=chunkserver_ctl.sh.j2 dest={{ deploy_dir }}/chunkserver_ctl.sh mode=0755
 

diff --git a/curve-ansible/roles/install_package/templates/chunkserver_ctl.sh.j2 b/curve-ansible/roles/install_package/templates/chunkserver_ctl.sh.j2
@@ -13,6 +13,7 @@ function help() {
     echo "    restart : restart chunkserver"
     echo "    status  : show the online status of chunkserver"
     echo "    deploy  : prepare the chunkserver running environment"
+    echo "    deploy-wal-pool  : prepare the wal pool"
     echo "    format  : format the chunkfile pool"
     echo "USAGE:"
     echo "    start all chunkservers              : ./chunkserver_ctl.sh start all"
@@ -26,13 +27,16 @@ function help() {
     echo "    record uuid meta in all disks       : ./chunkserver_ctl.sh record-meta"
     echo "    deploy all disk                     : ./chunkserver_ctl.sh deploy all"
     echo "    deploy one disk                     : ./chunkserver_ctl.sh deploy /dev/sd{id} /data/chunkserver{id}"
-    echo "    format by percent                   : ./chunkserver_ctl.sh format -allocatepercent=80 -filesystem_path=/data/chunkserver{id} "
-    echo "                                          -chunkfilepool_dir=/data/chunkserver{id}/chunkfilepool/"
-    echo "                                          -chunkfilepool_metapath=/data/chunkserver{id}/chunkfilepool.meta"
-    echo "    format by chunk numbers             : ./chunkserver_ctl.sh format -allocateByPercent=false -preallocateNum=100"
-    echo "                                          -filesystem_path=/data/chunkserver{id} "
-    echo "                                          -chunkfilepool_dir=/data/chunkserver{id}/chunkfilepool/"
-    echo "                                          -chunkfilepool_metapath==/data/chunkserver{id}/chunkfilepool.meta"
+    echo "    deploy all wal pool                 : ./chunkserver_ctl.sh deploy-wal-pool all"
+    echo "    deploy one wal pool                 : ./chunkserver_ctl.sh deploy-wal-pool {\$chunkserverId}"
+    echo "    format by percent                   : ./chunkserver_ctl.sh format -allocatePercent=80 -fileSystemPath=/data/chunkserver{id} "
+    echo "                                          -filePoolDir=/data/chunkserver{id}/filepool/"
+    echo "                                          -filePoolMetaPath=/data/chunkserver{id}/filepool.meta"
+    echo "    format by chunk numbers             : ./chunkserver_ctl.sh format -allocateByPercent=false -preAllocateNum=100"
+    echo "                                          -fileSystemPath=/data/chunkserver{id} "
+    echo "                                          -filePoolDir=/data/chunkserver{id}/filepool/"
+    echo "                                          -filePoolMetaPath==/data/chunkserver{id}/filepool.meta"
+
     echo "OPSTIONS:"
     echo "    [-c|--confPath path]  chunkserver conf path need for start command, default:/etc/curve/chunkserver.conf"
     echo "    [-nc|--noConfirm]  if specified, deploy no need to confirm"
@@ -146,20 +150,23 @@ function start_one() {
             -bthread_concurrency=18 -raft_max_segment_size=8388608 \
             -raft_max_install_snapshot_tasks_num=1 -raft_sync=true  \
             -conf=${confPath} \
-            -enableChunkfilepool={{ enable_chunkfilepool }} \
             -chunkFilePoolDir=${dataDir}/chunkserver$1 \
             -chunkFilePoolMetaPath=${dataDir}/chunkserver$1/chunkfilepool.meta \
+            -walFilePoolDir=${dataDir}/chunkserver$1 \
+            -walFilePoolMetaPath=${dataDir}/chunkserver$1/walfilepool.meta \
             -chunkServerIp=$internal_ip \
             -enableExternalServer=$enableExternalServer \
             -chunkServerExternalIp=$external_ip \
             -chunkServerPort=$((${port}+${1})) \
             -chunkServerMetaUri=local://${dataDir}/chunkserver$1/chunkserver.dat \
             -chunkServerStoreUri=local://${dataDir}/chunkserver$1/ \
             -copySetUri=local://${dataDir}/chunkserver$1/copysets \
-            -raftSnapshotUri=curve:///data/chunkserver$1/copysets \
+            -raftSnapshotUri=curve://${dataDir}/chunkserver$1/copysets \
+            -raftLogUri=curve://${dataDir}/chunkserver$1/copysets \
             -recycleUri=local://${dataDir}/chunkserver$1/recycler \
             -raft_sync_segments=true \
             -graceful_quit_on_sigterm=true \
+            -raft_use_fsync_rather_than_fdatasync=false \
             -log_dir=${dataDir}/log/chunkserver$1 > /dev/null 2>&1 &
 }
 
@@ -314,6 +321,22 @@ function recordmeta() {
     meta_record;
 }
 
+function deploy-wal-pool() {
+    if [ $# -lt 1 ]
+    then
+        help
+        return 1
+    fi
+    if [ "$1" = "all" ]
+    then
+        walfile_pool_prep
+        return $?
+    fi
+    deploy_one_walfile_pool $1
+    wait
+}
+
+
 function main() {
     if [ $# -lt 1 ]
     then
@@ -350,6 +373,11 @@ function main() {
         shift
         recordmeta
         ;;
+    "deploy-wal-pool")
+        shift
+        deploy-wal-pool $@
+        ;;
+
     *)
         help
         ;;

diff --git a/curve-ansible/roles/install_package/templates/chunkserver_deploy.sh.j2 b/curve-ansible/roles/install_package/templates/chunkserver_deploy.sh.j2
@@ -161,14 +161,34 @@ function chunkfile_pool_prep {
 ret=`lsblk|grep chunkserver|wc -l`
 for i in `seq 0 $((${ret}-1))`
 do
-  curve-format -allocatepercent={{ chunk_alloc_percent }} \
-  -chunkfilepool_dir=$dataDir/chunkserver$i/chunkfilepool \
-  -chunkfilepool_metapath=$dataDir/chunkserver$i/chunkfilepool.meta \
-  -filesystem_path=$dataDir/chunkserver$i/chunkfilepool &
+  curve-format -allocatePercent={{ chunk_alloc_percent }} \
+  -filePoolDir=/data/chunkserver$i/chunkfilepool \
+  -filePoolMetaPath=/data/chunkserver$i/chunkfilepool.meta \
+  -fileSize={{ chunk_size }} \
+  -fileSystemPath=/data/chunkserver$i/chunkfilepool  &
 done
 wait
 }
 
+function deploy_one_walfile_pool {
+  curve-format -allocatePercent={{ wal_segment_alloc_percent }} \
+  -filePoolDir=/data/chunkserver$1/walfilepool \
+  -filePoolMetaPath=/data/chunkserver$1/walfilepool.meta \
+  -fileSize={{ chunkserver_walfilepool_segment_size }} \
+  -fileSystemPath=/data/chunkserver$1/walfilepool  &
+}
+
+
+# format walfile pool
+function walfile_pool_prep {
+  ret=`lsblk|grep chunkserver|wc -l`
+  for i in `seq 0 $((${ret}-1))`
+  do
+    deploy_one_walfile_pool $i
+  done
+  wait
+}
+
 function usage {
   echo "HELP: this tool will prepare the chunkserver running env."
   echo "      you can deploy all the disks by setting all"
@@ -187,6 +207,7 @@ function deploy_all {
     fstab_record;
     meta_record;
     chunkfile_pool_prep;
+    walfile_pool_prep;
 }
 
 function deploy_one {
@@ -244,10 +265,15 @@ function deploy_one {
   echo "uuid=$uuid" > $dirname/disk.meta
   echo "uuidmd5=$uuidmd5" >> $dirname/disk.meta
   #格式化chunkfile pool
-  curve-format -allocatepercent=80 \
-  -chunkfilepool_dir=$dirname/chunkfilepool \
-  -chunkfilepool_metapath=$dirname/chunkfilepool.meta \
-  -filesystem_path=$dirname/chunkfilepool &
+  curve-format -allocatePercent=80 \
+  -filePoolDir=$dirname/chunkfilepool \
+  -filePoolMetaPath=$dirname/chunkfilepool.meta \
+  -fileSystemPath=$dirname/chunkfilepool  &
+  # prepare walfile pool
+  curve-format -allocatePercent=10 \
+  -filePoolDir=$dirname/walfilepool \
+  -filePoolMetaPath=$dirname/walfilepool.meta \
+  -fileSystemPath=$dirname/walfilepool  &
   wait
   exit
 }
diff --git a/curve-ansible/server.ini b/curve-ansible/server.ini
@@ -68,6 +68,7 @@ defined_copysets_status="Copysets are healthy"
 chunkserver_base_port=8200
 chunkserver_format_disk=false
 chunk_alloc_percent=80
+wal_segment_alloc_percent=10
 # 每台机器上的chunkserver的数量
 chunkserver_num=3
 chunkserver_need_sudo=true
@@ -89,6 +90,8 @@ client_chunkserver_op_max_retry=3
 client_chunkserver_max_stable_timeout_times=64
 client_turn_off_health_check=false
 disable_snapshot_clone=true
+chunk_size=16777216
+chunkserver_walfilepool_segment_size=8388608
 
 [snapshotclone_nginx:vars]
 snapshotcloneserver_nginx_dir=/etc/curve/nginx

diff --git a/curve-chunkserver/home/nbs/chunkserver_ctl.sh b/curve-chunkserver/home/nbs/chunkserver_ctl.sh
@@ -29,6 +29,7 @@ function help() {
     echo "    restart : restart chunkserver"
     echo "    status  : show the online status of chunkserver"
     echo "    deploy  : prepare the chunkserver running environment"
+    echo "    deploy-wal-pool  : prepare the wal pool"
     echo "    format  : format the chunkfile pool"
     echo "USAGE:"
     echo "    start all chunkservers              : ./chunkserver_ctl.sh start all"
@@ -42,13 +43,15 @@ function help() {
     echo "    record uuid meta in all disks       : ./chunkserver_ctl.sh record-meta"
     echo "    deploy all disk                     : ./chunkserver_ctl.sh deploy all"
     echo "    deploy one disk                     : ./chunkserver_ctl.sh deploy /dev/sd{id} /data/chunkserver{id}"
-    echo "    format by percent                   : ./chunkserver_ctl.sh format -allocatepercent=80 -filesystem_path=/data/chunkserver{id} "
-    echo "                                          -chunkfilepool_dir=/data/chunkserver{id}/chunkfilepool/"
-    echo "                                          -chunkfilepool_metapath=/data/chunkserver{id}/chunkfilepool.meta"
-    echo "    format by chunk numbers             : ./chunkserver_ctl.sh format -allocateByPercent=false -preallocateNum=100"
-    echo "                                          -filesystem_path=/data/chunkserver{id} "
-    echo "                                          -chunkfilepool_dir=/data/chunkserver{id}/chunkfilepool/"
-    echo "                                          -chunkfilepool_metapath==/data/chunkserver{id}/chunkfilepool.meta"
+    echo "    deploy all wal pool                 : ./chunkserver_ctl.sh deploy-wal-pool all"
+    echo "    deploy one wal pool                 : ./chunkserver_ctl.sh deploy-wal-pool {\$chunkserverId}"
+    echo "    format by percent                   : ./chunkserver_ctl.sh format -allocatePercent=80 -fileSystemPath=/data/chunkserver{id} "
+    echo "                                          -filePoolDir=/data/chunkserver{id}/filepool/"
+    echo "                                          -filePoolMetaPath=/data/chunkserver{id}/filepool.meta"
+    echo "    format by chunk numbers             : ./chunkserver_ctl.sh format -allocateByPercent=false -preAllocateNum=100"
+    echo "                                          -fileSystemPath=/data/chunkserver{id} "
+    echo "                                          -filePoolDir=/data/chunkserver{id}/filepool/"
+    echo "                                          -filePoolMetaPath==/data/chunkserver{id}/filepool.meta"
     echo "OPSTIONS:"
     echo "    [-c|--confPath path]  chunkserver conf path need for start command, default:/etc/curve/chunkserver.conf"
 }
@@ -163,6 +166,8 @@ function start_one() {
             -conf=${confPath} \
             -chunkFilePoolDir=${DATA_DIR}/chunkserver$1 \
             -chunkFilePoolMetaPath=${DATA_DIR}/chunkserver$1/chunkfilepool.meta \
+            -walFilePoolDir=${DATA_DIR}/chunkserver$1 \
+            -walFilePoolMetaPath=${DATA_DIR}/chunkserver$1/walfilepool.meta \
             -chunkServerIp=$internal_ip \
             -enableExternalServer=$enableExternalServer \
             -chunkServerExternalIp=$external_ip \
@@ -171,6 +176,7 @@ function start_one() {
             -chunkServerStoreUri=local:///data/chunkserver$1/ \
             -copySetUri=local:///data/chunkserver$1/copysets \
             -raftSnapshotUri=curve:///data/chunkserver$1/copysets \
+            -raftLogUri=curve:///data/chunkserver$1/copysets \
             -recycleUri=local:///data/chunkserver$1/recycler \
             -raft_sync_segments=true \
             -graceful_quit_on_sigterm=true \
@@ -327,6 +333,21 @@ function recordmeta() {
     meta_record;
 }
 
+function deploy-wal-pool() {
+    if [ $# -lt 1 ]
+    then
+        help
+        return 1
+    fi
+    if [ "$1" = "all" ]
+    then
+        walfile_pool_prep
+        return $?
+    fi
+    deploy_one_walfile_pool $1
+    wait
+}
+
 function main() {
     if [ $# -lt 1 ]
     then
@@ -363,6 +384,10 @@ function main() {
         shift
         recordmeta
         ;;
+    "deploy-wal-pool")
+        shift
+        deploy-wal-pool $@
+        ;;
     *)
         help
         ;;