From eb8a47d0ce470767d7cc6f454e4b2b3e712e19ab Mon Sep 17 00:00:00 2001 From: baijiaruo Date: Tue, 5 Jan 2021 19:16:23 +0800 Subject: [PATCH] client/mds: merge stripe feature and unit test --- curvefs_python/cbd_client.cpp | 6 + curvefs_python/cbd_client.h | 2 + curvefs_python/curve_type.h | 2 + curvefs_python/curvefs.py | 11 + curvefs_python/curvefs_tool.py | 7 +- curvefs_python/curvefs_wrap.cxx | 212 +++++++++++++++++- curvefs_python/parser.py | 2 + include/client/libcurve.h | 16 ++ proto/nameserver2.proto | 4 + src/client/client_common.h | 7 + src/client/libcurve_file.cpp | 30 +++ src/client/libcurve_file.h | 15 ++ src/client/mds_client.cpp | 8 +- src/client/mds_client.h | 4 +- src/client/mds_client_base.cpp | 8 +- src/client/mds_client_base.h | 2 + src/client/service_helper.cpp | 6 + src/client/splitor.cpp | 101 ++++++--- src/mds/nameserver2/curvefs.cpp | 40 +++- src/mds/nameserver2/curvefs.h | 11 +- src/mds/nameserver2/namespace_service.cpp | 3 +- test/client/iotracker_splitor_unittest.cpp | 89 ++++++++ test/client/libcurve_interface_unittest.cpp | 66 ++++++ .../client/common/file_operation.cpp | 26 +++ .../client/common/file_operation.h | 3 + .../client/config/topo_example.json | 2 +- .../integration/client/mds_exception_test.cpp | 129 +++++++++++ test/mds/nameserver2/curvefs_test.cpp | 65 +++++- 28 files changed, 822 insertions(+), 55 deletions(-) diff --git a/curvefs_python/cbd_client.cpp b/curvefs_python/cbd_client.cpp index 5f9229ef6e..3eb341daae 100644 --- a/curvefs_python/cbd_client.cpp +++ b/curvefs_python/cbd_client.cpp @@ -54,6 +54,12 @@ int CBDClient::Create(const char* filename, UserInfo_t* userInfo, size_t size) { return client_->Create(filename, ToCurveClientUserInfo(userInfo), size); } +int CBDClient::Create2(const char* filename, UserInfo_t* userInfo, size_t size, + uint64_t stripeUnit, uint64_t stripeCount) { + return client_->Create2(filename, ToCurveClientUserInfo(userInfo), + size, stripeUnit, stripeCount); +} + int CBDClient::Unlink(const char* filename, UserInfo_t* userInfo) { return client_->Unlink(filename, ToCurveClientUserInfo(userInfo)); } diff --git a/curvefs_python/cbd_client.h b/curvefs_python/cbd_client.h index dec996a008..836218414f 100644 --- a/curvefs_python/cbd_client.h +++ b/curvefs_python/cbd_client.h @@ -48,6 +48,8 @@ class CBDClient { int Close(int fd); int Create(const char* filename, UserInfo_t* userInfo, size_t size); + int Create2(const char* filename, UserInfo_t* userInfo, size_t size, + uint64_t stripeUnit, uint64_t stripeCount); int Unlink(const char* filename, UserInfo_t* info); int DeleteForce(const char* filename, UserInfo_t* info); int Rename(UserInfo_t* info, const char* oldpath, const char* newpath); diff --git a/curvefs_python/curve_type.h b/curvefs_python/curve_type.h index e3bb16a85e..be4353c184 100644 --- a/curvefs_python/curve_type.h +++ b/curvefs_python/curve_type.h @@ -123,6 +123,8 @@ typedef struct FileInfo { char filename[256]; char owner[256]; int fileStatus; + uint64_t stripeUnit; + uint64_t stripeCount; } FileInfo_t; typedef struct DirInfos { diff --git a/curvefs_python/curvefs.py b/curvefs_python/curvefs.py index e9fe8cfc0a..9650a1f7b9 100644 --- a/curvefs_python/curvefs.py +++ b/curvefs_python/curvefs.py @@ -259,6 +259,14 @@ class FileInfo_t(_object): __swig_getmethods__["fileStatus"] = _curvefs.FileInfo_t_fileStatus_get if _newclass: fileStatus = _swig_property(_curvefs.FileInfo_t_fileStatus_get, _curvefs.FileInfo_t_fileStatus_set) + __swig_setmethods__["stripeUnit"] = _curvefs.FileInfo_t_stripeUnit_set + __swig_getmethods__["stripeUnit"] = _curvefs.FileInfo_t_stripeUnit_get + if _newclass: + stripeUnit = _swig_property(_curvefs.FileInfo_t_stripeUnit_get, _curvefs.FileInfo_t_stripeUnit_set) + __swig_setmethods__["stripeCount"] = _curvefs.FileInfo_t_stripeCount_set + __swig_getmethods__["stripeCount"] = _curvefs.FileInfo_t_stripeCount_get + if _newclass: + stripeCount = _swig_property(_curvefs.FileInfo_t_stripeCount_get, _curvefs.FileInfo_t_stripeCount_set) def __init__(self): this = _curvefs.new_FileInfo_t() @@ -420,6 +428,9 @@ def Close(self, fd): def Create(self, filename, userInfo, size): return _curvefs.CBDClient_Create(self, filename, userInfo, size) + def Create2(self, filename, userInfo, size, stripeUnit, stripeCount): + return _curvefs.CBDClient_Create2(self, filename, userInfo, size, stripeUnit, stripeCount) + def Unlink(self, filename, info): return _curvefs.CBDClient_Unlink(self, filename, info) diff --git a/curvefs_python/curvefs_tool.py b/curvefs_python/curvefs_tool.py index c5d81a89f4..5150a28bdc 100644 --- a/curvefs_python/curvefs_tool.py +++ b/curvefs_python/curvefs_tool.py @@ -81,7 +81,10 @@ def getRetCodeMsg(ret): user.password = args.password if args.optype == "create": - ret = cbd.Create(args.filename, user, args.length * kGB) + if args.stripeUnit and args.stripeCount: + ret = cbd.Create2(args.filename, user, args.length * kGB, args.stripeUnit, args.stripeCount) + else: + ret = cbd.Create(args.filename, user, args.length * kGB) elif args.optype == "delete": ret = cbd.Unlink(args.filename, user) elif args.optype == "extend": @@ -98,6 +101,8 @@ def getRetCodeMsg(ret): print "user: " + finfo.owner print "filename: " + finfo.filename print "fileStatus: " + fileStatus[finfo.fileStatus] + print "stripeUnit: " + str(finfo.stripeUnit) + print "stripeCount: " + str(finfo.stripeCount) elif args.optype == "rename": ret = cbd.Rename(user, args.filename, args.newname) elif args.optype == "mkdir": diff --git a/curvefs_python/curvefs_wrap.cxx b/curvefs_python/curvefs_wrap.cxx index c5f4540dab..9e2e9161ca 100644 --- a/curvefs_python/curvefs_wrap.cxx +++ b/curvefs_python/curvefs_wrap.cxx @@ -3615,6 +3615,29 @@ SWIG_From_unsigned_SS_long_SS_long (unsigned long long value) #endif +SWIGINTERN int +SWIG_AsVal_unsigned_SS_int (PyObject * obj, unsigned int *val) +{ + unsigned long v; + int res = SWIG_AsVal_unsigned_SS_long (obj, &v); + if (SWIG_IsOK(res)) { + if ((v > UINT_MAX)) { + return SWIG_OverflowError; + } else { + if (val) *val = static_cast< unsigned int >(v); + } + } + return res; +} + + +SWIGINTERNINLINE PyObject* + SWIG_From_unsigned_SS_int (unsigned int value) +{ + return PyInt_FromSize_t((size_t) value); +} + + @@ -4579,11 +4602,11 @@ SWIGINTERN PyObject *_wrap_FileInfo_t_fileStatus_get(PyObject *SWIGUNUSEDPARM(se int res1 = 0 ; PyObject * obj0 = 0 ; int result; - + if (!PyArg_ParseTuple(args,(char *)"O:FileInfo_t_fileStatus_get",&obj0)) SWIG_fail; res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_FileInfo, 0 | 0 ); if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_fileStatus_get" "', argument " "1"" of type '" "FileInfo *""'"); + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_fileStatus_get" "', argument " "1"" of type '" "FileInfo *""'"); } arg1 = reinterpret_cast< FileInfo * >(argp1); result = (int) ((arg1)->fileStatus); @@ -4594,6 +4617,110 @@ SWIGINTERN PyObject *_wrap_FileInfo_t_fileStatus_get(PyObject *SWIGUNUSEDPARM(se } +SWIGINTERN PyObject *_wrap_FileInfo_t_stripeUnit_set(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; + FileInfo *arg1 = (FileInfo *) 0 ; + uint32_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + unsigned int val2 ; + int ecode2 = 0 ; + PyObject * obj0 = 0 ; + PyObject * obj1 = 0 ; + + if (!PyArg_ParseTuple(args,(char *)"OO:FileInfo_t_stripeUnit_set",&obj0,&obj1)) SWIG_fail; + res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_FileInfo, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_stripeUnit_set" "', argument " "1"" of type '" "FileInfo *""'"); + } + arg1 = reinterpret_cast< FileInfo * >(argp1); + ecode2 = SWIG_AsVal_unsigned_SS_int(obj1, &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FileInfo_t_stripeUnit_set" "', argument " "2"" of type '" "uint32_t""'"); + } + arg2 = static_cast< uint32_t >(val2); + if (arg1) (arg1)->stripeUnit = arg2; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FileInfo_t_stripeUnit_get(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; + FileInfo *arg1 = (FileInfo *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject * obj0 = 0 ; + uint32_t result; + + if (!PyArg_ParseTuple(args,(char *)"O:FileInfo_t_stripeUnit_get",&obj0)) SWIG_fail; + res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_FileInfo, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_stripeUnit_get" "', argument " "1"" of type '" "FileInfo *""'"); + } + arg1 = reinterpret_cast< FileInfo * >(argp1); + result = (uint64_t) ((arg1)->stripeUnit); + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FileInfo_t_stripeCount_set(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; + FileInfo *arg1 = (FileInfo *) 0 ; + uint32_t arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + unsigned int val2 ; + int ecode2 = 0 ; + PyObject * obj0 = 0 ; + PyObject * obj1 = 0 ; + + if (!PyArg_ParseTuple(args,(char *)"OO:FileInfo_t_stripeCount_set",&obj0,&obj1)) SWIG_fail; + res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_FileInfo, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_stripeCount_set" "', argument " "1"" of type '" "FileInfo *""'"); + } + arg1 = reinterpret_cast< FileInfo * >(argp1); + ecode2 = SWIG_AsVal_unsigned_SS_int(obj1, &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FileInfo_t_stripeCount_set" "', argument " "2"" of type '" "uint32_t""'"); + } + arg2 = static_cast< uint32_t >(val2); + if (arg1) (arg1)->stripeCount = arg2; + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_FileInfo_t_stripeCount_get(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; + FileInfo *arg1 = (FileInfo *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject * obj0 = 0 ; + uint32_t result; + + if (!PyArg_ParseTuple(args,(char *)"O:FileInfo_t_stripeCount_get",&obj0)) SWIG_fail; + res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_FileInfo, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_stripeCount_get" "', argument " "1"" of type '" "FileInfo *""'"); + } + arg1 = reinterpret_cast< FileInfo * >(argp1); + result = (uint64_t) ((arg1)->stripeCount); + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result)); + return resultobj; +fail: + return NULL; +} + + SWIGINTERN PyObject *_wrap_new_FileInfo_t(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { PyObject *resultobj = 0; FileInfo *result = 0 ; @@ -5696,7 +5823,7 @@ SWIGINTERN PyObject *_wrap_GetClusterId(PyObject *SWIGUNUSEDPARM(self), PyObject PyObject * obj1 = 0 ; int result; int retlen; - + if (!PyArg_ParseTuple(args,(char *)"|OO:GetClusterId",&obj0,&obj1)) SWIG_fail; if (obj0) { res1 = SWIG_AsCharPtrAndSize(obj0, &buf1, NULL, &alloc1); @@ -5840,7 +5967,7 @@ SWIGINTERN PyObject *_wrap_CBDClient_Open(PyObject *SWIGUNUSEDPARM(self), PyObje PyObject * obj1 = 0 ; PyObject * obj2 = 0 ; int result; - + if (!PyArg_ParseTuple(args,(char *)"OOO:CBDClient_Open",&obj0,&obj1,&obj2)) SWIG_fail; res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CBDClient, 0 | 0 ); if (!SWIG_IsOK(res1)) { @@ -5938,7 +6065,7 @@ SWIGINTERN PyObject *_wrap_CBDClient_Create(PyObject *SWIGUNUSEDPARM(self), PyOb ecode4 = SWIG_AsVal_size_t(obj3, &val4); if (!SWIG_IsOK(ecode4)) { SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "CBDClient_Create" "', argument " "4"" of type '" "size_t""'"); - } + } arg4 = static_cast< size_t >(val4); result = (int)(arg1)->Create((char const *)arg2,arg3,arg4); resultobj = SWIG_From_int(static_cast< int >(result)); @@ -5950,6 +6077,76 @@ SWIGINTERN PyObject *_wrap_CBDClient_Create(PyObject *SWIGUNUSEDPARM(self), PyOb } +SWIGINTERN PyObject *_wrap_CBDClient_Create2(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *resultobj = 0; + CBDClient *arg1 = (CBDClient *) 0 ; + char *arg2 = (char *) 0 ; + UserInfo_t *arg3 = (UserInfo_t *) 0 ; + size_t arg4 ; + uint32_t arg5 ; + uint32_t arg6 ; + void *argp1 = 0 ; + int res1 = 0 ; + int res2 ; + char *buf2 = 0 ; + int alloc2 = 0 ; + void *argp3 = 0 ; + int res3 = 0 ; + size_t val4 ; + int ecode4 = 0 ; + unsigned int val5 ; + int ecode5 = 0 ; + unsigned int val6 ; + int ecode6 = 0 ; + PyObject * obj0 = 0 ; + PyObject * obj1 = 0 ; + PyObject * obj2 = 0 ; + PyObject * obj3 = 0 ; + PyObject * obj4 = 0 ; + PyObject * obj5 = 0 ; + int result; + + if (!PyArg_ParseTuple(args,(char *)"OOOOOO:CBDClient_Create2",&obj0,&obj1,&obj2,&obj3,&obj4,&obj5)) SWIG_fail; + res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CBDClient, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CBDClient_Create2" "', argument " "1"" of type '" "CBDClient *""'"); + } + arg1 = reinterpret_cast< CBDClient * >(argp1); + res2 = SWIG_AsCharPtrAndSize(obj1, &buf2, NULL, &alloc2); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "CBDClient_Create2" "', argument " "2"" of type '" "char const *""'"); + } + arg2 = reinterpret_cast< char * >(buf2); + res3 = SWIG_ConvertPtr(obj2, &argp3,SWIGTYPE_p_UserInfo, 0 | 0 ); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "CBDClient_Create2" "', argument " "3"" of type '" "UserInfo_t *""'"); + } + arg3 = reinterpret_cast< UserInfo_t * >(argp3); + ecode4 = SWIG_AsVal_size_t(obj3, &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "CBDClient_Create2" "', argument " "4"" of type '" "size_t""'"); + } + arg4 = static_cast< size_t >(val4); + ecode5 = SWIG_AsVal_unsigned_SS_int(obj4, &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "CBDClient_Create2" "', argument " "5"" of type '" "uint32_t""'"); + } + arg5 = static_cast< uint32_t >(val5); + ecode6 = SWIG_AsVal_unsigned_SS_int(obj5, &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "CBDClient_Create2" "', argument " "6"" of type '" "uint32_t""'"); + } + arg6 = static_cast< uint32_t >(val6); + result = (int)(arg1)->Create2((char const *)arg2,arg3,arg4,arg5,arg6); + resultobj = SWIG_From_int(static_cast< int >(result)); + if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + return resultobj; +fail: + if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + return NULL; +} + + SWIGINTERN PyObject *_wrap_CBDClient_Unlink(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { PyObject *resultobj = 0; CBDClient *arg1 = (CBDClient *) 0 ; @@ -6748,6 +6945,10 @@ static PyMethodDef SwigMethods[] = { { (char *)"FileInfo_t_owner_get", _wrap_FileInfo_t_owner_get, METH_VARARGS, NULL}, { (char *)"FileInfo_t_fileStatus_set", _wrap_FileInfo_t_fileStatus_set, METH_VARARGS, NULL}, { (char *)"FileInfo_t_fileStatus_get", _wrap_FileInfo_t_fileStatus_get, METH_VARARGS, NULL}, + { (char *)"FileInfo_t_stripeUnit_set", _wrap_FileInfo_t_stripeUnit_set, METH_VARARGS, NULL}, + { (char *)"FileInfo_t_stripeUnit_get", _wrap_FileInfo_t_stripeUnit_get, METH_VARARGS, NULL}, + { (char *)"FileInfo_t_stripeCount_set", _wrap_FileInfo_t_stripeCount_set, METH_VARARGS, NULL}, + { (char *)"FileInfo_t_stripeCount_get", _wrap_FileInfo_t_stripeCount_get, METH_VARARGS, NULL}, { (char *)"new_FileInfo_t", _wrap_new_FileInfo_t, METH_VARARGS, NULL}, { (char *)"delete_FileInfo_t", _wrap_delete_FileInfo_t, METH_VARARGS, NULL}, { (char *)"FileInfo_t_swigregister", FileInfo_t_swigregister, METH_VARARGS, NULL}, @@ -6792,6 +6993,7 @@ static PyMethodDef SwigMethods[] = { { (char *)"CBDClient_Open", _wrap_CBDClient_Open, METH_VARARGS, NULL}, { (char *)"CBDClient_Close", _wrap_CBDClient_Close, METH_VARARGS, NULL}, { (char *)"CBDClient_Create", _wrap_CBDClient_Create, METH_VARARGS, NULL}, + { (char *)"CBDClient_Create2", _wrap_CBDClient_Create2, METH_VARARGS, NULL}, { (char *)"CBDClient_Unlink", _wrap_CBDClient_Unlink, METH_VARARGS, NULL}, { (char *)"CBDClient_DeleteForce", _wrap_CBDClient_DeleteForce, METH_VARARGS, NULL}, { (char *)"CBDClient_Rename", _wrap_CBDClient_Rename, METH_VARARGS, NULL}, diff --git a/curvefs_python/parser.py b/curvefs_python/parser.py index 05e5dde6d4..7ff2bb2583 100644 --- a/curvefs_python/parser.py +++ b/curvefs_python/parser.py @@ -33,6 +33,8 @@ def get_parser(): subparser.add_argument("--length", help='length of file(GB)', type=long, required=True) subparser.add_argument("--user", help='user of file', type=str, required=True) subparser.add_argument("--password", help='password of user', type=str) + subparser.add_argument("--stripeUnit", help='stripe unit size', type=long) + subparser.add_argument("--stripeCount", help='stripe width', type=long) # delete option subparser = subparsers.add_parser("delete", help="delete file") diff --git a/include/client/libcurve.h b/include/client/libcurve.h index f47d7b86c0..2928b20cea 100644 --- a/include/client/libcurve.h +++ b/include/client/libcurve.h @@ -134,6 +134,8 @@ typedef struct FileStatInfo { char filename[NAME_MAX_SIZE]; char owner[NAME_MAX_SIZE]; int fileStatus; + uint64_t stripeUnit; + uint64_t stripeCount; } FileStatInfo_t; // 存储用户信息 @@ -195,6 +197,20 @@ int Create(const char* filename, const C_UserInfo_t* userinfo, size_t size); +/** + * create file with stripe + * @param: filename file name + * @param: userinfo user info + * @param: size file size + * @param: stripeUnit block in stripe size + * @param: stripeCount stripe count in one stripe + * + * @return: success return 0, fail return less than 0 + */ +int Create2(const char* filename, + const C_UserInfo_t* userinfo, + size_t size, uint64_t stripeUnit, uint64_t stripeCount); + /** * 同步模式读 * @param: fd为当前open返回的文件描述符 diff --git a/proto/nameserver2.proto b/proto/nameserver2.proto index 7033a3c62e..7da80029f5 100644 --- a/proto/nameserver2.proto +++ b/proto/nameserver2.proto @@ -67,6 +67,8 @@ message FileInfo { // cloneLength 克隆源文件的长度，用于clone过程中进行extent optional uint64 cloneLength = 14; + optional uint64 stripeUnit = 15; + optional uint64 stripeCount = 16; } // status code @@ -163,6 +165,8 @@ message CreateFileRequest { required string owner = 2; optional string signature = 5; required uint64 date = 6; + optional uint64 stripeUnit = 7; + optional uint64 stripeCount = 8; }; message CreateFileResponse { diff --git a/src/client/client_common.h b/src/client/client_common.h index aca34666bf..7d730ce7f1 100644 --- a/src/client/client_common.h +++ b/src/client/client_common.h @@ -156,7 +156,12 @@ typedef struct FInfo { std::string filename; std::string fullPathName; FileStatus filestatus; + CloneSourceInfo sourceInfo; + std::string cloneSource; + uint64_t cloneLength{0}; + uint64_t stripeUnit; + uint64_t stripeCount; FInfo() { id = 0; @@ -165,6 +170,8 @@ typedef struct FInfo { length = 0; chunksize = 4 * 1024 * 1024; segmentsize = 1 * 1024 * 1024 * 1024ul; + stripeUnit = 0; + stripeCount = 0; } } FInfo_t; diff --git a/src/client/libcurve_file.cpp b/src/client/libcurve_file.cpp index 14dd664e71..7ea2d98076 100644 --- a/src/client/libcurve_file.cpp +++ b/src/client/libcurve_file.cpp @@ -271,6 +271,22 @@ int FileClient::Create(const std::string& filename, return -ret; } +int FileClient::Create2(const std::string& filename, + const UserInfo_t& userinfo, size_t size, + uint64_t stripeUnit, uint64_t stripeCount) { + LIBCURVE_ERROR ret; + if (mdsClient_ != nullptr) { + ret = mdsClient_->CreateFile(filename, userinfo, size, true, + stripeUnit, stripeCount); + LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK) + << "Create file failed, filename: " << filename << ", ret: " << ret; + } else { + LOG(ERROR) << "global mds client not inited!"; + return -LIBCURVE_ERROR::FAILED; + } + return -ret; +} + int FileClient::Read(int fd, char* buf, off_t offset, size_t len) { // 长度为0，直接返回，不做任何操作 if (len == 0) { @@ -431,6 +447,8 @@ int FileClient::StatFile(const std::string& filename, finfo->ctime = fi.ctime; finfo->length = fi.length; finfo->filetype = fi.filetype; + finfo->stripeUnit = fi.stripeUnit; + finfo->stripeCount = fi.stripeCount; memcpy(finfo->filename, fi.filename.c_str(), std::min(sizeof(finfo->filename), fi.filename.size() + 1)); @@ -733,6 +751,18 @@ int Create(const char* filename, const C_UserInfo_t* userinfo, size_t size) { UserInfo(userinfo->owner, userinfo->password), size); } +int Create2(const char* filename, const C_UserInfo_t* userinfo, size_t size, + uint64_t stripeUnit, uint64_t stripeCount) { + if (globalclient == nullptr) { + LOG(ERROR) << "not inited!"; + return -LIBCURVE_ERROR::FAILED; + } + + return globalclient->Create2(filename, + UserInfo(userinfo->owner, userinfo->password), + size, stripeUnit, stripeCount); +} + int Rename(const C_UserInfo_t* userinfo, const char* oldpath, const char* newpath) { if (globalclient == nullptr) { diff --git a/src/client/libcurve_file.h b/src/client/libcurve_file.h index e3d1928382..2e37d1fc24 100644 --- a/src/client/libcurve_file.h +++ b/src/client/libcurve_file.h @@ -111,6 +111,21 @@ class FileClient { const UserInfo_t& userinfo, size_t size); + /** + * create file with stripe + * @param: filename file name + * @param: userinfo user info + * @param: size file size + * @param: stripeUnit block in stripe size + * @param stripeCount stripe count in one stripe + * @return: success return 0, fail return less than 0 + * + */ + virtual int Create2(const std::string& filename, + const UserInfo_t& userinfo, + size_t size, uint64_t stripeUnit, + uint64_t stripeCount); + /** * 同步模式读 * @param: fd为当前open返回的文件描述符 diff --git a/src/client/mds_client.cpp b/src/client/mds_client.cpp index bea0600eb1..f77cb2bd53 100644 --- a/src/client/mds_client.cpp +++ b/src/client/mds_client.cpp @@ -318,13 +318,15 @@ LIBCURVE_ERROR MDSClient::OpenFile(const std::string& filename, LIBCURVE_ERROR MDSClient::CreateFile(const std::string& filename, const UserInfo_t& userinfo, size_t size, - bool normalFile) { + bool normalFile, + uint64_t stripeUnit, + uint64_t stripeCount) { auto task = RPCTaskDefine { CreateFileResponse response; mdsClientMetric_.createFile.qps.count << 1; LatencyGuard lg(&mdsClientMetric_.createFile.latency); mdsClientBase_.CreateFile(filename, userinfo, size, normalFile, - &response, cntl, channel); + stripeUnit, stripeCount, &response, cntl, channel); if (cntl->Failed()) { mdsClientMetric_.createFile.eps.count << 1; @@ -393,8 +395,6 @@ LIBCURVE_ERROR MDSClient::GetFileInfo(const std::string& filename, if (cntl->Failed()) { mdsClientMetric_.getFile.eps.count << 1; - LOG(WARNING) << "get file info failed, error content:" - << cntl->ErrorText() << ", log id = " << cntl->log_id(); return -cntl->ErrorCode(); } diff --git a/src/client/mds_client.h b/src/client/mds_client.h index e8fe4623d5..1bef7bcb3c 100644 --- a/src/client/mds_client.h +++ b/src/client/mds_client.h @@ -81,7 +81,9 @@ class MDSClient { LIBCURVE_ERROR CreateFile(const std::string& filename, const UserInfo_t& userinfo, size_t size = 0, - bool normalFile = true); + bool normalFile = true, + uint64_t stripeUnit = 0, + uint64_t stripeCount = 0); /** * 打开文件 * @param: filename是文件名 diff --git a/src/client/mds_client_base.cpp b/src/client/mds_client_base.cpp index 35ff229c95..c3e6e8b586 100644 --- a/src/client/mds_client_base.cpp +++ b/src/client/mds_client_base.cpp @@ -55,6 +55,8 @@ void MDSClientBase::CreateFile(const std::string& filename, const UserInfo_t& userinfo, size_t size, bool normalFile, + uint64_t stripeUnit, + uint64_t stripeCount, CreateFileResponse* response, brpc::Controller* cntl, brpc::Channel* channel) { @@ -67,12 +69,16 @@ void MDSClientBase::CreateFile(const std::string& filename, request.set_filetype(curve::mds::FileType::INODE_DIRECTORY); } + request.set_stripeunit(stripeUnit); + request.set_stripecount(stripeCount); FillUserInfo(&request, userinfo); LOG(INFO) << "CreateFile: filename = " << filename << ", owner = " << userinfo.owner << ", is nomalfile: " << normalFile - << ", log id = " << cntl->log_id(); + << ", log id = " << cntl->log_id() + << ", stripeUnit = " << stripeUnit + << ", stripeCount = " << stripeCount; curve::mds::CurveFSService_Stub stub(channel); stub.CreateFile(cntl, &request, response, NULL); diff --git a/src/client/mds_client_base.h b/src/client/mds_client_base.h index 4703098662..33e9f50a61 100644 --- a/src/client/mds_client_base.h +++ b/src/client/mds_client_base.h @@ -130,6 +130,8 @@ class MDSClientBase { const UserInfo_t& userinfo, size_t size, bool normalFile, + const uint64_t stripeUnit, + const uint64_t stripeCount, CreateFileResponse* response, brpc::Controller* cntl, brpc::Channel* channel); diff --git a/src/client/service_helper.cpp b/src/client/service_helper.cpp index 5bdb8bb65b..4e7acf9897 100644 --- a/src/client/service_helper.cpp +++ b/src/client/service_helper.cpp @@ -72,6 +72,12 @@ void ServiceHelper::ProtoFileInfo2Local(const curve::mds::FileInfo& finfo, if (finfo.has_filestatus()) { fi->filestatus = (FileStatus)finfo.filestatus(); } + if (finfo.has_stripeunit()) { + fi->stripeUnit = finfo.stripeunit(); + } + if (finfo.has_stripecount()) { + fi->stripeCount = finfo.stripecount(); + } } void ServiceHelper::ProtoCloneSourceInfo2Local( diff --git a/src/client/splitor.cpp b/src/client/splitor.cpp index 617e0c2271..c18373f577 100644 --- a/src/client/splitor.cpp +++ b/src/client/splitor.cpp @@ -60,34 +60,30 @@ int Splitor::IO2ChunkRequests(IOTracker* iotracker, MetaCache* metaCache, targetlist->reserve(length / (iosplitopt_.fileIOSplitMaxSizeKB * 1024) + 1); const uint64_t chunksize = fileInfo->chunksize; - uint64_t currentChunkIndex = offset / chunksize; - const uint64_t endChunkIndex = (offset + length - 1) / chunksize; - uint64_t currentRequestOffset = offset; - const uint64_t endRequestOffest = offset + length; - uint64_t currentChunkOffset = offset % chunksize; - uint64_t dataOffset = 0; + uint64_t stripeUnit = fileInfo->stripeUnit; + const uint64_t stripeCount = fileInfo->stripeCount; + bool isStripe = true; + + if (((stripeUnit == 0) && (stripeCount == 0)) || stripeCount == 1) { + isStripe = false; + } - while (currentChunkIndex <= endChunkIndex) { - const uint64_t currentChunkEndOffset = + if (!isStripe) { + uint64_t currentChunkIndex = offset / chunksize; + const uint64_t endChunkIndex = (offset + length - 1) / chunksize; + uint64_t currentRequestOffset = offset; + const uint64_t endRequestOffest = offset + length; + uint64_t currentChunkOffset = offset % chunksize; + uint64_t dataOffset = 0; + + while (currentChunkIndex <= endChunkIndex) { + const uint64_t currentChunkEndOffset = chunksize * (currentChunkIndex + 1); - uint64_t requestLength = + uint64_t requestLength = std::min(currentChunkEndOffset, endRequestOffest) - currentRequestOffset; - DVLOG(9) << "request split" - << ", off = " << currentChunkOffset - << ", len = " << requestLength - << ", seqnum = " << fileInfo->seqnum - << ", endoff = " << endRequestOffest - << ", chunkendpos = " << currentChunkEndOffset - << ", chunksize = " << chunksize - << ", chunkindex = " << currentChunkIndex - << ", endchunkindex = " << endChunkIndex; - - if (!AssignInternal(iotracker, metaCache, targetlist, data, - currentChunkOffset, requestLength, mdsclient, - fileInfo, currentChunkIndex)) { - LOG(ERROR) << "request split failed" + DVLOG(9) << "request split" << ", off = " << currentChunkOffset << ", len = " << requestLength << ", seqnum = " << fileInfo->seqnum @@ -96,16 +92,63 @@ int Splitor::IO2ChunkRequests(IOTracker* iotracker, MetaCache* metaCache, << ", chunksize = " << chunksize << ", chunkindex = " << currentChunkIndex << ", endchunkindex = " << endChunkIndex; - return -1; + + if (!AssignInternal(iotracker, metaCache, targetlist, data, + currentChunkOffset, requestLength, mdsclient, + fileInfo, currentChunkIndex)) { + LOG(ERROR) << "request split failed" + << ", off = " << currentChunkOffset + << ", len = " << requestLength + << ", seqnum = " << fileInfo->seqnum + << ", endoff = " << endRequestOffest + << ", chunkendpos = " << currentChunkEndOffset + << ", chunksize = " << chunksize + << ", chunkindex = " << currentChunkIndex + << ", endchunkindex = " << endChunkIndex; + return -1; + } + + currentChunkOffset = 0; + currentChunkIndex++; + + dataOffset += requestLength; + currentRequestOffset += requestLength; } + } else { + const uint64_t stripesPerChunk = chunksize / stripeUnit; + uint64_t cur = offset; + uint64_t left = length; + uint64_t curChunkIndex = 0; + while (left > 0) { + uint64_t blockIndex = cur / stripeUnit; + uint64_t stripeIndex = blockIndex / stripeCount; + uint64_t stripepos = blockIndex % stripeCount; + uint64_t curChunkSetIndex = stripeIndex / stripesPerChunk; + uint64_t curChunkIndex = curChunkSetIndex * stripeCount + stripepos; + + uint64_t blockInChunkStartOff = (stripeIndex % stripesPerChunk) + * stripeUnit; + uint64_t blockOff = cur % stripeUnit; + uint64_t curChunkOffset = blockInChunkStartOff + blockOff; + uint64_t requestLength = std::min((stripeUnit - blockOff), left); + + if (!AssignInternal(iotracker, metaCache, targetlist, data, + curChunkOffset, requestLength, mdsclient, + fileInfo, curChunkIndex)) { + LOG(ERROR) << "request split failed" + << ", off = " << curChunkOffset + << ", len = " << requestLength + << ", seqnum = " << fileInfo->seqnum + << ", chunksize = " << chunksize + << ", chunkindex = " << curChunkIndex; - currentChunkOffset = 0; - currentChunkIndex++; + return -1; + } - dataOffset += requestLength; - currentRequestOffset += requestLength; + left -= requestLength; + cur += requestLength; + } } - return 0; } diff --git a/src/mds/nameserver2/curvefs.cpp b/src/mds/nameserver2/curvefs.cpp index d6bc020505..0b1eb4710f 100644 --- a/src/mds/nameserver2/curvefs.cpp +++ b/src/mds/nameserver2/curvefs.cpp @@ -208,7 +208,8 @@ StatusCode CurveFS::SnapShotFile(const FileInfo * origFileInfo, StatusCode CurveFS::CreateFile(const std::string & fileName, const std::string& owner, - FileType filetype, uint64_t length) { + FileType filetype, uint64_t length, + uint64_t stripeUnit, uint64_t stripeCount) { FileInfo parentFileInfo; std::string lastEntry; @@ -243,7 +244,11 @@ StatusCode CurveFS::CreateFile(const std::string & fileName, } } - auto ret = WalkPath(fileName, &parentFileInfo, &lastEntry); + auto ret = CheckStripeParam(stripeUnit, stripeCount); + if (ret != StatusCode::kOK) { + return ret; + } + ret = WalkPath(fileName, &parentFileInfo, &lastEntry); if ( ret != StatusCode::kOK ) { return ret; } @@ -277,6 +282,8 @@ StatusCode CurveFS::CreateFile(const std::string & fileName, fileInfo.set_ctime(::curve::common::TimeUtility::GetTimeofDayUs()); fileInfo.set_seqnum(kStartSeqNum); fileInfo.set_filestatus(FileStatus::kFileCreated); + fileInfo.set_stripeunit(stripeUnit); + fileInfo.set_stripecount(stripeCount); ret = PutFile(fileInfo); return ret; @@ -1993,6 +2000,35 @@ uint64_t CurveFS::GetDefaultChunkSize() { return defaultChunkSize_; } +StatusCode CurveFS::CheckStripeParam(uint64_t stripeUnit, + uint64_t stripeCount) { + if ((stripeUnit == 0) && (stripeCount == 0 )) { + return StatusCode::kOK; + } + + if ((stripeUnit && !stripeCount) || + (!stripeUnit && stripeCount)) { + LOG(ERROR) << "can't just one is zero. stripeUnit:" + << stripeUnit << ",stripeCount:" << stripeCount; + return StatusCode::kParaError; + } + + if (stripeUnit > defaultChunkSize_) { + LOG(ERROR) << "stripeUnit more than chunksize.stripeUnit:" + << stripeUnit; + return StatusCode::kParaError; + } + + if ((defaultChunkSize_ % stripeUnit != 0) || + (defaultChunkSize_ % stripeCount != 0)) { + LOG(ERROR) << "is not divisible by chunksize. stripeUnit:" + << stripeUnit << ",stripeCount:" << stripeCount; + return StatusCode::kParaError; + } + + return StatusCode::kOK; +} + CurveFS &kCurveFS = CurveFS::GetInstance(); uint64_t GetOpenFileNum(void *varg) { diff --git a/src/mds/nameserver2/curvefs.h b/src/mds/nameserver2/curvefs.h index f16a354459..05abf36dcd 100644 --- a/src/mds/nameserver2/curvefs.h +++ b/src/mds/nameserver2/curvefs.h @@ -119,12 +119,16 @@ class CurveFS { * owner: the owner of the file * filetype：the type of the file * length：file length + * stripeUnit: the smallest unit of stripe + * stripeCount: stripe width * @return return StatusCode::kOK if succeeded */ StatusCode CreateFile(const std::string & fileName, const std::string& owner, FileType filetype, - uint64_t length); + uint64_t length, + uint64_t stripeUnit, + uint64_t stripeCount); /** * @brief get file information * @param filename @@ -614,6 +618,7 @@ class CurveFS { const std::string &owner, bool *isHasCloneRely); + /** * @brief check whether mds has started for enough time, based on the * file record expiration time(mds.file.expiredTimeUs) @@ -636,6 +641,10 @@ class CurveFS { StatusCode ListCloneSourceFileSegments( const FileInfo* fileInfo, CloneSourceSegment* cloneSourceSegment) const; + StatusCode CheckStripeParam(uint64_t stripeUnit, + uint64_t stripeCount); + + private: FileInfo rootFileInfo_; std::shared_ptr storage_; diff --git a/src/mds/nameserver2/namespace_service.cpp b/src/mds/nameserver2/namespace_service.cpp index 86ef6fd4ac..7393c19712 100644 --- a/src/mds/nameserver2/namespace_service.cpp +++ b/src/mds/nameserver2/namespace_service.cpp @@ -85,7 +85,8 @@ void NameSpaceService::CreateFile(::google::protobuf::RpcController* controller, } retCode = kCurveFS.CreateFile(request->filename(), request->owner(), - request->filetype(), request->filelength()); + request->filetype(), request->filelength(), request->stripeunit(), + request->stripecount()); if (retCode != StatusCode::kOK) { response->set_statuscode(retCode); // TODO(hzsunjianliang): check if we should really print error here diff --git a/test/client/iotracker_splitor_unittest.cpp b/test/client/iotracker_splitor_unittest.cpp index 70017fb263..f89a1388c5 100644 --- a/test/client/iotracker_splitor_unittest.cpp +++ b/test/client/iotracker_splitor_unittest.cpp @@ -1047,6 +1047,95 @@ TEST(SplitorTest, RequestSourceInfoTest) { ASSERT_EQ(sourceInfo.cloneFileOffset, 0); } +TEST_F(IOTrackerSplitorTest, stripeTest) { + MockRequestScheduler mockschuler; + mockschuler.DelegateToFake(); + + FInfo_t fi; + uint64_t offset = 1 * 1024 * 1024 - 64 * 1024; + uint64_t length = 128 * 1024; + butil::IOBuf dataCopy; + char* buf = new char[length]; + + fi.seqnum = 0; + fi.chunksize = 4 * 1024 * 1024; + fi.segmentsize = 1 * 1024 * 1024 * 1024ul; + fi.stripeUnit = 1 * 1024 * 1024; + fi.stripeCount = 4; + memset(buf, 'a', length); // 64KB + dataCopy.append(buf, length); + curve::client::IOManager4File* iomana = fileinstance_->GetIOManager4File(); + MetaCache* mc = iomana->GetMetaCache(); + + IOTracker* iotracker = new IOTracker(iomana, mc, &mockschuler); + iotracker->SetOpType(OpType::WRITE); + curve::client::ChunkIDInfo chinfo(1, 2, 3); + curve::client::ChunkIDInfo chinfo1(4, 5, 6); + mc->UpdateChunkInfoByIndex(0, chinfo); + mc->UpdateChunkInfoByIndex(1, chinfo1); + + std::vector reqlist; + ASSERT_EQ(0, curve::client::Splitor::IO2ChunkRequests(iotracker, mc, + &reqlist, + &dataCopy, + offset, + length, + &mdsclient_, + &fi)); + + ASSERT_EQ(2, reqlist.size()); + RequestContext* first = reqlist.front(); + reqlist.erase(reqlist.begin()); + RequestContext* second = reqlist.front(); + reqlist.erase(reqlist.begin()); + + ASSERT_EQ(1, first->idinfo_.cid_); + ASSERT_EQ(3, first->idinfo_.cpid_); + ASSERT_EQ(2, first->idinfo_.lpid_); + ASSERT_EQ(1 * 1024 * 1024 - 64 * 1024, first->offset_); + ASSERT_EQ(64 * 1024, first->rawlength_); + + ASSERT_EQ(4, second->idinfo_.cid_); + ASSERT_EQ(6, second->idinfo_.cpid_); + ASSERT_EQ(5, second->idinfo_.lpid_); + ASSERT_EQ(0, second->offset_); + ASSERT_EQ(64 * 1024, second->rawlength_); + + reqlist.clear(); + offset = 16 * 1024 * 1024 - 64 * 1024; + length = 128 * 1024; + memset(buf, 'b', length); + dataCopy.append(buf, length); + mc->UpdateChunkInfoByIndex(3, chinfo); + mc->UpdateChunkInfoByIndex(4, chinfo1); + ASSERT_EQ(0, curve::client::Splitor::IO2ChunkRequests(iotracker, mc, + &reqlist, + &dataCopy, + offset, + length, + &mdsclient_, + &fi)); + ASSERT_EQ(2, reqlist.size()); + first = reqlist.front(); + reqlist.erase(reqlist.begin()); + second = reqlist.front(); + reqlist.erase(reqlist.begin()); + + ASSERT_EQ(1, first->idinfo_.cid_); + ASSERT_EQ(3, first->idinfo_.cpid_); + ASSERT_EQ(2, first->idinfo_.lpid_); + ASSERT_EQ(4 * 1024 * 1024 - 64 * 1024, first->offset_); + ASSERT_EQ(64 * 1024, first->rawlength_); + + ASSERT_EQ(4, second->idinfo_.cid_); + ASSERT_EQ(6, second->idinfo_.cpid_); + ASSERT_EQ(5, second->idinfo_.lpid_); + ASSERT_EQ(0, second->offset_); + ASSERT_EQ(64 * 1024, second->rawlength_); + + delete[] buf; +} + // read the chunks all haven't been write from normal volume with no clonesource TEST_F(IOTrackerSplitorTest, StartReadNotAllocateSegment) { curvefsservice.SetGetOrAllocateSegmentFakeReturn(notallocatefakeret); diff --git a/test/client/libcurve_interface_unittest.cpp b/test/client/libcurve_interface_unittest.cpp index 719babb51e..63d9cbf9da 100644 --- a/test/client/libcurve_interface_unittest.cpp +++ b/test/client/libcurve_interface_unittest.cpp @@ -950,5 +950,71 @@ TEST(TestLibcurveInterface, ResumeTimeoutBackoff) { delete[] buffer; } +TEST(TestLibcurveInterface, InterfaceStripeTest) { + FLAGS_chunkserver_list = + "127.0.0.1:9115:0,127.0.0.1:9116:0,127.0.0.1:9117:0"; + + std::string filename = "/1"; + std::string filename2 = "/2"; + UserInfo_t userinfo; + userinfo.owner = "userinfo"; + uint64_t size = 100 * 1024 * 1024 * 1024ul; + FileClient fc; + + // 设置leaderid + EndPoint ep; + butil::str2endpoint("127.0.0.1", 9115, &ep); + PeerId pd(ep); + + // init mds service + FakeMDS mds(filename); + mds.Initialize(); + mds.StartCliService(pd); + mds.StartService(); + mds.CreateCopysetNode(true); + + ASSERT_EQ(0, fc.Init(configpath)); + + FakeMDSCurveFSService *service = NULL; + service = mds.GetMDSService(); + ::curve::mds::CreateFileResponse response; + response.set_statuscode(::curve::mds::StatusCode::kOK); + FakeReturn* fakeret + = new FakeReturn(nullptr, static_cast(&response)); + service->SetCreateFileFakeReturn(fakeret); + int ret = fc.Create2(filename, userinfo, size, 0, 0); + ASSERT_EQ(LIBCURVE_ERROR::OK, ret); + + response.set_statuscode(::curve::mds::StatusCode::kFileExists); + fakeret = new FakeReturn(nullptr, static_cast(&response)); + service->SetCreateFileFakeReturn(fakeret); + ret = fc.Create2(filename2, userinfo, size, 1024*1024, 4); + ASSERT_EQ(LIBCURVE_ERROR::EXISTS, -ret); + + FileStatInfo_t fsinfo; + ::curve::mds::FileInfo * info = new curve::mds::FileInfo; + ::curve::mds::GetFileInfoResponse getinforesponse; + info->set_filename(filename2); + info->set_id(1); + info->set_parentid(0); + info->set_filetype(curve::mds::FileType::INODE_PAGEFILE); + info->set_chunksize(4 * 1024 * 1024); + info->set_length(4 * 1024 * 1024 * 1024ul); + info->set_ctime(12345678); + info->set_segmentsize(1 * 1024 * 1024 * 1024ul); + info->set_stripeunit(1024*1024); + info->set_stripecount(4); + getinforesponse.set_allocated_fileinfo(info); + getinforesponse.set_statuscode(::curve::mds::StatusCode::kOK); + FakeReturn* fakegetinfo = + new FakeReturn(nullptr, static_cast(&getinforesponse)); + service->SetGetFileInfoFakeReturn(fakegetinfo); + ret = fc.StatFile(filename2, userinfo, &fsinfo); + ASSERT_EQ(1024*1024, fsinfo.stripeUnit); + ASSERT_EQ(4, fsinfo.stripeCount); + mds.UnInitialize(); + fc.UnInit(); +} + } // namespace client } // namespace curve diff --git a/test/integration/client/common/file_operation.cpp b/test/integration/client/common/file_operation.cpp index a3aeeb29e2..1be458f930 100644 --- a/test/integration/client/common/file_operation.cpp +++ b/test/integration/client/common/file_operation.cpp @@ -66,5 +66,31 @@ void FileCommonOperation::Close(int fd) { ::Close(fd); } +int FileCommonOperation::Open(const std::string& filename, + const std::string& owner, + uint64_t stripeUnit, uint64_t stripeCount) { + C_UserInfo_t userinfo; + memset(userinfo.owner, 0, 256); + memcpy(userinfo.owner, owner.c_str(), owner.size()); + + // 先创建文件 + int ret = ::Create2(filename.c_str(), &userinfo, + 100*1024*1024*1024ul, stripeUnit, stripeCount); + if (ret != LIBCURVE_ERROR::OK && ret != -LIBCURVE_ERROR::EXISTS) { + LOG(ERROR) << "file create failed! " << ret + << ", filename = " << filename; + return -1; + } + + // 再打开文件 + int fd = ::Open(filename.c_str(), &userinfo); + if (fd < 0 && ret != -LIBCURVE_ERROR::FILE_OCCUPIED) { + LOG(ERROR) << "Open file failed!"; + return -1; + } + + return fd; +} + } // namespace test } // namespace curve diff --git a/test/integration/client/common/file_operation.h b/test/integration/client/common/file_operation.h index 51977d8786..0414146eff 100644 --- a/test/integration/client/common/file_operation.h +++ b/test/integration/client/common/file_operation.h @@ -36,6 +36,9 @@ class FileCommonOperation { static int Open(const std::string& filename, const std::string& owner); static void Close(int fd); + + static int Open(const std::string& filename, const std::string& owner, + uint64_t stripeUnit, uint64_t stripeCount); }; } // namespace test } // namespace curve diff --git a/test/integration/client/config/topo_example.json b/test/integration/client/config/topo_example.json index 8661446fab..a46a983212 100644 --- a/test/integration/client/config/topo_example.json +++ b/test/integration/client/config/topo_example.json @@ -1,7 +1,7 @@ { "logicalpools": [ { - "copysetnum": 300, + "copysetnum": 30, "name": "logicalPool1", "physicalpool": "pool1", "replicasnum": 3, diff --git a/test/integration/client/mds_exception_test.cpp b/test/integration/client/mds_exception_test.cpp index df5faa90b9..71fada95cf 100644 --- a/test/integration/client/mds_exception_test.cpp +++ b/test/integration/client/mds_exception_test.cpp @@ -46,6 +46,8 @@ uint64_t ioFailedCount = 0; std::mutex resumeMtx; std::condition_variable resumeCV; curve::client::InflightControl inflightContl; +bool testIOWrite = false; +bool testIORead = false; using curve::CurveCluster; const std::vector mdsConf{ @@ -386,6 +388,87 @@ class MDSModuleException : public ::testing::Test { return AioWrite(fd, context) == 0; } + /** 下发一个写请求并读取进行数据验证 + * @param: fd 卷fd + * @param: 当前需要下发io的偏移 + * @param：下发io的大小 + * @return: 数据是否一致 + */ + void VerifyDataConsistency(int fd, uint64_t offset, uint64_t size) { + char* writebuf = new char[size]; + char* readbuf = new char[size]; + unsigned int i; + + LOG(INFO) << "VerifyDataConsistency(): offset " << + offset << ", size " << size; + for (i = 0; i < size; i++) { + writebuf[i] = ('a' + std::rand() % 26); + } + + // 开始写 + auto wcb = [](CurveAioContext* context) { + if (context->ret == context->length) { + testIOWrite = true; + } + std::unique_lock lk(resumeMtx); + resumeCV.notify_all(); + delete context; + }; + + auto writefunc = [&]() { + CurveAioContext* context = new CurveAioContext;; + context->op = LIBCURVE_OP::LIBCURVE_OP_WRITE; + context->offset = offset; + context->length = size; + context->buf = writebuf; + context->cb = wcb; + ASSERT_EQ(LIBCURVE_ERROR::OK, AioWrite(fd, context)); + }; + + std::thread writeThtread(writefunc); + { + std::unique_lock lk(resumeMtx); + resumeCV.wait_for(lk, std::chrono::seconds(300)); + } + + writeThtread.join(); + ASSERT_TRUE(testIOWrite); + + // 开始读 + auto rcb = [](CurveAioContext* context) { + if (context->ret == context->length) { + testIORead = true; + } + std::unique_lock lk(resumeMtx); + resumeCV.notify_all(); + delete context; + }; + + auto readfunc = [&]() { + CurveAioContext* context = new CurveAioContext;; + context->op = LIBCURVE_OP::LIBCURVE_OP_READ; + context->offset = offset; + context->length = size; + context->buf = readbuf; + context->cb = rcb; + ASSERT_EQ(LIBCURVE_ERROR::OK, AioRead(fd, context)); + }; + + std::thread readThread(readfunc); + { + std::unique_lock lk(resumeMtx); + resumeCV.wait_for(lk, std::chrono::seconds(300)); + } + + readThread.join(); + ASSERT_TRUE(testIORead); + ASSERT_EQ(strcmp(writebuf, readbuf), 0); + + delete[] writebuf; + delete[] readbuf; + return; + } + int fd; // 是否出现挂卸载失败 @@ -925,3 +1008,49 @@ TEST_F(MDSModuleException, MDSExceptionTest) { // 10. 对集群没有影响 ASSERT_TRUE(MonitorResume(0, 4096, 1)); } + +TEST_F(MDSModuleException, StripeMDSExceptionTest) { + LOG(INFO) << "current case: StripeMDSExceptionTest"; + // 1. 创建一个条带的卷 + int stripefd = curve::test::FileCommonOperation::Open("/test2", + "curve", 1024 * 1024, 8); + ASSERT_NE(stripefd, -1); + uint64_t offset = std::rand() % 5 * segment_size; + + // 2. 进行数据的读写校验 + VerifyDataConsistency(stripefd, offset, 128 *1024 *1024); + std::this_thread::sleep_for(std::chrono::seconds(60)); + // 3. kill 一台当前为leader的mds + LOG(INFO) << "stop mds."; + int serviceMDSID = 0; + cluster->CurrentServiceMDS(&serviceMDSID); + ASSERT_EQ(0, cluster->StopMDS(serviceMDSID)); + // 4. 启动后台挂卸载线程 + CreateOpenFileBackend(); + + // 5. 继续随机写数据进行校验 + offset = std::rand() % 5 * segment_size; + LOG(INFO) << "when stop mds, write and read data."; + VerifyDataConsistency(stripefd, offset, 128 *1024 *1024); + + // 6. 等待挂卸载检测结果 + WaitBackendCreateDone(); + + // 7. 挂卸载服务正常 + ASSERT_TRUE(createOrOpenFailed); + + LOG(INFO) <<"start mds."; + pid_t pid = cluster->StartSingleMDS(serviceMDSID, ipmap[serviceMDSID], + 22240 + serviceMDSID, + configmap[serviceMDSID], false); + LOG(INFO) << "mds " << serviceMDSID << " started on " << ipmap[serviceMDSID] + << ", pid = " << pid; + ASSERT_GT(pid, 0); + + + LOG(INFO) << "start mds, write and read data."; + offset = std::rand() % 5 * segment_size; + VerifyDataConsistency(stripefd, offset, 128 *1024 *1024); + + ::Close(stripefd); +} diff --git a/test/mds/nameserver2/curvefs_test.cpp b/test/mds/nameserver2/curvefs_test.cpp index 3260cc86e7..78bbf61f8f 100644 --- a/test/mds/nameserver2/curvefs_test.cpp +++ b/test/mds/nameserver2/curvefs_test.cpp @@ -122,16 +122,18 @@ class CurveFSTest: public ::testing::Test { TEST_F(CurveFSTest, testCreateFile1) { // test parm error ASSERT_EQ(curvefs_->CreateFile("/file1", "owner1", FileType::INODE_PAGEFILE, - kMiniFileLength - 1), StatusCode::kFileLengthNotSupported); + kMiniFileLength - 1, 0, 0), + StatusCode::kFileLengthNotSupported); ASSERT_EQ(curvefs_->CreateFile("/file1", "owner1", FileType::INODE_PAGEFILE, - kMaxFileLength + 1), StatusCode::kFileLengthNotSupported); + kMaxFileLength + 1, 0, 0), + StatusCode::kFileLengthNotSupported); ASSERT_EQ(curvefs_->CreateFile("/flie1", "owner1", FileType::INODE_PAGEFILE, - kMiniFileLength + 1), + kMiniFileLength + 1, 0, 0), StatusCode::kFileLengthNotSupported); - ASSERT_EQ(curvefs_->CreateFile("/", "", FileType::INODE_DIRECTORY, 0), + ASSERT_EQ(curvefs_->CreateFile("/", "", FileType::INODE_DIRECTORY, 0, 0, 0), StatusCode::kFileExists); { @@ -141,7 +143,7 @@ TEST_F(CurveFSTest, testCreateFile1) { .WillOnce(Return(StoreStatus::OK)); auto statusCode = curvefs_->CreateFile("/file1", "owner1", - FileType::INODE_PAGEFILE, kMiniFileLength); + FileType::INODE_PAGEFILE, kMiniFileLength, 0, 0); ASSERT_EQ(statusCode, StatusCode::kFileExists); } @@ -152,7 +154,7 @@ TEST_F(CurveFSTest, testCreateFile1) { .WillOnce(Return(StoreStatus::InternalError)); auto statusCode = curvefs_->CreateFile("/file1", "owner1", - FileType::INODE_PAGEFILE, kMiniFileLength); + FileType::INODE_PAGEFILE, kMiniFileLength, 0, 0); ASSERT_EQ(statusCode, StatusCode::kStorageError); } @@ -171,7 +173,7 @@ TEST_F(CurveFSTest, testCreateFile1) { .WillOnce(Return(true)); auto statusCode = curvefs_->CreateFile("/file1", "owner1", - FileType::INODE_PAGEFILE, kMiniFileLength); + FileType::INODE_PAGEFILE, kMiniFileLength, 0, 0); ASSERT_EQ(statusCode, StatusCode::kStorageError); } @@ -191,7 +193,7 @@ TEST_F(CurveFSTest, testCreateFile1) { auto statusCode = curvefs_->CreateFile("/file1", "owner1", - FileType::INODE_PAGEFILE, kMiniFileLength); + FileType::INODE_PAGEFILE, kMiniFileLength, 0, 0); ASSERT_EQ(statusCode, StatusCode::kOK); } @@ -206,11 +208,56 @@ TEST_F(CurveFSTest, testCreateFile1) { .WillOnce(Return(false)); auto statusCode = curvefs_->CreateFile("/file1", "owner1", - FileType::INODE_PAGEFILE, kMiniFileLength); + FileType::INODE_PAGEFILE, kMiniFileLength, 0, 0); ASSERT_EQ(statusCode, StatusCode::kStorageError); } } +TEST_F(CurveFSTest, testCreateStripeFile) { + { + // test create ok + EXPECT_CALL(*storage_, GetFile(_, _, _)) + .Times(AtLeast(1)) + .WillOnce(Return(StoreStatus::KeyNotExist)); + + EXPECT_CALL(*storage_, PutFile(_)) + .Times(AtLeast(1)) + .WillOnce(Return(StoreStatus::OK)); + + EXPECT_CALL(*inodeIdGenerator_, GenInodeID(_)) + .Times(1) + .WillOnce(Return(true)); + + ASSERT_EQ(curvefs_->CreateFile("/file1", "owner1", + FileType::INODE_PAGEFILE, kMiniFileLength, + 1 * 1024 * 1024, 4), StatusCode::kOK); + } + + { + // test stripeStripe and stripeCount is not all zero + ASSERT_EQ(curvefs_->CreateFile("/file1", "owner1", + FileType::INODE_PAGEFILE, kMiniFileLength, 0, 1), + StatusCode::kParaError); + ASSERT_EQ(curvefs_->CreateFile("/file1", "owner1", + FileType::INODE_PAGEFILE, kMiniFileLength, 1024*1024ul, 0), + StatusCode::kParaError); + } + + { + // test stripeUnit more then chunksize + ASSERT_EQ(curvefs_->CreateFile("/file1", "owner1", + FileType::INODE_PAGEFILE, kMiniFileLength, 16*1024*1024ul + 1, 0), + StatusCode::kParaError); + } + + { + // test stripeUnit is not divisible by chunksize + ASSERT_EQ(curvefs_->CreateFile("/file1", "owner1", + FileType::INODE_PAGEFILE, kMiniFileLength, + 4*1024*1024ul + 1, 0), StatusCode::kParaError); + } +} + TEST_F(CurveFSTest, testGetFileInfo) { // test parm error FileInfo fileInfo;