From 11dd7355c179d9dc6b68282c2c2fbe513b481554 Mon Sep 17 00:00:00 2001 From: jumao Date: Fri, 8 Sep 2023 12:09:48 -0400 Subject: [PATCH] =?UTF-8?q?*=20[saidump]=20=E2=80=A2=09Saidump=20for=20DNX?= =?UTF-8?q?-SAI=20https://github.com/sonic-net/sonic-buildimage/issues/135?= =?UTF-8?q?61?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- saidump/saidump.cpp | 75 +++++++++++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index 72f997723..cd19378ce 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -5,6 +5,7 @@ #include #include #include +#include extern "C" { #include @@ -22,17 +23,19 @@ extern "C" { using namespace swss; using json = nlohmann::json; -// 100 MB -constexpr int64_t RDB_FILE_MAX_SIZE = 1024 * 1024 * 100; +// Default value: 100 MB +constexpr int64_t RDB_JSON_MAX_SIZE = 1024 * 1024 * 100; struct CmdOptions { bool skipAttributes; bool dumpTempView; bool dumpGraph; - std::string rdbFile; + std::string rdbJsonFile; + uint64_t rdbJSonSizeLimit; }; + static CmdOptions g_cmdOptions; static std::map g_oid_map; @@ -40,13 +43,15 @@ void printUsage() { SWSS_LOG_ENTER(); - std::cout << "Usage: saidump [-t] [-g] [-r] [-h]" << std::endl; + std::cout << "Usage: saidump [-t] [-g] [-r] [-m] [-h]" << std::endl; std::cout << " -t --tempView:" << std::endl; std::cout << " Dump temp view" << std::endl; std::cout << " -g --dumpGraph:" << std::endl; std::cout << " Dump current graph" << std::endl; std::cout << " -r --rdb:" << std::endl; - std::cout << " Dump by parsing the Redis persistence file dump.rdb that is generated by SAVE command" << std::endl; + std::cout << " Dump by parsing the RDB JSON file, which is created by rdbtools based on Redis dump.rdb that is generated by Redis SAVE command" << std::endl; + std::cout << " -m --max:" << std::endl; + std::cout << " Config the the RDB JSON file's max size in MB, which is optional with default value 100MB" << std::endl; std::cout << " -h --help:" << std::endl; std::cout << " Print out this message" << std::endl; } @@ -60,7 +65,8 @@ CmdOptions handleCmdLine(int argc, char **argv) options.dumpTempView = false; options.dumpGraph = false; - const char* const optstring = "gtr:h"; + const char* const optstring = "gtr:m:h"; + uint64_t result = 0; while (true) { @@ -69,6 +75,7 @@ CmdOptions handleCmdLine(int argc, char **argv) { "dumpGraph", no_argument, 0, 'g' }, { "tempView", no_argument, 0, 't' }, { "rdb", required_argument, 0, 'r' }, + { "max", required_argument, 0, 'm' }, { "help", no_argument, 0, 'h' }, { 0, 0, 0, 0 } }; @@ -96,7 +103,36 @@ CmdOptions handleCmdLine(int argc, char **argv) case 'r': SWSS_LOG_NOTICE("Dumping from %s", optarg); - options.rdbFile = std::string(optarg); + options.rdbJsonFile = std::string(optarg); + break; + + case 'm': + if(!regex_match(optarg, std::regex(R"([+]?\d+)"))) //only positive numeric chars are valid, such as 3984, +3232, etc. + { + SWSS_LOG_WARN("invalid option -m %s", optarg); + printUsage(); + exit(EXIT_SUCCESS); + } + + result = strtoull(optarg, NULL, 0); + + if((errno == ERANGE && (result == ULLONG_MAX || result == 0)) || (errno != 0 && result == 0) || result >= INT_MAX) + { + SWSS_LOG_WARN("invalid option -m %s", optarg); + printUsage(); + exit(EXIT_SUCCESS); + } + + if(result == 0) + { + options.rdbJSonSizeLimit = RDB_JSON_MAX_SIZE; + } + else + { + options.rdbJSonSizeLimit = result * 1024 * 1024; + } + + SWSS_LOG_NOTICE("Configure the RDB JSON MAX size to %llu MB", options.rdbJSonSizeLimit / 1024 / 1024); break; case 'h': @@ -419,9 +455,9 @@ void dumpGraph(const TableDump& td) #define SWSS_LOG_ERROR_AND_STDERR(format, ...) { fprintf(stderr, format"\n", ##__VA_ARGS__); SWSS_LOG_ERROR(format, ##__VA_ARGS__); } /** - * @brief Preprocess the input json file to make sure it's a valid json file. + * @brief Preprocess the input JSON file to make sure it's a valid JSON file for Nlohmann JSON library. */ -static sai_status_t preProcessFile(std::string file_name) +static sai_status_t preProcessFile(const std::string file_name) { SWSS_LOG_ENTER(); @@ -434,11 +470,11 @@ static sai_status_t preProcessFile(std::string file_name) } input_file.seekg(0, std::ios::end); // Move to the end of the file - int64_t file_size = input_file.tellg(); // Get the current position + uint64_t file_size = input_file.tellg(); // Get the current position - if (file_size >= RDB_FILE_MAX_SIZE) + if (file_size >= g_cmdOptions.rdbJSonSizeLimit) { - SWSS_LOG_ERROR_AND_STDERR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); + SWSS_LOG_ERROR_AND_STDERR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, g_cmdOptions.rdbJSonSizeLimit / 1024 / 1024); return SAI_STATUS_FAILURE; } @@ -471,7 +507,7 @@ static sai_status_t preProcessFile(std::string file_name) return SAI_STATUS_SUCCESS; } -static sai_status_t dumpFromRedisRdbFile(std::string file_name) +static sai_status_t dumpFromRedisRdbJson(const std::string file_name) { SWSS_LOG_ENTER(); @@ -479,7 +515,7 @@ static sai_status_t dumpFromRedisRdbFile(std::string file_name) if (!input_file.is_open()) { - SWSS_LOG_ERROR_AND_STDERR("The file %s does not exist for dumping from redis dump.rdb file.", file_name.c_str()); + SWSS_LOG_ERROR_AND_STDERR("The file %s does not exist for dumping from Redis RDB JSON file.", file_name.c_str()); return SAI_STATUS_FAILURE; } @@ -539,9 +575,9 @@ static sai_status_t dumpFromRedisRdbFile(std::string file_name) map[itt.key()] = itt.value(); } - size_t indent = 4; + constexpr size_t LINE_IDENT = 4; size_t max_len = get_max_attr_len(map); - std::string str_indent = pad_string("", indent); + std::string str_indent = pad_string("", LINE_IDENT); for (const auto&field: map) { @@ -575,14 +611,15 @@ int main(int argc, char **argv) g_cmdOptions = handleCmdLine(argc, argv); - if (g_cmdOptions.rdbFile.size() > 0) + + if (g_cmdOptions.rdbJsonFile.size() > 0) { - if (SAI_STATUS_FAILURE == preProcessFile(g_cmdOptions.rdbFile)) + if (SAI_STATUS_FAILURE == preProcessFile(g_cmdOptions.rdbJsonFile)) { return EXIT_FAILURE; } - return dumpFromRedisRdbFile(g_cmdOptions.rdbFile); + return dumpFromRedisRdbJson(g_cmdOptions.rdbJsonFile); } swss::DBConnector db("ASIC_DB", 0);