From 5ee5c122547b093fc1ba16f649bcfe2a49fa0e99 Mon Sep 17 00:00:00 2001 From: jumao Date: Fri, 1 Sep 2023 17:25:20 -0400 Subject: [PATCH 01/13] =?UTF-8?q?*=20[saidump]=20=E2=80=A2=09Saidump=20for?= =?UTF-8?q?=20DNX-SAI=20https://github.com/sonic-net/sonic-buildimage/issu?= =?UTF-8?q?es/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, #1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null #2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" #3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null #4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" #5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- saidump/saidump.cpp | 187 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 184 insertions(+), 3 deletions(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index 6a3321e58..cf99f8dc0 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -2,6 +2,9 @@ #include #include #include +#include +#include +#include extern "C" { #include @@ -10,18 +13,26 @@ extern "C" { #include "swss/table.h" #include "meta/sai_serialize.h" #include "sairediscommon.h" +#include "swss/json.h" +#include "swss/json.hpp" #include // TODO split to multiple cpp using namespace swss; +using namespace std; +using json = nlohmann::json; + +// 100 MB +const int64_t RDB_FILE_MAX_SIZE = 1024 * 1024 * 100; struct CmdOptions { bool skipAttributes; bool dumpTempView; bool dumpGraph; + string rdb_file; }; CmdOptions g_cmdOptions; @@ -31,11 +42,13 @@ void printUsage() { SWSS_LOG_ENTER(); - std::cout << "Usage: saidump [-t] [-g] [-h]" << std::endl; + std::cout << "Usage: saidump [-t] [-g] [-r] [-h]" << std::endl; std::cout << " -t --tempView:" << std::endl; std::cout << " Dump temp view" << std::endl; std::cout << " -g --dumpGraph:" << std::endl; std::cout << " Dump current graph" << std::endl; + std::cout << " -r --rdb:" << std::endl; + std::cout << " Dump by parsing the Redis persistence file dump.rdb that is generated by SAVE command." << std::endl; std::cout << " -h --help:" << std::endl; std::cout << " Print out this message" << std::endl; } @@ -49,7 +62,7 @@ CmdOptions handleCmdLine(int argc, char **argv) options.dumpTempView = false; options.dumpGraph = false; - const char* const optstring = "gth"; + const char* const optstring = "gtr:h"; while (true) { @@ -57,6 +70,7 @@ CmdOptions handleCmdLine(int argc, char **argv) { { "dumpGraph", no_argument, 0, 'g' }, { "tempView", no_argument, 0, 't' }, + { "rdb", required_argument, 0, 'r' }, { "help", no_argument, 0, 'h' }, { 0, 0, 0, 0 } }; @@ -82,6 +96,11 @@ CmdOptions handleCmdLine(int argc, char **argv) options.dumpTempView = true; break; + case 'r': + SWSS_LOG_WARN("Dumping from dump.rdb"); + options.rdb_file = string(optarg); + break; + case 'h': printUsage(); exit(EXIT_SUCCESS); @@ -399,7 +418,159 @@ void dumpGraph(const TableDump& td) std::cout << "}" << std::endl; } -int main(int argc, char ** argv) +/* + preprocess the input json file to make make sure it's a valid json file. +*/ +int pre_process_file(string file_name) +{ + ifstream input_file(file_name); + + if (!input_file.is_open()) + { + cerr << "Failed to open the input file." << endl; + return SAI_STATUS_FAILURE; + } + + input_file.seekg(0, ios::end); // Move to the end of the file + int64_t file_size = input_file.tellg(); // Get the current position + + if (file_size >= RDB_FILE_MAX_SIZE) + { + SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.\n", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); + + // Close the input file + input_file.close(); + return SAI_STATUS_FAILURE; + } + input_file.seekg(0, ios::beg); // Move to the begin of the file + + // Read the content of the input file into a string + string content((istreambuf_iterator(input_file)), + istreambuf_iterator()); + + + input_file.close(); + + content = regex_replace(content, regex("\\},\\{\\r"), ","); + + //erase the 1st and last char. + if (content.length() >= 2 && content[0] == '[' && content[content.length()-1] == ']') { + content.erase(0, 1); + content.erase(content.length() - 1); + } + + ofstream outputFile(file_name); + + if (!outputFile.is_open()) + { + cerr << "Failed to open the output file." << endl; + return SAI_STATUS_FAILURE; + } + + outputFile << content; + outputFile.close(); + + return SAI_STATUS_SUCCESS; +} + +int dump_from_redis_rdb_file(string file_name) +{ + ifstream input_file(file_name); + if (!input_file.is_open()) + { + SWSS_LOG_NOTICE("The file %s is not exsisted for dumping from redis dump.rdb file", file_name); + return SAI_STATUS_FAILURE; + } + + input_file.seekg(0, ios::end); // Move to the end of the file + int64_t file_size = input_file.tellg(); // Get the current position + if (file_size >= RDB_FILE_MAX_SIZE) + { + SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.\n", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); + + // Close the input file + input_file.close(); + return SAI_STATUS_FAILURE; + } + input_file.seekg(0, ios::beg); // Move to the begin of the file + + try + { + // Parse the JSON data from the file (validation) + nlohmann::json jsonData; + input_file >> jsonData; + input_file.close(); + + SWSS_LOG_DEBUG("%s\n","JSON file is valid.\n"); + + for (json::iterator it = jsonData.begin(); it != jsonData.end(); ++it) + { + json jj_key = it.key(); + + string keystr = jj_key; + string item_name = keystr; + size_t pos = keystr.find_first_of(":"); + if (pos != string::npos) + { + if("ASIC_STATE" != keystr.substr(0, pos)) // filter out non ASIC_STATE + { + continue; + } + item_name = keystr.substr(pos + 1); + if (item_name.find(":") != string::npos) + { + item_name.replace(item_name.find_first_of(":"), 1, " "); + } + } + else + { + continue; + } + + cout << item_name << " " << endl; + + json jj = it.value(); + + if (!(*it).is_object()) + { + continue; + } + + TableMap map; + for (json::iterator itt = jj.begin(); itt != jj.end(); ++itt) + { + if (itt.key() == "NULL") + { + continue; + } + map[itt.key()] = itt.value(); + } + + size_t indent = 4; + size_t max_len = get_max_attr_len(map); + string str_indent = pad_string("", indent); + + for (const auto&field: map) + { + stringstream ss; + ss << str_indent << pad_string(field.first, max_len) << " : "; + ss << field.second; + cout << ss.str() << std::endl; + } + cout << endl; + } + return SAI_STATUS_SUCCESS; + } + catch (exception &ex) + { + input_file.close(); + cerr << "JSON file:" << file_name << " is invalid." << endl; + cerr << "JSON parsing error: " << ex.what() << endl; + } + return SAI_STATUS_FAILURE; +} + +int main(int argc, char **argv) { swss::Logger::getInstance().setMinPrio(swss::Logger::SWSS_DEBUG); @@ -411,6 +582,16 @@ int main(int argc, char ** argv) g_cmdOptions = handleCmdLine(argc, argv); + if (g_cmdOptions.rdb_file.size() > 0) + { + if (SAI_STATUS_FAILURE == pre_process_file(g_cmdOptions.rdb_file)) + { + return EXIT_FAILURE; + } + dump_from_redis_rdb_file(g_cmdOptions.rdb_file); + return EXIT_SUCCESS; + } + swss::DBConnector db("ASIC_DB", 0); std::string table = ASIC_STATE_TABLE; From 049947e6e4e7d1070c382c404ec15b40b2bb0551 Mon Sep 17 00:00:00 2001 From: jumao Date: Fri, 1 Sep 2023 17:25:20 -0400 Subject: [PATCH 02/13] =?UTF-8?q?*=20[saidump]=20=E2=80=A2=09Saidump=20for?= =?UTF-8?q?=20DNX-SAI=20https://github.com/sonic-net/sonic-buildimage/issu?= =?UTF-8?q?es/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- saidump/saidump.cpp | 187 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 184 insertions(+), 3 deletions(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index 6a3321e58..cf99f8dc0 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -2,6 +2,9 @@ #include #include #include +#include +#include +#include extern "C" { #include @@ -10,18 +13,26 @@ extern "C" { #include "swss/table.h" #include "meta/sai_serialize.h" #include "sairediscommon.h" +#include "swss/json.h" +#include "swss/json.hpp" #include // TODO split to multiple cpp using namespace swss; +using namespace std; +using json = nlohmann::json; + +// 100 MB +const int64_t RDB_FILE_MAX_SIZE = 1024 * 1024 * 100; struct CmdOptions { bool skipAttributes; bool dumpTempView; bool dumpGraph; + string rdb_file; }; CmdOptions g_cmdOptions; @@ -31,11 +42,13 @@ void printUsage() { SWSS_LOG_ENTER(); - std::cout << "Usage: saidump [-t] [-g] [-h]" << std::endl; + std::cout << "Usage: saidump [-t] [-g] [-r] [-h]" << std::endl; std::cout << " -t --tempView:" << std::endl; std::cout << " Dump temp view" << std::endl; std::cout << " -g --dumpGraph:" << std::endl; std::cout << " Dump current graph" << std::endl; + std::cout << " -r --rdb:" << std::endl; + std::cout << " Dump by parsing the Redis persistence file dump.rdb that is generated by SAVE command." << std::endl; std::cout << " -h --help:" << std::endl; std::cout << " Print out this message" << std::endl; } @@ -49,7 +62,7 @@ CmdOptions handleCmdLine(int argc, char **argv) options.dumpTempView = false; options.dumpGraph = false; - const char* const optstring = "gth"; + const char* const optstring = "gtr:h"; while (true) { @@ -57,6 +70,7 @@ CmdOptions handleCmdLine(int argc, char **argv) { { "dumpGraph", no_argument, 0, 'g' }, { "tempView", no_argument, 0, 't' }, + { "rdb", required_argument, 0, 'r' }, { "help", no_argument, 0, 'h' }, { 0, 0, 0, 0 } }; @@ -82,6 +96,11 @@ CmdOptions handleCmdLine(int argc, char **argv) options.dumpTempView = true; break; + case 'r': + SWSS_LOG_WARN("Dumping from dump.rdb"); + options.rdb_file = string(optarg); + break; + case 'h': printUsage(); exit(EXIT_SUCCESS); @@ -399,7 +418,159 @@ void dumpGraph(const TableDump& td) std::cout << "}" << std::endl; } -int main(int argc, char ** argv) +/* + preprocess the input json file to make make sure it's a valid json file. +*/ +int pre_process_file(string file_name) +{ + ifstream input_file(file_name); + + if (!input_file.is_open()) + { + cerr << "Failed to open the input file." << endl; + return SAI_STATUS_FAILURE; + } + + input_file.seekg(0, ios::end); // Move to the end of the file + int64_t file_size = input_file.tellg(); // Get the current position + + if (file_size >= RDB_FILE_MAX_SIZE) + { + SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.\n", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); + + // Close the input file + input_file.close(); + return SAI_STATUS_FAILURE; + } + input_file.seekg(0, ios::beg); // Move to the begin of the file + + // Read the content of the input file into a string + string content((istreambuf_iterator(input_file)), + istreambuf_iterator()); + + + input_file.close(); + + content = regex_replace(content, regex("\\},\\{\\r"), ","); + + //erase the 1st and last char. + if (content.length() >= 2 && content[0] == '[' && content[content.length()-1] == ']') { + content.erase(0, 1); + content.erase(content.length() - 1); + } + + ofstream outputFile(file_name); + + if (!outputFile.is_open()) + { + cerr << "Failed to open the output file." << endl; + return SAI_STATUS_FAILURE; + } + + outputFile << content; + outputFile.close(); + + return SAI_STATUS_SUCCESS; +} + +int dump_from_redis_rdb_file(string file_name) +{ + ifstream input_file(file_name); + if (!input_file.is_open()) + { + SWSS_LOG_NOTICE("The file %s is not exsisted for dumping from redis dump.rdb file", file_name); + return SAI_STATUS_FAILURE; + } + + input_file.seekg(0, ios::end); // Move to the end of the file + int64_t file_size = input_file.tellg(); // Get the current position + if (file_size >= RDB_FILE_MAX_SIZE) + { + SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.\n", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); + + // Close the input file + input_file.close(); + return SAI_STATUS_FAILURE; + } + input_file.seekg(0, ios::beg); // Move to the begin of the file + + try + { + // Parse the JSON data from the file (validation) + nlohmann::json jsonData; + input_file >> jsonData; + input_file.close(); + + SWSS_LOG_DEBUG("%s\n","JSON file is valid.\n"); + + for (json::iterator it = jsonData.begin(); it != jsonData.end(); ++it) + { + json jj_key = it.key(); + + string keystr = jj_key; + string item_name = keystr; + size_t pos = keystr.find_first_of(":"); + if (pos != string::npos) + { + if("ASIC_STATE" != keystr.substr(0, pos)) // filter out non ASIC_STATE + { + continue; + } + item_name = keystr.substr(pos + 1); + if (item_name.find(":") != string::npos) + { + item_name.replace(item_name.find_first_of(":"), 1, " "); + } + } + else + { + continue; + } + + cout << item_name << " " << endl; + + json jj = it.value(); + + if (!(*it).is_object()) + { + continue; + } + + TableMap map; + for (json::iterator itt = jj.begin(); itt != jj.end(); ++itt) + { + if (itt.key() == "NULL") + { + continue; + } + map[itt.key()] = itt.value(); + } + + size_t indent = 4; + size_t max_len = get_max_attr_len(map); + string str_indent = pad_string("", indent); + + for (const auto&field: map) + { + stringstream ss; + ss << str_indent << pad_string(field.first, max_len) << " : "; + ss << field.second; + cout << ss.str() << std::endl; + } + cout << endl; + } + return SAI_STATUS_SUCCESS; + } + catch (exception &ex) + { + input_file.close(); + cerr << "JSON file:" << file_name << " is invalid." << endl; + cerr << "JSON parsing error: " << ex.what() << endl; + } + return SAI_STATUS_FAILURE; +} + +int main(int argc, char **argv) { swss::Logger::getInstance().setMinPrio(swss::Logger::SWSS_DEBUG); @@ -411,6 +582,16 @@ int main(int argc, char ** argv) g_cmdOptions = handleCmdLine(argc, argv); + if (g_cmdOptions.rdb_file.size() > 0) + { + if (SAI_STATUS_FAILURE == pre_process_file(g_cmdOptions.rdb_file)) + { + return EXIT_FAILURE; + } + dump_from_redis_rdb_file(g_cmdOptions.rdb_file); + return EXIT_SUCCESS; + } + swss::DBConnector db("ASIC_DB", 0); std::string table = ASIC_STATE_TABLE; From 21bdccd32240d343e17b36025d07054d103e6899 Mon Sep 17 00:00:00 2001 From: jumao Date: Wed, 6 Sep 2023 11:26:17 -0400 Subject: [PATCH 03/13] =?UTF-8?q?=20=20=20=20*=20[saidump]=20=20=20=20=20?= =?UTF-8?q?=E2=80=A2=20=20=20=20=20=20=20Saidump=20for=20DNX-SAI=20https:/?= =?UTF-8?q?/github.com/sonic-net/sonic-buildimage/issues/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- saidump/saidump.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index cf99f8dc0..865bd2888 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -478,7 +478,7 @@ int dump_from_redis_rdb_file(string file_name) ifstream input_file(file_name); if (!input_file.is_open()) { - SWSS_LOG_NOTICE("The file %s is not exsisted for dumping from redis dump.rdb file", file_name); + SWSS_LOG_NOTICE("The file %s does not exist for dumping from redis dump.rdb file.", file_name); return SAI_STATUS_FAILURE; } From 1464f573dbebcf6245b9359a21d7289f72fc7547 Mon Sep 17 00:00:00 2001 From: jumao Date: Wed, 6 Sep 2023 11:34:57 -0400 Subject: [PATCH 04/13] =?UTF-8?q?=20=20=20=20*=20[saidump]=20=20=20=20=20?= =?UTF-8?q?=E2=80=A2=20=20=20=20=20=20=20Saidump=20for=20DNX-SAI=20https:/?= =?UTF-8?q?/github.com/sonic-net/sonic-buildimage/issues/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- saidump/saidump.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index 865bd2888..57c38d4fd 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -70,7 +70,7 @@ CmdOptions handleCmdLine(int argc, char **argv) { { "dumpGraph", no_argument, 0, 'g' }, { "tempView", no_argument, 0, 't' }, - { "rdb", required_argument, 0, 'r' }, + { "rdb", required_argument, 0, 'r' }, { "help", no_argument, 0, 'h' }, { 0, 0, 0, 0 } }; @@ -419,7 +419,7 @@ void dumpGraph(const TableDump& td) } /* - preprocess the input json file to make make sure it's a valid json file. + preprocess the input json file to make sure it's a valid json file. */ int pre_process_file(string file_name) { From e0b6f9835dde37b9fab3c4fe2a05849051ec5902 Mon Sep 17 00:00:00 2001 From: jumao Date: Wed, 6 Sep 2023 16:54:23 -0400 Subject: [PATCH 05/13] =?UTF-8?q?=20=20=20=20=20=20=20=20*=20[saidump]=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=E2=80=A2=20=20=20=20=20=20=20Saidump?= =?UTF-8?q?=20for=20DNX-SAI=20https://github.com/sonic-net/sonic-buildimag?= =?UTF-8?q?e/issues/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- saidump/saidump.cpp | 80 +++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index 57c38d4fd..fa8ec2680 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -21,18 +21,17 @@ extern "C" { // TODO split to multiple cpp using namespace swss; -using namespace std; using json = nlohmann::json; // 100 MB -const int64_t RDB_FILE_MAX_SIZE = 1024 * 1024 * 100; +constexpr int64_t RDB_FILE_MAX_SIZE = 1024 * 1024 * 100; struct CmdOptions { bool skipAttributes; bool dumpTempView; bool dumpGraph; - string rdb_file; + std::string rdbFile; }; CmdOptions g_cmdOptions; @@ -97,8 +96,8 @@ CmdOptions handleCmdLine(int argc, char **argv) break; case 'r': - SWSS_LOG_WARN("Dumping from dump.rdb"); - options.rdb_file = string(optarg); + SWSS_LOG_NOTICE("Dumping from dump.rdb"); + options.rdbFile = std::string(optarg); break; case 'h': @@ -421,17 +420,17 @@ void dumpGraph(const TableDump& td) /* preprocess the input json file to make sure it's a valid json file. */ -int pre_process_file(string file_name) +static int preProcessFile(std::string file_name) { - ifstream input_file(file_name); + std::ifstream input_file(file_name); if (!input_file.is_open()) { - cerr << "Failed to open the input file." << endl; + std::cerr << "Failed to open the input file." << std::endl; return SAI_STATUS_FAILURE; } - input_file.seekg(0, ios::end); // Move to the end of the file + input_file.seekg(0, std::ios::end); // Move to the end of the file int64_t file_size = input_file.tellg(); // Get the current position if (file_size >= RDB_FILE_MAX_SIZE) @@ -442,16 +441,16 @@ int pre_process_file(string file_name) input_file.close(); return SAI_STATUS_FAILURE; } - input_file.seekg(0, ios::beg); // Move to the begin of the file + input_file.seekg(0, std::ios::beg); // Move to the begin of the file // Read the content of the input file into a string - string content((istreambuf_iterator(input_file)), - istreambuf_iterator()); + std::string content((std::istreambuf_iterator(input_file)), + std::istreambuf_iterator()); input_file.close(); - content = regex_replace(content, regex("\\},\\{\\r"), ","); + content = regex_replace(content, std::regex("\\},\\{\\r"), ","); //erase the 1st and last char. if (content.length() >= 2 && content[0] == '[' && content[content.length()-1] == ']') { @@ -459,11 +458,11 @@ int pre_process_file(string file_name) content.erase(content.length() - 1); } - ofstream outputFile(file_name); + std::ofstream outputFile(file_name); if (!outputFile.is_open()) { - cerr << "Failed to open the output file." << endl; + std::cerr << "Failed to open the output file." << std::endl; return SAI_STATUS_FAILURE; } @@ -473,16 +472,16 @@ int pre_process_file(string file_name) return SAI_STATUS_SUCCESS; } -int dump_from_redis_rdb_file(string file_name) +int dump_from_redis_rdb_file(std::string file_name) { - ifstream input_file(file_name); + std::ifstream input_file(file_name); if (!input_file.is_open()) { SWSS_LOG_NOTICE("The file %s does not exist for dumping from redis dump.rdb file.", file_name); return SAI_STATUS_FAILURE; } - input_file.seekg(0, ios::end); // Move to the end of the file + input_file.seekg(0, std::ios::end); // Move to the end of the file int64_t file_size = input_file.tellg(); // Get the current position if (file_size >= RDB_FILE_MAX_SIZE) { @@ -492,7 +491,7 @@ int dump_from_redis_rdb_file(string file_name) input_file.close(); return SAI_STATUS_FAILURE; } - input_file.seekg(0, ios::beg); // Move to the begin of the file + input_file.seekg(0, std::ios::beg); // Move to the begin of the file try { @@ -507,17 +506,17 @@ int dump_from_redis_rdb_file(string file_name) { json jj_key = it.key(); - string keystr = jj_key; - string item_name = keystr; + std::string keystr = jj_key; + std::string item_name = keystr; size_t pos = keystr.find_first_of(":"); - if (pos != string::npos) + if (pos != std::string::npos) { - if("ASIC_STATE" != keystr.substr(0, pos)) // filter out non ASIC_STATE + if(ASIC_STATE_TABLE != keystr.substr(0, pos)) // filter out non ASIC_STATE { continue; } item_name = keystr.substr(pos + 1); - if (item_name.find(":") != string::npos) + if (item_name.find(":") != std::string::npos) { item_name.replace(item_name.find_first_of(":"), 1, " "); } @@ -527,16 +526,17 @@ int dump_from_redis_rdb_file(string file_name) continue; } - cout << item_name << " " << endl; + std::cout << item_name << " \r\n"; json jj = it.value(); - if (!(*it).is_object()) + if (!it->is_object()) { continue; } TableMap map; + for (json::iterator itt = jj.begin(); itt != jj.end(); ++itt) { if (itt.key() == "NULL") @@ -548,24 +548,26 @@ int dump_from_redis_rdb_file(string file_name) size_t indent = 4; size_t max_len = get_max_attr_len(map); - string str_indent = pad_string("", indent); + std::string str_indent = pad_string("", indent); for (const auto&field: map) { - stringstream ss; + std::stringstream ss; ss << str_indent << pad_string(field.first, max_len) << " : "; ss << field.second; - cout << ss.str() << std::endl; + std::cout << ss.str() << "\r\n"; } - cout << endl; + std::cout << "\r\n"; } - return SAI_STATUS_SUCCESS; - } - catch (exception &ex) + return SAI_STATUS_SUCCESS; + } + catch (std::exception &ex) { input_file.close(); - cerr << "JSON file:" << file_name << " is invalid." << endl; - cerr << "JSON parsing error: " << ex.what() << endl; + std::cerr << "JSON file:" << file_name << " is invalid." << std::endl; + std::cerr << "JSON parsing error: " << ex.what() << std::endl; + SWSS_LOG_ERROR("JSON file %s is invalid\n", file_name); + SWSS_LOG_ERROR("JSON parsing error: %s", ex.what()); } return SAI_STATUS_FAILURE; } @@ -582,13 +584,13 @@ int main(int argc, char **argv) g_cmdOptions = handleCmdLine(argc, argv); - if (g_cmdOptions.rdb_file.size() > 0) + if (g_cmdOptions.rdbFile.size() > 0) { - if (SAI_STATUS_FAILURE == pre_process_file(g_cmdOptions.rdb_file)) + if (SAI_STATUS_FAILURE == preProcessFile(g_cmdOptions.rdbFile)) { return EXIT_FAILURE; - } - dump_from_redis_rdb_file(g_cmdOptions.rdb_file); + } + dump_from_redis_rdb_file(g_cmdOptions.rdbFile); return EXIT_SUCCESS; } From 957dc96e9e2b27412ad2c3876803835b5ba003ed Mon Sep 17 00:00:00 2001 From: jumao Date: Wed, 6 Sep 2023 17:44:36 -0400 Subject: [PATCH 06/13] =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20*=20[?= =?UTF-8?q?saidump]=20=20=20=20=20=20=20=20=20=20=20=20=20=E2=80=A2=20=20?= =?UTF-8?q?=20=20=20=20=20Saidump=20for=20DNX-SAI=20https://github.com/son?= =?UTF-8?q?ic-net/sonic-buildimage/issues/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- saidump/saidump.cpp | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index fa8ec2680..6524c4375 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -426,7 +426,8 @@ static int preProcessFile(std::string file_name) if (!input_file.is_open()) { - std::cerr << "Failed to open the input file." << std::endl; + std::cerr << "Failed to open the input file " << file_name << std::endl; + SWSS_LOG_ERROR("Failed to open the input file %s.", file_name.c_str()); return SAI_STATUS_FAILURE; } @@ -435,25 +436,25 @@ static int preProcessFile(std::string file_name) if (file_size >= RDB_FILE_MAX_SIZE) { - SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.\n", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); - + std::cerr << "Get " << file_name.c_str() << "'s size failure or its size " << file_size << " >= "<< RDB_FILE_MAX_SIZE / 1024 / 1024<<" MB." << std::endl; + SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); // Close the input file input_file.close(); return SAI_STATUS_FAILURE; } + input_file.seekg(0, std::ios::beg); // Move to the begin of the file // Read the content of the input file into a string std::string content((std::istreambuf_iterator(input_file)), std::istreambuf_iterator()); - - input_file.close(); - content = regex_replace(content, std::regex("\\},\\{\\r"), ","); + content = regex_replace(content, std::regex("\\},\\{\\r"), ","); //erase the 1st and last char. - if (content.length() >= 2 && content[0] == '[' && content[content.length()-1] == ']') { + if (content.length() >= 2 && content[0] == '[' && content[content.length()-1] == ']') + { content.erase(0, 1); content.erase(content.length() - 1); } @@ -462,7 +463,8 @@ static int preProcessFile(std::string file_name) if (!outputFile.is_open()) { - std::cerr << "Failed to open the output file." << std::endl; + std::cerr << "Failed to open the output file " << file_name << std::endl; + SWSS_LOG_ERROR("Failed to open the output file %s.", file_name.c_str()); return SAI_STATUS_FAILURE; } @@ -475,22 +477,27 @@ static int preProcessFile(std::string file_name) int dump_from_redis_rdb_file(std::string file_name) { std::ifstream input_file(file_name); + if (!input_file.is_open()) { - SWSS_LOG_NOTICE("The file %s does not exist for dumping from redis dump.rdb file.", file_name); + std::cerr << "Failed to open the input file " << file_name << std::endl; + SWSS_LOG_ERROR("The file %s does not exist for dumping from redis dump.rdb file.", file_name.c_str()); return SAI_STATUS_FAILURE; } input_file.seekg(0, std::ios::end); // Move to the end of the file int64_t file_size = input_file.tellg(); // Get the current position + if (file_size >= RDB_FILE_MAX_SIZE) { - SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.\n", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); + std::cerr << "Get " << file_name.c_str() << "'s size failure or its size " << file_size << " >= "<< RDB_FILE_MAX_SIZE / 1024 / 1024<<" MB." << std::endl; + SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); // Close the input file input_file.close(); return SAI_STATUS_FAILURE; } + input_file.seekg(0, std::ios::beg); // Move to the begin of the file try @@ -500,7 +507,7 @@ int dump_from_redis_rdb_file(std::string file_name) input_file >> jsonData; input_file.close(); - SWSS_LOG_DEBUG("%s\n","JSON file is valid.\n"); + SWSS_LOG_DEBUG("JSON file is valid."); for (json::iterator it = jsonData.begin(); it != jsonData.end(); ++it) { @@ -509,6 +516,7 @@ int dump_from_redis_rdb_file(std::string file_name) std::string keystr = jj_key; std::string item_name = keystr; size_t pos = keystr.find_first_of(":"); + if (pos != std::string::npos) { if(ASIC_STATE_TABLE != keystr.substr(0, pos)) // filter out non ASIC_STATE @@ -566,8 +574,8 @@ int dump_from_redis_rdb_file(std::string file_name) input_file.close(); std::cerr << "JSON file:" << file_name << " is invalid." << std::endl; std::cerr << "JSON parsing error: " << ex.what() << std::endl; - SWSS_LOG_ERROR("JSON file %s is invalid\n", file_name); - SWSS_LOG_ERROR("JSON parsing error: %s", ex.what()); + SWSS_LOG_ERROR("JSON file %s is invalid.", file_name.c_str()); + SWSS_LOG_ERROR("JSON parsing error: %s.", ex.what()); } return SAI_STATUS_FAILURE; } @@ -590,6 +598,7 @@ int main(int argc, char **argv) { return EXIT_FAILURE; } + dump_from_redis_rdb_file(g_cmdOptions.rdbFile); return EXIT_SUCCESS; } From fc2b38bf32c70cdd50e3504345ff4d7778a91b0c Mon Sep 17 00:00:00 2001 From: jumao Date: Wed, 6 Sep 2023 11:26:17 -0400 Subject: [PATCH 07/13] =?UTF-8?q?=20=20=20=20*=20[saidump]=20=20=20=20=20?= =?UTF-8?q?=E2=80=A2=20=20=20=20=20=20=20Saidump=20for=20DNX-SAI=20https:/?= =?UTF-8?q?/github.com/sonic-net/sonic-buildimage/issues/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- saidump/saidump.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index cf99f8dc0..865bd2888 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -478,7 +478,7 @@ int dump_from_redis_rdb_file(string file_name) ifstream input_file(file_name); if (!input_file.is_open()) { - SWSS_LOG_NOTICE("The file %s is not exsisted for dumping from redis dump.rdb file", file_name); + SWSS_LOG_NOTICE("The file %s does not exist for dumping from redis dump.rdb file.", file_name); return SAI_STATUS_FAILURE; } From 7b947b171ec609ca38620af469748c48b2f6e3fd Mon Sep 17 00:00:00 2001 From: jumao Date: Wed, 6 Sep 2023 11:34:57 -0400 Subject: [PATCH 08/13] =?UTF-8?q?=20=20=20=20*=20[saidump]=20=20=20=20=20?= =?UTF-8?q?=E2=80=A2=20=20=20=20=20=20=20Saidump=20for=20DNX-SAI=20https:/?= =?UTF-8?q?/github.com/sonic-net/sonic-buildimage/issues/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- saidump/saidump.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index 865bd2888..57c38d4fd 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -70,7 +70,7 @@ CmdOptions handleCmdLine(int argc, char **argv) { { "dumpGraph", no_argument, 0, 'g' }, { "tempView", no_argument, 0, 't' }, - { "rdb", required_argument, 0, 'r' }, + { "rdb", required_argument, 0, 'r' }, { "help", no_argument, 0, 'h' }, { 0, 0, 0, 0 } }; @@ -419,7 +419,7 @@ void dumpGraph(const TableDump& td) } /* - preprocess the input json file to make make sure it's a valid json file. + preprocess the input json file to make sure it's a valid json file. */ int pre_process_file(string file_name) { From dbeac2721ae305beb0dda2c57e7a58f5974ad978 Mon Sep 17 00:00:00 2001 From: jumao Date: Wed, 6 Sep 2023 16:54:23 -0400 Subject: [PATCH 09/13] =?UTF-8?q?=20=20=20=20=20=20=20=20*=20[saidump]=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=E2=80=A2=20=20=20=20=20=20=20Saidump?= =?UTF-8?q?=20for=20DNX-SAI=20https://github.com/sonic-net/sonic-buildimag?= =?UTF-8?q?e/issues/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- saidump/saidump.cpp | 80 +++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index 57c38d4fd..fa8ec2680 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -21,18 +21,17 @@ extern "C" { // TODO split to multiple cpp using namespace swss; -using namespace std; using json = nlohmann::json; // 100 MB -const int64_t RDB_FILE_MAX_SIZE = 1024 * 1024 * 100; +constexpr int64_t RDB_FILE_MAX_SIZE = 1024 * 1024 * 100; struct CmdOptions { bool skipAttributes; bool dumpTempView; bool dumpGraph; - string rdb_file; + std::string rdbFile; }; CmdOptions g_cmdOptions; @@ -97,8 +96,8 @@ CmdOptions handleCmdLine(int argc, char **argv) break; case 'r': - SWSS_LOG_WARN("Dumping from dump.rdb"); - options.rdb_file = string(optarg); + SWSS_LOG_NOTICE("Dumping from dump.rdb"); + options.rdbFile = std::string(optarg); break; case 'h': @@ -421,17 +420,17 @@ void dumpGraph(const TableDump& td) /* preprocess the input json file to make sure it's a valid json file. */ -int pre_process_file(string file_name) +static int preProcessFile(std::string file_name) { - ifstream input_file(file_name); + std::ifstream input_file(file_name); if (!input_file.is_open()) { - cerr << "Failed to open the input file." << endl; + std::cerr << "Failed to open the input file." << std::endl; return SAI_STATUS_FAILURE; } - input_file.seekg(0, ios::end); // Move to the end of the file + input_file.seekg(0, std::ios::end); // Move to the end of the file int64_t file_size = input_file.tellg(); // Get the current position if (file_size >= RDB_FILE_MAX_SIZE) @@ -442,16 +441,16 @@ int pre_process_file(string file_name) input_file.close(); return SAI_STATUS_FAILURE; } - input_file.seekg(0, ios::beg); // Move to the begin of the file + input_file.seekg(0, std::ios::beg); // Move to the begin of the file // Read the content of the input file into a string - string content((istreambuf_iterator(input_file)), - istreambuf_iterator()); + std::string content((std::istreambuf_iterator(input_file)), + std::istreambuf_iterator()); input_file.close(); - content = regex_replace(content, regex("\\},\\{\\r"), ","); + content = regex_replace(content, std::regex("\\},\\{\\r"), ","); //erase the 1st and last char. if (content.length() >= 2 && content[0] == '[' && content[content.length()-1] == ']') { @@ -459,11 +458,11 @@ int pre_process_file(string file_name) content.erase(content.length() - 1); } - ofstream outputFile(file_name); + std::ofstream outputFile(file_name); if (!outputFile.is_open()) { - cerr << "Failed to open the output file." << endl; + std::cerr << "Failed to open the output file." << std::endl; return SAI_STATUS_FAILURE; } @@ -473,16 +472,16 @@ int pre_process_file(string file_name) return SAI_STATUS_SUCCESS; } -int dump_from_redis_rdb_file(string file_name) +int dump_from_redis_rdb_file(std::string file_name) { - ifstream input_file(file_name); + std::ifstream input_file(file_name); if (!input_file.is_open()) { SWSS_LOG_NOTICE("The file %s does not exist for dumping from redis dump.rdb file.", file_name); return SAI_STATUS_FAILURE; } - input_file.seekg(0, ios::end); // Move to the end of the file + input_file.seekg(0, std::ios::end); // Move to the end of the file int64_t file_size = input_file.tellg(); // Get the current position if (file_size >= RDB_FILE_MAX_SIZE) { @@ -492,7 +491,7 @@ int dump_from_redis_rdb_file(string file_name) input_file.close(); return SAI_STATUS_FAILURE; } - input_file.seekg(0, ios::beg); // Move to the begin of the file + input_file.seekg(0, std::ios::beg); // Move to the begin of the file try { @@ -507,17 +506,17 @@ int dump_from_redis_rdb_file(string file_name) { json jj_key = it.key(); - string keystr = jj_key; - string item_name = keystr; + std::string keystr = jj_key; + std::string item_name = keystr; size_t pos = keystr.find_first_of(":"); - if (pos != string::npos) + if (pos != std::string::npos) { - if("ASIC_STATE" != keystr.substr(0, pos)) // filter out non ASIC_STATE + if(ASIC_STATE_TABLE != keystr.substr(0, pos)) // filter out non ASIC_STATE { continue; } item_name = keystr.substr(pos + 1); - if (item_name.find(":") != string::npos) + if (item_name.find(":") != std::string::npos) { item_name.replace(item_name.find_first_of(":"), 1, " "); } @@ -527,16 +526,17 @@ int dump_from_redis_rdb_file(string file_name) continue; } - cout << item_name << " " << endl; + std::cout << item_name << " \r\n"; json jj = it.value(); - if (!(*it).is_object()) + if (!it->is_object()) { continue; } TableMap map; + for (json::iterator itt = jj.begin(); itt != jj.end(); ++itt) { if (itt.key() == "NULL") @@ -548,24 +548,26 @@ int dump_from_redis_rdb_file(string file_name) size_t indent = 4; size_t max_len = get_max_attr_len(map); - string str_indent = pad_string("", indent); + std::string str_indent = pad_string("", indent); for (const auto&field: map) { - stringstream ss; + std::stringstream ss; ss << str_indent << pad_string(field.first, max_len) << " : "; ss << field.second; - cout << ss.str() << std::endl; + std::cout << ss.str() << "\r\n"; } - cout << endl; + std::cout << "\r\n"; } - return SAI_STATUS_SUCCESS; - } - catch (exception &ex) + return SAI_STATUS_SUCCESS; + } + catch (std::exception &ex) { input_file.close(); - cerr << "JSON file:" << file_name << " is invalid." << endl; - cerr << "JSON parsing error: " << ex.what() << endl; + std::cerr << "JSON file:" << file_name << " is invalid." << std::endl; + std::cerr << "JSON parsing error: " << ex.what() << std::endl; + SWSS_LOG_ERROR("JSON file %s is invalid\n", file_name); + SWSS_LOG_ERROR("JSON parsing error: %s", ex.what()); } return SAI_STATUS_FAILURE; } @@ -582,13 +584,13 @@ int main(int argc, char **argv) g_cmdOptions = handleCmdLine(argc, argv); - if (g_cmdOptions.rdb_file.size() > 0) + if (g_cmdOptions.rdbFile.size() > 0) { - if (SAI_STATUS_FAILURE == pre_process_file(g_cmdOptions.rdb_file)) + if (SAI_STATUS_FAILURE == preProcessFile(g_cmdOptions.rdbFile)) { return EXIT_FAILURE; - } - dump_from_redis_rdb_file(g_cmdOptions.rdb_file); + } + dump_from_redis_rdb_file(g_cmdOptions.rdbFile); return EXIT_SUCCESS; } From 4beed6115064f76db181b44752e624ae09fa1179 Mon Sep 17 00:00:00 2001 From: jumao Date: Wed, 6 Sep 2023 17:44:36 -0400 Subject: [PATCH 10/13] =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20*=20[?= =?UTF-8?q?saidump]=20=20=20=20=20=20=20=20=20=20=20=20=20=E2=80=A2=20=20?= =?UTF-8?q?=20=20=20=20=20Saidump=20for=20DNX-SAI=20https://github.com/son?= =?UTF-8?q?ic-net/sonic-buildimage/issues/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- saidump/saidump.cpp | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index fa8ec2680..6524c4375 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -426,7 +426,8 @@ static int preProcessFile(std::string file_name) if (!input_file.is_open()) { - std::cerr << "Failed to open the input file." << std::endl; + std::cerr << "Failed to open the input file " << file_name << std::endl; + SWSS_LOG_ERROR("Failed to open the input file %s.", file_name.c_str()); return SAI_STATUS_FAILURE; } @@ -435,25 +436,25 @@ static int preProcessFile(std::string file_name) if (file_size >= RDB_FILE_MAX_SIZE) { - SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.\n", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); - + std::cerr << "Get " << file_name.c_str() << "'s size failure or its size " << file_size << " >= "<< RDB_FILE_MAX_SIZE / 1024 / 1024<<" MB." << std::endl; + SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); // Close the input file input_file.close(); return SAI_STATUS_FAILURE; } + input_file.seekg(0, std::ios::beg); // Move to the begin of the file // Read the content of the input file into a string std::string content((std::istreambuf_iterator(input_file)), std::istreambuf_iterator()); - - input_file.close(); - content = regex_replace(content, std::regex("\\},\\{\\r"), ","); + content = regex_replace(content, std::regex("\\},\\{\\r"), ","); //erase the 1st and last char. - if (content.length() >= 2 && content[0] == '[' && content[content.length()-1] == ']') { + if (content.length() >= 2 && content[0] == '[' && content[content.length()-1] == ']') + { content.erase(0, 1); content.erase(content.length() - 1); } @@ -462,7 +463,8 @@ static int preProcessFile(std::string file_name) if (!outputFile.is_open()) { - std::cerr << "Failed to open the output file." << std::endl; + std::cerr << "Failed to open the output file " << file_name << std::endl; + SWSS_LOG_ERROR("Failed to open the output file %s.", file_name.c_str()); return SAI_STATUS_FAILURE; } @@ -475,22 +477,27 @@ static int preProcessFile(std::string file_name) int dump_from_redis_rdb_file(std::string file_name) { std::ifstream input_file(file_name); + if (!input_file.is_open()) { - SWSS_LOG_NOTICE("The file %s does not exist for dumping from redis dump.rdb file.", file_name); + std::cerr << "Failed to open the input file " << file_name << std::endl; + SWSS_LOG_ERROR("The file %s does not exist for dumping from redis dump.rdb file.", file_name.c_str()); return SAI_STATUS_FAILURE; } input_file.seekg(0, std::ios::end); // Move to the end of the file int64_t file_size = input_file.tellg(); // Get the current position + if (file_size >= RDB_FILE_MAX_SIZE) { - SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.\n", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); + std::cerr << "Get " << file_name.c_str() << "'s size failure or its size " << file_size << " >= "<< RDB_FILE_MAX_SIZE / 1024 / 1024<<" MB." << std::endl; + SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); // Close the input file input_file.close(); return SAI_STATUS_FAILURE; } + input_file.seekg(0, std::ios::beg); // Move to the begin of the file try @@ -500,7 +507,7 @@ int dump_from_redis_rdb_file(std::string file_name) input_file >> jsonData; input_file.close(); - SWSS_LOG_DEBUG("%s\n","JSON file is valid.\n"); + SWSS_LOG_DEBUG("JSON file is valid."); for (json::iterator it = jsonData.begin(); it != jsonData.end(); ++it) { @@ -509,6 +516,7 @@ int dump_from_redis_rdb_file(std::string file_name) std::string keystr = jj_key; std::string item_name = keystr; size_t pos = keystr.find_first_of(":"); + if (pos != std::string::npos) { if(ASIC_STATE_TABLE != keystr.substr(0, pos)) // filter out non ASIC_STATE @@ -566,8 +574,8 @@ int dump_from_redis_rdb_file(std::string file_name) input_file.close(); std::cerr << "JSON file:" << file_name << " is invalid." << std::endl; std::cerr << "JSON parsing error: " << ex.what() << std::endl; - SWSS_LOG_ERROR("JSON file %s is invalid\n", file_name); - SWSS_LOG_ERROR("JSON parsing error: %s", ex.what()); + SWSS_LOG_ERROR("JSON file %s is invalid.", file_name.c_str()); + SWSS_LOG_ERROR("JSON parsing error: %s.", ex.what()); } return SAI_STATUS_FAILURE; } @@ -590,6 +598,7 @@ int main(int argc, char **argv) { return EXIT_FAILURE; } + dump_from_redis_rdb_file(g_cmdOptions.rdbFile); return EXIT_SUCCESS; } From 66a477db28969fe249db5b7c9c680231bd273cbd Mon Sep 17 00:00:00 2001 From: jumao Date: Thu, 7 Sep 2023 17:43:08 -0400 Subject: [PATCH 11/13] =?UTF-8?q?*=20[saidump]=20=E2=80=A2=20=20=20=20=20?= =?UTF-8?q?=20=20Saidump=20for=20DNX-SAI=20https://github.com/sonic-net/so?= =?UTF-8?q?nic-buildimage/issues/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json --- saidump/saidump.cpp | 106 ++++++++++++++++++-------------------------- 1 file changed, 44 insertions(+), 62 deletions(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index 6524c4375..72f997723 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -13,7 +13,6 @@ extern "C" { #include "swss/table.h" #include "meta/sai_serialize.h" #include "sairediscommon.h" -#include "swss/json.h" #include "swss/json.hpp" #include @@ -34,8 +33,8 @@ struct CmdOptions std::string rdbFile; }; -CmdOptions g_cmdOptions; -std::map g_oid_map; +static CmdOptions g_cmdOptions; +static std::map g_oid_map; void printUsage() { @@ -47,7 +46,7 @@ void printUsage() std::cout << " -g --dumpGraph:" << std::endl; std::cout << " Dump current graph" << std::endl; std::cout << " -r --rdb:" << std::endl; - std::cout << " Dump by parsing the Redis persistence file dump.rdb that is generated by SAVE command." << std::endl; + std::cout << " Dump by parsing the Redis persistence file dump.rdb that is generated by SAVE command" << std::endl; std::cout << " -h --help:" << std::endl; std::cout << " Print out this message" << std::endl; } @@ -96,7 +95,7 @@ CmdOptions handleCmdLine(int argc, char **argv) break; case 'r': - SWSS_LOG_NOTICE("Dumping from dump.rdb"); + SWSS_LOG_NOTICE("Dumping from %s", optarg); options.rdbFile = std::string(optarg); break; @@ -417,17 +416,20 @@ void dumpGraph(const TableDump& td) std::cout << "}" << std::endl; } -/* - preprocess the input json file to make sure it's a valid json file. -*/ -static int preProcessFile(std::string file_name) +#define SWSS_LOG_ERROR_AND_STDERR(format, ...) { fprintf(stderr, format"\n", ##__VA_ARGS__); SWSS_LOG_ERROR(format, ##__VA_ARGS__); } + +/** + * @brief Preprocess the input json file to make sure it's a valid json file. + */ +static sai_status_t preProcessFile(std::string file_name) { + SWSS_LOG_ENTER(); + std::ifstream input_file(file_name); if (!input_file.is_open()) { - std::cerr << "Failed to open the input file " << file_name << std::endl; - SWSS_LOG_ERROR("Failed to open the input file %s.", file_name.c_str()); + SWSS_LOG_ERROR_AND_STDERR("Failed to open the input file %s.", file_name.c_str()); return SAI_STATUS_FAILURE; } @@ -435,77 +437,57 @@ static int preProcessFile(std::string file_name) int64_t file_size = input_file.tellg(); // Get the current position if (file_size >= RDB_FILE_MAX_SIZE) - { - std::cerr << "Get " << file_name.c_str() << "'s size failure or its size " << file_size << " >= "<< RDB_FILE_MAX_SIZE / 1024 / 1024<<" MB." << std::endl; - SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); - // Close the input file - input_file.close(); + { + SWSS_LOG_ERROR_AND_STDERR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); return SAI_STATUS_FAILURE; } - input_file.seekg(0, std::ios::beg); // Move to the begin of the file + input_file.seekg(0); // Move to the begin of the file // Read the content of the input file into a string std::string content((std::istreambuf_iterator(input_file)), std::istreambuf_iterator()); - input_file.close(); content = regex_replace(content, std::regex("\\},\\{\\r"), ","); - //erase the 1st and last char. - if (content.length() >= 2 && content[0] == '[' && content[content.length()-1] == ']') - { - content.erase(0, 1); - content.erase(content.length() - 1); - } - std::ofstream outputFile(file_name); if (!outputFile.is_open()) { - std::cerr << "Failed to open the output file " << file_name << std::endl; - SWSS_LOG_ERROR("Failed to open the output file %s.", file_name.c_str()); + SWSS_LOG_ERROR_AND_STDERR("Failed to open the output file %s.", file_name.c_str()); return SAI_STATUS_FAILURE; } - outputFile << content; - outputFile.close(); + //Romove the 1st and last char to make sure its format is same as previous saidump's output + if (content.size() >= 2 && content[0] == '[' && content[content.length()-1] == ']') + { + outputFile << content.substr(1, content.size()-2); + } + else + { + outputFile << content; + } return SAI_STATUS_SUCCESS; } -int dump_from_redis_rdb_file(std::string file_name) +static sai_status_t dumpFromRedisRdbFile(std::string file_name) { + SWSS_LOG_ENTER(); + std::ifstream input_file(file_name); if (!input_file.is_open()) { - std::cerr << "Failed to open the input file " << file_name << std::endl; - SWSS_LOG_ERROR("The file %s does not exist for dumping from redis dump.rdb file.", file_name.c_str()); + SWSS_LOG_ERROR_AND_STDERR("The file %s does not exist for dumping from redis dump.rdb file.", file_name.c_str()); return SAI_STATUS_FAILURE; } - input_file.seekg(0, std::ios::end); // Move to the end of the file - int64_t file_size = input_file.tellg(); // Get the current position - - if (file_size >= RDB_FILE_MAX_SIZE) - { - std::cerr << "Get " << file_name.c_str() << "'s size failure or its size " << file_size << " >= "<< RDB_FILE_MAX_SIZE / 1024 / 1024<<" MB." << std::endl; - SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); - - // Close the input file - input_file.close(); - return SAI_STATUS_FAILURE; - } - - input_file.seekg(0, std::ios::beg); // Move to the begin of the file - try { // Parse the JSON data from the file (validation) nlohmann::json jsonData; input_file >> jsonData; - input_file.close(); SWSS_LOG_DEBUG("JSON file is valid."); @@ -523,7 +505,9 @@ int dump_from_redis_rdb_file(std::string file_name) { continue; } + item_name = keystr.substr(pos + 1); + if (item_name.find(":") != std::string::npos) { item_name.replace(item_name.find_first_of(":"), 1, " "); @@ -534,7 +518,7 @@ int dump_from_redis_rdb_file(std::string file_name) continue; } - std::cout << item_name << " \r\n"; + std::cout << item_name << " " << std::endl; json jj = it.value(); @@ -551,6 +535,7 @@ int dump_from_redis_rdb_file(std::string file_name) { continue; } + map[itt.key()] = itt.value(); } @@ -560,23 +545,21 @@ int dump_from_redis_rdb_file(std::string file_name) for (const auto&field: map) { - std::stringstream ss; - ss << str_indent << pad_string(field.first, max_len) << " : "; - ss << field.second; - std::cout << ss.str() << "\r\n"; + std::cout << str_indent << pad_string(field.first, max_len) << " : "; + std::cout << field.second << std::endl; } - std::cout << "\r\n"; + + std::cout << std::endl; } + return SAI_STATUS_SUCCESS; } catch (std::exception &ex) - { - input_file.close(); - std::cerr << "JSON file:" << file_name << " is invalid." << std::endl; - std::cerr << "JSON parsing error: " << ex.what() << std::endl; - SWSS_LOG_ERROR("JSON file %s is invalid.", file_name.c_str()); - SWSS_LOG_ERROR("JSON parsing error: %s.", ex.what()); + { + SWSS_LOG_ERROR_AND_STDERR("JSON file %s is invalid.", file_name.c_str()); + SWSS_LOG_ERROR_AND_STDERR("JSON parsing error: %s.", ex.what()); } + return SAI_STATUS_FAILURE; } @@ -599,8 +582,7 @@ int main(int argc, char **argv) return EXIT_FAILURE; } - dump_from_redis_rdb_file(g_cmdOptions.rdbFile); - return EXIT_SUCCESS; + return dumpFromRedisRdbFile(g_cmdOptions.rdbFile); } swss::DBConnector db("ASIC_DB", 0); From 363589acccae57b6b9b0171251bbefacc3dce6f3 Mon Sep 17 00:00:00 2001 From: jumao Date: Fri, 8 Sep 2023 12:09:48 -0400 Subject: [PATCH 12/13] =?UTF-8?q?*=20[saidump]=20=E2=80=A2=09Saidump=20for?= =?UTF-8?q?=20DNX-SAI=20https://github.com/sonic-net/sonic-buildimage/issu?= =?UTF-8?q?es/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Add a new script file: files/scripts/saidump.sh, to do the below steps For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files sudo python /usr/local/bin/rdb --command json /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh --- saidump/saidump.cpp | 75 +++++++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 23 deletions(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index 72f997723..f0a9bc282 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -5,6 +5,7 @@ #include #include #include +#include extern "C" { #include @@ -22,17 +23,19 @@ extern "C" { using namespace swss; using json = nlohmann::json; -// 100 MB -constexpr int64_t RDB_FILE_MAX_SIZE = 1024 * 1024 * 100; +// Default value: 100 MB +constexpr int64_t RDB_JSON_MAX_SIZE = 1024 * 1024 * 100; struct CmdOptions { bool skipAttributes; bool dumpTempView; bool dumpGraph; - std::string rdbFile; + std::string rdbJsonFile; + uint64_t rdbJSonSizeLimit; }; + static CmdOptions g_cmdOptions; static std::map g_oid_map; @@ -40,13 +43,15 @@ void printUsage() { SWSS_LOG_ENTER(); - std::cout << "Usage: saidump [-t] [-g] [-r] [-h]" << std::endl; + std::cout << "Usage: saidump [-t] [-g] [-r] [-m] [-h]" << std::endl; std::cout << " -t --tempView:" << std::endl; std::cout << " Dump temp view" << std::endl; std::cout << " -g --dumpGraph:" << std::endl; std::cout << " Dump current graph" << std::endl; std::cout << " -r --rdb:" << std::endl; - std::cout << " Dump by parsing the Redis persistence file dump.rdb that is generated by SAVE command" << std::endl; + std::cout << " Dump by parsing the RDB JSON file, which is created by rdbtools based on Redis dump.rdb that is generated by Redis SAVE command" << std::endl; + std::cout << " -m --max:" << std::endl; + std::cout << " Config the the RDB JSON file's max size in MB, which is optional with default value 100MB" << std::endl; std::cout << " -h --help:" << std::endl; std::cout << " Print out this message" << std::endl; } @@ -59,8 +64,10 @@ CmdOptions handleCmdLine(int argc, char **argv) options.dumpTempView = false; options.dumpGraph = false; + options.rdbJSonSizeLimit = RDB_JSON_MAX_SIZE; - const char* const optstring = "gtr:h"; + const char* const optstring = "gtr:m:h"; + uint64_t result = 0; while (true) { @@ -69,6 +76,7 @@ CmdOptions handleCmdLine(int argc, char **argv) { "dumpGraph", no_argument, 0, 'g' }, { "tempView", no_argument, 0, 't' }, { "rdb", required_argument, 0, 'r' }, + { "max", required_argument, 0, 'm' }, { "help", no_argument, 0, 'h' }, { 0, 0, 0, 0 } }; @@ -96,7 +104,29 @@ CmdOptions handleCmdLine(int argc, char **argv) case 'r': SWSS_LOG_NOTICE("Dumping from %s", optarg); - options.rdbFile = std::string(optarg); + options.rdbJsonFile = std::string(optarg); + break; + + case 'm': + if(!regex_match(optarg, std::regex(R"([+]?\d+)"))) //only positive numeric chars are valid, such as 3984, +3232, etc. + { + SWSS_LOG_WARN("invalid option -m %s", optarg); + printUsage(); + exit(EXIT_SUCCESS); + } + + result = strtoull(optarg, NULL, 0); + + if((errno == ERANGE && result == ULLONG_MAX) || result == 0 || result >= INT_MAX) + { + SWSS_LOG_WARN("invalid option -m %s", optarg); + printUsage(); + exit(EXIT_SUCCESS); + } + + options.rdbJSonSizeLimit = result * 1024 * 1024; + SWSS_LOG_NOTICE("Configure the RDB JSON MAX size to %llu MB", options.rdbJSonSizeLimit / 1024 / 1024); + break; case 'h': @@ -419,9 +449,9 @@ void dumpGraph(const TableDump& td) #define SWSS_LOG_ERROR_AND_STDERR(format, ...) { fprintf(stderr, format"\n", ##__VA_ARGS__); SWSS_LOG_ERROR(format, ##__VA_ARGS__); } /** - * @brief Preprocess the input json file to make sure it's a valid json file. + * @brief Preprocess the input JSON file to make sure it's a valid JSON file for Nlohmann JSON library. */ -static sai_status_t preProcessFile(std::string file_name) +static sai_status_t preProcessFile(const std::string file_name) { SWSS_LOG_ENTER(); @@ -434,11 +464,11 @@ static sai_status_t preProcessFile(std::string file_name) } input_file.seekg(0, std::ios::end); // Move to the end of the file - int64_t file_size = input_file.tellg(); // Get the current position + uint64_t file_size = input_file.tellg(); // Get the current position - if (file_size >= RDB_FILE_MAX_SIZE) + if (file_size >= g_cmdOptions.rdbJSonSizeLimit) { - SWSS_LOG_ERROR_AND_STDERR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024); + SWSS_LOG_ERROR_AND_STDERR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, g_cmdOptions.rdbJSonSizeLimit / 1024 / 1024); return SAI_STATUS_FAILURE; } @@ -471,7 +501,7 @@ static sai_status_t preProcessFile(std::string file_name) return SAI_STATUS_SUCCESS; } -static sai_status_t dumpFromRedisRdbFile(std::string file_name) +static sai_status_t dumpFromRedisRdbJson(const std::string file_name) { SWSS_LOG_ENTER(); @@ -479,7 +509,7 @@ static sai_status_t dumpFromRedisRdbFile(std::string file_name) if (!input_file.is_open()) { - SWSS_LOG_ERROR_AND_STDERR("The file %s does not exist for dumping from redis dump.rdb file.", file_name.c_str()); + SWSS_LOG_ERROR_AND_STDERR("The file %s does not exist for dumping from Redis RDB JSON file.", file_name.c_str()); return SAI_STATUS_FAILURE; } @@ -531,17 +561,15 @@ static sai_status_t dumpFromRedisRdbFile(std::string file_name) for (json::iterator itt = jj.begin(); itt != jj.end(); ++itt) { - if (itt.key() == "NULL") + if (itt.key() != "NULL") { - continue; + map[itt.key()] = itt.value(); } - - map[itt.key()] = itt.value(); } - size_t indent = 4; + constexpr size_t LINE_IDENT = 4; size_t max_len = get_max_attr_len(map); - std::string str_indent = pad_string("", indent); + std::string str_indent = pad_string("", LINE_IDENT); for (const auto&field: map) { @@ -575,14 +603,15 @@ int main(int argc, char **argv) g_cmdOptions = handleCmdLine(argc, argv); - if (g_cmdOptions.rdbFile.size() > 0) + + if (g_cmdOptions.rdbJsonFile.size() > 0) { - if (SAI_STATUS_FAILURE == preProcessFile(g_cmdOptions.rdbFile)) + if (SAI_STATUS_FAILURE == preProcessFile(g_cmdOptions.rdbJsonFile)) { return EXIT_FAILURE; } - return dumpFromRedisRdbFile(g_cmdOptions.rdbFile); + return dumpFromRedisRdbJson(g_cmdOptions.rdbJsonFile); } swss::DBConnector db("ASIC_DB", 0); From c710885d5517ff72ad2a65f15dd1f21176bbe957 Mon Sep 17 00:00:00 2001 From: jumao Date: Wed, 13 Sep 2023 11:57:49 -0400 Subject: [PATCH 13/13] =?UTF-8?q?*=20[saidump]=20=E2=80=A2=20Saidump=20for?= =?UTF-8?q?=20DNX-SAI=20https://github.com/sonic-net/sonic-buildimage/issu?= =?UTF-8?q?es/13561?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solution and modification: To use the Redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it. (1) Updated platform/broadcom/docker-syncd-brcm-dnx/Dockerfile.j2, install Python library rdbtools into the syncd containter. (2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format. (3) Updated sonic-buildimage/build_debian.sh, to add a new script file: files/scripts/saidump.sh into the host. This shell file does the below steps: For each ASIC0, such as ASIC0, 1. Save the Redis data. sudo sonic-db-cli -n asic$1 SAVE > /dev/null 2. Move dump files to /var/run/redisX/ docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/" 3. Run rdb command to convert the dump files into JSON files docker exec syncd$1 sh -c "rdb --command json /var/run/redis$1/dump.rdb | tee /var/run/redis$1/dump.json > /dev/null" 4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output. docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json -m 100" 5. clear sudo rm -f /var/run/redis$1/dump.rdb sudo rm -f /var/run/redis$1/dump.json (4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, to check the asic db size and if it is larger than xxx entries, then do with REDIS SAVE, otherwise, to do with old method: looping through each entry of Redis DB. --- saidump/saidump.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/saidump/saidump.cpp b/saidump/saidump.cpp index f0a9bc282..4677f48aa 100644 --- a/saidump/saidump.cpp +++ b/saidump/saidump.cpp @@ -14,7 +14,7 @@ extern "C" { #include "swss/table.h" #include "meta/sai_serialize.h" #include "sairediscommon.h" -#include "swss/json.hpp" +#include #include @@ -125,7 +125,7 @@ CmdOptions handleCmdLine(int argc, char **argv) } options.rdbJSonSizeLimit = result * 1024 * 1024; - SWSS_LOG_NOTICE("Configure the RDB JSON MAX size to %llu MB", options.rdbJSonSizeLimit / 1024 / 1024); + SWSS_LOG_NOTICE("Configure the RDB JSON MAX size to %llu MB", options.rdbJSonSizeLimit / 1024 / 1024); break; @@ -449,7 +449,7 @@ void dumpGraph(const TableDump& td) #define SWSS_LOG_ERROR_AND_STDERR(format, ...) { fprintf(stderr, format"\n", ##__VA_ARGS__); SWSS_LOG_ERROR(format, ##__VA_ARGS__); } /** - * @brief Preprocess the input JSON file to make sure it's a valid JSON file for Nlohmann JSON library. + * @brief Process the input JSON file to make sure it's a valid JSON file for the JSON library. */ static sai_status_t preProcessFile(const std::string file_name) { @@ -465,10 +465,11 @@ static sai_status_t preProcessFile(const std::string file_name) input_file.seekg(0, std::ios::end); // Move to the end of the file uint64_t file_size = input_file.tellg(); // Get the current position + SWSS_LOG_NOTICE("Get %s's size %" PRIu64 " Bytes, limit: %" PRIu64 " MB.", file_name.c_str(), file_size, g_cmdOptions.rdbJSonSizeLimit / 1024 / 1024); if (file_size >= g_cmdOptions.rdbJSonSizeLimit) - { - SWSS_LOG_ERROR_AND_STDERR("Get %s's size failure or its size %ld >= %ld MB.", file_name.c_str(), file_size, g_cmdOptions.rdbJSonSizeLimit / 1024 / 1024); + { + SWSS_LOG_ERROR_AND_STDERR("Get %s's size failure or its size %" PRIu64 " >= %" PRIu64 " MB.", file_name.c_str(), file_size, g_cmdOptions.rdbJSonSizeLimit / 1024 / 1024); return SAI_STATUS_FAILURE; } @@ -488,7 +489,7 @@ static sai_status_t preProcessFile(const std::string file_name) return SAI_STATUS_FAILURE; } - //Romove the 1st and last char to make sure its format is same as previous saidump's output + //Remove the 1st and last char to make sure its format is same as previous output if (content.size() >= 2 && content[0] == '[' && content[content.length()-1] == ']') { outputFile << content.substr(1, content.size()-2); @@ -583,11 +584,11 @@ static sai_status_t dumpFromRedisRdbJson(const std::string file_name) return SAI_STATUS_SUCCESS; } catch (std::exception &ex) - { + { SWSS_LOG_ERROR_AND_STDERR("JSON file %s is invalid.", file_name.c_str()); SWSS_LOG_ERROR_AND_STDERR("JSON parsing error: %s.", ex.what()); } - + return SAI_STATUS_FAILURE; }