From 7a44b510abb89b89eafaef1cc7eba5b56503899f Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Fri, 25 Dec 2020 23:39:14 +0000 Subject: [PATCH 01/25] Implement Pubsub listen_message() and punsubscribe() --- common/dbconnector.cpp | 7 +++++++ common/dbconnector.h | 2 ++ common/pubsub.cpp | 25 +++++++++++++++++++++++-- common/pubsub.h | 4 +++- common/redisselect.cpp | 13 +++++++++++++ common/redisselect.h | 3 +++ 6 files changed, 51 insertions(+), 3 deletions(-) diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index 635fa9532..e881b5b84 100644 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -791,6 +791,13 @@ void DBConnector::psubscribe(const std::string &pattern) RedisReply r(this, s, REDIS_REPLY_ARRAY); } +void DBConnector::punsubscribe(const std::string &pattern) +{ + std::string s("PUNSUBSCRIBE "); + s += pattern; + RedisReply r(this, s, REDIS_REPLY_ARRAY); +} + int64_t DBConnector::publish(const string &channel, const string &message) { RedisCommand publish; diff --git a/common/dbconnector.h b/common/dbconnector.h index 0a6a79ac5..9628e0ec0 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -202,6 +202,8 @@ class DBConnector : public RedisContext void psubscribe(const std::string &pattern); + void punsubscribe(const std::string &pattern); + int64_t publish(const std::string &channel, const std::string &message); void config_set(const std::string &key, const std::string &value); diff --git a/common/pubsub.cpp b/common/pubsub.cpp index 0388a7d0b..02f5702f0 100644 --- a/common/pubsub.cpp +++ b/common/pubsub.cpp @@ -21,6 +21,12 @@ void PubSub::psubscribe(const std::string &pattern) m_select.addSelectable(this); } +void PubSub::punsubscribe(const std::string &pattern) +{ + RedisSelect::punsubscribe(pattern); + m_select.removeSelectable(this); +} + uint64_t PubSub::readData() { redisReply *reply = nullptr; @@ -71,7 +77,7 @@ bool PubSub::hasCachedData() return m_keyspace_event_buffer.size() > 1; } -map PubSub::get_message() +map PubSub::get_message(double timeout) { map ret; if (!m_subscribe) @@ -80,7 +86,7 @@ map PubSub::get_message() } Selectable *selected; - int rc = m_select.select(&selected, 0); + int rc = m_select.select(&selected, int(timeout)); switch (rc) { case Select::ERROR: @@ -111,6 +117,21 @@ map PubSub::get_message() return ret; } +// Note: it is not straightforward to implement redis-py PubSub.listen() directly in c++ +// due to the `yield` syntax, so we implement this function for blocking listen one message +std::map PubSub::listen_message() +{ + const double GET_MESSAGE_INTERVAL = 600.0; // in seconds + for (;;) + { + auto ret = get_message(GET_MESSAGE_INTERVAL); + if (!ret.empty()) + { + return ret; + } + } +} + shared_ptr PubSub::popEventBuffer() { if (m_keyspace_event_buffer.empty()) diff --git a/common/pubsub.h b/common/pubsub.h index 1ac433465..e2fbc5643 100644 --- a/common/pubsub.h +++ b/common/pubsub.h @@ -15,9 +15,11 @@ class PubSub : protected RedisSelect public: PubSub(DBConnector *other); - std::map get_message(); + std::map get_message(double timeout = 0.0); + std::map listen_message(); void psubscribe(const std::string &pattern); + void punsubscribe(const std::string &pattern); /* Read keyspace event from redis */ uint64_t readData() override; diff --git a/common/redisselect.cpp b/common/redisselect.cpp index 4827eebd9..77d2b5063 100644 --- a/common/redisselect.cpp +++ b/common/redisselect.cpp @@ -92,6 +92,19 @@ void RedisSelect::psubscribe(DBConnector* db, const std::string &channelName) m_subscribe->psubscribe(channelName); } +/* PUNSUBSCRIBE */ +void RedisSelect::punsubscribe(const std::string &channelName) +{ + /* + * Send PUNSUBSCRIBE #channel command on the + * non-blocking subscriber DBConnector + */ + if (m_subscribe) + { + m_subscribe->psubscribe(channelName); + } +} + void RedisSelect::setQueueLength(long long int queueLength) { m_queueLength = queueLength; diff --git a/common/redisselect.h b/common/redisselect.h index ea395200a..65bfe8d98 100644 --- a/common/redisselect.h +++ b/common/redisselect.h @@ -3,6 +3,7 @@ #include #include #include "selectable.h" +#include "dbconnector.h" namespace swss { @@ -27,6 +28,8 @@ class RedisSelect : public Selectable /* PSUBSCRIBE */ void psubscribe(DBConnector* db, const std::string &channelName); + /* PUNSUBSCRIBE */ + void punsubscribe(const std::string &channelName); void setQueueLength(long long int queueLength); From 8d22032a3f9bbf778f37cf4676e389103cd6d8d7 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 26 Dec 2020 04:03:32 +0000 Subject: [PATCH 02/25] Implement ConfigDBConnector::connect() --- common/Makefile.am | 1 + common/configdb.cpp | 48 +++++++++++++++++++++++++++++++++++++++++++++ common/configdb.h | 23 ++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 common/configdb.cpp create mode 100644 common/configdb.h diff --git a/common/Makefile.am b/common/Makefile.am index d77ea3109..1e82f97a6 100644 --- a/common/Makefile.am +++ b/common/Makefile.am @@ -30,6 +30,7 @@ endif libswsscommon_la_SOURCES = \ logger.cpp \ redisreply.cpp \ + configdb.cpp \ dbconnector.cpp \ dbinterface.cpp \ sonicv2connector.cpp \ diff --git a/common/configdb.cpp b/common/configdb.cpp new file mode 100644 index 000000000..6963180f5 --- /dev/null +++ b/common/configdb.cpp @@ -0,0 +1,48 @@ +#include "configdb.h" +#include "pubsub.h" + +using namespace std; +using namespace swss; + +ConfigDBConnector::ConfigDBConnector(bool use_unix_socket_path, const char *netns) + : SonicV2Connector(use_unix_socket_path, netns) +{ +} + +void ConfigDBConnector::connect(bool wait_for_init, bool retry_on) +{ + m_db_name = "CONFIG_DB"; + SonicV2Connector::connect(m_db_name, retry_on); + + if (wait_for_init) + { + auto client = get_redis_client(m_db_name); + auto pubsub = client.pubsub(); + auto initialized = client.get(INIT_INDICATOR); + if (initialized) + { + string pattern = "__keyspace@" + to_string(get_dbid(m_db_name)) + "__:" + INIT_INDICATOR; + pubsub->psubscribe(pattern); + for (;;) + { + auto item = pubsub->listen_message(); + if (item["type"] == "pmessage") + { + string channel = item["channel"]; + size_t pos = channel.find(':'); + string key = channel.substr(pos + 1); + if (key == INIT_INDICATOR) + { + initialized = client.get(INIT_INDICATOR); + if (initialized && !initialized->empty()) + { + break; + } + } + } + } + pubsub->punsubscribe(pattern); + } + } +} + diff --git a/common/configdb.h b/common/configdb.h new file mode 100644 index 000000000..99fb8827c --- /dev/null +++ b/common/configdb.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include "sonicv2connector.h" + +namespace swss { + +class ConfigDBConnector : public SonicV2Connector +{ +public: + ConfigDBConnector(bool use_unix_socket_path = false, const char *netns = ""); + + void connect(bool wait_for_init = true, bool retry_on = false); + +protected: + static constexpr const char *INIT_INDICATOR = "CONFIG_DB_INITIALIZED"; + static constexpr const char *TABLE_NAME_SEPARATOR = "|"; + static constexpr const char *KEY_SEPARATOR = "|"; + + std::string m_db_name; +}; + +} From ae54684fcb887dc0f634ec2e5543efda8df3ab28 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 26 Dec 2020 06:55:38 +0000 Subject: [PATCH 03/25] Move complex class constructor as explicit, and fix several mistaken copy constructor usage --- common/dbconnector.h | 2 +- common/dbinterface.cpp | 2 +- common/pubsub.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/dbconnector.h b/common/dbconnector.h index 9628e0ec0..f21a37271 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -120,7 +120,7 @@ class DBConnector : public RedisContext * Timeout - The time in milisecond until exception is been thrown. For * infinite wait, set this value to 0 */ - DBConnector(const DBConnector &other); + explicit DBConnector(const DBConnector &other); DBConnector(int dbId, const RedisContext &ctx); DBConnector(int dbId, const std::string &hostname, int port, unsigned int timeout); DBConnector(int dbId, const std::string &unixPath, unsigned int timeout); diff --git a/common/dbinterface.cpp b/common/dbinterface.cpp index 5b798cc6c..492ffef9c 100644 --- a/common/dbinterface.cpp +++ b/common/dbinterface.cpp @@ -326,7 +326,7 @@ void DBInterface::_onetime_connect(int dbId, const string& dbName) bool inserted = rc.second; if (inserted) { - auto redisClient = rc.first->second; + auto& redisClient = rc.first->second; redisClient.config_set("notify-keyspace-events", KEYSPACE_EVENTS); } } diff --git a/common/pubsub.h b/common/pubsub.h index e2fbc5643..46b414d71 100644 --- a/common/pubsub.h +++ b/common/pubsub.h @@ -13,7 +13,7 @@ namespace swss { class PubSub : protected RedisSelect { public: - PubSub(DBConnector *other); + explicit PubSub(DBConnector *other); std::map get_message(double timeout = 0.0); std::map listen_message(); From 651ea835d7180716542e0907fc3d6022fd4a5b35 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 26 Dec 2020 06:58:38 +0000 Subject: [PATCH 04/25] Implement set_entry() and get_entry() --- common/configdb.cpp | 49 ++++++++++++++++++++++++++++++++++++++++++++- common/configdb.h | 4 ++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/common/configdb.cpp b/common/configdb.cpp index 6963180f5..9ef1995c0 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -1,5 +1,8 @@ +#include +#include #include "configdb.h" #include "pubsub.h" +#include "converter.h" using namespace std; using namespace swss; @@ -16,7 +19,7 @@ void ConfigDBConnector::connect(bool wait_for_init, bool retry_on) if (wait_for_init) { - auto client = get_redis_client(m_db_name); + auto& client = get_redis_client(m_db_name); auto pubsub = client.pubsub(); auto initialized = client.get(INIT_INDICATOR); if (initialized) @@ -46,3 +49,47 @@ void ConfigDBConnector::connect(bool wait_for_init, bool retry_on) } } +// Write a table entry to config db. +// Remove extra fields in the db which are not in the data. +// Args: +// table: Table name. +// key: Key of table entry, or a tuple of keys if it is a multi-key table. +// data: Table row data in a form of dictionary {'column_key': 'value', ...}. +// Pass {} as data will delete the entry. +void ConfigDBConnector::set_entry(string table, string key, const unordered_map& data) +{ + auto& client = get_redis_client(m_db_name); + string _hash = to_upper(table) + TABLE_NAME_SEPARATOR + key; + if (data.empty()) + { + client.del(_hash); + } + else + { + auto original = get_entry(table, key); + client.hmset(_hash, data.begin(), data.end()); + for (auto& it: original) + { + auto& k = it.first; + bool found = data.find(k) == data.end(); + if (!found) + { + client.hdel(_hash, k); + } + } + } +} + +// Read a table entry from config db. +// Args: +// table: Table name. +// key: Key of table entry, or a tuple of keys if it is a multi-key table. +// Returns: +// Table row data in a form of dictionary {'column_key': 'value', ...} +// Empty dictionary if table does not exist or entry does not exist. +unordered_map ConfigDBConnector::get_entry(string table, string key) +{ + auto& client = get_redis_client(m_db_name); + string _hash = to_upper(table) + TABLE_NAME_SEPARATOR + key; + return client.hgetall(_hash); +} diff --git a/common/configdb.h b/common/configdb.h index 99fb8827c..fb34f9112 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "sonicv2connector.h" namespace swss { @@ -12,6 +13,9 @@ class ConfigDBConnector : public SonicV2Connector void connect(bool wait_for_init = true, bool retry_on = false); + void set_entry(std::string table, std::string key, const std::unordered_map& data); + std::unordered_map get_entry(std::string table, std::string key); + protected: static constexpr const char *INIT_INDICATOR = "CONFIG_DB_INITIALIZED"; static constexpr const char *TABLE_NAME_SEPARATOR = "|"; From 28728c10e06c7c0393e72f0bd02582358059af17 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 26 Dec 2020 19:22:11 +0000 Subject: [PATCH 05/25] Implement mod_entry() --- common/configdb.cpp | 21 +++++++++++++++++++++ common/configdb.h | 1 + 2 files changed, 22 insertions(+) diff --git a/common/configdb.cpp b/common/configdb.cpp index 9ef1995c0..78e465015 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -80,6 +80,27 @@ void ConfigDBConnector::set_entry(string table, string key, const unordered_map< } } +// Modify a table entry to config db. +// Args: +// table: Table name. +// key: Key of table entry, or a tuple of keys if it is a multi-key table. +// data: Table row data in a form of dictionary {'column_key': 'value', ...}. +// Pass {} as data will create an entry with no column if not already existed. +// Pass None as data will delete the entry. +void ConfigDBConnector::mod_entry(string table, string key, const unordered_map& data) +{ + auto& client = get_redis_client(m_db_name); + string _hash = to_upper(table) + TABLE_NAME_SEPARATOR + key; + if (data.empty()) + { + client.del(_hash); + } + else + { + client.hmset(_hash, data.begin(), data.end()); + } +} + // Read a table entry from config db. // Args: // table: Table name. diff --git a/common/configdb.h b/common/configdb.h index fb34f9112..b0394aaef 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -14,6 +14,7 @@ class ConfigDBConnector : public SonicV2Connector void connect(bool wait_for_init = true, bool retry_on = false); void set_entry(std::string table, std::string key, const std::unordered_map& data); + void mod_entry(std::string table, std::string key, const std::unordered_map& data); std::unordered_map get_entry(std::string table, std::string key); protected: From 21ced2723d510efcf6e1d89fc88996eb66388028 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 26 Dec 2020 19:41:39 +0000 Subject: [PATCH 06/25] Implement db_connect() --- common/configdb.cpp | 9 +++++++-- common/configdb.h | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/common/configdb.cpp b/common/configdb.cpp index 78e465015..2dd096236 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -12,9 +12,9 @@ ConfigDBConnector::ConfigDBConnector(bool use_unix_socket_path, const char *netn { } -void ConfigDBConnector::connect(bool wait_for_init, bool retry_on) +void ConfigDBConnector::db_connect(string db_name, bool wait_for_init, bool retry_on) { - m_db_name = "CONFIG_DB"; + m_db_name = db_name; SonicV2Connector::connect(m_db_name, retry_on); if (wait_for_init) @@ -49,6 +49,11 @@ void ConfigDBConnector::connect(bool wait_for_init, bool retry_on) } } +void ConfigDBConnector::connect(bool wait_for_init, bool retry_on) +{ + db_connect("CONFIG_DB", wait_for_init, retry_on); +} + // Write a table entry to config db. // Remove extra fields in the db which are not in the data. // Args: diff --git a/common/configdb.h b/common/configdb.h index b0394aaef..d2e211b2e 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -11,6 +11,7 @@ class ConfigDBConnector : public SonicV2Connector public: ConfigDBConnector(bool use_unix_socket_path = false, const char *netns = ""); + void db_connect(std::string db_name, bool wait_for_init, bool retry_on); void connect(bool wait_for_init = true, bool retry_on = false); void set_entry(std::string table, std::string key, const std::unordered_map& data); From bce0d4e47e1cc1dc186d1cfaa9c5604e2fb4dce2 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 26 Dec 2020 20:09:28 +0000 Subject: [PATCH 07/25] Implement get_keys() --- common/configdb.cpp | 30 ++++++++++++++++++++++++++++++ common/configdb.h | 1 + 2 files changed, 31 insertions(+) diff --git a/common/configdb.cpp b/common/configdb.cpp index 2dd096236..df8997131 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "configdb.h" #include "pubsub.h" #include "converter.h" @@ -119,3 +120,32 @@ unordered_map ConfigDBConnector::get_entry(string table, string string _hash = to_upper(table) + TABLE_NAME_SEPARATOR + key; return client.hgetall(_hash); } + +// Read all keys of a table from config db. +// Args: +// table: Table name. +// split: split the first part and return second. +// Useful for keys with two parts : +// Returns: +// List of keys. +vector ConfigDBConnector::get_keys(string table, bool split) +{ + auto& client = get_redis_client(m_db_name); + string pattern = to_upper(table) + TABLE_NAME_SEPARATOR + "*"; + const auto& keys = client.keys(pattern); + vector data; + for (auto& key: keys) + { + if (split) + { + size_t pos = key.find(TABLE_NAME_SEPARATOR); + string row = key.substr(pos + 1); + data.push_back(row); + } + else + { + data.push_back(key); + } + } + return data; +} diff --git a/common/configdb.h b/common/configdb.h index d2e211b2e..9b7dceec7 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -17,6 +17,7 @@ class ConfigDBConnector : public SonicV2Connector void set_entry(std::string table, std::string key, const std::unordered_map& data); void mod_entry(std::string table, std::string key, const std::unordered_map& data); std::unordered_map get_entry(std::string table, std::string key); + std::vector get_keys(std::string table, bool split = true); protected: static constexpr const char *INIT_INDICATOR = "CONFIG_DB_INITIALIZED"; From b18735b2653f71caa25797563ee167013ebb9af3 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 26 Dec 2020 20:43:15 +0000 Subject: [PATCH 08/25] Implement get_table() --- common/configdb.cpp | 25 +++++++++++++++++++++++++ common/configdb.h | 1 + 2 files changed, 26 insertions(+) diff --git a/common/configdb.cpp b/common/configdb.cpp index df8997131..35524bb17 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -149,3 +149,28 @@ vector ConfigDBConnector::get_keys(string table, bool split) } return data; } + +// Read an entire table from config db. +// Args: +// table: Table name. +// Returns: +// Table data in a dictionary form of +// { 'row_key': {'column_key': value, ...}, ...} +// or { ('l1_key', 'l2_key', ...): {'column_key': value, ...}, ...} for a multi-key table. +// Empty dictionary if table does not exist. +unordered_map> ConfigDBConnector::get_table(string table) +{ + auto& client = get_redis_client(m_db_name); + string pattern = to_upper(table) + TABLE_NAME_SEPARATOR + "*"; + const auto& keys = client.keys(pattern); + unordered_map> data; + for (auto& key: keys) + { + auto const& entry = client.hgetall(key); + size_t pos = key.find(TABLE_NAME_SEPARATOR); + string row = key.substr(pos + 1); + data[row] = entry; + } + return data; +} + diff --git a/common/configdb.h b/common/configdb.h index 9b7dceec7..14d792f3b 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -18,6 +18,7 @@ class ConfigDBConnector : public SonicV2Connector void mod_entry(std::string table, std::string key, const std::unordered_map& data); std::unordered_map get_entry(std::string table, std::string key); std::vector get_keys(std::string table, bool split = true); + std::unordered_map> get_table(std::string table); protected: static constexpr const char *INIT_INDICATOR = "CONFIG_DB_INITIALIZED"; From ab958f6a61742a0d9d060260398da91465fc40dd Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 26 Dec 2020 20:47:46 +0000 Subject: [PATCH 09/25] Implement delete_table() --- common/configdb.cpp | 14 ++++++++++++++ common/configdb.h | 1 + 2 files changed, 15 insertions(+) diff --git a/common/configdb.cpp b/common/configdb.cpp index 35524bb17..acb544d24 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -174,3 +174,17 @@ unordered_map> ConfigDBConnector::get_tabl return data; } +// Delete an entire table from config db. +// Args: +// table: Table name. +void ConfigDBConnector::delete_table(string table) +{ + auto& client = get_redis_client(m_db_name); + string pattern = to_upper(table) + TABLE_NAME_SEPARATOR + "*"; + const auto& keys = client.keys(pattern); + for (auto& key: keys) + { + client.del(key); + } +} + diff --git a/common/configdb.h b/common/configdb.h index 14d792f3b..7b4268d3b 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -19,6 +19,7 @@ class ConfigDBConnector : public SonicV2Connector std::unordered_map get_entry(std::string table, std::string key); std::vector get_keys(std::string table, bool split = true); std::unordered_map> get_table(std::string table); + void delete_table(std::string table); protected: static constexpr const char *INIT_INDICATOR = "CONFIG_DB_INITIALIZED"; From c7f50fe59630015c5c8440b9c63dad9f5b80880a Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sat, 26 Dec 2020 21:44:49 +0000 Subject: [PATCH 10/25] Implement mod_config(), get_config() --- common/configdb.cpp | 62 ++++++++++++++++++++++++++++++++++++++++++++- common/configdb.h | 2 ++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/common/configdb.cpp b/common/configdb.cpp index acb544d24..6356d5629 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -15,7 +15,7 @@ ConfigDBConnector::ConfigDBConnector(bool use_unix_socket_path, const char *netn void ConfigDBConnector::db_connect(string db_name, bool wait_for_init, bool retry_on) { - m_db_name = db_name; + m_db_name = db_name; SonicV2Connector::connect(m_db_name, retry_on); if (wait_for_init) @@ -188,3 +188,63 @@ void ConfigDBConnector::delete_table(string table) } } +// Write multiple tables into config db. +// Extra entries/fields in the db which are not in the data are kept. +// Args: +// data: config data in a dictionary form +// { +// 'TABLE_NAME': { 'row_key': {'column_key': 'value', ...}, ...}, +// 'MULTI_KEY_TABLE_NAME': { ('l1_key', 'l2_key', ...) : {'column_key': 'value', ...}, ...}, +// ... +// } +void ConfigDBConnector::mod_config(const unordered_map>>& data) +{ + for (auto const& it: data) + { + string table_name = it.first; + auto const& table_data = it.second; + if (table_data.empty()) + { + delete_table(table_name); + continue; + } + for (auto const& ie: table_data) + { + string key = ie.first; + auto const& fvs = ie.second; + mod_entry(table_name, key, fvs); + } + } +} + +// Read all config data. +// Returns: +// Config data in a dictionary form of +// { +// 'TABLE_NAME': { 'row_key': {'column_key': 'value', ...}, ...}, +// 'MULTI_KEY_TABLE_NAME': { ('l1_key', 'l2_key', ...) : {'column_key': 'value', ...}, ...}, +// ... +// } +unordered_map>> ConfigDBConnector::get_config() +{ + auto& client = get_redis_client(m_db_name); + auto const& keys = client.keys("*"); + unordered_map>> data; + for (string key: keys) + { + size_t pos = key.find(TABLE_NAME_SEPARATOR); + if (pos == string::npos) + { + continue; + } + string table_name = key.substr(0, pos); + string row = key.substr(pos + 1); + auto const& entry = client.hgetall(key); + + if (!entry.empty()) + { + data[table_name][row] = entry; + } + } + return data; +} diff --git a/common/configdb.h b/common/configdb.h index 7b4268d3b..4ce42abf9 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -20,6 +20,8 @@ class ConfigDBConnector : public SonicV2Connector std::vector get_keys(std::string table, bool split = true); std::unordered_map> get_table(std::string table); void delete_table(std::string table); + void mod_config(const std::unordered_map>>& data); + std::unordered_map>> get_config(); protected: static constexpr const char *INIT_INDICATOR = "CONFIG_DB_INITIALIZED"; From 6f4a7014dec79a0bfbe1ae6c16599c9d8f9e6320 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sun, 27 Dec 2020 00:35:38 +0000 Subject: [PATCH 11/25] On-par with python implementation --- common/configdb.cpp | 21 ++++++++++++++++++--- common/configdb.h | 4 ++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/common/configdb.cpp b/common/configdb.cpp index 6356d5629..e293a77d8 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -10,12 +10,15 @@ using namespace swss; ConfigDBConnector::ConfigDBConnector(bool use_unix_socket_path, const char *netns) : SonicV2Connector(use_unix_socket_path, netns) + , TABLE_NAME_SEPARATOR("|") + , KEY_SEPARATOR("|") { } void ConfigDBConnector::db_connect(string db_name, bool wait_for_init, bool retry_on) { m_db_name = db_name; + KEY_SEPARATOR = TABLE_NAME_SEPARATOR = get_db_separator(db_name); SonicV2Connector::connect(m_db_name, retry_on); if (wait_for_init) @@ -34,7 +37,11 @@ void ConfigDBConnector::db_connect(string db_name, bool wait_for_init, bool retr { string channel = item["channel"]; size_t pos = channel.find(':'); - string key = channel.substr(pos + 1); + string key; + if (pos != string::npos) + { + key = channel.substr(pos + 1); + } if (key == INIT_INDICATOR) { initialized = client.get(INIT_INDICATOR); @@ -139,7 +146,11 @@ vector ConfigDBConnector::get_keys(string table, bool split) if (split) { size_t pos = key.find(TABLE_NAME_SEPARATOR); - string row = key.substr(pos + 1); + string row; + if (pos != string::npos) + { + row = key.substr(pos + 1); + } data.push_back(row); } else @@ -168,7 +179,11 @@ unordered_map> ConfigDBConnector::get_tabl { auto const& entry = client.hgetall(key); size_t pos = key.find(TABLE_NAME_SEPARATOR); - string row = key.substr(pos + 1); + string row; + if (pos != string::npos) + { + row = key.substr(pos + 1); + } data[row] = entry; } return data; diff --git a/common/configdb.h b/common/configdb.h index 4ce42abf9..0381db8fb 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -25,8 +25,8 @@ class ConfigDBConnector : public SonicV2Connector protected: static constexpr const char *INIT_INDICATOR = "CONFIG_DB_INITIALIZED"; - static constexpr const char *TABLE_NAME_SEPARATOR = "|"; - static constexpr const char *KEY_SEPARATOR = "|"; + std::string TABLE_NAME_SEPARATOR = "|"; + std::string KEY_SEPARATOR = "|"; std::string m_db_name; }; From 23d0b42e2a359023709be4a852c5f3259fe59373 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sun, 27 Dec 2020 00:37:12 +0000 Subject: [PATCH 12/25] Implement dynamic functions --- common/configdb.h | 94 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/common/configdb.h b/common/configdb.h index 0381db8fb..02025b230 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -29,6 +29,100 @@ class ConfigDBConnector : public SonicV2Connector std::string KEY_SEPARATOR = "|"; std::string m_db_name; + +#ifdef SWIG + // Dynamic typed functions used in python + %pythoncode %{ + @staticmethod + def raw_to_typed(raw_data): + if raw_data is None: + return None + typed_data = {} + for raw_key in raw_data: + key = raw_key + + # "NULL:NULL" is used as a placeholder for objects with no attributes + if key == "NULL": + pass + # A column key with ending '@' is used to mark list-typed table items + # TODO: Replace this with a schema-based typing mechanism. + elif key.endswith("@"): + value = raw_data[raw_key].split(',') + typed_data[key[:-1]] = value + else: + typed_data[key] = raw_data[raw_key] + return typed_data + + @staticmethod + def typed_to_raw(typed_data): + if typed_data is None: + return {} + elif len(typed_data) == 0: + return { "NULL": "NULL" } + raw_data = {} + for key in typed_data: + value = typed_data[key] + if type(value) is list: + raw_data[key+'@'] = ','.join(value) + else: + raw_data[key] = str(value) + return raw_data + + @staticmethod + def serialize_key(key): + if type(key) is tuple: + return ConfigDBConnector.KEY_SEPARATOR.join(key) + else: + return str(key) + + @staticmethod + def deserialize_key(key): + tokens = key.split(ConfigDBConnector.KEY_SEPARATOR) + if len(tokens) > 1: + return tuple(tokens) + else: + return key + %} +#endif + }; +#ifdef SWIG +%pythoncode %{ + // TRICK! + // Note: there is no easy way for SWIG to map ctor parameter netns(C++) to namespace(python), + // so we use python patch to achieve this + // TODO: implement it with formal SWIG syntax, which will be target language independent + _old_ConfigDBConnector__init__ = ConfigDBConnector.__init__ + def _new_ConfigDBConnector__init__(self, use_unix_socket_path = False, namespace = '', **kwargs): + if 'decode_responses' in kwargs and kwargs.pop('decode_responses') != True: + raise ValueError('decode_responses must be True if specified, False is not supported') + if namespace is None: + namespace = '' + _old_ConfigDBConnector__init__(self, use_unix_socket_path = use_unix_socket_path, netns = namespace) + ConfigDBConnector.__init__ = _new_ConfigDBConnector__init__ + + _old_ConfigDBConnector_set_entry = ConfigDBConnector.set_entry + def _new_ConfigDBConnector_set_entry(self, table, key, data): + key = self.serialize_key(key) + raw_data = self.typed_to_raw(data) + _old_ConfigDBConnector_set_entry(self, table, key, raw_data) + ConfigDBConnector.set_entry = _new_ConfigDBConnector_set_entry + + _old_ConfigDBConnector_mod_entry = ConfigDBConnector.mod_entry + def _new_ConfigDBConnector_mod_entry(self, table, key, data): + key = self.serialize_key(key) + raw_data = self.typed_to_raw(data) + _old_ConfigDBConnector_mod_entry(self, table, key, raw_data) + ConfigDBConnector.mod_entry = _new_ConfigDBConnector_mod_entry + + _old_ConfigDBConnector_get_entry = ConfigDBConnector.get_entry + def _new_ConfigDBConnector_get_entry(self, table, key): + key = self.serialize_key(key) + raw_data = _old_ConfigDBConnector_get_entry(self, table, key) + return self.raw_to_typed(raw_data) + ConfigDBConnector.get_entry = _new_ConfigDBConnector_get_entry + +%} +#endif } From 4fb9658932fc26f884c05b0eae717d437650cefe Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sun, 27 Dec 2020 00:51:10 +0000 Subject: [PATCH 13/25] Implement getKeySeparator(), move static functions to public --- common/configdb.cpp | 5 +++++ common/configdb.h | 19 ++++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/common/configdb.cpp b/common/configdb.cpp index e293a77d8..58362624b 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -263,3 +263,8 @@ unordered_map>> Conf } return data; } + +std::string ConfigDBConnector::getKeySeparator() const +{ + return KEY_SEPARATOR; +} diff --git a/common/configdb.h b/common/configdb.h index 02025b230..5cd0ab92a 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -23,16 +23,15 @@ class ConfigDBConnector : public SonicV2Connector void mod_config(const std::unordered_map>>& data); std::unordered_map>> get_config(); -protected: - static constexpr const char *INIT_INDICATOR = "CONFIG_DB_INITIALIZED"; - std::string TABLE_NAME_SEPARATOR = "|"; - std::string KEY_SEPARATOR = "|"; - - std::string m_db_name; + std::string getKeySeparator() const; #ifdef SWIG - // Dynamic typed functions used in python %pythoncode %{ + __swig_getmethods__["KEY_SEPARATOR"] = getKeySeparator + __swig_setmethods__["KEY_SEPARATOR"] = None + if _newclass: KEY_SEPARATOR = property(getKeySeparator, None) + + ## Dynamic typed functions used in python @staticmethod def raw_to_typed(raw_data): if raw_data is None: @@ -85,6 +84,12 @@ class ConfigDBConnector : public SonicV2Connector %} #endif +protected: + static constexpr const char *INIT_INDICATOR = "CONFIG_DB_INITIALIZED"; + std::string TABLE_NAME_SEPARATOR = "|"; + std::string KEY_SEPARATOR = "|"; + + std::string m_db_name; }; #ifdef SWIG From 27a35f70b79780b545b34d74746b7ce1c33bb33c Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sun, 27 Dec 2020 02:38:38 +0000 Subject: [PATCH 14/25] Implement listen() --- common/configdb.h | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/common/configdb.h b/common/configdb.h index 5cd0ab92a..a6c0ea3a4 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -31,6 +31,24 @@ class ConfigDBConnector : public SonicV2Connector __swig_setmethods__["KEY_SEPARATOR"] = None if _newclass: KEY_SEPARATOR = property(getKeySeparator, None) + ## Note: callback is difficult to implement by SWIG C++, so keep in python + def listen(self): + ## Start listen Redis keyspace events and will trigger corresponding handlers when content of a table changes. + self.pubsub = self.get_redis_client(self.m_db_name).pubsub() + self.pubsub.psubscribe("__keyspace@{}__:*".format(self.get_dbid(self.m_db_name))) + while True: + item = self.pubsub.listen_message(): + if item['type'] == 'pmessage': + key = item['channel'].split(':', 1)[1] + try: + (table, row) = key.split(self.TABLE_NAME_SEPARATOR, 1) + if table in self.handlers: + client = self.get_redis_client(self.db_name) + data = self.raw_to_typed(client.hgetall(key)) + self.__fire(table, row, data) + except ValueError: + pass #Ignore non table-formated redis entries + ## Dynamic typed functions used in python @staticmethod def raw_to_typed(raw_data): @@ -90,6 +108,15 @@ class ConfigDBConnector : public SonicV2Connector std::string KEY_SEPARATOR = "|"; std::string m_db_name; + +#ifdef SWIG + %pythoncode %{ + def __fire(self, table, key, data): + if table in self.handlers: + handler = self.handlers[table] + handler(table, key, data) + %} +#endif }; #ifdef SWIG @@ -105,6 +132,8 @@ class ConfigDBConnector : public SonicV2Connector if namespace is None: namespace = '' _old_ConfigDBConnector__init__(self, use_unix_socket_path = use_unix_socket_path, netns = namespace) + ## Note: callback is difficult to implement by SWIG C++, so keep in python + self.handlers = {} ConfigDBConnector.__init__ = _new_ConfigDBConnector__init__ _old_ConfigDBConnector_set_entry = ConfigDBConnector.set_entry From 10418199054b69920854e88e51ae142bc1b86051 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sun, 27 Dec 2020 03:09:49 +0000 Subject: [PATCH 15/25] Fix RedisTransactioner: handle empty deque --- common/redistran.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/common/redistran.cpp b/common/redistran.cpp index 8c3c87936..0b50086d0 100644 --- a/common/redistran.cpp +++ b/common/redistran.cpp @@ -73,6 +73,10 @@ void RedisTransactioner::enqueue(const std::string &command, int expectedType) redisReply *RedisTransactioner::dequeueReply() { + if (m_results.empty()) + { + return NULL; + } redisReply *ret = m_results.front(); m_results.pop_front(); return ret; @@ -80,7 +84,10 @@ redisReply *RedisTransactioner::dequeueReply() void RedisTransactioner::clearResults() { - if (m_results.empty()) return; + if (m_results.empty()) + { + return; + } for (const auto& r: m_results) { freeReplyObject(r); From 9a18482e85c9d9a14a955b47af2e6e5f94f69082 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sun, 27 Dec 2020 04:06:05 +0000 Subject: [PATCH 16/25] Implement DBConnector scan() --- common/dbconnector.cpp | 20 ++++++++++++++++++++ common/dbconnector.h | 2 ++ common/dbinterface.cpp | 5 +++++ common/dbinterface.h | 1 + common/redisreply.cpp | 12 ++++++++++++ common/redisreply.h | 5 +++++ common/sonicv2connector.cpp | 5 +++++ common/sonicv2connector.h | 2 ++ pyext/py2/Makefile.am | 2 +- pyext/py3/Makefile.am | 2 +- pyext/swsscommon.i | 1 + 11 files changed, 55 insertions(+), 2 deletions(-) diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index e881b5b84..0832835c7 100644 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -681,6 +681,26 @@ vector DBConnector::keys(const string &key) return list; } +pair> DBConnector::scan(int64_t cursor, const char *match, uint32_t count) +{ + RedisCommand sscan; + sscan.format("SCAN %lld %s %lld", cursor, match, count); + RedisReply r(this, sscan, REDIS_REPLY_ARRAY); + + RedisReply r0(r.releaseChild(0)); + RedisReply r1(r.releaseChild(1)); + r1.checkReplyType(REDIS_REPLY_ARRAY); + + pair> ret; + ret.first = r0.getReply(); + for (size_t i = 0; i < r1.getChildCount(); i++) + { + RedisReply r11(r1.releaseChild(i)); + ret.second.emplace_back(r11.getReply()); + } + return ret; +} + int64_t DBConnector::incr(const string &key) { RedisCommand sincr; diff --git a/common/dbconnector.h b/common/dbconnector.h index f21a37271..c7d505b0c 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -175,6 +175,8 @@ class DBConnector : public RedisContext std::vector keys(const std::string &key); + std::pair> scan(int64_t cursor = 0, const char *match = "", uint32_t count = 10); + void set(const std::string &key, const std::string &value); void hset(const std::string &key, const std::string &field, const std::string &value); diff --git a/common/dbinterface.cpp b/common/dbinterface.cpp index 492ffef9c..939777f13 100644 --- a/common/dbinterface.cpp +++ b/common/dbinterface.cpp @@ -120,6 +120,11 @@ std::vector DBInterface::keys(const std::string& dbName, const char return blockable>(innerfunc, dbName, blocking); } +std::pair> DBInterface::scan(const std::string& db_name, int64_t cursor, const char *match, uint32_t count) +{ + return m_redisClient.at(db_name).scan(cursor, match, count); +} + int64_t DBInterface::publish(const std::string& dbName, const std::string& channel, const std::string& message) { return m_redisClient.at(dbName).publish(channel, message); diff --git a/common/dbinterface.h b/common/dbinterface.h index 9922b73da..fb55362a5 100644 --- a/common/dbinterface.h +++ b/common/dbinterface.h @@ -41,6 +41,7 @@ class DBInterface bool hexists(const std::string& dbName, const std::string& hash, const std::string& key); std::map get_all(const std::string& dbName, const std::string& hash, bool blocking = false); std::vector keys(const std::string& dbName, const char *pattern = "*", bool blocking = false); + std::pair> scan(const std::string& db_name, int64_t cursor, const char *match, uint32_t count); int64_t publish(const std::string& dbName, const std::string& channel, const std::string& message); int64_t set(const std::string& dbName, const std::string& hash, const std::string& key, const std::string& value, bool blocking = false); DBConnector& get_redis_client(const std::string& dbName); diff --git a/common/redisreply.cpp b/common/redisreply.cpp index abf2c650e..fadefaf5b 100644 --- a/common/redisreply.cpp +++ b/common/redisreply.cpp @@ -99,6 +99,11 @@ redisReply *RedisReply::getContext() return m_reply; } +size_t RedisReply::getChildCount() +{ + return m_reply->elements; +} + redisReply *RedisReply::getChild(size_t index) { if (index >= m_reply->elements) @@ -108,6 +113,13 @@ redisReply *RedisReply::getChild(size_t index) return m_reply->element[index]; } +redisReply *RedisReply::releaseChild(size_t index) +{ + auto ret = getChild(index); + m_reply->element[index] = NULL; + return ret; +} + void RedisReply::checkStatus(const char *status) { if (strcmp(m_reply->str, status) != 0) diff --git a/common/redisreply.h b/common/redisreply.h index c758ea5dc..85930efaf 100644 --- a/common/redisreply.h +++ b/common/redisreply.h @@ -74,8 +74,13 @@ class RedisReply /* Return the actual reply object */ redisReply *getContext(); + size_t getChildCount(); + redisReply *getChild(size_t index); + /* Return the actual child reply object and release the ownership */ + redisReply *releaseChild(size_t index); + void checkReplyType(int expectedType); template diff --git a/common/sonicv2connector.cpp b/common/sonicv2connector.cpp index aa86ce5c8..f20658378 100644 --- a/common/sonicv2connector.cpp +++ b/common/sonicv2connector.cpp @@ -69,6 +69,11 @@ std::vector SonicV2Connector::keys(const std::string& db_name, cons return m_dbintf.keys(db_name, pattern, blocking); } +std::pair> SonicV2Connector::scan(const std::string& db_name, int64_t cursor, const char *match, uint32_t count) +{ + return m_dbintf.scan(db_name, cursor, match, count); +} + std::string SonicV2Connector::get(const std::string& db_name, const std::string& _hash, const std::string& key, bool blocking) { return m_dbintf.get(db_name, _hash, key, blocking); diff --git a/common/sonicv2connector.h b/common/sonicv2connector.h index 1cbc1fa53..7c1b13230 100644 --- a/common/sonicv2connector.h +++ b/common/sonicv2connector.h @@ -41,6 +41,8 @@ class SonicV2Connector std::vector keys(const std::string& db_name, const char *pattern="*", bool blocking=false); + std::pair> scan(const std::string& db_name, int64_t cursor = 0, const char *match = "", uint32_t count = 10); + std::string get(const std::string& db_name, const std::string& _hash, const std::string& key, bool blocking=false); bool hexists(const std::string& db_name, const std::string& _hash, const std::string& key); diff --git a/pyext/py2/Makefile.am b/pyext/py2/Makefile.am index b01c75262..d6fe5b328 100644 --- a/pyext/py2/Makefile.am +++ b/pyext/py2/Makefile.am @@ -9,6 +9,6 @@ _swsscommon_la_LDFLAGS = -module _swsscommon_la_LIBADD = ../../common/libswsscommon.la -lpython$(PYTHON_VERSION) swsscommon_wrap.cpp: $(SWIG_SOURCES) - $(SWIG) -Wall -c++ -python -keyword -I../../common -o $@ $< + $(SWIG) -DSWIGWORDSIZE64 -Wall -c++ -python -keyword -I../../common -o $@ $< CLEANFILES = swsscommon_wrap.cpp diff --git a/pyext/py3/Makefile.am b/pyext/py3/Makefile.am index b1a46b3d6..65463ac8f 100644 --- a/pyext/py3/Makefile.am +++ b/pyext/py3/Makefile.am @@ -9,6 +9,6 @@ _swsscommon_la_LDFLAGS = -module _swsscommon_la_LIBADD = ../../common/libswsscommon.la $(PYTHON3_BLDLIBRARY) swsscommon_wrap.cpp: $(SWIG_SOURCES) - $(SWIG) -Wall -c++ -python -keyword -I../../common -o $@ $< + $(SWIG) -DSWIGWORDSIZE64 -Wall -c++ -python -keyword -I../../common -o $@ $< CLEANFILES = swsscommon_wrap.cpp diff --git a/pyext/swsscommon.i b/pyext/swsscommon.i index 6ed58c53a..3f15ccb7f 100644 --- a/pyext/swsscommon.i +++ b/pyext/swsscommon.i @@ -60,6 +60,7 @@ %template(FieldValuePairs) std::vector>; %template(FieldValueMap) std::map; %template(VectorString) std::vector; +%template(ScanResult) std::pair>; %pythoncode %{ def _FieldValueMap__get(self, key, default=None): From e6a8dcbddaf152275df7ff32de3dbf9c426feb06 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sun, 27 Dec 2020 23:46:09 +0000 Subject: [PATCH 17/25] Simply refactor DBConnector hgetall() --- common/dbconnector.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/dbconnector.h b/common/dbconnector.h index c7d505b0c..c87e7c8ce 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -223,9 +223,9 @@ class DBConnector : public RedisContext template void DBConnector::hgetall(const std::string &key, OutputIterator result) { - RedisCommand sincr; - sincr.format("HGETALL %s", key.c_str()); - RedisReply r(this, sincr, REDIS_REPLY_ARRAY); + RedisCommand shgetall; + shgetall.format("HGETALL %s", key.c_str()); + RedisReply r(this, shgetall, REDIS_REPLY_ARRAY); auto ctx = r.getContext(); From 1a9bf7383db3ba58da5656b6e47d3092279e86c6 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Sun, 27 Dec 2020 23:46:38 +0000 Subject: [PATCH 18/25] Implement class ConfigDBPipeConnector --- common/configdb.cpp | 174 ++++++++++++++++++++++++++++++++++++++++++++ common/configdb.h | 41 ++++++++++- 2 files changed, 213 insertions(+), 2 deletions(-) diff --git a/common/configdb.cpp b/common/configdb.cpp index 58362624b..5b1ef72fa 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -268,3 +268,177 @@ std::string ConfigDBConnector::getKeySeparator() const { return KEY_SEPARATOR; } + +// Helper method to delete table entries from config db using Redis pipeline +// with batch size of REDIS_SCAN_BATCH_SIZE. +// The caller should call pipeline execute once ready +// Args: +// client: Redis client +// pipe: Redis DB pipe +// pattern: key pattern +// cursor: position to start scanning from +// +// Returns: +// cur: poition of next item to scan +int64_t ConfigDBPipeConnector::_delete_entries(DBConnector& client, RedisTransactioner& pipe, const char *pattern, int64_t cursor) +{ + const auto& rc = client.scan(cursor, pattern, REDIS_SCAN_BATCH_SIZE); + auto cur = rc.first; + auto& keys = rc.second; + for (auto const& key: keys) + { + RedisCommand sdel; + sdel.format("DEL %s", key.c_str()); + pipe.enqueue(sdel.c_str(), REDIS_REPLY_INTEGER); + } + + return cur; +} + +// Helper method to delete table entries from config db using Redis pipeline. +// The caller should call pipeline execute once ready +// Args: +// client: Redis client +// pipe: Redis DB pipe +// table: Table name. +void ConfigDBPipeConnector::_delete_table(DBConnector& client, RedisTransactioner& pipe, string table) +{ + string pattern = to_upper(table) + TABLE_NAME_SEPARATOR + "*"; + auto cur = _delete_entries(client, pipe, pattern.c_str(), 0); + while (cur != 0) + { + cur = _delete_entries(client, pipe, pattern.c_str(), cur); + } +} + +// Modify a table entry to config db. +// Args: +// table: Table name. +// pipe: Redis DB pipe +// table: Table name. +// key: Key of table entry, or a tuple of keys if it is a multi-key table. +// data: Table row data in a form of dictionary {'column_key': 'value', ...}. +// Pass {} as data will create an entry with no column if not already existed. +// Pass None as data will delete the entry. +void ConfigDBPipeConnector::_mod_entry(RedisTransactioner& pipe, string table, string key, const unordered_map& data) +{ + string _hash = to_upper(table) + TABLE_NAME_SEPARATOR + key; + if (data.empty()) + { + RedisCommand sdel; + sdel.format("DEL %s", _hash.c_str()); + pipe.enqueue(sdel.c_str(), REDIS_REPLY_INTEGER); + } + else + { + RedisCommand shmset; + shmset.formatHMSET(_hash, data.begin(), data.end()); + pipe.enqueue(shmset.c_str(), REDIS_REPLY_STATUS); + } +} +// Write multiple tables into config db. +// Extra entries/fields in the db which are not in the data are kept. +// Args: +// data: config data in a dictionary form +// { +// 'TABLE_NAME': { 'row_key': {'column_key': 'value', ...}, ...}, +// 'MULTI_KEY_TABLE_NAME': { ('l1_key', 'l2_key', ...) : {'column_key': 'value', ...}, ...}, +// ... +// } +void ConfigDBPipeConnector::mod_config(const unordered_map>>& data) +{ + auto& client = get_redis_client(m_db_name); + RedisTransactioner pipe(&client); + pipe.multi(); + for (auto const& id: data) + { + auto& table_name = id.first; + auto& table_data = id.second; + if (table_data.empty()) + { + _delete_table(client, pipe, table_name); + continue; + } + for (auto const& it: table_data) + { + auto& key = it.first; + _mod_entry(pipe, table_name, key, it.second); + } + } + pipe.exec(); +} + +// Read config data in batches of size REDIS_SCAN_BATCH_SIZE using Redis pipelines +// Args: +// client: Redis client +// pipe: Redis DB pipe +// data: config dictionary +// cursor: position to start scanning from +// +// Returns: +// cur: poition of next item to scan +int64_t ConfigDBPipeConnector::_get_config(DBConnector& client, RedisTransactioner& pipe, unordered_map>>& data, int64_t cursor) +{ + auto const& rc = client.scan(cursor, "*", REDIS_SCAN_BATCH_SIZE); + auto cur = rc.first; + auto const& keys = rc.second; + pipe.multi(); + for (auto const& key: keys) + { + if (key == INIT_INDICATOR) + { + continue; + } + RedisCommand shgetall; + shgetall.format("HGETALL %s", key.c_str()); + pipe.enqueue(shgetall.c_str(), REDIS_REPLY_ARRAY); + } + pipe.exec(); + + for (auto const& key: keys) + { + if (key == INIT_INDICATOR) + { + continue; + } + + size_t pos = key.find(TABLE_NAME_SEPARATOR); + if (pos == string::npos) + { + continue; + } + string table_name = key.substr(0, pos - 1); + string row = key.substr(pos + 1); + + auto reply = pipe.dequeueReply(); + if (reply == NULL) + { + continue; + } + RedisReply r(reply); + + auto dataentry = data[table_name][row]; + for (unsigned int i = 0; i < r.getChildCount(); i += 2) + { + string field = r.getChild(i)->str; + string value = r.getChild(i+1)->str; + dataentry.emplace(field, value); + } + } + return cur; +} + +unordered_map>> ConfigDBPipeConnector::get_config() +{ + auto& client = get_redis_client(m_db_name); + RedisTransactioner pipe(&client); + + unordered_map>> data; + auto cur = _get_config(client, pipe, data, 0); + while (cur != 0) + { + cur = _get_config(client, pipe, data, cur); + } + + return data; +} diff --git a/common/configdb.h b/common/configdb.h index a6c0ea3a4..eb90be05e 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -3,6 +3,7 @@ #include #include #include "sonicv2connector.h" +#include "redistran.h" namespace swss { @@ -20,8 +21,8 @@ class ConfigDBConnector : public SonicV2Connector std::vector get_keys(std::string table, bool split = true); std::unordered_map> get_table(std::string table); void delete_table(std::string table); - void mod_config(const std::unordered_map>>& data); - std::unordered_map>> get_config(); + virtual void mod_config(const std::unordered_map>>& data); + virtual std::unordered_map>> get_config(); std::string getKeySeparator() const; @@ -159,4 +160,40 @@ class ConfigDBConnector : public SonicV2Connector %} #endif + +class ConfigDBPipeConnector: public ConfigDBConnector +{ +public: + ConfigDBPipeConnector(bool use_unix_socket_path = false, const char *netns = ""); + + void mod_config(const std::unordered_map>>& data) override; + std::unordered_map>> get_config() override; + +private: + static const int64_t REDIS_SCAN_BATCH_SIZE = 30; + + int64_t _delete_entries(DBConnector& client, RedisTransactioner& pipe, const char *pattern, int64_t cursor); + void _delete_table(DBConnector& client, RedisTransactioner& pipe, std::string table); + void _mod_entry(RedisTransactioner& pipe, std::string table, std::string key, const std::unordered_map& data); + int64_t _get_config(DBConnector& client, RedisTransactioner& pipe, std::unordered_map>>& data, int64_t cursor); +}; + +#ifdef SWIG +%pythoncode %{ + // TRICK! + // Note: there is no easy way for SWIG to map ctor parameter netns(C++) to namespace(python), + // so we use python patch to achieve this + // TODO: implement it with formal SWIG syntax, which will be target language independent + _old_ConfigDBPipeConnector__init__ = ConfigDBPipeConnector.__init__ + def _new_ConfigDBPipeConnector__init__(self, use_unix_socket_path = False, namespace = '', **kwargs): + if 'decode_responses' in kwargs and kwargs.pop('decode_responses') != True: + raise ValueError('decode_responses must be True if specified, False is not supported') + if namespace is None: + namespace = '' + _old_ConfigDBPipeConnector__init__(self, use_unix_socket_path = use_unix_socket_path, netns = namespace) + ## Note: callback is difficult to implement by SWIG C++, so keep in python + self.handlers = {} + ConfigDBPipeConnector.__init__ = _new_ConfigDBPipeConnector__init__ +%} +#endif } From d09b28f379b7d711c60c86335aa533f7fe412e35 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Mon, 28 Dec 2020 06:53:44 +0000 Subject: [PATCH 19/25] Fix bugs Signed-off-by: Qi Luo --- common/configdb.cpp | 5 +++++ common/configdb.h | 18 +++++++++--------- pyext/swsscommon.i | 2 ++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/common/configdb.cpp b/common/configdb.cpp index 5b1ef72fa..8fade9fcf 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -269,6 +269,11 @@ std::string ConfigDBConnector::getKeySeparator() const return KEY_SEPARATOR; } +ConfigDBPipeConnector::ConfigDBPipeConnector(bool use_unix_socket_path, const char *netns) + : ConfigDBConnector(use_unix_socket_path, netns) +{ +} + // Helper method to delete table entries from config db using Redis pipeline // with batch size of REDIS_SCAN_BATCH_SIZE. // The caller should call pipeline execute once ready diff --git a/common/configdb.h b/common/configdb.h index eb90be05e..a59bdaf66 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -38,7 +38,7 @@ class ConfigDBConnector : public SonicV2Connector self.pubsub = self.get_redis_client(self.m_db_name).pubsub() self.pubsub.psubscribe("__keyspace@{}__:*".format(self.get_dbid(self.m_db_name))) while True: - item = self.pubsub.listen_message(): + item = self.pubsub.listen_message() if item['type'] == 'pmessage': key = item['channel'].split(':', 1)[1] try: @@ -122,10 +122,10 @@ class ConfigDBConnector : public SonicV2Connector #ifdef SWIG %pythoncode %{ - // TRICK! - // Note: there is no easy way for SWIG to map ctor parameter netns(C++) to namespace(python), - // so we use python patch to achieve this - // TODO: implement it with formal SWIG syntax, which will be target language independent + ## TRICK! + ## Note: there is no easy way for SWIG to map ctor parameter netns(C++) to namespace(python), + ## so we use python patch to achieve this + ## TODO: implement it with formal SWIG syntax, which will be target language independent _old_ConfigDBConnector__init__ = ConfigDBConnector.__init__ def _new_ConfigDBConnector__init__(self, use_unix_socket_path = False, namespace = '', **kwargs): if 'decode_responses' in kwargs and kwargs.pop('decode_responses') != True: @@ -180,10 +180,10 @@ class ConfigDBPipeConnector: public ConfigDBConnector #ifdef SWIG %pythoncode %{ - // TRICK! - // Note: there is no easy way for SWIG to map ctor parameter netns(C++) to namespace(python), - // so we use python patch to achieve this - // TODO: implement it with formal SWIG syntax, which will be target language independent + ## TRICK! + ## Note: there is no easy way for SWIG to map ctor parameter netns(C++) to namespace(python), + ## so we use python patch to achieve this + ## TODO: implement it with formal SWIG syntax, which will be target language independent _old_ConfigDBPipeConnector__init__ = ConfigDBPipeConnector.__init__ def _new_ConfigDBPipeConnector__init__(self, use_unix_socket_path = False, namespace = '', **kwargs): if 'decode_responses' in kwargs and kwargs.pop('decode_responses') != True: diff --git a/pyext/swsscommon.i b/pyext/swsscommon.i index 3f15ccb7f..19e6e2ffe 100644 --- a/pyext/swsscommon.i +++ b/pyext/swsscommon.i @@ -13,6 +13,7 @@ #define SWIG_PYTHON_2_UNICODE #include "schema.h" +#include "configdb.h" #include "dbconnector.h" #include "dbinterface.h" #include "sonicv2connector.h" @@ -106,6 +107,7 @@ T castSelectableObj(swss::Selectable *temp) %template(CastSelectableToSubscriberTableObj) castSelectableObj; %include "schema.h" +%include "configdb.h" %include "dbconnector.h" %include "sonicv2connector.h" %include "pubsub.h" From 3fdcd313ce6b7a5172919b6704dad4c1d6ef4116 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Wed, 30 Dec 2020 18:51:07 +0000 Subject: [PATCH 20/25] Fix bug: use instance variable instead of class variable in ConfigDBConnector --- common/configdb.h | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/common/configdb.h b/common/configdb.h index a59bdaf66..46d65803f 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -86,20 +86,29 @@ class ConfigDBConnector : public SonicV2Connector raw_data[key] = str(value) return raw_data + # Note: we could not use a class variable for KEY_SEPARATOR, but original dependent code is using + # these static functions. So we implement both static and instance functions with the same name. + # The static function will behave according to ConfigDB separators. @staticmethod - def serialize_key(key): + def serialize_key(key, separator='|'): if type(key) is tuple: - return ConfigDBConnector.KEY_SEPARATOR.join(key) + return separator.join(key) else: return str(key) + def _serialize_key(self, key): + return ConfigDBConnector.serialize_key(key, self.KEY_SEPARATOR) + @staticmethod - def deserialize_key(key): - tokens = key.split(ConfigDBConnector.KEY_SEPARATOR) + def deserialize_key(key, separator='|'): + tokens = key.split(separator) if len(tokens) > 1: return tuple(tokens) else: return key + + def _deserialize_key(self, key): + return ConfigDBConnector.deserialize_key(key, self.KEY_SEPARATOR) %} #endif @@ -133,6 +142,12 @@ class ConfigDBConnector : public SonicV2Connector if namespace is None: namespace = '' _old_ConfigDBConnector__init__(self, use_unix_socket_path = use_unix_socket_path, netns = namespace) + + # Trick: to achieve static/instance method "overload", we must use initize the function in ctor + # ref: https://stackoverflow.com/a/28766809/2514803 + self.serialize_key = self._serialize_key + self.deserialize_key = self._deserialize_key + ## Note: callback is difficult to implement by SWIG C++, so keep in python self.handlers = {} ConfigDBConnector.__init__ = _new_ConfigDBConnector__init__ From 3088bb8576b2502fd97d733634665f5719ca8a9d Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Thu, 31 Dec 2020 03:12:35 +0000 Subject: [PATCH 21/25] Fix: SonicDBConfig load_sonic_global_db_config --- common/dbconnector.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/common/dbconnector.h b/common/dbconnector.h index c87e7c8ce..029eb67b0 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -40,6 +40,15 @@ class SonicDBConfig static constexpr const char *DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE = "/var/run/redis/sonic-db/database_global.json"; static void initialize(const std::string &file = DEFAULT_SONIC_DB_CONFIG_FILE); static void initializeGlobalConfig(const std::string &file = DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE); +#ifdef SWIG + %pythoncode %{ + ## TODO: the python function and C++ one is not on-par + @staticmethod + def load_sonic_global_db_config(global_db_file_path=DEFAULT_SONIC_DB_GLOBAL_CONFIG_FILE, namespace=None): + SonicDBConfig.initializeGlobalConfig(global_db_file_path) + %} +#endif + static void validateNamespace(const std::string &netns); static std::string getDbInst(const std::string &dbName, const std::string &netns = EMPTY_NAMESPACE); static int getDbId(const std::string &dbName, const std::string &netns = EMPTY_NAMESPACE); From c32d955ffdfa2fdd5e450a4ee1ec34eeaa69d7bc Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Thu, 31 Dec 2020 03:55:22 +0000 Subject: [PATCH 22/25] Fix SWIG compilation under 32-bit Signed-off-by: Qi Luo --- configure.ac | 1 + pyext/py2/Makefile.am | 7 ++++++- pyext/py3/Makefile.am | 7 ++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 97b295339..a108f111c 100644 --- a/configure.ac +++ b/configure.ac @@ -29,6 +29,7 @@ AC_ARG_ENABLE(debug, *) AC_MSG_ERROR(bad value ${enableval} for --enable-debug) ;; esac],[debug=false]) AM_CONDITIONAL(DEBUG, test x$debug = xtrue) +AM_CONDITIONAL(ARCH64, test `getconf LONG_BIT` = "64") AC_PATH_PROG(SWIG, [swig3.0]) diff --git a/pyext/py2/Makefile.am b/pyext/py2/Makefile.am index d6fe5b328..f7f772778 100644 --- a/pyext/py2/Makefile.am +++ b/pyext/py2/Makefile.am @@ -8,7 +8,12 @@ _swsscommon_la_CPPFLAGS = -std=c++11 -I../../common -I/usr/include/python$(PYTHO _swsscommon_la_LDFLAGS = -module _swsscommon_la_LIBADD = ../../common/libswsscommon.la -lpython$(PYTHON_VERSION) +SWIG_FLAG = -Wall -c++ -python -keyword +if ARCH64 +SWIG_FLAG += -DSWIGWORDSIZE64 +endif + swsscommon_wrap.cpp: $(SWIG_SOURCES) - $(SWIG) -DSWIGWORDSIZE64 -Wall -c++ -python -keyword -I../../common -o $@ $< + $(SWIG) $(SWIG_FLAG) -I../../common -o $@ $< CLEANFILES = swsscommon_wrap.cpp diff --git a/pyext/py3/Makefile.am b/pyext/py3/Makefile.am index 65463ac8f..30d50af44 100644 --- a/pyext/py3/Makefile.am +++ b/pyext/py3/Makefile.am @@ -8,7 +8,12 @@ _swsscommon_la_CPPFLAGS = -std=c++11 -I../../common -I/usr/include/python$(PYTHO _swsscommon_la_LDFLAGS = -module _swsscommon_la_LIBADD = ../../common/libswsscommon.la $(PYTHON3_BLDLIBRARY) +SWIG_FLAG = -Wall -c++ -python -keyword +if ARCH64 +SWIG_FLAG += -DSWIGWORDSIZE64 +endif + swsscommon_wrap.cpp: $(SWIG_SOURCES) - $(SWIG) -DSWIGWORDSIZE64 -Wall -c++ -python -keyword -I../../common -o $@ $< + $(SWIG) $(SWIG_FLAG) -I../../common -o $@ $< CLEANFILES = swsscommon_wrap.cpp From 825b8f670a310a3cf50ed6f13ff460ff9e3da504 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Thu, 31 Dec 2020 07:21:18 +0000 Subject: [PATCH 23/25] Change unordered_map to map, because swig3.0 does not support Signed-off-by: Qi Luo --- common/configdb.cpp | 28 ++++++++++++++-------------- common/configdb.h | 20 ++++++++++---------- common/dbconnector.cpp | 4 ++-- common/dbconnector.h | 3 ++- pyext/swsscommon.i | 1 + 5 files changed, 29 insertions(+), 27 deletions(-) diff --git a/common/configdb.cpp b/common/configdb.cpp index 8fade9fcf..ed54b806a 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include "configdb.h" #include "pubsub.h" @@ -69,7 +69,7 @@ void ConfigDBConnector::connect(bool wait_for_init, bool retry_on) // key: Key of table entry, or a tuple of keys if it is a multi-key table. // data: Table row data in a form of dictionary {'column_key': 'value', ...}. // Pass {} as data will delete the entry. -void ConfigDBConnector::set_entry(string table, string key, const unordered_map& data) +void ConfigDBConnector::set_entry(string table, string key, const map& data) { auto& client = get_redis_client(m_db_name); string _hash = to_upper(table) + TABLE_NAME_SEPARATOR + key; @@ -100,7 +100,7 @@ void ConfigDBConnector::set_entry(string table, string key, const unordered_map< // data: Table row data in a form of dictionary {'column_key': 'value', ...}. // Pass {} as data will create an entry with no column if not already existed. // Pass None as data will delete the entry. -void ConfigDBConnector::mod_entry(string table, string key, const unordered_map& data) +void ConfigDBConnector::mod_entry(string table, string key, const map& data) { auto& client = get_redis_client(m_db_name); string _hash = to_upper(table) + TABLE_NAME_SEPARATOR + key; @@ -121,7 +121,7 @@ void ConfigDBConnector::mod_entry(string table, string key, const unordered_map< // Returns: // Table row data in a form of dictionary {'column_key': 'value', ...} // Empty dictionary if table does not exist or entry does not exist. -unordered_map ConfigDBConnector::get_entry(string table, string key) +map ConfigDBConnector::get_entry(string table, string key) { auto& client = get_redis_client(m_db_name); string _hash = to_upper(table) + TABLE_NAME_SEPARATOR + key; @@ -169,12 +169,12 @@ vector ConfigDBConnector::get_keys(string table, bool split) // { 'row_key': {'column_key': value, ...}, ...} // or { ('l1_key', 'l2_key', ...): {'column_key': value, ...}, ...} for a multi-key table. // Empty dictionary if table does not exist. -unordered_map> ConfigDBConnector::get_table(string table) +map> ConfigDBConnector::get_table(string table) { auto& client = get_redis_client(m_db_name); string pattern = to_upper(table) + TABLE_NAME_SEPARATOR + "*"; const auto& keys = client.keys(pattern); - unordered_map> data; + map> data; for (auto& key: keys) { auto const& entry = client.hgetall(key); @@ -212,7 +212,7 @@ void ConfigDBConnector::delete_table(string table) // 'MULTI_KEY_TABLE_NAME': { ('l1_key', 'l2_key', ...) : {'column_key': 'value', ...}, ...}, // ... // } -void ConfigDBConnector::mod_config(const unordered_map>>& data) +void ConfigDBConnector::mod_config(const map>>& data) { for (auto const& it: data) { @@ -240,11 +240,11 @@ void ConfigDBConnector::mod_config(const unordered_map>> ConfigDBConnector::get_config() +map>> ConfigDBConnector::get_config() { auto& client = get_redis_client(m_db_name); auto const& keys = client.keys("*"); - unordered_map>> data; + map>> data; for (string key: keys) { size_t pos = key.find(TABLE_NAME_SEPARATOR); @@ -325,7 +325,7 @@ void ConfigDBPipeConnector::_delete_table(DBConnector& client, RedisTransactione // data: Table row data in a form of dictionary {'column_key': 'value', ...}. // Pass {} as data will create an entry with no column if not already existed. // Pass None as data will delete the entry. -void ConfigDBPipeConnector::_mod_entry(RedisTransactioner& pipe, string table, string key, const unordered_map& data) +void ConfigDBPipeConnector::_mod_entry(RedisTransactioner& pipe, string table, string key, const map& data) { string _hash = to_upper(table) + TABLE_NAME_SEPARATOR + key; if (data.empty()) @@ -350,7 +350,7 @@ void ConfigDBPipeConnector::_mod_entry(RedisTransactioner& pipe, string table, s // 'MULTI_KEY_TABLE_NAME': { ('l1_key', 'l2_key', ...) : {'column_key': 'value', ...}, ...}, // ... // } -void ConfigDBPipeConnector::mod_config(const unordered_map>>& data) +void ConfigDBPipeConnector::mod_config(const map>>& data) { auto& client = get_redis_client(m_db_name); RedisTransactioner pipe(&client); @@ -382,7 +382,7 @@ void ConfigDBPipeConnector::mod_config(const unordered_map>>& data, int64_t cursor) +int64_t ConfigDBPipeConnector::_get_config(DBConnector& client, RedisTransactioner& pipe, map>>& data, int64_t cursor) { auto const& rc = client.scan(cursor, "*", REDIS_SCAN_BATCH_SIZE); auto cur = rc.first; @@ -433,12 +433,12 @@ int64_t ConfigDBPipeConnector::_get_config(DBConnector& client, RedisTransaction return cur; } -unordered_map>> ConfigDBPipeConnector::get_config() +map>> ConfigDBPipeConnector::get_config() { auto& client = get_redis_client(m_db_name); RedisTransactioner pipe(&client); - unordered_map>> data; + map>> data; auto cur = _get_config(client, pipe, data, 0); while (cur != 0) { diff --git a/common/configdb.h b/common/configdb.h index 46d65803f..61f26be3c 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -15,14 +15,14 @@ class ConfigDBConnector : public SonicV2Connector void db_connect(std::string db_name, bool wait_for_init, bool retry_on); void connect(bool wait_for_init = true, bool retry_on = false); - void set_entry(std::string table, std::string key, const std::unordered_map& data); - void mod_entry(std::string table, std::string key, const std::unordered_map& data); - std::unordered_map get_entry(std::string table, std::string key); + void set_entry(std::string table, std::string key, const std::map& data); + void mod_entry(std::string table, std::string key, const std::map& data); + std::map get_entry(std::string table, std::string key); std::vector get_keys(std::string table, bool split = true); - std::unordered_map> get_table(std::string table); + std::map> get_table(std::string table); void delete_table(std::string table); - virtual void mod_config(const std::unordered_map>>& data); - virtual std::unordered_map>> get_config(); + virtual void mod_config(const std::map>>& data); + virtual std::map>> get_config(); std::string getKeySeparator() const; @@ -181,16 +181,16 @@ class ConfigDBPipeConnector: public ConfigDBConnector public: ConfigDBPipeConnector(bool use_unix_socket_path = false, const char *netns = ""); - void mod_config(const std::unordered_map>>& data) override; - std::unordered_map>> get_config() override; + void mod_config(const std::map>>& data) override; + std::map>> get_config() override; private: static const int64_t REDIS_SCAN_BATCH_SIZE = 30; int64_t _delete_entries(DBConnector& client, RedisTransactioner& pipe, const char *pattern, int64_t cursor); void _delete_table(DBConnector& client, RedisTransactioner& pipe, std::string table); - void _mod_entry(RedisTransactioner& pipe, std::string table, std::string key, const std::unordered_map& data); - int64_t _get_config(DBConnector& client, RedisTransactioner& pipe, std::unordered_map>>& data, int64_t cursor); + void _mod_entry(RedisTransactioner& pipe, std::string table, std::string key, const std::map& data); + int64_t _get_config(DBConnector& client, RedisTransactioner& pipe, std::map>>& data, int64_t cursor); }; #ifdef SWIG diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index 0832835c7..9819a97d9 100644 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -659,9 +659,9 @@ void DBConnector::config_set(const std::string &key, const std::string &value) RedisReply r(this, sset, REDIS_REPLY_STATUS); } -unordered_map DBConnector::hgetall(const string &key) +map DBConnector::hgetall(const string &key) { - unordered_map map; + map map; hgetall(key, std::inserter(map, map.end())); return map; } diff --git a/common/dbconnector.h b/common/dbconnector.h index 029eb67b0..9f8870d7c 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -177,7 +178,7 @@ class DBConnector : public RedisContext void del(const std::vector& keys); - std::unordered_map hgetall(const std::string &key); + std::map hgetall(const std::string &key); template void hgetall(const std::string &key, OutputIterator result); diff --git a/pyext/swsscommon.i b/pyext/swsscommon.i index 19e6e2ffe..74748a60d 100644 --- a/pyext/swsscommon.i +++ b/pyext/swsscommon.i @@ -62,6 +62,7 @@ %template(FieldValueMap) std::map; %template(VectorString) std::vector; %template(ScanResult) std::pair>; +%template(GetTableResult) std::map>; %pythoncode %{ def _FieldValueMap__get(self, key, default=None): From 19f8b231bde3ecdf18c23d947f71e43f2bbe6d7a Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Fri, 1 Jan 2021 00:07:44 +0000 Subject: [PATCH 24/25] Fix: db_connect wait for a non-empty INIT_INDICATOR --- common/configdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/configdb.cpp b/common/configdb.cpp index ed54b806a..bf8b12684 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -26,7 +26,7 @@ void ConfigDBConnector::db_connect(string db_name, bool wait_for_init, bool retr auto& client = get_redis_client(m_db_name); auto pubsub = client.pubsub(); auto initialized = client.get(INIT_INDICATOR); - if (initialized) + if (!initialized || initialized->empty()) { string pattern = "__keyspace@" + to_string(get_dbid(m_db_name)) + "__:" + INIT_INDICATOR; pubsub->psubscribe(pattern); From bfedf15fae406d4a1728fb4e5aa59af77bfbe288 Mon Sep 17 00:00:00 2001 From: Qi Luo Date: Fri, 1 Jan 2021 01:37:08 +0000 Subject: [PATCH 25/25] Implement more dynamic functions --- common/configdb.h | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/common/configdb.h b/common/configdb.h index 61f26be3c..32e578c91 100644 --- a/common/configdb.h +++ b/common/configdb.h @@ -173,6 +173,35 @@ class ConfigDBConnector : public SonicV2Connector return self.raw_to_typed(raw_data) ConfigDBConnector.get_entry = _new_ConfigDBConnector_get_entry + _old_ConfigDBConnector_get_keys = ConfigDBConnector.get_keys + def _new_ConfigDBConnector_get_keys(self, table, split=True): + keys = _old_ConfigDBConnector_get_keys(self, table, split) + ret = [] + for key in keys: + ret.append(self.deserialize_key(key)) + return ret + ConfigDBConnector.get_key = _new_ConfigDBConnector_get_keys + + _old_ConfigDBConnector_get_table = ConfigDBConnector.get_table + def _new_ConfigDBConnector_get_table(self, table): + data = _old_ConfigDBConnector_get_table(self, table) + ret = {} + for row, entry in data.items(): + entry = self.raw_to_typed(entry) + ret[self.deserialize_key(row)] = entry + return ret + ConfigDBConnector.get_table = _new_ConfigDBConnector_get_table + + _old_ConfigDBConnector_get_config = ConfigDBConnector.get_config + def _new_ConfigDBConnector_get_config(self): + data = _old_ConfigDBConnector_get_config(self) + ret = {} + for table_name, table in data.items(): + for row, entry in table.items(): + entry = self.raw_to_typed(entry) + ret.setdefault(table_name, {})[self.deserialize_key(row)] = entry + return ret + ConfigDBConnector.get_config = _new_ConfigDBConnector_get_config %} #endif