Skip to content

Commit

Permalink
[muxorch] Refactoring bulk switchover
Browse files Browse the repository at this point in the history 
 - using existing add/removeNeighbor functions to do bulker operations.
 - Created addNeighborPost and removeNeighborPost
 - Created addRoutes and removeRoutes to deal with bulk operations
 - Created enableNeighbors and disableNeighbors to handle bulk
 switchover.

Signed-off-by: Nikola Dancejic <ndancejic@microsoft.com>
  • Loading branch information
@Ndancejic
Ndancejic committed Jun 3, 2024
1 parent fc9f317 commit 411da0f
Show file tree
Hide file tree
Showing 5 changed files with 354 additions and 397 deletions.
231 changes: 130 additions & 101 deletions orchagent/muxorch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -744,20 +744,21 @@ void MuxNbrHandler::update(NextHopKey nh, sai_object_id_t tunnelId, bool add, Mu
bool MuxNbrHandler::enable(bool update_rt)
{
NeighborEntry neigh;
std::list<NeighborBulkContext> bulk_neigh_ctx_list;
std::list<MuxRouteBulkContext> bulk_route_ctx_list;
std::list<NeighborContext> neigh_ctx_list;
std::list<MuxRouteBulkContext> route_ctx_list;

auto it = neighbors_.begin();
while (it != neighbors_.end())
{
SWSS_LOG_INFO("Enabling neigh %s on %s", it->first.to_string().c_str(), alias_.c_str());

neigh = NeighborEntry(it->first, alias_);
bulk_neigh_ctx_list.push_back(NeighborBulkContext(neigh, true));
// Create neighbor context with bulk_op enabled
neigh_ctx_list.push_back(NeighborContext(neigh, true));
it++;
}

if (!gNeighOrch->createBulkNeighborEntries(bulk_neigh_ctx_list) || !gNeighOrch->flushBulkNeighborEntries(bulk_neigh_ctx_list))
if (!gNeighOrch->enableNeighbors(neigh_ctx_list))
{
return false;
}
Expand Down Expand Up @@ -804,23 +805,16 @@ bool MuxNbrHandler::enable(bool update_rt)
IpPrefix pfx = it->first.to_string();
if (update_rt)
{
bulk_route_ctx_list.push_back(MuxRouteBulkContext(pfx, false));
route_ctx_list.push_back(MuxRouteBulkContext(pfx));
}

it++;
}

if (update_rt)
{
if (!createBulkRouteEntries(bulk_route_ctx_list))
if (!removeRoutes(route_ctx_list))
{
gRouteBulker.clear();
return false;
}
gRouteBulker.flush();
if (!processBulkRouteEntries(bulk_route_ctx_list))
{
gRouteBulker.clear();
return false;
}

Expand All @@ -844,8 +838,8 @@ bool MuxNbrHandler::enable(bool update_rt)
bool MuxNbrHandler::disable(sai_object_id_t tnh)
{
NeighborEntry neigh;
std::list<NeighborBulkContext> bulk_neigh_ctx_list;
std::list<MuxRouteBulkContext> bulk_route_ctx_list;
std::list<NeighborContext> neigh_ctx_list;
std::list<MuxRouteBulkContext> route_ctx_list;

auto it = neighbors_.begin();
while (it != neighbors_.end())
Expand Down Expand Up @@ -887,32 +881,25 @@ bool MuxNbrHandler::disable(sai_object_id_t tnh)
updateTunnelRoute(nh_key, true);

IpPrefix pfx = it->first.to_string();
bulk_route_ctx_list.push_back(MuxRouteBulkContext(pfx, it->second, true));
route_ctx_list.push_back(MuxRouteBulkContext(pfx, it->second));

neigh = NeighborEntry(it->first, alias_);
bulk_neigh_ctx_list.push_back(NeighborBulkContext(neigh, false));
// Create neighbor context with bulk_op enabled
neigh_ctx_list.push_back(NeighborContext(neigh, true));

it++;
}

if (!createBulkRouteEntries(bulk_route_ctx_list))
{
gRouteBulker.clear();
return false;
}
gRouteBulker.flush();
if (!processBulkRouteEntries(bulk_route_ctx_list))
if (!addRoutes(route_ctx_list))
{
gRouteBulker.clear();
return false;
}

if (!gNeighOrch->createBulkNeighborEntries(bulk_neigh_ctx_list) || !gNeighOrch->flushBulkNeighborEntries(bulk_neigh_ctx_list))
if (!gNeighOrch->disableNeighbors(neigh_ctx_list))
{
return false;
}

gRouteBulker.clear();
return true;
}

Expand All @@ -927,13 +914,13 @@ sai_object_id_t MuxNbrHandler::getNextHopId(const NextHopKey nhKey)
return SAI_NULL_OBJECT_ID;
}

bool MuxNbrHandler::createBulkRouteEntries(std::list<MuxRouteBulkContext>& bulk_ctx_list)
bool MuxNbrHandler::addRoutes(std::list<MuxRouteBulkContext>& bulk_ctx_list)
{
int count = 0;

SWSS_LOG_INFO("Creating %d bulk route entries", (int)bulk_ctx_list.size());
sai_status_t status;
bool ret = true;

for (auto ctx = bulk_ctx_list.begin(); ctx != bulk_ctx_list.end(); ctx++)
auto ctx = bulk_ctx_list.begin();
while (ctx != bulk_ctx_list.end())
{
auto& object_statuses = ctx->object_statuses;
sai_route_entry_t route_entry;
Expand All @@ -942,111 +929,153 @@ bool MuxNbrHandler::createBulkRouteEntries(std::list<MuxRouteBulkContext>& bulk_
copy(route_entry.destination, ctx->pfx);
subnet(route_entry.destination, route_entry.destination);

SWSS_LOG_INFO("Creating route entry %s, nh %" PRIx64 ", add:%d", ctx->pfx.getIp().to_string().c_str(), ctx->nh, ctx->add);
SWSS_LOG_INFO("Adding route entry %s, nh %" PRIx64 " to bulker", ctx->pfx.getIp().to_string().c_str(), ctx->nh);

object_statuses.emplace_back();
if (ctx->add)
sai_attribute_t attr;
vector<sai_attribute_t> attrs;

attr.id = SAI_ROUTE_ENTRY_ATTR_PACKET_ACTION;
attr.value.s32 = SAI_PACKET_ACTION_FORWARD;
attrs.push_back(attr);

attr.id = SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID;
attr.value.oid = ctx->nh;
attrs.push_back(attr);

status = gRouteBulker.create_entry(&object_statuses.back(), &route_entry, (uint32_t)attrs.size(), attrs.data());
if (status == SAI_STATUS_ITEM_ALREADY_EXISTS)
{
sai_attribute_t attr;
vector<sai_attribute_t> attrs;
// entry exists in bulker, erase and continue
ctx = bulk_ctx_list.erase(ctx);
continue;
}

attr.id = SAI_ROUTE_ENTRY_ATTR_PACKET_ACTION;
attr.value.s32 = SAI_PACKET_ACTION_FORWARD;
attrs.push_back(attr);
ctx++;
}

attr.id = SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID;
attr.value.oid = ctx->nh;
attrs.push_back(attr);
gRouteBulker.flush();

sai_status_t status = gRouteBulker.create_entry(&object_statuses.back(), &route_entry, (uint32_t)attrs.size(), attrs.data());
if (status == SAI_STATUS_ITEM_ALREADY_EXISTS)
{
SWSS_LOG_ERROR("Failed to create add entry for tunnel route in bulker object, entry already exists %s,nh %" PRIx64,
ctx->pfx.getIp().to_string().c_str(), ctx->nh);
for (auto ctx = bulk_ctx_list.begin(); ctx != bulk_ctx_list.end(); ctx++)
{
auto& object_statuses = ctx->object_statuses;

if (object_statuses.empty())
{
continue;
}

auto it_status = object_statuses.begin();
status = *it_status++;

sai_route_entry_t route_entry;
route_entry.switch_id = gSwitchId;
route_entry.vr_id = gVirtualRouterId;
copy(route_entry.destination, ctx->pfx);
subnet(route_entry.destination, route_entry.destination);

if (status != SAI_STATUS_SUCCESS)
{
if (status == SAI_STATUS_ITEM_ALREADY_EXISTS) {
SWSS_LOG_INFO("Tunnel route to %s already exists", ctx->pfx.to_string().c_str());
continue;
}
SWSS_LOG_ERROR("Failed to create tunnel route %s,nh %" PRIx64 " rv:%d",
ctx->pfx.getIp().to_string().c_str(), ctx->nh, status);
ret = false;
continue;
}

if (route_entry.destination.addr_family == SAI_IP_ADDR_FAMILY_IPV4)
{
gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE);
}
else
{
sai_status_t status = gRouteBulker.remove_entry(&object_statuses.back(), &route_entry);
if (status == SAI_STATUS_ITEM_ALREADY_EXISTS)
{
SWSS_LOG_ERROR("Failed to create remove entry for tunnel route in bulker object, entry already exists %s,nh %" PRIx64,
ctx->pfx.getIp().to_string().c_str(), ctx->nh);
continue;
}
gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE);
}
count++;

SWSS_LOG_NOTICE("Created tunnel route to %s ", ctx->pfx.to_string().c_str());
}

SWSS_LOG_INFO("Successfully created %d bulk route entries", count);
return true;
gRouteBulker.clear();
return ret;
}

bool MuxNbrHandler::processBulkRouteEntries(std::list<MuxRouteBulkContext>& bulk_ctx_list)
bool MuxNbrHandler::removeRoutes(std::list<MuxRouteBulkContext>& bulk_ctx_list)
{
sai_status_t status;
bool ret = true;

auto ctx = bulk_ctx_list.begin();
while (ctx != bulk_ctx_list.end())
{
auto& object_statuses = ctx->object_statuses;
sai_route_entry_t route_entry;
route_entry.switch_id = gSwitchId;
route_entry.vr_id = gVirtualRouterId;
copy(route_entry.destination, ctx->pfx);
subnet(route_entry.destination, route_entry.destination);

SWSS_LOG_INFO("Removing route entry %s, nh %" PRIx64 "", ctx->pfx.getIp().to_string().c_str(), ctx->nh);

object_statuses.emplace_back();
status = gRouteBulker.remove_entry(&object_statuses.back(), &route_entry);
if (status == SAI_STATUS_SUCCESS)
{
// entry already removed, clear from list and continue
ctx = bulk_ctx_list.erase(ctx);
continue;
}
ctx++;
}

gRouteBulker.flush();

for (auto ctx = bulk_ctx_list.begin(); ctx != bulk_ctx_list.end(); ctx++)
{
auto& object_statuses = ctx->object_statuses;

if (object_statuses.empty())
{
continue;
}

auto it_status = object_statuses.begin();
sai_status_t status = *it_status++;
status = *it_status++;

sai_route_entry_t route_entry;
route_entry.switch_id = gSwitchId;
route_entry.vr_id = gVirtualRouterId;
copy(route_entry.destination, ctx->pfx);
subnet(route_entry.destination, route_entry.destination);

if (ctx->add)
if (status != SAI_STATUS_SUCCESS)
{
if (status != SAI_STATUS_SUCCESS)
{
if (status == SAI_STATUS_ITEM_ALREADY_EXISTS) {
SWSS_LOG_NOTICE("Tunnel route to %s already exists", ctx->pfx.to_string().c_str());
continue;
}
SWSS_LOG_ERROR("Failed to create tunnel route %s,nh %" PRIx64 " rv:%d",
ctx->pfx.getIp().to_string().c_str(), ctx->nh, status);
return false;
}

if (route_entry.destination.addr_family == SAI_IP_ADDR_FAMILY_IPV4)
{
gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE);
}
else
{
gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE);
if (status == SAI_STATUS_ITEM_NOT_FOUND) {
SWSS_LOG_INFO("Tunnel route to %s already removed", ctx->pfx.to_string().c_str());
continue;
}
SWSS_LOG_ERROR("Failed to remove tunnel route %s, rv:%d",
ctx->pfx.getIp().to_string().c_str(), status);
ret = false;
continue;
}

SWSS_LOG_NOTICE("Created tunnel route to %s ", ctx->pfx.to_string().c_str());
if (route_entry.destination.addr_family == SAI_IP_ADDR_FAMILY_IPV4)
{
gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE);
}
else
{
if (status != SAI_STATUS_SUCCESS)
{
if (status == SAI_STATUS_ITEM_NOT_FOUND) {
SWSS_LOG_NOTICE("Tunnel route to %s already removed", ctx->pfx.to_string().c_str());
continue;
}
SWSS_LOG_ERROR("Failed to remove tunnel route %s, rv:%d",
ctx->pfx.getIp().to_string().c_str(), status);
return false;
}

if (route_entry.destination.addr_family == SAI_IP_ADDR_FAMILY_IPV4)
{
gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE);
}
else
{
gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE);
}

SWSS_LOG_NOTICE("Removed tunnel route to %s ", ctx->pfx.to_string().c_str());
gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE);
}

SWSS_LOG_NOTICE("Removed tunnel route to %s ", ctx->pfx.to_string().c_str());
}
return true;

gRouteBulker.clear();
return ret;
}

void MuxNbrHandler::updateTunnelRoute(NextHopKey nh, bool add)
Expand Down
13 changes: 6 additions & 7 deletions orchagent/muxorch.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,14 @@ struct MuxRouteBulkContext
std::deque<sai_status_t> object_statuses; // Bulk statuses
IpPrefix pfx; // Route prefix
sai_object_id_t nh; // nexthop id
bool add; // add route bool

MuxRouteBulkContext(IpPrefix pfx, bool add)
: pfx(pfx), add(add)
MuxRouteBulkContext(IpPrefix pfx)
: pfx(pfx)
{
}

MuxRouteBulkContext(IpPrefix pfx, sai_object_id_t nh, bool add)
: pfx(pfx), nh(nh), add(add)
MuxRouteBulkContext(IpPrefix pfx, sai_object_id_t nh)
: pfx(pfx), nh(nh)
{
}
};
Expand Down Expand Up @@ -97,8 +96,8 @@ class MuxNbrHandler
string getAlias() const { return alias_; };

private:
bool createBulkRouteEntries(std::list<MuxRouteBulkContext>& bulk_ctx_list);
bool processBulkRouteEntries(std::list<MuxRouteBulkContext>& bulk_ctx_list);
bool removeRoutes(std::list<MuxRouteBulkContext>& bulk_ctx_list);
bool addRoutes(std::list<MuxRouteBulkContext>& bulk_ctx_list);

inline void updateTunnelRoute(NextHopKey, bool = true);

Expand Down
Loading

0 comments on commit 411da0f

Please sign in to comment.