From ac3ede5cd626bfa722cd2f4f635598af3d9b6d6a Mon Sep 17 00:00:00 2001 From: Kamil Cudnik Date: Fri, 18 Nov 2016 11:33:30 -0800 Subject: [PATCH] Add support for scheduler groups and trap group workaround (#127) * Add support for scheduler groups and trap group workaround * Apply default routes first on syncd reinit * Address comments * Address comments --- meta/sai_meta.cpp | 2 + syncd/syncd.cpp | 10 ++- syncd/syncd.h | 1 + syncd/syncd_hard_reinit.cpp | 162 ++++++++++++++++++++++++++++++------ syncd/syncd_reinit.cpp | 153 +++++++++++++++++++++++++++++++++- 5 files changed, 301 insertions(+), 27 deletions(-) diff --git a/meta/sai_meta.cpp b/meta/sai_meta.cpp index ab31b0e25a0b..a6c872d9e971 100644 --- a/meta/sai_meta.cpp +++ b/meta/sai_meta.cpp @@ -327,6 +327,8 @@ class SaiAttrWrapper ~SaiAttrWrapper() { + SWSS_LOG_ENTER(); + sai_deserialize_free_attribute_value(m_meta->serializationtype, m_attr); } diff --git a/syncd/syncd.cpp b/syncd/syncd.cpp index 0078ac49542d..22d882149f1f 100644 --- a/syncd/syncd.cpp +++ b/syncd/syncd.cpp @@ -1324,13 +1324,19 @@ int main(int argc, char **argv) gProfileMap[SAI_KEY_BOOT_TYPE] = std::to_string(options.startType); - sai_api_initialize(0, (service_method_table_t*)&test_services); + sai_status_t status = sai_api_initialize(0, (service_method_table_t*)&test_services); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("fail to sai_api_initialize: %d", status); + exit_and_notify(EXIT_FAILURE); + } populate_sai_apis(); initialize_common_api_pointers(); - sai_status_t status = sai_switch_api->initialize_switch(0, "", "", &switch_notifications); + status = sai_switch_api->initialize_switch(0, "", "", &switch_notifications); if (status != SAI_STATUS_SUCCESS) { diff --git a/syncd/syncd.h b/syncd/syncd.h index a5ee01328002..58020a5a79ce 100644 --- a/syncd/syncd.h +++ b/syncd/syncd.h @@ -63,6 +63,7 @@ extern void exit_and_notify(int status) __attribute__ ((__noreturn__)); extern std::mutex g_mutex; extern std::set g_defaultPriorityGroupsRids; +extern std::set g_defaultSchedulerGroupsRids; extern std::set g_defaultQueuesRids; void onSyncdStart(bool warmStart); diff --git a/syncd/syncd_hard_reinit.cpp b/syncd/syncd_hard_reinit.cpp index 25a78abb232f..081b11da2cf7 100644 --- a/syncd/syncd_hard_reinit.cpp +++ b/syncd/syncd_hard_reinit.cpp @@ -28,7 +28,7 @@ void processVlans(); void processNeighbors(); void processOids(); void processFdbs(); -void processRoutes(); +void processRoutes(bool defaultOnly); void processTraps(); sai_object_type_t getObjectTypeFromAsicKey(const std::string &key); @@ -266,8 +266,9 @@ void hardReinit() processFdbs(); processNeighbors(); processOids(); - processRoutes(); processTraps(); + processRoutes(true); + processRoutes(false); checkAllIds(); } @@ -300,6 +301,82 @@ bool shouldSkipCreateion( return false; } +void trapGroupWorkaround( + sai_object_id_t vid, + sai_object_id_t& rid, + bool& createObject, + uint32_t attrCount, + const sai_attribute_t* attrList) +{ + SWSS_LOG_ENTER(); + + if (createObject) + { + createObject = false; // force to "SET" left attributes + } + else + { + // default trap group + return; + } + + sai_object_type_t objectType = SAI_OBJECT_TYPE_TRAP_GROUP; + + SWSS_LOG_INFO("creating trap group and setting attributes 1 by 1 as workaround"); + + const sai_attribute_t* queue_attr = get_attribute_by_id(SAI_HOSTIF_TRAP_GROUP_ATTR_QUEUE, attrCount, attrList); + + if (queue_attr == NULL) + { + SWSS_LOG_ERROR("missing QUEUE attribute on TRAP_GROUP creation even if it's not MANDATORY"); + + exit_and_notify(EXIT_FAILURE); + } + + create_fn create = common_create[objectType]; + + if (create == NULL) + { + SWSS_LOG_ERROR("create function is not defined for object type %llx", objectType); + + exit_and_notify(EXIT_FAILURE); + } + + sai_status_t status = create(&rid, 1, queue_attr); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("failed to create TRAP_GROUP %s", sai_serialize_status(status).c_str()); + + exit_and_notify(EXIT_FAILURE); + } + + SWSS_LOG_DEBUG("created TRAP_GROUP, processed VID %llx to RID %llx", objectType, vid, rid); +} + +void listFailedAttributes( + sai_object_type_t objectType, + uint32_t attrCount, + const sai_attribute_t* attrList) +{ + SWSS_LOG_ENTER(); + + for (uint32_t idx = 0; idx < attrCount; idx++) + { + const sai_attribute_t *attr = &attrList[idx]; + + auto meta = get_attribute_metadata(objectType, attr->id); + + if (meta == NULL) + { + SWSS_LOG_ERROR("failed to get atribute metadata %d %d", objectType, attr->id); + exit_and_notify(EXIT_FAILURE); + } + + SWSS_LOG_ERROR("%s = %s", meta->attridname, sai_serialize_attr_value(*meta, *attr).c_str()); + } +} + sai_object_id_t processSingleVid(sai_object_id_t vid) { SWSS_LOG_ENTER(); @@ -328,6 +405,25 @@ sai_object_id_t processSingleVid(sai_object_id_t vid) std::string strVid = sai_serialize_object_id(vid); + auto oit = g_oids.find(strVid); + + if (oit == g_oids.end()) + { + SWSS_LOG_ERROR("failed to find VID %s in OIDs map", strVid.c_str()); + + exit_and_notify(EXIT_FAILURE); + } + + std::string asicKey = oit->second;; + + std::shared_ptr list = g_attributesLists[asicKey]; + + processAttributesForOids(objectType, list); // recursion + + sai_attribute_t *attrList = list->get_attr_list(); + + uint32_t attrCount = list->get_attr_count(); + bool createObject = true; if (objectType == SAI_OBJECT_TYPE_VIRTUAL_ROUTER) @@ -351,12 +447,21 @@ sai_object_id_t processSingleVid(sai_object_id_t vid) SWSS_LOG_DEBUG("default priority group will not be created, processed VID %llx to RID %llx", vid, rid); } } + else if (objectType == SAI_OBJECT_TYPE_SCHEDULER_GROUP) + { + if (shouldSkipCreateion(vid, rid, createObject, [&](sai_object_id_t sgId) { return g_defaultSchedulerGroupsRids.find(sgId) != g_defaultSchedulerGroupsRids.end(); })) + { + SWSS_LOG_DEBUG("default scheduler group will not be created, processed VID %llx to RID %llx", vid, rid); + } + } else if (objectType == SAI_OBJECT_TYPE_TRAP_GROUP) { if (shouldSkipCreateion(vid, rid, createObject, [](sai_object_id_t id) { return id == redisGetDefaultTrapGroupId(); })) { SWSS_LOG_INFO("default trap group will not be created, processed VID %llx to RID %llx", vid, rid); } + + trapGroupWorkaround(vid, rid, createObject, attrCount, attrList); } else if (objectType == SAI_OBJECT_TYPE_PORT) { @@ -366,25 +471,6 @@ sai_object_id_t processSingleVid(sai_object_id_t vid) } } - auto oit = g_oids.find(strVid); - - if (oit == g_oids.end()) - { - SWSS_LOG_ERROR("failed to find VID %s in OIDs map", strVid.c_str()); - - exit_and_notify(EXIT_FAILURE); - } - - std::string asicKey = oit->second;; - - std::shared_ptr list = g_attributesLists[asicKey]; - - processAttributesForOids(objectType, list); // recursion - - sai_attribute_t *attrList = list->get_attr_list(); - - uint32_t attrCount = list->get_attr_count(); - if (createObject) { create_fn create = common_create[objectType]; @@ -402,6 +488,8 @@ sai_object_id_t processSingleVid(sai_object_id_t vid) { SWSS_LOG_ERROR("failed to create object %s: %s", sai_serialize_object_type(objectType).c_str(), sai_serialize_status(status).c_str()); + listFailedAttributes(objectType, attrCount, attrList); + exit_and_notify(EXIT_FAILURE); } @@ -421,7 +509,19 @@ sai_object_id_t processSingleVid(sai_object_id_t vid) if (status != SAI_STATUS_SUCCESS) { - SWSS_LOG_ERROR("failed to set attribute for object type %llx attr id %llx: %d", objectType, attr->id, status); + auto meta = get_attribute_metadata(objectType, attr->id); + + if (meta == NULL) + { + SWSS_LOG_ERROR("failed to get atribute metadata %d %d", objectType, attr->id); + exit_and_notify(EXIT_FAILURE); + } + + SWSS_LOG_ERROR( + "failed to set %s value %s: %s", + meta->attridname, + sai_serialize_attr_value(*meta, *attr).c_str(), + sai_serialize_status(status).c_str()); exit_and_notify(EXIT_FAILURE); } @@ -717,15 +817,24 @@ sai_unicast_route_entry_t getRouteEntryFromString(const std::string &strRouteEnt return routeEntry; } -void processRoutes() +void processRoutes(bool defaultOnly) { SWSS_LOG_ENTER(); + SWSS_LOG_TIMER("apply routes"); + for (auto &kv: g_routes) { const std::string &strRouteEntry = kv.first; const std::string &asicKey = kv.second; + bool isDefault = strRouteEntry.find("/0") != std::string::npos; + + if (defaultOnly ^ isDefault) + { + continue; + } + sai_unicast_route_entry_t routeEntry = getRouteEntryFromString(strRouteEntry); routeEntry.vr_id = processSingleVid(routeEntry.vr_id); @@ -742,7 +851,12 @@ void processRoutes() if (status != SAI_STATUS_SUCCESS) { - SWSS_LOG_ERROR("failed to create_route_entry: %d", status); + SWSS_LOG_ERROR( + "failed to create ROUTE %s: %s", + sai_serialize_route_entry(routeEntry).c_str(), + sai_serialize_status(status).c_str()); + + listFailedAttributes(SAI_OBJECT_TYPE_ROUTE, attrCount, attrList); exit_and_notify(EXIT_FAILURE); } diff --git a/syncd/syncd_reinit.cpp b/syncd/syncd_reinit.cpp index d991a247a57e..8863357b84d8 100644 --- a/syncd/syncd_reinit.cpp +++ b/syncd/syncd_reinit.cpp @@ -691,7 +691,7 @@ std::vector saiGetPortPriorityGroups(sai_object_id_t portId) { SWSS_LOG_ENTER(); - uint32_t pgCount = saiGetPortNumberOfPriorityGroups(portId); + uint32_t pgCount = saiGetPortNumberOfPriorityGroups(portId); std::vector pgList; @@ -748,6 +748,155 @@ void helperCheckPriorityGroupsIds() } } +uint32_t saiGetMaxNumberOfChildsPerSchedulerGroup() +{ + SWSS_LOG_ENTER(); + + sai_attribute_t attr; + + attr.id = SAI_SWITCH_ATTR_QOS_MAX_NUMBER_OF_CHILDS_PER_SCHEDULER_GROUP; + + sai_status_t status = sai_switch_api->get_switch_attribute(1, &attr); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("faled to obtain SAI_SWITCH_ATTR_QOS_MAX_NUMBER_OF_CHILDS_PER_SCHEDULER_GROUP"); + + exit_and_notify(EXIT_FAILURE); + } + + return attr.value.u32; +} + +uint32_t saiGetQosNumberOfSchedulerGroups(sai_object_id_t portId) +{ + SWSS_LOG_ENTER(); + + sai_attribute_t attr; + + attr.id = SAI_PORT_ATTR_QOS_NUMBER_OF_SCHEDULER_GROUPS; + + sai_status_t status = sai_port_api->get_port_attribute(portId, 1, &attr); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to get number of scheduler groups for port: 0x%llx", portId); + + exit_and_notify(EXIT_FAILURE); + } + + // total groups list on the port + return attr.value.u32; +} + +std::vector saiGetSchedulerGroupList(sai_object_id_t portId) +{ + SWSS_LOG_ENTER(); + + uint32_t groupsCount = saiGetQosNumberOfSchedulerGroups(portId); + + std::vector groups; + + groups.resize(groupsCount); + + sai_attribute_t attr; + + attr.id = SAI_PORT_ATTR_QOS_SCHEDULER_GROUP_LIST; + attr.value.objlist.list = groups.data(); + attr.value.objlist.count = (uint32_t)groups.size(); + + sai_status_t status = sai_port_api->get_port_attribute(portId, 1, &attr); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to get scheduler group list for port: 0x%llx", portId); + + exit_and_notify(EXIT_FAILURE); + } + + // just in case + groups.resize(attr.value.objlist.count); + + SWSS_LOG_DEBUG("got %zu scheduler groups on port 0x%llx", groups.size(), portId); + + return groups; +} + +uint32_t saiGetSchedulerGroupChildCount(sai_object_id_t schedulerGroupId) +{ + SWSS_LOG_ENTER(); + + sai_attribute_t attr; + + attr.id = SAI_SCHEDULER_GROUP_ATTR_CHILD_COUNT; // queues + sched groups + + sai_status_t status = sai_scheduler_group_api->get_scheduler_group_attribute(schedulerGroupId, 1, &attr); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to get child count for scheduler group: 0x%llx", schedulerGroupId); + + exit_and_notify(EXIT_FAILURE); + } + + return attr.value.u32; +} + +std::vector saiGetSchedulerGroupChildList(sai_object_id_t schedulerGroupId) +{ + SWSS_LOG_ENTER(); + + uint32_t childCount = saiGetSchedulerGroupChildCount(schedulerGroupId); + + std::vector childs; + + childs.resize(childCount); + + sai_attribute_t attr; + + attr.id = SAI_SCHEDULER_GROUP_ATTR_CHILD_LIST; + attr.value.objlist.list = childs.data(); + attr.value.objlist.count = (uint32_t)childs.size(); + + sai_status_t status = sai_scheduler_group_api->get_scheduler_group_attribute(schedulerGroupId, 1, &attr); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to get child list for scheduler group: 0x%llx", schedulerGroupId); + + exit_and_notify(EXIT_FAILURE); + } + + childs.resize(attr.value.objlist.count); + + SWSS_LOG_DEBUG("got %zu childs on scheduler group 0x%llx", childs.size(), schedulerGroupId); + + return childs; // scheduler groups + queues +} + +std::set g_defaultSchedulerGroupsRids; + +void helperCheckSchedulerGroupsIds() +{ + SWSS_LOG_ENTER(); + + for (const auto& portId: saiGetPortList()) + { + // each group can contain next scheduler group or queue + + for (const auto& schedGroupId: saiGetSchedulerGroupList(portId)) + { + // create entry in asic view if missing + // we assume here that SchedulerGroups numbers will + // not be changed during restarts + + redisCreateDummyEntryInAsicView(schedGroupId); + + g_defaultSchedulerGroupsRids.insert(schedGroupId); + } + } +} + void onSyncdStart(bool warmStart) { // it may happen that after initialize we will receive @@ -779,6 +928,8 @@ void onSyncdStart(bool warmStart) helperCheckPriorityGroupsIds(); + helperCheckSchedulerGroupsIds(); + if (warmStart) { SWSS_LOG_NOTICE("skipping hard reinit since WARM start was performed");