From 9cedd8cba472785257c8fdd4b1bf605bf7b81997 Mon Sep 17 00:00:00 2001 From: Longxiang Lyu Date: Tue, 21 Jun 2022 08:34:07 +0000 Subject: [PATCH 1/6] [sonic-conffig-engine][dualtor] Support parsing `soc_ip` out of dpg Signed-off-by: Longxiang Lyu --- src/sonic-config-engine/minigraph.py | 127 ++++++++++++------ .../tests/simple-sample-graph-case.xml | 91 ++++++++----- .../tests/test_minigraph_case.py | 9 +- 3 files changed, 145 insertions(+), 82 deletions(-) diff --git a/src/sonic-config-engine/minigraph.py b/src/sonic-config-engine/minigraph.py index 2bcef2232aef..93f1aee60cad 100644 --- a/src/sonic-config-engine/minigraph.py +++ b/src/sonic-config-engine/minigraph.py @@ -185,6 +185,7 @@ def formulate_fine_grained_ecmp(version, dpg_ecmp_content, port_device_map, port fine_grained_content = {"FG_NHG_MEMBER": FG_NHG_MEMBER, "FG_NHG": FG_NHG, "NEIGH": NEIGH} return fine_grained_content + def parse_png(png, hname, dpg_ecmp_content = None): neighbors = {} devices = {} @@ -400,9 +401,9 @@ def parse_asic_png(png, asic_name, hostname): device_data['lo_addr_v6']= lo_prefix_v6 devices[name] = device_data - return (neighbors, devices, port_speeds) + def parse_loopback_intf(child): lointfs = child.find(str(QName(ns, "LoopbackIPInterfaces"))) lo_intfs = {} @@ -412,6 +413,7 @@ def parse_loopback_intf(child): lo_intfs[(intfname, ipprefix)] = {} return lo_intfs + def parse_dpg(dpg, hname): aclintfs = None mgmtintfs = None @@ -455,7 +457,7 @@ def parse_dpg(dpg, hname): ipprefix = ipintf.find(str(QName(ns, "Prefix"))).text intfs[(intfname, ipprefix)] = {} ip_intfs_map[ipprefix] = intfalias - lo_intfs = parse_loopback_intf(child) + lo_intfs = parse_loopback_intf(child) subintfs = child.find(str(QName(ns, "SubInterfaces"))) if subintfs is not None: @@ -757,7 +759,6 @@ def parse_dpg(dpg, hname): return None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None - def parse_host_loopback(dpg, hname): for child in dpg: hostname = child.find(str(QName(ns, "Hostname"))) @@ -766,6 +767,7 @@ def parse_host_loopback(dpg, hname): lo_intfs = parse_loopback_intf(child) return lo_intfs + def parse_cpg(cpg, hname, local_devices=[]): bgp_sessions = {} bgp_internal_sessions = {} @@ -891,6 +893,7 @@ def parse_meta(meta, hname): max_cores = None kube_data = {} macsec_profile = {} + redundancy_type = None device_metas = meta.find(str(QName(ns, "Devices"))) for device in device_metas.findall(str(QName(ns1, "DeviceMetadata"))): if device.find(str(QName(ns1, "Name"))).text.lower() == hname.lower(): @@ -933,7 +936,9 @@ def parse_meta(meta, hname): kube_data["ip"] = value elif name == 'MacSecProfile': macsec_profile = parse_macsec_profile(value) - return syslog_servers, dhcp_servers, dhcpv6_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, resource_type, downstream_subrole, switch_id, switch_type, max_cores, kube_data, macsec_profile + elif name == "RedundancyType": + redundancy_type = redundancy_type + return syslog_servers, dhcp_servers, dhcpv6_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, resource_type, downstream_subrole, switch_id, switch_type, max_cores, kube_data, macsec_profile, redundancy_type def parse_system_defaults(meta): @@ -1313,6 +1318,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw static_routes = {} system_defaults = {} macsec_profile = {} + redundancy_type = None hwsku_qn = QName(ns, "HwSku") hostname_qn = QName(ns, "Hostname") @@ -1343,7 +1349,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw elif child.tag == str(QName(ns, "UngDec")): (u_neighbors, u_devices, _, _, _, _, _, _) = parse_png(child, hostname, None) elif child.tag == str(QName(ns, "MetadataDeclaration")): - (syslog_servers, dhcp_servers, dhcpv6_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, resource_type, downstream_subrole, switch_id, switch_type, max_cores, kube_data, macsec_profile) = parse_meta(child, hostname) + (syslog_servers, dhcp_servers, dhcpv6_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, resource_type, downstream_subrole, switch_id, switch_type, max_cores, kube_data, macsec_profile, redundancy_type) = parse_meta(child, hostname) elif child.tag == str(QName(ns, "LinkMetadataDeclaration")): linkmetas = parse_linkmeta(child, hostname) elif child.tag == str(QName(ns, "DeviceInfos")): @@ -1567,11 +1573,6 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw if macsec_enabled and 'PrimaryKey' in macsec_profile: port['macsec'] = macsec_profile['PrimaryKey'] - # If connected to a smart cable, get the connection position - for port_name, port in ports.items(): - if port_name in mux_cable_ports: - port['mux_cable'] = "true" - # set port description if parsed from deviceinfo for port_name in port_descriptions: # ignore port not in port_config.ini @@ -1713,7 +1714,13 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw # Add src_ip and qos remapping config into TUNNEL table if tunnel_qos_remap is enabled results['TUNNEL'] = get_tunnel_entries(tunnel_intfs, tunnel_intfs_qos_remap_config, lo_intfs, system_defaults.get('tunnel_qos_remap', {}), mux_tunnel_name, peer_switch_ip) - results['MUX_CABLE'] = get_mux_cable_entries(mux_cable_ports, neighbors, devices) + active_active_ports = get_ports_in_active_active(root, devices, neighbors) + results['MUX_CABLE'] = get_mux_cable_entries(ports, mux_cable_ports, active_active_ports, neighbors, devices, redundancy_type) + + # If connected to a smart cable, get the connection position + for port_name, port in results['PORT'].items(): + if port_name in results['MUX_CABLE']: + port['mux_cable'] = "true" if static_routes: results['STATIC_ROUTE'] = static_routes @@ -1826,43 +1833,75 @@ def get_tunnel_entries(tunnel_intfs, tunnel_intfs_qos_remap_config, lo_intfs, tu return tunnels -def get_mux_cable_entries(mux_cable_ports, neighbors, devices): + +def get_ports_in_active_active(root, devices, neighbors): + """Parse out ports in active-active cable type.""" + servers = {hostname.lower(): device_data for hostname, device_data in devices.items() if device_data["type"] == "Server"} + ports_in_active_active = {} + dpg_section = root.find(str(QName(ns, "DpgDec"))) + neighbor_to_port_mapping = {neighbor["name"].lower(): port for port, neighbor in neighbors.items()} + if dpg_section is not None: + for child in dpg_section: + hostname = child.find(str(QName(ns, "Hostname"))) + if hostname is None: + continue + hostname = hostname.text.lower() + if hostname not in servers: + continue + lo_intfs = parse_loopback_intf(child) + soc_intfs = {} + for intfname, ipprefix in lo_intfs.keys(): + intfname_lower = intfname.lower() + if hostname in intfname_lower and "soc" in intfname_lower: + ipprefix = str(ipaddress.ip_network(UNICODE_TYPE(ipprefix.split("/")[0]))) + if "." in ipprefix: + soc_intfs["soc_ipv4"] = ipprefix + elif ":" in ipprefix: + soc_intfs["soc_ipv6"] = ipprefix + if hostname in neighbor_to_port_mapping and soc_intfs: + ports_in_active_active[neighbor_to_port_mapping[hostname]] = soc_intfs + return ports_in_active_active + + +def get_mux_cable_entries(ports, mux_cable_ports, active_active_ports, neighbors, devices, redundancy_type): mux_cable_table = {} + if redundancy_type: + redundancy_type = redundancy_type.lower() + + for port in ports: + is_active_active = port in active_active_ports + is_active_standby = port in mux_cable_ports + if is_active_active and is_active_standby: + print(f"Warning: skip {port} as it is defined as active-standby and actie-active", file=sys.stderr) + continue + if not (is_active_active or is_active_standby): + continue - for intf, cable_name in mux_cable_ports.items(): - if intf in neighbors: - entry = {} - neighbor = neighbors[intf]['name'] - entry['state'] = 'auto' - - if devices[neighbor]['lo_addr'] is not None: - # Always force a /32 prefix for server IPv4 loopbacks - server_ipv4_lo_addr = devices[neighbor]['lo_addr'].split("/")[0] - server_ipv4_lo_prefix = ipaddress.ip_network(UNICODE_TYPE(server_ipv4_lo_addr)) - entry['server_ipv4'] = str(server_ipv4_lo_prefix) - - if 'lo_addr_v6' in devices[neighbor] and devices[neighbor]['lo_addr_v6'] is not None: - server_ipv6_lo_addr = devices[neighbor]['lo_addr_v6'].split('/')[0] - server_ipv6_lo_prefix = ipaddress.ip_network(UNICODE_TYPE(server_ipv6_lo_addr)) - entry['server_ipv6'] = str(server_ipv6_lo_prefix) - mux_cable_table[intf] = entry - else: - print("Warning: no server IPv4 loopback found for {}, skipping mux cable table entry".format(neighbor), file=sys.stderr) + entry = {} + neighbor = neighbors[port]['name'] + entry['state'] = 'auto' - if cable_name in devices: - cable_type = devices[cable_name].get('subtype') - if cable_type is None: - continue - if cable_type in dualtor_cable_types: - mux_cable_table[intf]['cable_type'] = cable_type - if cable_type == 'active-active': - soc_ipv4 = devices[cable_name]['lo_addr'].split('/')[0] - soc_ipv4_prefix = ipaddress.ip_network(UNICODE_TYPE(soc_ipv4)) - mux_cable_table[intf]['soc_ipv4'] = str(soc_ipv4_prefix) - else: - print("Warning: skip parsing device %s for mux cable entry, cable type %s not supported" % (cable_name, cable_type), file=sys.stderr) + if devices[neighbor]['lo_addr'] is not None: + # Always force a /32 prefix for server IPv4 loopbacks + server_ipv4_lo_addr = devices[neighbor]['lo_addr'].split("/")[0] + server_ipv4_lo_prefix = ipaddress.ip_network(UNICODE_TYPE(server_ipv4_lo_addr)) + entry['server_ipv4'] = str(server_ipv4_lo_prefix) + + if 'lo_addr_v6' in devices[neighbor] and devices[neighbor]['lo_addr_v6'] is not None: + server_ipv6_lo_addr = devices[neighbor]['lo_addr_v6'].split('/')[0] + server_ipv6_lo_prefix = ipaddress.ip_network(UNICODE_TYPE(server_ipv6_lo_addr)) + entry['server_ipv6'] = str(server_ipv6_lo_prefix) else: - print("Warning: skip parsing device %s for mux cable entry, device definition not found" % cable_name, file=sys.stderr) + print("Warning: no server IPv4 loopback found for {}, skipping mux cable table entry".format(neighbor), file=sys.stderr) + + if is_active_active: + if redundancy_type is None or redundancy_type not in ("libra", "mixed"): + print(f"Warning: skip {port} as it is defined as active-active but with redundancy_type as {redundancy_type}", file=sys.stderr) + # continue + entry['cable_type'] = 'active-active' + entry.update(active_active_ports[port]) + + mux_cable_table[port] = entry return mux_cable_table diff --git a/src/sonic-config-engine/tests/simple-sample-graph-case.xml b/src/sonic-config-engine/tests/simple-sample-graph-case.xml index 7bbef28aaa38..4165647a9aa3 100644 --- a/src/sonic-config-engine/tests/simple-sample-graph-case.xml +++ b/src/sonic-config-engine/tests/simple-sample-graph-case.xml @@ -198,6 +198,62 @@ + + + + + LoopbackInterface + HostIP + Loopback0 + + 10.10.10.2/32 + + 10.10.10.2/32 + + + LoopbackInterface + HostIP1 + Loopback0 + + fe80::0002/128 + + fe80::0002/128 + + + LoopbackInterface + SoCHostIP0 + server2SOC + + 10.10.10.3/32 + + 10.10.10.3/32 + + + LoopbackInterface + SoCHostIP1 + server2SOC + + fe80::0003/128 + + fe80::0003/128 + + + + + + + + server2 + + + + + + + + + + @@ -262,17 +318,6 @@ L true - - LogicalLink - 10000 - false - switch-t0 - fortyGigE0/8 - true - server2-SC - U - true - LogicalLink 0 @@ -349,25 +394,6 @@ server1 server-sku - - SmartCable - active-active -
- 10.10.10.3/32 -
- - ::/0 - - - 0.0.0.0/0 - - - ::/0 - - - server2-SC - smartcable-sku - Server
@@ -506,6 +532,11 @@ Storage + + RedundancyType + + Mixed + diff --git a/src/sonic-config-engine/tests/test_minigraph_case.py b/src/sonic-config-engine/tests/test_minigraph_case.py index 16ad019032f2..bfee76c7546e 100644 --- a/src/sonic-config-engine/tests/test_minigraph_case.py +++ b/src/sonic-config-engine/tests/test_minigraph_case.py @@ -236,14 +236,6 @@ def test_minigraph_neighbor_metadata(self): 'lo_addr_v6': '::/0', 'mgmt_addr': '0.0.0.0/0', 'type': 'SmartCable' - }, - 'server2-SC': { - 'hwsku': 'smartcable-sku', - 'lo_addr': '10.10.10.3/32', - 'lo_addr_v6': '::/0', - 'mgmt_addr': '0.0.0.0/0', - 'type': 'SmartCable', - 'subtype': 'active-active' } } output = self.run_script(argument) @@ -421,6 +413,7 @@ def test_minigraph_mux_cable_table(self): 'server_ipv4': '10.10.10.2/32', 'server_ipv6': 'fe80::2/128', 'soc_ipv4': '10.10.10.3/32', + 'soc_ipv6': 'fe80::3/128', 'cable_type': 'active-active' } } From 696a8ebda2b2e14e81ae03394b3c9663f2fdacca Mon Sep 17 00:00:00 2001 From: Longxiang Lyu Date: Tue, 21 Jun 2022 08:45:49 +0000 Subject: [PATCH 2/6] Modify soc check Signed-off-by: Longxiang Lyu --- src/sonic-config-engine/minigraph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-config-engine/minigraph.py b/src/sonic-config-engine/minigraph.py index 93f1aee60cad..9c19332d2b47 100644 --- a/src/sonic-config-engine/minigraph.py +++ b/src/sonic-config-engine/minigraph.py @@ -1852,7 +1852,7 @@ def get_ports_in_active_active(root, devices, neighbors): soc_intfs = {} for intfname, ipprefix in lo_intfs.keys(): intfname_lower = intfname.lower() - if hostname in intfname_lower and "soc" in intfname_lower: + if hostname + "soc" == intfname_lower: ipprefix = str(ipaddress.ip_network(UNICODE_TYPE(ipprefix.split("/")[0]))) if "." in ipprefix: soc_intfs["soc_ipv4"] = ipprefix From 082e26f54ac8feb547b65b08c30f481ce87e63ec Mon Sep 17 00:00:00 2001 From: Longxiang Lyu Date: Wed, 22 Jun 2022 00:53:00 +0000 Subject: [PATCH 3/6] Fix `redundancy_type` assignment Signed-off-by: Longxiang Lyu --- src/sonic-config-engine/minigraph.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sonic-config-engine/minigraph.py b/src/sonic-config-engine/minigraph.py index 9c19332d2b47..7511728dd9c6 100644 --- a/src/sonic-config-engine/minigraph.py +++ b/src/sonic-config-engine/minigraph.py @@ -938,6 +938,7 @@ def parse_meta(meta, hname): macsec_profile = parse_macsec_profile(value) elif name == "RedundancyType": redundancy_type = redundancy_type + redundancy_type = value return syslog_servers, dhcp_servers, dhcpv6_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, resource_type, downstream_subrole, switch_id, switch_type, max_cores, kube_data, macsec_profile, redundancy_type From 921c671c9bc53152831b10e4cf0d72966fb1fd61 Mon Sep 17 00:00:00 2001 From: Longxiang Lyu Date: Wed, 22 Jun 2022 05:30:10 +0000 Subject: [PATCH 4/6] Use %s to format string Signed-off-by: Longxiang Lyu --- src/sonic-config-engine/minigraph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sonic-config-engine/minigraph.py b/src/sonic-config-engine/minigraph.py index 7511728dd9c6..852d76d3df41 100644 --- a/src/sonic-config-engine/minigraph.py +++ b/src/sonic-config-engine/minigraph.py @@ -1873,7 +1873,7 @@ def get_mux_cable_entries(ports, mux_cable_ports, active_active_ports, neighbors is_active_active = port in active_active_ports is_active_standby = port in mux_cable_ports if is_active_active and is_active_standby: - print(f"Warning: skip {port} as it is defined as active-standby and actie-active", file=sys.stderr) + print("Warning: skip %s as it is defined as active-standby and actie-active" % port, file=sys.stderr) continue if not (is_active_active or is_active_standby): continue @@ -1897,7 +1897,7 @@ def get_mux_cable_entries(ports, mux_cable_ports, active_active_ports, neighbors if is_active_active: if redundancy_type is None or redundancy_type not in ("libra", "mixed"): - print(f"Warning: skip {port} as it is defined as active-active but with redundancy_type as {redundancy_type}", file=sys.stderr) + print("Warning: skip %s as it is defined as active-active but with redundancy_type as %s" % (port, redundancy_type), file=sys.stderr) # continue entry['cable_type'] = 'active-active' entry.update(active_active_ports[port]) From 0245672501b20883d48e9c6bc9093ab7d7fc86e2 Mon Sep 17 00:00:00 2001 From: Longxiang Lyu Date: Tue, 5 Jul 2022 01:22:51 +0000 Subject: [PATCH 5/6] Improve code logic Signed-off-by: Longxiang Lyu --- src/sonic-config-engine/minigraph.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/sonic-config-engine/minigraph.py b/src/sonic-config-engine/minigraph.py index 852d76d3df41..20fc381c3b6d 100644 --- a/src/sonic-config-engine/minigraph.py +++ b/src/sonic-config-engine/minigraph.py @@ -1870,7 +1870,7 @@ def get_mux_cable_entries(ports, mux_cable_ports, active_active_ports, neighbors redundancy_type = redundancy_type.lower() for port in ports: - is_active_active = port in active_active_ports + is_active_active = redundancy_type in ("libra", "mixed") and port in active_active_ports is_active_standby = port in mux_cable_ports if is_active_active and is_active_standby: print("Warning: skip %s as it is defined as active-standby and actie-active" % port, file=sys.stderr) @@ -1892,20 +1892,18 @@ def get_mux_cable_entries(ports, mux_cable_ports, active_active_ports, neighbors server_ipv6_lo_addr = devices[neighbor]['lo_addr_v6'].split('/')[0] server_ipv6_lo_prefix = ipaddress.ip_network(UNICODE_TYPE(server_ipv6_lo_addr)) entry['server_ipv6'] = str(server_ipv6_lo_prefix) - else: - print("Warning: no server IPv4 loopback found for {}, skipping mux cable table entry".format(neighbor), file=sys.stderr) - if is_active_active: - if redundancy_type is None or redundancy_type not in ("libra", "mixed"): - print("Warning: skip %s as it is defined as active-active but with redundancy_type as %s" % (port, redundancy_type), file=sys.stderr) - # continue - entry['cable_type'] = 'active-active' - entry.update(active_active_ports[port]) + if is_active_active: + entry['cable_type'] = 'active-active' + entry.update(active_active_ports[port]) - mux_cable_table[port] = entry + mux_cable_table[port] = entry + else: + print("Warning: no server IPv4 loopback found for {}, skipping mux cable table entry".format(neighbor), file=sys.stderr) return mux_cable_table + def parse_device_desc_xml(filename): root = ET.parse(filename).getroot() (lo_prefix, lo_prefix_v6, mgmt_prefix, mgmt_prefix_v6, hostname, hwsku, d_type, _, _, _) = parse_device(root) From 494941be31a46af207d65850aadf8df855a99ff2 Mon Sep 17 00:00:00 2001 From: Longxiang Lyu Date: Tue, 5 Jul 2022 02:31:26 +0000 Subject: [PATCH 6/6] Fix lgtm Signed-off-by: Longxiang Lyu --- src/sonic-config-engine/minigraph.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sonic-config-engine/minigraph.py b/src/sonic-config-engine/minigraph.py index 20fc381c3b6d..b79ffff039c6 100644 --- a/src/sonic-config-engine/minigraph.py +++ b/src/sonic-config-engine/minigraph.py @@ -937,7 +937,6 @@ def parse_meta(meta, hname): elif name == 'MacSecProfile': macsec_profile = parse_macsec_profile(value) elif name == "RedundancyType": - redundancy_type = redundancy_type redundancy_type = value return syslog_servers, dhcp_servers, dhcpv6_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, resource_type, downstream_subrole, switch_id, switch_type, max_cores, kube_data, macsec_profile, redundancy_type