Skip to content

Commit

Permalink
Merge branch 'sonic-net:master' into test3
Browse files Browse the repository at this point in the history
  • Loading branch information
FengPan-Frank committed May 21, 2024
2 parents 3779113 + 7298cd2 commit 9c581e5
Show file tree
Hide file tree
Showing 9 changed files with 598 additions and 44 deletions.
89 changes: 76 additions & 13 deletions config/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,15 @@
from jsonpointer import JsonPointerException
from collections import OrderedDict
from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat, extract_scope
from generic_config_updater.gu_common import HOST_NAMESPACE, GenericConfigUpdaterError
from minigraph import parse_device_desc_xml, minigraph_encoder
from natsort import natsorted
from portconfig import get_child_ports
from socket import AF_INET, AF_INET6
from sonic_py_common import device_info, multi_asic
from sonic_py_common.general import getstatusoutput_noshell
from sonic_py_common.interface import get_interface_table_name, get_port_table_name, get_intf_longname
from sonic_yang_cfg_generator import SonicYangCfgDbGenerator
from utilities_common import util_base
from swsscommon import swsscommon
from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector
Expand Down Expand Up @@ -1155,25 +1157,75 @@ def validate_gre_type(ctx, _, value):
return gre_type_value
except ValueError:
raise click.UsageError("{} is not a valid GRE type".format(value))



def multi_asic_save_config(db, filename):
"""A function to save all asic's config to single file
"""
all_current_config = {}
cfgdb_clients = db.cfgdb_clients

for ns, config_db in cfgdb_clients.items():
current_config = config_db.get_config()
sonic_cfggen.FormatConverter.to_serialized(current_config)
asic_name = "localhost" if ns == DEFAULT_NAMESPACE else ns
all_current_config[asic_name] = sort_dict(current_config)
click.echo("Integrate each ASIC's config into a single JSON file {}.".format(filename))
with open(filename, 'w') as file:
json.dump(all_current_config, file, indent=4)

# Function to apply patch for a single ASIC.
def apply_patch_for_scope(scope_changes, results, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path):
scope, changes = scope_changes
# Replace localhost to DEFAULT_NAMESPACE which is db definition of Host
if scope.lower() == "localhost" or scope == "":
if scope.lower() == HOST_NAMESPACE or scope == "":
scope = multi_asic.DEFAULT_NAMESPACE
scope_for_log = scope if scope else "localhost"

scope_for_log = scope if scope else HOST_NAMESPACE
try:
# Call apply_patch with the ASIC-specific changes and predefined parameters
GenericUpdater(namespace=scope).apply_patch(jsonpatch.JsonPatch(changes), config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path)
GenericUpdater(namespace=scope).apply_patch(jsonpatch.JsonPatch(changes),
config_format,
verbose,
dry_run,
ignore_non_yang_tables,
ignore_path)
results[scope_for_log] = {"success": True, "message": "Success"}
log.log_notice(f"'apply-patch' executed successfully for {scope_for_log} by {changes}")
except Exception as e:
results[scope_for_log] = {"success": False, "message": str(e)}
log.log_error(f"'apply-patch' executed failed for {scope_for_log} by {changes} due to {str(e)}")


def validate_patch(patch):
try:
command = ["show", "runningconfiguration", "all"]
proc = subprocess.Popen(command, text=True, stdout=subprocess.PIPE)
all_running_config, returncode = proc.communicate()
if returncode:
log.log_notice(f"Fetch all runningconfiguration failed as output:{all_running_config}")
return False

# Structure validation and simulate apply patch.
all_target_config = patch.apply(json.loads(all_running_config))

# Verify target config by YANG models
target_config = all_target_config.pop(HOST_NAMESPACE) if multi_asic.is_multi_asic() else all_target_config
target_config.pop("bgpraw", None)
if not SonicYangCfgDbGenerator().validate_config_db_json(target_config):
return False

if multi_asic.is_multi_asic():
for asic in multi_asic.get_namespace_list():
target_config = all_target_config.pop(asic)
target_config.pop("bgpraw", None)
if not SonicYangCfgDbGenerator().validate_config_db_json(target_config):
return False

return True
except Exception as e:
raise GenericConfigUpdaterError(f"Validate json patch: {patch} failed due to:{e}")

# This is our main entrypoint - the main 'config' command
@click.group(cls=clicommon.AbbreviationGroup, context_settings=CONTEXT_SETTINGS)
@click.pass_context
Expand Down Expand Up @@ -1241,7 +1293,8 @@ def config(ctx):
@click.option('-y', '--yes', is_flag=True, callback=_abort_if_false,
expose_value=False, prompt='Existing files will be overwritten, continue?')
@click.argument('filename', required=False)
def save(filename):
@clicommon.pass_db
def save(db, filename):
"""Export current config DB to a file on disk.\n
<filename> : Names of configuration file(s) to save, separated by comma with no spaces in between
"""
Expand All @@ -1256,7 +1309,13 @@ def save(filename):
if filename is not None:
cfg_files = filename.split(',')

if len(cfg_files) != num_cfg_file:
# If only one filename is provided in multi-ASIC mode,
# save all ASIC configurations to that single file.
if len(cfg_files) == 1 and multi_asic.is_multi_asic():
filename = cfg_files[0]
multi_asic_save_config(db, filename)
return
elif len(cfg_files) != num_cfg_file:
click.echo("Input {} config file(s) separated by comma for multiple files ".format(num_cfg_file))
return

Expand Down Expand Up @@ -1381,6 +1440,9 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i
patch_as_json = json.loads(text)
patch = jsonpatch.JsonPatch(patch_as_json)

if not validate_patch(patch):
raise GenericConfigUpdaterError(f"Failed validating patch:{patch}")

results = {}
config_format = ConfigFormat[format.upper()]
# Initialize a dictionary to hold changes categorized by scope
Expand All @@ -1403,7 +1465,8 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i
# Empty case to force validate YANG model.
if not changes_by_scope:
asic_list = [multi_asic.DEFAULT_NAMESPACE]
asic_list.extend(multi_asic.get_namespace_list())
if multi_asic.is_multi_asic():
asic_list.extend(multi_asic.get_namespace_list())
for asic in asic_list:
changes_by_scope[asic] = []

Expand All @@ -1416,7 +1479,7 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i

if failures:
failure_messages = '\n'.join([f"- {failed_scope}: {results[failed_scope]['message']}" for failed_scope in failures])
raise Exception(f"Failed to apply patch on the following scopes:\n{failure_messages}")
raise GenericConfigUpdaterError(f"Failed to apply patch on the following scopes:\n{failure_messages}")

log.log_notice(f"Patch applied successfully for {patch}.")
click.secho("Patch applied successfully.", fg="cyan", underline=True)
Expand Down Expand Up @@ -1620,9 +1683,9 @@ def reload(db, filename, yes, load_sysinfo, no_service_restart, force, file_form
file_input = read_json_file(file)

platform = file_input.get("DEVICE_METADATA", {}).\
get("localhost", {}).get("platform")
get(HOST_NAMESPACE, {}).get("platform")
mac = file_input.get("DEVICE_METADATA", {}).\
get("localhost", {}).get("mac")
get(HOST_NAMESPACE, {}).get("mac")

if not platform or not mac:
log.log_warning("Input file does't have platform or mac. platform: {}, mac: {}"
Expand Down Expand Up @@ -1995,8 +2058,8 @@ def override_config_table(db, input_config_db, dry_run):
if multi_asic.is_multi_asic() and len(config_input):
# Golden Config will use "localhost" to represent host name
if ns == DEFAULT_NAMESPACE:
if "localhost" in config_input.keys():
ns_config_input = config_input["localhost"]
if HOST_NAMESPACE in config_input.keys():
ns_config_input = config_input[HOST_NAMESPACE]
else:
click.secho("Wrong config format! 'localhost' not found in host config! cannot override.. abort")
sys.exit(1)
Expand Down
29 changes: 12 additions & 17 deletions generic_config_updater/generic_updater.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import jsonpointer
import os
from enum import Enum
from .gu_common import GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \
from .gu_common import HOST_NAMESPACE, GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \
DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging
from .patch_sorter import StrictPatchSorter, NonStrictPatchSorter, ConfigSplitter, \
TablesWithoutYangConfigSplitter, IgnorePathsFromYangConfigSplitter
Expand All @@ -16,28 +16,26 @@ def extract_scope(path):
if not path:
raise Exception("Wrong patch with empty path.")

try:
pointer = jsonpointer.JsonPointer(path)
parts = pointer.parts
except Exception as e:
raise Exception(f"Error resolving path: '{path}' due to {e}")
pointer = jsonpointer.JsonPointer(path)
parts = pointer.parts

if not parts:
raise Exception("Wrong patch with empty path.")
raise GenericConfigUpdaterError("Wrong patch with empty path.")
if parts[0].startswith("asic"):
if not parts[0][len("asic"):].isnumeric():
raise Exception(f"Error resolving path: '{path}' due to incorrect ASIC number.")
raise GenericConfigUpdaterError(f"Error resolving path: '{path}' due to incorrect ASIC number.")
scope = parts[0]
remainder = "/" + "/".join(parts[1:])
elif parts[0] == "localhost":
scope = "localhost"
elif parts[0] == HOST_NAMESPACE:
scope = HOST_NAMESPACE
remainder = "/" + "/".join(parts[1:])
else:
scope = ""
remainder = path

return scope, remainder


class ConfigLock:
def acquire_lock(self):
# TODO: Implement ConfigLock
Expand Down Expand Up @@ -67,7 +65,7 @@ def __init__(self,
self.changeapplier = changeapplier if changeapplier is not None else ChangeApplier(namespace=self.namespace)

def apply(self, patch, sort=True):
scope = self.namespace if self.namespace else 'localhost'
scope = self.namespace if self.namespace else HOST_NAMESPACE
self.logger.log_notice(f"{scope}: Patch application starting.")
self.logger.log_notice(f"{scope}: Patch: {patch}")

Expand All @@ -84,10 +82,10 @@ def apply(self, patch, sort=True):
self.config_wrapper.validate_field_operation(old_config, target_config)

# Validate target config does not have empty tables since they do not show up in ConfigDb
self.logger.log_notice(f"{scope}: alidating target config does not have empty tables, " \
"since they do not show up in ConfigDb.")
self.logger.log_notice(f"""{scope}: validating target config does not have empty tables,
since they do not show up in ConfigDb.""")
empty_tables = self.config_wrapper.get_empty_tables(target_config)
if empty_tables: # if there are empty tables
if empty_tables: # if there are empty tables
empty_tables_txt = ", ".join(empty_tables)
raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables " \
"which is not allowed in ConfigDb. " \
Expand All @@ -105,9 +103,6 @@ def apply(self, patch, sort=True):
self.logger.log_notice(f"The {scope} patch was converted into {changes_len} " \
f"change{'s' if changes_len != 1 else ''}{':' if changes_len > 0 else '.'}")

for change in changes:
self.logger.log_notice(f" * {change}")

# Apply changes in order
self.logger.log_notice(f"{scope}: applying {changes_len} change{'s' if changes_len != 1 else ''} " \
f"in order{':' if changes_len > 0 else '.'}")
Expand Down
1 change: 1 addition & 0 deletions generic_config_updater/gu_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
SYSLOG_IDENTIFIER = "GenericConfigUpdater"
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
GCU_FIELD_OP_CONF_FILE = f"{SCRIPT_DIR}/gcu_field_operation_validators.conf.json"
HOST_NAMESPACE = "localhost"

class GenericConfigUpdaterError(Exception):
pass
Expand Down
67 changes: 59 additions & 8 deletions scripts/fabricstat
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,49 @@ class FabricIsolation(FabricStat):
print(tabulate(body, header, tablefmt='simple', stralign='right'))
return

class FabricRate(FabricStat):
def rate_print(self):
# Connect to database
self.db = multi_asic.connect_to_all_dbs_for_ns(self.namespace)
# Get the set of all fabric ports
port_keys = self.db.keys(self.db.STATE_DB, FABRIC_PORT_STATUS_TABLE_PREFIX + '*')
# Create a new dictionary. The keys are the local port values in integer format.
# Only fabric ports that have remote port data are added.
port_dict = {}
for port_key in port_keys:
port_data = self.db.get_all(self.db.STATE_DB, port_key)
port_number = int(port_key.replace("FABRIC_PORT_TABLE|PORT", ""))
port_dict.update({port_number: port_data})
# Create ordered table of fabric ports.
rxRate = 0
rxData = 0
txRate = 0
txData = 0
time = 0
local_time = ""
# RX data , Tx data , Time are for testing
asic = "asic0"
if self.namespace:
asic = self.namespace
header = ["ASIC", "Link ID", "Rx Data Mbps", "Tx Data Mbps"]
body = []
for port_number in sorted(port_dict.keys()):
port_data = port_dict[port_number]
if "OLD_RX_RATE_AVG" in port_data:
rxRate = port_data["OLD_RX_RATE_AVG"]
if "OLD_RX_DATA" in port_data:
rxData = port_data["OLD_RX_DATA"]
if "OLD_TX_RATE_AVG" in port_data:
txRate = port_data["OLD_TX_RATE_AVG"]
if "OLD_TX_DATA" in port_data:
txData = port_data["OLD_TX_DATA"]
if "LAST_TIME" in port_data:
time = int(port_data["LAST_TIME"])
local_time = datetime.fromtimestamp(time)
body.append((asic, port_number, rxRate, txRate));
click.echo()
click.echo(tabulate(body, header, tablefmt='simple', stralign='right'))

def main():
global cnstat_dir
global cnstat_fqn_file_port
Expand All @@ -415,6 +458,8 @@ Examples:
fabricstat -q -n asic0
fabricstat -c
fabricstat -c -n asic0
fabricstat -s
fabricstat -s -n asic0
fabricstat -C
fabricstat -D
""")
Expand All @@ -425,6 +470,7 @@ Examples:
parser.add_argument('-e', '--errors', action='store_true', help='Display errors')
parser.add_argument('-c','--capacity',action='store_true', help='Display fabric capacity')
parser.add_argument('-i','--isolation', action='store_true', help='Display fabric ports isolation status')
parser.add_argument('-s','--rate', action='store_true', help='Display fabric counters rate')
parser.add_argument('-C','--clear', action='store_true', help='Copy & clear fabric counters')
parser.add_argument('-D','--delete', action='store_true', help='Delete saved stats')

Expand All @@ -433,6 +479,7 @@ Examples:
reachability = args.reachability
capacity_status = args.capacity
isolation_status = args.isolation
rate = args.rate
namespace = args.namespace
errors_only = args.errors

Expand All @@ -455,17 +502,21 @@ Examples:

def nsStat(ns, errors_only):
if queue:
stat = FabricQueueStat(ns)
stat = FabricQueueStat(ns)
elif reachability:
stat = FabricReachability(ns)
stat.reachability_print()
return
stat = FabricReachability(ns)
stat.reachability_print()
return
elif isolation_status:
stat = FabricIsolation(ns)
stat.isolation_print()
return
stat = FabricIsolation(ns)
stat.isolation_print()
return
elif rate:
stat = FabricRate(ns)
stat.rate_print()
return
else:
stat = FabricPortStat(ns)
stat = FabricPortStat(ns)
cnstat_dict = stat.get_cnstat_dict()
if save_fresh_stats:
stat.save_fresh_stats()
Expand Down
10 changes: 10 additions & 0 deletions show/fabric.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,3 +74,13 @@ def queue(namespace):
if namespace is not None:
cmd += ['-n', str(namespace)]
clicommon.run_command(cmd)


@counters.command()
@multi_asic_util.multi_asic_click_option_namespace
def rate(namespace):
"""Show fabric counters rate"""
cmd = ['fabricstat', '-s']
if namespace is not None:
cmd += ['-n', str(namespace)]
clicommon.run_command(cmd)
5 changes: 5 additions & 0 deletions tests/config_save_output/all_config_db.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"localhost": {},
"asic0": {},
"asic1": {}
}
Loading

0 comments on commit 9c581e5

Please sign in to comment.