[yang-models]: Add yang extension to control translation configDB<->yang from YANG models.

src/sonic-yang-mgmt/_sonic_yang_ext.py

@@ -19,11 +19,15 @@
 def loadYangModel(self):
 
     try:
-        yangDir = self.yang_dir
-        self.yangFiles = glob(yangDir +"/*.yang")
+        # get all files
+        self.yangFiles = glob(self.yang_dir +"/*.yang")
+        # load yang modules
         for file in self.yangFiles:
-            if (self.load_schema_module(file) == False):
-                return False
+            m = self.load_schema_module(file)
+            if m is not None:
+                self.logInFile("module: {} is loaded successfully".format(m.name()))
+            else:
+                raise(Exception("Could not load module {}".format(file)))
 
         # keep only modules name in self.yangFiles
         self.yangFiles = [f.split('/')[-1] for f in self.yangFiles]
@@ -53,6 +57,7 @@ def loadJsonYangModel(self):
             if m is not None:
                 xml = m.print_mem(ly.LYD_JSON, ly.LYP_FORMAT)
                 self.yJson.append(parse(xml))
+                self.logInFile("Parsed Json for {}".format(m.name()))
     except Exception as e:
         print('JSON conversion for yang models failed')
         raise e
@@ -70,7 +75,8 @@ def createDBTableToModuleMap(self):
     for j in self.yJson:
         # get module name
         moduleName = j['module']['@name']
-        if "sonic-head" in moduleName or "sonic-common" in moduleName:
+        # topLevelContainer does not exist in sonic-head and sonic-extension.
+        if "sonic-head" in moduleName or "sonic-extension" in moduleName:
             continue;
         # get all top level container
         topLevelContainer = j['module']['container']
@@ -129,31 +135,28 @@ def cropConfigDB(self, croppedFile=None, allowExtraTables=True):
 
 """
 Extract keys from table entry in Config DB and return in a dict
-For Example: regex = <vlan_name>| and tableKey = "Vlan111|2a04:5555:45:6709::1/64"
 
-1.) first code will extract key list from regex, i.e. vlan_name and ip_prefix.
-2.) then will create another regex(regexV) to extract Values from tableKey by
-    replacing " --> extractor i.e. (.*?)" in regex.
-3.) Then will extract values from tableKey with regexV.
-4.) Resulting Dict will be:
+Input:
+tableKey: Config DB Primary Key, Example tableKey = "Vlan111|2a04:5555:45:6709::1/64"
+keys: key string from YANG list, i.e. 'vlan_name ip-prefix'.
+regex: A regex to extract keys from tableKeys, good to have it as accurate as possible.
+
+Return:
 KeyDict = {"vlan_name": "Vlan111", "ip-prefix": "2a04:5555:45:6709::1/64"}
 """
-def extractKey(self, tableKey, regex):
+def extractKey(self, tableKey, keys, regex):
 
-    # get the keys from regex of key extractor
-    keyList = re.findall(r'<(.*?)>', regex)
-    # create a regex to get values from tableKey
-    # and change separator to text in regexV
-    regexV = re.sub('<.*?>', '(.*?)', regex)
-    regexV = re.sub('\|', '\\|', regexV)
+    keyList = keys.split()
+    #self.logInFile("extractKey {}".format(keyList))
     # get the value groups
-    value = re.match(r'^'+regexV+'$', tableKey)
+    value = re.match(regex, tableKey)
     # create the keyDict
     i = 1
     keyDict = dict()
     for k in keyList:
         if value.group(i):
             keyDict[k] = value.group(i)
+            # self.logInFile("extractKey {} {}".format(k, keyDict[k]))
         else:
             raise Exception("Value not found for {} in {}".format(k, tableKey))
         i = i + 1
@@ -251,40 +254,40 @@ def yangConvert(val):
 """
 def xlateList(self, model, yang, config, table):
 
-    # TODO: define a keyExt dict as of now, but we should be able to extract
-    # this from YANG model extentions.
-    keyExt = {
-        "VLAN_INTERFACE": "<vlan_name>|<ip-prefix>",
-        "ACL_RULE": "<ACL_TABLE_NAME>|<RULE_NAME>",
-        "VLAN": "<vlan_name>",
-        "VLAN_MEMBER": "<vlan_name>|<port>",
-        "ACL_TABLE": "<ACL_TABLE_NAME>",
-        "INTERFACE": "<interface>|<ip-prefix>",
-        "PORT": "<port_name>"
-    }
     #create a dict to map each key under primary key with a dict yang model.
     #This is done to improve performance of mapping from values of TABLEs in
     #config DB to leaf in YANG LIST.
 
     leafDict = self.createLeafDict(model)
 
-    self.logInFile("Xlate {}".format(table))
-    # Find and extracts key from each dict in config
-    for pkey in config:
+    # fetch regex from YANG models.
+    keyRegEx = model['ext:key-regex-configdb-to-yang']['@value']
+    # seperator `|` has special meaning in regex, so change it appropriately.
+    keyRegEx = re.sub('\|', '\\|', keyRegEx)
+
+    # get keys from YANG model list itself
+    listKeys = model['key']['@value']
+
+    for pkey in config.keys():
         try:
             vKey = None
-            self.logInFile("xlate Extract pkey {} {}".format(pkey,keyExt[table]))
-            keyDict = self.extractKey(pkey, keyExt[table])
+            self.logInFile("xlateList Extract pkey:{} regex:{} keyList:{}\
+                ".format(pkey, keyRegEx, listKeys))
+            # Find and extracts key from each dict in config
+            keyDict = self.extractKey(pkey, listKeys, keyRegEx)
             # fill rest of the values in keyDict
             for vKey in config[pkey]:
-                self.logInFile("xlate vkey {}".format(vKey), keyExt[table])
+                self.logInFile("xlateList vkey {}".format(vKey))
                 keyDict[vKey] = self.findYangTypedValue(vKey, \
                                     config[pkey][vKey], leafDict)
             yang.append(keyDict)
+            # delete pkey from config, done to match one key with one list
+            del config[pkey]
+
         except Exception as e:
-            print("Exception while Config DB --> YANG: pkey:{}, "\
-            "vKey:{}, value: {}".format(pkey, vKey, config[pkey].get(vKey)))
-            raise e
+            self.logInFile("xlateList Exception {}".format(e))
+            # with multilist, we continue matching other keys.
+            continue
 
     return
 
@@ -295,19 +298,37 @@ def xlateList(self, model, yang, config, table):
 """
 def xlateContainer(self, model, yang, config, table):
 
-    # if container contains single list with containerName_LIST and
-    # config is not empty then xLate the list
+    # To Handle multiple List, Make a copy of config, because we delete keys
+    # from config after each match. This is done to match one pkey with one list.
+    configC = config.copy()
+
     clist = model.get('list')
+    # If single list exists in container,
     if clist and isinstance(clist, dict) and \
-       clist['@name'] == model['@name']+"_LIST" and bool(config):
+       clist['@name'] == model['@name']+"_LIST" and bool(configC):
             #print(clist['@name'])
             yang[clist['@name']] = list()
-            self.xlateList(model['list'], yang[clist['@name']], \
-                           config, table)
+            self.logInFile("xlateContainer listD {}".format(clist['@name']))
+            self.xlateList(clist, yang[clist['@name']], \
+                           configC, table)
+            # clean empty lists
+            if len(yang[clist['@name']]) == 0:
+                del yang[clist['@name']]
             #print(yang[clist['@name']])
 
-    # TODO: Handle mupltiple list and rest of the field in Container.
-    # We do not have any such instance in Yang model today.
+    # If multi-list exists in container,
+    elif clist and isinstance(clist, list) and bool(configC):
+        for modelList in clist:
+            yang[modelList['@name']] = list()
+            self.logInFile("xlateContainer listL {}".format(modelList['@name']))
+            self.xlateList(modelList, yang[modelList['@name']], configC, table)
+            # clean empty lists
+            if len(yang[modelList['@name']]) == 0:
+                del yang[modelList['@name']]
+
+    if len(configC):
+        self.logInFile("Alert: Remaining keys in Config", obj=configC, json=True)
+        raise(Exception("All Keys are not parsed in {}".format(table)))
 
     return
 
@@ -325,6 +346,7 @@ def xlateConfigDBtoYang(self, jIn, yangJ):
         # Add new top level container for first table in this container
         yangJ[key] = dict() if yangJ.get(key) is None else yangJ[key]
         yangJ[key][subkey] = dict()
+        self.logInFile("xlateConfigDBtoYang {}:{}".format(key, subkey))
         self.xlateContainer(cmap['container'], yangJ[key][subkey], \
                             jIn[table], table)
 
@@ -389,35 +411,26 @@ def revYangConvert(val):
 
     return vValue
 
-
 """
 Rev xlate from <TABLE>_LIST to table in config DB
 """
 def revXlateList(self, model, yang, config, table):
 
-    # TODO: define a keyExt dict as of now, but we should be able to
-    # extract this from YANG model extentions.
-    keyExt = {
-        "VLAN_INTERFACE": "<vlan_name>|<ip-prefix>",
-        "ACL_RULE": "<ACL_TABLE_NAME>|<RULE_NAME>",
-        "VLAN": "<vlan_name>",
-        "VLAN_MEMBER": "<vlan_name>|<port>",
-        "ACL_TABLE": "<ACL_TABLE_NAME>",
-        "INTERFACE": "<interface>|<ip-prefix>",
-        "PORT": "<port_name>"
-    }
+    # fetch regex from YANG models
+    keyRegEx = model['ext:key-regex-yang-to-configdb']['@value']
 
     # create a dict to map each key under primary key with a dict yang model.
     # This is done to improve performance of mapping from values of TABLEs in
     # config DB to leaf in YANG LIST.
     leafDict = self.createLeafDict(model)
 
-    # list with name <TABLE>_LIST should be removed,
-    # right now we have only this instance of LIST
-    if model['@name'] == table + "_LIST":
+    # list with name <NAME>_LIST should be removed,
+    if "_LIST" in model['@name']:
         for entry in yang:
             # create key of config DB table
-            pkey, pkeydict = self.createKey(entry, keyExt[table])
+            self.logInFile("revXlateList regex:{}".format(keyRegEx))
+            pkey, pkeydict = self.createKey(entry, keyRegEx)
+            self.logInFile("revXlateList pkey:{}".format(pkey))
             config[pkey]= dict()
             # fill rest of the entries
             for key in entry:
@@ -437,11 +450,12 @@ def revXlateContainer(self, model, yang, config, table):
     if isinstance(model['list'], dict):
         modelList = model['list']
         # Pass matching list from Yang Json
+        self.logInFile("revXlateContainer {}".format(modelList['@name']))
         self.revXlateList(modelList, yang[modelList['@name']], config, table)
-    else:
-        # TODO: Container[TABLE] contains multiple lists. [Test Pending]
-        # No instance now.
+
+    elif isinstance(model['list'], list):
         for modelList in model['list']:
+            self.logInFile("revXlateContainer {}".format(modelList['@name']))
             self.revXlateList(modelList, yang[modelList['@name']], config, table)
 
     return
@@ -464,6 +478,7 @@ def revXlateYangtoConfigDB(self, yangJ, cDbJson):
             cmap = self.confDbYangMap[table]
             cDbJson[table] = dict()
             #print(key + "--" + subkey)
+            self.logInFile("revXlateYangtoConfigDB {}".format(table))
             self.revXlateContainer(cmap['container'], yangJ[module_top][container], \
                 cDbJson[table], table)
 
@@ -575,10 +590,11 @@ def load_data(self, configdbJson, allowExtraTables=True):
       # reset xlate
       self.xlateJson = dict()
       # self.jIn will be cropped
-      self.cropConfigDB("cropped.json", allowExtraTables)
+      self.cropConfigDB(allowExtraTables=allowExtraTables)
       # xlated result will be in self.xlateJson
       self.xlateConfigDB()
       #print(self.xlateJson)
+      self.logInFile("Try to load Data in the tree")
       self.root = self.ctx.parse_data_mem(dumps(self.xlateJson), \
                     ly.LYD_JSON, ly.LYD_OPT_CONFIG|ly.LYD_OPT_STRICT)
 

src/sonic-yang-mgmt/sonic_yang.py

@@ -71,7 +71,7 @@ def logInFile(self, header="", obj=None, json=False):
     """
     def load_schema_module(self, yang_file):
         try:
-            self.ctx.parse_module_path(yang_file, ly.LYS_IN_YANG)
+            return self.ctx.parse_module_path(yang_file, ly.LYS_IN_YANG)
         except Exception as e:
             print("Failed to load yang module file: " + yang_file)
             self.fail(e)

src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py

@@ -7,12 +7,18 @@
 import getopt
 import subprocess
 import glob
+import logging
 from ijson import items as ijson_itmes
 
 test_path = os.path.dirname(os.path.abspath(__file__))
 modules_path = os.path.dirname(test_path)
 sys.path.insert(0, modules_path)
 
+logging.basicConfig(level=logging.DEBUG)
+log = logging.getLogger("YANG-TEST")
+log.setLevel(logging.INFO)
+log.addHandler(logging.NullHandler())
+
 class Test_SonicYang(object):
     # class vars
     yang_test_file = "/sonic/src/sonic-yang-mgmt/tests/yang-model-tests/yangTest.json"
@@ -228,17 +234,23 @@ def test_xlate_rev_xlate(self):
 
         syc.get_data()
 
-        if syc.jIn == syc.revXlateJson:
+        if syc.jIn and syc.jIn == syc.revXlateJson:
             print("Xlate and Rev Xlate Passed")
-        else:
+        elif syc.jIn :
             # Right now, interface and vlan_interface will have default diff due to ip_prefix
             from jsondiff import diff
             configDiff = diff(syc.jIn, syc.revXlateJson, syntax='symmetric')
+            log.info(configDiff)
             for key in configDiff.keys():
                 if 'INTERFACE' not in key:
                     print("Xlate and Rev Xlate failed")
-                    sys.exit(1)
+                    # lets make it fail
+                    assert True == False
             print("Xlate and Rev Xlate Passed")
+        else:
+            print("Xlate and Rev Xlate failed")
+            # lets make it fail
+            assert True == False
 
         return
 

src/sonic-yang-mgmt/tests/yang-model-tests/yangModelTesting.py

@@ -7,6 +7,7 @@
 import ijson
 import json
 #import sonic_yang as sy
+from glob import glob
 from os import listdir
 from os.path import isfile, join, splitext
 
@@ -134,27 +135,21 @@ def __init__(self, tests, yangDir, jsonFile):
         load all YANG models before test run
     """
     def loadYangModel(self, yangDir):
+
         try:
-            # get all files
-            yangFiles = [f for f in listdir(yangDir) if isfile(join(yangDir, f))]
-            # get all yang files
-            yangFiles = [f for f in yangFiles if splitext(f)[-1].lower()==".yang"]
-            yangFiles = [f.split('.')[0] for f in yangFiles]
-            # load yang mdoules
+            # create context
             self.ctx = ly.Context(yangDir)
-            log.debug(yangFiles)
-            for f in yangFiles:
-                # load a module
-                log.debug(f)
-                m = self.ctx.get_module(f)
+            # get all files
+            yangFiles = glob(yangDir +"/*.yang")
+            # load yang modules
+            for file in yangFiles:
+                log.debug(file)
+                m = self.ctx.parse_module_path(file, ly.LYS_IN_YANG)
                 if m is not None:
-                    log.error("Could not get module: {}".format(m.name()))
+                    log.info("module: {} is loaded successfully".format(m.name()))
                 else:
-                    m = self.ctx.load_module(f)
-                    if m is not None:
-                        log.info("module: {} is loaded successfully".format(m.name()))
-                    else:
-                        return
+                    log.info("Could not load module: {}".format(file))
+
         except Exception as e:
             printExceptionDetails()
             raise e
@@ -275,7 +270,8 @@ def main():
 
         yTest = YangModelTesting(tests, yangDir, jsonFile)
         if (listTests):
-            log.info(yTest.ExceptionTests.keys())
+            for key in yTest.ExceptionTests.keys():
+                log.info(key)
             sys.exit(0)
 
         ret = yTest.run()