Skip to content

Commit

Permalink
[sonic-package-manager] support sonic-cli-gen and packages with YANG …
Browse files Browse the repository at this point in the history
…model (#1650)

- What I did
This PR brings in support for packages with YANG models and CLI auto generation capabilities for 3rd party packages.

- How I did it
Packages can set two new flags in manifest - "auto-generate-show" and "auto-generate-config" in addition to YANG module recorded in package image label "com.azure.sonic.yang-module".

- How to verify it
Build and run. Prepare some package with YANG model and test CLI is generated for it.

Signed-off-by: Stepan Blyshchak <stepanb@nvidia.com>
Co-authored-by: Vadym Hlushko <vadymh@nvidia.com>
  • Loading branch information
stepanblyschak and vadymhlushko-mlnx authored Nov 24, 2021
1 parent 64777a4 commit f5e5a56
Show file tree
Hide file tree
Showing 9 changed files with 380 additions and 96 deletions.
110 changes: 91 additions & 19 deletions config/config_mgmt.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,13 @@
config_mgmt.py provides classes for configuration validation and for Dynamic
Port Breakout.
'''

import os
import re
import shutil
import syslog
import tempfile
import yang as ly
from json import load
from sys import flags
from time import sleep as tsleep
Expand Down Expand Up @@ -46,34 +51,38 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True):
try:
self.configdbJsonIn = None
self.configdbJsonOut = None
self.source = source
self.allowTablesWithoutYang = allowTablesWithoutYang

# logging vars
self.SYSLOG_IDENTIFIER = "ConfigMgmt"
self.DEBUG = debug

self.sy = sonic_yang.SonicYang(YANG_DIR, debug=debug)
# load yang models
self.sy.loadYangModel()
# load jIn from config DB or from config DB json file.
if source.lower() == 'configdb':
self.readConfigDB()
# treat any other source as file input
else:
self.readConfigDBJson(source)
# this will crop config, xlate and load.
self.sy.loadData(self.configdbJsonIn)

# Raise if tables without YANG models are not allowed but exist.
if not allowTablesWithoutYang and len(self.sy.tablesWithOutYang):
raise Exception('Config has tables without YANG models')
self.__init_sonic_yang()

except Exception as e:
self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e))
raise Exception('ConfigMgmt Class creation failed')

return

def __init_sonic_yang(self):
self.sy = sonic_yang.SonicYang(YANG_DIR, debug=self.DEBUG)
# load yang models
self.sy.loadYangModel()
# load jIn from config DB or from config DB json file.
if self.source.lower() == 'configdb':
self.readConfigDB()
# treat any other source as file input
else:
self.readConfigDBJson(self.source)
# this will crop config, xlate and load.
self.sy.loadData(self.configdbJsonIn)

# Raise if tables without YANG models are not allowed but exist.
if not self.allowTablesWithoutYang and len(self.sy.tablesWithOutYang):
raise Exception('Config has tables without YANG models')

def __del__(self):
pass

Expand Down Expand Up @@ -213,6 +222,69 @@ def writeConfigDB(self, jDiff):

return

def add_module(self, yang_module_str):
"""
Validate and add new YANG module to the system.
Parameters:
yang_module_str (str): YANG module in string representation.
Returns:
None
"""

module_name = self.get_module_name(yang_module_str)
module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name))
if os.path.exists(module_path):
raise Exception('{} already exists'.format(module_name))
with open(module_path, 'w') as module_file:
module_file.write(yang_module_str)
try:
self.__init_sonic_yang()
except Exception:
os.remove(module_path)
raise

def remove_module(self, module_name):
"""
Remove YANG module from the system and validate.
Parameters:
module_name (str): YANG module name.
Returns:
None
"""

module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name))
if not os.path.exists(module_path):
return
temp = tempfile.NamedTemporaryFile(delete=False)
try:
shutil.move(module_path, temp.name)
self.__init_sonic_yang()
except Exception:
shutil.move(temp.name, module_path)
raise

@staticmethod
def get_module_name(yang_module_str):
"""
Read yangs module name from yang_module_str
Parameters:
yang_module_str(str): YANG module string.
Returns:
str: Module name
"""

# Instantiate new context since parse_module_mem() loads the module into context.
sy = sonic_yang.SonicYang(YANG_DIR)
module = sy.ctx.parse_module_mem(yang_module_str, ly.LYS_IN_YANG)
return module.name()


# End of Class ConfigMgmt

class ConfigMgmtDPB(ConfigMgmt):
Expand Down Expand Up @@ -417,8 +489,8 @@ def _deletePorts(self, ports=list(), force=False):
deps.extend(dep)

# No further action with no force and deps exist
if force == False and deps:
return configToLoad, deps, False;
if not force and deps:
return configToLoad, deps, False

# delets all deps, No topological sort is needed as of now, if deletion
# of deps fails, return immediately
Expand All @@ -436,8 +508,8 @@ def _deletePorts(self, ports=list(), force=False):
self.sy.deleteNode(str(xPathPort))

# Let`s Validate the tree now
if self.validateConfigData()==False:
return configToLoad, deps, False;
if not self.validateConfigData():
return configToLoad, deps, False

# All great if we are here, Lets get the diff
self.configdbJsonOut = self.sy.getData()
Expand Down
4 changes: 3 additions & 1 deletion sonic_package_manager/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -414,10 +414,11 @@ def reset(ctx, name, force, yes, skip_host_plugins):

@cli.command()
@add_options(PACKAGE_COMMON_OPERATION_OPTIONS)
@click.option('--keep-config', is_flag=True, help='Keep features configuration in CONFIG DB.')
@click.argument('name')
@click.pass_context
@root_privileges_required
def uninstall(ctx, name, force, yes):
def uninstall(ctx, name, force, yes, keep_config):
""" Uninstall package. """

manager: PackageManager = ctx.obj
Expand All @@ -428,6 +429,7 @@ def uninstall(ctx, name, force, yes):

uninstall_opts = {
'force': force,
'keep_config': keep_config,
}

try:
Expand Down
21 changes: 17 additions & 4 deletions sonic_package_manager/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,11 @@

import docker
import filelock
from config import config_mgmt
from sonic_py_common import device_info

from sonic_cli_gen.generator import CliGenerator

from sonic_package_manager import utils
from sonic_package_manager.constraint import (
VersionConstraint,
Expand Down Expand Up @@ -45,7 +48,10 @@
run_command
)
from sonic_package_manager.service_creator.feature import FeatureRegistry
from sonic_package_manager.service_creator.sonic_db import SonicDB
from sonic_package_manager.service_creator.sonic_db import (
INIT_CFG_JSON,
SonicDB
)
from sonic_package_manager.service_creator.utils import in_chroot
from sonic_package_manager.source import (
PackageSource,
Expand Down Expand Up @@ -435,13 +441,16 @@ def install_from_source(self,

@under_lock
@opt_check
def uninstall(self, name: str, force=False):
def uninstall(self, name: str,
force: bool = False,
keep_config: bool = False):
""" Uninstall SONiC Package referenced by name. The uninstallation
can be forced if force argument is True.
Args:
name: SONiC Package name.
force: Force the installation.
keep_config: Keep feature configuration in databases.
Raises:
PackageManagerError
"""
Expand Down Expand Up @@ -482,7 +491,7 @@ def uninstall(self, name: str, force=False):
self._systemctl_action(package, 'stop')
self._systemctl_action(package, 'disable')
self._uninstall_cli_plugins(package)
self.service_creator.remove(package)
self.service_creator.remove(package, keep_config=keep_config)
self.service_creator.generate_shutdown_sequence_files(
self._get_installed_packages_except(package)
)
Expand Down Expand Up @@ -1000,9 +1009,13 @@ def get_manager() -> 'PackageManager':
docker_api = DockerApi(docker.from_env(), ProgressManager())
registry_resolver = RegistryResolver()
metadata_resolver = MetadataResolver(docker_api, registry_resolver)
cfg_mgmt = config_mgmt.ConfigMgmt(source=INIT_CFG_JSON)
cli_generator = CliGenerator(log)
feature_registry = FeatureRegistry(SonicDB)
service_creator = ServiceCreator(feature_registry,
SonicDB)
SonicDB,
cli_generator,
cfg_mgmt)

return PackageManager(docker_api,
registry_resolver,
Expand Down
4 changes: 3 additions & 1 deletion sonic_package_manager/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,9 @@ def unmarshal(self, value):
ManifestField('mandatory', DefaultMarshaller(bool), False),
ManifestField('show', DefaultMarshaller(str), ''),
ManifestField('config', DefaultMarshaller(str), ''),
ManifestField('clear', DefaultMarshaller(str), '')
ManifestField('clear', DefaultMarshaller(str), ''),
ManifestField('auto-generate-show', DefaultMarshaller(bool), False),
ManifestField('auto-generate-config', DefaultMarshaller(bool), False),
])
])

Expand Down
6 changes: 4 additions & 2 deletions sonic_package_manager/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import json
import tarfile
from typing import Dict
from typing import Dict, Optional

from sonic_package_manager import utils
from sonic_package_manager.errors import MetadataError
Expand Down Expand Up @@ -54,6 +54,7 @@ class Metadata:

manifest: Manifest
components: Dict[str, Version] = field(default_factory=dict)
yang_module_str: Optional[str] = None


class MetadataResolver:
Expand Down Expand Up @@ -163,5 +164,6 @@ def from_labels(cls, labels: Dict[str, str]) -> Metadata:
except ValueError as err:
raise MetadataError(f'Failed to parse component version: {err}')

return Metadata(Manifest.marshal(manifest_dict), components)
yang_module_str = sonic_metadata.get('yang-module')

return Metadata(Manifest.marshal(manifest_dict), components, yang_module_str)
Loading

0 comments on commit f5e5a56

Please sign in to comment.