Gnmi poc update

.gitmodules

@@ -103,12 +103,14 @@
 [submodule "src/ptf-py3"]
 	path = src/ptf-py3
 	url = https://github.com/p4lang/ptf.git
+[submodule "src/sonic-gnmi"]
+	path = src/sonic-gnmi
+	url = https://github.com/ganglyu/sonic-gnmi.git
+	branch = gnmi_init_draft
 [submodule "src/dhcprelay"]
 	path = src/dhcprelay
 	url = https://github.com/sonic-net/sonic-dhcp-relay.git
 [submodule "src/sonic-host-services"]
 	path = src/sonic-host-services
-	url = https://github.com/sonic-net/sonic-host-services
-[submodule "src/sonic-gnmi"]
-	path = src/sonic-gnmi
-	url = https://github.com/sonic-net/sonic-gnmi.git
+	url = https://github.com/ganglyu/sonic-host-services.git
+	branch = update_dbus

dockers/docker-sonic-gnmi/Dockerfile.j2

@@ -0,0 +1,45 @@
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+ARG docker_container_name
+ARG image_version
+RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
+
+## Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Pass the image_version to container
+ENV IMAGE_VERSION=$image_version
+
+RUN apt-get update         && \
+    apt-get install -y        \
+        pkg-config build-essential libssl-dev swig python3-dev
+
+{% if docker_sonic_gnmi_debs.strip() -%}
+# Copy locally-built Debian package dependencies
+{{ copy_files("debs/", docker_sonic_gnmi_debs.split(' '), "/debs/") }}
+
+# Install locally-built Debian packages and implicitly install their dependencies
+{{ install_debian_packages(docker_sonic_gnmi_debs.split(' ')) }}
+{%- endif %}
+
+{% if docker_sonic_gnmi_whls.strip() %}
+# Copy locally-built Python wheel dependencies
+{{ copy_files("python-wheels/", docker_sonic_gnmi_whls.split(' '), "/python-wheels/") }}
+
+# Install locally-built Python wheel dependencies
+{{ install_python_wheels(docker_sonic_gnmi_whls.split(' ')) }}
+{% endif %}
+
+RUN apt-get clean -y      && \
+    apt-get autoclean -   && \
+    apt-get autoremove -y && \
+    rm -rf /debs /python-wheels ~/.cache
+
+COPY ["start.sh", "gnmi.sh", "/usr/bin/"]
+COPY ["gnmi_vars.j2", "/usr/share/sonic/templates/"]
+COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
+
+ENTRYPOINT ["/usr/local/bin/supervisord"]

dockers/docker-sonic-gnmi/base_image_files/monit_gnmi

@@ -0,0 +1,5 @@
+###############################################################################
+## Monit configuration for gnmi container
+###############################################################################
+check program container_memory_gnmi with path "/usr/bin/memory_checker gnmi 419430400"
+    if status == 3 for 10 times within 20 cycles then exec "/usr/bin/restart_service gnmi" repeat every 2 cycles

dockers/docker-sonic-gnmi/critical_processes

@@ -0,0 +1 @@
+program:gnxi

dockers/docker-sonic-gnmi/dialout.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Start with default config
+export CVL_SCHEMA_PATH=/usr/sbin/schema
+exec /usr/sbin/dialout_client_cli -insecure -logtostderr  -v 2
+

dockers/docker-sonic-gnmi/gnmi.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+EXIT_GNXI_VARS_FILE_NOT_FOUND=1
+GNXI_VARS_FILE=/usr/share/sonic/templates/gnmi_vars.j2
+
+if [ ! -f "$GNXI_VARS_FILE" ]; then
+    echo "GNMI vars template file not found"
+    exit $EXIT_GNXI_VARS_FILE_NOT_FOUND
+fi
+
+# Try to read gnxi and certs config from ConfigDB.
+# Use default value if no valid config exists
+GNXI_VARS=$(sonic-cfggen -d -t $GNXI_VARS_FILE)
+GNXI_VARS=${GNXI_VARS//[\']/\"}
+X509=$(echo $GNXI_VARS | jq -r '.x509')
+GNMI=$(echo $GNXI_VARS | jq -r '.gnmi')
+CERTS=$(echo $GNXI_VARS | jq -r '.certs')
+
+GNXI_ARGS=" -logtostderr"
+export CVL_SCHEMA_PATH=/usr/sbin/schema
+
+if [ -n "$CERTS" ]; then
+    SERVER_CRT=$(echo $CERTS | jq -r '.server_crt')
+    SERVER_KEY=$(echo $CERTS | jq -r '.server_key')
+    if [ -z $SERVER_CRT  ] || [ -z $SERVER_KEY  ]; then
+        GNXI_ARGS+=" --insecure"
+    else
+        GNXI_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY "
+    fi
+
+    CA_CRT=$(echo $CERTS | jq -r '.ca_crt')
+    if [ ! -z $CA_CRT ]; then
+        GNXI_ARGS+=" --ca_crt $CA_CRT"
+    fi
+elif [ -n "$X509" ]; then
+    SERVER_CRT=$(echo $X509 | jq -r '.server_crt')
+    SERVER_KEY=$(echo $X509 | jq -r '.server_key')
+    if [ -z $SERVER_CRT  ] || [ -z $SERVER_KEY  ]; then
+        GNXI_ARGS+=" --insecure"
+    else
+        GNXI_ARGS+=" --server_crt $SERVER_CRT --server_key $SERVER_KEY "
+    fi
+
+    CA_CRT=$(echo $X509 | jq -r '.ca_crt')
+    if [ ! -z $CA_CRT ]; then
+        GNXI_ARGS+=" --ca_crt $CA_CRT"
+    fi
+else
+    GNXI_ARGS+=" --noTLS"
+fi
+
+# If no configuration entry exists for GNXI, create one default port
+if [ -z "$GNMI" ]; then
+    PORT=8080
+else
+    PORT=$(echo $GNMI | jq -r '.port')
+fi
+GNXI_ARGS+=" --port $PORT"
+
+CLIENT_AUTH=$(echo $GNMI | jq -r '.client_auth')
+if [ -z $CLIENT_AUTH ] || [ $CLIENT_AUTH == "false" ]; then
+    GNXI_ARGS+=" --allow_no_client_auth"
+fi
+
+LOG_LEVEL=$(echo $GNMI | jq -r '.log_level')
+if [ ! -z $LOG_LEVEL ]; then
+    GNXI_ARGS+=" -v=$LOG_LEVEL"
+else
+    GNXI_ARGS+=" -v=2"
+fi
+
+exec /usr/sbin/gnxi ${GNXI_ARGS}

dockers/docker-sonic-gnmi/gnmi_vars.j2

@@ -0,0 +1,5 @@
+{
+    "certs": {% if "certs" in TELEMETRY.keys() %}{{ TELEMETRY["certs"] }}{% else %}""{% endif %},
+    "gnmi" : {% if "gnmi" in TELEMETRY.keys() %}{{ TELEMETRY["gnmi"] }}{% else %}""{% endif %},
+    "x509" : {% if "x509" in DEVICE_METADATA.keys() %}{{ DEVICE_METADATA["x509"] }}{% else %}""{% endif %}
+}

dockers/docker-sonic-gnmi/start.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+if [ "${RUNTIME_OWNER}" == "" ]; then
+    RUNTIME_OWNER="kube"
+fi
+
+CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py"
+if test -f ${CTR_SCRIPT}
+then
+    ${CTR_SCRIPT} -f gnmi -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION}
+fi
+
+mkdir -p /var/sonic
+echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status

dockers/docker-sonic-gnmi/supervisord.conf

@@ -0,0 +1,51 @@
+[supervisord]
+logfile_maxbytes=1MB
+logfile_backups=2
+nodaemon=true
+
+[eventlistener:dependent-startup]
+command=python3 -m supervisord_dependent_startup
+autostart=true
+autorestart=unexpected
+startretries=0
+exitcodes=0,3
+events=PROCESS_STATE
+buffer_size=1024
+
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener --container-name gnmi
+events=PROCESS_STATE_EXITED,PROCESS_STATE_RUNNING
+autostart=true
+autorestart=false
+buffer_size=1024
+
+[program:rsyslogd]
+command=/usr/sbin/rsyslogd -n -iNONE
+priority=1
+autostart=false
+autorestart=true
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+
+[program:start]
+command=/usr/bin/start.sh
+priority=2
+autostart=false
+autorestart=false
+startsecs=0
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=rsyslogd:running
+
+[program:gnxi]
+command=/usr/bin/gnmi.sh
+priority=3
+autostart=false
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=start:exited
+

files/build_templates/gnmi.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=GNMI container
+Requires=database.service
+After=database.service swss.service syncd.service
+Before=ntp-config.service
+BindsTo=sonic.target
+After=sonic.target
+StartLimitIntervalSec=1200
+StartLimitBurst=3
+
+[Service]
+User={{ sonicadmin_user }}
+ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
+ExecStart=/usr/bin/{{docker_container_name}}.sh wait
+ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+RestartSec=30

files/build_templates/gnmi.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Delays gnmi container until SONiC has started
+PartOf=gnmi.service
+
+[Timer]
+OnUnitActiveSec=0 sec
+OnBootSec=3min 30 sec
+Unit=gnmi.service
+
+[Install]
+WantedBy=timers.target sonic.target sonic-delayed.target

files/build_templates/init_cfg.json.j2

@@ -53,6 +53,7 @@
 {%- if include_sflow == "y" %}{% do features.append(("sflow", "disabled", false, "enabled")) %}{% endif %}
 {%- if include_macsec == "y" %}{% do features.append(("macsec", "disabled", false, "enabled")) %}{% endif %}
 {%- if include_system_telemetry == "y" %}{% do features.append(("telemetry", "enabled", true, "enabled")) %}{% endif %}
+{%- if include_system_gnmi == "y" %}{% do features.append(("gnmi", "enabled", true, "enabled")) %}{% endif %}
     "FEATURE": {
 {# has_timer field if set, will start the feature systemd .timer unit instead of .service unit #}
 {%- for feature, state, has_timer, autorestart in features %}
@@ -69,7 +70,7 @@
             "check_up_status" : "false",
 {%- endif %}
 {%- if include_kubernetes == "y" %}
-{%- if feature in ["lldp", "pmon", "radv", "snmp", "telemetry"] %}
+{%- if feature in ["lldp", "pmon", "radv", "snmp", "telemetry", "gnmi"] %}
             "set_owner": "kube", {% else %}
             "set_owner": "local", {% endif %} {% endif %}
             "high_mem_alert": "disabled"

files/build_templates/sonic_debian_extension.j2

@@ -850,6 +850,11 @@ sudo cp $BUILD_TEMPLATES/telemetry.timer $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
 echo "telemetry.timer" | sudo tee -a $GENERATED_SERVICE_FILE
 {% endif %}
 
+{% if include_system_gnmi == 'y' %}
+sudo cp $BUILD_TEMPLATES/gnmi.timer $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
+echo "gnmi.timer" | sudo tee -a $GENERATED_SERVICE_FILE
+{% endif %}
+
 {% if include_mgmt_framework == 'y' %}
 sudo cp $BUILD_TEMPLATES/mgmt-framework.timer $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
 echo "mgmt-framework.timer" | sudo tee -a $GENERATED_SERVICE_FILE

files/image_config/config-setup/config-setup

@@ -313,7 +313,7 @@ check_all_config_db_present()
 do_config_migration()
 {
     # Identify list of files to migrate
-    copy_list="minigraph.xml snmp.yml acl.json port_config.json frr telemetry"
+    copy_list="minigraph.xml snmp.yml acl.json port_config.json frr telemetry gnxi"
 
     # Migrate all configuration files from old to new
     copy_config_files_and_directories $copy_list

files/image_config/logrotate/rsyslog.j2

@@ -28,6 +28,7 @@
 /var/log/syslog
 /var/log/teamd.log
 /var/log/telemetry.log
+/var/log/gnmi.log
 /var/log/frr/bgpd.log
 /var/log/frr/zebra.log
 /var/log/swss/sairedis*.rec

files/image_config/rsyslog/rsyslog.d/00-sonic.conf

@@ -22,3 +22,9 @@ if $msg startswith  " telemetry" or ($msg startswith  " dialout" )then {
     /var/log/telemetry.log
     stop
 }
+
+## gnmi rules
+if $msg startswith " gnmi" then {
+    /var/log/gnmi.log
+    stop
+}

rules/config

@@ -125,6 +125,9 @@ DEFAULT_VS_PREPARE_MEM = yes
 # INCLUDE_SYSTEM_TELEMETRY - build docker-sonic-telemetry for system telemetry support
 INCLUDE_SYSTEM_TELEMETRY = y
 
+# INCLUDE_SYSTEM_GNMI - build docker-sonic-gnmi for system gnmi support
+INCLUDE_SYSTEM_GNMI = y
+
 # INCLUDE_ICCPD - build docker-iccpd for mclag support
 INCLUDE_ICCPD = n
 
@@ -139,7 +142,7 @@ INCLUDE_MGMT_FRAMEWORK = y
 
 # ENABLE_HOST_SERVICE_ON_START - enable sonic-host-server for mgmt-framework and/or
 # telemetry containers to access host functionality by default
-ENABLE_HOST_SERVICE_ON_START = n
+ENABLE_HOST_SERVICE_ON_START = y
 
 # INCLUDE_RESTAPI - build docker-sonic-restapi for configuring the switch using REST APIs
 INCLUDE_RESTAPI = n

rules/docker-gnmi.dep

@@ -0,0 +1,11 @@
+
+DPATH       := $($(DOCKER_GNMI)_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/docker-gnmi.mk rules/docker-gnmi.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(DPATH))
+
+$(DOCKER_GNMI)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(DOCKER_GNMI)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(DOCKER_GNMI)_DEP_FILES   := $(DEP_FILES)
+
+$(eval $(call add_dbg_docker,$(DOCKER_GNMI),$(DOCKER_GNMI_DBG)))

rules/docker-gnmi.mk

@@ -0,0 +1,36 @@
+# docker image for gnmi agent
+
+DOCKER_GNMI_STEM = docker-sonic-gnmi
+DOCKER_GNMI = $(DOCKER_GNMI_STEM).gz
+DOCKER_GNMI_DBG = $(DOCKER_GNMI_STEM)-$(DBG_IMAGE_MARK).gz
+
+$(DOCKER_GNMI)_PATH = $(DOCKERS_PATH)/$(DOCKER_GNMI_STEM)
+
+$(DOCKER_GNMI)_DEPENDS += $(SONIC_GNMI)
+$(DOCKER_GNMI)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_DEPENDS)
+
+$(DOCKER_GNMI)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_BULLSEYE)
+
+$(DOCKER_GNMI)_VERSION = 1.0.0
+$(DOCKER_GNMI)_PACKAGE_NAME = gnmi
+
+$(DOCKER_GNMI)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_IMAGE_PACKAGES)
+
+SONIC_DOCKER_IMAGES += $(DOCKER_GNMI)
+ifeq ($(INCLUDE_SYSTEM_GNMI), y)
+SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_GNMI)
+endif
+
+SONIC_DOCKER_DBG_IMAGES += $(DOCKER_GNMI_DBG)
+ifeq ($(INCLUDE_SYSTEM_GNMI), y)
+SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_GNMI_DBG)
+endif
+
+$(DOCKER_GNMI)_CONTAINER_NAME = gnmi
+$(DOCKER_GNMI)_RUN_OPT += --privileged -t
+$(DOCKER_GNMI)_RUN_OPT += -v /etc/sonic:/etc/sonic:rw
+$(DOCKER_GNMI)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro
+$(DOCKER_GNMI)_RUN_OPT += -v /var/run/dbus:/var/run/dbus:rw
+
+$(DOCKER_GNMI)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
+$(DOCKER_GNMI)_BASE_IMAGE_FILES += monit_gnmi:/etc/monit/conf.d

rules/gnmi.dep

@@ -0,0 +1,13 @@
+#DPKG FRK
+SPATH       := $($(SONIC_GNMI)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/gnmi.mk rules/gnmi.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+
+$(SONIC_GNMI)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(SONIC_GNMI)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_GNMI)_DEP_FILES   := $(DEP_FILES)
+$(SONIC_GNMI)_SMDEP_FILES := $(SMDEP_FILES)
+$(SONIC_GNMI)_SMDEP_PATHS := $(SPATH)
+