From cf5ca9d27ce1cc24136dc9e932b187aaa37bbe8c Mon Sep 17 00:00:00 2001 From: "Richard.Yu" Date: Fri, 17 Feb 2023 07:42:12 +0800 Subject: [PATCH] [SAI-PTF][mlnx]Enable saiserver test container on mlnx platforms (#13311) * Why I did it Enable Test sai api on bfn container with a lightweight container(saiserver). [SAI-PTF][mlnx]Enable saiserver test container on mlnx container How I did it enable saiserver container on mlnx platform. add docker-saiserver-mlnx.mk for building saiserver container in platform/barefoot/docker-saiserver-mlnx, add necessary files that needs in saiserver container How to verify it Signed-off-by: richardyu-ms --- .azure-pipelines/azure-pipelines-build.yml | 23 ++---- .azure-pipelines/build-template.yml | 13 +--- platform/mellanox/docker-saiserver-mlnx.mk | 21 +++++- .../docker-saiserver-mlnx/Dockerfile.j2 | 72 +++++++++++++++---- .../docker-saiserver-mlnx/old_start.sh | 13 ++++ .../docker-saiserver-mlnx/saiserver_start.sh | 32 +++++++++ .../mellanox/docker-saiserver-mlnx/start.sh | 10 --- .../{supervisord.conf => supervisord.conf.j2} | 6 +- platform/mellanox/libsaithrift-dev.mk | 10 ++- 9 files changed, 143 insertions(+), 57 deletions(-) create mode 100644 platform/mellanox/docker-saiserver-mlnx/old_start.sh create mode 100644 platform/mellanox/docker-saiserver-mlnx/saiserver_start.sh rename platform/mellanox/docker-saiserver-mlnx/{supervisord.conf => supervisord.conf.j2} (67%) diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml index 51ad919817b1..992bbb4097ee 100644 --- a/.azure-pipelines/azure-pipelines-build.yml +++ b/.azure-pipelines/azure-pipelines-build.yml @@ -148,35 +148,20 @@ jobs: if [ $(raw_image) == yes ]; then make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw fi - if [ $(GROUP_NAME) == marvell-armhf ]; then - make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y PLATFORM_ARCH=armhf target/docker-saiserverv2-mrvl.gz - pushd ./src/sonic-sairedis/SAI - git stash - popd - fi if [ $(docker_syncd_rpc_image) == yes ]; then # workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz - pushd ./src/sonic-sairedis/SAI - git stash - popd if [ $(GROUP_NAME) == broadcom ]; then make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-dnx-rpc.gz - pushd ./src/sonic-sairedis/SAI - git stash - popd make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz - pushd ./src/sonic-sairedis/SAI - git stash - popd fi if [ $(GROUP_NAME) == marvell-armhf ]; then make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y PLATFORM_ARCH=armhf target/docker-saiserverv2-mrvl.gz - pushd ./src/sonic-sairedis/SAI - git stash - popd fi - fi + if [ $(GROUP_NAME) == mellanox ]; then + make $BUILD_OPTIONS SAITHRIFT_V2=y ENABLE_SYNCD_RPC=y target/docker-saiserverv2-mlnx.gz + fi + fi if [ $(syncd_rpc_image) == yes ]; then make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/sonic-$(GROUP_NAME).bin mv target/sonic-mellanox.bin target/sonic-$(GROUP_NAME)-rpc.bin diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index eb1ec4cab2ae..48bc3293b964 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -105,21 +105,14 @@ jobs: fi if [ ${{ parameters.sync_rpc_image }} == true ]; then make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz - # workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache - pushd ./src/sonic-sairedis/SAI - git stash - popd if [ ${{ parameters.platform }} == broadcom ]; then make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz - pushd ./src/sonic-sairedis/SAI - git stash - popd fi if [ $(GROUP_NAME) == marvell-armhf ]; then make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y PLATFORM_ARCH=armhf target/docker-saiserverv2-mrvl.gz - pushd ./src/sonic-sairedis/SAI - git stash - popd + fi + if [ ${{ parameters.platform }} == mellanox ]; then + make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) SAITHRIFT_V2=y ENABLE_SYNCD_RPC=y target/docker-saiserverv2-mlnx.gz fi fi diff --git a/platform/mellanox/docker-saiserver-mlnx.mk b/platform/mellanox/docker-saiserver-mlnx.mk index 74fb96f91d1b..920510194257 100644 --- a/platform/mellanox/docker-saiserver-mlnx.mk +++ b/platform/mellanox/docker-saiserver-mlnx.mk @@ -18,7 +18,26 @@ DOCKER_SAISERVER_MLNX = docker-saiserver$(SAITHRIFT_VER)-mlnx.gz $(DOCKER_SAISERVER_MLNX)_PATH = $(PLATFORM_PATH)/docker-saiserver-mlnx -$(DOCKER_SAISERVER_MLNX)_DEPENDS += $(SAISERVER) $(PYTHON_SDK_API) +$(DOCKER_SAISERVER_MLNX)_DEPENDS += $(SAISERVER) + +$(DOCKER_SAISERVER_MLNX)_DEPENDS += $(PYTHON_SDK_API) $(MFT) + +ifeq ($(ENABLE_ASAN), y) +$(DOCKER_SAISERVER_MLNX)_DEPENDS += $(SAISERVER_DBG) +endif + +$(DOCKER_SAISERVER_MLNX)_FILES += $(ISSU_VERSION_FILE) + +ifeq ($(SDK_FROM_SRC), y) +$(DOCKER_SAISERVER_MLNX)_DBG_DEPENDS += $(MLNX_SDK_DBG_DEBS) $(MLNX_SAI_DBGSYM) +endif + +# Use syncd_init_common.sh to init hardware platform +SYNCD_INIT_COMMON_SCRIPT = syncd_init_common.sh +$(SYNCD_INIT_COMMON_SCRIPT)_PATH = $(SRC_PATH)/sonic-sairedis/syncd/scripts +SONIC_COPY_FILES += $(SYNCD_INIT_COMMON_SCRIPT) + +$(DOCKER_SAISERVER_MLNX)_FILES += $(SYNCD_INIT_COMMON_SCRIPT) $(DOCKER_SAISERVER_MLNX)_PYTHON_DEBS += $(MLNX_SFPD) $(DOCKER_SAISERVER_MLNX)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_BULLSEYE) SONIC_DOCKER_IMAGES += $(DOCKER_SAISERVER_MLNX) diff --git a/platform/mellanox/docker-saiserver-mlnx/Dockerfile.j2 b/platform/mellanox/docker-saiserver-mlnx/Dockerfile.j2 index 0f6745d580ab..47545f710b5d 100644 --- a/platform/mellanox/docker-saiserver-mlnx/Dockerfile.j2 +++ b/platform/mellanox/docker-saiserver-mlnx/Dockerfile.j2 @@ -14,42 +14,84 @@ ## See the License for the specific language governing permissions and ## limitations under the License. ## -{% from "dockers/dockerfile-macros.j2" import install_debian_packages %} +{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %} FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}} ARG docker_container_name RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf +#Install syncd dependences +RUN mkdir -p /var/run/sx_sdk + ## Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y \ + libxml2 \ + python3-pip \ + python3-dev \ + python-is-python3 \ +{%- if ENABLE_ASAN == "y" %} + libasan6 \ +{%- endif %} + python3-setuptools + +RUN pip3 install --upgrade pip +RUN apt-get purge -y python-pip + +#install syncd_rpc dependences ## Pre-install the fundamental packages RUN apt-get update \ && apt-get -y install \ gdb \ + net-tools \ + python-setuptools \ + build-essential \ + libssl-dev \ + libffi-dev \ + python-dev \ + wget \ + cmake \ + libqt5core5a \ + libqt5network5 \ libboost-atomic1.74.0 -COPY \ -{% for deb in docker_saiserver_mlnx_debs.split(' ') -%} -debs/{{ deb }}{{' '}} -{%- endfor -%} -debs/ +# Build and install python-scapy +RUN curl http://ftp.us.debian.org/debian/pool/main/s/scapy/python-scapy_2.4.0-2_all.deb --output python-scapy_2.4.0-2_all.deb \ + && dpkg -i python-scapy_2.4.0-2_all.deb \ + && apt install -f -COPY \ -{% for deb in docker_saiserver_mlnx_pydebs.split(' ') -%} -python-debs/{{ deb }}{{' '}} -{%- endfor -%} -debs/ - -RUN apt-get install -y --no-install-recommends libxml2 iptables libbsd0 protobuf-c-compiler protobuf-compiler python3-protobuf libprotobuf-c1 python3-future python3-ipaddr libnet1 pkg-config asciidoc xmlto +{% if docker_saiserver_mlnx_debs.strip() -%} +# Copy locally-built Debian package dependencies +{{ copy_files("debs/", docker_saiserver_mlnx_debs.split(' '), "/debs/") }} +# Install locally-built Debian packages and implicitly install their dependencies {{ install_debian_packages(docker_saiserver_mlnx_debs.split(' ')) }} +{% endif %} -COPY ["start.sh", "/usr/bin/"] +{% if docker_saiserver_mlnx_pydebs.strip() -%} +# Copy locally-built Debian package dependencies +{{ copy_files("python-debs/", docker_saiserver_mlnx_pydebs.split(' '), "/debs/") }} -COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] +# Install locally-built Debian packages and implicitly install their dependencies +{{ install_debian_packages(docker_saiserver_mlnx_pydebs.split(' ')) }} +{% endif %} + +RUN apt-get install -y --no-install-recommends libxml2 iptables libbsd0 protobuf-c-compiler protobuf-compiler python3-protobuf libprotobuf-c1 python3-future python3-ipaddr libnet1 pkg-config asciidoc xmlto + +COPY ["start.sh", "/usr/bin/"] +COPY ["old_start.sh", "/usr/bin/"] +COPY ["saiserver_start.sh", "/usr/bin/"] +COPY ["files/syncd_init_common.sh", "/usr/bin/"] +RUN chmod +x /usr/bin/saiserver_start.sh /usr/bin/syncd_init_common.sh /usr/bin/old_start.sh COPY ["profile.ini", "portmap.ini", "/etc/sai/"] +COPY ["supervisord.conf.j2", "/usr/share/sonic/templates/"] + +RUN mkdir -p /etc/supervisor/conf.d/ +RUN sonic-cfggen -a "{\"ENABLE_ASAN\":\"{{ENABLE_ASAN}}\"}" -t /usr/share/sonic/templates/supervisord.conf.j2 > /etc/supervisor/conf.d/supervisord.conf +RUN rm -f /usr/share/sonic/templates/supervisord.conf.j2 COPY ["sai_2700.xml", "/usr/share/"] diff --git a/platform/mellanox/docker-saiserver-mlnx/old_start.sh b/platform/mellanox/docker-saiserver-mlnx/old_start.sh new file mode 100644 index 000000000000..ed2dc278c23b --- /dev/null +++ b/platform/mellanox/docker-saiserver-mlnx/old_start.sh @@ -0,0 +1,13 @@ +start_mlnx() +{ + [ -e /dev/sxdevs/sxcdev ] || ( mkdir -p /dev/sxdevs && mknod /dev/sxdevs/sxcdev c 231 193 ) +} + + +rm -f /var/run/rsyslogd.pid + +supervisorctl start rsyslogd + +start_mlnx + +supervisorctl start saiserver \ No newline at end of file diff --git a/platform/mellanox/docker-saiserver-mlnx/saiserver_start.sh b/platform/mellanox/docker-saiserver-mlnx/saiserver_start.sh new file mode 100644 index 000000000000..06fcf6c3f3ae --- /dev/null +++ b/platform/mellanox/docker-saiserver-mlnx/saiserver_start.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copy from src/sonic-sairedis/syncd/scripts/syncd_start.sh +# Re-use the structure for syncd setup +# Use it to start saiserver +# Script to start syncd using supervisord +# + +# Source the file that holds common code for systemd and supervisord +. /usr/bin/syncd_init_common.sh + +get_saiserver_param() +{ + IFS=' ' read -r -a array <<< "$CMD_ARGS" + for index in "${!array[@]}" + do + #echo "$index ${array[index]}" + if [[ "${array[index]}" == *"-p"* ]]; then + SAI_PROFILE="${array[index+1]}" + fi + if [[ "${array[index]}" == *"-m"* ]]; then + PORT_CONFIG="${array[index+1]}" + fi + done +} + +ENABLE_SAITHRIFT=1 +config_syncd +get_saiserver_param + +echo exec /usr/sbin/saiserver -p ${SAI_PROFILE} -f ${PORT_CONFIG} +exec /usr/sbin/saiserver -p ${SAI_PROFILE} -f ${PORT_CONFIG} +#exec ${CMD} ${CMD_ARGS} diff --git a/platform/mellanox/docker-saiserver-mlnx/start.sh b/platform/mellanox/docker-saiserver-mlnx/start.sh index d5ac04c5e323..271f9c6abbc0 100755 --- a/platform/mellanox/docker-saiserver-mlnx/start.sh +++ b/platform/mellanox/docker-saiserver-mlnx/start.sh @@ -16,16 +16,6 @@ # limitations under the License. # -start_mlnx() -{ - [ -e /dev/sxdevs/sxcdev ] || ( mkdir -p /dev/sxdevs && mknod /dev/sxdevs/sxcdev c 231 193 ) -} - - rm -f /var/run/rsyslogd.pid - supervisorctl start rsyslogd - -start_mlnx - supervisorctl start saiserver diff --git a/platform/mellanox/docker-saiserver-mlnx/supervisord.conf b/platform/mellanox/docker-saiserver-mlnx/supervisord.conf.j2 similarity index 67% rename from platform/mellanox/docker-saiserver-mlnx/supervisord.conf rename to platform/mellanox/docker-saiserver-mlnx/supervisord.conf.j2 index e90e41645042..81f08526224e 100644 --- a/platform/mellanox/docker-saiserver-mlnx/supervisord.conf +++ b/platform/mellanox/docker-saiserver-mlnx/supervisord.conf.j2 @@ -1,3 +1,4 @@ +{% set asan_extra_options = ':print_suppressions=0' %} [supervisord] logfile_maxbytes=1MB logfile_backups=2 @@ -20,9 +21,12 @@ stdout_logfile=syslog stderr_logfile=syslog [program:saiserver] -command=/usr/sbin/saiserver -p /etc/sai/profile.ini -f /etc/sai/portmap.ini +command=/usr/bin/saiserver_start.sh priority=3 autostart=false autorestart=false stdout_logfile=syslog stderr_logfile=syslog +{% if ENABLE_ASAN == "y" %} +environment=ASAN_OPTIONS="log_path=/var/log/asan/syncd-asan.log{{ asan_extra_options }}" +{% endif %} diff --git a/platform/mellanox/libsaithrift-dev.mk b/platform/mellanox/libsaithrift-dev.mk index 4ee241eab4c7..675b483773d3 100644 --- a/platform/mellanox/libsaithrift-dev.mk +++ b/platform/mellanox/libsaithrift-dev.mk @@ -20,9 +20,17 @@ SAI_VER = 0.9.4 LIBSAITHRIFT_DEV = libsaithrift$(SAITHRIFT_VER)-dev_$(SAI_VER)_amd64.deb $(LIBSAITHRIFT_DEV)_SRC_PATH = $(SRC_PATH)/sonic-sairedis/SAI + +#Support two different versions of thrift +ifeq ($(SAITHRIFT_V2),y) +$(LIBSAITHRIFT_DEV)_DEPENDS += $(LIBTHRIFT_0_14_1) $(LIBTHRIFT_0_14_1_DEV) $(PYTHON3_THRIFT_0_14_1) $(THRIFT_0_14_1_COMPILER) +$(LIBSAITHRIFT_DEV)_RDEPENDS += $(LIBTHRIFT_0_14_1) +$(LIBSAITHRIFT_DEV)_BUILD_ENV += SAITHRIFTV2=true GEN_SAIRPC_OPTS="--adapter_logger" +else $(LIBSAITHRIFT_DEV)_DEPENDS += $(LIBTHRIFT) $(LIBTHRIFT_DEV) $(PYTHON_THRIFT) $(THRIFT_COMPILER) $(LIBSAITHRIFT_DEV)_RDEPENDS += $(LIBTHRIFT) -$(LIBSAITHRIFT_DEV)_BUILD_ENV = SAITHRIFT_VER=v2 # Workaround to use python3 in build +endif +$(LIBSAITHRIFT_DEV)_BUILD_ENV += SAITHRIFT_VER=v2 # Workaround to use python3 in build $(LIBSAITHRIFT_DEV)_DEPENDS += $(MLNX_SAI) $(MLNX_SAI_DEV) $(LIBSAITHRIFT_DEV)_RDEPENDS += $(MLNX_SAI)