Skip to content

Commit

Permalink
[ci]: add vstest to azure pipeline (sonic-net#1436)
Browse files Browse the repository at this point in the history
Signed-off-by: Guohan Lu <lguohan@gmail.com>
  • Loading branch information
lguohan authored Mar 29, 2021
1 parent 28b64ec commit 6b51bcd
Show file tree
Hide file tree
Showing 5 changed files with 314 additions and 61 deletions.
55 changes: 55 additions & 0 deletions .azure-pipelines/build-docker-sonic-vs-template.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
parameters:
- name: timeout
type: number
default: 60

- name: artifact_name
type: string

jobs:
- job:
displayName: "docker-sonic-vs"
timeoutInMinutes: ${{ parameters.timeout }}

pool:
vmImage: 'ubuntu-20.04'

steps:
- task: DownloadPipelineArtifact@2
inputs:
artifact: wheels
displayName: "Download sonic swss artifact"

- task: DownloadPipelineArtifact@2
inputs:
source: specific
project: build
pipeline: 1
artifact: sonic-buildimage.vs
runVersion: 'latestFromBranch'
runBranch: 'refs/heads/master'
displayName: "Download sonic buildimage"

- script: |
set -ex
echo $(Build.DefinitionName).$(Build.BuildNumber)
docker load < ../target/docker-sonic-vs.gz
mkdir -p .azure-pipelines/docker-sonic-vs/wheels
cp -v ../*.whl .azure-pipelines/docker-sonic-vs/wheels
pushd .azure-pipelines
docker build --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) docker-sonic-vs
popd
docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
displayName: "Build docker-sonic-vs image"
- publish: $(Build.ArtifactStagingDirectory)/
artifact: ${{ parameters.artifact_name }}
displayName: "Archive sonic docker vs image"
87 changes: 87 additions & 0 deletions .azure-pipelines/build_and_install_module.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
#!/bin/bash
#
# build and install team/vrf driver
#

set -e

source /etc/os-release

function build_and_install_kmodule()
{
if sudo modprobe team 2>/dev/null && sudo modprobe vrf 2>/dev/null && sudo modprobe macsec 2>/dev/null; then
echo "The module team, vrf and macsec exist."
return
fi

[ -z "$WORKDIR" ] && WORKDIR=$(mktemp -d)
cd $WORKDIR

KERNEL_RELEASE=$(uname -r)
KERNEL_MAINVERSION=$(echo $KERNEL_RELEASE | cut -d- -f1)
EXTRAVERSION=$(echo $KERNEL_RELEASE | cut -d- -f2)
LOCALVERSION=$(echo $KERNEL_RELEASE | cut -d- -f3)
VERSION=$(echo $KERNEL_MAINVERSION | cut -d. -f1)
PATCHLEVEL=$(echo $KERNEL_MAINVERSION | cut -d. -f2)
SUBLEVEL=$(echo $KERNEL_MAINVERSION | cut -d. -f3)

# Install the required debian packages to build the kernel modules
apt-get install -y build-essential linux-headers-${KERNEL_RELEASE} autoconf pkg-config fakeroot
apt-get install -y flex bison libssl-dev libelf-dev
apt-get install -y libnl-route-3-200 libnl-route-3-dev libnl-cli-3-200 libnl-cli-3-dev libnl-3-dev

# Add the apt source mirrors and download the linux image source code
cp /etc/apt/sources.list /etc/apt/sources.list.bk
sed -i "s/^# deb-src/deb-src/g" /etc/apt/sources.list
apt-get update
apt-get source linux-image-unsigned-$(uname -r) > source.log

# Recover the original apt sources list
cp /etc/apt/sources.list.bk /etc/apt/sources.list
apt-get update

# Build the Linux kernel module drivers/net/team and vrf
cd $(find . -maxdepth 1 -type d | grep -v "^.$")
make allmodconfig
mv .config .config.bk
cp /boot/config-$(uname -r) .config
grep NET_TEAM .config.bk >> .config
echo CONFIG_NET_VRF=m >> .config
echo CONFIG_MACSEC=m >> .config
make VERSION=$VERSION PATCHLEVEL=$PATCHLEVEL SUBLEVEL=$SUBLEVEL EXTRAVERSION=-${EXTRAVERSION} LOCALVERSION=-${LOCALVERSION} modules_prepare
make M=drivers/net/team
mv drivers/net/Makefile drivers/net/Makefile.bak
echo 'obj-$(CONFIG_NET_VRF) += vrf.o' > drivers/net/Makefile
echo 'obj-$(CONFIG_MACSEC) += macsec.o' >> drivers/net/Makefile
make M=drivers/net

# Install the module
TEAM_DIR=$(echo /lib/modules/$(uname -r)/kernel/net/team)
NET_DIR=$(echo /lib/modules/$(uname -r)/kernel/net)
if [ ! -e "$TEAM_DIR/team.ko" ]; then
mkdir -p $TEAM_DIR
cp drivers/net/team/*.ko $TEAM_DIR/
modinfo $TEAM_DIR/team.ko
depmod
modprobe team
fi
if [ ! -e "$NET_DIR/vrf.ko" ]; then
mkdir -p $NET_DIR
cp drivers/net/vrf.ko $NET_DIR/
modinfo $NET_DIR/vrf.ko
depmod
modprobe vrf
fi
if [ ! -e "$NET_DIR/macsec.ko" ]; then
mkdir -p $NET_DIR
cp drivers/net/macsec.ko $NET_DIR/
modinfo $NET_DIR/macsec.ko
depmod
modprobe macsec
fi

cd /tmp
rm -rf $WORKDIR
}

build_and_install_kmodule
7 changes: 7 additions & 0 deletions .azure-pipelines/docker-sonic-vs/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
FROM docker-sonic-vs

ARG docker_container_name

ADD ["wheels", "/wheels"]

RUN pip3 install --no-deps --force-reinstall /wheels/sonic_utilities-1.2-py3-none-any.whl
75 changes: 75 additions & 0 deletions .azure-pipelines/test-docker-sonic-vs-template.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
parameters:
- name: timeout
type: number
default: 180

- name: log_artifact_name
type: string

jobs:
- job:
displayName: vstest
timeoutInMinutes: ${{ parameters.timeout }}

pool:
vmImage: 'ubuntu-20.04'

steps:
- task: DownloadPipelineArtifact@2
inputs:
artifact: docker-sonic-vs
displayName: "Download docker sonic vs image"

- task: DownloadPipelineArtifact@2
inputs:
source: specific
project: build
pipeline: 9
artifact: sonic-swss-common.amd64.ubuntu20_04
runVersion: 'latestFromBranch'
runBranch: 'refs/heads/master'
displayName: "Download sonic swss common deb packages"

- checkout: self
displayName: "Checkout sonic-swss-common"
- checkout: sonic-swss
displayName: "Checkout sonic-swss"

- script: |
set -x
sudo sonic-utilities/.azure-pipelines/build_and_install_module.sh
sudo apt-get install -y libhiredis0.14
sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f
sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb
# install packages for vs test
sudo apt-get install -y net-tools bridge-utils vlan
sudo apt-get install -y python3-pip
sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker==4.4.1 redis==3.3.4 flaky==3.7.0
displayName: "Install dependencies"
- script: |
set -x
sudo docker load -i ../docker-sonic-vs.gz
docker ps
ip netns list
pushd sonic-swss/tests
sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
displayName: "Run vs tests"
- task: PublishTestResults@2
inputs:
testResultsFiles: '**/tr.xml'
testRunTitle: vstest
condition: always()

- script: |
cp -r sonic-swss/tests/log $(Build.ArtifactStagingDirectory)/
displayName: "Collect logs"
condition: always()
- publish: $(Build.ArtifactStagingDirectory)/
artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt)
displayName: "Publish logs"
condition: always()
151 changes: 90 additions & 61 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,73 +6,102 @@
trigger:
- master

pool:
vmImage: ubuntu-20.04
resources:
repositories:
- repository: sonic-swss
type: github
name: Azure/sonic-swss
endpoint: build

container:
image: sonicdev-microsoft.azurecr.io:443/sonic-slave-buster:latest
stages:
- stage: Build

steps:
- task: DownloadPipelineArtifact@2
inputs:
source: specific
project: build
pipeline: 1
artifact: sonic-buildimage.vs
runVersion: 'latestFromBranch'
runBranch: 'refs/heads/master'
displayName: "Download artifacts from latest sonic-buildimage build"
jobs:
- job:
displayName: "Python3"
pool:
vmImage: ubuntu-20.04

- script: |
set -xe
sudo dpkg -i libnl-3-200_*.deb
sudo dpkg -i libnl-genl-3-200_*.deb
sudo dpkg -i libnl-route-3-200_*.deb
sudo dpkg -i libnl-nf-3-200_*.deb
sudo dpkg -i libhiredis0.14_*.deb
sudo dpkg -i libswsscommon_1.0.0_amd64.deb
sudo dpkg -i python3-swsscommon_1.0.0_amd64.deb
sudo dpkg -i libyang_1.0.73_amd64.deb
sudo dpkg -i libyang-cpp_1.0.73_amd64.deb
sudo dpkg -i python3-yang_1.0.73_amd64.deb
workingDirectory: $(Pipeline.Workspace)/target/debs/buster/
displayName: 'Install Debian dependencies'
container:
image: sonicdev-microsoft.azurecr.io:443/sonic-slave-buster:latest

- script: |
set -xe
sudo pip3 install swsssdk-2.0.1-py3-none-any.whl
sudo pip3 install sonic_py_common-1.0-py3-none-any.whl
sudo pip3 install sonic_config_engine-1.0-py3-none-any.whl
sudo pip3 install sonic_platform_common-1.0-py3-none-any.whl
sudo pip3 install sonic_yang_mgmt-1.0-py3-none-any.whl
sudo pip3 install sonic_yang_models-1.0-py3-none-any.whl
workingDirectory: $(Pipeline.Workspace)/target/python-wheels/
displayName: 'Install Python dependencies'
steps:
- task: DownloadPipelineArtifact@2
inputs:
source: specific
project: build
pipeline: 1
artifact: sonic-buildimage.vs
runVersion: 'latestFromBranch'
runBranch: 'refs/heads/master'
displayName: "Download artifacts from latest sonic-buildimage build"

- script: |
python3 setup.py test
displayName: 'Test Python 3'
- script: |
set -xe
sudo dpkg -i libnl-3-200_*.deb
sudo dpkg -i libnl-genl-3-200_*.deb
sudo dpkg -i libnl-route-3-200_*.deb
sudo dpkg -i libnl-nf-3-200_*.deb
sudo dpkg -i libhiredis0.14_*.deb
sudo dpkg -i libswsscommon_1.0.0_amd64.deb
sudo dpkg -i python3-swsscommon_1.0.0_amd64.deb
sudo dpkg -i libyang_1.0.73_amd64.deb
sudo dpkg -i libyang-cpp_1.0.73_amd64.deb
sudo dpkg -i python3-yang_1.0.73_amd64.deb
workingDirectory: $(Pipeline.Workspace)/target/debs/buster/
displayName: 'Install Debian dependencies'
- task: PublishTestResults@2
inputs:
testResultsFiles: '$(System.DefaultWorkingDirectory)/test-results.xml'
testRunTitle: Python 3
failTaskOnFailedTests: true
condition: succeededOrFailed()
displayName: 'Publish Python 3 test results'
- script: |
set -xe
sudo pip3 install swsssdk-2.0.1-py3-none-any.whl
sudo pip3 install sonic_py_common-1.0-py3-none-any.whl
sudo pip3 install sonic_config_engine-1.0-py3-none-any.whl
sudo pip3 install sonic_platform_common-1.0-py3-none-any.whl
sudo pip3 install sonic_yang_mgmt-1.0-py3-none-any.whl
sudo pip3 install sonic_yang_models-1.0-py3-none-any.whl
workingDirectory: $(Pipeline.Workspace)/target/python-wheels/
displayName: 'Install Python dependencies'
- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov/'
displayName: 'Publish Python 3 test coverage'
- script: |
python3 setup.py test
displayName: 'Test Python 3'
- script: |
set -e
python3 setup.py bdist_wheel
displayName: 'Build Python 3 wheel'
- task: PublishTestResults@2
inputs:
testResultsFiles: '$(System.DefaultWorkingDirectory)/test-results.xml'
testRunTitle: Python 3
failTaskOnFailedTests: true
condition: succeededOrFailed()
displayName: 'Publish Python 3 test results'

- publish: '$(System.DefaultWorkingDirectory)/dist/'
artifact: wheels
displayName: "Publish Python wheels"
- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov/'
displayName: 'Publish Python 3 test coverage'

- script: |
set -e
python3 setup.py bdist_wheel
displayName: 'Build Python 3 wheel'
- publish: '$(System.DefaultWorkingDirectory)/dist/'
artifact: wheels
displayName: "Publish Python wheels"

- stage: BuildDocker
dependsOn: Build
condition: succeeded('Build')
jobs:
- template: .azure-pipelines/build-docker-sonic-vs-template.yml
parameters:
artifact_name: docker-sonic-vs

- stage: Test
dependsOn: BuildDocker
condition: succeeded('BuildDocker')
jobs:
- template: .azure-pipelines/test-docker-sonic-vs-template.yml
parameters:
log_artifact_name: log

0 comments on commit 6b51bcd

Please sign in to comment.