Test semgrep

.azure-pipelines/azure-pipelines-build.yml

@@ -121,6 +121,7 @@ jobs:
               mv target/docker-sonic-vs.gz target/docker-sonic-vs-asan.gz
             fi
             make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+            make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-ptf-sai.gz
             if [ $(Build.Reason) != 'PullRequest' ];then
               gzip -kd target/sonic-vs.img.gz
               SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make sonic-slave-run
@@ -138,9 +139,25 @@ jobs:
               make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
             fi
             if [ $(docker_syncd_rpc_image) == yes ]; then
+              # workaround for issue in rules/sairedis.dep,  git ls-files will list un-exist files for cache
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz
+              pushd ./src/sonic-sairedis/SAI
+              git stash
+              popd  
               if [ $(GROUP_NAME) == broadcom ]; then
                 make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-dnx-rpc.gz
+                pushd ./src/sonic-sairedis/SAI
+                git stash
+                popd 
+                make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
+                pushd ./src/sonic-sairedis/SAI
+                git stash
+                popd 
+                echo BUILD_OPTIONS $BUILD_OPTIONS
+                make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-syncd-brcm-rpcv2.gz
+                pushd ./src/sonic-sairedis/SAI
+                git stash
+                popd 
               fi
             fi
             if [ $(syncd_rpc_image) == yes ]; then

.azure-pipelines/azure-pipelines-image-template.yml

@@ -36,7 +36,7 @@ jobs:
         displayName: "Set cache options"
       - checkout: self
         submodules: recursive
-        condition: eq(variables.SKIP_CHECKOUT, '') 
+        condition: and(succeeded(), eq(variables.SKIP_CHECKOUT, ''))
         displayName: 'Checkout code'
       - script: |
           BRANCH_NAME=$(Build.SourceBranchName)

.azure-pipelines/build-template.yml

@@ -92,6 +92,7 @@ jobs:
         fi
 
         make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+        make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-ptf-sai.gz
       else
         if [ ${{ parameters.dbg_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) INSTALL_DEBUG_TOOLS=y target/sonic-${{ parameters.platform }}.bin && \
@@ -105,6 +106,20 @@ jobs:
         fi
         if [ ${{ parameters.sync_rpc_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz
+          # workaround for issue in rules/sairedis.dep,  git ls-files will list un-exist files for cache
+          pushd ./src/sonic-sairedis/SAI
+          git stash
+          popd 
+          if [ ${{ parameters.platform }} == broadcom ]; then
+            make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) SAITHRIFT_V2=y ENABLE_SYNCD_RPC=y target/docker-syncd-brcm-rpcv2.gz
+            pushd ./src/sonic-sairedis/SAI
+            git stash
+            popd 
+            make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
+            pushd ./src/sonic-sairedis/SAI
+            git stash
+            popd 
+          fi
         fi
 
         make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin

.azure-pipelines/official-build.yml

@@ -9,7 +9,7 @@ schedules:
   branches:
     include:
     - master
-    - 202012
+    - 202???
   always: true
 - cron: "0 4 * * *"
   displayName: nightly build for release

.azure-pipelines/run-test-scheduler-template.yml

@@ -0,0 +1,139 @@
+parameters:
+- name: TOPOLOGY
+  type: string
+
+- name: POLL_INTERVAL
+  type: number
+  default: 10
+
+- name: POLL_TIMEOUT
+  type: number
+  default: 36000
+
+- name: MIN_WORKER
+  type: number
+  default: 1
+
+- name: MAX_WORKER
+  type: number
+  default: 2
+
+- name: TEST_SET
+  type: string
+  default: ""
+
+- name: DEPLOY_MG_EXTRA_PARAMS
+  type: string
+  default: ""
+
+- name: COMMON_EXTRA_PARAMS
+  type: string
+  default: ""
+
+- name: VM_TYPE
+  type: string
+  default: "ceos"
+
+- name: SPECIFIED_PARAMS
+  type: string
+  default: "{}"
+
+- name: MGMT_BRANCH
+  type: string
+  default: master
+
+- name: NUM_ASIC
+  type: number
+  default: 1
+
+steps:
+  - script: |
+      set -ex
+      wget -O ./.azure-pipelines/test_plan.py  https://github.com/raw/sonic-net/sonic-mgmt/master/.azure-pipelines/test_plan.py
+      wget -O ./.azure-pipelines/pr_test_scripts.yaml  https://github.com/raw/sonic-net/sonic-mgmt/master/.azure-pipelines/pr_test_scripts.yaml
+    displayName: Download TestbedV2 scripts
+
+  - script: |
+      set -ex
+      pip install PyYAML
+      rm -f new_test_plan_id.txt
+      python ./.azure-pipelines/test_plan.py create -t ${{ parameters.TOPOLOGY }} -o new_test_plan_id.txt \
+      --min-worker ${{ parameters.MIN_WORKER }} --max-worker ${{ parameters.MAX_WORKER }} \
+      --test-set ${{ parameters.TEST_SET }} --kvm-build-id $(KVM_BUILD_ID) \
+      --deploy-mg-extra-params "${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" --common-extra-params "${{ parameters.COMMON_EXTRA_PARAMS }}" \
+      --mgmt-branch ${{ parameters.MGMT_BRANCH }} --vm-type ${{ parameters.VM_TYPE }} --specified-params "${{ parameters.SPECIFIED_PARAMS }}" \
+      --num-asic ${{ parameters.NUM_ASIC }}
+      TEST_PLAN_ID=`cat new_test_plan_id.txt`
+
+      echo "Created test plan $TEST_PLAN_ID"
+      echo "Check https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID for test plan status"
+      echo "##vso[task.setvariable variable=TEST_PLAN_ID]$TEST_PLAN_ID"
+    env:
+      TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
+      TENANT_ID: $(TESTBED_TOOLS_MSAL_TENANT_ID)
+      CLIENT_ID: $(TESTBED_TOOLS_MSAL_CLIENT_ID)
+      CLIENT_SECRET: $(TESTBED_TOOLS_MSAL_CLIENT_SECRET)
+    displayName: Trigger test
+
+  - script: |
+      set -ex
+      echo "Lock testbed"
+      echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
+      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
+      # When "LOCK_TESTBED" finish, it changes into "PREPARE_TESTBED"
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 43200 --expected-states PREPARE_TESTBED EXECUTING KVMDUMP FINISHED CANCELLED FAILED
+    env:
+      TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
+    displayName: Lock testbed
+    timeoutInMinutes: 240
+
+  - script: |
+      set -ex
+      echo "Prepare testbed"
+      echo "Preparing the testbed(add-topo, deploy-mg) may take 15-30 minutes. Before the testbed is ready, the progress of the test plan keeps displayed as 0, please be patient(We will improve the indication in a short time)"
+      echo "If the progress keeps as 0 for more than 1 hour, please cancel and retry this pipeline"
+      echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
+      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
+      # When "PREPARE_TESTBED" finish, it changes into "EXECUTING"
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 2400 --expected-states EXECUTING KVMDUMP FINISHED CANCELLED FAILED
+    env:
+      TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
+    displayName: Prepare testbed
+    timeoutInMinutes: 40
+
+  - script: |
+      set -ex
+      echo "Run test"
+      echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
+      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
+      # When "EXECUTING" finish, it changes into "KVMDUMP", "FAILED", "CANCELLED" or "FINISHED"
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 18000 --expected-states KVMDUMP FINISHED CANCELLED FAILED
+    env:
+      TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
+    displayName: Run test
+    timeoutInMinutes: 300
+
+  - script: |
+      set -ex
+      echo "KVM dump"
+      echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com"
+      echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID"
+      # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED"
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 43200 --expected-states FINISHED CANCELLED FAILED
+    condition: succeededOrFailed()
+    env:
+      TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
+    displayName: KVM dump
+    timeoutInMinutes: 20
+
+  - script: |
+      set -ex
+      echo "Try to cancel test plan $TEST_PLAN_ID, cancelling finished test plan has no effect."
+      python ./.azure-pipelines/test_plan.py cancel -i "$(TEST_PLAN_ID)"
+    condition: always()
+    env:
+      TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL)
+      TENANT_ID: $(TESTBED_TOOLS_MSAL_TENANT_ID)
+      CLIENT_ID: $(TESTBED_TOOLS_MSAL_CLIENT_ID)
+      CLIENT_SECRET: $(TESTBED_TOOLS_MSAL_CLIENT_SECRET)
+    displayName: Finalize running test plan

.azure-pipelines/run-test-template.yml

@@ -22,10 +22,15 @@ parameters:
   - sonic-6asic-vs.img.gz
 
 steps:
+- template: cleanup.yml
 - checkout: self
   clean: true
   displayName: 'checkout sonic-mgmt repo'
 
+- script: |
+    sudo rm -rf ../target
+  displayName: "Cleanup"
+
 - task: DownloadPipelineArtifact@2
   inputs:
     artifact: sonic-buildimage.vs

.azure-pipelines/trigger-publish-artifacts-build.yml

@@ -53,7 +53,7 @@ steps:
               **/*.raw\n
               **/*.img.gz\n
               **/*-rpc.gz\n
-              **/python-saithrift*.deb"},
+              **/*.deb"},
       publishContext: {"publishPrefix":"${{ parameters.publishPrefix }}",
         "keepArtifactName":false,
         "dockerImagePatterns":"target/*-rpc.gz",

.github/codeql/codeql-config.yml

@@ -0,0 +1,4 @@
+name: "CodeQL config"
+queries:
+  - uses: security-and-quality
+  - uses: security-extended

.github/pull_request_template.md

@@ -39,7 +39,7 @@ Write a short (one line) summary that describes the changes in this
 pull request for inclusion in the changelog:
 -->
 
-#### Ensure to add label/tag for the feature raised. example - [PR#2174](https://github.com/sonic-net/sonic-utilities/pull/2174) where, Generic Config and Update feature has been labelled as GCU.
+#### Ensure to add label/tag for the feature raised. example - PR#2174 under sonic-utilities repo. where, Generic Config and Update feature has been labelled as GCU.
 
 #### Link to config_db schema for YANG module changes
 <!--

.github/workflows/README.md

@@ -0,0 +1,38 @@
+#  Github actions README
+This is an introduction about auto-cherry-pick workflow.
+take 202205 branch for example:
+1. pr_cherrypick_prestep:
+```mermaid
+graph
+Start(Origin PR) --> A{merged?}
+A -- NO --> STOP
+A -- YES --> A1{Approved<br> for 202205<br> Branch?}
+A1 -- NO --> STOP
+A1 -- YES --> A2(pr_cherrypick_prestep)
+B(pr_cherrypick_prestep)
+B --> B1{cherry pick<br>conflict?}
+B1 -- YES --> B2(Add tag:<br>Cherry Pick Confclit_202205) --> B3(Add comment:<br>refer author code conflict) --> STOP1(STOP)
+B1 -- NO --> B4(Create New PR) -- success --> B5(New PR add tag:<br> automerge) --> B6(New PR add comment:<br>Origin PR link) --> B7(Origin PR add tag:<br>Created PR to 202205 Branch) --> B8(Origin PR add comment:<br>New PR link)
+B4 -- fail --> STOP1
+```
+
+2. automerge:
+```mermaid
+graph
+Start(PR azp finished successfully) --> A{author:<br>mssonicbld?}
+A -- NO --> STOP
+A -- YES --> B{tag:<br>automerge?} -- YES --> C(Merge PR)
+B -- NO --> STOP
+```
+
+3. pr_cherrypick_poststep:
+```mermaid
+graph
+A(PR is Merged) --> B{tag:<br>automerge?}
+B -- YES --> B1{author:<br>mssonicbld?}
+B1 -- YES --> B2{"title starts:<br>[action] [PR:123]"}
+B2 -- YES --> C(Origin PR remove tag:<br> Created PR to 202205 Branch) --> D(Origin PR add tag:<br> Included in 202205 Branch)
+B -- NO --> STOP
+B1 -- NO --> STOP
+B2 -- NO --> STOP
+```

.github/workflows/automerge.yml

@@ -13,10 +13,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: automerge
-      uses: 'pascalgn/automerge-action@v0.13.1'
+      uses: 'pascalgn/automerge-action@v0.15.5'
       env:
         GITHUB_TOKEN: '${{ secrets.TOKEN }}'
         MERGE_LABELS: 'automerge'
         MERGE_METHOD: 'squash'
         MERGE_FILTER_AUTHOR: 'mssonicbld'
         MERGE_DELETE_BRANCH: true
+        LOG: "TRACE"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,43 @@
+# For more infomation, please visit: https://github.com/github/codeql-action
+
+name: "CodeQL"
+
+on:
+  push:
+    branches:
+      - 'master'
+      - '202[0-9][0-9][0-9]'
+  pull_request_target:
+    branches:
+      - 'master'
+      - '202[0-9][0-9][0-9]'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        config-file: ./.github/codeql/codeql-config.yml
+        languages: ${{ matrix.language }}
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"