[pytest/ansible] Add support for multi-duts (#1432)

* [pytest/ansible] Add support for multi-duts * review comment: use duts instead of dut * Fixing various test cases to use duthost instead of testbed_devices['dut'] * trim spaces if any when present in the duts list signed-off-by: Tamer Ahmed <tamer.ahmed@microsoft.com>
sonic-net · May 5, 2020 · c0cdedb · c0cdedb
1 parent 979616a
commit c0cdedb
Show file tree

Hide file tree

Showing 43 changed files with 364 additions and 414 deletions.
diff --git a/ansible/config_sonic_basedon_testbed.yml b/ansible/config_sonic_basedon_testbed.yml
@@ -41,12 +41,17 @@
         testbed_file: testbed.csv
       when: testbed_file is not defined
 
+    - name: Set default dut index
+      set_fact:
+        dut_index: 0
+      when: dut_index is not defined
+
     - name: Gathering testbed information
       test_facts: testbed_name="{{ testbed_name }}" testbed_file="{{ testbed_file }}"
       delegate_to: localhost
 
     - fail: msg="The DUT you are trying to run test does not belongs to this testbed"
-      when: testbed_facts['dut'] != inventory_hostname
+      when: testbed_facts['duts'][dut_index] != inventory_hostname
 
     - name: set testbed_type
       set_fact:

diff --git a/ansible/library/test_facts.py b/ansible/library/test_facts.py
@@ -3,6 +3,7 @@
 import traceback
 import ipaddr as ipaddress
 import csv
+import string
 from operator import itemgetter
 from itertools import groupby
 import yaml
@@ -109,7 +110,7 @@ def __init__(self, testbed_file):
     def read_testbed_topo(self):
         CSV_FIELDS = ('conf-name', 'group-name', 'topo', 'ptf_image_name', 'ptf', 'ptf_ip', 'server', 'vm_base', 'dut', 'comment')
         with open(self.testbed_filename) as f:
-            topo = csv.DictReader(f, fieldnames=CSV_FIELDS)
+            topo = csv.DictReader(f, fieldnames=CSV_FIELDS, delimiter=',')
 
             # Validate all field are in the same order and are present
             header = next(topo)
@@ -125,6 +126,9 @@ def read_testbed_topo(self):
                     line['ptf_ip'] = str(ptfaddress.ip)
                     line['ptf_netmask'] = str(ptfaddress.netmask)
 
+                line['duts'] = line['dut'].translate(string.maketrans("", ""), "[] ").split(';')
+                del line['dut']
+
                 self.testbed_topo[line['conf-name']] = line
         return
 

diff --git a/ansible/roles/test/tasks/sonic.yml b/ansible/roles/test/tasks/sonic.yml
@@ -63,8 +63,13 @@
       test_facts: testbed_name="{{ testbed_name }}" testbed_file="{{ testbed_file }}"
       delegate_to: localhost
 
+    - name: Set default dut index
+      set_fact:
+        dut_index: 0
+      when: dut_index is not defined
+
     - fail: msg="The DUT you are trying to run test does not belongs to this testbed"
-      when: testbed_facts['dut'] != inventory_hostname
+      when: testbed_facts['duts'][dut_index] != inventory_hostname
 
     - name: set testbed_type
       set_fact:

diff --git a/ansible/testbed-cli.sh b/ansible/testbed-cli.sh
@@ -57,37 +57,38 @@ function usage
 
 function read_file
 {
- echo reading
+  echo reading
 
- # Filter testbed names in the first column in the testbed definition file
- line=$(cat $tbfile | grep "^$1,")
+  # Filter testbed names in the first column in the testbed definition file
+  line=$(cat $tbfile | grep "^$1,")
 
- if [ $? -ne 0 ]
- then
+  if [ $? -ne 0 ]
+  then
    echo "Couldn't find topology name '$1'"
    exit
- fi
+  fi
 
- NL='
+  NL='
 '
- case $line in
-  *"$NL"*) echo "Find more than one topology names in $tbfile"
-           exit
-           ;;
-        *) echo Found topology $1
-           ;;
- esac
-
- IFS=, read -r -a line_arr <<< $line
-
- testbed_name=${line_arr[1]}
- topo=${line_arr[2]}
- ptf_imagename=${line_arr[3]}
- ptf=${line_arr[4]}
- ptf_ip=${line_arr[5]}
- server=${line_arr[6]}
- vm_base=${line_arr[7]}
- dut=${line_arr[8]}
+  case $line in
+    *"$NL"*) echo "Find more than one topology names in $tbfile"
+             exit
+             ;;
+          *) echo Found topology $1
+             ;;
+  esac
+
+  IFS=, read -r -a line_arr <<< $line
+
+  testbed_name=${line_arr[1]}
+  topo=${line_arr[2]}
+  ptf_imagename=${line_arr[3]}
+  ptf=${line_arr[4]}
+  ptf_ip=${line_arr[5]}
+  server=${line_arr[6]}
+  vm_base=${line_arr[7]}
+  dut=${line_arr[8]//;/,}
+  duts=${dut//[\[\] ]/}
 }
 
 function start_vms
@@ -123,9 +124,9 @@ function add_topo
 
   read_file ${topology}
 
-  ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_add_vm_topology.yml --vault-password-file="${passwd}" -l "$server" -e topo_name="$topo_name" -e dut_name="$dut" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$testbed_name" -e ptf_imagename="$ptf_imagename" -e vm_type="$vm_type" $@ 
+  ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_add_vm_topology.yml --vault-password-file="${passwd}" -l "$server" -e topo_name="$topo_name" -e dut_name="$duts" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$testbed_name" -e ptf_imagename="$ptf_imagename" -e vm_type="$vm_type" $@ 
 
-  ansible-playbook fanout_connect.yml -i $vmfile --limit "$server" --vault-password-file="${passwd}" -e "dut=$dut" $@
+  ansible-playbook fanout_connect.yml -i $vmfile --limit "$server" --vault-password-file="${passwd}" -e "dut=$duts" $@
 
   # Delete the obsoleted arp entry for the PTF IP
   ip neighbor flush $ptf_ip
@@ -143,7 +144,7 @@ function remove_topo
 
   read_file ${topology}
 
-  ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_remove_vm_topology.yml --vault-password-file="${passwd}" -l "$server" -e topo_name="$topo_name" -e dut_name="$dut" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$testbed_name" -e ptf_imagename="$ptf_imagename" -e vm_type="$vm_type" $@
+  ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_remove_vm_topology.yml --vault-password-file="${passwd}" -l "$server" -e topo_name="$topo_name" -e dut_name="$duts" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$testbed_name" -e ptf_imagename="$ptf_imagename" -e vm_type="$vm_type" $@
 
   echo Done
 }
@@ -158,9 +159,9 @@ function renumber_topo
 
   read_file ${topology}
 
-  ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_renumber_vm_topology.yml --vault-password-file="${passwd}" -l "$server" -e topo_name="$topo_name" -e dut_name="$dut" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$testbed_name" -e ptf_imagename="$ptf_imagename" $@
+  ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_renumber_vm_topology.yml --vault-password-file="${passwd}" -l "$server" -e topo_name="$topo_name" -e dut_name="$duts" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$testbed_name" -e ptf_imagename="$ptf_imagename" $@
 
-  ansible-playbook fanout_connect.yml -i $vmfile --limit "$server" --vault-password-file="${passwd}" -e "dut=$dut" $@
+  ansible-playbook fanout_connect.yml -i $vmfile --limit "$server" --vault-password-file="${passwd}" -e "dut=$duts" $@
 
   echo Done
 }
@@ -171,11 +172,11 @@ function refresh_dut
   passwd=$2
   shift
   shift
-  echo "Refresh $dut in  '${topology}'"
+  echo "Refresh $duts in  '${topology}'"
 
   read_file ${topology}
 
-  ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_refresh_dut.yml --vault-password-file="${passwd}" -l "$server" -e topo_name="$topo_name" -e dut_name="$dut" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$testbed_name" -e ptf_imagename="$ptf_imagename" $@
+  ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_refresh_dut.yml --vault-password-file="${passwd}" -l "$server" -e topo_name="$topo_name" -e dut_name="$duts" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$testbed_name" -e ptf_imagename="$ptf_imagename" $@
 
   echo Done
 }
@@ -186,7 +187,7 @@ function connect_vms
 
   read_file $1
 
-  ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_connect_vms.yml --vault-password-file="$2" -l "$server" -e topo_name="$topo_name" -e dut_name="$dut" -e VM_base="$vm_base" -e topo="$topo" -e vm_set_name="$testbed_name"
+  ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_connect_vms.yml --vault-password-file="$2" -l "$server" -e topo_name="$topo_name" -e dut_name="$duts" -e VM_base="$vm_base" -e topo="$topo" -e vm_set_name="$testbed_name"
 
   echo Done
 }
@@ -197,7 +198,7 @@ function disconnect_vms
 
   read_file $1
 
-  ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_disconnect_vms.yml --vault-password-file="$2" -l "$server" -e topo_name="$topo_name" -e dut_name="$dut" -e VM_base="$vm_base" -e topo="$topo" -e vm_set_name="$testbed_name"
+  ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_disconnect_vms.yml --vault-password-file="$2" -l "$server" -e topo_name="$topo_name" -e dut_name="$duts" -e VM_base="$vm_base" -e topo="$topo" -e vm_set_name="$testbed_name"
 
   echo Done
 }
@@ -216,7 +217,7 @@ function generate_minigraph
 
   read_file $topology
 
-  ansible-playbook -i "$inventory" config_sonic_basedon_testbed.yml --vault-password-file="$passfile" -l "$dut" -e testbed_name="$topology" -e testbed_file=$tbfile -e vm_file=$vmfile -e local_minigraph=true $@
+  ansible-playbook -i "$inventory" config_sonic_basedon_testbed.yml --vault-password-file="$passfile" -l "$duts" -e testbed_name="$topology" -e testbed_file=$tbfile -e vm_file=$vmfile -e local_minigraph=true $@
 
   echo Done
 }
@@ -234,7 +235,7 @@ function deploy_minigraph
 
   read_file $topology
 
-  ansible-playbook -i "$inventory" config_sonic_basedon_testbed.yml --vault-password-file="$passfile" -l "$dut" -e testbed_name="$topology" -e testbed_file=$tbfile -e vm_file=$vmfile -e deploy=true -e save=true $@
+  ansible-playbook -i "$inventory" config_sonic_basedon_testbed.yml --vault-password-file="$passfile" -l "$duts" -e testbed_name="$topology" -e testbed_file=$tbfile -e vm_file=$vmfile -e deploy=true -e save=true $@
 
   echo Done
 }
@@ -252,7 +253,7 @@ function test_minigraph
 
   read_file $topology
 
-  ansible-playbook -i "$inventory" --diff --connection=local --check config_sonic_basedon_testbed.yml --vault-password-file="$passfile" -l "$dut" -e testbed_name="$topology" -e testbed_file=$tbfile -e vm_file=$vmfile -e local_minigraph=true $@
+  ansible-playbook -i "$inventory" --diff --connection=local --check config_sonic_basedon_testbed.yml --vault-password-file="$passfile" -l "$duts" -e testbed_name="$topology" -e testbed_file=$tbfile -e vm_file=$vmfile -e local_minigraph=true $@
 
   echo Done
 }
@@ -274,7 +275,7 @@ function connect_topo
 
   read_file $1
 
-  ansible-playbook fanout_connect.yml -i $vmfile --limit "$server" --vault-password-file="$2" -e "dut=$dut"
+  ansible-playbook fanout_connect.yml -i $vmfile --limit "$server" --vault-password-file="$2" -e "dut=$duts"
 }
 
 vmfile=veos

diff --git a/ansible/vtestbed.csv b/ansible/vtestbed.csv
@@ -1,4 +1,4 @@
 # conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,server,vm_base,dut,comment
-vms-kvm-t0,vms6-1,t0,docker-ptf,ptf-unknown,10.250.0.102/24,server_1,VM0100,vlab-01,Tests virtual switch vm
-vms-kvm-t0-64,vms6-1,t0-64,docker-ptf,ptf-unknown,10.250.0.102/24,server_1,VM0100,vlab-02,Tests virtual switch vm
-vms-kvm-t1-lag,vms6-2,t1-lag,docker-ptf,ptf-unknown,10.250.0.106/24,server_1,VM0104,vlab-03,Tests virtual switch vm
+vms-kvm-t0,vms6-1,t0,docker-ptf,ptf-unknown,10.250.0.102/24,server_1,VM0100,[vlab-01],Tests virtual switch vm
+vms-kvm-t0-64,vms6-1,t0-64,docker-ptf,ptf-unknown,10.250.0.102/24,server_1,VM0100,[vlab-02],Tests virtual switch vm
+vms-kvm-t1-lag,vms6-2,t1-lag,docker-ptf,ptf-unknown,10.250.0.106/24,server_1,VM0104,[vlab-03],Tests virtual switch vm
diff --git a/tests/bgp/test_bgp_fact.py b/tests/bgp/test_bgp_fact.py
@@ -1,13 +1,11 @@
 from ansible_host import AnsibleHost
 
-def test_bgp_facts(ansible_adhoc, testbed,duthost):
+def test_bgp_facts(duthost):
     """compare the bgp facts between observed states and target state"""
 
-    hostname = testbed['dut']
-    ans_host = AnsibleHost(ansible_adhoc, hostname)
     npus = duthost.num_npus()
-    bgp_facts = ans_host.bgp_facts(num_npus=npus)['ansible_facts']
-    mg_facts  = ans_host.minigraph_facts(host=hostname)['ansible_facts']
+    bgp_facts = duthost.bgp_facts()['ansible_facts']
+    mg_facts  = duthost.minigraph_facts(host=duthost.hostname)['ansible_facts']
 
     for k, v in bgp_facts['bgp_neighbors'].items():
         # Verify bgp sessions are established

diff --git a/tests/common/fixtures/advanced_reboot.py b/tests/common/fixtures/advanced_reboot.py
@@ -25,10 +25,11 @@ class AdvancedReboot:
     inboot/preboot list. The class transfers number of configuration files to the dut/ptf in preparation for reboot test.
     Test cases can trigger test start utilizing runRebootTestcase API.
     '''
-    def __init__(self, request, testbed_devices, testbed, **kwargs):
+    def __init__(self, request, duthost, testbed_devices, testbed, **kwargs):
         '''
         Class contructor.
         @param request: pytest request object
+        @param duthost: AnsibleHost instance of DUT
         @param testbed_devices: fixture provides information about testbed devices
         @param testbed: fixture provides information about testbed
         @param kwargs: extra parameters including reboot type
@@ -38,7 +39,7 @@ def __init__(self, request, testbed_devices, testbed, **kwargs):
         )
 
         self.request = request
-        self.duthost = testbed_devices['dut']
+        self.duthost = duthost
         self.ptfhost = testbed_devices['ptf']
         self.localhost = testbed_devices['localhost']
         self.testbed = testbed
@@ -479,10 +480,11 @@ def tearDown(self):
             self.__restorePrevImage()
 
 @pytest.fixture
-def get_advanced_reboot(request, testbed_devices, testbed):
+def get_advanced_reboot(request, duthost, testbed_devices, testbed):
     '''
     Pytest test fixture that provides access to AdvancedReboot test fixture
         @param request: pytest request object
+        @param duthost: AnsibleHost instance of DUT
         @param testbed_devices: fixture provides information about testbed devices
         @param testbed: fixture provides information about testbed
     '''
@@ -493,7 +495,7 @@ def get_advanced_reboot(**kwargs):
         API that returns instances of AdvancedReboot class
         '''
         assert len(instances) == 0, "Only one instance of reboot data is allowed"
-        advancedReboot = AdvancedReboot(request, testbed_devices, testbed, **kwargs)
+        advancedReboot = AdvancedReboot(request, duthost, testbed_devices, testbed, **kwargs)
         instances.append(advancedReboot)
         return advancedReboot
 

diff --git a/tests/common/fixtures/conn_graph_facts.py b/tests/common/fixtures/conn_graph_facts.py
@@ -3,9 +3,8 @@
 import json
 
 @pytest.fixture(scope="module")
-def conn_graph_facts(testbed_devices):
+def conn_graph_facts(duthost, testbed_devices):
     conn_graph_facts = dict()
-    dut = testbed_devices["dut"]
     localhost = testbed_devices["localhost"]
 
     base_path = os.path.dirname(os.path.realpath(__file__))
@@ -14,9 +13,9 @@ def conn_graph_facts(testbed_devices):
     if os.path.exists(inv_mapping_file):
         with open(inv_mapping_file) as fd:
             inv_map = json.load(fd)
-        inv_file = dut.host.options['inventory'].split('/')[-1]
+        inv_file = duthost.host.options['inventory'].split('/')[-1]
         if inv_map and inv_file in inv_map:
             lab_conn_graph_file = os.path.join(base_path, "../../../ansible/files/{}".format(inv_map[inv_file]))
 
-            conn_graph_facts = localhost.conn_graph_facts(host=dut.hostname, filename=lab_conn_graph_file)['ansible_facts']
+            conn_graph_facts = localhost.conn_graph_facts(host=duthost.hostname, filename=lab_conn_graph_file)['ansible_facts']
     return conn_graph_facts
diff --git a/tests/common/plugins/dut_monitor/pytest_dut_monitor.py b/tests/common/plugins/dut_monitor/pytest_dut_monitor.py
@@ -29,14 +29,14 @@ def __init__(self, thresholds):
         self.thresholds = thresholds
 
     @pytest.fixture(autouse=True, scope="module")
-    def dut_ssh(self, testbed, creds):
+    def dut_ssh(self, duthost, creds):
         """Establish SSH connection with DUT"""
-        ssh = DUTMonitorClient(host=testbed["dut"], user=creds["sonicadmin_user"],
+        ssh = DUTMonitorClient(host=duthost.hostname, user=creds["sonicadmin_user"],
                                password=creds["sonicadmin_password"])
         yield ssh
 
     @pytest.fixture(autouse=True, scope="function")
-    def dut_monitor(self, dut_ssh, localhost, duthost, testbed_devices):
+    def dut_monitor(self, dut_ssh, localhost, duthost):
         """
         For each test item starts monitoring of hardware resources consumption on the DUT
         """
@@ -50,8 +50,8 @@ def dut_monitor(self, dut_ssh, localhost, duthost, testbed_devices):
             general_thresholds = yaml.safe_load(stream)
         dut_thresholds = general_thresholds["default"]
 
-        dut_platform = testbed_devices["dut"].facts["platform"]
-        dut_hwsku = testbed_devices["dut"].facts["hwsku"]
+        dut_platform = duthost.facts["platform"]
+        dut_hwsku = duthost.facts["hwsku"]
         if dut_platform in general_thresholds:
             dut_thresholds.update(general_thresholds[dut_platform]["default"])
             if dut_hwsku in general_thresholds[dut_platform]["hwsku"]:

diff --git a/tests/common/plugins/sanity_check/__init__.py b/tests/common/plugins/sanity_check/__init__.py
@@ -49,10 +49,9 @@ def _update_check_items(old_items, new_items, supported_items):
 
 
 @pytest.fixture(scope="module", autouse=True)
-def sanity_check(testbed_devices, request, fanouthosts):
+def sanity_check(testbed_devices, duthost, request, fanouthosts):
     logger.info("Start pre-test sanity check")
 
-    dut = testbed_devices["dut"]
     localhost = testbed_devices["localhost"]
 
     skip_sanity = False
@@ -102,8 +101,8 @@ def sanity_check(testbed_devices, request, fanouthosts):
         logger.info("No sanity check item is specified, no post-test sanity check")
         return
 
-    print_logs(dut, constants.PRINT_LOGS)
-    check_results = do_checks(dut, check_items)
+    print_logs(duthost, constants.PRINT_LOGS)
+    check_results = do_checks(duthost, check_items)
     logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check results: !!!!!!!!!!!!!!!!\n%s" % \
                 json.dumps(check_results, indent=4))
     if any([result["failed"] for result in check_results]):
@@ -112,9 +111,9 @@ def sanity_check(testbed_devices, request, fanouthosts):
             return
 
         logger.info("Pre-test sanity check failed, try to recover, recover_method=%s" % recover_method)
-        recover(dut, localhost, fanouthosts, check_results, recover_method)
+        recover(duthost, localhost, fanouthosts, check_results, recover_method)
         logger.info("Run sanity check again after recovery")
-        new_check_results = do_checks(dut, check_items)
+        new_check_results = do_checks(duthost, check_items)
         logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check after recovery results: !!!!!!!!!!!!!!!!\n%s" % \
                     json.dumps(new_check_results, indent=4))
         if any([result["failed"] for result in new_check_results]):
@@ -131,7 +130,7 @@ def sanity_check(testbed_devices, request, fanouthosts):
         logger.info("No post-test check is required. Done post-test sanity check")
         return
 
-    post_check_results = do_checks(dut, check_items)
+    post_check_results = do_checks(duthost, check_items)
     logger.info("!!!!!!!!!!!!!!!! Post-test sanity check results: !!!!!!!!!!!!!!!!\n%s" % \
                 json.dumps(post_check_results, indent=4))
     if any([result["failed"] for result in post_check_results]):