rdma sw fix.

fpgasystems · Sep 26, 2023 · 4629886 · 4629886
1 parent 42e63d5
commit 4629886
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 31 deletions.
diff --git a/flow_alveo.sh b/flow_alveo.sh
@@ -95,36 +95,31 @@ if [ $DRV_INSERT -eq 1 ]; then
 
     echo "*** Rescan PCIe ..."	
     echo " ** "
-	    #parallel-ssh -H "$hostlist" -x '-tt' 'sudo /opt/cli/program/pci_hot_plug "$(hostname -s)"'
+	    #parallel-ssh -H "$hostlist" -x '-tt' 'sudo /opt/sgrt/cli/program/pci_hot_plug "$(hostname -s)"'
         # read -p "Hot-reset done. Press enter to load the driver or Ctrl-C to exit."
-	    parallel-ssh -H "$hostlist" -x '-tt' 'upstream_port=$(/opt/cli/get/get_fpga_device_param 1 upstream_port) && root_port=$(/opt/cli/get/get_fpga_device_param 1 root_port) && LinkCtl=$(/opt/cli/get/get_fpga_device_param 1 LinkCtl) && sudo /opt/cli/program/pci_hot_plug 1 $upstream_port $root_port $LinkCtl'
+	    parallel-ssh -H "$hostlist" -x '-tt' 'upstream_port=$(/opt/sgrt/cli/get/get_fpga_device_param 1 upstream_port) && root_port=$(/opt/sgrt/cli/get/get_fpga_device_param 1 root_port) && LinkCtl=$(/opt/sgrt/cli/get/get_fpga_device_param 1 LinkCtl) && sudo /opt/sgrt/cli/program/pci_hot_plug 1 $upstream_port $root_port $LinkCtl'
 	    # read -p "Hot-reset done. Press enter to load the driver or Ctrl-C to exit."
 	echo "*** Compiling the driver ..."
     echo " ** "
 	    parallel-ssh -H "$hostlist" "make -C $BASE_PATH/$DRV_PATH"
 
     echo "*** Loading the driver ..."
     echo " ** "
-        qsfp_ip="DEVICE_$3_IP_ADDRESS_HEX"
-        qsfp_mac="DEVICE_$3_MAC_ADDRESS"
+        qsfp_ip="DEVICE_1_IP_ADDRESS_HEX_$3"
+        qsfp_mac="DEVICE_1_MAC_ADDRESS_$3"
 
 	    parallel-ssh -H "$hostlist" -x '-tt' "sudo insmod $BASE_PATH/$DRV_PATH/coyote_drv.ko ip_addr_q$3=\$$qsfp_ip mac_addr_q$3=\$$qsfp_mac"
-        parallel-ssh -H "$hostlist" -x '-tt' "sudo /opt/cli/program/fpga_chmod 0"
+        parallel-ssh -H "$hostlist" -x '-tt' "sudo /opt/sgrt/cli/program/fpga_chmod 0"
 
         # shopt -s nullglob
         # dev_arr=(/dev/fpga*)
         # for fdev in "${!dev_arr[@]}"; do
         #     echo $fdev
-        #     parallel-ssh -H "$hostlist" -x '-tt' "sudo /opt/cli/program/fpga_chmod $fdev"
+        #     parallel-ssh -H "$hostlist" -x '-tt' "sudo /opt/sgrt/cli/program/fpga_chmod $fdev"
         # done
 
     echo "*** Driver loaded"
     echo " ** "
-
-        #parallel-ssh -H "$hostlist" -x '-tt' "sudo insmod $BASE_PATH/$DRV_PATH/coyote_drv.ko ip_addr_q0=$DEVICE_1_IP_ADDRESS_HEX_0 mac_addr_q0=$DEVICE_1_MAC_ADDRESS_0 && "
-        #parallel-ssh -H "$hostlist" -x '-tt' "sudo /opt/cli/program/fpga_chmod 0"
-    #echo "*** Driver loaded"
-    #echo " ** "
 fi
 
 
diff --git a/sw/examples/perf_rdma/main.cpp b/sw/examples/perf_rdma/main.cpp
@@ -20,7 +20,7 @@
 #include "ibvQpMap.hpp"
 
 #define EN_THR_TESTS
-//#define EN_LAT_TESTS
+#define EN_LAT_TESTS
 
 using namespace std;
 using namespace std::chrono;
@@ -39,10 +39,10 @@ constexpr auto const port = 18488;
 
 /* Bench */
 constexpr auto const defNBenchRuns = 1; 
-constexpr auto const defNRepsThr = 1;
-constexpr auto const defNRepsLat = 1;
-constexpr auto const defMinSize = 1024;
-constexpr auto const defMaxSize = 1024;
+constexpr auto const defNRepsThr = 1000;
+constexpr auto const defNRepsLat = 100;
+constexpr auto const defMinSize = 128;
+constexpr auto const defMaxSize = 32 * 1024;
 constexpr auto const defOper = 0;
 
 int main(int argc, char *argv[])  
@@ -125,9 +125,9 @@ int main(int argc, char *argv[])
     struct ibvSendWr wr;
 
     memset(&sg, 0, sizeof(sg));
-    sg.type.rdma.local_offs = 0;
-    sg.type.rdma.remote_offs = 0;
-    sg.type.rdma.len = size;
+    sg.local_offs = 0;
+    sg.remote_offs = 0;
+    sg.len = size;
 
     memset(&wr, 0, sizeof(wr));
     wr.sg_list = &sg;
@@ -145,7 +145,7 @@ int main(int argc, char *argv[])
     } 
 
     PR_HEADER("RDMA BENCHMARK");
-    while(sg.type.rdma.len <= max_size) {
+    while(sg.len <= max_size) {
         // Setup
         iqp->ibvClear();
         iqp->ibvSync(mstr);
@@ -178,10 +178,8 @@ int main(int argc, char *argv[])
             };
             bench.runtime(benchmark_thr);
             std::cout << std::fixed << std::setprecision(2);
-            std::cout << std::setw(8) << sg.type.rdma.len << " [bytes], thoughput: " 
-                      << std::setw(8) << ((1 + oper) * ((1000 * sg.type.rdma.len))) / ((bench.getAvg()) / n_reps_thr) << " [MB/s], latency: "; 
-
-            std::cout << std::endl << std::endl << "ACKs: " << cproc->ibvCheckAcks() << std::endl;
+            std::cout << std::setw(8) << sg.len << " [bytes], thoughput: " 
+                      << std::setw(8) << ((1 + oper) * ((1000 * sg.len))) / ((bench.getAvg()) / n_reps_thr) << " [MB/s], latency: "; 
 #endif
 
             // Reset
@@ -244,7 +242,8 @@ int main(int argc, char *argv[])
             }
         }  
 
-        sg.type.rdma.len *= 2;
+        //std::cout << std::endl << std::endl << "ACKs: " << cproc->ibvCheckAcks() << std::endl;
+        sg.len *= 2;
     }
     std::cout << std::endl;
 

diff --git a/sw/src/cProcess.cpp b/sw/src/cProcess.cpp
@@ -44,8 +44,6 @@ cProcess::cProcess(int32_t vfid, pid_t pid, cSched *csched) : vfid(vfid), pid(pi
 	// Registration
 	uint64_t tmp[2];
 	tmp[0] = pid;
-
-	std::cout << "HR 0" << std::endl;
 
 	// register pid
 	if(ioctl(fd, IOCTL_REGISTER_PID, &tmp))
@@ -608,7 +606,6 @@ int32_t cProcess::ibvGetCompleted(int32_t &cpid) {
  * @param wr - operation struct
  */
 void cProcess::ibvPostSend(ibvQp *qp, ibvSendWr *wr) {
-    std::cout << "HERE SEND ENTRY" << std::endl;
     if(fcnfg.en_rdma) {
         if(qp->local.ip_addr == qp->remote.ip_addr) {
             for(int i = 0; i < wr->num_sge; i++) {

diff --git a/sw/src/ibvQpConn.cpp b/sw/src/ibvQpConn.cpp
@@ -179,11 +179,16 @@ void ibvQpConn::ibvClear() {
  */
 uint32_t ibvQpConn::readAck() {
     uint32_t ack;
-
+
+    while(::read(connection, &ack, sizeof(uint32_t)) != sizeof(uint32_t)) {
+        usleep(1000);
+    }
+    /*
     if (::read(connection, &ack, sizeof(uint32_t)) != sizeof(uint32_t)) {
-        ::close(connection);
-        throw std::runtime_error("Could not read ack\n");
+        //::close(connection);
+        //throw std::runtime_error("Could not read ack\n");
     }
+    */
 
     return ack;
 }