Add route rules for additional workload networks

Previously, HAProxy would not route to additional workload networks
without user customization because we did not give them the option
to provide workload networks interfaces via the UI.

This change implements the ability for the user to provide workload
networks the user wishes to route to. Since we expect workload
networks to be routable to each other, we can program route rules
to user-provided CIDR ranges in which routes will exit via the
workload default gateway. These routes are configurable via a new
file located at /etc/vmware/workload-networks.cfg. This file is
written once just before cloud-init performs the bootstrapping.

This change also fixes a few bugs in route table configuration.

Closes #15, #11, #10

ansible/roles/cloudinit/files/var/lib/vmware/ovf-to-cloud-init.sh

@@ -38,7 +38,7 @@ management_gw_key="network.management_gateway"
 workload_gw_key="network.workload_gateway"
 frontend_gw_key="network.frontend_gateway"
 
-workload_routes_key="network.workload.routes"
+workload_networks_key="network.additional_workload_networks"
 
 # These are the display names for the nics
 management_net_name="management"
@@ -86,8 +86,13 @@ escapeString () {
     echo "$escaped"
 }
 
-getWorkloadRoutes() {
-  routes=$(ovf-rpctool get.ovf "${workload_routes_key}")
+# Extract the additional workload networks and store them in the appropriate file.
+# These CIDRs will be picked up by the route-tables service and the appropriate routes will be created.
+writeWorkloadNetworks() {
+  networks=$(ovf-rpctool get.ovf "${workload_networks_key}")
+  if [ -n "${networks}" ]; then
+    echo "${networks//,/$'\n'}" > /etc/vmware/workload-networks.cfg
+  fi
 }
 
 # Persist a string to a file
@@ -364,6 +369,7 @@ if [ ! -f "$first_boot_path" ]; then
     setDataPlaneAPIPort
     writeCAfiles
     writeAnyipConfig
+    writeWorkloadNetworks
     writeNetPostConfig
 else
     ensureMetadata

ansible/roles/vmware/files/var/lib/vmware/routetablectl.sh

@@ -62,6 +62,11 @@ CONFIG_FILE="${CONFIG_FILE:-/etc/vmware/route-tables.cfg}"
 # The path to the file with the route table identifiers.
 RT_TABLES_FILE="${RT_TABLES_FILE:-/etc/iproute2/rt_tables}"
 
+# Path to the file for additional workload networks.
+WORKLOAD_NETWORKS_FILE="${WORKLOAD_NETWORKS_FILE:-/etc/vmware/workload-networks.cfg}"
+
+# Name of the route table for workload networks.
+WORKLOAD_RT="${WORKLOAD_RT:-${RT_TABLE_NAME_PREFIX}workload}"
 
 ################################################################################
 ##                                   funcs
@@ -74,12 +79,19 @@ function error() {
   echo "${@}" 1>&2
   return "${exit_code}"
 }
+
 function fatal() {
   error "${@}"
   exit 1
 }
+
 function echo2() {
-  echo "${@}" 1>&2
+  echo "${@}" 2>&1
+}
+
+function call() {
+  echo2 "${@}"
+  eval "${@}"
 }
 
 # Returns the name of the device that has the provided MAC address.
@@ -100,43 +112,60 @@ function down_routes() {
       route_table_name="$(echo "${line}" | awk -F' ' '{print $2}')"
       echo2 "discovered route table ${route_table_name}"
 
-      # Remove the rule for this route table. If the route table file does not match
-      # the actual route tables present, ignore the failure so that tables can be re-added gracefully.
-      route_rule="$(ip rule | grep -F "${route_table_name}" | awk -F':[[:space:]]' '{print $2}')" || \
-      echo2 "ignoring failed attempt to find existing table"
-      echo2 "removing ip rule: ${route_rule}"
-      # shellcheck disable=SC2086
-      ip rule del ${route_rule} || echo2 "ignoring failed attempt to delete route rule \"${route_rule}\""
-
-      # Remove the default route for this route table. If the route file does not match
-      # the actual route tables present, ignore the failure so that route can be re-added gracefully.
-      echo2 "removing default route for ${route_table_name}"
-      ip route del table "${route_table_name}" default || \
-      echo "ignoring failed attempt to delete route table \"${route_table_name}\""
+      # Remove the rules for this route table.
+      while ip call "rule del from 0/0 to 0/0 table ${route_table_name} 2>/dev/null"; do true; done
+
+      # Remove any existing routes from our route tables.
+      while IFS= read -r route; do
+        call "ip route del table ${route_table_name} ${route}"
+      done < <(ip route show table "${route_table_name}")
     fi
-  done <"${RT_TABLES_FILE}"
+  done < "${RT_TABLES_FILE}"
   mv -f "${RT_TABLES_FILE}.tmp" "${RT_TABLES_FILE}"
 }
 
 # Adds route tables to the route tables file. Prevents duplicates from being added.
-add_route_tables() {
-  tables=$(grep -E '^\w' "${CONFIG_FILE}" | cut -d, -f1,2 | uniq)
+function add_route_tables() {
+  tables=$(grep -E '^\w' "${CONFIG_FILE}" || echo "" | cut -d, -f1,2 | uniq)
   for table in ${tables}; do
     IFS=, read -ra line <<< "${table}"
-    cfg_table_id="${line[0]}"
-    cfg_table_name="${line[1]}"
-    echo2 "create new route table id=${cfg_table_id} name=${route_table_name}"
-    printf '%d\t%s\n' "${cfg_table_id}" "${route_table_name}" >>"${RT_TABLES_FILE}"
+    route_table_id="${line[0]}"
+    route_table_name="${RT_TABLE_NAME_PREFIX}${line[1]}"
+    echo2 "create new route table id=${route_table_id} name=${route_table_name}"
+    printf '%d\t%s\n' "${route_table_id}" "${route_table_name}" >>"${RT_TABLES_FILE}"
   done
 }
 
+# Adds lookup rules for workload routes. The net result is that additional workloads can be reached
+# via the default gateway of the workload network route table.
+function add_workload_network_rules() {
+  if [ ! -f "${WORKLOAD_NETWORKS_FILE}" ]; then
+    echo2 "no additional workload networks detected"
+    return
+  fi
+
+  while IFS= read -r cfg_cidr; do
+    # Skip empty and commented lines.
+    if [ -z "${cfg_cidr}" ] || [ "${cfg_cidr::1}" == "#" ]; then
+      continue
+    fi
+    call "ip rule add to ${cfg_cidr} lookup ${WORKLOAD_RT}"
+  done < "${WORKLOAD_NETWORKS_FILE}"
+}
+
 # Enables the custom route tables.
 function up_routes() {
   # Enabling the custom route tables first requires removing any custom route
   # tables.
   down_routes
 
+  if [ ! -f "${CONFIG_FILE}" ]; then
+    echo2 "missing config file ${CONFIG_FILE}"
+    return 0
+  fi
+
   add_route_tables
+  add_workload_network_rules
 
   while IFS= read -r line; do
     # Skip empty and commented lines.
@@ -148,7 +177,6 @@ function up_routes() {
     IFS=, read -ra line_parts <<<"${line}"
 
     # Store route table configuration's parts.
-    cfg_table_id="${line_parts[0]}"
     cfg_table_name="${line_parts[1]}"
     cfg_mac_addr="${line_parts[2]}"
     cfg_cidr="${line_parts[3]}"
@@ -164,18 +192,14 @@ function up_routes() {
     if [[ "${cfg_gateway}" == "" ]]; then
         cfg_destination=$(python3 -c "import sys; import ipaddress; print(ipaddress.ip_network(sys.argv[1], strict=False))" "${cfg_cidr}")
         host="$(echo "${cfg_cidr}" | cut -d/ -f 1)"
-        cmd="ip route add table ${route_table_name} ${cfg_destination} dev ${cfg_dev} proto kernel scope link src ${host}"
-        echo2 "create route with cmd: ${cmd}"
-        eval "${cmd}"
+        call "ip route add table ${route_table_name} ${cfg_destination} dev ${cfg_dev} proto kernel scope link src ${host}"
     else
         # Create default route for new route table.
-        echo2 "create default route for ${route_table_name}"
-        ip route add table "${route_table_name}" default via "${cfg_gateway}" dev "${cfg_dev}" proto static
+        call "ip route add table ${route_table_name} default via ${cfg_gateway} dev ${cfg_dev} proto static"
+        # Create IP rule for new route table.
+        call "ip rule add from ${cfg_cidr} lookup ${route_table_name}"
     fi
 
-    # Create IP rule for new route table.
-    echo2 "create IP rule for ${route_table_name}"
-    ip rule add from "${cfg_cidr}" lookup "${route_table_name}"
   done <"${CONFIG_FILE}"
 }
 

hack/image-build-ova.py

@@ -611,12 +611,16 @@ def stream_optimize_vmdk_files(inlist):
         <Label>2.6. Workload Gateway</Label>
         <Description>The gateway address for the workload network</Description>
       </Property>
+      <Property ovf:key="additional_workload_networks" ovf:type="string" ovf:userConfigurable="true">
+        <Label>2.7. Additional Workload Networks</Label>
+        <Description>(Optional) A comma-separated list of networks in CIDR notation (e.g. 192.168.0.1/24) to the workload networks. These networks must be routable via the Workload Gateway. This list must not include the primary workload network.</Description>
+      </Property>
       <Property ovf:key="frontend_ip" ovf:type="string" ovf:userConfigurable="true" ovf:configuration="frontend">
-        <Label>2.7. Frontend IP</Label>
+        <Label>2.8. Frontend IP</Label>
         <Description>(Optional) The static IP address for the appliance on the Frontend Port Group in CIDR format (Eg. ip/subnet mask bits). This IP must be outside of the Load Balancer IP Range</Description>
       </Property>
       <Property ovf:key="frontend_gateway" ovf:type="string" ovf:userConfigurable="true" ovf:configuration="frontend">
-        <Label>2.8. Frontend Gateway</Label>
+        <Label>2.9. Frontend Gateway</Label>
         <Description>(Optional) The gateway address for the frontend network</Description>
       </Property>
     </ProductSection>

hack/test-route-programs.sh

@@ -235,6 +235,9 @@ function test_anyiproutectl() {
 set -o errexit
 set -o nounset
 set -o pipefail
+set -v
+
+echo "starting test" >&2
 
 # Ping each of the IP addresses and expect an error for each one.
 ! ping -c2 -W1 "${ANYIP_IP_SLASH_32}"
@@ -331,6 +334,7 @@ EOF
 set -o errexit
 set -o nounset
 set -o pipefail
+set -v
 
 # Run the program with an empty config file and expect no errors.
 /var/lib/vmware/routetablectl.sh up
@@ -344,6 +348,11 @@ cat <<EOD >/etc/vmware/route-tables.cfg
 3,workload,${DOCKER_NET_3_MAC},${DOCKER_IP_NET_3}
 EOD
 
+# Create the networks file.
+# It's a newline-delimited list of CIDRs.
+echo "10.169.10.0/24" > /etc/vmware/workload-networks.cfg
+echo "10.169.20.0/24" >> /etc/vmware/workload-networks.cfg
+
 # Run the program with a populated config file and expect no errors.
 /var/lib/vmware/routetablectl.sh up
 
@@ -352,8 +361,12 @@ grep $'2\trtctl_frontend' /etc/iproute2/rt_tables
 grep $'3\trtctl_workload' /etc/iproute2/rt_tables
 
 # Assert the expected IP rules exist.
-ip rule | grep rtctl_frontend
-ip rule | grep rtctl_workload
+ip rule show table rtctl_frontend
+ip rule show table rtctl_workload
+
+# Assert that rules for our workload networks exist.
+ip rule show table rtctl_workload | grep "10.169.10.0 /24"
+ip rule show table rtctl_workload | grep "10.169.20.0 /24"
 
 # Assert the expected default gateways exist.
 ip route show table rtctl_frontend | grep default
@@ -366,14 +379,18 @@ ip route show table rtctl_workload | grep default
 ! grep $'2\trtctl_frontend' /etc/iproute2/rt_tables
 ! grep $'3\trtctl_workload' /etc/iproute2/rt_tables
 
-# Assert the expected IP rules DO NOT exist.
+# Assert the expected IP rules DO NOT exist. This also applies to workload networks.
 ! ip rule | grep rtctl_frontend'
 ! ip rule | grep rtctl_workload'
 
 # Assert the expected default gateways DO NOT exist.
 ! ip route show table rtctl_frontend 2>/dev/null
 ! ip route show table rtctl_workload 2>/dev/null
 
+# Assert that rules for our workload networks DO NOT exist.
+! ip rule show table rtctl_workload | grep "10.169.10.0 /24" 2>/dev/null
+! ip rule show table rtctl_workload | grep "10.169.20.0 /24" 2>/dev/null
+
 # Truncate the config file.
 printf '' >/etc/vmware/route-tables.cfg
 
@@ -384,18 +401,19 @@ printf '' >/etc/vmware/route-tables.cfg
 sleep 1
 
 # Update the config file and assert the appropriate actions occur.
-echo "2,frontend,${DOCKER_NET_2_MAC},${DOCKER_NET_2_CIDR},${DOCKER_NET_2_GATEWAY}" >/etc/vmware/route-tables.cfg
+echo "2,workload,${DOCKER_NET_3_MAC},${DOCKER_NET_3_CIDR},${DOCKER_NET_3_GATEWAY}" >/etc/vmware/route-tables.cfg
 sleep 2
-grep $'2\trtctl_frontend' /etc/iproute2/rt_tables
-ip rule | grep rtctl_frontend
-ip route show table rtctl_frontend | grep default
+grep $'2\trtctl_workload' /etc/iproute2/rt_tables
+ip rule | grep rtctl_workload
+ip route show table rtctl_workload | grep default
 
 # Update the config file and assert the appropriate actions occur.
-echo "3,workload,${DOCKER_NET_3_MAC},${DOCKER_NET_3_CIDR},${DOCKER_NET_3_GATEWAY}" >/etc/vmware/route-tables.cfg
+echo "3,frontend,${DOCKER_NET_2_MAC},${DOCKER_NET_2_CIDR},${DOCKER_NET_2_GATEWAY}" >> /etc/vmware/route-tables.cfg
 sleep 2
-grep $'3\trtctl_workload' /etc/iproute2/rt_tables
-ip rule | grep rtctl_workload
-ip route show table rtctl_workload | grep default
+grep $'3\trtctl_frontend' /etc/iproute2/rt_tables
+ip rule | grep rtctl_frontend
+ip route show table rtctl_frontend | grep default
+
 EOF
 
   # Copy the test script to the container.