Docker compose 2.22.0

linux/just_files/docker_compose_override

@@ -229,11 +229,8 @@ function generate_docker_compose_override()
   local volumes_docker
   local volumes_flags
 
-  # parse_docker_compose_volumes/docker_compose_volumes vars
-  local DOCKER_VOLUME_LINES
-  local DOCKER_VOLUME_SOURCES
-  local DOCKER_VOLUME_TARGETS
-  local DOCKER_VOLUME_FLAGS
+  # docker_compose_volumes vars
+  local vsi_compose_volume_sources vsi_compose_volume_targets vsi_compose_volume_flags vsi_compose_volume_types
   # _env_echo
   local new_environment
 
@@ -303,31 +300,21 @@ function generate_docker_compose_override()
     volumes_name="${JUST_PROJECT_PREFIX}_${services_upper[service_index]}_VOLUMES"
     indirect="${volumes_name}[@]"
 
-    parse_docker_compose_volumes ${services_name[service_index]} <<< "${compose_config}"
-    docker_compose_volumes
+    vsi::docker::get_compose_volumes ${services_name[service_index]} < <(yarp <<< "${compose_config}")
 
     # Get internal volumes of service
-    for volume in "${!DOCKER_VOLUME_SOURCES[@]}"; do
-
-      # If directory doesn't exist, attempt to make it as this user. Better than root
-      # When the argument is a docker volume, nothing should happen.
-      docker_premkdir "${DOCKER_VOLUME_SOURCES[volume]}"
-
+    for volume in "${!vsi_compose_volume_sources[@]}"; do
       # Get internal volumes of service
-      if is_internal_docker_volume "${DOCKER_VOLUME_SOURCES[volume]}"; then
-        if [ "${DOCKER_VOLUME_FORMATS[volume]}" == "short" ]; then
-          if is_readonly_docker_volume "${DOCKER_VOLUME_FLAGS[volume]}"; then
-            just_docker_entrypoint_internal_ro_volumes+=("${DOCKER_VOLUME_TARGETS[volume]}")
-          else
-            just_docker_entrypoint_internal_volumes+=("${DOCKER_VOLUME_TARGETS[volume]}")
-          fi
+      if [ "${vsi_compose_volume_types[volume]}" = "volume" ]; then
+        if vsi::docker::is_docker_compose_volume_readonly "${vsi_compose_volume_flags[volume]}"; then
+          just_docker_entrypoint_internal_ro_volumes+=("${vsi_compose_volume_targets[volume]}")
         else
-          if is_readonly_docker_compose_long_volume "${DOCKER_VOLUME_FLAGS[volume]}"; then
-            just_docker_entrypoint_internal_ro_volumes+=("${DOCKER_VOLUME_TARGETS[volume]}")
-          else
-            just_docker_entrypoint_internal_volumes+=("${DOCKER_VOLUME_TARGETS[volume]}")
-          fi
+          just_docker_entrypoint_internal_volumes+=("${vsi_compose_volume_targets[volume]}")
         fi
+      else
+        # If directory doesn't exist, attempt to make it as this user. Better than root
+        # When the argument is a docker volume, nothing should happen.
+        docker_premkdir "${vsi_compose_volume_sources[volume]}"
       fi
     done
 
@@ -343,8 +330,6 @@ function generate_docker_compose_override()
 
       docker_parse_volume_string "${volume}" # sets volume_host, volume_docker, and volume_flags
 
-      # If it doesn't exist, attempt to make it as this user. Better than root
-      docker_premkdir "${volume_host}"
       # If it's a docker volume, track that too!
       if is_internal_docker_volume "${volume_host}"; then
         if is_readonly_docker_volume "${volume_flags}"; then
@@ -354,6 +339,8 @@ function generate_docker_compose_override()
         fi
         all_dynamic_internal_volumes+=("${volume_host}")
       else # Else the volume is a path, get the real path
+        # If it doesn't exist, attempt to make it as this user. Better than root
+        docker_premkdir "${volume_host}"
         volume_host="$(real_path "${volume_host}")"
       fi
 

linux/just_files/docker_functions.bsh

@@ -27,7 +27,7 @@ source "${VSI_COMMON_DIR}/linux/aliases.bsh"
 #
 # Set of functions to make using dockers easier
 #
-# .. var:: DOCKER_VOLUME_PATTERN
+# .. var:: vsi_docker_short_format_volume_pattern
 #
 # Regex pattern for matching docker volume short form
 #
@@ -41,7 +41,7 @@ source "${VSI_COMMON_DIR}/linux/aliases.bsh"
 #    3 - flags
 #**
 # https://www.debuggex.com/r/lJRoX_sSnhOKpEER
-DOCKER_VOLUME_PATTERN='^([a-zA-Z]:[/\][^:]*|[^:]+):([a-zA-Z]:[/\][^:]*|[^:]+)((:ro|:rw|:z|:Z|:r?shared|:r?slave|:r?private|:delegated|:cached|:consistent|:nocopy)*)$'
+vsi_docker_short_format_volume_pattern='^([a-zA-Z]:[/\][^:]*|[^:]+):([a-zA-Z]:[/\][^:]*|[^:]+)((:ro|:rw|:z|:Z|:r?shared|:r?slave|:r?private|:delegated|:cached|:consistent|:nocopy)*)$'
 #**
 # .. function:: is_dir_and_not_exist
 #
@@ -151,6 +151,7 @@ function is_readonly_docker_volume()
 
 function is_readonly_docker_compose_long_volume()
 {
+  echo "${FUNCNAME[0]} is deprecated: Use vsi::docker::is_docker_compose_volume_readonly instead" >&2
   local IFS=$'\n'
   local flags=(${1}) # noquotes
   local flag
@@ -164,6 +165,41 @@ function is_readonly_docker_compose_long_volume()
   return 1
 }
 
+#**
+# .. function:: vsi::docker::is_docker_compose_volume_readonly
+#
+# :Arguments: * ``$1`` - A ``vsi_compose_volume_flags`` string to check (Not the whole array)
+# :Return Value: * ``0`` if it is readonly
+#                * ``1`` it if is writeable (according to docker flags only)
+#
+# Checks to see if a volume flags string contains the read only attribute.
+#
+# .. note::
+#
+#   It is still possible the underlying directory is not writeable, this will not check that. Used mostly for docker volumes, where this is not an issue.
+#
+# .. seealso::
+#
+#   vsi::docker::get_compose_volumes
+#     Parses docker compose volumes and generates ``vsi_compose_volume_flags``
+#**
+
+function vsi::docker::is_docker_compose_volume_readonly()
+{
+  local IFS=$'\n'
+  local flags=(${1}) # noquotes
+  local flag
+
+  for flag in ${flags[@]+"${flags[@]}"}; do
+    if [ "${flag}" == "read_only = true" ]; then
+      return 0
+    fi
+  done
+
+  return 1
+}
+
+
 #**
 # .. function:: container_get_label
 #
@@ -315,7 +351,7 @@ function docker_host_dir()
 
 function docker_parse_volume_string()
 {
-  if [[ ${1} =~ ${DOCKER_VOLUME_PATTERN} ]]; then
+  if [[ ${1} =~ ${vsi_docker_short_format_volume_pattern} ]]; then
     volume_host="${BASH_REMATCH[1]}"
     volume_docker="${BASH_REMATCH[2]}"
     volume_flags="${BASH_REMATCH[3]}"
@@ -560,6 +596,7 @@ function _Docker()
 
 function parse_docker_compose_volumes()
 {
+  echo "${FUNCNAME[0]} is deprecated: Use vsi::docker::get_compose_volumes instead" >&2
   local PREFIX="${2-}"
   local IFS=$'\n'
   DOCKER_VOLUME_LINES=(\
@@ -702,6 +739,8 @@ function docker_compose_list_internal_volumes()
 
 function docker_compose_volumes()
 {
+  echo "${FUNCNAME[0]} is deprecated: Use vsi::docker::get_compose_volumes instead" >&2
+
   local volume_line
   DOCKER_VOLUME_SOURCES=()
   DOCKER_VOLUME_TARGETS=()
@@ -723,7 +762,7 @@ function docker_compose_volumes()
 
     if [ "${volume_line:0:1}" = "S" ]; then # Short format
       volume_line="${volume_line:1}"
-      if [[ ${volume_line} =~ ${DOCKER_VOLUME_PATTERN} ]]; then
+      if [[ ${volume_line} =~ ${vsi_docker_short_format_volume_pattern} ]]; then
         DOCKER_VOLUME_SOURCES[count]="${BASH_REMATCH[1]}"
         DOCKER_VOLUME_TARGETS[count]="${BASH_REMATCH[2]}"
         DOCKER_VOLUME_FORMATS[count]="short"
@@ -754,6 +793,125 @@ function docker_compose_volumes()
   done
 }
 
+#**
+# :Arguments: * ``$1`` - Name of service of interest
+#             * [``$2``] - Prefix to volume names
+# :Input: **stdin** - Yarped docker compose yaml file
+# :Output: * ``DOCKER_VOLUME_SOURCES`` - List of sources
+#          * ``DOCKER_VOLUME_TARGETS`` - Corresponding list of targets
+#          * ``DOCKER_VOLUME_FLAGS`` - Corresponding list of flags. Can be a mix of short and long format flags. Needs more parsing before this mix is useful.
+#          * ``DOCKER_VOLUME_FORMATS`` - Corresponding list specifying either "long" or "short" as the format of each volume
+#
+# Get volume lists from ``DOCKER_VOLUME_LINES``
+#
+# .. rubric:: Usage
+#
+# .. code-block:: bash
+#
+#    vsi::docker::get_compose_volumes nb < <(docker compose config | yarp)
+#    docker_compose_volumes
+#
+# .. seealso::
+#    yarp
+#**
+function vsi::docker::get_compose_volumes()
+{
+  local volume_line
+  vsi_compose_volume_sources=()
+  vsi_compose_volume_targets=()
+  vsi_compose_volume_flags=()
+  vsi_compose_volume_types=()
+  local -i count=0
+
+  local short_flag
+
+  local pattern="services\\.${1}\\.volumes\\[([0-9]+)](\\.[^ ]*)? = (.*)"
+  #BASH_REMATCH 0 - line, 1 - index, 2 - .key 3 - value/short format
+
+  local line
+  local index
+  while read -r line; do
+    if [[ ${line} =~ ${pattern} ]]; then
+      index=${BASH_REMATCH[1]}
+
+      case "${BASH_REMATCH[2]#.}" in
+        source)
+          vsi_compose_volume_sources[index]=${BASH_REMATCH[3]}
+          ;;
+        target)
+          vsi_compose_volume_targets[index]=${BASH_REMATCH[3]}
+          ;;
+        type)
+          vsi_compose_volume_types[index]=${BASH_REMATCH[3]}
+          ;;
+
+        # short format - Docker-compose 2 never outputs short format when you call
+        # the config command, so this is largely for backwards compatibility and
+        # corner cases
+        "")
+          if [[ ${BASH_REMATCH[3]} =~ ${vsi_docker_short_format_volume_pattern} ]]; then
+            declare -p BASH_REMATCH >&2
+            vsi_compose_volume_sources[index]=${BASH_REMATCH[1]}
+            vsi_compose_volume_targets[index]=${BASH_REMATCH[2]}
+            short_flag=${BASH_REMATCH[3]}
+
+            if is_internal_docker_volume "${vsi_compose_volume_sources[index]}"; then
+              vsi_compose_volume_types[index]=volume
+            else
+              vsi_compose_volume_types[index]=bind
+            fi
+
+            vsi_compose_volume_flags[index]="bind.create_host_path = true"
+
+            # These are the only 3 mentioned in https://docs.docker.com/compose/compose-file/05-services/#short-syntax-5 that need addressing
+            if [[ ${short_flag} = *:ro* ]]; then
+              vsi_compose_volume_flags[index]+=$'\nread_only = true'
+            fi
+            # Cannot be z and Z at the same time
+            if [[ ${short_flag} = *:Z* ]]; then
+              vsi_compose_volume_flags[index]+=$'\nbind.selinux = Z'
+            elif [[ ${short_flag} = *:z* ]]; then
+              vsi_compose_volume_flags[index]+=$'\nbind.selinux = z'
+            fi
+          else
+            echo "${FUNCNAME[0]}: Unrecognized short volume format: ${line}" >&2
+          fi
+          ;;
+
+        # All other flags are added to vsi_compose_volume_flags, new line separated
+        *)
+          vsi_compose_volume_flags[index]="${vsi_compose_volume_flags[index]+${vsi_compose_volume_flags[index]}}${vsi_compose_volume_flags[index]:+$'\n'}${BASH_REMATCH[2]#.} = ${BASH_REMATCH[3]}"
+          ;;
+      esac
+    fi
+  done
+
+  # The flags array can be discontiguous. Fill in empty holes, to prevent unbound variable errors
+  if [ "${#vsi_compose_volume_flags[@]}" != "${#vsi_compose_volume_sources[@]}" ]; then
+    for index in "${!vsi_compose_volume_sources[@]}"; do
+      if [ -z "${vsi_compose_volume_flags[index]+set}" ]; then
+        vsi_compose_volume_flags[index]=""
+      fi
+    done
+  fi
+}
+
+
+#     Expected: Element 4 (DOCKER_VOLUME_LINES[4]) is different:
+#     Ltest:/opt != Stest:/opt
+#     declare -a Reference=([0]="S/tmp:/mnt" [1]="Lsource: /home/user/src" [2]="ltarget: /src" [3]="ltype: bind" [4]="Stest:/opt" [5]="Lsource: internal" [6]="ltarget: /src" [7]="ltype: volume" [8]="Ltarget: /tmp2" [9]="ltype: tmpfs")
+#     Actual: declare -a DOCKER_VOLUME_LINES=([0]="S/tmp:/mnt" [1]="Lsource: /home/user/src" [2]="ltarget: /src" [3]="ltype: bind" [4]="Ltest:/opt" [5]="Lsource: internal" [6]="ltarget: /src" [7]="ltype: volume" [8]="Ltarget: /tmp2" [9]="ltype: tmpfs")
+
+#   104:services.wine.volumes =
+# 105:services.wine.volumes[0].type = volume
+# 106:services.wine.volumes[0].source = wine_home
+# 107:services.wine.volumes[0].target = /home/.user_wine
+# 108:services.wine.volumes[1].type = bind
+# 109:services.wine.volumes[1].source = /home/andy/src/terra_dsm/external/terra/external/vsi_common
+# 110:services.wine.volumes[1].target = /vsi
+# 111:services.wine.volumes[1].read_only = true
+
+
 #**
 # .. function:: parse_docker_compose
 #

linux/yarp.awk

@@ -64,11 +64,13 @@ function process_line(str)
 
   # Split the line into key and the remainder after the colon
   remain = substr(str, 1+max(RLENGTH, indent))
-  key = match(remain, /^[^ '":]+ *: */)   #"# VS Code parser error
+  # key = match(remain, /^[^ '":]+ *: */) #"# VS Code parser error
+  key = match(remain, /^[^ '":]+ *(: *$|:  *)/) #"# VS Code parser error
   if ( key )
   {
     tmp = RLENGTH
-    match(remain, / *: */)
+    # match(remain, / *: */)
+    match(remain, / *(: *$|:  *)/)
     key = substr(remain, 1, tmp-RLENGTH)
     remain = substr(remain, tmp+1)
   }

tests/int/test-new_just.bsh

@@ -125,8 +125,9 @@ services:
   yaan:
 "
   if [ -d "/tmp/.X11-unix" ]; then
+    x11_dir=$(real_path /tmp/.X11-unix)
     ans+="    volumes:
-      - /tmp/.X11-unix:/tmp/.X11-unix:ro
+      - ${x11_dir}:/tmp/.X11-unix:ro
 "
   fi
   ans+="    environment:
@@ -142,7 +143,7 @@ services:
 "
   if [ -d "/tmp/.X11-unix" ]; then
     ans+="    volumes:
-      - /tmp/.X11-unix:/tmp/.X11-unix:ro
+      - ${x11_dir}:/tmp/.X11-unix:ro
 "
   fi
   ans+="    environment:
@@ -222,8 +223,9 @@ services:
   example:
 "
   if [ -d "/tmp/.X11-unix" ]; then
+    x11_dir=$(real_path /tmp/.X11-unix)
     ans+="    volumes:
-      - /tmp/.X11-unix:/tmp/.X11-unix:ro
+      - ${x11_dir}:/tmp/.X11-unix:ro
 "
   fi
   ans+="    environment:

tests/run_tests

@@ -25,6 +25,7 @@ if [ -z "${VSI_COMMON_DIR+set}" ]; then
   VSI_COMMON_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd)"
 fi
 
+source "${VSI_COMMON_DIR}/linux/source_once.bsh"
 source "${VSI_COMMON_DIR}/env.bsh"
 source "${VSI_COMMON_DIR}/linux/common_source.sh"
 source "${VSI_COMMON_DIR}/tests/test_colors.sh"

tests/test-circular_source.bsh

@@ -104,7 +104,7 @@ begin_test "Looking for infinite source loops"
       # Only "library" files should be checked, all other programs and non-bash files should be added to this exception list
       */example_just|*/just_diff|*/just_entrypoint.sh|*/just_env|*/new_just|*/Just_wrap|*/git_safe_update|*.py|*.awk|*/check_shell) ;;
       *)
-        ( timeout 10 bash -c "source \"${x}\"" )
+        ( timeout 10 bash -c "source_once(){ return 1; }; source \"${x}\"" )
         ;;
     esac
   done