Fix get_python_install_dir issues

catkin_tools/jobs/cmake.py

@@ -78,7 +78,7 @@ def copy_install_manifest(
     return 0
 
 
-def get_python_install_dir():
+def get_python_install_dir(context):
     """Returns the same value as the CMake variable PYTHON_INSTALL_DIR
 
     The PYTHON_INSTALL_DIR variable is normally set from the CMake file:
@@ -88,15 +88,17 @@ def get_python_install_dir():
     :returns: Python install directory for the system Python
     :rtype: str
     """
-    python_install_dir = 'lib'
-    if os.name != 'nt':
-        python_version_xdoty = str(sys.version_info[0]) + '.' + str(sys.version_info[1])
-        python_install_dir = os.path.join(python_install_dir, 'python' + python_version_xdoty)
-
-    python_use_debian_layout = os.path.exists('/etc/debian_version')
-    python_packages_dir = 'dist-packages' if python_use_debian_layout else 'site-packages'
-    python_install_dir = os.path.join(python_install_dir, python_packages_dir)
-    return python_install_dir
+    cmake_command = [CMAKE_EXEC]
+    cmake_command.extend(context.cmake_args)
+    script_path = os.path.join(os.path.dirname(__file__), 'cmake', 'python_install_dir.cmake')
+    cmake_command.extend(['-P', script_path])
+    p = subprocess.Popen(
+        cmake_command,
+        cwd=os.path.join(os.path.dirname(__file__), 'cmake'),
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # only our message (containing the install directory) is written to stderr
+    _, out = p.communicate()
+    return out.decode().strip()
 
 
 def get_multiarch():
@@ -185,7 +187,7 @@ def generate_setup_file(logger, event_queue, context, install_target):
     subs = {}
     subs['cmake_prefix_path'] = install_target + ":"
     subs['ld_path'] = os.path.join(install_target, 'lib') + ":"
-    pythonpath = os.path.join(install_target, get_python_install_dir())
+    pythonpath = os.path.join(install_target, get_python_install_dir(context))
     subs['pythonpath'] = pythonpath + ':'
     subs['pkgcfg_path'] = os.path.join(install_target, 'lib', 'pkgconfig') + ":"
     subs['path'] = os.path.join(install_target, 'bin') + ":"

catkin_tools/jobs/cmake/python.cmake

@@ -0,0 +1,38 @@
+# the CMake variable PYTHON_INSTALL_DIR has the same value as the Python function catkin.builder.get_python_install_dir()
+
+set(PYTHON_VERSION "$ENV{ROS_PYTHON_VERSION}" CACHE STRING "Specify specific Python version to use ('major.minor' or 'major')")
+find_package(PythonInterp ${PYTHON_VERSION} REQUIRED)
+
+message(STATUS "Using PYTHON_EXECUTABLE: ${PYTHON_EXECUTABLE}")
+
+set(_PYTHON_PATH_VERSION_SUFFIX "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}")
+
+set(enable_setuptools_deb_layout OFF)
+if(EXISTS "/etc/debian_version")
+  set(enable_setuptools_deb_layout ON)
+endif()
+option(SETUPTOOLS_DEB_LAYOUT "Enable debian style python package layout" ${enable_setuptools_deb_layout})
+
+if(SETUPTOOLS_DEB_LAYOUT)
+  message(STATUS "Using Debian Python package layout")
+  set(PYTHON_PACKAGES_DIR dist-packages)
+  set(SETUPTOOLS_ARG_EXTRA "--install-layout=deb")
+  # use major version only when installing 3.x with debian layout
+  if("${PYTHON_VERSION_MAJOR}" STREQUAL "3")
+    set(_PYTHON_PATH_VERSION_SUFFIX "${PYTHON_VERSION_MAJOR}")
+  endif()
+else()
+  message(STATUS "Using default Python package layout")
+  set(PYTHON_PACKAGES_DIR site-packages)
+  # setuptools is fussy about windows paths, make sure the install prefix is in native format
+  file(TO_NATIVE_PATH "${CMAKE_INSTALL_PREFIX}" SETUPTOOLS_INSTALL_PREFIX)
+endif()
+
+if(NOT WIN32)
+  set(PYTHON_INSTALL_DIR lib/python${_PYTHON_PATH_VERSION_SUFFIX}/${PYTHON_PACKAGES_DIR}
+    CACHE INTERNAL "This needs to be in PYTHONPATH when 'setup.py install' is called.  And it needs to match.  But setuptools won't tell us where it will install things.")
+else()
+  # Windows setuptools installs to lib/site-packages not lib/python2.7/site-packages
+  set(PYTHON_INSTALL_DIR lib/${PYTHON_PACKAGES_DIR}
+    CACHE INTERNAL "This needs to be in PYTHONPATH when 'setup.py install' is called.  And it needs to match.  But setuptools won't tell us where it will install things.")
+endif()

catkin_tools/jobs/cmake/python_install_dir.cmake

@@ -0,0 +1,4 @@
+# Include a copy of ros/catkin/cmake/python.cmake to reproduce catkin's behavior
+include(python.cmake)
+# Print resulting PYTHON_INSTALL_DIR for further processing
+message(${PYTHON_INSTALL_DIR})

tests/system/verbs/catkin_build/test_pythonpath.py

@@ -0,0 +1,82 @@
+import os
+import re
+import shutil
+import subprocess
+
+from tests.system.workspace_factory import workspace_factory
+from tests.utils import redirected_stdio, catkin_success
+
+RESOURCES_DIR = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
+
+"""
+These tests check if the PYTHONPATH environment variable is set correctly when building a
+plain cmake package. The tests are run for python version 2 and 3 and for install and
+devel workspaces. The workspaces are set to merged because for linked workspaces, 
+the catkin that is injected by catkin_tools_prebuild generates the setup files.
+"""
+
+
+def test_python2_devel():
+    with workspace_factory() as wf:
+        os.mkdir(os.path.join(wf.workspace, 'src'))
+        shutil.copytree(
+            os.path.join(RESOURCES_DIR, 'cmake_pkgs', 'cmake_pkg'),
+            os.path.join('src', 'cmake_pkg'))
+        assert catkin_success(['config', '--merge-devel'])
+        assert catkin_success(['build', '-DPYTHON_VERSION=2'])
+        pythonpaths = subprocess.check_output(
+            ['bash', '-c', 'source ' + wf.workspace + '/devel/setup.sh && echo $PYTHONPATH']).decode().split(':')
+        ws_pythonpath = [p for p in pythonpaths if p.startswith(wf.workspace)][0]
+        assert ws_pythonpath
+        # it might be dist-packages (debian) or site-packages
+        assert re.match('^' + wf.workspace + '/devel/lib/python2\.\d/(site|dist)-packages$', ws_pythonpath)
+
+
+def test_python3_devel():
+    with workspace_factory() as wf:
+        os.mkdir(os.path.join(wf.workspace, 'src'))
+        shutil.copytree(
+            os.path.join(RESOURCES_DIR, 'cmake_pkgs', 'cmake_pkg'),
+            os.path.join('src', 'cmake_pkg'))
+        assert catkin_success(['config', '--merge-devel'])
+        assert catkin_success(['build', '-DPYTHON_VERSION=3'])
+        pythonpaths = subprocess.check_output(
+            ['bash', '-c', 'source ' + wf.workspace + '/devel/setup.sh && echo $PYTHONPATH']).decode().split(':')
+        ws_pythonpath = [p for p in pythonpaths if p.startswith(wf.workspace)][0]
+        assert ws_pythonpath
+        # it might be python3/dist-packages (debian) or python3.x/site-packages
+        assert (ws_pythonpath == wf.workspace + '/devel/lib/python3/dist-packages' or
+                re.match('^' + wf.workspace + '/devel/lib/python3\.\d/site-packages$', ws_pythonpath))
+
+
+def test_python2_install():
+    with workspace_factory() as wf:
+        os.mkdir(os.path.join(wf.workspace, 'src'))
+        shutil.copytree(
+            os.path.join(RESOURCES_DIR, 'cmake_pkgs', 'cmake_pkg'),
+            os.path.join('src', 'cmake_pkg'))
+        assert catkin_success(['config', '--merge-devel', '--merge-install', '--install'])
+        assert catkin_success(['build', '-DPYTHON_VERSION=2'])
+        pythonpaths = subprocess.check_output(
+            ['bash', '-c', 'source ' + wf.workspace + '/install/setup.sh && echo $PYTHONPATH']).decode().split(':')
+        ws_pythonpath = [p for p in pythonpaths if p.startswith(wf.workspace)][0]
+        assert ws_pythonpath
+        # it might be dist-packages (debian) or site-packages
+        assert re.match('^' + wf.workspace + '/install/lib/python2\.\d/(site|dist)-packages$', ws_pythonpath)
+
+
+def test_python3_install():
+    with workspace_factory() as wf:
+        os.mkdir(os.path.join(wf.workspace, 'src'))
+        shutil.copytree(
+            os.path.join(RESOURCES_DIR, 'cmake_pkgs', 'cmake_pkg'),
+            os.path.join('src', 'cmake_pkg'))
+        assert catkin_success(['config', '--merge-devel', '--merge-install', '--install'])
+        assert catkin_success(['build', '-DPYTHON_VERSION=3'])
+        pythonpaths = subprocess.check_output(
+            ['bash', '-c', 'source ' + wf.workspace + '/install/setup.sh && echo $PYTHONPATH']).decode().split(':')
+        ws_pythonpath = [p for p in pythonpaths if p.startswith(wf.workspace)][0]
+        assert ws_pythonpath
+        # it might be python3/dist-packages (debian) or python3.x/site-packages
+        assert (ws_pythonpath == wf.workspace + '/install/lib/python3/dist-packages' or
+                re.match('^' + wf.workspace + '/install/lib/python3\.\d/site-packages$', ws_pythonpath))