WIP: Merge with 18.x. Fix cmake

tarunprabhu · Jul 24, 2024 · 44b75a0 · 44b75a0
1 parent ddcb3c6
commit 44b75a0
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 22 deletions.
diff --git a/kitsune/CMakeLists.txt b/kitsune/CMakeLists.txt
@@ -93,19 +93,27 @@ if (KITSUNE_KOKKOS_ENABLE)
   # kitsune/runtime, just so everything that needs to be fetched and built is
   # there, even if this is not technically a runtime.
 
+  set(KITSUNE_KOKKOS_BUILD_CONFIGURE_FLAGS ""
+    CACHE STRING
+  "Additional options to be passed to be CMake when configuring Kokkos. These are only used when building Kitsune")
+
+  set(KITSUNE_KOKKOS_BUILD_COMPILE_FLAGS ""
+    CACHE STRING
+    "Additional C++ compiler flags needed to build Kokkos. These are only used when building Kitsune")
+
   set(KITSUNE_KOKKOS_SRC_DIR  ${KITSUNE_TARGETS_BINARY_DIR}/kokkos/kokkos)
   set(KITSUNE_KOKKOS_BUILD_DIR ${KITSUNE_TARGETS_BINARY_DIR}/kokkos/build)
   set(KITSUNE_KOKKOS_STAMP_DIR ${KITSUNE_TARGETS_BINARY_DIR}/kokkos/stamp)
-  set(KITSUNE_KOKKOS_BUILD_ARGS
-    -DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang
+  set(KITSUNE_KOKKOS_BUILD_CMAKE_FLAGS
     -DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++
-    -DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}
     -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}
+    -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
     -DCMAKE_INSTALL_PREFIX=${CLANG_RESOURCE_INSTALL_DIR}
-    -DCMAKE_BUILD_TYPE=Release
+    -DCMAKE_CXX_FLAGS=${KITSUNE_KOKKOS_BUILD_COMPILE_FLAGS}
     -DKokkos_ENABLE_SERIAL=ON
     -DBUILD_TESTING=OFF
-    -DBUILD_SHARED_LIBS=ON)
+    -DBUILD_SHARED_LIBS=ON
+    ${KITSUNE_KOKKOS_BUILD_CONFIGURE_FLAGS})
 
   # We have to patch kokkos to disable some sections of code that are using
   # exceptions.
@@ -136,7 +144,7 @@ if (KITSUNE_KOKKOS_ENABLE)
     SOURCE_DIR          ${KITSUNE_KOKKOS_SRC_DIR}
     BINARY_DIR          ${KITSUNE_KOKKOS_BUILD_DIR}
     STAMP_DIR           ${KITSUNE_KOKKOS_STAMP_DIR}
-    CMAKE_ARGS          ${KITSUNE_KOKKOS_BUILD_ARGS}
+    CMAKE_ARGS          ${KITSUNE_KOKKOS_BUILD_CMAKE_FLAGS}
     PATCH_COMMAND       patch -p0 --input=${KITSUNE_KOKKOS_PATCH_EXCS}
     UPDATE_COMMAND      ""
     UPDATE_DISCONNECTED FALSE
@@ -217,9 +225,12 @@ if (KITSUNE_CUDA_ENABLE)
     PATHS ${KITSUNE_CUDA_LIBRARY_DIR}
     NO_DEFAULT_PATH)
 
+  # This should almost always be the libcuda that is installed on the system,
+  # so look for it in the "usual" places. In some cases, both /usr/lib and
+  # /usr/lib64 are present with libcuda in the latter. But cmake does not seem
+  # to look there, so force it to.
   find_library(KITSUNE_CUDA_LIB_CUDA cuda REQUIRED
-    PATHS ${KITSUNE_CUDA_STUBS_DIR}
-    NO_DEFAULT_PATH)
+    HINTS /usr/lib64 /usr/local/lib64)
 
   find_library(KITSUNE_CUDA_LIB_NVPTX_STATIC nvptxcompiler_static REQUIRED
     PATHS ${KITSUNE_CUDA_LIBRARY_DIR}
@@ -260,8 +271,8 @@ if (KITSUNE_HIP_ENABLE)
     "Path to hip install directory")
 
   # Probably should not be hard-coding /opt/rocm/* here, but it seems to be
-  # where it goes in several distributions and it seems we cannot otherwise
-  # reliably find rocm.
+  # where it goes in several distributions and we cannot reliably find rocm
+  # otherwise
   find_package(hip REQUIRED
     PATHS $ENV{ROCM_PATH}/hip $ENV{ROCM_PATH}
     /opt/rocm/hip /opt/rocm)
@@ -304,25 +315,36 @@ if (KITSUNE_OPENCILK_ENABLE)
   # vanilla clang build of the same version, but it's probably not worth all the
   # error checking hassles that would be needed to detect if the versions were
   # mismatched.
-  #
+
+  set(KITSUNE_OPENCILK_BUILD_CONFIGURE_FLAGS ""
+    CACHE STRING
+  "Additional options to be passed to be CMake when configuring Kokkos. These are only used when building Kitsune")
+
+  set(KITSUNE_OPENCILK_BUILD_COMPILE_FLAGS ""
+    CACHE STRING
+    "Additional C++ compiler flags needed to build Kokkos. These are only used when building Kitsune")
+
   # We pass some LLVM_* variables to make Cheetah behave as if it were an
   # in-tree build.
   set(KITSUNE_CHEETAH_SOURCE_DIR ${KITSUNE_TARGETS_BINARY_DIR}/cheetah/cheetah)
   set(KITSUNE_CHEETAH_BINARY_DIR ${KITSUNE_TARGETS_BINARY_DIR}/cheetah/build)
-  set(KITSUNE_CHEETAH_BUILD_ARGS
+  set(KITSUNE_CHEETAH_BUILD_CMAKE_FLAGS
     -DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang
     -DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++
-    -DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang
-    -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-    -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
     -DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}
     -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}
-    -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
+    -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+    -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
+    -DCMAKE_INSTALL_PREFIX=${CLANG_RESOURCE_INSTALL_DIR}
+    -DCMAKE_CXX_FLAGS=${KITSUNE_OPENCILK_BUILD_COMPILE_FLAGS}
     -DCHEETAH_DEFAULT_TARGET_TRIPLE=${LLVM_DEFAULT_TARGET_TRIPLE}
     -DCHEETAH_DIRECTORY_PER_TARGET=ON
+    -DLLVM_CMAKE_DIR=${CMAKE_BINARY_DIR}
+    -DLLVM_COMMON_CMAKE_UTILS=${CMAKE_SOURCE_DIR}/llvm/cmake
     -DLLVM_LIBRARY_OUTPUT_INTDIR=${LLVM_LIBRARY_OUTPUT_INTDIR}
     -DLLVM_RUNTIME_OUTPUT_INTDIR=${LLVM_RUNTIME_OUTPUT_INTDIR}
     -DPACKAGE_VERSION=${PACKAGE_VERSION}
+    ${KITSUNE_OPENCILK_BUILD_CONFIGURE_FLAGS}
   )
 
   ExternalProject_Add(cheetah
@@ -331,7 +353,7 @@ if (KITSUNE_OPENCILK_ENABLE)
     GIT_TAG        opencilk/v2.1
     SOURCE_DIR     ${KITSUNE_CHEETAH_SOURCE_DIR}
     BINARY_DIR     ${KITSUNE_CHEETAH_BINARY_DIR}
-    CMAKE_ARGS     ${KITSUNE_CHEETAH_BUILD_ARGS})
+    CMAKE_ARGS     ${KITSUNE_CHEETAH_BUILD_CMAKE_FLAGS})
 
   # The values here are populated given that we know where Cheetah will be
   # installed. The code to actual fetch, configure, build etc. is in
@@ -506,7 +528,7 @@ endif()
 ExternalProject_Add(kitsune-runtimes
   DEPENDS llvm-config clang LLVM
   SOURCE_DIR ${KITSUNE_SOURCE_DIR}/runtime
-  STAMP_DIR ${KITSUNE_BINARY_DIR}/runtime-stamp
+  STAMP_DIR ${KITSUNE_BINARY_DIR}/stamp
   BINARY_DIR ${KITSUNE_BINARY_DIR}/runtime
   CMAKE_ARGS -DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang
              -DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++
@@ -515,6 +537,7 @@ ExternalProject_Add(kitsune-runtimes
              -DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}
              -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}
              -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
+             -DCMAKE_COLOR_DIAGNOSTICS=${CMAKE_COLOR_DIAGNOSTICS}
              -DKITSUNE_CUDA_ENABLE=${KITSUNE_CUDA_ENABLE}
              -DKITSUNE_CUDA_INCLUDE_DIR=${KITSUNE_CUDA_INCLUDE_DIR}
              -DKITSUNE_CUDA_BINARY_DIR=${KITSUNE_CUDA_BINARY_DIR}
@@ -550,7 +573,9 @@ ExternalProject_Add(kitsune-runtimes
              -DLLVM_RUNTIME_OUTPUT_INTDIR=${LLVM_RUNTIME_OUTPUT_INTDIR}
              -DCUDAToolkit_INCLUDE_DIRS=${CUDAToolkit_INCLUDE_DIRS}
              -DCUDAToolkit_LIBRARY_DIR=${CUDAToolkit_LIBRARY_DIR}
-  USES_TERMINAL_CONFIGURE 1
-  USES_TERMINAL_BUILD 1
-  USES_TERMINAL_INSTALL 1)
+  USES_TERMINAL_CONFIGURE TRUE
+  USES_TERMINAL_BUILD TRUE
+  USES_TERMINAL_INSTALL TRUE
+  BUILD_ALWAYS TRUE
+  CONFIGURE_HANDLED_BY_BUILD TRUE)
 
diff --git a/kitsune/runtime/cuda/kitcuda.cpp b/kitsune/runtime/cuda/kitcuda.cpp
@@ -97,7 +97,7 @@ extern "C" {
 bool __kitcuda_initialize() {
   KIT_NVTX_PUSH("kitcuda: initialize", KIT_NVTX_INIT);
   if (_kitcuda_initialized) {
-    if (__kitrt_verbose_mode()) 
+    if (__kitrt_verbose_mode())
       fprintf(stderr, "kitcuda: warning, multiple initialization calls!\n");
     return true;
   }