Analyze core dumps on test failure (#4430)

ci/tests/run-core-tests.sh

@@ -1,8 +1,4 @@
 #!/bin/bash
 set -euo pipefail
 
-source "$(dirname "$BASH_SOURCE")/common.sh"
-
-BUILD_DIR=${1-${PWD}}
-
-${BUILD_DIR}/core_test$(get_exec_extension)
+$(dirname "$BASH_SOURCE")/run-tests.sh core_test

ci/tests/run-qt-tests.sh

@@ -1,10 +1,6 @@
 #!/bin/bash
 set -euo pipefail
 
-source "$(dirname "$BASH_SOURCE")/common.sh"
-
-BUILD_DIR=${1-${PWD}}
-
 # Alpine doesn't offer an xvfb
 xvfb_run_()
 {
@@ -20,4 +16,4 @@ xvfb_run_()
     return ${res}
 }
 
-xvfb_run_ ${BUILD_DIR}/qt_test$(get_exec_extension)
+xvfb_run_ $(dirname "$BASH_SOURCE")/run-tests.sh qt_test

ci/tests/run-rpc-tests.sh

@@ -1,8 +1,4 @@
 #!/bin/bash
 set -euo pipefail
 
-source "$(dirname "$BASH_SOURCE")/common.sh"
-
-BUILD_DIR=${1-${PWD}}
-
-${BUILD_DIR}/rpc_test$(get_exec_extension)
+$(dirname "$BASH_SOURCE")/run-tests.sh rpc_test

ci/tests/run-tests.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -uo pipefail
+
+source "$(dirname "$BASH_SOURCE")/common.sh"
+
+target=$1
+if [ -z "${target-}" ]; then
+    echo "Target not specified"
+    exit 1
+fi
+
+echo "Running tests for target: ${target}"
+
+# Enable core dumps
+DEFAULT_COREDUMP_DIR="/cores"
+case "$(uname -s)" in
+    Linux*)
+        # Ensure directory exists and is writable for core dumps
+        sudo mkdir -p "${DEFAULT_COREDUMP_DIR}"
+        sudo chmod a+w "${DEFAULT_COREDUMP_DIR}"
+        # Enable core dumps
+        ulimit -c unlimited
+        echo "${DEFAULT_COREDUMP_DIR}/core-%e.%p" | sudo tee /proc/sys/kernel/core_pattern
+        export COREDUMP_DIR=${DEFAULT_COREDUMP_DIR}
+
+        echo "Core dumps enabled (Linux)"
+        ;;
+    Darwin*)
+        # Ensure directory exists and is writable for core dumps
+        sudo mkdir -p "${DEFAULT_COREDUMP_DIR}"
+        sudo chmod a+w "${DEFAULT_COREDUMP_DIR}"
+        # Enable core dumps
+        ulimit -c unlimited
+        # By default, macOS writes core dumps to /cores
+        export COREDUMP_DIR=${DEFAULT_COREDUMP_DIR}
+
+        echo "Core dumps enabled (macOS)"
+        ;;
+    CYGWIN*|MINGW32*|MSYS*|MINGW*)
+        # TODO: Support core dumps on Windows
+        echo "Core dumps not supported on Windows"
+        ;;
+    *)
+        echo "Unknown OS"
+        exit 1
+        ;;
+esac
+
+# Run the test
+executable=./${target}$(get_exec_extension)
+"${executable}"
+status=$?
+
+if [ $status -ne 0 ]; then
+    echo "::error::Test failed: ${target}"
+
+    # Show core dumps
+    export EXECUTABLE=${executable}
+    "$(dirname "$BASH_SOURCE")/show-core-dumps.sh"
+
+    exit $status
+else
+    exit 0
+fi

ci/tests/show-core-dumps.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+set -uo pipefail
+
+echo "Analyzing core dumps..."
+
+if [ -z "${COREDUMP_DIR-}" ]; then
+    echo "COREDUMP_DIR environment variable is not set."
+    exit 1
+fi
+
+if [ -z "${EXECUTABLE-}" ]; then
+    echo "EXECUTABLE environment variable is not set."
+    exit 1
+fi
+
+echo "Core dump location: ${COREDUMP_DIR}"
+echo "Executable: ${EXECUTABLE}"
+
+analyze_core_dump() {
+    local core_dump=$1
+    local executable=$2
+
+    case "$(uname)" in
+        Darwin)
+            # macOS, use lldb
+            echo "Using lldb for analysis..."
+            lldb "${executable}" -c "$core_dump" --batch -o "thread backtrace all" -o "quit"
+            ;;
+        Linux)
+            # Linux, use gdb
+            echo "Using gdb for analysis..."
+            gdb -quiet -batch -ex "thread apply all bt full" -ex "quit" "${executable}" "$core_dump"
+            ;;
+        *)
+            echo "Unsupported OS."
+            return 1
+            ;;
+    esac
+
+    # Remove the analyzed core dump file
+    echo "Removing analyzed core dump: $core_dump"
+    rm "$core_dump"
+}
+
+# List core dump files
+echo "::group::Core dump files"
+ls -al "${COREDUMP_DIR}"
+echo "::endgroup::"
+
+# Use a glob pattern to match core dumps
+shopt -s nullglob
+core_dumps=("${COREDUMP_DIR}"/core*)
+
+if [ ${#core_dumps[@]} -gt 0 ]; then
+    for core_dump in "${core_dumps[@]}"; do
+        echo "::group::Analyzing core dump: $core_dump"
+        analyze_core_dump "$core_dump" "${EXECUTABLE}"
+        echo "::endgroup::"
+    done
+else
+    echo "No core dump file found."
+fi