From 953e1743e93757518b42b845b4f5116ae1b1fca2 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Fri, 16 Feb 2024 11:24:13 +0800 Subject: [PATCH 1/3] X --- tools/java_api/src/jni/kuzu_java.cpp | 22 +++++++++++++++++++ .../src/main/java/com/kuzudb/KuzuNative.java | 11 +++++++++- .../java/com/kuzudb/test/ExtensionTest.java | 7 +++--- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/tools/java_api/src/jni/kuzu_java.cpp b/tools/java_api/src/jni/kuzu_java.cpp index 6add4a319b..756e344ba2 100644 --- a/tools/java_api/src/jni/kuzu_java.cpp +++ b/tools/java_api/src/jni/kuzu_java.cpp @@ -1,5 +1,11 @@ #include +#ifdef _WIN32 +// Do nothing on Windows +#else +#include +#endif + // This header is generated at build time. See CMakeLists.txt. #include "com_kuzudb_KuzuNative.h" #include "common/exception/conversion.h" @@ -149,6 +155,22 @@ std::unordered_map> javaMapToCPPMap( /** * All Database native functions */ +// protected static native void kuzu_native_reload_library(String lib_path); +JNIEXPORT void JNICALL Java_com_kuzudb_KuzuNative_kuzu_1native_1reload_1library( + JNIEnv* env, jclass, jstring lib_path) { +#ifdef _WIN32 +// Do nothing on Windows +#else + const char* path = env->GetStringUTFChars(lib_path, JNI_FALSE); + void* handle = dlopen(path, RTLD_NOW | RTLD_GLOBAL); + env->ReleaseStringUTFChars(lib_path, path); + if (handle == nullptr) { + jclass Exception = env->FindClass("java/lang/Exception"); + auto error = dlerror(); + env->ThrowNew(Exception, error); + } +#endif +} JNIEXPORT jlong JNICALL Java_com_kuzudb_KuzuNative_kuzu_1database_1init(JNIEnv* env, jclass, jstring database_path, jlong buffer_pool_size, jboolean enable_compression, diff --git a/tools/java_api/src/main/java/com/kuzudb/KuzuNative.java b/tools/java_api/src/main/java/com/kuzudb/KuzuNative.java index 87ec6ca98d..9f37e9f0ed 100644 --- a/tools/java_api/src/main/java/com/kuzudb/KuzuNative.java +++ b/tools/java_api/src/main/java/com/kuzudb/KuzuNative.java @@ -54,12 +54,21 @@ public class KuzuNative { } Files.copy(lib_res.openStream(), lib_file, StandardCopyOption.REPLACE_EXISTING); new File(lib_file.toString()).deleteOnExit(); - System.load(lib_file.toAbsolutePath().toString()); + String lib_path = lib_file.toAbsolutePath().toString(); + System.load(lib_path); + if(os_name.equals("linux")) { + kuzu_native_reload_library(lib_path); + } } catch (IOException e) { e.printStackTrace(); } } + // Hack: Reload the native library again in JNI bindings to work around the + // extension loading issue on Linux as System.load() does not set + // `RTLD_GLOBAL` flag and there is no way to set it in Java. + protected static native void kuzu_native_reload_library(String lib_path); + // Database protected static native long kuzu_database_init(String database_path, long buffer_pool_size, boolean enable_compression, boolean read_only); diff --git a/tools/java_api/src/test/java/com/kuzudb/test/ExtensionTest.java b/tools/java_api/src/test/java/com/kuzudb/test/ExtensionTest.java index bab76e9c93..ec8d65321f 100644 --- a/tools/java_api/src/test/java/com/kuzudb/test/ExtensionTest.java +++ b/tools/java_api/src/test/java/com/kuzudb/test/ExtensionTest.java @@ -11,9 +11,8 @@ void HttpfsInstallAndLoad() throws KuzuObjectRefDestroyedException { KuzuQueryResult result = conn.query("INSTALL httpfs"); assertTrue(result.isSuccess()); result.destroy(); - // Skip loading the extension for now until the fix is in place - // result = conn.query("LOAD EXTENSION httpfs"); - // assertTrue(result.isSuccess()); - // result.destroy(); + result = conn.query("LOAD EXTENSION httpfs"); + assertTrue(result.isSuccess()); + result.destroy(); } } From e2e2f22da700fac8676a493ac0d86ddb674ec556 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Fri, 16 Feb 2024 11:40:30 +0800 Subject: [PATCH 2/3] Fix linter error --- tools/java_api/src/jni/kuzu_java.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/java_api/src/jni/kuzu_java.cpp b/tools/java_api/src/jni/kuzu_java.cpp index 756e344ba2..d12c75f576 100644 --- a/tools/java_api/src/jni/kuzu_java.cpp +++ b/tools/java_api/src/jni/kuzu_java.cpp @@ -166,7 +166,7 @@ JNIEXPORT void JNICALL Java_com_kuzudb_KuzuNative_kuzu_1native_1reload_1library( env->ReleaseStringUTFChars(lib_path, path); if (handle == nullptr) { jclass Exception = env->FindClass("java/lang/Exception"); - auto error = dlerror(); + auto error = dlerror(); // NOLINT(concurrency-mt-unsafe): load can only be executed in single thread. env->ThrowNew(Exception, error); } #endif From b552879fc2a763fb1d1a0682d27226feef264433 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Fri, 16 Feb 2024 11:44:35 +0800 Subject: [PATCH 3/3] Try lazy loading --- tools/java_api/src/jni/kuzu_java.cpp | 2 +- tools/python_api/src_py/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/java_api/src/jni/kuzu_java.cpp b/tools/java_api/src/jni/kuzu_java.cpp index d12c75f576..159c677272 100644 --- a/tools/java_api/src/jni/kuzu_java.cpp +++ b/tools/java_api/src/jni/kuzu_java.cpp @@ -162,7 +162,7 @@ JNIEXPORT void JNICALL Java_com_kuzudb_KuzuNative_kuzu_1native_1reload_1library( // Do nothing on Windows #else const char* path = env->GetStringUTFChars(lib_path, JNI_FALSE); - void* handle = dlopen(path, RTLD_NOW | RTLD_GLOBAL); + void* handle = dlopen(path, RTLD_LAZY | RTLD_GLOBAL); env->ReleaseStringUTFChars(lib_path, path); if (handle == nullptr) { jclass Exception = env->FindClass("java/lang/Exception"); diff --git a/tools/python_api/src_py/__init__.py b/tools/python_api/src_py/__init__.py index b478c830d5..6a71049e29 100644 --- a/tools/python_api/src_py/__init__.py +++ b/tools/python_api/src_py/__init__.py @@ -40,11 +40,11 @@ import sys import os -# Set RTLD_GLOBAL and RTLD_NOW flags on Linux to fix the issue with loading +# Set RTLD_GLOBAL and RTLD_LAZY flags on Linux to fix the issue with loading # extensions if sys.platform == "linux": original_dlopen_flags = sys.getdlopenflags() - sys.setdlopenflags(os.RTLD_GLOBAL | os.RTLD_NOW) + sys.setdlopenflags(os.RTLD_GLOBAL | os.RTLD_LAZY) from .database import * from .connection import *