From c10d328045a4067d76e25c775dc76ec520477b5e Mon Sep 17 00:00:00 2001 From: marktwtn Date: Wed, 3 Apr 2019 12:50:09 +0800 Subject: [PATCH 01/13] build: Detect the OpenCL library automatically --- mk/opencl.mk | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/mk/opencl.mk b/mk/opencl.mk index 1005507..bf9dcc1 100644 --- a/mk/opencl.mk +++ b/mk/opencl.mk @@ -1,16 +1,17 @@ -# Default the path to the NVIDIA Run time Library -OPENCL_LIB_PATH ?= /usr/local/cuda/lib64 -OPENCL_LIB_DIR := $(shell test -d $(OPENCL_LIB_PATH); echo $$?) - -ifneq ($(OPENCL_LIB_DIR),0) -$(error "Please specify the path to OpenCL library.") -endif - -OPENCL_LIB_AVAIL := $(shell ls $(OPENCL_LIB_PATH) | grep -oi libOpenCL.so > /dev/null; echo $$?) +# The auto-detected path of the OpenCL library +OPENCL_LIB_PATH := $(shell pkg-config --variable=libdir OpenCL 2> /dev/null) +OPENCL_LIB_AVAIL := $(shell test -e $(OPENCL_LIB_PATH)/libOpenCL.so; echo $$?) +UNAME_S := $(shell uname -s) ifneq ($(OPENCL_LIB_AVAIL),0) -$(error "Please check installation of OpenCL dynamic library in $(OPENCL_LIB_PATH)") +$(error "Please install the OpenCL library correctly.") endif CFLAGS += -DENABLE_OPENCL -LDFLAGS += -L$(OPENCL_LIB_PATH) -lOpenCL +ifeq ($(UNAME_S),Darwin) + # macOS + LDFLAGS += -F$(OPENCL_LIB_PATH) -framework OpenCL +else + # Default to Linux + LDFLAGS += -L$(OPENCL_LIB_PATH) -lOpenCL +endif From ab451b326b93a9181e614cd3635391f688e2abd1 Mon Sep 17 00:00:00 2001 From: marktwtn Date: Sat, 6 Apr 2019 15:30:07 +0800 Subject: [PATCH 02/13] CI: Remove the installation of unused tools --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e7bd4a6..6973a27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,10 +5,6 @@ compiler: before_install: - sudo apt-get install -y build-essential - - sudo apt-get install -y python3-pip - - sudo apt-get install -y python3-cffi - - sudo pip3 install --upgrade pip setuptools - - sudo pip3 install --ignore-installed pyota script: - make check From a8fe0ae7853ec2948f137eb8ad526d323ee45378 Mon Sep 17 00:00:00 2001 From: marktwtn Date: Sat, 6 Apr 2019 16:09:24 +0800 Subject: [PATCH 03/13] CI: Test the source code on macOS --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6973a27..4b2d057 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,14 @@ language: c +os: + - linux + - osx + compiler: - gcc before_install: - - sudo apt-get install -y build-essential + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y build-essential; fi script: - make check From 011178bb50933a1d2c9cc9a90a869a8849f8d2eb Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Wed, 10 Apr 2019 16:07:04 +0800 Subject: [PATCH 04/13] Relicense list utility As the original author for this file, I would like to simplify the terms of use. --- src/list.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/list.h b/src/list.h index c447d58..f2292c7 100644 --- a/src/list.h +++ b/src/list.h @@ -1,13 +1,13 @@ /* - * Copyright (C) 2018 National Cheng Kung University, Taiwan. + * Copyright (C) 2018 dcurl Developers. * Use of this source code is governed by MIT license that can be * found in the LICENSE file. */ /* Linux-like double-linked list implementation */ -#ifndef SYSPROG21_LIST_H -#define SYSPROG21_LIST_H +#ifndef LIST_UTIL_H +#define LIST_UTIL_H #ifdef __cplusplus extern "C" { @@ -441,4 +441,4 @@ static inline void list_move_tail(struct list_head *node, } #endif -#endif /* SYSPROG21_LIST_H */ +#endif /* LIST_UTIL_H */ From 78f6fc8d0e5e7a3a7ee1a0453318a974bf60280d Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Wed, 10 Apr 2019 16:10:24 +0800 Subject: [PATCH 05/13] fix: Suppress compilation warnings on macOS Close #99 --- src/list.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/list.h b/src/list.h index f2292c7..dd8f2aa 100644 --- a/src/list.h +++ b/src/list.h @@ -15,6 +15,11 @@ extern "C" { #include +/* suppress compilation warnings on macOS */ +#ifdef LIST_HEAD +#undef LIST_HEAD +#endif + /* "typeof" is a GNU extension. * Reference: https://gcc.gnu.org/onlinedocs/gcc/Typeof.html */ From f1c92c948cc92c00d5f0150742a418f6220eab02 Mon Sep 17 00:00:00 2001 From: marktwtn Date: Fri, 12 Apr 2019 02:42:48 +0800 Subject: [PATCH 06/13] docs: Add the libtuv threading model document Close #102. --- docs/threading-model.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docs/threading-model.md diff --git a/docs/threading-model.md b/docs/threading-model.md new file mode 100644 index 0000000..caf79e1 --- /dev/null +++ b/docs/threading-model.md @@ -0,0 +1,28 @@ +# Threading model +In `dcurl`, it calls the APIs provided by the `libtuv` submodule to use the threads and thread pool. + +## Thread pool size +The thread pool size initialization is written in the function `init_once()`.\ +The environment variable `UV_THREADPOOL_SIZE` can be used to overwrite the default pool size.\ +Default pool size: 4\ +Maximum pool size: 128 + +The current `dcurl` implementation can not affect the thread pool size. + +## Thread pool mechanism +The thread pool and the work request queue are global.\ +Work request can be treated as the function you would like to request the thread to execute. + +In the thread pool initialization phase, the threads are created and execute the infinite loop function `worker()`.\ +The infinite loop makes the thread blocked with the function `uv_cond_wait()`, and it can be unblocked with the function `uv_cond_signal()`. + +The function `uv_queue_work()` push the work request into the queue and unblock one of the blocked threads. +Then the unblocked thread pops out the work request from the queue and execute it.\ +If all the threads are busy, the first thread finishing its job would check the non-empty queue, pop out the work request and execute it. + +In `dcurl`, it calls the function `uv_queue_work()` and passes the arguments with the PoW calculation function pointer and necessary data.\ +If the thread pool is large enough, the specified number of threads (which is the argument of `dcurl_entry()`) can be unblocked and fully utilized.\ +Otherwise, the `dcurl` would not be able to use as many threads as it specified at the same time. + +## Thread affinity +The current thread pool mechanism does not take thread affinity into consideration. From c122cca9e4c75e0c692e673a17686b0eb0b56de1 Mon Sep 17 00:00:00 2001 From: marktwtn Date: Tue, 9 Apr 2019 17:10:44 +0800 Subject: [PATCH 07/13] docs: Change the badge from travis CI to buildkite CI --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 994e094..db83f35 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # dcurl - Multi-threaded Curl implementation -[![Build Status](https://travis-ci.org/DLTcollab/dcurl.svg?branch=develop)](https://travis-ci.org/DLTcollab/dcurl) +[![Build Status](https://badge.buildkite.com/46ec07b122bde13f984c241fe8b38e64698c5c0d816ee6c7e4.svg)](https://buildkite.com/dltcollab/dcurl-test) ![Supported IRI version](https://img.shields.io/badge/Supported%20IRI%20Version-1.6.0-brightgreen.svg) ![Release version](https://img.shields.io/github/release-pre/DLTcollab/dcurl.svg) From c911e8a4ab11ab6e2a6d9c7596492b76ec7d66c0 Mon Sep 17 00:00:00 2001 From: marktwtn Date: Fri, 12 Apr 2019 04:44:35 +0800 Subject: [PATCH 08/13] docs: Change the prerequisite of the GPU part The OpenCL paths and flags have been able to be selected automatically . --- docs/build-n-test.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build-n-test.md b/docs/build-n-test.md index 42c2e7c..1692db8 100644 --- a/docs/build-n-test.md +++ b/docs/build-n-test.md @@ -1,9 +1,9 @@ # Building and Testing ## Prerequisites -* You need to configure paths and flags of OpenCL installation in `mk/opencl.mk`. * Check JDK installation and set environment variable `JAVA_HOME` if you wish to specify. * If target platform lacks of Intel SSE instructions, multi-threaded pure C implementation would be used as the fallback. +* Install the OpenCL and GPU driver before calculating the PoW with GPU. * For FPGA-based hardware accelerator, [Lampa Lab's Cyclone V FPGA PoW](https://github.com/LampaLab/iota_fpga) is taken as the basis. - File `soc_system.rbf` is only valid for DE10-nano board, and you have to synthesize to get appropriate `soc_system.rbf` for Arrow SoCKit board. - [RBF file](https://github.com/ajblane/dcurl/releases/tag/v1.0-SoCKit) can be downloaded from our release. From 7c99849a373f42a8fb2b61e42bfaac15701f0698 Mon Sep 17 00:00:00 2001 From: marktwtn Date: Sat, 13 Apr 2019 21:54:52 +0800 Subject: [PATCH 09/13] docs: Update the link of RBF file --- docs/build-n-test.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build-n-test.md b/docs/build-n-test.md index 1692db8..21f4387 100644 --- a/docs/build-n-test.md +++ b/docs/build-n-test.md @@ -6,7 +6,7 @@ * Install the OpenCL and GPU driver before calculating the PoW with GPU. * For FPGA-based hardware accelerator, [Lampa Lab's Cyclone V FPGA PoW](https://github.com/LampaLab/iota_fpga) is taken as the basis. - File `soc_system.rbf` is only valid for DE10-nano board, and you have to synthesize to get appropriate `soc_system.rbf` for Arrow SoCKit board. - - [RBF file](https://github.com/ajblane/dcurl/releases/tag/v1.0-SoCKit) can be downloaded from our release. + - [RBF file](https://github.com/DLTcollab/iota_fpga/releases/tag/v0.3-sockit) can be downloaded from our release. - Moreover, you need to download [Lampa Lab-provided Linux image](https://github.com/LampaLab/iota_fpga/releases/tag/v0.1) to flash into the micro-SD card. The root password is `123456`. From b78fda7e796bc811d8dcea55d3d7a1430f129e9f Mon Sep 17 00:00:00 2001 From: marktwtn Date: Sat, 20 Apr 2019 09:45:59 +0800 Subject: [PATCH 10/13] fix: Fix the JNI related function name of IRI Change the name from `hash` to `crypto` and use the correct method name. The IRI can apply the dcurl library successfully now. --- jni/iri-pearldiver-exlib.c | 8 ++++---- mk/libdcurl.version | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jni/iri-pearldiver-exlib.c b/jni/iri-pearldiver-exlib.c index 61cc062..bc1deba 100644 --- a/jni/iri-pearldiver-exlib.c +++ b/jni/iri-pearldiver-exlib.c @@ -5,7 +5,7 @@ #include "../src/trinary.h" JNIEXPORT jboolean JNICALL -Java_com_iota_iri_hash_PearlDiver_exlib_1init(JNIEnv *env, jclass clazz) +Java_com_iota_iri_crypto_PearlDiver_exlibInit(JNIEnv *env, jclass clazz) { if (!dcurl_init()) return JNI_FALSE; @@ -13,7 +13,7 @@ Java_com_iota_iri_hash_PearlDiver_exlib_1init(JNIEnv *env, jclass clazz) } JNIEXPORT jboolean JNICALL -Java_com_iota_iri_hash_PearlDiver_exlib_1search(JNIEnv *env, +Java_com_iota_iri_crypto_PearlDiver_exlibSearch(JNIEnv *env, jclass clazz, jbyteArray trits, jint mwm, @@ -48,13 +48,13 @@ Java_com_iota_iri_hash_PearlDiver_exlib_1search(JNIEnv *env, } JNIEXPORT void JNICALL -Java_com_iota_iri_hash_PearlDiver_exlib_1cancel(JNIEnv *env, jclass clazz) +Java_com_iota_iri_crypto_PearlDiver_exlibCancel(JNIEnv *env, jclass clazz) { /* Do nothing */ } JNIEXPORT void JNICALL -Java_com_iota_iri_hash_PearlDiver_exlib_1destroy(JNIEnv *env, jclass clazz) +Java_com_iota_iri_crypto_PearlDiver_exlibDestroy(JNIEnv *env, jclass clazz) { dcurl_destroy(); } diff --git a/mk/libdcurl.version b/mk/libdcurl.version index 205dc04..a4acd8b 100644 --- a/mk/libdcurl.version +++ b/mk/libdcurl.version @@ -7,7 +7,7 @@ CODEABI_1.0 { ccurl_*; /* JNI */ - Java_com_iota_iri_hash_PearlDiver*; + Java_com_iota_iri_crypto_PearlDiver*; local: *; }; From 22526dacf106a4f1585ad32cbd3b10d8ec2a8827 Mon Sep 17 00:00:00 2001 From: marktwtn Date: Fri, 19 Apr 2019 04:11:13 +0800 Subject: [PATCH 11/13] feat: Create JAR file for Java native interface Build with option `BUILD_JNI=1` would compile the Java code and create the JAR file. The dcurl shared library `libdcurl.so` would be packaged into the JAR file and can be extracted from it for Jave native interface easily. Close #106. --- .gitignore | 5 +++- Makefile | 12 +++++++++- java/org/dltcollab/Dcurl.java | 43 +++++++++++++++++++++++++++++++++++ mk/java.mk | 20 ++++++++++++---- 4 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 java/org/dltcollab/Dcurl.java diff --git a/.gitignore b/.gitignore index 34cb540..bb17d56 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ docs/html *.obj *.elf -# Precompiled Headers +# Precompiled headers *.gch *.pch @@ -23,3 +23,6 @@ docs/html *.so *.so.* *.dylib + +# Java bytecode +*.class diff --git a/Makefile b/Makefile index 0987a4e..475bb46 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +VERSION = 0.2.0 + OUT ?= ./build SRC := src @@ -89,7 +91,15 @@ TESTS := $(addprefix $(OUT)/test-, $(TESTS)) LIBS = libdcurl.so LIBS := $(addprefix $(OUT)/, $(LIBS)) -all: config $(TESTS) $(LIBS) +JARS := dcurljni-$(VERSION).jar +JARS := $(addprefix $(OUT)/, $(JARS)) + +PREQ := config $(TESTS) $(LIBS) +ifeq ("$(BUILD_JNI)","1") +PREQ += $(JARS) +endif + +all: $(PREQ) .DEFAULT_GOAL := all OBJS = \ diff --git a/java/org/dltcollab/Dcurl.java b/java/org/dltcollab/Dcurl.java new file mode 100644 index 0000000..c1085bf --- /dev/null +++ b/java/org/dltcollab/Dcurl.java @@ -0,0 +1,43 @@ +package org.dltcollab; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +public class Dcurl { + private static final String libraryName = "dcurl"; + private static final String libraryPrefix = "lib"; + private static final String librarySuffix = ".so"; + private static final String libraryFileName = libraryPrefix + libraryName + librarySuffix; + + public void loadLibraryFromJar() { + final File temp; + try { + temp = File.createTempFile(libraryPrefix + libraryName, librarySuffix); + if (temp.exists() && !temp.delete()) { + throw new RuntimeException("File: " + temp.getAbsolutePath() + " already exists and cannot be removed."); + } + if (!temp.createNewFile()) { + throw new RuntimeException("File: " + temp.getAbsolutePath() + " could not be created."); + } + + if (!temp.exists()) { + throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist."); + } else { + temp.deleteOnExit(); + } + + // attempt to copy the library from the Jar file to the temp destination + try (final InputStream is = getClass().getClassLoader().getResourceAsStream(libraryFileName)) { + if (is == null) { + throw new RuntimeException(libraryFileName + " was not found inside JAR."); + } else { + Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } + + System.load(temp.getAbsolutePath()); + } catch (IOException e) { + } + } +} diff --git a/mk/java.mk b/mk/java.mk index af0e09d..b8c6812 100644 --- a/mk/java.mk +++ b/mk/java.mk @@ -1,16 +1,17 @@ UNAME_S := $(shell uname -s) +JAVAC := $(shell which javac) +ifndef JAVAC +$(error "javac is not available. Please check JDK installation") +endif + # if JAVA_HOME is not set, guess it according to system configurations ifndef JAVA_HOME ifeq ($(UNAME_S),Darwin) # macOS JAVA_HOME := $(shell /usr/libexec/java_home) else -# Default to Linux - JAVAC := $(shell which javac) - ifndef JAVAC - $(error "javac is not available. Please check JDK installation") - endif + # Default to Linux JAVA_HOME := $(shell readlink -f $(JAVAC) | sed "s:bin/javac::") endif endif # JAVA_HOME @@ -60,3 +61,12 @@ jni/iri-pearldiver-exlib.c: $(OUT)/jni/iri-pearldiver-exlib.h $(OUT)/jni/%.o: jni/%.c $(VECHO) " CC\t$@\n" $(Q)$(CC) -o $@ $(CFLAGS) $(CFLAGS_JNI) -c -MMD -MF $@.d $< + +java/org/dltcollab/%.class: java/org/dltcollab/%.java + $(VECHO) " JAVAC\t$@\n" + $(Q)$(JAVAC) $< + +$(OUT)/dcurljni-$(VERSION).jar: $(OUT)/libdcurl.so java/org/dltcollab/Dcurl.class + $(VECHO) " JAR\t$@\n" + $(Q)jar -cf $@ -C $(OUT) libdcurl.so + $(Q)jar -uf $@ -C java org/dltcollab/Dcurl.class From b008fa4c899f0e6d1ec684c5637c97336d38853a Mon Sep 17 00:00:00 2001 From: marktwtn Date: Tue, 23 Apr 2019 22:54:13 +0800 Subject: [PATCH 12/13] docs: Change the information of JNI and IRI --- README.md | 10 ++++++++-- docs/build-n-test.md | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index db83f35..f4076b2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # dcurl - Multi-threaded Curl implementation [![Build Status](https://badge.buildkite.com/46ec07b122bde13f984c241fe8b38e64698c5c0d816ee6c7e4.svg)](https://buildkite.com/dltcollab/dcurl-test) -![Supported IRI version](https://img.shields.io/badge/Supported%20IRI%20Version-1.6.0-brightgreen.svg) +![Supported IRI version](https://img.shields.io/badge/Supported%20IRI%20Version-1.7.0-brightgreen.svg) ![Release version](https://img.shields.io/github/release-pre/DLTcollab/dcurl.svg) Hardware-accelerated implementation for IOTA PearlDiver, which utilizes multi-threaded SIMD, FPGA and GPU. @@ -28,7 +28,13 @@ After integrating dcurl into IRI, performance of [attachToTangle](https://iota.r ## IRI Adaptation [Modified IRI accepting external PoW Library](https://github.com/DLTcollab/iri) -Supported IRI version: 1.6.0 +Supported IRI version: 1.7.0 + +* ```$ cd ~/iri && mvn compile && mvn package``` +* ```$ java -jar target/iri.jar -p --pearldiver-exlib dcurl``` + +or with the **deprecated** commands + * ```$ cd ~/iri && mvn compile && mvn package``` * ```$ cp ~/dcurl/build/libdcurl.so ~/iri``` * ```$ cd ~/iri && java -Djava.library.path=./ -jar target/iri.jar -p --pearldiver-exlib dcurl``` diff --git a/docs/build-n-test.md b/docs/build-n-test.md index 21f4387..fe9551c 100644 --- a/docs/build-n-test.md +++ b/docs/build-n-test.md @@ -17,7 +17,7 @@ - ``BUILD_SSE``: build the Intel SSE-accelerated Curl backend. - ``BUILD_GPU``: build the OpenCL-based GPU accelerations. - ``BUILD_FPGA_ACCEL``: build the interface interacting with the Cyclone V FPGA based accelerator. Verified on DE10-nano board and Arrow SoCKit board. - - ``BUILD_JNI``: build a shared library for IRI. The build system would generate JNI header file + - ``BUILD_JNI``: build a JAR file including the shared library and the JAVA bytecode for IRI. The build system would generate JNI header file downloading from [latest JAVA source](https://github.com/DLTcollab/iri). - ``BUILD_COMPAT``: build extra cCurl compatible interface. - ``BUILD_STAT``: show the statistics of the PoW information. From b3d30de074f4e0a96a03fe0ec3a9f6f4339082ad Mon Sep 17 00:00:00 2001 From: marktwtn Date: Thu, 25 Apr 2019 15:58:00 +0800 Subject: [PATCH 13/13] Release version 0.3.0 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 475bb46..ea3d45f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -VERSION = 0.2.0 +VERSION = 0.3.0 OUT ?= ./build SRC := src