diff --git a/.gitignore b/.gitignore index 34cb540..bb17d56 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ docs/html *.obj *.elf -# Precompiled Headers +# Precompiled headers *.gch *.pch @@ -23,3 +23,6 @@ docs/html *.so *.so.* *.dylib + +# Java bytecode +*.class diff --git a/.travis.yml b/.travis.yml index e7bd4a6..4b2d057 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,14 @@ language: c +os: + - linux + - osx + compiler: - gcc before_install: - - sudo apt-get install -y build-essential - - sudo apt-get install -y python3-pip - - sudo apt-get install -y python3-cffi - - sudo pip3 install --upgrade pip setuptools - - sudo pip3 install --ignore-installed pyota + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y build-essential; fi script: - make check diff --git a/Makefile b/Makefile index 0987a4e..ea3d45f 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +VERSION = 0.3.0 + OUT ?= ./build SRC := src @@ -89,7 +91,15 @@ TESTS := $(addprefix $(OUT)/test-, $(TESTS)) LIBS = libdcurl.so LIBS := $(addprefix $(OUT)/, $(LIBS)) -all: config $(TESTS) $(LIBS) +JARS := dcurljni-$(VERSION).jar +JARS := $(addprefix $(OUT)/, $(JARS)) + +PREQ := config $(TESTS) $(LIBS) +ifeq ("$(BUILD_JNI)","1") +PREQ += $(JARS) +endif + +all: $(PREQ) .DEFAULT_GOAL := all OBJS = \ diff --git a/README.md b/README.md index 994e094..f4076b2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # dcurl - Multi-threaded Curl implementation -[![Build Status](https://travis-ci.org/DLTcollab/dcurl.svg?branch=develop)](https://travis-ci.org/DLTcollab/dcurl) -![Supported IRI version](https://img.shields.io/badge/Supported%20IRI%20Version-1.6.0-brightgreen.svg) +[![Build Status](https://badge.buildkite.com/46ec07b122bde13f984c241fe8b38e64698c5c0d816ee6c7e4.svg)](https://buildkite.com/dltcollab/dcurl-test) +![Supported IRI version](https://img.shields.io/badge/Supported%20IRI%20Version-1.7.0-brightgreen.svg) ![Release version](https://img.shields.io/github/release-pre/DLTcollab/dcurl.svg) Hardware-accelerated implementation for IOTA PearlDiver, which utilizes multi-threaded SIMD, FPGA and GPU. @@ -28,7 +28,13 @@ After integrating dcurl into IRI, performance of [attachToTangle](https://iota.r ## IRI Adaptation [Modified IRI accepting external PoW Library](https://github.com/DLTcollab/iri) -Supported IRI version: 1.6.0 +Supported IRI version: 1.7.0 + +* ```$ cd ~/iri && mvn compile && mvn package``` +* ```$ java -jar target/iri.jar -p --pearldiver-exlib dcurl``` + +or with the **deprecated** commands + * ```$ cd ~/iri && mvn compile && mvn package``` * ```$ cp ~/dcurl/build/libdcurl.so ~/iri``` * ```$ cd ~/iri && java -Djava.library.path=./ -jar target/iri.jar -p --pearldiver-exlib dcurl``` diff --git a/docs/build-n-test.md b/docs/build-n-test.md index 42c2e7c..fe9551c 100644 --- a/docs/build-n-test.md +++ b/docs/build-n-test.md @@ -1,12 +1,12 @@ # Building and Testing ## Prerequisites -* You need to configure paths and flags of OpenCL installation in `mk/opencl.mk`. * Check JDK installation and set environment variable `JAVA_HOME` if you wish to specify. * If target platform lacks of Intel SSE instructions, multi-threaded pure C implementation would be used as the fallback. +* Install the OpenCL and GPU driver before calculating the PoW with GPU. * For FPGA-based hardware accelerator, [Lampa Lab's Cyclone V FPGA PoW](https://github.com/LampaLab/iota_fpga) is taken as the basis. - File `soc_system.rbf` is only valid for DE10-nano board, and you have to synthesize to get appropriate `soc_system.rbf` for Arrow SoCKit board. - - [RBF file](https://github.com/ajblane/dcurl/releases/tag/v1.0-SoCKit) can be downloaded from our release. + - [RBF file](https://github.com/DLTcollab/iota_fpga/releases/tag/v0.3-sockit) can be downloaded from our release. - Moreover, you need to download [Lampa Lab-provided Linux image](https://github.com/LampaLab/iota_fpga/releases/tag/v0.1) to flash into the micro-SD card. The root password is `123456`. @@ -17,7 +17,7 @@ - ``BUILD_SSE``: build the Intel SSE-accelerated Curl backend. - ``BUILD_GPU``: build the OpenCL-based GPU accelerations. - ``BUILD_FPGA_ACCEL``: build the interface interacting with the Cyclone V FPGA based accelerator. Verified on DE10-nano board and Arrow SoCKit board. - - ``BUILD_JNI``: build a shared library for IRI. The build system would generate JNI header file + - ``BUILD_JNI``: build a JAR file including the shared library and the JAVA bytecode for IRI. The build system would generate JNI header file downloading from [latest JAVA source](https://github.com/DLTcollab/iri). - ``BUILD_COMPAT``: build extra cCurl compatible interface. - ``BUILD_STAT``: show the statistics of the PoW information. diff --git a/docs/threading-model.md b/docs/threading-model.md new file mode 100644 index 0000000..caf79e1 --- /dev/null +++ b/docs/threading-model.md @@ -0,0 +1,28 @@ +# Threading model +In `dcurl`, it calls the APIs provided by the `libtuv` submodule to use the threads and thread pool. + +## Thread pool size +The thread pool size initialization is written in the function `init_once()`.\ +The environment variable `UV_THREADPOOL_SIZE` can be used to overwrite the default pool size.\ +Default pool size: 4\ +Maximum pool size: 128 + +The current `dcurl` implementation can not affect the thread pool size. + +## Thread pool mechanism +The thread pool and the work request queue are global.\ +Work request can be treated as the function you would like to request the thread to execute. + +In the thread pool initialization phase, the threads are created and execute the infinite loop function `worker()`.\ +The infinite loop makes the thread blocked with the function `uv_cond_wait()`, and it can be unblocked with the function `uv_cond_signal()`. + +The function `uv_queue_work()` push the work request into the queue and unblock one of the blocked threads. +Then the unblocked thread pops out the work request from the queue and execute it.\ +If all the threads are busy, the first thread finishing its job would check the non-empty queue, pop out the work request and execute it. + +In `dcurl`, it calls the function `uv_queue_work()` and passes the arguments with the PoW calculation function pointer and necessary data.\ +If the thread pool is large enough, the specified number of threads (which is the argument of `dcurl_entry()`) can be unblocked and fully utilized.\ +Otherwise, the `dcurl` would not be able to use as many threads as it specified at the same time. + +## Thread affinity +The current thread pool mechanism does not take thread affinity into consideration. diff --git a/java/org/dltcollab/Dcurl.java b/java/org/dltcollab/Dcurl.java new file mode 100644 index 0000000..c1085bf --- /dev/null +++ b/java/org/dltcollab/Dcurl.java @@ -0,0 +1,43 @@ +package org.dltcollab; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; + +public class Dcurl { + private static final String libraryName = "dcurl"; + private static final String libraryPrefix = "lib"; + private static final String librarySuffix = ".so"; + private static final String libraryFileName = libraryPrefix + libraryName + librarySuffix; + + public void loadLibraryFromJar() { + final File temp; + try { + temp = File.createTempFile(libraryPrefix + libraryName, librarySuffix); + if (temp.exists() && !temp.delete()) { + throw new RuntimeException("File: " + temp.getAbsolutePath() + " already exists and cannot be removed."); + } + if (!temp.createNewFile()) { + throw new RuntimeException("File: " + temp.getAbsolutePath() + " could not be created."); + } + + if (!temp.exists()) { + throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist."); + } else { + temp.deleteOnExit(); + } + + // attempt to copy the library from the Jar file to the temp destination + try (final InputStream is = getClass().getClassLoader().getResourceAsStream(libraryFileName)) { + if (is == null) { + throw new RuntimeException(libraryFileName + " was not found inside JAR."); + } else { + Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } + + System.load(temp.getAbsolutePath()); + } catch (IOException e) { + } + } +} diff --git a/jni/iri-pearldiver-exlib.c b/jni/iri-pearldiver-exlib.c index 61cc062..bc1deba 100644 --- a/jni/iri-pearldiver-exlib.c +++ b/jni/iri-pearldiver-exlib.c @@ -5,7 +5,7 @@ #include "../src/trinary.h" JNIEXPORT jboolean JNICALL -Java_com_iota_iri_hash_PearlDiver_exlib_1init(JNIEnv *env, jclass clazz) +Java_com_iota_iri_crypto_PearlDiver_exlibInit(JNIEnv *env, jclass clazz) { if (!dcurl_init()) return JNI_FALSE; @@ -13,7 +13,7 @@ Java_com_iota_iri_hash_PearlDiver_exlib_1init(JNIEnv *env, jclass clazz) } JNIEXPORT jboolean JNICALL -Java_com_iota_iri_hash_PearlDiver_exlib_1search(JNIEnv *env, +Java_com_iota_iri_crypto_PearlDiver_exlibSearch(JNIEnv *env, jclass clazz, jbyteArray trits, jint mwm, @@ -48,13 +48,13 @@ Java_com_iota_iri_hash_PearlDiver_exlib_1search(JNIEnv *env, } JNIEXPORT void JNICALL -Java_com_iota_iri_hash_PearlDiver_exlib_1cancel(JNIEnv *env, jclass clazz) +Java_com_iota_iri_crypto_PearlDiver_exlibCancel(JNIEnv *env, jclass clazz) { /* Do nothing */ } JNIEXPORT void JNICALL -Java_com_iota_iri_hash_PearlDiver_exlib_1destroy(JNIEnv *env, jclass clazz) +Java_com_iota_iri_crypto_PearlDiver_exlibDestroy(JNIEnv *env, jclass clazz) { dcurl_destroy(); } diff --git a/mk/java.mk b/mk/java.mk index af0e09d..b8c6812 100644 --- a/mk/java.mk +++ b/mk/java.mk @@ -1,16 +1,17 @@ UNAME_S := $(shell uname -s) +JAVAC := $(shell which javac) +ifndef JAVAC +$(error "javac is not available. Please check JDK installation") +endif + # if JAVA_HOME is not set, guess it according to system configurations ifndef JAVA_HOME ifeq ($(UNAME_S),Darwin) # macOS JAVA_HOME := $(shell /usr/libexec/java_home) else -# Default to Linux - JAVAC := $(shell which javac) - ifndef JAVAC - $(error "javac is not available. Please check JDK installation") - endif + # Default to Linux JAVA_HOME := $(shell readlink -f $(JAVAC) | sed "s:bin/javac::") endif endif # JAVA_HOME @@ -60,3 +61,12 @@ jni/iri-pearldiver-exlib.c: $(OUT)/jni/iri-pearldiver-exlib.h $(OUT)/jni/%.o: jni/%.c $(VECHO) " CC\t$@\n" $(Q)$(CC) -o $@ $(CFLAGS) $(CFLAGS_JNI) -c -MMD -MF $@.d $< + +java/org/dltcollab/%.class: java/org/dltcollab/%.java + $(VECHO) " JAVAC\t$@\n" + $(Q)$(JAVAC) $< + +$(OUT)/dcurljni-$(VERSION).jar: $(OUT)/libdcurl.so java/org/dltcollab/Dcurl.class + $(VECHO) " JAR\t$@\n" + $(Q)jar -cf $@ -C $(OUT) libdcurl.so + $(Q)jar -uf $@ -C java org/dltcollab/Dcurl.class diff --git a/mk/libdcurl.version b/mk/libdcurl.version index 205dc04..a4acd8b 100644 --- a/mk/libdcurl.version +++ b/mk/libdcurl.version @@ -7,7 +7,7 @@ CODEABI_1.0 { ccurl_*; /* JNI */ - Java_com_iota_iri_hash_PearlDiver*; + Java_com_iota_iri_crypto_PearlDiver*; local: *; }; diff --git a/mk/opencl.mk b/mk/opencl.mk index 1005507..bf9dcc1 100644 --- a/mk/opencl.mk +++ b/mk/opencl.mk @@ -1,16 +1,17 @@ -# Default the path to the NVIDIA Run time Library -OPENCL_LIB_PATH ?= /usr/local/cuda/lib64 -OPENCL_LIB_DIR := $(shell test -d $(OPENCL_LIB_PATH); echo $$?) - -ifneq ($(OPENCL_LIB_DIR),0) -$(error "Please specify the path to OpenCL library.") -endif - -OPENCL_LIB_AVAIL := $(shell ls $(OPENCL_LIB_PATH) | grep -oi libOpenCL.so > /dev/null; echo $$?) +# The auto-detected path of the OpenCL library +OPENCL_LIB_PATH := $(shell pkg-config --variable=libdir OpenCL 2> /dev/null) +OPENCL_LIB_AVAIL := $(shell test -e $(OPENCL_LIB_PATH)/libOpenCL.so; echo $$?) +UNAME_S := $(shell uname -s) ifneq ($(OPENCL_LIB_AVAIL),0) -$(error "Please check installation of OpenCL dynamic library in $(OPENCL_LIB_PATH)") +$(error "Please install the OpenCL library correctly.") endif CFLAGS += -DENABLE_OPENCL -LDFLAGS += -L$(OPENCL_LIB_PATH) -lOpenCL +ifeq ($(UNAME_S),Darwin) + # macOS + LDFLAGS += -F$(OPENCL_LIB_PATH) -framework OpenCL +else + # Default to Linux + LDFLAGS += -L$(OPENCL_LIB_PATH) -lOpenCL +endif diff --git a/src/list.h b/src/list.h index c447d58..dd8f2aa 100644 --- a/src/list.h +++ b/src/list.h @@ -1,13 +1,13 @@ /* - * Copyright (C) 2018 National Cheng Kung University, Taiwan. + * Copyright (C) 2018 dcurl Developers. * Use of this source code is governed by MIT license that can be * found in the LICENSE file. */ /* Linux-like double-linked list implementation */ -#ifndef SYSPROG21_LIST_H -#define SYSPROG21_LIST_H +#ifndef LIST_UTIL_H +#define LIST_UTIL_H #ifdef __cplusplus extern "C" { @@ -15,6 +15,11 @@ extern "C" { #include +/* suppress compilation warnings on macOS */ +#ifdef LIST_HEAD +#undef LIST_HEAD +#endif + /* "typeof" is a GNU extension. * Reference: https://gcc.gnu.org/onlinedocs/gcc/Typeof.html */ @@ -441,4 +446,4 @@ static inline void list_move_tail(struct list_head *node, } #endif -#endif /* SYSPROG21_LIST_H */ +#endif /* LIST_UTIL_H */