Merge pull request #183 from DLTcollab/develop

Pre-release v0.4.1

LICENSE

@@ -1,4 +1,5 @@
 Copyright (C) 2018-2019 BiiLabs Co., Ltd. and Contributors.
+Copyright (C) 2018 Ievgen Korokyi.
 Copyright (C) 2017 IOTA AS, IOTA Foundation and Developers.
 Copyright (C) 2016 Shinya Yagyu.
 

Makefile

@@ -1,4 +1,4 @@
-VERSION = 0.4.0
+VERSION = 0.4.1
 
 OUT ?= ./build
 SRC := src
@@ -35,6 +35,9 @@ include mk/cpu-features.mk
 # Handle git submodule
 include mk/submodule.mk
 
+# Board specific compiler flags
+include mk/board.mk
+
 # Assign the hardware to CPU if no hardware is specified
 PLATFORMS := $(BUILD_AVX) $(BUILD_SSE) $(BUILD_GENERIC) $(BUILD_GPU) $(BUILD_FPGA_ACCEL)
 ENABLE_PLATFORMS := $(findstring 1,$(PLATFORMS))
@@ -99,7 +102,7 @@ LIBS := $(addprefix $(OUT)/, $(LIBS))
 JARS := dcurljni-$(VERSION).jar
 JARS := $(addprefix $(OUT)/, $(JARS))
 
-PREQ := config $(TESTS) $(LIBS)
+PREQ := $(SUBS) config $(TESTS) $(LIBS)
 ifeq ("$(BUILD_JNI)","1")
 PREQ += $(JARS)
 endif
@@ -152,27 +155,31 @@ OBJS += \
 	remote_common.o \
 	remote_interface.o
 
-WORKER_OBJS := $(addprefix $(OUT)/worker-,$(filter-out remote_interface.o, $(OBJS)))
+WORKER_EXCLUDE_OBJS := remote_interface.o
+ifeq ("$(BUILD_JNI)", "1")
+WORKER_EXCLUDE_OBJS += jni/iri-pearldiver-exlib.o
+endif
+WORKER_OBJS := $(addprefix $(OUT)/worker-,$(filter-out $(WORKER_EXCLUDE_OBJS), $(OBJS)))
 WORKER_CFLAGS := $(filter-out -DENABLE_REMOTE, $(CFLAGS))
 endif
 
 OBJS := $(addprefix $(OUT)/, $(OBJS))
 
-$(OUT)/test-%.o: tests/test-%.c $(LIBTUV_PATH)/include
+$(OUT)/test-%.o: tests/test-%.c
 	$(VECHO) "  CC\t$@\n"
-	$(Q)$(CC) -o $@ $(CFLAGS) -I $(SRC) $(LIBTUV_INCLUDE) -c -MMD -MF $@.d $<
+	$(Q)$(CC) -o $@ $(CFLAGS) -I $(SRC) $(SUB_INCLUDE) -c -MMD -MF $@.d $<
 
-$(OUT)/%.o: $(SRC)/%.c $(LIBTUV_PATH)/include $(LIBRABBITMQ_PATH)/build/include
+$(OUT)/%.o: $(SRC)/%.c $(SUB_OBJS)
 	$(VECHO) "  CC\t$@\n"
-	$(Q)$(CC) -o $@ $(CFLAGS) $(LIBTUV_INCLUDE) $(LIBRABBITMQ_INCLUDE) -c -MMD -MF $@.d $<
+	$(Q)$(CC) -o $@ $(CFLAGS) $(SUB_INCLUDE) -c -MMD -MF $@.d $<
 
-$(OUT)/test-%: $(OUT)/test-%.o $(OBJS) $(LIBTUV_LIBRARY) $(LIBRABBITMQ_LIBRARY)
+$(OUT)/test-%: $(OUT)/test-%.o $(OBJS) $(SUB_OBJS)
 	$(VECHO) "  LD\t$@\n"
-	$(Q)$(CC) -o $@ $^ $(LDFLAGS) $(LIBRABBITMQ_LINK)
+	$(Q)$(CC) -o $@ $^ $(LDFLAGS)
 
-$(OUT)/libdcurl.so: $(OBJS) $(LIBTUV_LIBRARY) $(LIBRABBITMQ_LIBRARY)
+$(OUT)/libdcurl.so: $(OBJS) $(SUB_OBJS)
 	$(VECHO) "  LD\t$@\n"
-	$(Q)$(CC) -shared -o $@ $^ $(LDFLAGS) $(LIBRABBITMQ_LINK)
+	$(Q)$(CC) -shared -o $@ $^ $(LDFLAGS)
 
 ifeq ("$(BUILD_REMOTE)", "1")
 include mk/remote.mk

README.md

@@ -13,19 +13,16 @@ In addition, dcurl supports FPGA-accelerated PoW, described in [docs/fpga-accele
 dcurl can be regarded as the drop-in replacement for [ccurl](https://github.com/iotaledger/ccurl).
 IOTA Reference Implementation (IRI) adaptation is available to benefit from hardware-accelerated PoW.
 
-
 ## Build Instructions
 Check [docs/build-n-test.md](docs/build-n-test.md) for details.
 
-
 ## Performance
 After integrating dcurl into IRI, performance of [attachToTangle](https://iota.readme.io/reference#attachtotangle) is measured as following.
 * Each sampling is measured with 30 transaction trytes and total 200 samples are measured.
 * mwm = 14, 26 CPU threads to find nonce
 * Settings: enable 2 pow tasks in CPU, 1 pow tasks in GPU at the same time
 ![](https://github.com/raw/DLTcollab/dcurl/develop/docs/benchmark.png)
 
-
 ## IRI Adaptation
 [Modified IRI accepting external PoW Library](https://github.com/DLTcollab/iri)
 Supported IRI version: 1.7.0
@@ -63,6 +60,27 @@ pull requests to dcurl.
   it faciliates dcurl to perform hardware-accelerated PoW operations on
   edge devices.
 
+## FAQ
+- What is **binary encoded ternary**?
+
+  It is a skill to transform the ternary trit value to two separate bits value.\
+  Hence multiple trits can be compressed to two separate data of the same data type and fully utilize the space.
+
+- Can the project [Batch Binary Encoded Ternary Curl](https://github.com/luca-moser/bct_curl) be applied to dcurl?
+
+  The answer is no.\
+  They both use the same skill, **binary encoded ternary**.\
+  However, their purpose are totally different.
+  - **bct_curl**:
+
+    Focus on hashing the multiple data of the same length at the same time.\
+    It ends when the hashing is finished.
+
+  - **dcurl**:
+
+    Focus on trying the different values of one transaction at the same time to find the nonce value.\
+    The procedure of finding the nonce value also does the hashing.\
+    However, it ends when the nonce value is found, which means one of the values is acceptable.
 
 ## Licensing
 `dcurl` is freely redistributable under the MIT License.

docs/build-n-test.md

@@ -17,9 +17,10 @@
     - ``BUILD_SSE``: build the Intel SSE-accelerated Curl backend.
     - ``BUILD_GPU``: build the OpenCL-based GPU accelerations.
     - ``BUILD_FPGA_ACCEL``: build the interface interacting with the Cyclone V FPGA based accelerator. Verified on DE10-nano board and Arrow SoCKit board.
+    - ``BUILD_REMOTE``: build with the remote interface and create the remote worker executable for calculating PoW on remote devices.
     - ``BUILD_JNI``: build a JAR file including the shared library and the JAVA bytecode for IRI. The build system would generate JNI header file
                      downloading from [latest JAVA source](https://github.com/DLTcollab/iri).
-    - ``BUILD_COMPAT``: build extra cCurl compatible interface.
+    - ``BUILD_COMPAT``: build extra cCurl compatible interface. **[deprecated]**
     - ``BUILD_STAT``: show the statistics of the PoW information.
     - ``BUILD_DEBUG``: dump verbose messages internally.
                        Build with the corresponding Sanitizer to detect software potential bugs if the value is `address`, `undefined` or `thread`.

docs/remote-interface.md

@@ -1,67 +1,76 @@
-# Remote interface
-## Introduction
-
-```
- +-----------------------------------------------+
- |               remote interface                |
- |    +----------------------------------------+ | 
- |   +----------------------------------------+| |
- |  +----------------------------------------+|+ |
- |  |     RabbitMQ-provided RPC              |+  |
- |  +----------------------------------------+   |
- |            |                     ^            |
- +------------|---------------------|------------+
- +------------|---------------------|------------+
- |            |   RabbitMQ broker   |            |
- |            |                     |            |
- |            |             +------------------+ |
- |            v            +------------------+| |
- |  +------------------+  +------------------+|| |
- |  | incoming queue   |  | private queue    ||| |
- |  |                  |  |                  ||| |
- |  | - trytes         |  | - PoW result     ||| |
- |  | - mwm            |  |                  ||+ |
- |  |                  |  |                  |+  |
- |  +------------------+  +------------------+   |
- |            |                     ^            |
- +------------|---------------------|------------+
-              v                     |
-   +---------------------------------------------+
-  +---------------------------------------------+|
- +---------------------------------------------+|+
- |                  remote worker              |+
- +---------------------------------------------+
-```
-To support asynchronous remote procedure call, remote interface in dcurl provides an interface named as `Remote_ImplContext` to implement it. dcurl currently uses RabbitMQ C client to implement asynchronous RPC in remote interface. Remote interface provides thread management to support an asynchronous RPC per thread.
-
-Here are detailed implementations of the RabbitMQ-provided RPC pattern as follows:
-* Asynchronous RPC requests are inserted into the message queue, `incoming_queue`, in RabbitMQ broker
-* Asynchronous RPCs with exclusive private queues (callback queues) with TTL = 10s property
-* Correlation ID is not used
-* An asynchronous RPC uses a connection to RabbitMQ broker
-* Remote workers can obtain requests from `incoming_queue` by default exchange of RabbitMQ broker
-
-## How to test remote interface in localhost
-You need to open three terminals
-
-Terminal 1: Run the RabbitMQ broker You can quickly use docker to run the RabbitMQ broker, rabbitmq
-```
-$ sudo docker run -d rabbitmq
-```
-
-Terminal 2: Run remote workers
-```
-$ ./build/remote-worker
-```
-How to build remote worker on FPGA board
-```
-$ make BUILD_REMOTE=1 BUILD_FPGA_ACCEL=1 BOARD=de10nano
-```
-
-Terminal 3: Run check
-```
-$ make BUILD_REMOTE=1 BUILD_DEBUG=1 check
-```
-
-## Requirements
-Remote interface requires RabbitMQ broker
+# Remote interface
+## Introduction
+
+```
+ +-----------------------------------------------+
+ |               remote interface                |
+ |    +----------------------------------------+ |
+ |   +----------------------------------------+| |
+ |  +----------------------------------------+|+ |
+ |  |     RabbitMQ-provided RPC              |+  |
+ |  +----------------------------------------+   |
+ |            |                     ^            |
+ +------------|---------------------|------------+
+ +------------|---------------------|------------+
+ |            |   RabbitMQ broker   |            |
+ |            |                     |            |
+ |            |             +------------------+ |
+ |            v            +------------------+| |
+ |  +------------------+  +------------------+|| |
+ |  | incoming queue   |  | private queue    ||| |
+ |  |                  |  |                  ||| |
+ |  | - trytes         |  | - PoW result     ||| |
+ |  | - mwm            |  |                  ||+ |
+ |  |                  |  |                  |+  |
+ |  +------------------+  +------------------+   |
+ |            |                     ^            |
+ +------------|---------------------|------------+
+              v                     |
+   +---------------------------------------------+
+  +---------------------------------------------+|
+ +---------------------------------------------+|+
+ |                  remote worker              |+
+ +---------------------------------------------+
+```
+To support asynchronous remote procedure call, remote interface in dcurl provides an interface named as `Remote_ImplContext` to implement it.\
+dcurl currently uses RabbitMQ C client to implement asynchronous RPC in remote interface.\
+Remote interface provides thread management to support an asynchronous RPC per thread.
+
+Here are detailed implementations of the RabbitMQ-provided RPC pattern as follows:
+* Asynchronous RPC requests are inserted into the message queue, `incoming_queue`, in RabbitMQ broker
+* Asynchronous RPCs with exclusive private queues (callback queues) with TTL = 10s property
+* Correlation ID is not used
+* An asynchronous RPC uses a connection to RabbitMQ broker
+* Remote workers can obtain requests from `incoming_queue` by default exchange of RabbitMQ broker
+
+## How to test remote interface in localhost
+You need to open three terminals
+
+**Terminal 1:**\
+Run the RabbitMQ broker\
+You can quickly use docker to run the RabbitMQ broker, rabbitmq
+```
+$ sudo docker run -d -p 5672:5672 rabbitmq
+```
+
+---
+
+**Terminal 2:**\
+Build and run the remote worker
+```
+$ cd dcurl
+$ make BUILD_REMOTE=1 BUILD_DEBUG=1
+$ ./build/remote-worker
+```
+
+---
+
+**Terminal 3:**\
+Run the tests to send requests to remote worker
+```
+$ cd dcurl
+$ make BUILD_REMOTE=1 BUILD_DEBUG=1 check
+```
+
+## Requirements
+Remote interface requires RabbitMQ broker

jni/iri-pearldiver-exlib.c

@@ -1,3 +1,11 @@
+/*
+ * Copyright (C) 2018-2019 BiiLabs Co., Ltd. and Contributors
+ * All Rights Reserved.
+ * This is free software; you can redistribute it and/or modify it under the
+ * terms of the MIT license. A copy of the license can be found in the file
+ * "LICENSE" at the root of this distribution.
+ */
+
 #include "iri-pearldiver-exlib.h"
 #include <jni.h>
 #include <stdint.h>

mk/board.mk

@@ -0,0 +1,3 @@
+ifeq ($(BOARD),de10nano)
+    CFLAGS += -mcpu=cortex-a9 -mtune=cortex-a9 -mfloat-abi=hard -mfpu=neon
+endif

mk/common.mk

@@ -26,7 +26,7 @@ NO_COLOR = \e[0m
 $(OUT)/test-%.done: $(OUT)/test-%
 	$(Q)$(PRINTF) "*** Validating $< ***\n"
 	$(Q)./$< && $(PRINTF) "\t$(PASS_COLOR)[ Verified ]$(NO_COLOR)\n"
-check: $(addsuffix .done, $(TESTS))
+check: $(SUBS) $(addsuffix .done, $(TESTS))
 
 config: $(OUT)/config-timestamp
 

mk/remote.mk

@@ -1,8 +1,8 @@
 # Build remote-worker
-$(OUT)/worker-%.o: $(SRC)/%.c $(LIBTUV_PATH)/include $(LIBRABBITMQ_PATH)/build/include
+$(OUT)/worker-%.o: $(SRC)/%.c
 	$(VECHO) "  CC\t$@\n"
-	$(Q)$(CC) -o $@ $(WORKER_CFLAGS) $(LIBTUV_INCLUDE) $(LIBRABBITMQ_INCLUDE) -c -MMD -MF $@.d $<
+	$(Q)$(CC) -o $@ $(WORKER_CFLAGS) $(SUB_INCLUDE) -c -MMD -MF $@.d $<
 
-$(OUT)/remote-worker: $(OUT)/remote_worker.o $(WORKER_OBJS) $(LIBTUV_LIBRARY) $(LIBRABBITMQ_LIBRARY)
+$(OUT)/remote-worker: $(OUT)/remote_worker.o $(WORKER_OBJS) $(SUB_OBJS)
 	$(VECHO) "  LD\t$@\n"
-	$(Q)$(CC) -o $@ $^ $(LDFLAGS) $(LIBRABBITMQ_LINK)
+	$(Q)$(CC) -o $@ $^ $(LDFLAGS)

mk/submodule.mk

@@ -18,42 +18,54 @@ LIBTUV_PATH = deps/libtuv
 LIBTUV_INCLUDE := -I $(LIBTUV_PATH)/include
 LIBTUV_PLATFORM := $(UNAME_M)-$(UNAME_S)
 LIBTUV_BOARD := $(BOARD)
-# PIC (Position-Independent-Code) library
-LIBTUV_LIBRARY := $(LIBTUV_PATH)/build/$(LIBTUV_PLATFORM)/release/lib/libtuv.o
+# PIC (Position-Independent-Code) object file
+LIBTUV_OBJS := $(LIBTUV_PATH)/build/$(LIBTUV_PLATFORM)/release/lib/libtuv.o
 
 $(LIBTUV_PATH)/include:
 	git submodule update --init $(LIBTUV_PATH)
 
-$(LIBTUV_LIBRARY):
+$(LIBTUV_OBJS):
 	$(MAKE) -C $(LIBTUV_PATH) TUV_BUILD_TYPE=release TUV_CREATE_PIC_LIB=yes TUV_PLATFORM=$(LIBTUV_PLATFORM) TUV_BOARD=$(LIBTUV_BOARD)
 
 # librabbitmq related variables
 LIBRABBITMQ_PATH = deps/rabbitmq-c
 LIBRABBITMQ_INCLUDE := -I $(LIBRABBITMQ_PATH)/build/include
-LIBRABBITMQ_LIB_PATH := $(LIBRABBITMQ_PATH)/build/librabbitmq/
+LIBRABBITMQ_OBJS := $(LIBRABBITMQ_PATH)/build/librabbitmq/CMakeFiles/rabbitmq.dir/*.o
 ifeq ($(UNAME_S),darwin)
-    # macOS
-    LIBRABBITMQ_LINK := -Wl,-rpath,$(LIBRABBITMQ_LIB_PATH) -L$(LIBRABBITMQ_LIB_PATH) -lrabbitmq
-    LIBRABBITMQ_LIBRARY := $(LIBRABBITMQ_LIB_PATH)/librabbitmq.dylib
-else
-    LIBRABBITMQ_LINK := -Wl,-rpath=$(LIBRABBITMQ_LIB_PATH) -L$(LIBRABBITMQ_LIB_PATH) -lrabbitmq
-    LIBRABBITMQ_LIBRARY := $(LIBRABBITMQ_LIB_PATH)/librabbitmq.so
+    OPENSSL_PATH := /usr/local/opt/openssl
+    LDFLAGS += -L$(OPENSSL_PATH)/lib -lcrypto -lssl
 endif
 
-$(LIBRABBITMQ_PATH)/build/include:
+$(LIBRABBITMQ_PATH)/librabbitmq:
 	git submodule update --init $(LIBRABBITMQ_PATH)
-	mkdir $(LIBRABBITMQ_PATH)/build
+
+$(LIBRABBITMQ_OBJS):
+	mkdir -p $(LIBRABBITMQ_PATH)/build
 ifeq ($(UNAME_S),darwin)
 	# macOS
 	cd $(LIBRABBITMQ_PATH)/build && \
-         cmake -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl/ -DCMAKE_INSTALL_PREFIX=. .. && \
+         cmake -DOPENSSL_ROOT_DIR=$(OPENSSL_PATH) -DCMAKE_INSTALL_PREFIX=. .. && \
          cmake --build . --target install
 else
 	cd $(LIBRABBITMQ_PATH)/build && \
          cmake -DCMAKE_INSTALL_PREFIX=. .. && \
          cmake --build . --target install
 endif
-
-$(LIBRABBITMQ_LIBRARY):
 	cd $(LIBRABBITMQ_PATH)/build && \
          cmake --build .
+
+# Submodules
+SUBS := $(LIBTUV_PATH)/include
+ifeq ($(BUILD_REMOTE),1)
+    SUBS += $(LIBRABBITMQ_PATH)/librabbitmq
+endif
+# Submodule related objects
+SUB_OBJS := $(LIBTUV_OBJS)
+ifeq ($(BUILD_REMOTE),1)
+    SUB_OBJS += $(LIBRABBITMQ_OBJS)
+endif
+# Submodule C flags for including header files
+SUB_INCLUDE := $(LIBTUV_INCLUDE)
+ifeq ($(BUILD_REMOTE),1)
+    SUB_INCLUDE += $(LIBRABBITMQ_INCLUDE)
+endif

src/clcontext.c

@@ -1,8 +1,10 @@
 /*
- * Copyright (C) 2018 dcurl Developers.
- * Copyright (C) 2017 IOTA AS, IOTA Foundation and Developers.
- * Use of this source code is governed by MIT license that can be
- * found in the LICENSE file.
+ * Copyright (C) 2018 BiiLabs Co., Ltd. and Contributors
+ * Copyright (C) 2017 IOTA AS, IOTA Foundation and Developers
+ * All Rights Reserved.
+ * This is free software; you can redistribute it and/or modify it under the
+ * terms of the MIT license. A copy of the license can be found in the file
+ * "LICENSE" at the root of this distribution.
  */
 
 #include "clcontext.h"