Integrate FPGA-accelerated PoW

Makefile

@@ -39,6 +39,10 @@ ifeq ("$(BUILD_GPU)","1")
 include mk/opencl.mk
 endif
 
+ifeq ("$(BUILD_FPGA_ACCEL)","1")
+CFLAGS += -DENABLE_FPGA_ACCEL
+endif
+
 ifeq ("$(BUILD_JNI)","1")
 include mk/java.mk
 endif
@@ -71,6 +75,10 @@ ifeq ("$(BUILD_COMPAT)", "1")
 TESTS += ccurl-multi_pow
 endif
 
+ifeq ("$(BUILD_FPGA_ACCEL)","1")
+TESTS += pow_fpga_accel
+endif
+
 TESTS := $(addprefix $(OUT)/test-, $(TESTS))
 
 LIBS = libdcurl.so
@@ -112,6 +120,11 @@ OBJS += \
 	compat-ccurl.o
 endif
 
+ifeq ("$(BUILD_FPGA_ACCEL)","1")
+OBJS += \
+	pow_fpga_accel.o
+endif
+
 OBJS := $(addprefix $(OUT)/, $(OBJS))
 
 $(OUT)/test-%.o: tests/test-%.c

README.md

@@ -3,18 +3,19 @@
 [![Build Status](https://travis-ci.org/DLTcollab/dcurl.svg?branch=dev)](https://travis-ci.org/DLTcollab/dcurl)
 ![Supported IRI version](https://img.shields.io/badge/Supported%20IRI%20Version-1.5.3-brightgreen.svg)
 
-Hardware-accelerated implementation for IOTA PearlDiver, which utilizes multi-threaded SIMD and GPU.
+Hardware-accelerated implementation for IOTA PearlDiver, which utilizes multi-threaded SIMD, FPGA and GPU.
 
 # Introduction
 dcurl exploits SIMD instructions on CPU and OpenCL on GPU. Both CPU and GPU accelerations can be
 enabled in multi-threaded execuction fashion, resulting in much faster proof-of-work (PoW) for IOTA
-Reference Implementation (IRI).
+Reference Implementation (IRI). Additionally, dcurl also supports the FPGA-accelerated solution further described in docs/FPGA-ACCEL.md
 
 # Warning
 * You need to configure paths and flags of OpenCL installation in ```mk/opencl.mk```
 * dcurl will automatically configure all the GPU divices on your platform.
 * Check JDK installation and set JAVA_HOME if you wish to specify.
 * If your platform doesn't support Intel SSE, dcurl would be compiled with naive implementation.
+* For the IOTA hardware accelerator, we integrate [Lampa Lab's Cyclone V FPGA PoW](https://github.com/LampaLab/iota_fpga) into dcurl. Lampa Lab provides soc_system.rbf only for DE10-nano board. You need to synthesize to get soc_system.rbf for using Arrow SoCKit board and [this RBF file](https://github.com/ajblane/dcurl/releases/tag/v1.0-SoCKit) can be downloaded from our release. Moreover, you need to download [Lampa Lab-provided Linux image](https://github.com/LampaLab/iota_fpga/releases/tag/v0.1) to flash into the micro-SD card and root password is 123456. Finally, you also need to download dcurl into root directory.
 
 # Build Instructions
 * dcurl allows various combinations of build configurations to fit final use scenarios.
@@ -24,6 +25,7 @@ Reference Implementation (IRI).
     - ``BUILD_JNI``: build a shared library for IRI. The build system would generate JNI header file
                      downloading from [latest JAVA source](https://github.com/chenwei-tw/iri/tree/feat/new_pow_interface).
     - ``BUILD_COMPAT``: build extra cCurl compatible interface.
+    - ``BUILD_FPGA_ACCEL``: build the interface interacting with the Cyclone V FPGA based accelerator. Verified on DE10-nano board and Arrow SoCKit board.
 * Alternatively, you can specify conditional build as following:
 ```shell
 $ make BUILD_GPU=0 BUILD_JNI=1 BUILD_AVX=1
@@ -68,6 +70,27 @@ $ make BUILD_AVX=1 check
         [ Verified ]
 ```
 
+* Test with Arrow SoCKit board
+```shell
+root@lampa:~# sh init_curl_pow.sh 
+root@lampa:~# cd dcurl
+root@lampa:~/dcurl# make BUILD_FPGA_ACCEL=1 check
+```
+
+* Expected Results
+```
+*** Validating build/test-trinary ***
+        [ Verified ]
+*** Validating build/test-curl ***
+        [ Verified ]
+*** Validating build/test-pow_c ***
+        [ Verified ]
+*** Validating build/test-multi_pow_cpu ***
+        [ Verified ]
+*** Validating build/test-pow_fpga_accel *** 
+        [ Verified ] 
+```
+
 # Tweaks
 * Number of threads to find nonce in CPU
     * ```$ export DCURL_NUM_CPU=26```

docs/FPGA-ACCEL.md

@@ -0,0 +1,22 @@
+
+IOTA FPGA-accelerated solution for Dcurl
+----------------------------------------
+
+Dcurl supports IOTA FPGA-accelerated solutions to improve PoW performance. PoW calculation time for MWM=14 is between 0.001 and 0.8 second and 0.14 second in average and The time for MWM=15 is between 0.01 and 2 second and 0.42 second in average. Currently, it is experimented and verfied on Arrow Sockit board and Intel FPGA DE10-Nano board. We reuse the Lampa Lab-provied FPGA-accelerated solution. 
+
+Here is a brief summary of the tasks Lampa Lab have done:
+* Use Verilog to implement Curl & POW accelerators
+* Use System Verilog and UVM to verify the accelerators
+* Synthesize Curl & POW hardware accelerators for Intel FPGA DE10-Nano board and flash it into the board
+* Write Linux drivers in Gloden System Reference Design for Curl & POW hardware accelerators and verify them
+You want to known it much more and further look at [LampaLab/iota_fpga](https://github.com/LampaLab/iota_fpga)
+
+Here is a brief summary of the tasks we have done:
+* Resynthesize the POW hardware accelerator for Arrow Sockit board and flash it into the board
+* Integrate the IOTA PoW hardware accelerator into dcurl's implementation interface
+* Test and verify it
+
+
+
+
+

mk/defs.mk

@@ -12,3 +12,6 @@ BUILD_JNI ?= 0
 
 # Build cCurl compatible interface
 BUILD_COMPAT ?= 0
+
+# Build FPGA backend or not
+BUILD_FPGA_ACCEL ?= 0

src/dcurl.c

@@ -12,6 +12,9 @@
 #if defined(ENABLE_OPENCL)
 #include "pow_cl.h"
 #endif
+#if defined(ENABLE_FPGA_ACCEL)
+#include "pow_fpga_accel.h"
+#endif
 #include "trinary.h"
 #include "implcontext.h"
 #if defined(ENABLE_AVX)
@@ -51,6 +54,10 @@ extern ImplContext PoWC_Context;
 extern ImplContext PoWCL_Context;
 #endif
 
+#if defined(ENABLE_FPGA_ACCEL)
+extern ImplContext PoWFPGAAccel_Context;
+#endif
+
 bool dcurl_init()
 {
     bool ret = true;
@@ -67,6 +74,10 @@ bool dcurl_init()
     ret &= registerImplContext(&PoWCL_Context);
 #endif
 
+#if defined(ENABLE_FPGA_ACCEL)
+    ret &= registerImplContext(&PoWFPGAAccel_Context);
+#endif
+
 #ifdef __APPLE__
     notify = dispatch_semaphore_create(0);
 #else

src/pow_fpga_accel.c

@@ -0,0 +1,232 @@
+/*
+ * Copyright (C) 2018 dcurl Developers.
+ * Copyright (c) 2018 Ievgen Korokyi.
+ * Use of this source code is governed by MIT license that can be
+ * found in the LICENSE file.
+ */
+
+#include "pow_fpga_accel.h"
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include "implcontext.h"
+#include "trinary.h"
+
+#define HPS_TO_FPGA_BASE 0xC0000000
+#define HPS_TO_FPGA_SPAN 0x0020000
+#define HASH_CNT_REG_OFFSET 4
+#define TICK_CNT_LOW_REG_OFFSET 5
+#define TICK_CNT_HI_REG_OFFSET 6
+#define MWM_MASK_REG_OFFSET 3
+#define CPOW_BASE 0
+
+/* Set FPGA configuration for device files */
+#define DEV_CTRL_FPGA "/dev/cpow-ctrl"
+#define DEV_IDATA_FPGA "/dev/cpow-idata"
+#define DEV_ODATA_FPGA "/dev/cpow-odata"
+
+#define INT2STRING(I, S)         \
+    {                            \
+        S[0] = I & 0xff;         \
+        S[1] = (I >> 8) & 0xff;  \
+        S[2] = (I >> 16) & 0xff; \
+        S[3] = (I >> 24) & 0xff; \
+    }
+
+static int devmem_fd;
+static void *fpga_regs_map;
+static uint32_t *cpow_map;
+
+static bool PoWFPGAAccel(void *pow_ctx)
+{
+    PoW_FPGA_Accel_Context *ctx = (PoW_FPGA_Accel_Context *) pow_ctx;
+
+    int8_t fpga_out_nonce_trits[NonceTrinarySize];
+
+    char result[4];
+    char buf[4];
+
+    Trytes_t *object_trytes =
+        initTrytes(ctx->input_trytes, (transactionTrinarySize) / 3);
+    if (!object_trytes)
+        return false;
+
+    Trits_t *object_trits = trits_from_trytes(object_trytes);
+    if (!object_trits)
+        return false;
+
+    if (write(ctx->in_fd, (char *) object_trits->data, transactionTrinarySize) <
+        0)
+        return false;
+
+    INT2STRING(ctx->mwm, buf);
+    if (write(ctx->ctrl_fd, buf, sizeof(buf)) < 0)
+        return false;
+    if (read(ctx->ctrl_fd, result, sizeof(result)) < 0)
+        return false;
+
+    if (read(ctx->out_fd, (char *) fpga_out_nonce_trits, NonceTrinarySize) < 0)
+        return false;
+
+    Trits_t *object_nonce_trits =
+        initTrits(fpga_out_nonce_trits, NonceTrinarySize);
+    if (!object_nonce_trits)
+        return false;
+
+    Trytes_t *nonce_trytes = trytes_from_trits(object_nonce_trits);
+    if (!nonce_trytes)
+        return false;
+
+    memcpy(ctx->output_trytes, ctx->input_trytes, (NonceTrinaryOffset) / 3);
+    memcpy(ctx->output_trytes + ((NonceTrinaryOffset) / 3), nonce_trytes->data,
+           ((transactionTrinarySize) - (NonceTrinaryOffset)) / 3);
+
+    freeTrobject(object_trytes);
+    freeTrobject(object_trits);
+    freeTrobject(object_nonce_trits);
+    freeTrobject(nonce_trytes);
+
+    return true;
+}
+
+static bool PoWFPGAAccel_Context_Initialize(ImplContext *impl_ctx)
+{
+    int i = 0;
+    devmem_fd = 0;
+    fpga_regs_map = 0;
+    cpow_map = 0;
+
+    PoW_FPGA_Accel_Context *ctx = (PoW_FPGA_Accel_Context *) malloc(
+        sizeof(PoW_FPGA_Accel_Context) * impl_ctx->num_max_thread);
+    if (!ctx)
+        goto fail_to_malloc;
+
+    for (i = 0; i < impl_ctx->num_max_thread; i++) {
+        ctx[i].ctrl_fd = open(DEV_CTRL_FPGA, O_RDWR);
+        if (ctx[i].ctrl_fd < 0) {
+            perror("cpow-ctrl open fail");
+            goto fail_to_open_ctrl;
+        }
+        ctx[i].in_fd = open(DEV_IDATA_FPGA, O_RDWR);
+        if (ctx[i].in_fd < 0) {
+            perror("cpow-idata open fail");
+            goto fail_to_open_idata;
+        }
+        ctx[i].out_fd = open(DEV_ODATA_FPGA, O_RDWR);
+        if (ctx[i].out_fd < 0) {
+            perror("cpow-odata open fail");
+            goto fail_to_open_odata;
+        }
+        impl_ctx->bitmap = impl_ctx->bitmap << 1 | 0x1;
+    }
+    impl_ctx->context = ctx;
+    pthread_mutex_init(&impl_ctx->lock, NULL);
+
+    devmem_fd = open("/dev/mem", O_RDWR | O_SYNC);
+    if (devmem_fd < 0) {
+        perror("devmem open");
+        goto fail_to_open_memopen;
+    }
+
+    fpga_regs_map =
+        (uint32_t *) mmap(NULL, HPS_TO_FPGA_SPAN, PROT_READ | PROT_WRITE,
+                          MAP_SHARED, devmem_fd, HPS_TO_FPGA_BASE);
+    if (fpga_regs_map == MAP_FAILED) {
+        perror("devmem mmap");
+        goto fail_to_open_memmap;
+    }
+
+    cpow_map = (uint32_t *) (fpga_regs_map + CPOW_BASE);
+
+    return true;
+
+fail_to_open_memmap:
+    close(devmem_fd);
+fail_to_open_memopen:
+    close(ctx[i].out_fd);
+fail_to_open_odata:
+    close(ctx[i].in_fd);
+fail_to_open_idata:
+    close(ctx[i].ctrl_fd);
+fail_to_open_ctrl:
+fail_to_malloc:
+    for (int j = i - 1; j > 0; j--) {
+        close(ctx[j].in_fd);
+        close(ctx[j].out_fd);
+        close(ctx[j].ctrl_fd);
+    }
+    return false;
+}
+
+static void PoWFPGAAccel_Context_Destroy(ImplContext *impl_ctx)
+{
+    PoW_FPGA_Accel_Context *ctx = (PoW_FPGA_Accel_Context *) impl_ctx->context;
+    for (int i = 0; i < impl_ctx->num_max_thread; i++) {
+        close(ctx[i].in_fd);
+        close(ctx[i].out_fd);
+        close(ctx[i].ctrl_fd);
+    }
+    free(ctx);
+
+    int result = munmap(fpga_regs_map, HPS_TO_FPGA_SPAN);
+    if (result < 0) {
+        perror("devmem munmap");
+    }
+
+    close(devmem_fd);
+}
+
+static void *PoWFPGAAccel_getPoWContext(ImplContext *impl_ctx,
+                                        int8_t *trytes,
+                                        int mwm)
+{
+    pthread_mutex_lock(&impl_ctx->lock);
+    for (int i = 0; i < impl_ctx->num_max_thread; i++) {
+        if (impl_ctx->bitmap & (0x1 << i)) {
+            impl_ctx->bitmap &= ~(0x1 << i);
+            pthread_mutex_unlock(&impl_ctx->lock);
+            PoW_FPGA_Accel_Context *ctx =
+                impl_ctx->context + sizeof(PoW_FPGA_Accel_Context) * i;
+            memcpy(ctx->input_trytes, trytes, (transactionTrinarySize) / 3);
+            ctx->mwm = mwm;
+            ctx->indexOfContext = i;
+            return ctx;
+        }
+    }
+
+    pthread_mutex_unlock(&impl_ctx->lock);
+    return NULL; /* It should not happen */
+}
+
+static bool PoWFPGAAccel_freePoWContext(ImplContext *impl_ctx, void *pow_ctx)
+{
+    pthread_mutex_lock(&impl_ctx->lock);
+    impl_ctx->bitmap |= 0x1
+                        << ((PoW_FPGA_Accel_Context *) pow_ctx)->indexOfContext;
+    pthread_mutex_unlock(&impl_ctx->lock);
+    return true;
+}
+
+static int8_t *PoWFPGAAccel_getPoWResult(void *pow_ctx)
+{
+    int8_t *ret =
+        (int8_t *) malloc(sizeof(int8_t) * ((transactionTrinarySize) / 3));
+    if (!ret)
+        return NULL;
+    memcpy(ret, ((PoW_FPGA_Accel_Context *) pow_ctx)->output_trytes,
+           (transactionTrinarySize) / 3);
+    return ret;
+}
+
+ImplContext PoWFPGAAccel_Context = {
+    .context = NULL,
+    .bitmap = 0,
+    .num_max_thread = 1,  // num_max_thread >= 1
+    .num_working_thread = 0,
+    .initialize = PoWFPGAAccel_Context_Initialize,
+    .destroy = PoWFPGAAccel_Context_Destroy,
+    .getPoWContext = PoWFPGAAccel_getPoWContext,
+    .freePoWContext = PoWFPGAAccel_freePoWContext,
+    .doThePoW = PoWFPGAAccel,
+    .getPoWResult = PoWFPGAAccel_getPoWResult,
+};