From ac87850887f064752c2ad815367484c07eaf5449 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Wed, 26 Aug 2015 19:03:59 -0700 Subject: [PATCH 01/39] No need to squeeze the output of the network --- python/caffe/detector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/caffe/detector.py b/python/caffe/detector.py index 75cd3b1202f..ef1f91730bf 100644 --- a/python/caffe/detector.py +++ b/python/caffe/detector.py @@ -83,7 +83,7 @@ def detect_windows(self, images_windows): for ix, window_in in enumerate(window_inputs): caffe_in[ix] = self.transformer.preprocess(in_, window_in) out = self.forward_all(**{in_: caffe_in}) - predictions = out[self.outputs[0]].squeeze(axis=(2, 3)) + predictions = out[self.outputs[0]] # Package predictions with images and windows. detections = [] From 4e690b22ae30b0d483ccbe971007f2c6732cceb0 Mon Sep 17 00:00:00 2001 From: crazytan Date: Thu, 28 Apr 2016 18:45:13 -0400 Subject: [PATCH 02/39] fix problems in net_surgery.ipynb --- examples/net_surgery.ipynb | 45 +++++++++++++++----------------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/examples/net_surgery.ipynb b/examples/net_surgery.ipynb index a6092db0c40..d50d503bfe0 100644 --- a/examples/net_surgery.ipynb +++ b/examples/net_surgery.ipynb @@ -22,7 +22,6 @@ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", - "import Image\n", "\n", "# Make sure that caffe is on the python path:\n", "caffe_root = '../' # this file is expected to be in {caffe_root}/examples\n", @@ -3511,7 +3510,7 @@ "print(\"blobs {}\\nparams {}\".format(net.blobs.keys(), net.params.keys()))\n", "\n", "# load image and prepare as a single input batch for Caffe\n", - "im = np.array(Image.open('images/cat_gray.jpg'))\n", + "im = np.array(caffe.io.load_image('images/cat_gray.jpg', color=False)).squeeze()\n", "plt.title(\"original image\")\n", "plt.imshow(im)\n", "plt.axis('off')\n", @@ -4480,8 +4479,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "pre-surgery output mean -12.93\n", - "post-surgery output mean -11.93\n" + "pre-surgery output mean -0.02\n", + "post-surgery output mean 0.98\n" ] } ], @@ -4489,7 +4488,7 @@ "# pick first filter output\n", "conv0 = net.blobs['conv'].data[0, 0]\n", "print(\"pre-surgery output mean {:.2f}\".format(conv0.mean()))\n", - "# set first filter bias to 10\n", + "# set first filter bias to 1\n", "net.params['conv'][1].data[0] = 1.\n", "net.forward()\n", "print(\"post-surgery output mean {:.2f}\".format(conv0.mean()))" @@ -5494,13 +5493,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "1,2c1,2\r\n", + "1,2c1\r\n", "< # Fully convolutional network version of CaffeNet.\r\n", "< name: \"CaffeNetConv\"\r\n", "---\r\n", "> name: \"CaffeNet\"\r\n", - "> input: \"data\"\r\n", - "7,11c7\r\n", + "7,11c6\r\n", "< input_param {\r\n", "< # initial shape for a fully convolutional network:\r\n", "< # the shape can be set for each input by reshape.\r\n", @@ -5508,33 +5506,33 @@ "< }\r\n", "---\r\n", "> input_param { shape: { dim: 10 dim: 3 dim: 227 dim: 227 } }\r\n", - "157,158c153,154\r\n", + "157,158c152,153\r\n", "< name: \"fc6-conv\"\r\n", "< type: \"Convolution\"\r\n", "---\r\n", "> name: \"fc6\"\r\n", "> type: \"InnerProduct\"\r\n", - "160,161c156,157\r\n", + "160,161c155,156\r\n", "< top: \"fc6-conv\"\r\n", "< convolution_param {\r\n", "---\r\n", "> top: \"fc6\"\r\n", "> inner_product_param {\r\n", - "163d158\r\n", + "163d157\r\n", "< kernel_size: 6\r\n", - "169,170c164,165\r\n", + "169,170c163,164\r\n", "< bottom: \"fc6-conv\"\r\n", "< top: \"fc6-conv\"\r\n", "---\r\n", "> bottom: \"fc6\"\r\n", "> top: \"fc6\"\r\n", - "175,176c170,171\r\n", + "175,176c169,170\r\n", "< bottom: \"fc6-conv\"\r\n", "< top: \"fc6-conv\"\r\n", "---\r\n", "> bottom: \"fc6\"\r\n", "> top: \"fc6\"\r\n", - "182,186c177,181\r\n", + "182,186c176,180\r\n", "< name: \"fc7-conv\"\r\n", "< type: \"Convolution\"\r\n", "< bottom: \"fc6-conv\"\r\n", @@ -5546,21 +5544,21 @@ "> bottom: \"fc6\"\r\n", "> top: \"fc7\"\r\n", "> inner_product_param {\r\n", - "188d182\r\n", + "188d181\r\n", "< kernel_size: 1\r\n", - "194,195c188,189\r\n", + "194,195c187,188\r\n", "< bottom: \"fc7-conv\"\r\n", "< top: \"fc7-conv\"\r\n", "---\r\n", "> bottom: \"fc7\"\r\n", "> top: \"fc7\"\r\n", - "200,201c194,195\r\n", + "200,201c193,194\r\n", "< bottom: \"fc7-conv\"\r\n", "< top: \"fc7-conv\"\r\n", "---\r\n", "> bottom: \"fc7\"\r\n", "> top: \"fc7\"\r\n", - "207,211c201,205\r\n", + "207,211c200,204\r\n", "< name: \"fc8-conv\"\r\n", "< type: \"Convolution\"\r\n", "< bottom: \"fc7-conv\"\r\n", @@ -5572,9 +5570,9 @@ "> bottom: \"fc7\"\r\n", "> top: \"fc8\"\r\n", "> inner_product_param {\r\n", - "213d206\r\n", + "213d205\r\n", "< kernel_size: 1\r\n", - "219c212\r\n", + "219c211\r\n", "< bottom: \"fc8-conv\"\r\n", "---\r\n", "> bottom: \"fc8\"\r\n" @@ -5610,13 +5608,6 @@ } ], "source": [ - "# Make sure that caffe is on the python path:\n", - "caffe_root = '../' # this file is expected to be in {caffe_root}/examples\n", - "import sys\n", - "sys.path.insert(0, caffe_root + 'python')\n", - "\n", - "import caffe\n", - "\n", "# Load the original network and extract the fully connected layers' parameters.\n", "net = caffe.Net('../models/bvlc_reference_caffenet/deploy.prototxt', \n", " '../models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel', \n", From 42642936c2c29e539022e33bc0c691564d7e522d Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Mon, 9 May 2016 11:21:26 -0700 Subject: [PATCH 03/39] Catch MDB_MAP_FULL errors from mdb_txn_commit --- src/caffe/util/db_lmdb.cpp | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/caffe/util/db_lmdb.cpp b/src/caffe/util/db_lmdb.cpp index df83a52a633..4567cd7b93a 100644 --- a/src/caffe/util/db_lmdb.cpp +++ b/src/caffe/util/db_lmdb.cpp @@ -62,36 +62,42 @@ void LMDBTransaction::Commit() { MDB_CHECK(mdb_txn_begin(mdb_env_, NULL, 0, &mdb_txn)); MDB_CHECK(mdb_dbi_open(mdb_txn, NULL, 0, &mdb_dbi)); - bool out_of_memory = false; for (int i = 0; i < keys.size(); i++) { mdb_key.mv_size = keys[i].size(); mdb_key.mv_data = const_cast(keys[i].data()); mdb_data.mv_size = values[i].size(); mdb_data.mv_data = const_cast(values[i].data()); + // Add data to the transaction int put_rc = mdb_put(mdb_txn, mdb_dbi, &mdb_key, &mdb_data, 0); if (put_rc == MDB_MAP_FULL) { - out_of_memory = true; - break; - } else { - // Failed for some other reason - MDB_CHECK(put_rc); + // Out of memory - double the map size and retry + mdb_txn_abort(mdb_txn); + mdb_dbi_close(mdb_env_, mdb_dbi); + DoubleMapSize(); + Commit(); + return; } + // May have failed for some other reason + MDB_CHECK(put_rc); } - if (!out_of_memory) { - // Commit the transaction - MDB_CHECK(mdb_txn_commit(mdb_txn)); - mdb_dbi_close(mdb_env_, mdb_dbi); - keys.clear(); - values.clear(); - } else { - // Double the map size and retry - mdb_txn_abort(mdb_txn); + // Commit the transaction + int commit_rc = mdb_txn_commit(mdb_txn); + if (commit_rc == MDB_MAP_FULL) { + // Out of memory - double the map size and retry mdb_dbi_close(mdb_env_, mdb_dbi); DoubleMapSize(); Commit(); + return; } + // May have failed for some other reason + MDB_CHECK(commit_rc); + + // Cleanup after successful commit + mdb_dbi_close(mdb_env_, mdb_dbi); + keys.clear(); + values.clear(); } void LMDBTransaction::DoubleMapSize() { From a934ca54f3633479ea0573346c510df4f757df6c Mon Sep 17 00:00:00 2001 From: ray glover Date: Tue, 10 May 2016 15:44:47 +0100 Subject: [PATCH 04/39] [build] (CMake) customisable Caffe version/soversion --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c5d99cef9dd..da7142c9b3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,8 +10,8 @@ endif() project(Caffe C CXX) # ---[ Caffe version -set(CAFFE_TARGET_VERSION "1.0.0-rc3") -set(CAFFE_TARGET_SOVERSION "1.0.0-rc3") +set(CAFFE_TARGET_VERSION "1.0.0-rc3" CACHE STRING "Caffe logical version") +set(CAFFE_TARGET_SOVERSION "1.0.0-rc3" CACHE STRING "Caffe soname version") add_definitions(-DCAFFE_VERSION=${CAFFE_TARGET_VERSION}) # ---[ Using cmake scripts and modules From bb6ca4720ea41b8e9bdf162f63eb2757571a2e17 Mon Sep 17 00:00:00 2001 From: gdh1995 Date: Wed, 11 May 2016 20:51:07 +0800 Subject: [PATCH 05/39] a comment misses a space char --- src/caffe/util/db_lmdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/caffe/util/db_lmdb.cpp b/src/caffe/util/db_lmdb.cpp index 4567cd7b93a..fb1d4956aa1 100644 --- a/src/caffe/util/db_lmdb.cpp +++ b/src/caffe/util/db_lmdb.cpp @@ -10,7 +10,7 @@ namespace caffe { namespace db { void LMDB::Open(const string& source, Mode mode) { MDB_CHECK(mdb_env_create(&mdb_env_)); if (mode == NEW) { - CHECK_EQ(mkdir(source.c_str(), 0744), 0) << "mkdir " << source << "failed"; + CHECK_EQ(mkdir(source.c_str(), 0744), 0) << "mkdir " << source << " failed"; } int flags = 0; if (mode == READ) { From 078d9981a2c64b19834decdef3ce3dd032b667c0 Mon Sep 17 00:00:00 2001 From: Kyle Mills Date: Fri, 13 May 2016 11:15:33 -0400 Subject: [PATCH 06/39] fixed typo in io.py --- python/caffe/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/caffe/io.py b/python/caffe/io.py index cee5ace2e88..e1759beb587 100644 --- a/python/caffe/io.py +++ b/python/caffe/io.py @@ -46,7 +46,7 @@ def array_to_blobproto(arr, diff=None): return blob -def arraylist_to_blobprotovecor_str(arraylist): +def arraylist_to_blobprotovector_str(arraylist): """Converts a list of arrays to a serialized blobprotovec, which could be then passed to a network for processing. """ From 87c9dc397081248dd3d40e0dabce191557bcfc15 Mon Sep 17 00:00:00 2001 From: Yale Song Date: Fri, 13 May 2016 16:06:59 -0400 Subject: [PATCH 07/39] Fix Makefile CUDA_VERSION extraction on OSX Yosemite --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 568d9c2774d..403e00a38a1 100644 --- a/Makefile +++ b/Makefile @@ -272,7 +272,7 @@ endif ifeq ($(OSX), 1) CXX := /usr/bin/clang++ ifneq ($(CPU_ONLY), 1) - CUDA_VERSION := $(shell $(CUDA_DIR)/bin/nvcc -V | grep -o 'release [0-9.]*' | grep -o '[0-9.]*') + CUDA_VERSION := $(shell $(CUDA_DIR)/bin/nvcc -V | grep -o 'release [0-9.]*' | tr -d '[a-z ]') ifeq ($(shell echo | awk '{exit $(CUDA_VERSION) < 7.0;}'), 1) CXXFLAGS += -stdlib=libstdc++ LINKFLAGS += -stdlib=libstdc++ From e8ec9f806bd0051f2ee8d1d2737afdafe314f9e4 Mon Sep 17 00:00:00 2001 From: Bob Poekert Date: Fri, 13 May 2016 22:06:33 -0700 Subject: [PATCH 08/39] add check for background and foreground window size > 0 in WindowData layer --- src/caffe/layers/window_data_layer.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/caffe/layers/window_data_layer.cpp b/src/caffe/layers/window_data_layer.cpp index 4ca8315d791..103dd4b6af8 100644 --- a/src/caffe/layers/window_data_layer.cpp +++ b/src/caffe/layers/window_data_layer.cpp @@ -265,6 +265,9 @@ void WindowDataLayer::load_batch(Batch* batch) { const int num_samples[2] = { batch_size - num_fg, num_fg }; int item_id = 0; + CHECK_GT(fg_windows_.size(), 0); + CHECK_GT(bg_windows_.size(), 0); + // sample from bg set then fg set for (int is_fg = 0; is_fg < 2; ++is_fg) { for (int dummy = 0; dummy < num_samples[is_fg]; ++dummy) { From b43c8e43a95608a00033f8f8867d32a201e5eed4 Mon Sep 17 00:00:00 2001 From: Felix Abecassis Date: Mon, 16 May 2016 14:03:38 -0700 Subject: [PATCH 09/39] Add cuDNN v5 support, drop cuDNN v3 support cuDNN v4 is still supported. --- include/caffe/layers/cudnn_relu_layer.hpp | 1 + include/caffe/layers/cudnn_sigmoid_layer.hpp | 1 + include/caffe/layers/cudnn_tanh_layer.hpp | 1 + include/caffe/util/cudnn.hpp | 24 +++++++++++++++++--- src/caffe/layers/cudnn_conv_layer.cu | 12 ++-------- src/caffe/layers/cudnn_relu_layer.cpp | 1 + src/caffe/layers/cudnn_relu_layer.cu | 23 +++++++++++++++++-- src/caffe/layers/cudnn_sigmoid_layer.cpp | 2 ++ src/caffe/layers/cudnn_sigmoid_layer.cu | 23 +++++++++++++++++-- src/caffe/layers/cudnn_tanh_layer.cpp | 1 + src/caffe/layers/cudnn_tanh_layer.cu | 23 +++++++++++++++++-- 11 files changed, 93 insertions(+), 19 deletions(-) diff --git a/include/caffe/layers/cudnn_relu_layer.hpp b/include/caffe/layers/cudnn_relu_layer.hpp index e01f568abc9..a1cb29e7c5f 100644 --- a/include/caffe/layers/cudnn_relu_layer.hpp +++ b/include/caffe/layers/cudnn_relu_layer.hpp @@ -37,6 +37,7 @@ class CuDNNReLULayer : public ReLULayer { cudnnHandle_t handle_; cudnnTensorDescriptor_t bottom_desc_; cudnnTensorDescriptor_t top_desc_; + cudnnActivationDescriptor_t activ_desc_; }; #endif diff --git a/include/caffe/layers/cudnn_sigmoid_layer.hpp b/include/caffe/layers/cudnn_sigmoid_layer.hpp index 9c597958b0b..7b3486f8a7e 100644 --- a/include/caffe/layers/cudnn_sigmoid_layer.hpp +++ b/include/caffe/layers/cudnn_sigmoid_layer.hpp @@ -37,6 +37,7 @@ class CuDNNSigmoidLayer : public SigmoidLayer { cudnnHandle_t handle_; cudnnTensorDescriptor_t bottom_desc_; cudnnTensorDescriptor_t top_desc_; + cudnnActivationDescriptor_t activ_desc_; }; #endif diff --git a/include/caffe/layers/cudnn_tanh_layer.hpp b/include/caffe/layers/cudnn_tanh_layer.hpp index c0f0053f71e..59e758d7031 100644 --- a/include/caffe/layers/cudnn_tanh_layer.hpp +++ b/include/caffe/layers/cudnn_tanh_layer.hpp @@ -37,6 +37,7 @@ class CuDNNTanHLayer : public TanHLayer { cudnnHandle_t handle_; cudnnTensorDescriptor_t bottom_desc_; cudnnTensorDescriptor_t top_desc_; + cudnnActivationDescriptor_t activ_desc_; }; #endif diff --git a/include/caffe/util/cudnn.hpp b/include/caffe/util/cudnn.hpp index 8a7e17c6cd4..a7d8dbbad4c 100644 --- a/include/caffe/util/cudnn.hpp +++ b/include/caffe/util/cudnn.hpp @@ -91,8 +91,13 @@ template inline void createFilterDesc(cudnnFilterDescriptor_t* desc, int n, int c, int h, int w) { CUDNN_CHECK(cudnnCreateFilterDescriptor(desc)); +#if CUDNN_VERSION_MIN(5, 0, 0) CUDNN_CHECK(cudnnSetFilter4dDescriptor(*desc, dataType::type, - n, c, h, w)); + CUDNN_TENSOR_NCHW, n, c, h, w)); +#else + CUDNN_CHECK(cudnnSetFilter4dDescriptor_v4(*desc, dataType::type, + CUDNN_TENSOR_NCHW, n, c, h, w)); +#endif } template @@ -123,8 +128,21 @@ inline void createPoolingDesc(cudnnPoolingDescriptor_t* pool_desc, LOG(FATAL) << "Unknown pooling method."; } CUDNN_CHECK(cudnnCreatePoolingDescriptor(pool_desc)); - CUDNN_CHECK(cudnnSetPooling2dDescriptor(*pool_desc, *mode, h, w, - pad_h, pad_w, stride_h, stride_w)); +#if CUDNN_VERSION_MIN(5, 0, 0) + CUDNN_CHECK(cudnnSetPooling2dDescriptor(*pool_desc, *mode, + CUDNN_PROPAGATE_NAN, h, w, pad_h, pad_w, stride_h, stride_w)); +#else + CUDNN_CHECK(cudnnSetPooling2dDescriptor_v4(*pool_desc, *mode, + CUDNN_PROPAGATE_NAN, h, w, pad_h, pad_w, stride_h, stride_w)); +#endif +} + +template +inline void createActivationDescriptor(cudnnActivationDescriptor_t* activ_desc, + cudnnActivationMode_t mode) { + CUDNN_CHECK(cudnnCreateActivationDescriptor(activ_desc)); + CUDNN_CHECK(cudnnSetActivationDescriptor(*activ_desc, mode, + CUDNN_PROPAGATE_NAN, Dtype(0))); } } // namespace cudnn diff --git a/src/caffe/layers/cudnn_conv_layer.cu b/src/caffe/layers/cudnn_conv_layer.cu index 42c4fd0260c..8bc5346248c 100644 --- a/src/caffe/layers/cudnn_conv_layer.cu +++ b/src/caffe/layers/cudnn_conv_layer.cu @@ -30,19 +30,11 @@ void CuDNNConvolutionLayer::Forward_gpu( // Bias. if (this->bias_term_) { const Dtype* bias_data = this->blobs_[1]->gpu_data(); -#if CUDNN_VERSION_MIN(4, 0, 0) CUDNN_CHECK(cudnnAddTensor(handle_[g], cudnn::dataType::one, bias_desc_, bias_data + bias_offset_ * g, cudnn::dataType::one, top_descs_[i], top_data + top_offset_ * g)); -#else - CUDNN_CHECK(cudnnAddTensor(handle_[g], CUDNN_ADD_SAME_C, - cudnn::dataType::one, - bias_desc_, bias_data + bias_offset_ * g, - cudnn::dataType::one, - top_descs_[i], top_data + top_offset_ * g)); -#endif } } @@ -82,7 +74,7 @@ void CuDNNConvolutionLayer::Backward_gpu(const vector*>& top, // Gradient w.r.t. weights. if (this->param_propagate_down_[0]) { const Dtype* bottom_data = bottom[i]->gpu_data(); - CUDNN_CHECK(cudnnConvolutionBackwardFilter_v3( + CUDNN_CHECK(cudnnConvolutionBackwardFilter( handle_[1*this->group_ + g], cudnn::dataType::one, bottom_descs_[i], bottom_data + bottom_offset_ * g, @@ -100,7 +92,7 @@ void CuDNNConvolutionLayer::Backward_gpu(const vector*>& top, weight = this->blobs_[0]->gpu_data(); } Dtype* bottom_diff = bottom[i]->mutable_gpu_diff(); - CUDNN_CHECK(cudnnConvolutionBackwardData_v3( + CUDNN_CHECK(cudnnConvolutionBackwardData( handle_[2*this->group_ + g], cudnn::dataType::one, filter_desc_, weight + this->weight_offset_ * g, diff --git a/src/caffe/layers/cudnn_relu_layer.cpp b/src/caffe/layers/cudnn_relu_layer.cpp index c86c6907113..795e0a9efb0 100644 --- a/src/caffe/layers/cudnn_relu_layer.cpp +++ b/src/caffe/layers/cudnn_relu_layer.cpp @@ -13,6 +13,7 @@ void CuDNNReLULayer::LayerSetUp(const vector*>& bottom, CUDNN_CHECK(cudnnCreate(&handle_)); cudnn::createTensor4dDesc(&bottom_desc_); cudnn::createTensor4dDesc(&top_desc_); + cudnn::createActivationDescriptor(&activ_desc_, CUDNN_ACTIVATION_RELU); handles_setup_ = true; } diff --git a/src/caffe/layers/cudnn_relu_layer.cu b/src/caffe/layers/cudnn_relu_layer.cu index 9f617183baa..e7928bbd6e0 100644 --- a/src/caffe/layers/cudnn_relu_layer.cu +++ b/src/caffe/layers/cudnn_relu_layer.cu @@ -15,12 +15,21 @@ void CuDNNReLULayer::Forward_gpu(const vector*>& bottom, const Dtype* bottom_data = bottom[0]->gpu_data(); Dtype* top_data = top[0]->mutable_gpu_data(); +#if CUDNN_VERSION_MIN(5, 0, 0) CUDNN_CHECK(cudnnActivationForward(this->handle_, - CUDNN_ACTIVATION_RELU, + activ_desc_, cudnn::dataType::one, this->bottom_desc_, bottom_data, cudnn::dataType::zero, this->top_desc_, top_data)); +#else + CUDNN_CHECK(cudnnActivationForward_v4(this->handle_, + activ_desc_, + cudnn::dataType::one, + this->bottom_desc_, bottom_data, + cudnn::dataType::zero, + this->top_desc_, top_data)); +#endif } template @@ -40,13 +49,23 @@ void CuDNNReLULayer::Backward_gpu(const vector*>& top, const Dtype* top_diff = top[0]->gpu_diff(); const Dtype* bottom_data = bottom[0]->gpu_data(); Dtype* bottom_diff = bottom[0]->mutable_gpu_diff(); +#if CUDNN_VERSION_MIN(5, 0, 0) CUDNN_CHECK(cudnnActivationBackward(this->handle_, - CUDNN_ACTIVATION_RELU, + activ_desc_, cudnn::dataType::one, this->top_desc_, top_data, this->top_desc_, top_diff, this->bottom_desc_, bottom_data, cudnn::dataType::zero, this->bottom_desc_, bottom_diff)); +#else + CUDNN_CHECK(cudnnActivationBackward_v4(this->handle_, + activ_desc_, + cudnn::dataType::one, + this->top_desc_, top_data, this->top_desc_, top_diff, + this->bottom_desc_, bottom_data, + cudnn::dataType::zero, + this->bottom_desc_, bottom_diff)); +#endif } INSTANTIATE_LAYER_GPU_FUNCS(CuDNNReLULayer); diff --git a/src/caffe/layers/cudnn_sigmoid_layer.cpp b/src/caffe/layers/cudnn_sigmoid_layer.cpp index ccb955cdaff..3ce6aef1764 100644 --- a/src/caffe/layers/cudnn_sigmoid_layer.cpp +++ b/src/caffe/layers/cudnn_sigmoid_layer.cpp @@ -13,6 +13,8 @@ void CuDNNSigmoidLayer::LayerSetUp(const vector*>& bottom, CUDNN_CHECK(cudnnCreate(&handle_)); cudnn::createTensor4dDesc(&bottom_desc_); cudnn::createTensor4dDesc(&top_desc_); + cudnn::createActivationDescriptor(&activ_desc_, + CUDNN_ACTIVATION_SIGMOID); handles_setup_ = true; } diff --git a/src/caffe/layers/cudnn_sigmoid_layer.cu b/src/caffe/layers/cudnn_sigmoid_layer.cu index e2a4b460c6c..48d6cbab6de 100644 --- a/src/caffe/layers/cudnn_sigmoid_layer.cu +++ b/src/caffe/layers/cudnn_sigmoid_layer.cu @@ -10,12 +10,21 @@ void CuDNNSigmoidLayer::Forward_gpu(const vector*>& bottom, const vector*>& top) { const Dtype* bottom_data = bottom[0]->gpu_data(); Dtype* top_data = top[0]->mutable_gpu_data(); +#if CUDNN_VERSION_MIN(5, 0, 0) CUDNN_CHECK(cudnnActivationForward(this->handle_, - CUDNN_ACTIVATION_SIGMOID, + activ_desc_, cudnn::dataType::one, this->bottom_desc_, bottom_data, cudnn::dataType::zero, this->top_desc_, top_data)); +#else + CUDNN_CHECK(cudnnActivationForward_v4(this->handle_, + activ_desc_, + cudnn::dataType::one, + this->bottom_desc_, bottom_data, + cudnn::dataType::zero, + this->top_desc_, top_data)); +#endif } template @@ -30,13 +39,23 @@ void CuDNNSigmoidLayer::Backward_gpu(const vector*>& top, const Dtype* top_diff = top[0]->gpu_diff(); const Dtype* bottom_data = bottom[0]->gpu_data(); Dtype* bottom_diff = bottom[0]->mutable_gpu_diff(); +#if CUDNN_VERSION_MIN(5, 0, 0) CUDNN_CHECK(cudnnActivationBackward(this->handle_, - CUDNN_ACTIVATION_SIGMOID, + activ_desc_, cudnn::dataType::one, this->top_desc_, top_data, this->top_desc_, top_diff, this->bottom_desc_, bottom_data, cudnn::dataType::zero, this->bottom_desc_, bottom_diff)); +#else + CUDNN_CHECK(cudnnActivationBackward_v4(this->handle_, + activ_desc_, + cudnn::dataType::one, + this->top_desc_, top_data, this->top_desc_, top_diff, + this->bottom_desc_, bottom_data, + cudnn::dataType::zero, + this->bottom_desc_, bottom_diff)); +#endif } INSTANTIATE_LAYER_GPU_FUNCS(CuDNNSigmoidLayer); diff --git a/src/caffe/layers/cudnn_tanh_layer.cpp b/src/caffe/layers/cudnn_tanh_layer.cpp index 1a56418227c..e87dd9de0ab 100644 --- a/src/caffe/layers/cudnn_tanh_layer.cpp +++ b/src/caffe/layers/cudnn_tanh_layer.cpp @@ -13,6 +13,7 @@ void CuDNNTanHLayer::LayerSetUp(const vector*>& bottom, CUDNN_CHECK(cudnnCreate(&handle_)); cudnn::createTensor4dDesc(&bottom_desc_); cudnn::createTensor4dDesc(&top_desc_); + cudnn::createActivationDescriptor(&activ_desc_, CUDNN_ACTIVATION_TANH); handles_setup_ = true; } diff --git a/src/caffe/layers/cudnn_tanh_layer.cu b/src/caffe/layers/cudnn_tanh_layer.cu index 89df28a3e8b..6b5d7ae7ea7 100644 --- a/src/caffe/layers/cudnn_tanh_layer.cu +++ b/src/caffe/layers/cudnn_tanh_layer.cu @@ -10,12 +10,21 @@ void CuDNNTanHLayer::Forward_gpu(const vector*>& bottom, const vector*>& top) { const Dtype* bottom_data = bottom[0]->gpu_data(); Dtype* top_data = top[0]->mutable_gpu_data(); +#if CUDNN_VERSION_MIN(5, 0, 0) CUDNN_CHECK(cudnnActivationForward(this->handle_, - CUDNN_ACTIVATION_TANH, + activ_desc_, cudnn::dataType::one, this->bottom_desc_, bottom_data, cudnn::dataType::zero, this->top_desc_, top_data)); +#else + CUDNN_CHECK(cudnnActivationForward_v4(this->handle_, + activ_desc_, + cudnn::dataType::one, + this->bottom_desc_, bottom_data, + cudnn::dataType::zero, + this->top_desc_, top_data)); +#endif } template @@ -31,13 +40,23 @@ void CuDNNTanHLayer::Backward_gpu(const vector*>& top, const Dtype* bottom_data = bottom[0]->gpu_data(); Dtype* bottom_diff = bottom[0]->mutable_gpu_diff(); +#if CUDNN_VERSION_MIN(5, 0, 0) CUDNN_CHECK(cudnnActivationBackward(this->handle_, - CUDNN_ACTIVATION_TANH, + activ_desc_, cudnn::dataType::one, this->top_desc_, top_data, this->top_desc_, top_diff, this->bottom_desc_, bottom_data, cudnn::dataType::zero, this->bottom_desc_, bottom_diff)); +#else + CUDNN_CHECK(cudnnActivationBackward_v4(this->handle_, + activ_desc_, + cudnn::dataType::one, + this->top_desc_, top_data, this->top_desc_, top_diff, + this->bottom_desc_, bottom_data, + cudnn::dataType::zero, + this->bottom_desc_, bottom_diff)); +#endif } INSTANTIATE_LAYER_GPU_FUNCS(CuDNNTanHLayer); From 8730b146b7e19af189b9086e59fd1d5bc4214698 Mon Sep 17 00:00:00 2001 From: Felix Abecassis Date: Mon, 16 May 2016 14:32:34 -0700 Subject: [PATCH 10/39] Update Dockerfile to cuDNN v5 --- docker/Makefile | 2 +- docker/standalone/gpu/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index 0de887d0e19..3a6575b0c43 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -22,7 +22,7 @@ docker_files: standalone_files standalone_files: standalone/cpu/Dockerfile standalone/gpu/Dockerfile -FROM_GPU = "nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04" +FROM_GPU = "nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04" FROM_CPU = "ubuntu:14.04" GPU_CMAKE_ARGS = -DUSE_CUDNN=1 CPU_CMAKE_ARGS = -DCPU_ONLY=1 diff --git a/docker/standalone/gpu/Dockerfile b/docker/standalone/gpu/Dockerfile index 371aad5b1e9..daf6a7223ff 100644 --- a/docker/standalone/gpu/Dockerfile +++ b/docker/standalone/gpu/Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:7.5-cudnn4-devel-ubuntu14.04 +FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER caffe-maint@googlegroups.com RUN apt-get update && apt-get install -y --no-install-recommends \ From 1c3af7078b64ef71a5bb0c2cef6fee528917adac Mon Sep 17 00:00:00 2001 From: Felix Abecassis Date: Mon, 16 May 2016 14:35:40 -0700 Subject: [PATCH 11/39] Update supported cuDNN version in the documentation --- docs/installation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 1e29a49d82d..4aac7c42d27 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -40,14 +40,14 @@ Optional dependencies: * [OpenCV](http://opencv.org/) >= 2.4 including 3.0 * IO libraries: `lmdb`, `leveldb` (note: leveldb requires `snappy`) -* cuDNN for GPU acceleration (v4) +* cuDNN for GPU acceleration (v5) Pycaffe and Matcaffe interfaces have their own natural needs. * For Python Caffe: `Python 2.7` or `Python 3.3+`, `numpy (>= 1.7)`, boost-provided `boost.python` * For MATLAB Caffe: MATLAB with the `mex` compiler. -**cuDNN Caffe**: for fastest operation Caffe is accelerated by drop-in integration of [NVIDIA cuDNN](https://developer.nvidia.com/cudnn). To speed up your Caffe models, install cuDNN then uncomment the `USE_CUDNN := 1` flag in `Makefile.config` when installing Caffe. Acceleration is automatic. The current version is cuDNN v4; older versions are supported in older Caffe. +**cuDNN Caffe**: for fastest operation Caffe is accelerated by drop-in integration of [NVIDIA cuDNN](https://developer.nvidia.com/cudnn). To speed up your Caffe models, install cuDNN then uncomment the `USE_CUDNN := 1` flag in `Makefile.config` when installing Caffe. Acceleration is automatic. The current version is cuDNN v5; older versions are supported in older Caffe. **CPU-only Caffe**: for cold-brewed CPU-only Caffe uncomment the `CPU_ONLY := 1` flag in `Makefile.config` to configure and build Caffe without CUDA. This is helpful for cloud or cluster deployment. From a8cc860d6bef79edcdfa07d5da4195ba67714991 Mon Sep 17 00:00:00 2001 From: crazytan Date: Wed, 27 Apr 2016 01:01:30 -0400 Subject: [PATCH 12/39] handle image names with spaces --- examples/images/cat gray.jpg | Bin 0 -> 92726 bytes src/caffe/layers/image_data_layer.cpp | 9 +++-- src/caffe/test/test_image_data_layer.cpp | 44 +++++++++++++++++++++-- tools/convert_imageset.cpp | 9 +++-- 4 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 examples/images/cat gray.jpg diff --git a/examples/images/cat gray.jpg b/examples/images/cat gray.jpg new file mode 100644 index 0000000000000000000000000000000000000000..43c5ce377167a49624c3e9d61e83a04becb8df9e GIT binary patch literal 92726 zcmb5VWmsEJw+9-$P+W@!EyW!Q0aDzGJ1s7S76>lIrC4#70;NE4inIhPQrt?>0s#U9 zcefC@`M>vl&p98?bMA8|`@`NlOJz zy%)2M{TnA&8P=n=PF7|oI~i7EF-<{DHwAk~C)M{J_WJL&3~b-Ovz4@C1UOSlyJ{-4(Ztjzx+@p>o2D)*08W)n>vW(A0cJ+m0U2%oK> zkPx%DIKPmnn7FtQFSD?qkg$Lt`V!|85|t7YmlBp>{&!(TYxA&kkkV6B{ zjq>&N<@Xighj_dZ5R#OX{6~hcFdv$N&(q)4%i536)syW%5)|z{Z9SaayqqAe%>PKV zwt;wi$*`hp`v02X;-=^H#?j03zl8s6&h|f4{ojiIA^f)uQkvEt@9fdNsBG=*Y5%W3G&QBP(Uo^{ zwRTp7*m}FzyLzc9%CJ5c6%?0~5SNrt6jqWKlY6e9tRN^Nt}G!aFDNW3A*S#j-v7yi zR>N=SWG5vgrz9jOCLyONDJL!@q$KiOSV&A+Szbs`K?vQUV#5F7Rde<9vUat#|3}|H zyiWhkEBSx&N-22QTYEt~3?LBa{|qD@M~D~1(-Gpvte`K#Z0zJ}2l4g%r;pK+(KagD zdpP;n+bMfMT$umW+)_^ei*`Yw=jb6QAu1>#EGYh5Na(q+u$;V-g0P~pyqKuaa}m~m z^V80!DAJolzYS`tV<FSTBxQxm8QC@_3BJ8tgQk`gb_y_Q!$TE)FS8AuW$@-_7A++mwn^;>R;Zhx@Be19}CWQF#qd6M(01b>0n*l^{#0^#IOpfzwK5k@l6?s{B zCO7c|0GlT%30{7q(;G&Lqof1SQ@>Bu26ytcGj{D-YRn98xMc77#uLDEgdx4u(xCvd zHj)-3aeR$dKkK6fTEF!8Q{J(_8S`t)KT=m8%fyf{ssPDof?}_gYS%p>XiQ z<*wOAB2mBa(s#xmi^QO?u~4EG*-d)yN4%;SfM1tjF4>`Zym_qDm*O+-)&Q(P@#PLkI zCA5bgH!Z%f9`(0SbO*D0H5;j`VY4sJfzj)=W8!m?lj;z-;gd5{rAr z?;jYXe>DlGlW9Em&Dse|lowxb%b53J4`l4#Qv=6>73zo5Rf=i;3^7az<+fW#5WhOn zUnjYOLC(CQc57&I`pK-7%VARBP?4U%6KxH{Q384;fAg!l0JI*GrPl zJ+}!-%7t1&;Y%AOsbReK_8h>b*M2_qg^wQqA&|sRuU%AaDpQ12i)8jV!KdzXf&0=J zTjQ78pk2GaT)O_LEW#qy^k$=m}#o6)^0u0Fn@X*V^AGBOpGwT8PI)aJ6PjXZe(2&nM!RlRHA^%&*R zxKWABL&k#34ucr^ZN{L(5=;K)8L$)}+D#1f_iWvIv#Afg za`$Cf#$dg-wx^s1mKn4rf81w-?<{3kCSyJ7S`6Lk4C@W*km?y(^LW1YwBAge9GjUyexwB@-qZupT#US+g`St439&qEa!Y>AyLakhnSfQ6h5yhg5-q`E|f1K7~K z8qrA|+AVMQl!$WP8A`cUIcj$$%j_N((qqc1w?H~CWoQ~601?BN^&x#Hq(j&>?H`{c zY`joix3tlQcwWIhC;wLbWE*s<*q|`7OQi-KR#RSsJQzamqwkk=wvD`eRErT(FMX7B zBn+8}d8Gv+&MvnLi+>QSCu%)e@JW=EDa_T{AI4wOe*je30Iu&{^V&I{c{8~8Ik@S$ zmGq4`oI(4yLK4AKH`>PR=bS1qj0Uku>yF)3-)Fs1%YrYKlCJP7`M*y*Hy2MJ=6wK+ z2xL$w(+&W-3RFU(*tXmo9{}@)zt_2En~twCC3u6{uo?k|K1^UyD5`0%7U0sAs%&9O zM_N;-LoH9dEA1K-4S~b&QARF^ui~aTP>$!R9sU3~H`;FkfOpYG48X5BgxgLD{^rAg zk2HAPA07aEzp1p=8%f(S;8{m2`x+|u1Q*JM6QbU+4nK;7efQR?HY440*anvsfoFxP z`#&lM0wO&@894R=WJ`YMmG6&AIVR`wybqP$(BxOxpm?MBM&A*rnNikLl@r?Kz;cDw$O3oQEK)s-6d_Z-WC`H;E&Z0 zJB3^_l0AbZW8>19!dWTyfzLsJ^gAE+@T*KBxP}@i6f~>}dxa%;Y1ZaKw_S z<=Tt2;zwV%7%eNlafYZ^5Vf&t9BE8OgyS4zfC{aTM}3i~+q5=vZ2zu#t%EjF()^}V zlbch@17ty+)!-D#&G=&E@fJ*Pc?I)$K6EaBAf3wu$8#P4^a(r}m{E7y2gSweKsw(C z2HpjRO;KG3O#L{?dD}v7Xy_`Iy-6`dOZX|+KPiIq0|JFwa;;K%kBpqiCYXHP^pRb# zBFSg2=y9X6VVXx?Xb1!fFRHD(bLz=2a1K(_d!BFkQG>PG+9WvRL*Strb`m>q3Z*~h zR4pX+MXltg)s>4#%J+~A3QV9CE<9U#;CIhw*|N!tt-hgi^+{hF2W1capKYYw%E;im z&=Y9>t**CtOcG;y>{@NblHMgX)!J)l*W)@2C=B(pL;4~^fp(5;P(Pdt?t2NP=*{i| zA))Ieg&Vs^Mygh1qVK-H^kI0UBLQkirC2ZAL|)ito_odXYOXydY^QBn+cpf-!vaE{ zp3F!sT`ymUHw&;wV$dPeBgP?{mtQxMV8P}u@DJo9zL?1Ha+2%i&BV8!mr8XDp|CDsj5&-T!i5F74T%#)oEEx;QO(!^_d>j8*%n; zA_p7Dm%2U&G}!~`Q}K=-0N*r-K55z;2gS~->BA61lvH@X@9#C6;5*x|3Q{wi813S; z)SsjbWO1x(4a;3QcO4-;cI#C{UMwgcG$VSS_IRGkv;k9Eu2Jn?Bq-WwC63Fi z#801>&m|-szTYljd%C`l84laWqmfmk55z18u^lP*U)yYiP_IbRe~ZGi@&R%t0ij2; z6kw-_HudE$+`CsETD|vucc@UN_gU5KIfd~7G%WV)b%fF+Y|$;zNntZpz}(@rj~u4J zrfyS&g+eTL-F(=|N)jxjQ#VH@BnKNGZzJ{zj!3nr`Zb=qC^E`L5w@M5zfr)v9o^5Pk)TM=K zDG##ff_ncbm;+33V&^QYqOJJnZc zw`?C@KZmL6GPN#vHVN5%$|NHsozTdh@fERzWzS+#NbhgCEhp=Ud8u+iTgrC%ssb+~|Rk(_{*;9SD)d3`6cOk=8%)2Qir7 zS}gB>Qonv7Dad?x&Cfk@ zszeo~2B!HeLn{(4*c1(WXU12o+N0Da)=N`x>2fBLGVR`cFrwc*RAA5loO^ud780rQ z#WLZwQR6H;BC!4aJCoU|bOG&crpCUX(pBbD{m@uXzm_`qr&pHy z#*PXK!~%^&h;NQmMg9uSR~iX53DFgblr;FYCpf*eue*6k-p7&a-%9?aL+T2@^ss-o zI;vtxMPaTy_vEULU)L7I9Z_#x#{gxLDGR#0*_|ozYM*b5XGn}| zWo^o^(F$mE9UfXgS~-D12*e43rM<%fnEB2y40;<#9=Ape5{0g7jUcv;kQWp(zX6rq z41q5Y9UBJ81Rq?k9suX}C?yTw9Xlg(5tr5*vW9EC7!KvGgrtk@h|tCh#_05@H%jaz ze@GIV@XiasS72Dx%C7+jKRbfSLcBX9F*~LYuqw%z(7V2@NFRe>j5er1j-t4nx5OQk z=s?Jpd$b-+!2pYUrz|97+DvUiCaq zIn_wkm+4@>X;UsJ7LFcnK0kRTW`8RGHY!#HPeLhpCtUmJt$L-xI=5ji?MzKX?~Xm~ zy^^N8X&K{CgKxaZJyO!CH%46($Dk~op-XSaXGyeA&5R{eqAnyhUaBobmPH|e{8-kj zp+<}K$H4J|47K&bh(1ESND=kL{GfqkP;Hm~^a%^l)X5cob^7?LZKCu2J60hG5Gn%S|=XM3{ z@pY<9jQLtH5oHs|$W~B?@!$^f{O+f_(U>W<)U3*8^ku;c)=j(n3kjfSr^?HkNvGMw z_A&93oX40ZSLz2)il*i9y~`bsnM)qC&^|tYN?s4wK{@PH6OreZHh0c=2b0FTO#CC` zwskRBh2)A~grGY%!yPYSR3|Iw!19Xt@02mV4YO}@CgEfv88&-FLj_0(0!r4-Z%>So z3Q&^KrB@$K7-En-HPj)OgZ;kOPP%YGHF0j6U-sMDq|_{^&dLXkt@akO`s;DvSf z=(#t`CG%utIMX=hfVVNAf259JS+c;A?Ft1RoBUbh86ME$n~AGr`#he+30vyx-cuMb ziunkd7}mnvM5kJmWaws~dh1)N19eoE^wVCcwq!)<6t`3}q!_^Y|OW4i2$yy|-@QZe-C+yfBai)JoD1 z9I|mdfSzLqvl26cYLq7o04Dk?P8(umEfO>E?vANA`!Rv80<8~LGo2VuGdOJb%;k-j}@sxIWZx2n&{7>uW!%1B7>if?AtZ zc0RWA6AIPaz*1=SieAhC3papGD3TxAC8MA&MGTY0!C`_ENUEVSl2i+Rbn@rvY~eJW z!_|zA+PYcrc?_o7i=dKcK!^`=Ut$|s^wYcEj2-P*X_Y4r8G%j3R(-1A5cc~eN{)+< zFCDC{W{XU7NTO1(4lWJj>kv(&(l^_X!s+jui3(;Wb7=?M-R%vTXU)&$Ym5XM|X{lE-5zh7-L_T(qGSe5EwWP$-Y9LoN_$B;)NV~krozC^;j9(_}Hw%4S zBxA-*TcjV76cI9Wi1uOM3wJN+xoSPn*2*stw;%OMC(0kwHBvu)LEH<_>N#Ap+8*zj z{`1phs}|J^)+b#WNo}Oy$U0jUcgaG;Z2wx2FJ-D zIPtvVduG#2?v-TXhxI8s>vxksxQC~Y-taHR5!!6>v_7fTi*vZmz3kkZ$?*e=&bKaa zQsD8{IzT<9v&!&d2A-(~MAqDegRd=SE8t?su9m$}&&Gr&bj6|uhJ6VxJI9)q5led6ZjcCJusKo_Bf9^Bv%;8Zl8p(&xV(ma zemOcZIy}Pe8m?MusvbpDx6Y`FExxyPxr;lkK8dp!PTySW3lF-#SIJN6N0}dIT0GWm z9L($A8)RN8wZzW$&u@z!fc5tjyV5N@>5Ve9c@m&-LR_j63Yc#XJh*bL7o*{yAWOJD z!&WwF#FWr^mAM+aO`meUXS9*&{L|UKg=f_4`EVb5`B2A7Q;%PELKM;uR>{O2kQ*SH z(LiMvt|nQf?>PU4|IvJk@6sUb;AHlnK(xnJks)K4#v_*Ddx`p`MNiL=-zmS1QOG?j za1)4XJ*ThnT2A{`y!Ji$Zv|;3_7~wij|4nPpOcYP=a8|xCC9SAN7_QUNhb4yI^kRu zzwEB4v~S<4@-VL`IAa#VYCLXxwndCIX!mhA2S(T$$Hg4uXLzZy1)Y+UhL&F4DE9_> zYn>NUslq60el?7-kC8esC?s#Wtsk_%9$XB9!jRHW#x=O#2$k(tDY5!YSD(3U-eoAJ zd- z*ur5jg2b*5?-Du}vNLU9k+@4B zbOa%>;&o~-Zm^39U8c2*Ne0}&g1W(sZjDii^>O%yR(^wb6;x59d4)ixM=%dr6zTW= zuTNB!oXBxpWzF<-+bfyVT!u^r&128%Az%mg9I-#5KBiW>T`qlUSgYZOkLIZ90}t+& z^m+Gaey9(RQ8q+x5;h166FVDVN-P;1Umt5HR*pH;#PEJk@hMG_NLXP}b(!|JoV$Vb zuU}~9CV5*fyb8ff0qrXf`(Lk>0MVY=?5F7Y;bqZ`x4dxO*Ez_R=#gw&c|5RmTo*OhBE7AyG2YEJiC zO;B=XlpghWr^Z(rp>~xl0u;4_2)?6Yj7t#=+0~I zJ)HYw5S^2)k|__0NAC9EOgfLFmk3Gc~8tLZ!^g1|nSU}}1n%2U&iwD|AL z#At+gaq6b++mcer=ll_gkL4ZNb)RHJJRO#g+N_MRR*7;@uEkWQu+QzJ!k{JIN-tkk zZ$?S`3&5(BOjmyJ8r%PjyB&=e7CmUySxcE% zqWmS!mqHrX>C;PEby4EUBUmakSjaz+?Bsrl-}k$ZYb%e2!HaH_DxU|yE?`-l(C3B_ z?`T>zs*nEJrM2UXs)Kw9k7rO5p$bYHhy|NSQ=LonMlqMM`ipR-m|VNWu(QlJ!EeuP zCY$M6;(xxs{Nt7dqq_#EtHNBy2gr~&d)~oe!IW-v5gG;^bLV~*?ar$PG>H-zCPR-| zh(F|cBHd8XZMMCu6%FlykH|DaMi!J^xJoFUxgY*yo8CD~qZ-3!A$2H^N^ zthjXELOH9bzs~x>D64&FWpf=%5Euzx6Ji{%Oj$WyvZ=tqKOO+(KMX$^eI8t@VU996 zV<*|!y1d?*KNp!PzetpTK9y^`C`puD)uH{e8A?2F+af;<%fAyo zpIW!bw0%M@Y1cS#YT9JUz;QOxEWx*O=k@8`!i>A5nu!W16quA8rc=T5WNrD0#|;=L ztzs~2tn`KabpX!e#`ZAaK4yii1SjJ5-~r%4EmNt%&TUq$UNva^tk{Nl5sz$ziftSo zT+8#6oRe%cLkHR%naS3I-Zh(Njdz`)*U+aYQ*RHaEf-D>zPrB43tJbOi$DxAJ{Al9 zFz{w~F17C&O9QJ+l~6Zby^R{~3%pH|RNQ624rum?^zx@%lfz$UFKA${A`O+4Fdu@p zmc@gsIj<+9yLZiCO+8P#%8aEX_nhY$kF=1?@Q{Q%l7er+g{EVh4&U@N91x~%gK@s5 z8l-HX(88W+69 z(>9~nk<}%Qr2Z<<4`7$hknzhf`~rD)E9r!OWJ4x%k}Mn* za^R}AH2Lgi@aQpjU-RPW29%2Pg6LiawpH`_p6=N1rrvNFW%wo!x=2_YX?M`P?#s}A zhXysFN^tY@Onfh7ZpG`ynPg|o{TG_u8_lqR@kLapu^HAyo)ThzSK}%7$xSf7E-9ve zpdhk>6LtTdzz+4?WLwRKQg5kdvA^TEG=>8MWsBJN+3CM~<`g;UocV+=w}ms#F4e)t zGAK-ERduya7JQ?+XS2`=O6YaMH{+jqlPddK<;jqWC2uqYTO2XaE%Q!eCc*7wcmx}+ zzZhqdg0Z>H0ZN0Ok**)0z-M?nW7-^#3E9)!xoiiY6Rn4JjID=ir*)oQ?+#_pH2m<5 znSYA4+UvXtp+6*uy5bU;e&E!e()7u>Iqq5l^d|oo(mNX$X2rJ(yPgd*E70 z?+bH$LpB^I^E-tT8cWnDKwk0yFg%rp%-Z=s^EmxWrRL|5%|EuUv=|_d%VvL>loT-~ zw=*kSHKL|L0>YORB8PP5w;!Pe0;g)S-dldDGRmU~o^cWuWiMz`GpCPC>lvk`rd2_K zn3>$63#tQrS!v$eQ&XECNj{$&$?(k%4!k1N7_oUp@)o9mCzh#_!n`*mYZ{7r&(dFvDU3z4!exB#)Zf;-fM@4 z986+Wop}}Cw!NO%5LUPKFuow~YJ%-+tsI<9_N5Q0rF8xhiz%jlBuPF3C8bk9lHvoQ z7uyM?ZbIEQrE{+1RIO<$HjgKUV16#enC!=L_{SmN?%UM=ShT|i*WAyATc`u`{Zb#d zZ(M1>6wM3AQ(qWi6D7&U7@xZWrynWHSN5OwORAe2whXrsGn1``z-&9w!7K2Kl+j|R zMtYxNluuF|$;g_WjU=@e_*(S=FjaKlcE{n=CgdxW%(j%0jUr1>Q1(k}cMcRjMKp~e z?tXGg9x0wK&@eNbm|KH{4vQ=n%d9lQ4q*0J?z0Z50(Z9Le~3L4{0Tpy_ibhLpM6f& zzpbi>FN!+J8jp<-U{I=w!G%8D_7>Z=PxyvqsiOeo((zyQqwE zVZ+%`DSQ8R%z1dbn!ob5LEoh@xSXAq(BRKyWEQV+pE`&-#rbqu1FNmQ|M`{Wq^{JF z$=Z&JI*(1xXnj>{JcMa>Sn^HVRUW9YavIrthLsadT{Vjy9v?JWhHF2Tuo0Qz_V` z`87ym8C!nYWMv+Xox%>#PGtiDHHH3|Eq)nf&3JAx{U%sZ1UFF_gRL+j8h$8cB<5N* zIFjtoH(EdFkn-9Ac)--3u2$E*xd1V6o8X=mG+5r-T%+e1+7dlV)E!s|F&nD%p2#Wo!@ArV_OulM%tvg1EwRl`pwhvmOf z^H=vKs&4mkpD-WP!62O`w^*icygFVndQ|I6dzTV!IXnPL%r)HgNH}%y@%m9NTN4j} zUZa9WqpTARZ*rURZ1e$ceq#|muqG}#J*XZn*?6U|wEdmj5LWbPQke$$84Lo^a^Nq3!5yG#ML{G85j!xhkIho*(g5sj5u^<;0 z-98NWxYr$2$egruZxNS0X31D$XiAB>i#!ZuY$lhG#a|>rJ^-|0vq2dL6Tq9QZ-#TWlIY4(56re_Zfi>pC!Ag;*!{d6;?L zoCI-q5@`t+*iAw<708ZnwQ8!B^XK{MZm52*D36;fsl)cfgD zmNogM!8C36t?JLz+HzWhoAs{z0ras}2vCNdF52)@e!8E9aXV&Eq$ zHXkyH+c)rBJiH5A40LV_(bN3(?z(QCPJTKPgFv)4p+$ex!}|0Qz#qLYcnkbMV$QuK zMxFIXXTSmVQ!~3bdLZ36n>~jX5gja)#y;HM9tnhhQAb!oK5}tdskA3CS=#u7Pfymj)odGV(+xYjczj+4I(&?H6xaG= zIqY5T_05j%Z*5zlrLS|y6U(rd1oJq4Z))bkjTAl(&)UVSF51vg(iRaTJ5!QQI_?X&Ng_R~CDWt|yJ@$zcN+djV!0fQ_N3!&&@6I~;g7ix>HJ%|0WvoIA z&kKu;P9KnGk7&bR0*&gE>@8Ndg=NQJ0XK)+e7cggE z**_S_LPpb*g#upn!P2%P%D`n&jQ-wZ3yY)2o#$szBN>rhBV(GZfkjFVvk~Dw$9;?_ z>UqX1rY8W{%?TQXT$Xu0oif#JIHbVp!#5qJ^JzS-i--FIs1uqE4(9t4J>A*+-6HR8 zwZO^S(;wgHOXzF`1O1QV!Wc|F{Bd;CiX#jS|`a+O&?vQfR1Xz&hM{73G@ss6{n z7J4*rx9DA1ez?*IrTetqLQ5O57h|Q7D zQYa!CCSK3bbvUw%*0}@hUK@&~vePuAHK9uMVchYj>2g~${9)&R)o&riYv)%}qYQq$ zxLifC3*j$r7?Pvf6rXkm?xS7hQ^5ClZAxtq{+=&B0vhy>N!FuPX&K4zNmXD5>URNo zo+%--B-@wI&Sr9Do@KJ+Z(neONiZaw_*Xl5m!Wi*aZxR}yP6qk5du zt`Rru4Dsx+!&cr0zJAbYkXMe_RdJac$4D;>M}yfMjGaaZ@3VEthDs2o#ONIx~oxd`}YrwvWs{7pJJmkJzSgAK9n+$ z4lvgK(!|$o>i$J#bf_|xP6NsSPNB~=fZ(OJ6s^hA@Ur?Q2E+U?c;bq)uRVPjN~+oE zErk*0S zfW_juulQSTSX5GdcLhHd0X>?PupnoTPZ)GI4fvq4NYYt&dfxJW{Ogy7UxD7qHiT{Z zSk17GyYwtJ_fD3}e~yFYevFq)i+Bbb`ltHlZ@l^FZIbT6V5{HRE(9rT^FKMUNSdrX z=gDU0os?pF^%aKneH0ql)Z-C6_5jG|4#H$@?V}pUxWTcETBka`N24Jymw^uuMv?oJ z5Sl9HVjtgHV{0r44n6!O%e3Gx;`8lkTUR?aK`7R^O}0D9ldCEeWLir63$r1>-UFs zfcxNNlTz0gn0`1MUFbxxbG8Kq?t`=2)>RL7yI^jrxx-ieTkBEm0)sABsM}iAYWBIZ z9bV+MOOI=NlO=&PQ>SQ&lz^wVc`S27=Hn|__~VfQBxAHG`Rg^(>UJ$SOA-fgS5|@n zy7xm^I3aGNX+w2BmL8SOdVoadr?73mj?!SsjB9xLyJ%ubl7~qa!<9Lp^9&1l@uN^J zNV3<#-Mio%m)=tuQ>zlYYGn13ACAxCV`(qtp&j1Lba%Snr-qwVjM;zFgy=u$i7yI1 z0RB7x7KXG5gehO9frHEqUrQvk2qc6t1qLtiT2ybVf9<_A>$H0&%-`ia@X@s=Y_2R+ z79A%+Ki~;5_qCOHl2&wdVGal0CwGoC%Ss@TttDS8RfM_2mUzW|I1Cb95FvHzD#*lE zI{YIQVbm)!woJh+BYD8}aWJe2-m%ykkB0O%vE4FKer3W!JRySs6%qhCW`$kt5*(OT zu~#@qvLuH_r{SK8;~|OSlJy&9|-AlJA>$oA#wm3F&;a#{>;^My*{xcZm)nM`T9mRysSHl$hsND#lk2qOF_(^LW zVX0rm&S0MrC&RpF*dV-_T|f9`?JEXBN&Kf^bx$$d5?&>{?=_UZ$m&m}$uChwY!?-2 zyGG)fU(2YTA54r4qyZ* zL_ChCY&af2-U_KG%e@gG`lxSPJ*Kf}p~gC>NUoXwf!a942SxZ(`cA+yZfv(j+Hlar zkGDa0<_G$#k|&!z%cKG)yQ9bZdOunM#M#U**ApvC2e&(S2UZ=H!{oRsC=tx383*0W zYFzRo*haRAO1QMNY#ebpG&n>5PYX|do~sLj(4BLZkULZ< zd&oao%ZTjM=ik&@j75wjjf>QS0t=ChQQpl(bqnm5;e$O}M;*(u z64V=eRx1q+!xJMg*O!K|@yZPQJG6@+Vo+63hwRex`ZRpXaTQ0AfA!^1S1nIh(l5rbsVrpbPgv zl`_M&C)T=+G4$q?M4hB!dRM8nRMe_!P>tDoJT)Pm5lXbt1owOZ6l(am&T>Z*aV!4Z zQTSj%XlRbRF7j-#Dq`he3j+I$`(s8|k=-$SB7Ja}1t0(GNAt^fZV?Y;49N%LzUI!?{%lcs+^f;WJCrxms_qkKI znSo4ZQCc($>W#S1@)FOTja$eTQGW`+^%hM>+rV$PHWvg`Jz@6GONL>wRA^D~xfRG6 zv>?A~)}0Q@XkGyWd|^{-1egFM*gR3~A=*e~lOH%%>0Jro`_Zv${y1=Npoa?l;2wn` zWB7iya){9SPrFvQCEwFse0#l~E=OR|_za7i~ zsfrs-u|-s{ax%1rdLSvSsraC#`$xb|mb)d@nWEHRW54c{@Z7@XQzbam7R4U`qk$=B zGpFr8_>C1!ZalS?@2GsN7E#qNjvb&2=ZGiZ;)laE7S6*ZE&w8cqHwd~d-)+@yT`6|K7eIH=`BPmv+ zGD;tw0gA>COR-pk#c$R<8J<|q;no;x%P7fi^yO8w zv5^0mkf2&Bf602F5QC9ycEkooU96UOonOKp_$nkAQQDz~DLY!Mjn@hlzH&OPU z$U^;+Ys@-oS%#>P%zO%C<4;?uf~WE>iX9LwxRDZV0r z{ZFqdwL-LZ#xS9$e-KqYOs?x|APpQV`+|cQQ^zqf`s!4Z+EYad?j4fbN88qgcSN@` zHP+$%y7lfTS6SXqo9ZdXqzE|Y8zF}~!S;7uni)T>2KQR{%*oP;S_$SK07To2nC8RN&Ope$d$Dn#r0{1`DTrVcYq?P=7OX6y3Tk8T@I=%fU$+JC*+1U(MXlEL3n z7h}$%^^qFlmmkZFpy$`~UWroUh?B5aw2&D@qR*T5bigr%$8Pl>M;810B9yOTQ0nL76=FM3kj43SX3z_Up5oUn0oM|d@5_s;h0w9AU;zoo2?Wi<1y z|4}(WWu9Tc_qI*3h81ApL@RCt27W(~LS0jP8kyRW4N)oOj)MB0qAK1b(p)+#xG6wc zZZnL((hx-??UJi+*eU>@ERxtnt_7zTao4M2F~+4F0C9qrH0=2KT)EgwH`gb)C0zbQ zU3O;bynaRxX<-V1m0x?*OKYRyr4+8<(E#c#ZbNdN58g^%o6Cf5!{E<3Kx-7 z$Y?Iw)M<(EZXi>;AflL!aA2DJWhkAE!xH^H(7SGlY`1{pL~yvzW^EUro=FeL>22GZ zA3eYrt@pm{?`to=zuy=ga$h7+sM@PSx&0#=!i!om=p&j2N^NC!`P%F{rwqcidA`3@ zolg>3av1JU_Ito@oa5{^?Ea?<><`$4 z=9R6Z-RjT?tBERG!{}X%H&*r=5b}m73JsOh3~1v99pC0@xBdQZo53SbyM^CIWFqZ!$F-Qh3o4GX`bI^g)Es1LvFDiHUuK=bg$nv*0tM02Z{d=00lw%z8p@` zSd0|p%vGfG$ozTO{{ZFEpRLz9l#bKH^RmRnj74KW@sL9vBonYFkB!fdk8fB$o@pN8 zj#~i82^{0G`Hvf^ZbzG+f&F*i$DXy~h^nQL)yYw?P}_h2@=o65;(YvmuCXNoDCK6A z4{|WOF663!%s?Cd!~^HY`nrk|j@ezvZsy@wDN?<@AU4P6&;7?-myDpP`;+Cqc#bD+ zzCrQedFzakM25#D-ZgF_P1kM4>>qMIIg&bsK#{Wu{{S#|7;JXjf_DROe{u3RAJ?uh zM-2NMZ9?}WvVgZF*+5nZ5RHQFtGOF*@z6We zTopJPOEE73NMW&0)JqjoK=&PkY&?#scze=TQy`8Y>UTiRxPBwTjn9^1HUsQ>w?~>u z7BHMqZ^e(V~$om*Mvu_)2H|_7a+;9G!y7d$2_Lwtj%x(_=W{Tax6+1R*L?&v~ z?!0Pp!9#vz6^w>Om47lWOSiS#t(I$iZh)t!tK)K6fi_DC6JpJ}vn+nZ;rVvtm>%VO z5-QIU$g0u5F57#$(^sguH^OThnCw<}A=s<=)tX#}O00Ifc4lH}H5&)$RPu}-$PAdhiOTUbTuZ9yEYaX1{TOIgei1cnlJz_v9Um%6k+{39)Q9Wd_ zpS^|0W+0RJb~3$txqLHmuVeV8E<4!dF{vB(!&&cw;2<%VW*dg+w>|8>S4K;fyJmSH z#p5xwqgqn-vQ%=D%W63A{2EXVQCRz%W8HNJ&fJ!hr}J5Cb&2z{N`|$Zg&a;7u+v2L zWwkBWf*{foWSY`b$jM@N1S~btlZa(KFm(0RTpmY1sxw(Bavw{CuRfhcge^p?r0S&F zgCP`5NRFz=I}l6YabEYG$6W2}Xm0v_Usr2N6#MyFM>~s#M2#)9*{pe*2r_d1x;>0J zi}e7_6@}shKFS*Iu0sHDl@h%fGq6{l#&&4=rEHE)M7-`|L$oNc#)=UR$rLe%-+j7} zy{2$JpJ^JHJT^wfu6^8&Z!J8K!DB0a$8E04@y4LTGP1|Vak9kGk8_>6U$4a1tTC2e zHYQ@h3JGF`RDMfLjujg4-NHoNmOs}b^VvnePU7pWkuZihrgOZqwV5TeF-2Z%<-+$i z3OoNyw*W zGECuR+lj23@>t0&y8Ll&OYzY6OiwiTgNZ$`!e+YBxhN)-ut>X-0lRj_oZe;JJ=ikyVxD2m(yXHtxeJjhPO|U^Zpn-PK%5WtuA0pj8qqZOFGK zRgty?DE%$~1N%SUeu(;RtJ2%O8Ho1*Q^cyP9l3G}PbS1*=0$2 zdtsfHPj>$RultY>9sYlQtPxptbyu+ScF=oejzU~ct>6+)+xY&2Z*NuXw6e}*c%bdV z8+QD%BmA-Bk>BTb*q_;dr$OAAxAp7~bAqgp7{Ti&uaDAl%gXCrc_aj;RpjlO>UEY|tWG!F$*QD6%qkNH|C4nAtX!m<`JS ztfZaPjlM_Re{bv8$Ae@eyT#S_Ans1!gSZFupPkQ8R(sJSs4wIYBT%GA)6;bRoO$N%Li!lgE*p@y<{yulY$EL?uQKOU7^43utl6KTLf_6|h0ep~lX58=g`Ri4lwO|^B zjE`nE1|srqig)Ct!13}%!+-Yw0J@!l)^pODXsmONFvS887j2Iae0+Pi`S~9wezLRr zhCH+)xnV0wC>^z7$&TBG+^ZAI^X+MbW#i}?fD=1YmNeaK0SjWGH#IP(>7G1h^JsH(g zVDzp&SmDQ7$vVp(OA&f1_b?i!7}?r7T89lgq;|WN)?U@9T02oPq_9m4fm{}&)%RN) zXDNhyjg2rRuU{#M%j6p&CYWoZdh*FDpq9oxth|+y?5Y+(=$2h4Kb&CFo}y%R?O|V2 zTC&;vJyD9AHzjA2{w2~_c!7hx5~E3TQD7*TO3xBB`3x)1N^vyQp_!WO1-qEcE|tY) z)xNJG{2voq>_u%*v0o1zfv0l!OmKYX#_RJ(}>#OBtnd^CZ$lQy&Iz{$$WR_@x^VW9ETEY+gKPmTuyXu>En@ zmF*@A3bNdqS>(H769m-M%v?_crx96vqvRL|a3stM#`fPBi za?FxTVlislhiIzTi1VJ{SydHdEi9`a0_h(vu{CaAB`mT??G3M{;a0V192l_C*V&$H zF+^B@7IrZfE=twz?7Pq9E1$#fNm$;3YVbuIZ!DLxq8NFz@+}=mByM(gZTEKIr*Hz6 zZTg9+lGGJugXw9@dd;f>s#&WOToCX(2AUe`5+rYNW+U2JecXpeY;Ll(j(BQarFtcB z>W+{!>heZ+M_8D0_Z8eRazP);+x3;Jb!AJKbv6XKT%DuZ#6x2pXCIJuK4nWS3c8tM z=eaT!2l+{Fm|Yd7txK5Gx@tJgSv=gIS$2687;9o{0B_jg%YI53RD-{{{#UOxKmIJA z#0`&6^%vk@(OoG#%X=NAJwf$iN^n|O>D9&NGc|O^D)5CO=F`hAGVg8qDk?(2EArfV zWYa&1-%&CZ(ku2}7KxC-Aeq(J9e^7*lNS3E_xSLCS4Y}Dpky_b8m%TrjhWD_uyfoA zSvNZ_<8?fj!T8^yZ>w4tD>foqwXG-A#Z-M{QmAMpk7FWmkxubR;DScr{{T0hmk&&G zRdRZfadzbpS!RhLg_#dFg}*W;V0<0Vfxpilo|>9Um7ILd3^2zeb!Ol|_dA$K4ipW( z6qEPZbR~E$P`@%H{{WPddv=Y=0C=~S`||kPZ9{f>e3m z$5VPntTlHokh{q57-mI1xfgW>ym$j<*!=WCe5zK7UEK>Q?k@&N+BP6FY@2|s!ADeE z+JhsOyV`(+j$Y>b6485P3%Wl35GdSkM&GBaO#^dyrkoBayGg@+op=DuSI8q{^ZgH9 zxjXiE?#7QY$PW(RE@e*Ju>ko`f=1)V_4unzaqTj@a9I=nl5Rwd4&i_bd^Y@lZ`J*} zn4xN7s8C58I)Tez!GQLWxI2X{@=sD_CuU-S_{cyTac_m&ay|(ge@>?`a^S4pIGJ*5 z{81CTJ!l+QYQcueW^kY*F&_t)>haMkJSBLM37D|tr?)ub-bUg^@vz(X{{UB7P?|%; zq2i@O7WY(M~y?g1nJ0L|7-X}dsg z#Erlwa(JJe!VkC`4f^M1(#AQnGWRyzZNHBrbtCuKc-wzHSLB>Y9P!48W-3^3xFBrX zc~$ZKH~qTmW5y%}ST_3~J1aMqJNWkv`klgMtyXx0yVgOn@3AP}Kngj2rrVbNGMY;+ zCx*IioyoYVC66t@_aD`{L(6Tr{Q2mPB{4`5l9$wM1?p1a^Jbe6v`}7gk+G8J?OrVy$ z4qCgFm4ds6D#WQ`JcG~Ue;m5h52+ef#tS`gKqfJ~&-shtRh7vDklS!a?Z4^jH&5cE znJ1P;60#lR-HBgso+Xdz-M;?-eja>vE%cB{DH3+U1w(iPZO@S!vhC!4-Z$uXR(xPU%9c{}vQ!GM;V zEs+NZ{jJI{hFA<_NMv4KBw@0hkLo_(->*Wy;iKw)3lF0;JhZb>-C?*}HZqt(=juX)DGU3wNwXj*-*2RV0Emb@2U5 zVzpiPEY^}Z%og%`mlaCtu`=1C{AzoWTxNZbmZ#f|oktMycG11P6R7Ldsi-5+8ghlX zC&fpNn^2fr>k!znZgObFX@FO;3~iBqOn9QZuvQ7wDWja#KC0`Bm*SYoS2Z3stP@K$ zEL|Z~!xYg&G`kn4-A9dIW(f^@7FZ4U1^KlN8gNTCBO@e~Vv3SSdIp{v%u2DAdPX>j zs|4sIdJ%$JNF!TQGa-pytrnt=0F+iAprX;&F=jUbQJ3O{!ecS4} zyY0(jZk*K8 zi5={cTkT5mRG@oe2-THdJ26DU3bE_}UVBNo{XGPy^&&W@SrSW8HD@TY3wGK?k)s~gunj)5OU^ySN{ML z?^Sh-Mogxb(%M5xdUd3OGdQJ$(UcN`#TX}yb!W=9Qrc`gk{hN;K1W(hIC*L;GML0= zEC{{eEAirddHaA5jkepXT^yvChlX^O`9h(`7kz<707!N|<^X@+^>o*!qoJ%c-?FDU zG8B;+qad$v0SPa&P*VgSr8!WW)D{6}|VGO{dF^*ye;1G5p zH~oD*-ka<6%Fu>;F|cW%PU<%0yN(r04cFK=2ax-b@znNb1hy@$h!`w;cP3>6asgws zYJW%Ue_`>|_6Cg_vMiFPwndD4&mEk5oATdoD0kTGf0LqvWT?v+DkF#>5hxpMB8-9w z4168Gq1&NGtjg1xXMhePlQ7%P{{WgfZdC5R4~@U?(Bmg)Rgj82d9tZ=EKszMBPx&p z@OUWwheFK0S0vd973Cn!BOcPo+`A)Wvf)_2%F3m8Q@KAJe068S=&TktbDC^pR=PZwkVsH4 zV0I|OaCRW5BX1pFNsda>J$UMN+;uQ}8$W>qAn?yi}0R0}?hUS%(q; zWA-6`?ftsNP=6`oR`!xXWhF>bKX5lX-Dgy1ue2@Nd;e7u6M%J$KJD?5v8*re8{{T@W{++#_+o{R4`^iyR z4$eKhk>NM_*lc|7_CGwjtzzk1w~|`%G2RKlla}SXVDUUaJ_%Ab@#CXaQ)>W^a28nO zk|bnp*@H|~00(dj^60sybH;g~HWz55X$LZu$ zx(-~A`|rO}ncSvV2av6Ht20CGXDhpG9ZB4|U5VSq!2ETGm-SACe01`>oaUM`juFE+ z+#Ta8VFIJ>tLQHI9W#zug zcLlhO{{V2`JzzF@r!G^nB2uvh0CM68EET`|j=OH^9EDi;D>N~7$wJ(e74kL&9iROt z1&Z(A#I+o%h)I@%+7P8jih#J5CDd>SdHGtRhImyRhQO zHXaH7o=@AY#s2_?R)o*m#A_W-8fEIzv6_ymOLC_gpsH^o{bXW zF;=nDfRd>^ETTD*yfa7xMgWv2bQA3eSCXexN%ZssIj=QsiJE+FsJW%FRVbx3uPv&W z%xJvMGI0WvSe|Gnd7%sKsf~tW@ubo)rus#Udm4vXS+!!_%NZ&)BEv&kic3nw*+-u% ziKmXUGcwe_RMd(YIF>Rrk1>8O>(h_;Ui7N22Sv@6)GU{E7IvODA(%2p73sUPSF?AW zY1{zwHcDxE^Ya{4k04dmddAPZSw)fY*vGqXDPF`YTI>aDWHOfIl3bPSkV!j|VC6{^ z*onBc#f(-j6ee|0&ZDl>t5v-{#JPUzFVy5&i(#7F(*P)8=+-^WM`)r}S!a$VryAG# zX#0^yTn+5b$0aH~n;B~i=0guCsKEnP*<`nWCg|;Pl*zIzajGwIkx_ViNm@TO1sNf+ zEEMcaB8~;KQ!i;&twvcNv5&1Jhk*e$l39?qDLsj|j;_&Guc~cOIUoJqr?D~&?b`L4 zoY@UWVvu_u!wyF)0B6~_nLm~??=$RU3(_mBwKgg}LzasMXgccOrlgk+Vv;zWr>_(>WoeweRjA0_>yKI`QajG!h-D0UN$vgm zdp{Bx%YKJ`1-(=CKTpYahGQe??l!#{=*Z1tDf8LNwbE(U#@&+irsD1#ynBU2+jH0R z=JoF}kkuO7HL0_?Niw;dqq^U}JpJ{ZW?n^PR`A6`02nCo(le?k#cop3#<9Y)4`P5- zOis*D?fP4C#P9Mp9aeva8&_ehvcMU+@<2aOQ?oGGdw?5!eDC+)rZ3UWIO%FyjWo)i zlQhc1{#-Nh9FXiAwnxVQ0B)#XiET5Dt|GRvsId76U=E;$`DXZ5*|*#Mx)JC+;^Jmn zrYVdn>&&nqd$wZd#^OzfpFV%TO?D3}TFoO9dm``^JeCjc8@9*D{UmuHZ`4*+-_lO& z8B~wMqn`ps-;rbcx9RXk{SR}x(}Hwma6P^3)9Sc+mb~$h@<+U~k_sqd81N+c zWBkMYJh=Jkg}0>ek>e(KYeytz0%hQ;{{SU-D8zdKBYnId)zuEZCrubTkwF|Xhahu6 zLay67abdEy^SVBVZVjkd>@auT@$-PNkGMb+!g?FUG`uL4cmV| z9z1o08c5QFu6Ac&s(uH%$UBcJJbk)nboZu@CI+p?UQTR4v`s1 z?B0bvVp#l=2FLdQ0B(ue^A_L0-py#4pj-64@uw}DbK`=6 zyna9a9-?%6NhPY4tT8#1UT$nHsT)71@6e(k-osQ@*DF9 zZMORkom~g$d^)ET7D(2}lei$TRLQ@}o=SJz4ZpwnetJOv02F_REmP?>QqD#pVes~3 zg3kz)a3*P$91ua-L}PutZPJmb^yQ3}Dh}Lyk}A_ou2h|z#8hv&_6>;uc?W&^0L!3t zL~9XRlUjv}>`_SEo;v_r+8FQq_&z$bXn#&}S&czXg^IH(EX&`HR3m+l9xJkt!SL=6YBxv~oN@<=_so8I9m4dj*7V-|T9t9wQ6k3Uy*m=c zqJR*rQFjdc5$ES){W@;FG0|4A+J_1?EsaEQvRH;U{u_^%AZ0?~N{C>QW@ZQf0Ns`# zH^J-Gui+0?;qm^P($RQj)D_@%j{P*$mE@s%M38V^78*2b3{koJvT^;;li>^`M>E%FLxw zcqNg-`{QX;M&JBM;wDE3% zzODW+rTjVJzl6yiUJXN| z_p4vxBlzLcc}YD_Og^c{U@_ShM7EiHQeC0}D$riFgTppDp=2tuM=hBVg9V#@jee2; z0EiwJ@n35lR`l(p{{R&FCYiASxf^2R^=C*#exoCywdEM3cx~90q&ChQGgYOFy%%CY zj_8ARUW#hT;L}*WK|DIJH1cN?ZsqW021DaEZT%d~yX*d0rV zWc2Up7DN)+#NVTllB<}jr5l*ya!X%|mIvwHjaltPd!FIh8**N=Jq*ylR}Lm@vfQs$ zo__(7YMlpDg;S_4aB`NGfGFiCtat27KEKk_SM1V#tg2rZG zeiWi8LVXQeOEgs?uWP%+PYlLsrJm%xUOLd)jZ@l!%d@ftSXFo`Gr1?KmO~8K=EvBc ztK4Z6#Wkqm=K|Oh7;HI`6(Q8i-q;s;8?onx-CiTgX0n*exU6gjzt>f*Nny2F=7=nN zylW*G4^Ly$8Hr`Ewx`b9#>myFTPl`dS!YNk%Vg-lrYu@X zJnUkDOGv5jn}aH+OkbkDl`x8?LV8nPpo>Pamzs&Q)@(?PmE4iMn7&04IHLe8Q1NGD zG3cHCGM=N-&_Cga>P_sHQmz{Lo_;xs$!uKMU9P;;hF%Q^|7hri4upn*c-~q7T zmr9;CgzWLW08PyAAwQ|vM#K%gdE`7FKdAodO`2BR)!?rd#kj1S^DMjUzQ6)}KI3iA z+vlw9LzCwk@?>WoAx15^e+#hKcnC=I_v(=?9_mIosgg?tih@e)Kn^!M@%x>(>f=JG z5l31++x;>tO#cAO!GT2saq<_LA3i?+03BU4d|2tS;;n{Qq-b`G`Q2Q2QOUia`7GaK&O5v zM%$1#@;{rVMX3EYIxAIi@-qYu+=xx&4YO>^@frT3ZmT%2P-Cf>5ZsPv%PSxPQZ_~6 zJBUPn<{PjZeDryrTQzI}YLx`^~+XbwI)Xw zU1Fr1ox@RRCW-!xJsXGxJr;;;HP`ef}*nvvSHXCe6BiqOhjM!hsOcqkB$s#no z5FNR@6h2lnvU|PT_#O}Wy3{7C^vfrNcgfL;X|b}gc@Qf*z!>>(Sb2yao~cTmkb`C9 z=Zhb(+hRx{`PiTKC#-qv@-6K?PTO+-05^r?Kbzy{{asqLUYoUZAd?6tXk(B9-LPW~ zxb~1Ybi9#7A=t8b#ZJ*v-= zsV#XT=2p*QMB-Ka2^6N@;10)rtsbmtWvrEQf*4H86jhwCUO^G}LILx+ANSkOKrAMf zdpgY~h92Ooh$STpgyeaYa(>)@+-=h+@VIoT9p467(4!yo&BC$x#`FjnBBX8~F z$sJMj*QFT>*_OK1V0u|`QFw+fjnfJzt| zoFV(AfdCUX0sXmn^VY>b0=~E6wLS)&9HAdnYgYv(WgWX-)RX1EM|`kj`xQUarU&s~ zS5IqQ8*NrNUc_jQ{>5nrYxxyH1xS%c$AUknTBW7wKh?th+M#2!6saUVwe5gdBxS%J z-_$>#Bi-lEMb!L0Q=3Z3B$X@23`&m)jX-YTP_pu1yN*86q`oJzWWZNy+_4KTOTcW*v%Z#n<~f&8leFzaKdoQQs`8ImIW3L+u+W2jFew^8LDXvYHNs(T=H7x&(s4aSCOCats+6 zr-W{c@xJ@{`w`aFzYke&_;QXn6?VOg=AJ)^w^ikOmQNHL#w1A8wZLwUH*WXv#fr$G-iwM_Y3kID$m9~u ziFV}_QlS;3umcFy(b#c~y4eTeKk)MosLX{s~oS)q)8nNlY+?vTL;nLf?`} zNJsj`rIMZq)d59WBWD6(O8q+2TF+UmcW}70k=GM?aY_wCm0D9rUl(p! zofRt)w(|ou>kl)uhH> z)^n_x3J((@t1KA!Dedp3(uL+rUV;AriN3g>Qfb^DsTIyf4Mxz?#-?g4a!Zn$J(@G6 zbrzqwE-yKFS8m`%kt|tKcq_@sF6n>boekL?A?z6kKg(1*C>uMoeEzPj`m57@Kday~>MUJHo(7Ggo}DF$ zgkxqAIV#4_CENa>+j&fzbNHLlx-P~;If=(&yK?ZkT@3O?GGo<>L{4f?ue zy+`!&3mP8}a=mzr$Y5h4acNdCgg+MLza9Yj_}zNf9z#fWwH|7+Ni@orU7#ELhzHDoc^@8nlX(&; zA&M0Y@{SGQfM5AskbLj(r}ygDi_*68t{EbdLIGBG0P^Mh!`tO|Ujygg)V#VmDJ!he zlPpLYHeyN^JE&Oy0M*FsdbeVo{+1h1fQ0WQzlsNIs9%POE!FKla^7?sprb`-@n$OioJW8xgoYVpXC{$y* zrxsra-R>P%_1=VAe6pCq*}Q>-aiQa0LhKeqULF{R`SJGaE~FRE8l`(ecwRV|%p~R& z9Bww-_AEX}j)>5w}|6_4m-(v_?{A!>;w`vnOsoCvqcV8a7?L62r%j zJbZZRQQ-doP$QBi3>XPx*azE}7exokk>r1W>DI$N3-tOcRcjbvSs<|rKXODu9b6sv zJ014i`2G54arj2YJecT}BQ&o`#x>u#kp%@J3;zJa1UBA05!EA7VXD#8aoPbRj@pLc zqklg>8I#~B`TGvBDy}2ytkqJ1ZA8e*(XSxQuo23;5wZL2`*cOnUYy1CR%NRM3f4ea zk$l1d-)_-6n7om=2V>P6@d2SSc$y)E4E5S5`ISYJvfzRV+!Cp`-|x^<(rqlaE$6K~ zLEH-hHvsZ}Bn4ulZHWMQJM0HXtR9|4nw%`08LgR>K;AhRL*ZoG`F-F20MCwyS=~ER z6e71gYN;XyP6$*NB{qbD2Ipq;@zO{5fVpbjT!dmd6=zhf2bSO7;icuk9mf*8{r2gL z{37WcNcw*pt1GL~qOBzZ9~VJol!bdv0RI5;Hvw5sfL=D^gAKq$vugy&1x#GV%N4PD@ZI^F_Mg;yv!14)k2N?iby{`-MT4g{B1~HD>GTJ zDvEp?lr?b#6Uh^7u&(i7bM^a-AZLI0pmZ2Jl z6;3s4?^X+vnW|>1MIO-oJ1ba%UC3 zbt^h&9qQMocI_qAg9wxNQjVsjkdn;QEpfYx{?YSt#mHH5KRY+RNq#3NsuHMt2wNTA2cL(ATG>9Eq+e12N~GacG1PK}6d zStvi}S<)p~B!R+)hafn~+#UqZ+XwBj1;xQtttVo7-^0vx@v}t~#yH)Q!pxA~w*p2v zq&5^cdCup(3I0+Gd5-@88-K+VI?L9tM!yI+cC$8*)GL+M`tb#;6{5^wbmX*${5@TU z{rU1a`6wju9hI?|6JL@oxjON=_{;wQ5f8;aUr6h3!S+J_nW5;Wj`a&MtFg84S{F{? zeL%|IiW?I{jMW;t8gF+sRyCezE5T&6aYe!(a#ydzXC9CG1Fb&?k^UY%G1OkS;Qs&+ z-jx0;e+vGa{wp;0M+1s{C0|YsyPb}6(-A^k#!Brg*qawL_3U;$l*mbpr-*i80GxF$ly{ANQTh&qnNnyJLWc;h3GDsRSTk-2Po znZ|nAjh6CbvUq7Kq$>>8VltGm3|-cW8Lr7G;{|qe$$|GW4u9$B>@_|X;?l`sEUn0L zf;CwyK_gYissgVBF3~GX!Iis{k`KFW`Z#GzkY^BM1)?Uc8ozQFVu}dch;5QZV=St7 z-^uggo2E}f;8zbuOP8WwtBLFo-H_5n8h1O49x%-^y6(Vj%iHbW^p7Y%j}Js>=e=Xs zS^F|vmX4v#WhSvHp=95+6^jhfLt1CHd6){VK_ z_UT!J(fKpC^WFFw4YJ3H0hk0cFS!_>)Jl(m;EtRB071PxrKfUBQss46Vqnlq8TR(y zGy>0v)V#JeK zp-ROI^K*7;gSD~$05h*0iw+}jHXAof?gQ}IYKyIWoo}y3U&s$m};8f2*qBr}iM!kWz}gOB@qP^s_Xc_~4kR;!h#vPr?0ze@=_f_;0hQZrUpn z&`4tq9p{I`DyVJ|uN|dyKPPZD`|q=Lm6Y_WF^0jU!U$|FM(s~x0#pVJs-dHiK~+2Z zmvgZ_Q|kI_9#Zuiu+>THTl}|X2+p$y?Mq&ASN@`gW512TR$hzOy*nPC&>3AfH?la| zOL*cOMI$j%dm9a+l~{rS*cSGm^v&q4MINfFR<*cHb2BmRG(b$;NLB$r+vT?_@Cfne zqg|a2ws4Y-gA^h~*n-Ne2_O%J@HW`^{XHtW*U;3>H>I=~j|64AGv-p$@!SF9)~C zY{zZC*?K!=bn?ZU7hycUqCKP~ytLqr)bsEdZ{zmy)el-=#fq-)5%1RpkZ}rQ9DyuA z1Ht!f0qI=H<1Y)+it|jaT(n#_*_nOZ0A0M4NxYSIiVWL2G{U#A?{mtIOZ z22elDHX!_u_vysczMA7TmIm;cEUC#2nF11P>=%`H2gxK41Mkx3iTp7g5%u=#EX$6! zB(9AjH*zPJ^HobKY*dEvxcKVNi1b#?c84T?G(cvV2g*nos)PzZLWsuwRrPKNQ@duS zTl3+SO9cbU7fs?H8wXH4jsF0brIRt}#eHFtOf9m0m8^;6bsqPGIVDlPShm1#;O*AH z^hTP~x;IbO!^j||N(zxYX}~Hf)eHk3eU8O*W-e7S6%l>K zIR-HWAdUR|dF!RJ)vjF-2;&loinF^b_DeYfhR0@2l>t9~U14XiGGB)q7=j(%@;mcY zGO6XsyaC~N9zUnM@6r2A;$yuvYG{%OVTMo3J1huQ2XJ>JV647(1AV|7^jOu{EL}}8 zO2wO0s=!p&y&#S3l|T!S>ku1_zE8&c`Tgy_m+4dZEosgm2m;>uUUMM^tM&G@WS z{cXmSMtONJ#3&qxA8$q&@D*ms8PfYJMZ6DJEpnth<50pz?Lkra$PVg~2{~>0cJg|( zWapuQh6JPZHfFNCt5tz#79-#Ty^pJp{+}KrZhLMAo?T*T%=Q+ZdlJ)I6|(kn2xEpi zQr=3E!^ytBy_sN)&wW`-Mr%%`3UeqzjG-6zQYXX2gUrP6_nu`%t^&PD_9y*8dJ1t8c20bR>jD|tq7UmYC zNu`oHtlW`HZxeG5&6PTs8A`1h>1M_1?Sx0Mt-gIhTWl z_JB*t`n^|7XuVjH{Y;Wf(a@Wd*!vU5k}y8Cq<_{{$R z51U_$Nc2~#KAmUd)4siWNnLf8oygx7^7>aVSLG$`L50+KXeFa=G0HHj z&O02{*`eX>44>ei{5WCJmarNp(R^;D)LOq$O%$1(O@9ZF$767IWtK4>YIvuIzo_zo z-KMWjKH4k7oXj-Y%X?n*uhPvG_-p(=>5oXP3Rtzo;3GF7wj;g}EJC!)-*`oQm8X!+ zB>YbgsJAqcCdxWR{Y}z%d)X}hcLR@s;x=ofRC`Y(Nw>GBpVOEj4_+#3EqP~_ z7=w19`gbgRq~A{?1%4qtt_k14?*A91)oSIFPbA3bQB)(=B;<_=qQ zahSU?O&@-&EX&SKm3XAHyAlldk zeSigzg+J4%Us!()mh@#AHEs(ZVr)~-&m#;_mNic!yzARm~7jKS!dfW%2T7r|{>Z|rNeY9{krY-FG=9?@k+uKXjvJVU)Qnw7$q0K{2-=u_p6VUK6U_(sTMTP zD5XfE*$9xUcFxS(0wW*|_a~Rn&mO86?K)$y(W^3YR#;Kc7dtZ(x3_Yme<#Ty`j8qz ze2r--z1fm@q<+IL#Zg=h;o~9kx&0(<{@p10%AL!t8&TVgfT7=s20lTM?g8hwkH3S} zD#n+dv@B#z*j3~~+^Z5#H2Cn!S97@VKk|&gXnJ?l$sAh-m>DGDxo!6$3i^-B<0nUnG3>V3N!eEP_<-b{v?=8-RfaCEw&Q+n3+Q*mosj&Az0*{T}rX>vDx=PJTc;S1K@w+ z>8QqODDg{DT8S!gnH&WLSy=*tK-=yP`v6Y;9JJ1|7^^+|UCCm!S`3Y82;k*N?b(SN zz?Db2LFMC;pFK4da#(z(nXwpJjS5nE<9`ypvm(CC9Dbi``18KwZh_}h(#OcuyD`QD z2e7=nF3v+P?d6k>{{XAWEBo{i*0_|<&2H^=5%(-G2#d?KQ*>EE{gYrkkGVY!DBv>K ztgSh2R@|d9G5CUo~u`Q%5`?F@(hGP@p69l6>w) z-g?3Gnyhsz<+SaUD>XZ7Doj{{M<*P+t@?)~2gib2ufC`0s+YSrV`XV4Lmh<*#YtGt zBHwO%sN4NL1SN;~bt*ODCNldb%$sew*ah>q9f;fy{{Wv=Z8;u6vyTkVBc$`l(3E0O zZz}~MSZ}v2!TWo*`}ERNYPR93C4v!hv&O9MszUb2E6Ol^h6q{ZO3_J z*uU#t!zyx6KPVRC7zg(Ibt9;C3|I8r^lVD)!HC@otf~vJLZQaQ7C$G)M88dO_=LpN znzLnW#0XTHStz1J;XrudhLV}58-^{vw2nPmH8y%JKV!dLXDZcSdk-94#ef3il=g!d zLXt+v1iSwLZkHV2sd|R(#7L-S4F-|<#Zj)s5e_!)Ne|aS^%e|#K;M9l~4_PMjEG5F0owP}4# zidF#NKp(%L&Eu|Izu3uEkcc~XF0_^E*-*@_4=Ey-p8$nf!QYOD(C>@WxV+wP38^vk zui)Uw>Z$`wC6HxBR9wyqI+C;CDNSZy%%xm2RWfdZN@Wr-FXDSgP{nBL8k-+QZ|UeH zmReWw*$MAhN-)lXXqK6y3pQCNa?Q_mQiXXiCAtiL8$DX|mlxcB{q>NEW|Ez)TZzSg z$khj<0E<@B4ggY+k9z`l!k5AX2 z&`!8|(#cV&WwFl`^kcOY>hMD%vt@BIO8|2;n*kt%2Xrj{-q7ID-lbx+O62l1>|;$0=^w0yj*sk1XmD$i=n6uzDR0OV$|QyWd~6fvH}nEGW8 z=Khl-F>lk$;Pf(=2a#<_Qzr#oJ20gre@!+lPbh=s(gO^Ru}x!TV(qac{I=tdTRHR( z6!tL_XL7LAhRl+R_3K3(^^tpsfg}~ChzMt93%`PW+?B$QC;Xd;sLTwI$0W5LTlA#$HO3!#F8G;Mq-#GOgTwox!VzVB$XST_Z@53 z*DphA-%+%6fo5v;?4*wyR5C~TY=HL$$NBdgus#Z} zTKV<6&^^s7uJ(YtnGpdiw z9M6DuDtce@5jEwKyO`Zr$sDB=KI#xI>He+#+n>KcJ#*>hdEQ5MV1+NRw~?qly;eMTU)yVGAOw!oQaM&@n6s-B>@{ZydJ%UR_0|Oy+Xx8fkkIP27=u z$r_&txEt(Smy$j@Azs2oj)kmA8>rzNgpsz$N3g4HvnJcOpDVvW&0pIVrnQWaqsA6k z*&-;=%%_mzN6Hc#fSAZd`v)OWmzH zD%OGBJ%(VU7|*k~q<2RzgSh*6-_Kna@GSJ$1!odQ!Z1`e<^KSfarqoafIfN|P}BGE zStX{Vxw#@t+(1(rKaT(}w_s0?*mP^k=+5P2mMI*_BUgQiJOb{0>OkC`{(gF+T7p`# zO9TU%kj4-NW5QH!23}unz+t)Sa;~4bdW_Oos?w1QF(4j$aocGS9lQg+{{RGaLdWSj z@%gx9SPD0AIM5)#5QZn%8^MDFkOgc3(3a&?z@)XZ|T&AmepAMFx-eVOvY|d$tKDa0(mPdd~T!q z4yQAgs9dcbx^B_U0Deoh-KSlM17o>T%%pickJHx;E3BurkX%R6t68|BDP>2Dhm&F@ zAzR?=1A(yI{4ei2tTc4GQyE`X0Vt?pv_H}RFXvo zITRKQd?{c#NgqEYc?Bo$w^8}bbz?=Ojej<`=(WQ z+yWgv{@_Uaf;Zc$&Qo1m#AhBAcV=lKSrGwExe$kl`yNZ@{{R<6xb>XZg+aE>#XF0$ zey_0RqsGK+e0{p-m%U>lQR1isvTpp%PR)i&3F0b#kvM24@N3MmdIfBF~v$-hC^R^gNe~`Wu=Xf z%_cH@CX%^w=LnFMnzryT*&{sn9ZzWJJrUP7W~ZgSCfC}+npmE`L47*a9U*=S?xSaxF=gb!}_Cv%F88>PyJ)S=;D-TuNa>FylIC!UhEaB z+lHKrW*YTku`7!v2unDaLnJI25Snr_A#HY(77@sc2~F@2fMUXN-0LF$ed6Q0t#=TPXXG-(@oj2;^a zhtEE*mgV$@g;}!MSedcVraA1!)$9IblUPZlJbj&R^YGQ^j-Ju_D$LrKAslz@Kq%)DA@l#W4l1%h+YqdyWy+b(#{?Fwf ztF8rPAN1x3JT8Tb27-QJ$H0v;-}o- zryiE9fijie&PPq$tvn;@u~#l?8~F{GsChfLY{cfDaezS@5Iw_=?0PX^HAWjOw`mPK zrt&pwL)?zWAr|D(#~&1Z_Q|Z#vXBGDP%?X6k5-$Q%K2MVabHDR3wGK#=~o#WILg;j zU(5?!#~sMb>$m}(LYDAcb+8=;ippuNGe;9*CX!0^(s2S>jjg1vf<~=38soblUAJQz zh6DWFcIU4u&Ssw>VmQQRH^;qdv9#tg3zw3_Jbs-205D-2Zli98b__P_HM;RDNA(P2 zW>rK1M-DN>t+3zuj{Hv^L!p+jqaJ1$i&Y$Hq>#YTAVo4NFn-wKywCptr&`SQ`$0{b z!dtCkRBrr)E_f`dV+BQ!c?e5r*{;m9Vs$9*C=7znMILntHh8I}bW+pNwh;i=M zKzOuAek2jN05|IDyGL4Ovvwl>En$j0azny5aDM7|`SBn=K>HKWpHk-66{w_zTtv~a zXxnZ*#dIvHR3FqB{Qm$rC)?F>->CJLwTjYM3nYl}y0GFT4ZoLONAJiV-;Y%aIf@wx zD<$h8%+DWgQsst;fql%fb8Y z=b;{|z+~|km6D{1S-4Kpsu>`R&FqpkU_2q*F&|;EUaK0SW_on0WMYZPj7HNa4Hb~b zc;s0&_P1z!ZNInuT@NFR8Hs};(Mt+D)n*F8IUV*QGvnOcTXDbne}18{*-H5o5yE?_ zk||(=-yvn-;lbRg*zd6a05?+eWbI>GS`e!WoXCykJFpM2B=7paU$hS!f8pyUt<1%8 z3b=Z_c*x5oj!cj%FqL*5)*$#l2W~#o(RUG;IqKCCOHxT3Y9rXq#mbckjV9#oqx7AL z^VO3vr{Id7(A$;?gDswIys=0TLrR5Mllnq|#O^xAz}gfhLn|!r5arEI=zZLnu>*M5 zZyTNVJrwgAf#s)MLqB#K5wS7M!CEpGVhV%0fKJ^PMhjrQBh^%H7pRp2s5ALgot4fY@-t7a*Xq;q{6^PD3daKyuJbVL5Hj|Xo69$!6Eb>1o+IhdoHNCV^T zN5gdoXq-53#lLbxZb#eO=d1Ktj7~!b6^hQ1Zwzc12IUjZ=aC-by}gKUo`Df$;7v;` zQ%oa;=lx8`c!MVB~;i! z;Cy?x=-YPfT)v+ZE5!1~s7dfh-(kcMcH_ymKetQe`q?XuEhzGG*R@_pc>8|TJQ#q! zSI*J#+xH&ss%A{YH7uECk=_U5@;TV6s3l49zT?Kn{@o2C)tRVrO$;_&R31*Tje!V9 zCF95f+aEu_SKs}m+s9RLBfBd`j8WunOjmt?-0YF<{Hf82_QuSLeh43P(Ek8f z)Q3w{mTFKu5vK>5KMZ486rtkaACIv8hW!oc=i4t$mL@FQ7aK0yl{>QRPTOz&{(7}( z9JCb+LoigZWS7GLLxvvURkq^Wf17dp^+>&=mr`mj>#M9$I9VmG!Dj9f0OINh@xJ9) z5JPl9$Y@m4R4ZijSsAM1EXL-krvx_j7F{FQI$~(gBEtkc$qkUXB2!lOB3XaR>h+-Y zWl5_V_zKkYk{Om8c4OvgVdofjgqbtUUc@?Do}55v}#sS2rr z8~B|Q7A3t?imOKx80%HSVeiD^7Po(3?816Ci}eP!CrZAv)KxS-n#tqxI=j=~L-fRW z&q(lh^e!WXja%y%sPN!vik^vn8Sxd7jM|Sc9#2tbWtu#8kg0n8l)cmIQ`KB8=fvRa z<0;J}K0h0bq5l91L8LMZj7lq*rj3B*B)URL@HQsOMUS4u`5ixny++QUwEB}VG;voo zhp%|8KR!Cl*$hsNhRdw9Xya~w36mowSFt84taa)D)+$1_ELpYQ7EEGhZY^QUs-^2M z;yWLy^j@ITSQDtVhGzDd$YD~0EY$H=&S)8Bxk4+fMk=b-m02Bv$nO~90**Zv{{Rb~ zw&e8PTDYAndJ6c;Cly}Sn7dYtmT{4N7@>WaaIbKBwo*Anm+DW(c;e+;ESn9MWgb|m>6y&eub0PJf<{SVNS$MAaJw=} z04ige8KRI$zYWNBaI$jbz_(id^&`-{B{}TAp2B0YHY^9UV0$+%mYt+=W4#JgQ!F9m zJVbv|+)8uXZj{{KjmFvT(PqKvd(O8JAQ=G}v2M5Jtz3*3qGJt%H*;m<$PWjL4f=@F z4|7Z807Vw{3oLP@tnhTnPtW#DtACPd+vhm`0H%)zuG-Ryt-?vc$sE@r930~@PWg-;avao}YS)^_e`4D=ido!en~R%6yoJjzm@;ramjU z-~6RVyQ03Q(|F9adRckZWRhnNveV(mw(*h!4Zk`+{M7WYYn*aYIUxg%b)Ei#u{3WRL_b7u6M2uCQ2a~ux%2_!UU6_r? z1a13tO2J^O=IShX^;m3}ReNBUb!PzgBa3souw~qj3dhg$b^B9#d#JS3QRJ&E)GaEH zVnP}@VBw9b$EcLmH}Yvp4A@{{YYM zj^Q&BWAXN}eZtIz<5W}PVX=M(Z-yuR{hbQ3t#TaDwro2jy~hl!Nl?diM`rL&?Xdox z7_>L26_TxZY|inKY$I}hN6F$-=XGxh^ZR^`k2!4p3Gq@G+)7D3!b*-ZapM_MPm!_y z{Q`9^77NgS5DLGOyvobtw#&IZSp1Mj$oq8@qAx_zON*0{jf~^-5At#jhuOa*-HH8M z9lC_od6+5WDpE+1Woru0WbMj$nl1>#aqM8ORG%Z-2HW*nw~Dc5l#T9A^n{sM7T5sb zm0y_~h+nzrJ3fcZjIE0>O#;CgX*U8$xeh6^Fgu50IUlkA0Dhr0ZB|@Fa=9`zXkc=Y zktB?y0`39ue1W;}NA~DZ>A*(R1}MytMJ%$7h;q?5B#vRD@&P^%$DW(5KdAK%j~RI)hTZ>6l?g!iqbatAta3(RhDR)`zc8ha%IF*Pso(zqw^r-exaxN8hGtGgKNbPhj{%gh_cq~w z`g+XKl3%Nrn?4|h{x>5dHYXD~bpAChNep)KQw2Ps({<%8vbZSMX8!<`fr5$TTfVr+ zWHY$Sl4MrD)pmHD~EPd%YBfa z>$fw%kbI7v&qi{hm1-FsQ8d-?^Vpt_GMqLN46dliR4Mfo@%B(Nx4UUua%S-Ah zFc~JR6$<%fvjCc)H7Cf}sJ3leyLz3r(B7s&dYuhRj2_xtR7ntLJ4vycOa6a5ZS1|nHn@NJtr6I6b8%L zqO`jn!AI-?xr>*70j*f8v^dH@&~1$VP^}r{@Dla|dU%RKt?2&kOryXri0s5qn8$ z?A34KT8vfJSZLllb(<`ZOQ|q%s)%E$K6+|xXETmZ$SY>Ye$vGWM(;(+KCd31{uwFR z%GLfLttL1HZ{e~)pTu!ZT4J)o$yUew)qZ=o)KX&;hU6iv+b>%O!C>iWd|0E2lEhWf zJ4y96Cz>fFpJs`cNxPB5D-~sSF~%2lGDXRQH6PR)dW#u|p7nYd+qGh@S~?b?j#h#| zL6XI6Xo2B&R1M9HrZz3yf=JlxtA$k)+pm;JYQ<{9&eGRQGRkCzZGyFUo!ez!c;9tW zII&IFlNU77%-m&?X%Ql22^29|ndJqGaN`)+w_n`s2lVRoVidJXUhRC9Z))xWrFA@a z2*O{KjBoHhM;?JbqUnq#LYCGj+bRzzkJeb=vZDLm#~r)M!*lbuyW8i_T5kMd`dxyr zjF%^7#To&AqBe#TG>vPy62`E`j&>4{-1t3cD>voiNM z9k<)<`;tFyrD^(?avVGpLu-2RtdYwRjIoHS{%KWxx7ob^0A}bnmdWKZd9z{Xj@3)x zHpmQY#eL&w`E4fqdv^UrcKGXsoB{29Vk$7>$ww7Nc$|V1mH6B%1O8SW82vrUQp0wv z*9I8o9Bd})rsNJvyRbeMc%PFG>a4td-4HcCN(*_KP*;%yeI!yNMEnwFJC&L;hi+m& zJC7rexq64s>nnJ?VX1O({f+@AGO$L7mfMVfCNiL%hn`#gx)amM{eD6`j}&*2_`m59 z$fda>A93b3+;6z^(J3+USe83wQnFcKg=ZHzyr7(O7vLH`iD7#rW6rV)9^S3&b=;x%{?YJAH|6lbuVIY(%%^i^z3>e1E5+g^hDQNu6Hwl1m?Lj3BP@ zFdU2XQ|*1275(??A33OvoLfs0#N)v*x+IX2i@VDili`^8-)+Nx;pj?r1kzntRgyF^ zb8ZpwP7S&gUIW@eAEbQvE7e;Yr?V9l-K`wMtzoZa(agkecSg~L?>nkBWh$p3z zxmsvyix5pLYU%Rytw?Gq^lg3r8b^e)z$f+Mj!@3zQbl4^@BA@auC3k7FZ?mAzmQH7k#Eg z4i6oleYf-W>jpg~YO+1(r|h!s(!#vBG^)ggkOR2nHyi!>AYk^a@kt9Y{tO1TogNE@{OA6b?6A*jyFS6634o#G`Ho6_^_V!ZFjsbXhA|i6@d6>YEv| z_e_@V3CdQbrmD>l-nFaOriLps{*Kd43@WV(kz>VLxqj~mvYq_ldK!8umLXoIo6l;x zQ04Wmb|_2iveVKtW9U$mn=KgT?D9o~4zP8YsU2CdxS{+xWtKI9v}L1`rWYP8MW->) zdqYC272Ee^7d|!~2O{3yiP&sU z{JhR#Q)*a;)i0UfTbQL8qvJ{1i z%iB}A9AWL2L#oz($p10$0NgTMo5W{K^x0PIhBF>S(TLv$PUfR+z!X42TZOg z!z#ujjU*K1M*T8FJo`m2%I@=hRd{4m=aVaP*?w-~THIt!luO?Ofo>8-WbiU4vedAs(t;M!I0JVOj zxVZgCJ|ivk)Kh{_r>+U0wT!az2@@|$tZ*#x)me7$!6a=8G2H%4f%*i~XwlNb-|Yu+ zEveRJUD?8iuN;7QUgLWd4;{`Ymq%?McEmAPk^yP!vlM9x7$Hyw&D^tP`JZtcaT^dD z_Z<=JUpGqQnql;$cNr?iZFd6r-KLkxUppv1I;dxK)hh_@{{UHCf}!tC6mvavUm$jy zbOpB94?aELJ!^0AP3Z*;wl7d*YFd{dUJ1UXEm`hGVs${s?!0!5cj6AfaXvcp!RmcF z^|SYM`KwG>c9kWmP0-hg6#L#r9_5o@Jbdmq=^*t}NM`h%Vym(C@j9! z<~wI#%-#==^8Wyv$4mAf)R-;Xs}xA^Ad`55yKM^)1Ea6VZbcb*{0;vA&9_BG*LeNp zip0A&XWZL_i5Ttf-*dY2J23veM9r#h*@Ven!`F&Nb?!58Rd(W_l?UT=@HX?fZiybB zYAaH01=$*C<1IBOmPPuA(Fkc_NY9oC+p_rc8-GrweNMmpLlE~UayKAJUPx3%Ie@HO zY~LKCAdf#jIt}RUX$6x_EJ)GJtkLj!0OSKM;fWg+<-~$Kjwhr3KT`cTcwk{Ha;OU< z0VxXMOGZKciElpu4zNi?7)<#>1F~<$9^vXdxn4kwh{xujU>90GD#xef+N7Lh35=$69GIym;DPF0m-& zSva57c8p3FykiUlOc<4M}3H1Kzw}lY4s~9 zim`!LMnrKuZ4|7#?iiG1+<~#*hTDGMjiW0yhR>-V~W$+c6K*0e!{iF&%e?0>|LG<$_k)zt1 zyb$b#zI^TA>`DIJE!KTC$yLku!aP>tc(2f~JaL{j1}2S&1Q1mJ075q(&C|7~>&$fi zIL*d6q=cJozc{|BKoi0S-fOc4@OMxT--`55(>{tmzr)9G9~5#T?|zw55@@Wz`%PPI z_A0KdK71dao|tTYjEbg>$xz5w`i!0`r14H2IXXI^k}190{{SHeb^caA<>@_a%*&`D zp=P&kYON>ILffo&D>RPNNV9M53|r%Kw;h1nqC|^N7SEFo9+dI0bqJx=nl&zja`v|p z!}dPjq_CQ&9W>Re?zXGNwA@bJ(2O~i;-};)`*!=U9lBy`tnkg#>VC2)_Vs(Ve~9^^vA2jD^Z^PmPsXtwUStUKwQ;p|LtcELBlYaM)<%K0E++ zV%-_Fp+`%(2FERvES42#v1B=DAd*NasfaGn%^hToeaF}ka`yqyMjZ6=;^&WuTJ0K#vYB|_Hh#cBwv z;&E`R@s5;HtpsK~ecjJ@>E+i|RllCh*s*1vdofbTWh`Bgcezdzw+h1>0o+i|%C6jb zv??M27p~~BRw=^PRfVg9YWRwki4w;qcICJ#)uI`I??+z0ap&9SM*UlCIy&ji7idj9 z7I@Dh2aO-L5ci?9hWw4NApzmRLHQnkZi@P6AN)SVz(A2ev8|Icg1n-V+d(Ud9C+e; zN001x_+7eVbQYw%Z%JZdv11*zyOX_CH&FCPuO(61v*TC3~p5agBI= zMJrM{{{X8S6@8^$my;AcUw*H4wS~2_NY#ql(8x=6zt*g89Q>iT960*YD~;i`9Dzp9 z;o=Uwy~)+a9Y=mhCWS3gS+{1bKMw3`Y7-k+f@s+CSrdK_=k&Vs_7<+R!Z!J#Lk|s)(3rwa9mmo-HjKaJc7LFNx=v#6gm&(-4R-Sll*0ExC@4;>; ztJSe)f`KJCaw`;@mWnxiI6HDjdeRjIdhgWIh_JUTy!N7nR#rJGOD8-$!4cz@#8D>m zW7wiJvi98ym3bm~`JR`|-X7E!*HYo&d}dmNQDZX|AQD-jC3_Y722;(qmlI-~qA25I z%C{7SoTt-lEQ<{~-enbR@#4=Lk@X2wD{PTO^QhnymHl4f$IEnN(~!Ki(v_Ob{JVTo z&C=?d@9%M404~g1Wgb90dNb+e>?}anqfvtXPjr`zMt!{26+F2X3FXeyz6gq{vApw&lO+`8#Y!$?0js=y+_;%7K+^S=pL0q2pFZ-H60!P61k@3+J7mJP>f?2rQj^DSsHG)WO9{X#0V**75_jErDF<(!TXE{kjMmmQEk;GDvrgTS zv|AGXd=(xJ$Clr}F01tPHbpW);f<0eX*-I8V;gxpaSOVUyApiw@zFsHnrgjU%Oh9_ z6{L~;&_c2VhKckWS8o7WaQmx&Z1eHIu_g_bL{#kD5>(H1JX~ z?tCKcu>^QK?YB|5lVUrtx6$)UUBpS(iz(cc{T#i_r+v30`M2@YQHRoX@8DrcAxWY! z^R;1AnO1GT=_w#>5O0t^LF$~|o#X!ijY0G*uh^0&a0(TX=3Y!Vli~P}zrZ8+=oRU{ ze%6`EQdg0Zt%JKeasWuAkU{w3M}z+Who-+jIJJkXsBzM%y9pBZd8m$PWl$!8_9t-Q zZ|BG>%cv}Ne-WrL)@a=Aq;rBfuH36j9?Vdv`DQzC+iwf~`d#%_2P88y%Z$j%^+7S0 zjhSUJnR#)3ByJaP{Q4{4>Vt=hH=SL=4pK*pG8`E%hDjV{cEEYOG@t7<;@y?a31l zJc;)B8;$&QDAby&&5B?$M(HAN0FPt0B@B!d1Gk3x{=eU=&!*K^D_*INNn^3vy3_@IMQr9;hrZuHxS^j7GizAcL#r-jZo)kPOjtXgY1 zet1SSvrS$W42X9jk040o;1*y%uE(dR@aL{h4s6z%w^YYk%O+cOwQVa92FX0zl>RWX@bLbgU7{gJ2!y4w+j1<(?#cJft^yQKG6W@_iJ zAKQH5ZI2`V?cSQN7CQYFsR5~wTa}}E?@b(Sz*??0m$FaNZr)NT)H8j``0C53 zDo1WPv9aW=+nPCNZ>3CaX<+Y3G^}GbBx3}#ljn~X05RpwhW!gy1!}Jpk`K?3Ocj-- zjbx01OHULE+#!KNorpw6KcF725#Xy-j{!X4_sC%GW2L`+%ztuWw+9vkt9VZ%NIm zwH^*UXmO9i-WO}#o)Zmsy*$q(*I|N046y^{8UFw^-H`0OSEotz9JK7>vGi3M)aF}u z-9nkfHu;5;P^9ebAd!au00C7;xOIKhK9?iB9#5`<#b;J%VV<`nXqawf zqRlF9e@g!VHpiD!8CWb(qa@23_TYxZ(<84TcAbGcC74t*axvLe9MgAJ2&H-&=QW-J zIu?@EjiB1R_UzVnYf*h#D32vN0*y38G%wwY+y?06dz4l*{$EhxGZA91!Hv{(W?5&2 zF?nd4Ef(&OOPI!0y8X#ziWw+cYP&Ry^%+&m5Ez@KGIVs6%M>Ay{TFm4FE)AL35}#TR5F6vj@bMz%jACFg=mHSAHhZYt<#eJ!|y)S;G# zknC5UBP6iPx|kiEQcl~H=tXBtVzN-pO|o;rbt0B5R%WrWURkQeYqNh&>J%;`8%(g8 z?&|2_Lm3bh>al-I-o^&EVJ*AT-U*U3&F3)!3rWh$4{H?W0AN7=-DF}xdvsQ$a8667^w?>}#kC1)&o zcNu-#C@R0r*OEW+0Q8GM>5T2`c)C=ymO=|QWsaPJ^_ctoxB5xrVpcuRjzH``)7GQ( z-jICWR?j0Gw#>l8COK7vfe3%n!1otmOK-UxdMR7dx`$6fw<%yIy+Dq0O-_`OO$hFT z&FiTw(UIVlAFJeaJj~rWX+EbOy~n5GU&|tGqLsa}1OEU;mPO*guPuW6cIvTfQo~NH z(?Bddkrr!JLMt>*t+cJ-f#NsZe&_!HFy5%mRd?8QOZlyN(m zII{$7-q!&tMZmjN|`!BoF> zw^^LG5$AE?O0ga`*#7|YblYiMEnO>(ORzFJ#Cv;AJ>j_V3ch{>jG*pzA3Y4UUZu%c zy%cRVrddR<4)z;}VCuUDU!9R#Z`}U?ZiwETA+i@S9M&ycgvDBHYwo*-^WBj{cvnVd zjPJ6tGW1K-xcZc$tvnUlSIBn-nxU3NiBVQZb|jLVPr=*d9)S5=TzDL~w7mX>WJ@Ll z-?pKcyGOUk46HZ%Z|>;TfY-Gx^J~ac{}*qrFU0Qk4d}?@+++M)T9uCt{a!g3^zXG_Vd5=_@2|g zrxA#JTgpPnIH_(Wc@-cd0Czk5ay?b@S!?*{=D$Ikd3dzZ?#4FYcTOq};k@xXe}C?J z^gl*q{Z+-!X2LVul$m0dFJiC^!PDHncH#l}Jd^tMP1i}A(weHy0?aVdt4hhO^<|kP zhRY5ySRm{Tb{iM|{J&SpI~$@R)p)oq&n#m#QaHoGunA^18#r}U_eMble0)CrN#So% z(U&XM&cxDS<*>D_N>`D_KX9*b6Spehw#XBJhXni3ywYavG5c8`?iD(_$Oima_cINtBXTwfr6T46>=9IY6Y00*09?o)t_;{ z(>oO^Nbupd`SN1V`gxhvYIRO6)*H2}L@ZICes)qIdx(R7EH)c{`#0~<>MXXP$9+sy z&tYsrkWB;b0x}K|l%D(X4c6boO!1nLIfWUsQcU{^C)HTEA#apt=P1ZorIF4nORYoep zIiqD8?noo|W&QepJwD0da?MWT*RgzyRe8&jzbaLzjdBwjAE!=MCOfvt;Otv+y)dhp z)A;JM!YI_HpcyaOADdQ>dPYcTC1|Uuj@-K`72{{KJ2&S{k%EmEM+u4<0rd36%(aO) zTBy)SEu`oSp=?JpGEj;N+)J5WIMJkS&mb46$^ActuZ-?e#cYj;YvpcAreublYL(}L zb){0vY*GY0wCvGA2M(s^<%zGp%Bno)bS7juV?_~l#{I^GcPw>&r zcO#jSHTs-=YWUdVXg@XDSYuS}a|d`}3G=4jJ|n~x)lGXDUV zrV|C8t7MpNM@mGu9Llo5R$9|ak}*d~b_2H2gmR>YR(>t_>c6gOVHXWUFNvu;rE1nz zdbq0+!(s|CJKeJBS_x64j39ndsO}DFw@bD|)jKAc`dhZCy@YvSYg0`W)GXa{qqQB+ zWHC!%g>`RlCHEq+<C-KQm0T5{5Q-%lMN624l6yH=26vM@=zb1+Jhad_PW#SuxB)8M6yg5_B1 zT4awkYLiE2B?URFHOdPlQ>?j+C3%UGwb|vT3SFueNW1Ed&xENd{fN(1k0)+=cju#0 z!Gfl95Gp$3ttL(Tp-FiJBC*S{R{8CaTdZ100bk z{EG^Js|ekpU=*j$$~47gXk;gl{SBIP#O4N_t1NK`+>Q^p-S_?W{{1joVjHkiic||~ zCP5QJ68@xS19jYhiU9Mr&;D+RQ%O8kM3ciDX%@$@jm^LZa7=|*MmOX8o%*S2c?L%n zWnsBqorz04EYZ5$-F7Pnba+V0jgI~ZJ$Z=#02Lfw8FBdwQrfQBStGJ8Q?p;R;fQZA z#DFxB`F)%2dh=DFb=-6>*!~p#n>dT+MXxwyOBL2t5(u5V#lGyn=`0H?b?A@qO{cvh z_0J1NottzP#rQi?ncg_wt(W#ZPC*Y|b(DoW@4wH+&Cq}G@%TtzM{moh=;LwssLE8F z<|3R`#P$60{w4J4q11R;2?U*lp}soO4y@CZ=gCcWfRwi-OEr-9V>fVUrg~3e=cjiKF+kgq! z>^9u>U%j8THMGJ-Fp*wmM~z9?qaSLsZL%wTe19h{rEcgvyws)@qF($w&0`}s2^8$i zIqm-dSP`%v4Zl@wEq7CC`|`xgJ}$u~wZUVWH1%a-c}82uC6{+rUydJPzgMXBZJBbM zZT)?hHHxIgQ?`ZMc9D@!oZ`qm9T8P(*A)7lV1r@S~aAROejynuD z+1Y^D{2l!H>1$c*dW(Xon6tY`gO1Eg1!db{OhjzE4agq^dHZxB&pmBFZq;~#3G0=T zIbmYFgrB8ZejSIOpFZxtQ`Yd$4`U)@c&#A_{{RaSKo1e*C}72Z`g(N#00thm;&55q zu9SJAb&r-$8zU77D=!qt(nGi#y9AMc)wuigK>SX&o6YJPy1N{aM;+*$Z&e&fX@~3F zdH5T6`~Lv9O5TX|g55l#hG?H?{u zy+UBQ8oj>9m!l%_PEthw0Is`o_aC#qVBBbfK?<`=F8)w}8$6CJB_!`=SHn*-yURvt%Rg+NV}LU5gT;aj{Wi?NhQ#5KNu1+mM~fF{%06 ztXnwzMn>g}Fi8@lR|UARA!d`5p?CAJW;?FlPTO@!$Z6YIDp#Hwa$FTl2Qi;}1}(7d z?oP@{8~($ntr=e@hspaHOgzHQyty6#?b=jhw>2I%-+%k`#Wd1g!p&N_DPl4uuI?lu zj>-Y^;BCIgfA9Ua3Fv;SlT`X%m~5)AJ9ud##@m->acTu<>|?EbY{_CK5wY8G$KCpR z*Ke%;m`#_rV>f_}V9(?3GlTzeI^AHS6(d2Wk3QnKU)T2KjWDFr#WSmjB9E5c<9_Ps$ z7c6O4kmTFs6{vzN?f?N&@|mJ0%Aw^|zD-IZ)e z9EoJ}GkcUIe41LVYhIvcF}is*j{O9sVhqecXF-;b8}pr1)7z`sd^gg&8mE|+Sv7Mey>WhhGP?g$Ot1M*g#1k6;(m*Z0t9%|f*)3UfCylo(ahiAIup);2 zSsE{@#t9Rn@)RNYyu-b7xF2l>Ugb~SVm5JmhdoZhVc-fiHI9N97z7&nR zkjGXX^rIY+y=zt{zmLn&vpdNkSV9Px#l}klIpna`D*pi0kGvfMu=kYlIP8KoY<;$c z@8hhSRICdP9LZGBmCRLaMCfc&nHo7GD8AR&f$namu{3{;(7*NdEYZ(nY6=re2sQGz zadT}XGOSi&SaLAKWJbjCR{3QKCm`3|F=`sPzu}oznhcZ)^+M#ZwHLPX?E+hpl%-bP zS)(f%o(E$a0P>-3ptXKgX+b5JCYC#Ltclx!97k{+#{&KuHUxW!8e$9|ekE0(=7K_*5_Y|T0V$pf`#1rHM> zh`t1YfcV(3K2)EAV8~Zf62`MUPNjsfMAEPOt|#W9uy zfeg&M?&iOJf9tQ3J_hHnKA+-2Qx~gnnR>L1L6m`IMj06EMUd}C8a5$>LE~UK5S~$j zb>r*QtQJ>7dVyhVh@`J}Sp{ZD_O8%)a8N4yWHaykjruo!AHNW}&qei?H&kM+*<5$j zM7oC$8_4Nyk8x4$S6JJ{xZ?5;%c}6-mma+|OQ>{r;=9o*RnDOPB}+GjM#kkU*ou2q z>=+A06z)9p1Z<_gRPcU3Z5CYoFzRgsuQOUdQN<*brM+kPMx%}+-77qdy^A||=A&|> z!EiYI{EWR<^h;UjXw8>|U&OOWd$9wwyvsBp__p<>Dy%^G2gmB!dIC)qOYq2MjbXJA zhhb8trg<}x;WJXY<$Hi)r@OmJcBni%X&Lg<#!Dga z#gy;;dSW#TO1?^|g01;k4Wn7OnMq@_N>}uf&E$=?+R{1zLzUzVWDmd41-dLiEua#e=Kc+jzeMJM%?~uW9NS$a`^GlE7PqvlhZnf8#uQlGMff$W#pBE zqwrqkdB-8iPtWw}tMvvf#hg|?<;yO$yJ6*S=FGf71)qr=0}?-{jxEp&PGJ6$*vV46 zX0}EfKUa^n%fA91Y~87&@=BeDk>hSk!_Q4VD_0#wjFOM=(OP-v_HV}4L(}hV%1#W+ zv!@^(JaXYK-5EV4(%5gNO7_ug_GU3mB=%UZ+y$!`f^WASwpRP`UvE~7U*f`F!(ObD z8kN%RA&_*MNe{SU?W1y<9_az>#V;05zUlYn)pOLpU+8`yS_=@tkiRZJBV#9MOA}8F z_5Pg3wop8lZwY8)R)<5m^RBG3_@()RF5E!cKh{SY!<1-W8MW>C5d>MB#&f|Tmd-;`yj|!b7Vps?HxB2`#E2t#nMD*A+fxF#4gq@8D(367Hex~(>MYYl znk2IWjhi5~C3w4ww&g-Qbup98#|l!^c;4me-$^6hg1p6SEemduBa6H#mh$IwQ^`7C9_a~nOu27!nLrEz{pj%N8u>#x3tLx zTJx+GSc5PYWUX!(ds3%n1MJBfAWa8?0A7rSj)#kQm>wDPDmNq-TPYj834Uv_gy;99YXr(x_JZ%0Br!n;@)<_N0BqlYt=0ap* zcX;+-0olCle&2qr5yI7BNa?RQc#=%>oDET&*p4mE42GGYwj@5 zEZyi>vWH;H#2FWTw_gB%ZMx9k_<`z8OJP=7>*?E7BwH3JH7fAMced@_WcrHxGO%&E zN8FFo*Pi`pp>P`7t^4_E5mmJm(xVX(4+!(+J)3|}+<5WQSFCWD?Jty$7@&u@0!E$4 z`2?0BzZG@m?{dw*&+6NvXP}>pjQ8QgHfDz9t9l0Q&UuUs(-|SgiKYV0ym$n#i1#;W z!1lK8pQ*=N&Bgk=>8GrEN{*|eW{yuAm!OU;86oen5r$ogjp4{wW>)d!r%K+s$_+;b zLrLC}yp&4JF&PMXO42ADzSPjN?!{!`zn%9UI@iakewt*l_a=xW7UQn1H`Y}rnJfBv zqaDW<9)5mXrQab!<})P>{{T{I$~TfLb@y2+SdVFvNXFxF%YBq>us5Vmh zn&W6g1d=`8A!^&Rk+RQOWU<7ok@n+J;QLFi{?+K8qxBANM&U7doE_Ac7gEGx82Pgr zwdOAJz^nR1Ct%1Agf8CgiuKj%PE<>UzmYE+T(Z6XYZ2mDI{Q_C-+>#AfZQKyj)Gc8 z3riwwLz0<|*+NSNi1;#5%+q>bteZB-+) z8<^NkeEzWi00U;-TW|O4uM4K=*#?bT4QR9uyr5{Be%kzrP&ejCtK~@Dh#ua89=YfY z?uo>l+;Ca7fwCkL+EtLf7Lk0%VLn2im3_WW#Bb7Gv)YE#d^s(IkWE#GCPj(%GDPY& z+CmDQhmEEG08dSRiOYWx#X|(*TO7hiXzW876a(7i2Fl8I{{ZwSZ0%8E_B$10^I5Us z%6@M##DV5@EX0yOZ@>5GAB4$k`Zexd##NIcN-GUwvU1=Gjrj0Te6dwixj)Q)I&r-Z z^+h!)}o&tjHFf2!?O zLE1^x_i0RDrCp1Aqwo(dOV--`3)Gn1QKJ5ZG7+VavBQ3q$p~z1=-lzK8rO^#qQ{Fz z8!-&+u?LS=eK%$tef#re^CCo{-HkDoD#%Rgp|+nXaZ`I2NH4~?l~o!Ro1m9ZGQs4Uq|$?eNO2&xv8s?A~Bk6AT4nr7yj znw@>8n!UN|#eP!v^0QY%Zr%H??4p(~lC^$qx@nYWADQ1t5ceNRsfs2WJ*Yw}#b z4q|Ctr@f7aNnh%*u}{VAeO5suRkBnp;eRh^$3CCUIRLK>TtXBn7B?SK%~g;7T8>$H z+=f;}AonZ1o>Ow&o@s~aL~e|joTEH&FL5os9OfRhas%SyLL#aeHXh}5uQ!qa;;MgF zE~>h+)r@vximb+LY!~apyUGwyQ6ZMKm=GP}LX&Ra18Ceg$5A?{GFF-xrG;V2-L)`B zZR(nH`Kxk1p!&6JeQADVV0~V;|!u}(bKvz+G<{DE?SLvppN~3v24O_ z-Kx?>JS2eI?w$_yd@f2`(>^kdrL|%^P+G2JA#xStWHP}j-H8X)@pz|##6=Oyz_Txs z>(d*j>0#URP=B+5icob>3JlNgn1ok=Q&g0Y(xPW%rTMLr<(!Wn${@6p^4> z^7HKnJ|#%zjO^uou8F?PJTJSeggZ}S2&&qZYfZu8d1M@^SXE_+!t4~uQP+|%KHwYt z-Crt5f*TQAtf^wLPpYF@Sxs!TWPV;-6$Ff^v%&}b%qqz3>~!buRlZ{K1u^l*l=@7S zJ5KT9#oLV7%$O?W{)*J?vw zr#q)hL z`$tRv0OA|ae8cJ4^LYA`0US*<77`|BMQF(?N{yMnM&s@C)}R`fsCf*{yaM8H%aHoc z%rzr~Af7PgL#jC0p;z)w`*|NdJKl;XsUDWYS`>S7(3zsKn8gV!8s63|Gt*xkxrA)P z+qW(rSXc8L$Tk< z8#v#iAEp>=u)@3)=2J356U`wk$V&iP7LC7A+rjzRFZAmZQNfdfTX9{h8&q^iperHW zPQ-4Yjze}Wzn?0+ep{;#rW&sPhK5E&j!52EWqNb7N8PMEPgWAYp~n1_jkfM_>fx@d z=}%PmMU7S|>nIB}U8kusZsIu=hqiT`?g%?J-ro(>Ee}ufSn5iUqR#0MMA_I)XNXlU z6l^!xAw0J5e0**zQ&+Jk)njbL?J~$?bagDz5QLQjeYlVa`SQct^U#)_GSy7g+1BK> zI}Wo$9}=jri5r-YpN8T_;1UNG{{T-|zM~}58jnl~q@L04m{=9}q1~J**fVl%yn)Dd z)rr!X)vA{H-K^=`B}+RX`vJbdh2VeKj*MT2uS>I%)txU1p&aCw9Uq?UJCXAGm|0tH zK=5uO!3@6MTlG=(evY8UdY6XAW3R^8Or*jblze1TmtsL8NY3A;W-7ptzDC_yHC#(` z_(Uz4Xk9EM?gblp)NA;r>tFszq;Ii-xd~Xe=j;H|1+Cs?w{{Sky zoPEFY=wInDxsA)z?b+@xVNIAhZS0d|cZh}Z>FwY59XZ)t9uF;Ar}0IF>&~G?nU#O) zoq!V(4{WeL0rr;tJqYy=q&TWFNsz}&B~dhgRK2fsHn}RNIS4x|0tVY})NiBOC;l(i zEr!d+sZYF8TWVFMtadF@cjSG`-c$gci7FRvnSD*^rXW1}FE;e#GXpJqF?fx;tg)=} zC(7+19D(=)lKJUR^}q0`o6#C}O?jj3*@GvDl)h3r+`*AX@x1Z0dvVz}VBgLM_33x? zkJJq1n}5S|k{51n|=!iQ3{J;0;vrh>PMus}G7@xGo zQ*IGSBx~guCywFU^&da&*G{aDCz;2sXz0t?`gV-X5bh#OssQGyH)h+~e0cHIU(*d` zO9-u}HAE4`EqSF&g_Demk}>Vhz>ji*M*jeCI%p@-+7{-QmexAWG{0iCf+ei{=UYk$ z5zEBv(eA^6+w{8r+jOmIZ8c*fm6IgV8*7bXffFCQ>eF8dx{?w`sQV4kgQQ31x-u^EAYbT=eN zts+jgA*IW;G@+Sac>z{tK+4=tQ1*2`Z&l>^VC+j@FukfbHC|B1JQ{u`cN5N&Nf?f8 zKub(jluotB7~zOIsYyziN;Wb#Wt!dTF{_V*ON{PHRGTA~p%XQHozV@!@JyhIzw3 z=H9U&bwkhC?CEP*39P*gY}aj9(!I(#EQ^(Frl{8$&gF{|HE?+(>}+?XkI1bG#}De} zRHddMinMm^*qssR>L241Rfc4KbturDl0y`z0E#ne+On?4aJ<)!-CJ_lN?BZ8oRBfg zF?-ffH~lBa*KY9Sgl@2_LUtZ~rI-BORP_cOE#V&@VOBV$wUVBsBn7+$xSigzZ^)Tq zSQoSRak`%?3*+xeNI0iBxH!Ls_|%#fQ{MZyzPKo{GG4I^D;Z;}tDC9?-IoN(rrR zh+!gy8$2P@p>AD6YD}@Li)y7fa&;{!hReuWsywK7Xjq>#GC-ARG>3V6r$| zgwU*Y^7UFs>`3dpP#A{QYTW^h z(|V1s*RN1ZLU|ziii;C8m7{V|LqC!2Dhqw~{%){y`KnhbcI(LX#%8b88`qv#3*Lm{BQ7jx@0q$eCBf37tc^2R$q zLarJmJ@#{RwyeRTLJDrGK_E8eNc|`6(a%lmn-OCYEP_Dx9lLcOi^(uXP~C=fE#YKu zY5S{rBacMfXQuhxVN!g>EkAbsd1Deilx4@cav7y+vNIKub(LIj_ZBPn>3j7T(~Rvf z@J=!!`X3c1h*Q4=A&Uhtf!nzmMQzU|J15_&hA#||(zz9?k6X07AG!-l(+!UNw;;wy zZaWe-KdY!_^%|5CHZoZw_czl`6R?f>4UBRrVYco2dE9yFrMInXXh<4KYkyDuMvoaSfYqnQExM5;lO5rCik4VG;@n;)+^O6T{Kz^#*UOs24IVowQ&=(O#$+`lB>gz!DkQ3{6pbF=n05nlPQ!n8 z-5Gr^^&($T4p&TNp^lzHHkOk_=)y}08t$_|T>E}dpOTyY-Er19%~hshaXsrsjzM0o zI>!?vODZY<0G5?d?Hzze+vI(^3HoKObyl0!iJC|x?N#3!nLr*`tyh`9Dt~A5$12An zw&a4lFV9U}`tv=V)lq5-b@n*=k)LD(N4c=tD(W~23*&|12C zB;w{bAzfiI%d-ox71|>zk?)yPiV}9+W8>1%ti375q73#D1+OKCmE$$QUf5ztj1b!g z*_YaU5&YZrZ1k>Kv6%aVC?c9vnU!8$`G(wW9ErC5u;owh?)LSCo@DyV^K7;5B=T66 zzU{(D-`b>c+qnT!IFa$zXQvq4Mt&=i<|K_L?=(?~f(CfxNZWwtw-L;d{Yv#asq`Kr zO~Nv?`88Q&9mtR$`F00kBxWbRTR+xr-w1AX`8I$-N$DPpQrxjMC3?8V%g zrIrYi7V^&nk@<3;lgo8K8});d)7n2>vfS;}h|Iu9;q2Ky>A%Mif zyb(mMU8QbJHbvcw73I4#h`kw#shiZ24Xd{rwjiCgMK5;b{J7gHlm zGHnW?cxto`5-mJrHDyoYxhBa?dpUb!4jHnY+nMZhPBn;c#irx@rMfs{`gU@aX^~Gx@oKZ3E%`Q-J<_{hy+bwHH?9+VZxNLElgmF zw6O!e^1e=aHXpMyjrvyd^`*qiP8?K_n&K%l)z+kfMXiv#l8$Lv8HfzVOEwip>5*gz z+>>rISa-wa=@{b=^-@+Z43$p^tVImXR$BEd^WH0T=9RYJ+axn_1(YumZj`w;Ga%mU z7GhXb8*d3QUv^EaWa_hyQCT4NUMK6EKGUFPJ3|)E415tXwNE@ zXh8#U&uIdvBbQy6;Waw7s?Su(S|&4nD@<>gk(eZlkFexC^!9E@%-zIx;re*>k(6m7 zs`(8oX_CZJo>N66s8_V2My-^_%kaTY!L}&saAt*nB3dt6pTjM4g;& zZkSM(rJ6n}T0-{-6f5VaQ_-qLZc-w| z&`3#h^ypb+0C)%_PCQZeB%vIK+ouSsot2rk8#}NJj7uH6sOsL`iTD2iF2`fnQ}OnR z5UhNho(vi;3`p(G06qgMkPq40$DX?GRI7Hk(^*ks2{s`Qd_N+gmPp(lP`ZHR5PiLC zH}OF%`g%*LTR&4!;%nBt4Lgk^O^#j#ja#lg(Z~+nxep~JM<731`SAQtYRX!J5}K0_ zO|2~`Umo&zly)|e>@=>-OP0Fst{$aoD4L{jEpjr?DPT;ZMj^e;@C8tsuCF8kZ zz%qcG109bRFZ@)wf+=+DXpVt)Y^Af=MksJM*3LiDTx2l|b?0l=qn1Py%CJK;p+|4K7R|EA(T&3H zKtHSR)tS9H)Hs+wjZ6GG5spN=1}u(o+@@q%dvnf?c$rg>UuohA@)$2$k^DC4Ed}b& zqAwLQUhX)8L#=M zB9S1X&umLHb%@+{&yFs9AJjqd`}Diy^Hp^0Qmv>thEHrVvYu*;=Zf(gxCl2>`hoWE z()FFkVD$z>-L8Qga3m-TAlaGuKc8!j_F#MhzTRCP>F4X>GXZi9^-;7dEg2m0683g7 z#Vns@3GR>l!+*a|x8UQ_sy@HuwcV_AD+NqOK6=xb6++MTDjSH2d?TZT-29%kC-|lG zLUr}GUFnRLNiEH$ZY4T0OA~JuNdwu67UShuGoL;;ETy_ybmoNA`dZDGj~6wJXK^H& zS|Mya8-MYptFVwF5xHayxNVz{j-6jcr^mFY`H0q9my}eNxlhzs_>}_<*Z%-7)%`jl z>wix){x3IMJErIQEP@zqV{JyeIg%60w`3;8`5@SzxMHo#ewbU*XiaPBrK~n;7{0dc zN1j-#$QFEdYNULfD$2eF>=sD5aU58qA8$!MmeY96Ic~&ulsJ-mK#9i*e<|08a1pw> zQb_US^80Po_e|c$E=tW`(5z&NILR*`X;_bG`Bqco#=wW@Rj%+^w9Q8F#_vfTr=@x| ze%TCoBlWf9mwXmjytLQpfUy^eEYondKXPr z%in8MWGfYaeY&x!2_i&CHD*5(`6W-=&sVA%GN!YlwW6FX<{lO1KmJt*GE~IG4|HNe z%0c)lKg_)V{X^3E{{T;}4KYxMJki|;1T2>jyr?U<5*xd zmLiw%NhE6(Sy z=rbU}op_)^J#t7wO zj-+L5m(z-_vgPwxhenYCS28e=m}0LNvN6tKxi&nDTGZJ1DK)BkZwDI1r*w8cDC5d% z42_o}rMq;{GgGT(W@jDV<;!g{$znu{iuA|ebJ0O+%W5k%?AFuLe+#!ZK37dQM==)d zdmBo_9kX&rY>4Kv4+ownNP$K1d!?&!`ifk9RGz}aPQ+F0S*((YayO=tDpAJMFSog} z5$Ws#%Hrw#cOlp;F7M4j&@~%tWw5HU+b$^P&m~VZdD540W{3N zsIL814ax9YZ!b-&&2B7GVZL<33mSW3VS?0gvlXaJa4oAb z!*#B8-5Zsdff;be8>ydT<9#LbwKayi|{GX88)#v-vAF)X6MAEv-oR*BKLx5}Nz z9*J0MN+{SlD=}sH#?`JvltxsuRuD$lIioF`2_GHVxU#>wNaf-(g(+)fC2Sr^EaSNr zy`?i&Y{zaGkGeTAm+VI3cuM$H5-YCiId})F$<@%}tye*6thH`eQ6ZQgs)35KA|KYW zDECImCeQBhS$eJI^%NgdS1(ePTeTxB_uA5hauIPWBFXu3ab4M4 z^-(oev4P=MrN+%Uc{QAM=*Jvvvp*9xc@>yP41te?*;|(`0lLD%Y459*EMcd+jaw(Z zEOMy3jt=a*D;1mB8EkMx+iw*OaQM&xe=a*%AG}r=GZg2N5s1jD4}~J(d7&qDb~CBq4pkk+OmA>!Rhl z^(xt(^>Ug% z?TC2A8I2V#R+dw>tY7K@iP$S~SKqBZ>b*Cia^8|>t>b5=P1*Bxr-rSF?c=AMcXpPO z2Q3+3e5r0D`AF-{?v>29raq%*E!%~xQkAkWI+DNg^47I0SGN?hY&>vuZ}Tjhc-x{b zW7PZys6!bAz#~ZKl}V91TqKP*?ebzaK6cypUaxvuzMF(B)sW5GEEsOB8xyzcUS7g| zpo6yCH%Dr^M@?%i9f|9}&`go7dE!pW%T3u2ZOBjaqI+1LEEkFM)NY6TI^$x>Nt4vs zLwgrV#L`5ki#$N`JA?8<^84{%Tlec^x-;<|_+j*iNDMT1ToPqy)mX(>s~gQ=kLK<6ByG2jwS)MF{7qE#FDr8yq_C1=;ITQ5y@!op1H24Hn2=bgE#u>1?)_a3eK_=Q zSk$v$SWl?h7N;UwTJ@eq0!YfbaArHSVDjAj5OzHkeOL7Y{oEY51AWyRB;FV@HsTSm zA=DxX!hq}>&-%9LbLqdT)^H{)Roik;LRsOCNjDPBS~C16WqAQINSp2ZuwJ%8)v0B) zB(qhwQezf8%S!YH7PW7Ivxkki_6DU=Zw2~0p zIuhf^-M-&$f&&LfDT|usX-hn6XA(40$r}I^p^W^+xhhZn+s8tC!UexIYz1J|$8F}f z^wCWWgr7Ct6To6eh5e5I0B);T?MH>m;mme|#cd%g*%BW0Ipg97125`{xAyP0-sSyW zZ3Fx?y;%PM*}f0eSy7$?e$_-&NcUo_iXEF^c8}fiJ_Gfv&%{P!Qe?e*^>bV|cvZ@3 zt4#+aXRkdf@JAda3hm8753>!xB#(}(7|nA^Ok@KmOjgKC+>V`!zm(j;0ShqLs~034 z!Q6j0p04@}H(Mz`Kc`sPY#3*7h5IiTmtnwuP%<~q!)?|#s`vFZTUhJ(S#4RVR!IK5 zR7W#H7?8FS9sI_s-sJ%J^S4Hw3zS*0m2y|ASGcAn5?YUgIB+;aI}|&tq1cC49#yvA zsNH*|Q&!l6Qqy>9*AquuCgmPDv)S*E@NjXA(C~m z91|I8%9{mK8JBXqg>N1|TJ-LEQ>HY>s@{_1>fUP00tk>G`0OqPD7IPqRwN$!$I9;?0IQGFJzib-nDx?;n%N|p_WF~J<=q!_8#o-oK6 zn4dUd*eY%45$42HV-qrZTqc&d!6)u-D@_;jQXX3}a~YzN!~r4K$G&*A}fx_81Rvo@Sa^9G`y6v&yw4iK@$9X(6Jj0C=KF(Ik-ZAN37kCq4Xa#W()|FB`aw zwk8VqXyh^<#BD>hBOI;6mv!1W%x&y;0yzScxKI_F^>tl+b81GvikBk<_Aw1zuSX1r zxh)*Ry}FqY2F@$6hETJTk*IC+8P#VegF}bK9CW<0O1R@yrHa{QB7^q&D%5 ziR|MnaT5kg=p<3(uxXMRlo9|?3iID@(!CqfIO66M@#d)VDOHUDPis0VjscvL&3*|7 z$M5+a(advlebc$$XwhME$6*YGNm%6DhRilr{{A|Q&E+fLaaOW7s!u%%{L;c#Lu92# zaJ=k6F8t%&+mQf;L;G*nkdN_rmj$J>gB$7oa;6STx2)Nxg1u(SWGdsbQ+I6B!`_Lw zNsAIUC5BVFZrga`deWXh)Gc*^!^5WVxtJ3lc3Et#YZhd9OtPt29u41N5n;%$`nscH zeFL?qu5-xhd}(`9O)T-t%xyHy&xvk(mE5-9*nPUa$M{d76rjbOqxA4F9r-HC(M2gi z#R!g0avjJ7{QJ3eiLAXdpVMqgy_=QnSyfxSs@QoW5b|QG>hY^W4}tGrzmwJ$ko5|C zRiuR?mb=?#uOr0&0Gey53(8Ih+&K-``h0E6q8_c)mvq*UkUnm_pw&>Zq=qxP?BQhj zAwAxAK0|%FLLv1&JdUK5Yst;*&ehsU!ZB8mvhU;q03&byo{iW|F>56aTT_!e({!+$ zQ~@(FFTsJ>02g9O+xO_@Z(L;Q;9nUIMp-42zkBJ|U{zI#Xw`&Xvl}nDBO`Br zg*$$t547~&dNcTkrr66E*t?ynB4mvukg$!Ta0;p0WIRFG5xM^W(zmR#l{0yHa&X5j zc@?`7$H<_XMsXx_ZZ_c{<+=Nx-Fm9Wn#-H=MU{I}Vxnb?)n{ac!dQV>yzDmvaz_3S zM$EUXxVi9S1C5>JyCpxRHIcb?qT6IMqQ*GFyX*+^f7S8To6;OP#?_N2l33u5dH$)O zl@o-;M=!0~Pm$nx{fJ(bJl>mWGNCYS3`s3xDyZ%V+En*EP5yO$C^>JkmHwSD{)uI^ zwv5wxJ4ovtN$*8!jebQrMPIoc)wyraXLf}4nZZ8R@v-WgbJcnp>aIf?n5KJk+{mlT zXe8>bZRC+GL}MPwH*a$<-`(f*^eq@%O2%uwh}G-H>ohZv4<=GK54a_c=WYD?{+&T- zuT<&iT)D)WP1|M_UdVoZ=w<{gr-~6Iht+hpmCe=0 z+^&;7OEK7o+b?u9WLLKsUHgx-saBUWB!>6Y`u5(5I2*!B!l;cN{!< z9^XAO{*L;eZx3?qO4p4mHlR@@c65uq861dpVmFZ`efbSKO$@>roo&5B0^rO_QH?4Vz zeK^-Td|@xsygXEzO1$NXqR)*i)m#|f=kO7eX?Ne|4^x6~GvR&Z7^X`^`nAh@imN~kBJE|bq* z$QpQBnImAv%U;2PXzU7-~0+*?%VS@In6oz=yh1f$667((LzjE*@qK)fvO)vTlMmyz1{- z8fos|=%&aUlLFg!BdpUq1awfv)z4M24A};bpfcvKc3o5|z_s97@Wh;1yMZgX3*%$U zf1bJN)UoyXSW!Dw?p&d1rb!HuDjqQrD=^)JMh0&mr;*#DrFiSwdvdt19Eg%j1d0Mk zR?I*&_8ZR?C74E_4nNK1x{aZQb2*Tqk!FrG6C_*DAqGh#cHI8}sdpc8IvRSI&5y!f zvrvduI8AD5&lUq-Qg(xWwv3O}&|4FwAk}iko3UEOPp_;00KYWl((+cPlafO# z&5pRf`mwCc9TnBPWxxkx(wFgZ`pm_Jie4^ zEBK04)uw{AMRm6vi6bjb9De@Xd|0p`e=gg%Oh0ycNN8e?NESiBhF(rK!u z?frTYYi&uWG=@Z5$iq5QuW~~>Hz2Z7oGf#eA7Fs;^XK`xpR$y4x^`@ZYV7u4VQMP~ zfcIHYN!>s`3r0r!ZQ=R^>bNcH1KNdVHah{xM+gF=&QK@AYrY(>9qSG;cOzv0!HeUgD+kO7a$D)6& zn&(Yo@naE+WLuWMDiDhUSmLBFH3wjixlM=PpY71Q(rg8)mgCH297rXYOF4bdA_s`r zw%aIOxqJ_2`nKtY%=(D))0T}X7ByJfs-$}z#7Ie4$oMDZc=Au!^-70PLt+VRQi0H(wpxcS@X zp)M;GfY9}-iTiQMG(rgq+6^h>{DkVEh*=g!8-C%suWL*-*sM#baPviCjP*j=5wkFq zFav4S9n|i4hVl6OAXbx`)&8=*`YMUXRlArKb6R#lRV9r>$sCFX!26qH z@;Y1c+9e>G(_%7`L?YbwM6U3Noi;l{$8+az1K@vdsQ7(Fm8ERO8c95~!ATdlyfZ{j z%2ik3@#VQ6b?bYd;nC=~;G@=^F8M!QbM&caceJxt%4ed})^YaY0T#7K@22n_q9Cx@ zxqq;DHCLv8gYQq{yK7Hms%J5@mSW}MRuHS;Z zN24oC4T;lnYAbXZLe|ECep?M0^6H8D{to{D2mQL#oLbvba)k||UM8qO0c@R_`)G_%#WA5LgzLnI| zWiiseHib-qjs}ACNP*pD07+Wr+n)^d?gHdWcT2mJIK`hRTTp8nu zgQckU8~cP!8-B7kL*YkTI`#I0`oZY^yzZN0Jb3J662%%G;=Ssn`HhJ3jTqeh&f9d1 z`fL-_Dr=`RHF#RaQwt)-%8JPeSd5lqkwH!%jkfX%`RZNLu7kOjz|!$t$JuxZS1;W` zCvq~!Zs;U`a6URv^&UEwLs{F+wn|EpDB)j>gfcWl^6r9nRpK_{+k^iAhkmRW4NGSW zAT}dg804Mjlu;~`%@czQH`Do58y&Xz9J(g!{XM4rNYb^W#qh1YUuKirP@vr1c6Zt>FMXGczg%ZWgr<+nX5s*e;%4$D<1nPZb}keHi~>0X)LDXDSRaB|Z&;d|J4tzDF)N+OQlT4s`H z;_a^P$qW-Jj~*-e_MV}%=Bvr$r+l;%ytJgaE7PpR$qbP~WPF9$i1GCO*<_6f+jdSV zzf1<5$*h`oq^{AIn#-4ow;)3ZY2=K*^QoM~KK|5Y9(uS?xjMgWPX=AFB2+??O zHc*9NfQ~!)0ZAJyd;R)Ty=T-yuAiOeXoU7@%qy8?Dj=>ouT z^&)vS%u?55^jOPR{*J19UniZpS@HPCQct!D?KO$G_}EA1=cX$_Yf16dH1Y&*t;+uZ z5~*t4i8n7=w6y9~k_BDMqa!-;9!knv&n}tYPQ6yn*UjT@n1nH`HWJNFJR-Eza~5$~ zTI`N@F8JTfLf%6s+;7pLq?*V4)>hI=AiFcbwizRQt}ba()lpH2_e~3!WsNuI8#}79 zs~(B?4DK$yfFkZP6jEk|c+Bz0SG0_*0o()dciV{^x;N=s(dREEN|Ohflo3i^c1QOc-RE@o%2mb);^4$w3^){`2M9gH87^9Jdtc%T%@hEo#!hod-_#2VY zV@&EjJ)f3KX2Ud)uW64KOC9SpCCXXj1oE2I` z--uR5X7+xNyEj_g!s2rJYO><$!7Q%~4Nhm4PpF1E=%f%yKJCu{0-v8BJ9R{@l6^gs zy%t)y`ge z@_dhj=l=ll>BvkXnr5y#2y0M+0Bt(QBF=pNfZOfJe0=Sz>))%I* zX}s1`RK=Oe*Rq)I`Hu`N7-5d3r)Z7dV5xH(fP{|pW4PmY@DQie%1U4;2ycOBER!GYB$34%x=WYJs z`pCd&KUAi7b5`lCQ;o^mdlhG=YP-CA z3-9^igQSVzvba;Q`yux9$@;me{{Z8Rb`vL(nPrxYvCi>A>LfAG5Jr+R<7phX_VdSX zm5pB_>DyJNlI)OFm$;=Qj6))!@S<0L4m_`cFdUzXY}|(9lI`Peu=J)s{t(H@YZs3GAzIH77h&7C zDxz$T>Ia>GQL!8De}2DfD%P&NIVmulLWKhvHaKqRb+gSVI>eeoG+nz_IYd#{U2v72(nrq|kBa zW|gJP6q0+9)>fXZjEKc$X+r{$Fyr9(98dd1dKq3FU!Av&HiZne7C^Bi&HU%gh9_<( zc>(=_1NR+lRvdP1v2kK;$g;FBf|k`#Le1a_3XhKie~rFB-i+9M)hiclM5Y;KGQ}Lv zD1=7~jy5Mm<-ZXL+kXHS_ViHEFOaEdS%PF+nGVfp=l*n~C{cSPPQ_w^y*klDQYH38 zOV0eIWL3K9-FJC&`3&VHw=uzCrm2CTuiTaI%6l04aZ0Vaxtq5n>6`vO4n3JXx9>E@Czz#xx9(&fEV0%Zp1Y2itZs6ZaiA8VfC#&ER+L#a|sl zlCd0kC-kGVjT{U@^X+Bvc^o)@tx$9@H?_ZX6`9jDwR) zIZ#Ts6WovzPbnMx-4C@FtCn7lMHW|@#!8pfM`lI}1S@xn&Nm`-4m3|Dtrd~cgv0t!moV^)B8(8f>le$)k#9fk%P)Y=@YgfLOG8Pq=^;ml` z#~Kg^F4oK8dS|e>d1R?PwCAN3Lbds2ua2F2P<=*6h1fV*+sAOZ05Kc3!MZ3u7Vdu& z{4x3cOq|*qF@mBC5te320XxSstMT1F$B7(zkHcxbO^k1%$V-023meBBn2D8yYC;06 z7q}R1HXD#i{{SydKcRSCRglQiSQ5@cwJD4rQ6vVul>p>&o7@MIRDi*E>tR_8 zWy;uUaKma!c>}emqjcFAv|=UtfXUmAKl*_I{YTt{Y9o3Ww~NW#BbbYleABjcYXOO*pCO` z_y?=UrusuG6?tcd%TG=?jMK#&ukc5Pa&`=&HeI;yw%>lZ!$IFtSe0Rt&8oH$qjJo> zq2ZQ5+_qon0Q0}~>6z0x%elCP1)9*S9~7H|>{>|nV-iTK`MW6G?eWW_g*^jHH(47U zb_02Mhp#2W1%Lqu+cY~U+;6_*@9zHHH~L3UR>d*RvWG7An|g zWtKJskbVjFx4WhDGo|$AgEGaOrFKe^2_3Q~26c#eF{IJLrNRx*CjcqwUe_8Y5v!~m zwg4(F<&ogBk6G0T?a1(dBX8iiKO=9yRP1(yp2WtN9!3=BZ1aq$i-|0x50~HJN1xxv zR1H(5bJXl5Ip|uWT$E+*{!+X@%+4dUlkiXdp(x`p(`D>6{EyXH ziWg~^OR*}izinTT=67S~;BD<69emcf_0jr{-3=}j)VTyU40Y@gENn+U+%rqQ?CB~B|7!(zJ``-)lR z1H(Ty{Y}T?@9q8i8D;ft#?*!Bv-G_woI@VzfWvmltN# zSn`$W#t~zNwY4!wu#L7`3U(}^d2HXy;eStEUaRU%R#{S7kjD*)cYWb<&`KoSnLz{E z`>)R60nm=7r+2Rn8#gDq)YKU1OgES?Zv|Uzoj*?>zy12E>$FXC1yqW_jUs)!5-$b} zqjp0h5JPT02gcifHvM9HYo+Z_)RorMb6TxY#L#+P3`#MR1~wz1 za;v`Y4DtS%M&5njsY>+EelUF@$@)L0w8S_o^V82w^e_lak=l@3wIeqDpa^Q~6ml>> z8+*BV=`8gB0P$^}{v!UEYd9>nSlCMO&GgL@7I^|RjNp>cl2DKGmg$j;T8$}|sY^Fj zwOSXqn!TiSjlua;`j@CsZXs5?Cu8kRQ%tnuv@Rd6qo7zXmMlVe@_NB*OG&K@Q zDX5z~R?#ZX(TCY2Nq$7%U^n-F;p=bygMJ--6#QFg3wN~kvWH9SjBQ5YGS3R-v9=-w zZP&`sb`fQ2iNJ0rX8WV7b*4YXccvbtdKKy&ycVXhQ!|T>)1(>P6qe3D>`3e)w;Xc& z=t$9>o(VZ@T=8aJui6HDR;l#n42B+p#d5uNl4zkIGfb`e8*}cI5H=g^cyG|o(rL|j zEz-$J^cjqNlEku>Se1l9nNN9u54?a4j^0<>@zo3X&73>Ir=e>0YswlKnlm6*1ns=B z{5Q0y_U=4yzsFWsv{nlhJv{Cv;#beR*I{O?d;^I+($}5i^2$_k0hDe1`geUl%cRsJ zT7~^Wu z0G`s4ckow^{ZD_xIXpza!qFqWB(UyfNR=MrN~9%7(5Nv%@X{TXPxAcrH-z+-{fxb6 zA(~1|ak&Ele`$-%g5A|(_&q}tf)#o=$6ryq}CQ9jbF}g-t1R)^X%aadOe&*(<*O{;8wU1|9CmI#?wd?aybTQX zI&W@E$re##s|*eecrmoG_UziTD|=MrEaW7RBwlzg75z=o-UzZ80fx$?#UCXXH>#|{ z2qrdB;;C$>fXf4ZUme{_sSCGER*lJ7TeAICr<$sY(Mbz5VWOPEq%?(xxi;WQKbhn8 zap|_xx@NXgyfLChC9;1eVIf}J#W{%skI}gv2k9gF^wRoaj(kNcVrrrsygjlFaIhO3 zca>yGEGSulAFH3VkAv1@<+4|Gy+|azE>9C}qKf5lVIFz~z(10Kcy`2t;eVHQ<;SKb zG1`VT0Fsf2M4y&is*U0(`vsLs?d|d4{^0ejKk-BSMa$@qp_l4a!B3U{0E)5qAY7dr z6IaUMaxr+TVkqHY@RlTec?0=)@7IB^P)@G=LTen3PH3KzXoryRUQ$5% zt6@109|vN5eycXJ*LrjC@u62uVbOn!&jQp^0d}hMovm7uPnqX$ad-LT+n^mNzN;nEya)rFoiBC!bDwDG7LQ#*kBNL!Z>e3Q$e z_Dfo1Gz3;^&zO$JNtFxAWJQ2Uw!$?8gW3mz?tH2L0B*ZI8~jCU>2*Aq>6YF?qz(#6 zGsv}|48#U&1d}i-cH_5>UUu`=(zHIf(fZzsP}MZ?uw9xS&Af+r%8;vX8^)2SB(X++ z0DxHgiu7y4dRO>J#n?%8K4zNBk!aaYTKzg;5Q`@`C_ubWl1CK^!^mUq=&_ajDEcF+ zv5%AV*CT*=IEiEW{99YAG`vr2wV3$I2txkg#Qy-NtzZ5ie+(X?(aOn2hARF#J98%- zhAE}b;wOl=ffOn``1Ig^)yJR5x2307dN-~8HpH#u(|N46MmU~nR)q_BLhgs+7L5pV zyC4H#cYkv3I-sP#i7cFP!;188T&z;fA1Nil&Nhp}yNC|Ql7ssnckkz;h9A*CRAST9 zWHNr8>vzddQ=3OF8p`h@fU+S^3__2Ox7+)I^<+ASBNi)EWA2Bt)wvF3pJM6UF-i7k zBf_7Km0b6zTIRlT8y6||pn`Xf&ARf<*k(rrhgTab2H%h!TJ8GkUaT1TqaIZwM@CgC zA@-I8ey=BDPlLZ6u+F=)W0bJaHdWBb(Li}gDoDW`2)G@N>Un>E->$~=?&hhvUp+>X z;00HVxU>XvSW>%!gXBWHdEEKq(K?lB@wA>S)#$9p19pKN8W|Zu0F~vsub2M-^!Uru z8aofDa`o~xvQW}hm$xKMKb4o6Q6ecG2E*O*Ht4^T^g9`->|{nKEfiPcsUfm}H(~A_ zgi_UIZMeLN`hg?om-B9@0G#n_UhtH=qGSX+~NX{~o5On08PM;`+Qi|X5Lqo2Hu+i*AEbH5&( z&t1P0+UHYi%x)jl90klydrD7kwD_6j-bn++8X;o94T$aqKkD)FdK=_4tUAY2L9H=3 zsUGz-sdhu1wNioZr$w%L%d`=uR&ce# z!yt5u(t{b2sRwrFu^Tsm&x!HU{7CCd$;QOi1p_`>y{5Gr#)O&x2WW*)!!v$QctO`yDHqM!9b96_jvN8?bOev7`pmF zF5%>ey;#|uB??u(l?L%zj40w{Rwt5=l_!54Sw@b;WbNhh^)YZ0VUla+oFjfH3rBoL zyMY@31MT*HUU>CK%KA%;xp(NW)%$lXs?SEs$|r^fnVEve7(C)q+xwNa8~GguwWgTP z=={uBT%=|1-5^)91n%*cQUrfmn}&){Xx&H~a{Whcj9!oA^$q<)9v>%0zBV|=1hIW= zr{xKWdBkeF$r)e2{{Y3-$u!h|!z!ZFWs@l-We``Z0EMeSTmEA0fKj8PT~R}!DqWdJ zmOWZ3#}1#Crc*NvSq#PNkpP{(C))pyV7mi1$SzXCxQ(I=; zk6O zg}aBA3sBaYSY1p`5cfo@8v_ilzx_Fs1weU!FCLg5OS4(5Zcdd8P_4{vN;jSfVU@QL z@gb2OSjQ<~n}W{)1=oXd@XFn6q6p-pq@b{{)&}h)lOYCO<&GH8#ug(=$B84ZQ2S$2 zO7a_jTK0p+In3HgUIpj*^x}dRj=WvCN4mc;$Vsu-o%Uh={TXw45r@R(GMrL*G4!Oe zm84`yp4?HH5fQd68F$%1+=2G~zEuAJ#0LxO=dYfk+q+*-T*B%3Ws-~OSfXhfambD? zL-~X!j%Yp)!t{h){{Rl(#GO1He5RSf(WO@KUN$4CBgh(Ei6Rm}u_GaGKdbM{rF&if z0EM@#I*TV2Ek^k z?s{T$PxwuG7f!WIPE#X)JuK3vCHk3(<0mHAg>O4I9{_paIx=Rz!H@8#Qyn7L!An{{RQyNnn>LPa!wcs+BNTn)S;q8i2uG2*BUF z=k%8U0DigSKf(*r3YVs_nzdBXNguT!qhs>`+j!)TX8hfw9^iIR8<$5NC;lD(0EfJK z8klS?J1G=qIb>++lnlNMfnRWh$-m#_{zp*y*ZeI%4;h&S)X_9DR@;M24GL2#?A}#f zy{zDQ@%L}l)w@Ce0EIu{(kRwjnkqWd2tf+8>BAf`0-z0`UC!H|o&Nx@L=1oUarzgh z@&w}a+-7aT0yrW>;J(%7+kRjvw#0qkmJ;=JsyLWq{{Zp{w)u=#*#6%mh#&Y*Mi_kx(R!T& z$))hMZpNLiMud^fAyg?p9O}$RAlO;u+6cjD-$yW zJD<{^f#7ey`*cLb>ufUK5#s9Zi4Z8O1Y%Ag?Su z%#SZ9il20x)rTQBJ(*Y$x$whovNi4+zE39>x}|a}^WJFfSf^f2GFy*vxB#Crg*y^< zAHPx_sOh-l(0ILNB{<@ucr>;+qqoz@$f`MyAw-U&ef*GmTXFYhf`0ZY{?8EHRc*LF z>f7-n$lXuJ>^k(r{tMCQ=(KOCo~Ua(wVNoA#f-*dEg0I=m1@S)v%*-Bvo|k;zxup& ztWU;|s8|nCeNoGLnTw7gKm1Ra##xFK5i8=>+ocN}NCIY44&!1sEZr9U1Yh|3LfwN4 zkx26pNi>feN~vP5(QNNJF5mLax{>xj*P(ttPi8cxqq*JG2JT@mah9y5JFPI0T@Wbw zgs;Z_e14vfzNl&)VXv|HJVlvoTC0po;i)v1C5alc2|-xXeh3^X46DB+Eywna^xpm3 zGT*H=T(oy7r)pW6DVRJHDMbt8^QPp#(_!=HqMn}1ded80VDeaPJCV!RrE(UtR^V8Z zB^U-2Vs>vqJx-Gik<=J^m>Fo&i5k3Rp`K{~;Lyz)sS~i@TS9fW3OVQ3j}5n zZ;US}iIa(m7B}}WJ8$;it{#(mC9d>NM<%(Ph0KmF^yJi*aasxSZ12WOQw1bRS=-1` zd=dLL-4d;OFJl>&p?*7Y#F9wvKAsp=;}VitH)cFQe{zB2W9RjC7pOl4_BH0AsjRY^ zvN1I%l>Hf)K`fG~C1sL0(`gSMBYnv0B0o#9nrl+UQ-Hm~BPVgYl13UZqyrd5Ex04) zfFy78)!yE%OeChAEPQcFSCnuF8drO*vO{+%EKAPU0>Pq)zX7tZ4g7RS(YVSG-iI-f zkG*3VOyu$zER3rps{H$eO%d5Kl_MOupxK$^ZXmZz@1d54psjaRTZJK-#m2i1`dFGyBVfA$;ks<8=W4#2Yw|>{WXeqlSfUHUDJBaaB_xMQE3&X}0ZVZ` zCO?U7D_c=u@{>CI1T#jf8UzGClwb(l5aZ9=`CG3!+RdcVny)UAc4Ugg_9Lw=ba+)A zHHMcck}x(_H8MYrM5?Nd);+=!dTKEdVA;}I z_y&PiPWxN43M4c(LN}3}ha<$Q#DFt906zU#@U-e*pDC5L(_Ottxv^d;jcT;x<=dl9 z#Nt5H%BamW?cq$DLl`_*^j*?;`dJAq;H(&`y4%OcRM$izNu{qgPTZ4_H!QI%Xs3V6 zVn0iC%W3@*`1-c%Q?k{5q6iwKfkzu1kraz0ATPDRc;qA7xK?lMf70<;vSGyJ`yW=Y zb{1)55y3QzBJ3ZL;U6z)QhXNs^!R!$rE!mLNbggIN$X1$#ix+RnBud*h~FVvXpxm| z#CGMwVJg4sV!jUEqg<|#Q^&Pw#F7PgZAVF3C?i5cD%f+fFa^9S^4tBolhF~_uZof9 zCT30|s~DOiEopangsjWE@m~RlgZ=s&diCn|o%HJkEj*rDzAoj)+j~e&cdatNiKHXj zAtT4`2hUorlKzm?8SI{V##)`l9OuW$8Hl3G8TK;B$K#ZYIUV=ks62nFx{a^ZuX8Tb zV=Lqz=&{^+Bf|;eth;=$A17hIz~a3eFdnca+`n@#qUvNZIO#atvm{I0kr44a?FkY$ z6(^6#A0K|Nxx1c)>znX>K9y1#LABeB)<~PlLmq>>D-2NGNMB>vrJWAph;wvQ4+Ca`Ca6+kt<1VN+^Yg#^ zbViTy0VZmAWxYy8%D{*uuU@M~95JcjS)aGGfg6$Bo=2A^UZz%QadZA2GND0hFsoqoRy{9oBe8T?m%XFPclqkje!3ES5u3iFf=Q7 zsOO1@##YRD=2tzZBV}Nxi>JXO%YL4u;Lhm{FHTclX)am?R2D~(LjWTQz$ce&()3(EL)ng~>hAU6Pa!B(%Xq}^$d6f)n#Q9G$ z5_s?ON6T;S>mx?$Jbt~9*~VKfHx0@%!jSHFAyBjZwp0HA%ge4=Pf{>i9@FEqI-?$! zXMWGpv=B?YlJBr99D%n1eEq$hSTOli)JYZiEkgCDj7c2Touh(Q<+qT`8C!_(cm95* z*HB(tQC7#}{-RDrMV*|MPzWl4izys;9__aysLfC6wH+4q*HYtNR+2Sg91*+B;Epfb zNcY%*`ipo6u8;k{X_l(8;k1qRat`Xzx{%v%$f;6J%k$&O>-ow0b*yzYdQwAw6tj%N zp|0_8E#2T_cqQh$ob@sQlCe3WteGlkSVJnX8wBDPky0CR)!WmLMJweo z(&}A0@->QK-bzeD*akK|+S7(zqeJc=8-U$UYJ5DlG~JO7A{ZVZK?--V3z-h^qU=MU zJdc6!{U_<9^U>-w$mc3pIK+`7inLX%yvnNK!nARz_ki01K6!EJkZ()!n)0F@Kcw?^ zrj9U?ESTw}C73Git;du^?Y`E%4K*nWX z(g+@SuP&NT!-wIg)Xj5|7;DrnVzJA@8LVY15j-r<;Tafv6a(%)IqkO*^Va5czxaN7 zU6Im|>0eT8Hl)Y3SmBc%WpP?K#H(kx7A8`EGi|)@vhn@8F<^hfTkv6f9~MFkEgWTu zWR5WWCRSaJ&G}-UElT}_zm;T@ZYz(Bq{cf#i4n}qzzu?NCx4&q)gv^y9Z9P!L2l#~YR0Q7 zJvn1WhA`cv_3;mU$*@nbY-)Mr(QbU%duMbwVCGLL>q0$sT+{W4ZmDU@od4X)|Jaj zgJ4ypVm~psozJuadL~rIW_1Qpau`XjNl3#Oj>K}x(T8Q&5{JRu{Qkgyzgd=aPI^0n z^^59QCeuVC7I z)D>aVbY<^QhSXKgKC*VH<#NTi#a6}Gvm}+IjHt0D%u(trWooztkcvc# z51;y;emW3(#b8N_zX6Ybrj})r7Cu!bB=@#E76p^Z`0LJJt=LcE{-IrhqL{5ok}GDh zA*;v(afZT;xA!m4+;RBn!1R|?Y2QrnUrlcq{{X~c#dq($g)K|ABjn);@K~ShN5}H{ zxJT;N8(ZnQrPDKTwD#Co?Sitw+9bDZ$lvne5CYD32&=gFFGQ^o3O!vrPad-5?BwRZ zdU+f_zU=+La@SC|F&t=yI=k}Zs?K=t?QHa=d`>>BnRww@aoCwMH7-iP{Y>}VcHOPa z$7t)3EG&d{;rCXRIPBSw_VyqD#kM;j}h5@v5*^uj||@)s|*NWsyR!R7P zM0=8X0oJ(muDuJh0rIr}#--N?!xUIJRmtI9kKLB_P#B^}0t2CM#GwJO~ zr7{vnVPa!%htxTLYK4ixB=*O~!{N4Wi8^OoUcZ>hRL)~`-5eV5k}RbRf+4-&DBd4E zr{b@${$4|uo{G5pct~l>4l!v)^DvPunWAmt3(d+l-?8!RP(q$ z8!q1*wqdcDPfu_-FuJGcqRrfcy&BiA3p%usf=?xH%ap5lCw=&zj;ZkhR&;levU4Y#WbH1>Ryu!6bzcdzwS@ z;B3Bq+jP!oOE{lQAz5;w9NXFg6AHv`MutEPuNjl{@f+>9_#dl&u6ca!gUM8qYS?^k zHqE$M)u)LfkA2FhP|^Si+_6*V+wi?AxnE6m+vJ}ocTv^H&odGtr8KJ?h{dEVRb*k! zRg?mwiwEP-pVh4!e>sjCdroBLv%Gtjtp#_GA#Kt|eYnm7@5%Q&2J`&AP<=nrUZCkp z&q`bNX{c`6+D3^(9zZ(HviR8#&wvAUDVV`cOCt@urYh#9+557}D6N@QVk~!vabw>A zY&Ilq)u&AQ_ou1Y{s~V`>gpD206w1167_15aM=Qf`ce+($N-JE=&MWAO>tt~S?J(i zEMe_;M5{VX>&UkbM!0VPc_59C{&HHsT2D2YY{jG={g#ZDd8*qe8NBW}1MLI)dF9n> z6OPq+S(_mh2>lvF54U6r>XJbzBVpQokQ^A_^B${8$dk;RSo$7Wdj zJ4puRJDm0fNMq#qZHM2c1JoZ(EMf#br;Cczc@Y%L6z7idp(KO$@ymZ3 z9*NApyXjMQ7-p}XiZ_NDqG+SAmN*g4xk0`nFGIavOg9l^=|uz+#ewM>f>MV z-|1y5$dxqp8KH0jFt1y<#O?`DskqqqApCSs#QGsa6>h8G@OaAgQ)iMXApnHf6Uh}v zB6e-JJ~}Dh(i(cy5Y-w>74?0-26*a6xN-u9-H+6R%zz!#Ez5f8$_UfZo)SX2Y zG{~$Dv}ssZxEb4X#fLDe2IpXRJ9+l)*3tbB^(z6Lz+$ad)cSc1no*^g=dlXd9LC4> zo%i4N{{SxC9J1J*C4Ly0?BjA52n8M+mBCWOVn-L>Zy#^B>iOtjm(zM@A7ZtsZAU^X z@&dveK#9G@m3R&JAo(BFx}@d(QpDq~)Wn*1slj!dY^9ls!`nl@vzLxXZTx%>^>rhr zwdNlEn=>w{{XNXj)z$t zRgR;LgBep31JAwuT_b5 zW4J#gcs_pK4?_ID2U%Iv7>b$9e1m%`mFg@K*&gTOttQ8B3+1=m5AD{{v_^_uJ?Z== zz^fharAa4a8pct$1~Kqo)OO?jdIf4p@{-{$*@731Y~Gf#$|Nfxf+iu^8N4EpJ_pB1 zMzOVQ-L(W(WNL_HS!vBFSV0^)Ow%^+zz;to`gBnAs|hssF((C$ryS9=SLeTCKHSA6 zlYpEFM;g(xtM+$2!n?lRHyURm_40X0sQpE|bTR(`PbGHKww;-0P!8`cnz<_l6-L}gY_`i;0HkIvF^@wZfOQStHRvGn$usz4Eq_?^AdZmIfu+J~l)u066@%WV}}(PCmu zc$2t4CP~&5tqeeN?>$DENe14;WJAZ8>5HXyrJK15ki}jqnG01fz`k^g+s(lVd%#|4l80nZY z&BQmlK3kAOrsKydy)itGqS}F^bG&4a2Ov5;ylQ>U=Tl%(to@hx&hn*0~G1lG7~sOx4VBISAy( zy$PyKG({e%W#)U)Y=k$#ALi;Rub}laO>0x=2rkUjN-|8gAT(?-UQXgAa#;el<8Ryi z-B$G%rWv}FvL^OxCN7vhu+@&tl#&-w?H+6mwjV0Hl0Fx1uNqq?oYXpZ1qC~q>$x0Q zN>WQf*i(@dh|0vOH2u(XnM#aO0aX3=ZYL z)79?@WAPD7w~WnAVy+^zK_MvQF_nC*uD>J}9DFyo=lQy%>hD;85;}5tskf&%T|)#C z`?Gg)J@;5a2Xu_b%a?C;KRa^$`o8J?Q|jJoxyA?a=y<;>RspTFInx*t4ctJBwE%k#Y1BA_f5Wdt+_e#^d#L!(*v>la{Xe{C1~3B&s?HSI#;m`Lg(PLzy&`m<6+1A z$D&pnP1mbDH*tecWb2sG;763&b>jWS0a;hKCz1ml{Ev^fM9R9;Ok-JQOeJeKoOb7q z3c-Yp%z&i_AiEE1YzEvs@A`VX<6_de>|L9e^!&8uXyq#Iq9Qz<-?k3zx7~m5*LEvK zVlBeYM#Lg&RO9U$;>tqtyXSr%ju~qGb!DJ6_0g4*J8duS6|9<_O#A_ z#x9E&(^MfFu!RBAl*V|&m@9K<>4%?8xPeX00K}S zpSQc?uDe=uTI6Z0I;x%Aj3}s)&*c#Yz=bM&gd1!QF60*7I_n)VcZ0qqK~#hRIj1w|h-3CZ}d8eVa3g%E!NMK&Nr%amfAKp|($3`a@Ep z(Yvg&nCzqxw5g1?V=Ez2dBmS2sg0O_O~1|71%Fo-JXp!-ZBp0KgH>E^Lb6Q_6XgeQ z+6xVbgZm$MSeidrb$Jxc##p0xc>sGCJ;I9_UAbVU4lFSE% z-;-=lpYwGi>CdE^Z`2u0{Zn_bXhIyGU6?OI#a*lWU0JLL_bcpFgU79hdUN=8^ur}y z3}~BEX<8A|Yb}*n>RiU%c!~o(shf9=WA>5(;!u0IsOn=q_+ifAtvtFyOh#Uc%_c_H zTOW0L{F1Vpd=ya84S*T_qtuidSYT&y{ZuXVS`RJR`mn#5B5 zC5r~S$0f;ZFPM-X++3T$Db$8zAr-juMff#blrT*3ui-L z_`rn6EZE96yG?FZj87=+_V$a6k{4#$INNi`gRo`mQ+e-0a@i&RAiB+%+mXOjTNb(T zJa>hV9f2dmZ?Qc&UxQyq)6@GG^ZC4llL;B@JeP~}nAD7j>qP|a6iUwPyEnEaUPs;5 z)39-jtoXTc$^1ykJv(hmrx>2Ze7V-44|8RuiEB=uQ9AO8im zglS8%WhSqWg4J@fwYbwEa>O|FW?%mRWf4@fGZxsYFC`l{PA-}B$64c-7Zv?0k;+QT zUOw7KBv~nu(ToK4figd0K0uwF0>a7a*1%eU`UbW|=FsbZIep zxQjG*pyc}`u^vd{i+y zqC64Est;&NzZ>n$55HP({B>vzN0Rk&>C;%eb<7pKT}$~`FV`=NT)n!hEbVu-A95*V z3El1vLwP?=`}CDy@i~npr1JWMMc2+tdlw8-UemaCu%l*XTN2Wa(E?bN!)yQ?m-&44 zTB+);N7g!`#(M_;09S#vUmqa&=|yUG|edp6hI|#1kpL&M(HKu zm!(^{%27`g6)1_gfkeeA9`^a)@Av76tCG!MHS!OVwMqeGp<6WsvBP3L!iBR{+hiw? z>K@U*FP@GVOB$K4FZ^JXvRG?Xm1$%#5=`>P!X-snV{aU{W%1zby-nqG#zI^bk@VFl zU~0QdEsTd6mgk5P+*Wy;d8FHk0Dqfp`iRl`*FAJPMU^a-_6nN}lLq8_xRz$M3gROhRd#RIpEN>tbcGDp!hYGer$fB!C-&A0f8` z#>e%3p09ZMG|S|!!GwmMU8vU&a(a&xp=ES(9zM`x^1gTS0+01|1(Me~PYG_EdV?s` zFA4tu_!s|evm3frJickp|g{kpx|*Lrr_RlSAH!EVL4Rw(PkDu|h{r>VvkWzplJXMGDIPZRclbN;<=2M# zUY%+G08sQ~z5H!j`i@9syY=*n!(m=+xx=(yqlja<+j1M*&K;5VTDz@ONQ2*!KHDKvrG%8!yjW^7I4n*^$I!1>HyyUeYOHC{f!yRf3}KRTs|N zdHCtqtLW`qZa%%b3rGnV2cAM5fY^Js^Rn;beYX9&vuTY7TPGycZcj#(eAz{Wa0;&@ zyM+tk_g%lY>FAGRLgO14wdvTCBq+|R);Sa0VIc~zjfWLGd>{N>NZ7>7fhi@#W{$)$ z1b`0^DIV6@RP7-r%I)W@e06@r83CB>R)v^8J0}u`XN-VWIX?T3XdeTh->LapmLbQ} zt3oMlChRP#6<6D0S;UGr0C^n$rKJChxJVhsp2%wTGP@3K9xfxjYqPS>fuk_XebKY_rVv3`1J`Wy%TC!aBVq13M z)maJ^a$T4K03`q0+Qm^iR!3Fh?7ri?=hz*#@Ol@|>E@%c6gF_WqQ&{%Na4SbwP;Bs zOsu8m?oQjm7+w9O4gTI+p$2zMYPHqXg$&-F)GWwjnrW$5wA7VZl(A^kt8N1iBzfQ5 z-A-A;TEqA5U@rEW%u&*wmDuXkq-{s;vt}OVz=7^42gd8x6|Gg4r;9Y}RH=~4EFl9& zli<%UBgy{&G;D~VZ^!*V@6bor-&OrZUrv6H8>RA@$=;l``}%oKLD&e-9h5fUhYUV` zdPuzo^`}~D9cfEV>rFYTu+M%Kq`5!K*Q+l<^rnc? zweXncjx4QuSlX6m?$$?u%`9mB*ru`>`6zMi*bTpKw#VsLkToewj!}(Du2i z0mXR@y`j1uY5ijtu3A>i^l4)A6zS?5mJS@G)Sl$BWO2@6%3qE&fEebAO6?SqE4L=! zX5-ZZ)c*iWVXLJr$e60tg;?Y@-lT4kA>7E*d%*3fVZX-yJoV-8@x7=r&}1@|tIHhs zEB6wbOgPuEBvOGA&noT~yDr?f{{Si9&yJQYGp{DNnk!G+y(AigCb1jUD%OfBD#;LP z^=ZrcRvvy?-GC#{$6HOy&CdltTc7LsoE+EY{aMs{KtPgY~bjxUGtaqE?ZYt(;2&%tcBSklN|Qt zfqs$SbZXt`%2+VQ{Nn@M0V*~fQ#JRl8gJFTPnN`JYn1BV((5%nRXmhsxg?XzAO7s7 z+~6Bq?JR@GormcIszuB!_zgRzwQgStN)0b>Ct)(zEXy7?fo*M?)Q7T{iN_W zBU@P9wxY&McG!w&ara;in`>4G6;cI^{&j^Rc*Y5R{@oFv^v~4KN^fJb+PYV>qqPQ3 zym?%8DlhzMgjd3IziyLCMQBxcmu3mLW%l@Pt{VL-;-Z=K2C>w6+;mMDuV$rXSZnO6 z{1O>eO@`!nmQ_C+ZPH`vr{ZH>OEfxvE1ooQwWo@;i+QYV*rhRsRC1K13T1M4_k31?v?i^1v& z_F;!g#8`y>7Xqfim=@R*zeFrHGuA9Hp$<<*XQDueR>U-khRoAAMSAkg4%^6}jkX}C zyQXgq=+>at7+YW>*uE8HW&m`nrqEUi9|Z$78YioQds%+S;ImlxEpVv;DtMhU)ikPU8JOhO9cf9Y#7CMu{<*JJpSQ**i86Ij>GQ)OPF! zWg)E4Qwznux8AeMI`_>s1zE11!A&B<)>iLKC&q7I%vn>ok zNmig1=Iuo-7}a-p!U+|I?7Ye2?FGM^?egwV352l}Qep&EXGDgy)v+>6wR?k%bJ&q1 z0!~Bxqiw$H)#>iyaTOB8h8b4%AXd-trsD6#%SAFn-jEdFH?PO(v&Bs zUnP?N0Nl*NNu4ZNX~H276ErgOm&c9v+z*bnarig&lT~V*jGD(=;;Naj4;Za1kFc{W zZk%k%6@=v;{G0Xo1%2Im9`vtVMe5}YO{{t#$8w4ra91naLmLGw8Bd>Z-*9#XM%#hY zl5toZRy4@6{Xu9}1%g?*0K?ijF8gosu^%U_DKws+p!j@~ScYZDNjbY(NsLKV3hY7J zK*w)CKl%Li&K+-}w7t`hitTYpgp$Nxc?47c0F-P#Hc~u(ul{ckdY{$&C5V2f9fp?a z8?Sa~=6J{lV8wSIe~%k~FI78PZDW(hoTR#b^=xG*d%sIcHJWMCPlFPcCvvBC2gma$ z>X)aq#wSwhI}qe@mg!PJC10tt98#px@$R54=Zd-F;CNrvxE|h_I=VkWX`MY&_(q4D zF{>Wq!wzxeT)?Xj9nwH5?(oRoJ3+Pt{kq|T!eFaraf4Cj+Z`rtUhZzyimQdpBz?mg zQjTd)wM1jy03Z){x2v*xv8S|-A2X$NwjQ<~8d>6%FRV7Q2UO$4$Mb96Nfj$xRy>K@ z-MHvqlJyHs=5V(2c^vj(GL~|7Y9^S@VjkuqI8_XDT83#0IV_&-{{VLO`+Ba)>pDK1 z{{V*O&eww$$Pxc-cCVw17YX-^;`b{iLYDbi-OK3rY?CUgt*Jn ztzKUIVnNx4W6c$#Ra6IWKlpkc*L^`AVy-dk<>}MPWI4LxMPrSlMH}g3y?WmKLP({U zgXO+<>kCy<{{V(83vlarR2b)T41pNDyvAzcK77{7Me;WCe@|Wa^0^|G z4xaS!g-dWVRhu15;?>?Tc*S~YNM-?#oxaQao`bEb=v7Kl>n%T#ud1an#f`_yRN2Hu zByL2HT7CG+L~1?GRG;hTtel`(Nb(V@ofu!v>4l+r|`BSk6`MRzyN`&_*%N{1Z+fg_*>?CxR zD~JlW)!de2<@PGufLr{(9c?qw4S61Nyt1wOuPiWDX(Hq$mxW5|jV*$%K}K=9ug1&P z(|--Qui`$RYU_Gpx@EH(Zr!ZY((cCr$n66gqFGs38GwuO&~B)a1@gzILrGY}YYhci zp$io}o;7&KGdfOK#Lrvr1!~H%VH>$E&oOP;j z3iPz|32e`et$MTsb`sYnlau**EOKH==yCX@*EDJAJdNZ^)z%2+?!gR-ptHrq@0uMS_1k5aGYJyp3|O*C@iDA+SY5S7=nd8Co1iJCBXG0n?m@-}WKrC&j6ooQD! z7F5x*a@$CdRfdgk&9{ETEKyX5INC5~W#=O7LU>Qr*QuYv%=qmk=%$~~>j~;t$<1C9 zm9I?Lzh&-Pk~qyuvL?@A*1?Sms3={yF8enZD@L{RtTm*PNZ-REyNZJq2W4zelD7qF z_HqK$$^QW4$g|50ZcGBi1MwaQTM_gv^yabk3pJ+}3)M2(B0N>hor)6J6ky0@X?;v} zKB11;1N0TRkje?;++ZDm>AUpPC^c4}t*PayZmhR1<>QVElC!||YGo&ay}F(T^=2@~ zDi#V^A8n4^5}n(=RU9cu(YA1n9`d^>D51Hd@kRijVc+-PueYtiLWRtQuHmanrHhuk zQAs?e3b4G>cHoIwr+(aQ0w^rR?4xtHUUU6k(O#VTi|R&GQRuBBqqP1@GF_#mf}U|h zQ^wOXPU|W)f=#lt&{6BhYl9t|KaP{zc9WjuS1{v6yp_Rzm z##;*%owF@rWVB(O2s>=OI~Jv%6PK<0DY9MXuOdZC3yRo@%`nLY;0g$Ft0wHMrGvM} zLb}-;by)KA)zLFzu2{Q1Vtnnob&`foSoY>xT5~CpW06@@xex9wze8;9Q&(KOIL*&y zjaw^Z+Q-zkmod_-!X<|#40a`zt;C=xK$`_+Wj@}!bdBupCXmYIG{tQ}ilFmBPUJQ5 zl+wWM8GfE&LR9)$b}7Fo%duY{&DMTT7jI5t;q|VLM@8UijD#tV)A>P7?c2|z3liPH zb3ainx{&sxn0XsWr{}1fy3lW=c@#uvyA{zPj)>cG zs=Pe>aq81k)9p*1!o#g;WAj!h!yhUlsC8h-3;^onCSg& zYAAKT{8Mu&9Oz-IkZg5Kja3_QjaC@Xec16|J8$}%&rDyW--qu@biO60qky%D$z>;K zq#5yFFJMWqDQ#q3-mO+Vf9ajNX0RH6LSyTu*o+nIo;7SV)~!=34L6QI465-Kl0xsx z-M;?s-Fl^J3pKGb*s}&dPiCi(!zA<4TNwARG^I>&J$o?mk+`zr{_IG#8j57 z_9^8%@Xr%XK1U`d#9^f1IM#S$jI2?+gUfD1&-uEn>aS8psfLt!oGunTOo=JCm%`_3 zR?JjW8RLc#^qOW^UzXXS@o6=9S%E7>--O?n2DgtRfiXj$VG;|hGy<| zY@CK9PN zAzbYr1qf5)e;qKx);zzdk7V>QiB+;+miRJviZThij?^3@P zbm8Z!vN~3*^Fm_&6NSdtFyzny5(zD-Egf_|#$uqb=&4^&`p=uCEgKYdCT{tKmc@KM zcx!zGd!A8JyC7g;1hL$Zc3wwrqAL1%lzFj|%hfrQ|Yz(lh?*IYN6{%H!`)&ntl4qYM8Yqqe#hRE2}Jv@BW_`+uvhtuc(%8jgDw^ErF@2`D_Pau_9& zJZ3*(u`e?KC6jVb5x4{Tb!TWX>uxM`s*=lT8@L2QmtCmAzLqL<=8t~kG(8+kIwxt`HUyf%Ptce z7%V5fmB|7a_CP|B#H145M;@-YiyBEMqb^4M3iHJ$e#5>Fd9s%-R%qpxy(UV`vDCFG z@!aA#Atd#YtbIsoxh+e3Lq(0yF+DnRX1|75sBh(Eaw3jPX4`fvylf=cClCjp+o0}S zSN{OTi~Kfxy%^%g*Rzl#Cj7EiCcK@MlM!l1-bmfzXOR%?vv~IL>#OK8IZQ1qnYT(T zWlVxt$Jp;!iWt(gVm5~DC>h+Am&o^SO8dRNbG??eC7iLhLl+)BDah1p!D*@7UepRK zPVXx>31o6gENEGN{bAuXt(*1l77)J_(jsnK))01nr*@p;*1i7V5~*sl3Ji0#F?psupW(dC{VC&@v# z>Djz>X2AMA9YDdXx@N$SUn!q@Ot?2j^&lANQvO({LJ8y%*Q=WNB5@b@FgR7KDdS z)T=$n)<}f1LkuemNGB}I`Z%+vzivGMGM<^&^w+hEpAnav30R5KHrNU3=CQHtor-y+ zj6c<5*-G)WyKqM3C+X`mO?o$h)OguqdBiVbd3*88ILp~+tUQw<(?(`;ts5s94&aaU zj-qn^0EVno8jBmJ7D}||%^)~R5TyxjUYd|CblIq?4RDbbcioE-@DEkY$Kk)$-6;l3 zFX;wKoEIZmp|ejZKk(XDmN?-(QS^sZ z(~eHfyBXYVc{0%5PGzmfR;D}`Mf+^a$ozTtba3?3{86Z3AW4BtW{7QZ^J#_UG1h{<}Pdp*b2P;h@Ksd}$aTwDelk54(zDl)W&Z$-S(2@VyB-yyy=NN&Mr~EGX1q#F*w>LJ{{WbX zkqd@NLN`Y&PpaBKQfZw(sc>0}_GoIXd9{RXpOo`S6m4E9i+9(;=9{S?w^tr>o*2Zrz+W)X629?HG-f?C>D$ zr=pIO^zT=_k%_gb>QYvj(%B*_p$@+@1B18=fFSXo^a=>|&fJ13-lJjz{yV^&1UWwMs^?hb67IKSmB zRi64P<#I8?{$s7nbH6?Kw^`Y*PU=z1;B$G*z6yJqf-0$|yIN-ynqN-Utjf3a{y!rn ziDAE(n&a}ZFp(y+@fR)|bWxMh%}eynmc#3}!$n$SJseh;fh%G1FJXYOOF2UHGs0tu zxd^~2ptrp3k3F|0i=!|RhHrb`v5OV`3#f@TpMrYA3YP*yfqEEXp`Kg-nSb5-ej zscMydM@F+_DfWfRbT8MhYOzBlQx$}YRk2B>3)z@356hPxgEhTK)LOO(B+-|k%GiRd zrMYm~%CAM18NKFd=Eoy7{$5;riiQ4OyR*YwE7r+s{aAF?BF(~|#I55fWU*CYs|Pz$ zPF>77+oH6CM7VKsgRDLYj1bTT$9VK3Hc3d34FWOmdE zq>y`E)wUm}sVe@MdSxx6Eo}o09L(HliK{T43-#4|ajaQC%u^v@!DE#Ihh4hOn@oCg z*wd%-*w|}iezEGe*5jZ^qK*_cXM{Zw63(%rsPVfqKMxyq6`ZkNt_LEIrt1A{bQ1$Z zF^<0Y<9OJG&N_t5j`G}9iDPGpWmZt;K4F)tCXN39+pR-d&I4QN9VbRzJz|}xa#~7Q zANXRHuWZpbClIZ;19E9>7vFGLow;>U%<4S;fWh9Q>PN3{sxdKInAgbJmrrFO#j8ZN z;DELw{119T%)D@cxm5#xE7f00dZ&T)FCF!I6E$O2>pH7;jR~6s*lP~cOkM3uR^pXv z&XMiiclvnUloxJYTITfw)R?ei1z%rh{V4SV9gs-o#cFLHf*+u-#f@6|icm?9vsTm+ z2XgGr@c>lq)t?!ye-YEK(#K~?Yi7q}>y1gD7@8OANl{Qu5Aacll^Jcv9^aWNvc<&r zO7;A`tHY?7zlgjSI}oEGidQDLXnAI0n<_7#9|SM<>vn(PmH2$tUZ!cxj-$@vMjuJp z$U@zWv1p{qQ+D<}7zy2?Vu#2(uwXd+^}oFlqI3_Z@yAW)rm#XCp0rfi;#C_eMrDX` zSxfO7u}~NK^hj+@fu01pB(h$bw35k>#N$fE0>l$%HyL>MT~wkJh;*Q;e9u=!fm+p(pv^ln*}Rpb(_ zySHOQy)}@Z5wK?C=c_!FG!~u@xw0CXz@p7|douztP{r>d*lPMv&BtJmIq7H;}pqeUJX_LfJIO4MbJaER3Ynstyk z8?fD!l^qzdH+0=hgiB8yUbaxEHUt#}HcmR%-ca(fa_pLW`vYZ1eG)=w7=kU*Wtlqrm?Y^Sz3lvD2NCdQ14?w)Dt=2%xbE!4K{AwA zM_^V|*-IW<^^>Xa*_@5~$}nOKbxzz?gfJzDPBu9CKh{P9OyOC$0Z(FNmx1KI?!J0m zpfaADYg+awVyIxTdYNvdnVT1|i-?L_N=0cWnxu1F7TyG8L@tdRHwjy&3QZ*hbKQHH z*mLn;4+ODKW*gBJg`Om;)@)s6XyvZk5K(tA?pNA6Px1KM*qnB!hXe)|d_HRYc$Z|c zBr)Qt*>#5y`6>*?jJwOiJ`u4I#;y8#`=YdFw$^^4;`P>N7BjkI2biB3inC%niCen# z;)gYFm86K;$iXPu7KsEKFY7GFru#3ZX=ZUvj+yS`1nIhDVx=`ZwdQF;JhaG(+9^?$ zdr}r|AIjTxhZct&(2&KL%Fm2S_2@QKYLCY+QKiwEJQP$X|6Lc_xIUpos%7>UHbCDflY% zt1K}&9apDxJ~K?;5jtutW>s<)Ber0YS)uR51%Y%o1dXFtj%SdFw&2gz0@wIYS+DcxfubsTn54V?$3n#6V65tPRTj#=&H>e`yHvud+6++??2$K*`T zoEy(>RFRX;-APcy!;XrL41F3g!+oZmMT#)hu=Bzl!lYHxc0&|yVJ#yr+*|&qR2@ly zv07^`uPv(f5*C^m=Vi*72PzVsYjiWLR|TuviWW2^rF61>>$HL7H_ z1}bbn{UxcFlEP0mm66;;-P)LPz~Wb9O{7LsiB?Gfqbe_GJO zUGBVb1;)*k#7BzCLe+yIZbUfz-4im(BUs0F#}!i&N_TASZKbff%Q0UiAEo;5{{U#2S~RgU zW^ZZS^hsAU6@`-rRQqCTbsdCnq z)jD@T-Nxke)~mevD|K>ufK8G5Q&*7Yv#94-_Tpw!*M0pvgp&n_`ppk(O~aCcw5fXu zt8l%A$=gWk%PpNNYSoMQngyfmqr+*!bMB%=$bcc$UY>f{q%}oso|nVqt!W$URtmU0 z_3S1F4I4ZvQSZ>c?nbZ6UNOe7$2?**lENr*-rlqEx;t3mFuC1Ls&tmH)R-o!@@i~0 zVr*>;&Y!mLZpKG1ijd_dt9unJe@-pc`e`OsjTRD|!gMRtSMZ*m`c;3{YV+D%4UCCq zj@#+GPPxphSW4G4!NOv(jUanc$yx-p3_@C6wv&&SBA)ge}5I!|ZyH$^IR6R#rV7Yft*YpTS9|EnUIm zaoCejOX2L{F4m4J_N082E|+X~rP?_x6@!FL$=BpqFdv5=Ry^U9lmh3;c(9Sd!oHPmaK`PD&C>>ODqscLoa1D z43_OpkAW#<87x&xwbs3W_G*1aa`Eh_oich$OI^?D9Ih`%)ym^E#!*3!{vp6;@lA!N zkqhoFbo^3s65qb2iSe-L!sTs=H%|U+`nWk>7 bWv@?B+jK_~y&P~SxO literal 0 HcmV?d00001 diff --git a/src/caffe/layers/image_data_layer.cpp b/src/caffe/layers/image_data_layer.cpp index 62fda4accce..56d354655dc 100644 --- a/src/caffe/layers/image_data_layer.cpp +++ b/src/caffe/layers/image_data_layer.cpp @@ -37,10 +37,13 @@ void ImageDataLayer::DataLayerSetUp(const vector*>& bottom, const string& source = this->layer_param_.image_data_param().source(); LOG(INFO) << "Opening file " << source; std::ifstream infile(source.c_str()); - string filename; + string line; + size_t pos; int label; - while (infile >> filename >> label) { - lines_.push_back(std::make_pair(filename, label)); + while (std::getline(infile, line)) { + pos = line.find_last_of(' '); + label = atoi(line.substr(pos + 1).c_str()); + lines_.push_back(std::make_pair(line.substr(0, pos), label)); } if (this->layer_param_.image_data_param().shuffle()) { diff --git a/src/caffe/test/test_image_data_layer.cpp b/src/caffe/test/test_image_data_layer.cpp index a4080ccd145..ce5e0bc62d6 100644 --- a/src/caffe/test/test_image_data_layer.cpp +++ b/src/caffe/test/test_image_data_layer.cpp @@ -34,16 +34,24 @@ class ImageDataLayerTest : public MultiDeviceTest { std::ofstream outfile(filename_.c_str(), std::ofstream::out); LOG(INFO) << "Using temporary file " << filename_; for (int i = 0; i < 5; ++i) { - outfile << EXAMPLES_SOURCE_DIR "images/cat.jpg " << i; + outfile << EXAMPLES_SOURCE_DIR "images/cat.jpg " << i << std::endl; } outfile.close(); // Create test input file for images of distinct sizes. MakeTempFilename(&filename_reshape_); std::ofstream reshapefile(filename_reshape_.c_str(), std::ofstream::out); LOG(INFO) << "Using temporary file " << filename_reshape_; - reshapefile << EXAMPLES_SOURCE_DIR "images/cat.jpg " << 0; - reshapefile << EXAMPLES_SOURCE_DIR "images/fish-bike.jpg " << 1; + reshapefile << EXAMPLES_SOURCE_DIR "images/cat.jpg " << 0 << std::endl; + reshapefile << EXAMPLES_SOURCE_DIR "images/fish-bike.jpg " << 1 + << std::endl; reshapefile.close(); + // Create test input file for images with space in names + MakeTempFilename(&filename_space_); + std::ofstream spacefile(filename_space_.c_str(), std::ofstream::out); + LOG(INFO) << "Using temporary file " << filename_space_; + spacefile << EXAMPLES_SOURCE_DIR "images/cat.jpg " << 0 << std::endl; + spacefile << EXAMPLES_SOURCE_DIR "images/cat gray.jpg " << 1 << std::endl; + spacefile.close(); } virtual ~ImageDataLayerTest() { @@ -54,6 +62,7 @@ class ImageDataLayerTest : public MultiDeviceTest { int seed_; string filename_; string filename_reshape_; + string filename_space_; Blob* const blob_top_data_; Blob* const blob_top_label_; vector*> blob_bottom_vec_; @@ -177,5 +186,34 @@ TYPED_TEST(ImageDataLayerTest, TestShuffle) { } } +TYPED_TEST(ImageDataLayerTest, TestSpace) { + typedef typename TypeParam::Dtype Dtype; + LayerParameter param; + ImageDataParameter* image_data_param = param.mutable_image_data_param(); + image_data_param->set_batch_size(1); + image_data_param->set_source(this->filename_space_.c_str()); + image_data_param->set_shuffle(false); + ImageDataLayer layer(param); + layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + EXPECT_EQ(this->blob_top_label_->num(), 1); + EXPECT_EQ(this->blob_top_label_->channels(), 1); + EXPECT_EQ(this->blob_top_label_->height(), 1); + EXPECT_EQ(this->blob_top_label_->width(), 1); + // cat.jpg + layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_); + EXPECT_EQ(this->blob_top_data_->num(), 1); + EXPECT_EQ(this->blob_top_data_->channels(), 3); + EXPECT_EQ(this->blob_top_data_->height(), 360); + EXPECT_EQ(this->blob_top_data_->width(), 480); + EXPECT_EQ(this->blob_top_label_->cpu_data()[0], 0); + // cat gray.jpg + layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_); + EXPECT_EQ(this->blob_top_data_->num(), 1); + EXPECT_EQ(this->blob_top_data_->channels(), 3); + EXPECT_EQ(this->blob_top_data_->height(), 360); + EXPECT_EQ(this->blob_top_data_->width(), 480); + EXPECT_EQ(this->blob_top_label_->cpu_data()[0], 1); +} + } // namespace caffe #endif // USE_OPENCV diff --git a/tools/convert_imageset.cpp b/tools/convert_imageset.cpp index 9c52bfa0ef8..90cdb15d427 100644 --- a/tools/convert_imageset.cpp +++ b/tools/convert_imageset.cpp @@ -73,10 +73,13 @@ int main(int argc, char** argv) { std::ifstream infile(argv[2]); std::vector > lines; - std::string filename; + std::string line; + size_t pos; int label; - while (infile >> filename >> label) { - lines.push_back(std::make_pair(filename, label)); + while (std::getline(infile, line)) { + pos = line.find_last_of(' '); + label = atoi(line.substr(pos + 1).c_str()); + lines.push_back(std::make_pair(line.substr(0, pos), label)); } if (FLAGS_shuffle) { // randomly shuffle data From 4bf4b186076b054a0fa06103bc8989a3577468ba Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Tue, 24 May 2016 10:36:23 -0700 Subject: [PATCH 13/39] Overhaul TravisCI * Run on Ubuntu 14.04 * Test cuDNN builds * Build with OpenBLAS NOTE: Python3 build only works with CMake --- .travis.yml | 58 ++++++---- scripts/travis/build.sh | 13 +++ scripts/travis/configure-cmake.sh | 32 ++++++ scripts/travis/configure-make.sh | 36 ++++++ scripts/travis/configure.sh | 11 ++ scripts/travis/defaults.sh | 10 ++ scripts/travis/install-deps.sh | 105 ++++++++++++++++++ scripts/travis/install-python-deps.sh | 14 +++ scripts/travis/setup-venv.sh | 18 +++ scripts/travis/test.sh | 19 ++++ scripts/travis/travis_build_and_test.sh | 54 --------- scripts/travis/travis_install.sh | 101 ----------------- .../travis/travis_setup_makefile_config.sh | 31 ------ 13 files changed, 292 insertions(+), 210 deletions(-) create mode 100755 scripts/travis/build.sh create mode 100644 scripts/travis/configure-cmake.sh create mode 100644 scripts/travis/configure-make.sh create mode 100755 scripts/travis/configure.sh create mode 100755 scripts/travis/defaults.sh create mode 100755 scripts/travis/install-deps.sh create mode 100755 scripts/travis/install-python-deps.sh create mode 100755 scripts/travis/setup-venv.sh create mode 100755 scripts/travis/test.sh delete mode 100755 scripts/travis/travis_build_and_test.sh delete mode 100755 scripts/travis/travis_install.sh delete mode 100755 scripts/travis/travis_setup_makefile_config.sh diff --git a/.travis.yml b/.travis.yml index 4dc7ed72d6c..92d740cd88b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,40 +1,50 @@ -# Use a build matrix to do two builds in parallel: -# one using CMake, and one using make. +dist: trusty +sudo: required + +language: cpp +compiler: gcc + env: + global: + - NUM_THREADS=4 matrix: - - WITH_CUDA=false WITH_CMAKE=false WITH_IO=true - - WITH_CUDA=false WITH_CMAKE=true WITH_IO=true PYTHON_VERSION=3 - - WITH_CUDA=true WITH_CMAKE=false WITH_IO=true - - WITH_CUDA=true WITH_CMAKE=true WITH_IO=true - - WITH_CUDA=false WITH_CMAKE=false WITH_IO=false - - WITH_CUDA=false WITH_CMAKE=true WITH_IO=false PYTHON_VERSION=3 + # Use a build matrix to test many builds in parallel + # envvar defaults: + # WITH_CMAKE: false + # WITH_PYTHON3: false + # WITH_IO: true + # WITH_CUDA: false + # WITH_CUDNN: false + - BUILD_NAME="default-make" +# - BUILD_NAME="python3-make" WITH_PYTHON3=true + - BUILD_NAME="no-io-make" WITH_IO=false + - BUILD_NAME="cuda-make" WITH_CUDA=true + - BUILD_NAME="cudnn-make" WITH_CUDA=true WITH_CUDNN=true -language: cpp + - BUILD_NAME="default-cmake" WITH_CMAKE=true + - BUILD_NAME="python3-cmake" WITH_CMAKE=true WITH_PYTHON3=true + - BUILD_NAME="no-io-cmake" WITH_CMAKE=true WITH_IO=false + - BUILD_NAME="cuda-cmake" WITH_CMAKE=true WITH_CUDA=true + - BUILD_NAME="cudnn-cmake" WITH_CMAKE=true WITH_CUDA=true WITH_CUDNN=true -# Cache Ubuntu apt packages. cache: apt: true - directories: - - /home/travis/miniconda - - /home/travis/miniconda2 - - /home/travis/miniconda3 - -compiler: gcc before_install: - - export NUM_THREADS=4 - - export SCRIPTS=./scripts/travis - - export CONDA_DIR="/home/travis/miniconda$PYTHON_VERSION" + - source ./scripts/travis/defaults.sh install: - - sudo -E $SCRIPTS/travis_install.sh + - sudo -E ./scripts/travis/install-deps.sh + - ./scripts/travis/setup-venv.sh ~/venv + - source ~/venv/bin/activate + - ./scripts/travis/install-python-deps.sh before_script: - - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/cuda/lib64:$CONDA_DIR/lib - - export PATH=$CONDA_DIR/bin:$PATH - - if ! $WITH_CMAKE; then $SCRIPTS/travis_setup_makefile_config.sh; fi + - ./scripts/travis/configure.sh -script: $SCRIPTS/travis_build_and_test.sh +script: + - ./scripts/travis/build.sh + - ./scripts/travis/test.sh notifications: # Emails are sent to the committer's git-configured email address by default, diff --git a/scripts/travis/build.sh b/scripts/travis/build.sh new file mode 100755 index 00000000000..bb9406f046c --- /dev/null +++ b/scripts/travis/build.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# build the project + +BASEDIR=$(dirname $0) +source $BASEDIR/defaults.sh + +if ! $WITH_CMAKE ; then + make --jobs $NUM_THREADS all test pycaffe warn +else + cd build + make --jobs $NUM_THREADS all test.testbin +fi +make lint diff --git a/scripts/travis/configure-cmake.sh b/scripts/travis/configure-cmake.sh new file mode 100644 index 00000000000..772f1e2ce8d --- /dev/null +++ b/scripts/travis/configure-cmake.sh @@ -0,0 +1,32 @@ +# CMake configuration + +mkdir -p build +cd build + +ARGS="-DCMAKE_BUILD_TYPE=Release -DBLAS=Open" + +if $WITH_PYTHON3 ; then + ARGS="$ARGS -Dpython_version=3" +fi + +if $WITH_IO ; then + ARGS="$ARGS -DUSE_OPENCV=On -DUSE_LMDB=On -DUSE_LEVELDB=On" +else + ARGS="$ARGS -DUSE_OPENCV=Off -DUSE_LMDB=Off -DUSE_LEVELDB=Off" +fi + +if $WITH_CUDA ; then + # Only build SM50 + ARGS="$ARGS -DCPU_ONLY=Off -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN=\"50\" -DCUDA_ARCH_PTX=\"\"" +else + ARGS="$ARGS -DCPU_ONLY=On" +fi + +if $WITH_CUDNN ; then + ARGS="$ARGS -DUSE_CUDNN=On" +else + ARGS="$ARGS -DUSE_CUDNN=Off" +fi + +cmake .. $ARGS + diff --git a/scripts/travis/configure-make.sh b/scripts/travis/configure-make.sh new file mode 100644 index 00000000000..ddc40fffa9d --- /dev/null +++ b/scripts/travis/configure-make.sh @@ -0,0 +1,36 @@ +# raw Makefile configuration + +LINE () { + echo "$@" >> Makefile.config +} + +cp Makefile.config.example Makefile.config + +LINE "BLAS := open" +LINE "WITH_PYTHON_LAYER := 1" + +if $WITH_PYTHON3 ; then + # TODO(lukeyeager) this path is currently disabled because of test errors like: + # ImportError: dynamic module does not define init function (PyInit__caffe) + LINE "PYTHON_LIBRARIES := python3.4m boost_python-py34" + LINE "PYTHON_INCLUDE := /usr/include/python3.4 /usr/lib/python3/dist-packages/numpy/core/include" + LINE "INCLUDE_DIRS := \$(INCLUDE_DIRS) \$(PYTHON_INCLUDE)" +fi + +if ! $WITH_IO ; then + LINE "USE_OPENCV := 0" + LINE "USE_LEVELDB := 0" + LINE "USE_LMDB := 0" +fi + +if $WITH_CUDA ; then + # Only build SM50 + LINE "CUDA_ARCH := -gencode arch=compute_50,code=sm_50" +else + LINE "CPU_ONLY := 1" +fi + +if $WITH_CUDNN ; then + LINE "USE_CUDNN := 1" +fi + diff --git a/scripts/travis/configure.sh b/scripts/travis/configure.sh new file mode 100755 index 00000000000..ef740c8982e --- /dev/null +++ b/scripts/travis/configure.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# configure the project + +BASEDIR=$(dirname $0) +source $BASEDIR/defaults.sh + +if ! $WITH_CMAKE ; then + source $BASEDIR/configure-make.sh +else + source $BASEDIR/configure-cmake.sh +fi diff --git a/scripts/travis/defaults.sh b/scripts/travis/defaults.sh new file mode 100755 index 00000000000..d69c0a7d964 --- /dev/null +++ b/scripts/travis/defaults.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# set default environment variables + +set -e + +WITH_CMAKE=${WITH_CMAKE:-false} +WITH_PYTHON3=${WITH_PYTHON3:-false} +WITH_IO=${WITH_IO:-true} +WITH_CUDA=${WITH_CUDA:-false} +WITH_CUDNN=${WITH_CUDNN:-false} diff --git a/scripts/travis/install-deps.sh b/scripts/travis/install-deps.sh new file mode 100755 index 00000000000..f7bfe4c4df9 --- /dev/null +++ b/scripts/travis/install-deps.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# install dependencies +# (this script must be run as root) + +BASEDIR=$(dirname $0) +source $BASEDIR/defaults.sh + +apt-get -y update +apt-get install -y --no-install-recommends \ + build-essential \ + libboost-filesystem-dev \ + libboost-python-dev \ + libboost-system-dev \ + libboost-thread-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libhdf5-serial-dev \ + libopenblas-dev \ + python-virtualenv \ + wget + +if $WITH_CMAKE ; then + apt-get install -y --no-install-recommends cmake +fi + +if ! $WITH_PYTHON3 ; then + # Python2 + apt-get install -y --no-install-recommends \ + libprotobuf-dev \ + protobuf-compiler \ + python-dev \ + python-numpy \ + python-protobuf \ + python-skimage +else + # Python3 + apt-get install -y --no-install-recommends \ + python3-dev \ + python3-numpy \ + python3-skimage + + # build Protobuf3 since it's needed for Python3 + echo "Building protobuf3 from source ..." + pushd . + PROTOBUF3_DIR=~/protobuf3-build + rm -rf $PROTOBUF3_DIR + mkdir $PROTOBUF3_DIR + + # install some more dependencies required to build protobuf3 + apt-get install -y --no-install-recommends \ + curl \ + dh-autoreconf \ + unzip + + wget https://github.com/google/protobuf/archive/v3.0.0-beta-3.tar.gz -O protobuf3.tar.gz + tar -xzf protobuf3.tar.gz -C $PROTOBUF3_DIR --strip 1 + rm protobuf3.tar.gz + cd $PROTOBUF3_DIR + ./autogen.sh + ./configure --prefix=/usr + make --jobs=$NUM_THREADS + make install + popd +fi + +if $WITH_IO ; then + apt-get install -y --no-install-recommends \ + libleveldb-dev \ + liblmdb-dev \ + libopencv-dev \ + libsnappy-dev +fi + +if $WITH_CUDA ; then + # install repo packages + CUDA_REPO_PKG=cuda-repo-ubuntu1404_7.5-18_amd64.deb + wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1404/x86_64/$CUDA_REPO_PKG + dpkg -i $CUDA_REPO_PKG + rm $CUDA_REPO_PKG + + if $WITH_CUDNN ; then + ML_REPO_PKG=nvidia-machine-learning-repo_4.0-2_amd64.deb + wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1404/x86_64/$ML_REPO_PKG + dpkg -i $ML_REPO_PKG + fi + + # update package lists + apt-get -y update + + # install packages + CUDA_PKG_VERSION="7-5" + CUDA_VERSION="7.5" + apt-get install -y --no-install-recommends \ + cuda-core-$CUDA_PKG_VERSION \ + cuda-cudart-dev-$CUDA_PKG_VERSION \ + cuda-cublas-dev-$CUDA_PKG_VERSION \ + cuda-curand-dev-$CUDA_PKG_VERSION + # manually create CUDA symlink + ln -s /usr/local/cuda-$CUDA_VERSION /usr/local/cuda + + if $WITH_CUDNN ; then + apt-get install -y --no-install-recommends libcudnn5-dev + fi +fi + diff --git a/scripts/travis/install-python-deps.sh b/scripts/travis/install-python-deps.sh new file mode 100755 index 00000000000..eeec302791f --- /dev/null +++ b/scripts/travis/install-python-deps.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# install extra Python dependencies +# (must come after setup-venv) + +BASEDIR=$(dirname $0) +source $BASEDIR/defaults.sh + +if ! $WITH_PYTHON3 ; then + # Python2 + : +else + # Python3 + pip install --pre protobuf==3.0.0b3 +fi diff --git a/scripts/travis/setup-venv.sh b/scripts/travis/setup-venv.sh new file mode 100755 index 00000000000..81245f146da --- /dev/null +++ b/scripts/travis/setup-venv.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# setup a Python virtualenv +# (must come after install-deps) + +BASEDIR=$(dirname $0) +source $BASEDIR/defaults.sh + +VENV_DIR=${1:-~/venv} + +# setup our own virtualenv +if $WITH_PYTHON3; then + PYTHON_EXE='/usr/bin/python3' +else + PYTHON_EXE='/usr/bin/python2' +fi + +# use --system-site-packages so that Python will use deb packages +virtualenv $VENV_DIR -p $PYTHON_EXE --system-site-packages diff --git a/scripts/travis/test.sh b/scripts/travis/test.sh new file mode 100755 index 00000000000..fedd7e6b56e --- /dev/null +++ b/scripts/travis/test.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# test the project + +BASEDIR=$(dirname $0) +source $BASEDIR/defaults.sh + +if $WITH_CUDA ; then + echo "Skipping tests for CUDA build" + exit 0 +fi + +if ! $WITH_CMAKE ; then + make runtest + make pytest +else + cd build + make runtest + make pytest +fi diff --git a/scripts/travis/travis_build_and_test.sh b/scripts/travis/travis_build_and_test.sh deleted file mode 100755 index 174f1ee5a0a..00000000000 --- a/scripts/travis/travis_build_and_test.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# Script called by Travis to build and test Caffe. -# Travis CI tests are CPU-only for lack of compatible hardware. - -set -e -MAKE="make --jobs=$NUM_THREADS --keep-going" - -if $WITH_CMAKE; then - mkdir build - cd build - CPU_ONLY=" -DCPU_ONLY=ON" - if ! $WITH_CUDA; then - CPU_ONLY=" -DCPU_ONLY=OFF" - fi - PYTHON_ARGS="" - if [ "$PYTHON_VERSION" = "3" ]; then - PYTHON_ARGS="$PYTHON_ARGS -Dpython_version=3 -DBOOST_LIBRARYDIR=$CONDA_DIR/lib/" - fi - if $WITH_IO; then - IO_ARGS="-DUSE_OPENCV=ON -DUSE_LMDB=ON -DUSE_LEVELDB=ON" - else - IO_ARGS="-DUSE_OPENCV=OFF -DUSE_LMDB=OFF -DUSE_LEVELDB=OFF" - fi - cmake -DBUILD_python=ON -DCMAKE_BUILD_TYPE=Release $CPU_ONLY $PYTHON_ARGS -DCMAKE_INCLUDE_PATH="$CONDA_DIR/include/" -DCMAKE_LIBRARY_PATH="$CONDA_DIR/lib/" $IO_ARGS .. - $MAKE - $MAKE pytest - if ! $WITH_CUDA; then - $MAKE runtest - $MAKE lint - fi - $MAKE clean - cd - -else - if ! $WITH_CUDA; then - export CPU_ONLY=1 - fi - if $WITH_IO; then - export USE_LMDB=1 - export USE_LEVELDB=1 - export USE_OPENCV=1 - fi - $MAKE all test pycaffe warn lint || true - if ! $WITH_CUDA; then - $MAKE runtest - fi - $MAKE all - $MAKE test - $MAKE pycaffe - $MAKE pytest - $MAKE warn - if ! $WITH_CUDA; then - $MAKE lint - fi -fi diff --git a/scripts/travis/travis_install.sh b/scripts/travis/travis_install.sh deleted file mode 100755 index 091e92431f0..00000000000 --- a/scripts/travis/travis_install.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -# This script must be run with sudo. - -set -e - -MAKE="make --jobs=$NUM_THREADS" -# Install apt packages where the Ubuntu 12.04 default and ppa works for Caffe - -# This ppa is for gflags and glog -add-apt-repository -y ppa:tuleu/precise-backports -apt-get -y update -apt-get install \ - wget git curl \ - python-dev python-numpy python3-dev\ - libleveldb-dev libsnappy-dev libopencv-dev \ - libprotobuf-dev protobuf-compiler \ - libatlas-dev libatlas-base-dev \ - libhdf5-serial-dev libgflags-dev libgoogle-glog-dev \ - bc - -# Add a special apt-repository to install CMake 2.8.9 for CMake Caffe build, -# if needed. By default, Aptitude in Ubuntu 12.04 installs CMake 2.8.7, but -# Caffe requires a minimum CMake version of 2.8.8. -if $WITH_CMAKE; then - # cmake 3 will make sure that the python interpreter and libraries match - wget --no-check-certificate http://www.cmake.org/files/v3.2/cmake-3.2.3-Linux-x86_64.sh -O cmake3.sh - chmod +x cmake3.sh - ./cmake3.sh --prefix=/usr/ --skip-license --exclude-subdir -fi - -# Install CUDA, if needed -if $WITH_CUDA; then - CUDA_URL=http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1204/x86_64/cuda-repo-ubuntu1204_6.5-14_amd64.deb - CUDA_FILE=/tmp/cuda_install.deb - curl $CUDA_URL -o $CUDA_FILE - dpkg -i $CUDA_FILE - rm -f $CUDA_FILE - apt-get -y update - # Install the minimal CUDA subpackages required to test Caffe build. - # For a full CUDA installation, add 'cuda' to the list of packages. - apt-get -y install cuda-core-6-5 cuda-cublas-6-5 cuda-cublas-dev-6-5 cuda-cudart-6-5 cuda-cudart-dev-6-5 cuda-curand-6-5 cuda-curand-dev-6-5 - # Create CUDA symlink at /usr/local/cuda - # (This would normally be created by the CUDA installer, but we create it - # manually since we did a partial installation.) - ln -s /usr/local/cuda-6.5 /usr/local/cuda -fi - -# Install LMDB -LMDB_URL=https://github.com/LMDB/lmdb/archive/LMDB_0.9.14.tar.gz -LMDB_FILE=/tmp/lmdb.tar.gz -pushd . -wget $LMDB_URL -O $LMDB_FILE -tar -C /tmp -xzvf $LMDB_FILE -cd /tmp/lmdb*/libraries/liblmdb/ -$MAKE -$MAKE install -popd -rm -f $LMDB_FILE - -# Install the Python runtime dependencies via miniconda (this is much faster -# than using pip for everything). -export PATH=$CONDA_DIR/bin:$PATH -# clear any cached conda (see #3786) -rm -rf $CONDA_DIR -if [ ! -d $CONDA_DIR ]; then - if [ "$PYTHON_VERSION" -eq "3" ]; then - wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - else - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh - fi - chmod +x miniconda.sh - ./miniconda.sh -b -p $CONDA_DIR - - conda update --yes conda - # The version of boost we're using for Python 3 depends on 3.4 for now. - if [ "$PYTHON_VERSION" -eq "3" ]; then - conda install --yes python=3.4 - fi - conda install --yes numpy scipy matplotlib scikit-image pip - # Let conda install boost (so that boost_python matches) - conda install --yes -c https://conda.binstar.org/menpo boost=1.56.0 -fi - -# install protobuf 3 (just use the miniconda3 directory to avoid having to setup the path again) -if [ "$PYTHON_VERSION" -eq "3" ] && [ ! -e "$CONDA_DIR/bin/protoc" ]; then - pushd . - wget https://github.com/google/protobuf/archive/v3.0.0-alpha-3.1.tar.gz -O protobuf-3.tar.gz - tar -C /tmp -xzvf protobuf-3.tar.gz - cd /tmp/protobuf-3*/ - ./autogen.sh - ./configure --prefix=$CONDA_DIR - $MAKE - $MAKE install - popd -fi - -if [ "$PYTHON_VERSION" -eq "3" ]; then - pip install --pre protobuf==3.0.0b2 -else - pip install protobuf -fi diff --git a/scripts/travis/travis_setup_makefile_config.sh b/scripts/travis/travis_setup_makefile_config.sh deleted file mode 100755 index 83aacf11fb0..00000000000 --- a/scripts/travis/travis_setup_makefile_config.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -set -e - -mv Makefile.config.example Makefile.config - -if $WITH_CUDA; then - # Only generate compute_50. - GENCODE="-gencode arch=compute_50,code=sm_50" - GENCODE="$GENCODE -gencode arch=compute_50,code=compute_50" - echo "CUDA_ARCH := $GENCODE" >> Makefile.config -fi - -# Remove IO library settings from Makefile.config -# to avoid conflicts with CI configuration -sed -i -e '/USE_LMDB/d' Makefile.config -sed -i -e '/USE_LEVELDB/d' Makefile.config -sed -i -e '/USE_OPENCV/d' Makefile.config - -cat << 'EOF' >> Makefile.config -# Travis' nvcc doesn't like newer boost versions -NVCCFLAGS := -Xcudafe --diag_suppress=cc_clobber_ignored -Xcudafe --diag_suppress=useless_using_declaration -Xcudafe --diag_suppress=set_but_not_used -ANACONDA_HOME := $(CONDA_DIR) -PYTHON_INCLUDE := $(ANACONDA_HOME)/include \ - $(ANACONDA_HOME)/include/python2.7 \ - $(ANACONDA_HOME)/lib/python2.7/site-packages/numpy/core/include -PYTHON_LIB := $(ANACONDA_HOME)/lib -INCLUDE_DIRS := $(PYTHON_INCLUDE) /usr/local/include -LIBRARY_DIRS := $(PYTHON_LIB) /usr/local/lib /usr/lib -WITH_PYTHON_LAYER := 1 -EOF From 26879320898aacfcb5236c725938e259788c10fc Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Wed, 25 May 2016 16:39:55 -0700 Subject: [PATCH 14/39] Remove misleading comment from a test file --- src/caffe/test/test_caffe_main.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/caffe/test/test_caffe_main.cpp b/src/caffe/test/test_caffe_main.cpp index fccf6f1613b..6473b74d0a6 100644 --- a/src/caffe/test/test_caffe_main.cpp +++ b/src/caffe/test/test_caffe_main.cpp @@ -1,6 +1,3 @@ -// The main caffe test code. Your test cpp code should include this hpp -// to allow a main function to be compiled into the binary. - #include "caffe/caffe.hpp" #include "caffe/test/test_caffe_main.hpp" From a355f9c9d0bf28ac81552ddb4873b01d09581fb3 Mon Sep 17 00:00:00 2001 From: Siddarth Malreddy Date: Thu, 26 May 2016 23:31:31 +0530 Subject: [PATCH 15/39] Check for non-empty ImageData filelist. --- src/caffe/layers/image_data_layer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/caffe/layers/image_data_layer.cpp b/src/caffe/layers/image_data_layer.cpp index 56d354655dc..7ee7dc40714 100644 --- a/src/caffe/layers/image_data_layer.cpp +++ b/src/caffe/layers/image_data_layer.cpp @@ -46,6 +46,8 @@ void ImageDataLayer::DataLayerSetUp(const vector*>& bottom, lines_.push_back(std::make_pair(line.substr(0, pos), label)); } + CHECK(!lines_.empty()) << "File is empty"; + if (this->layer_param_.image_data_param().shuffle()) { // randomly shuffle data LOG(INFO) << "Shuffling data"; From 09546dbe9130789f0571a76a36b0fc265cd81fe3 Mon Sep 17 00:00:00 2001 From: Lumin Zhou Date: Mon, 30 May 2016 04:14:42 +0000 Subject: [PATCH 16/39] fix spelling error in memory_data_layer.cpp --- src/caffe/layers/memory_data_layer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/caffe/layers/memory_data_layer.cpp b/src/caffe/layers/memory_data_layer.cpp index 82909874054..975f4841723 100644 --- a/src/caffe/layers/memory_data_layer.cpp +++ b/src/caffe/layers/memory_data_layer.cpp @@ -107,7 +107,7 @@ void MemoryDataLayer::set_batch_size(int new_size) { template void MemoryDataLayer::Forward_cpu(const vector*>& bottom, const vector*>& top) { - CHECK(data_) << "MemoryDataLayer needs to be initalized by calling Reset"; + CHECK(data_) << "MemoryDataLayer needs to be initialized by calling Reset"; top[0]->Reshape(batch_size_, channels_, height_, width_); top[1]->Reshape(batch_size_, 1, 1, 1); top[0]->set_cpu_data(data_ + pos_ * size_); From 994a033a725c23811dc50e4b2874450a45f2ecd1 Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Wed, 1 Jun 2016 10:37:14 -0700 Subject: [PATCH 17/39] Cache protobuf3 build in TravisCI --- .travis.yml | 3 +++ scripts/travis/install-deps.sh | 37 +++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 92d740cd88b..4849a7ac289 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,10 @@ env: - BUILD_NAME="cudnn-cmake" WITH_CMAKE=true WITH_CUDA=true WITH_CUDNN=true cache: + timeout: 604800 # 1 week apt: true + directories: + - ~/protobuf3 before_install: - source ./scripts/travis/defaults.sh diff --git a/scripts/travis/install-deps.sh b/scripts/travis/install-deps.sh index f7bfe4c4df9..ee16d36a7fc 100755 --- a/scripts/travis/install-deps.sh +++ b/scripts/travis/install-deps.sh @@ -40,25 +40,30 @@ else python3-skimage # build Protobuf3 since it's needed for Python3 - echo "Building protobuf3 from source ..." + PROTOBUF3_DIR=~/protobuf3 pushd . - PROTOBUF3_DIR=~/protobuf3-build - rm -rf $PROTOBUF3_DIR - mkdir $PROTOBUF3_DIR + if [ -d "$PROTOBUF3_DIR" ] && [ -e "$PROTOBUF3_DIR/src/protoc" ]; then + echo "Using cached protobuf3 build ..." + cd $PROTOBUF3_DIR + else + echo "Building protobuf3 from source ..." + rm -rf $PROTOBUF3_DIR + mkdir $PROTOBUF3_DIR - # install some more dependencies required to build protobuf3 - apt-get install -y --no-install-recommends \ - curl \ - dh-autoreconf \ - unzip + # install some more dependencies required to build protobuf3 + apt-get install -y --no-install-recommends \ + curl \ + dh-autoreconf \ + unzip - wget https://github.com/google/protobuf/archive/v3.0.0-beta-3.tar.gz -O protobuf3.tar.gz - tar -xzf protobuf3.tar.gz -C $PROTOBUF3_DIR --strip 1 - rm protobuf3.tar.gz - cd $PROTOBUF3_DIR - ./autogen.sh - ./configure --prefix=/usr - make --jobs=$NUM_THREADS + wget https://github.com/google/protobuf/archive/v3.0.0-beta-3.tar.gz -O protobuf3.tar.gz + tar -xzf protobuf3.tar.gz -C $PROTOBUF3_DIR --strip 1 + rm protobuf3.tar.gz + cd $PROTOBUF3_DIR + ./autogen.sh + ./configure --prefix=/usr + make --jobs=$NUM_THREADS + fi make install popd fi From 5f2d845fafc8883aa16b437b79fa52b39f8a0ddb Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Sun, 15 Feb 2015 14:28:01 -0800 Subject: [PATCH 18/39] Add RecurrentLayer: an abstract superclass for other recurrent layer types --- include/caffe/layers/recurrent_layer.hpp | 187 ++++++++++++++ src/caffe/layers/recurrent_layer.cpp | 295 +++++++++++++++++++++++ src/caffe/layers/recurrent_layer.cu | 44 ++++ src/caffe/proto/caffe.proto | 22 +- 4 files changed, 547 insertions(+), 1 deletion(-) create mode 100644 include/caffe/layers/recurrent_layer.hpp create mode 100644 src/caffe/layers/recurrent_layer.cpp create mode 100644 src/caffe/layers/recurrent_layer.cu diff --git a/include/caffe/layers/recurrent_layer.hpp b/include/caffe/layers/recurrent_layer.hpp new file mode 100644 index 00000000000..ca17371b994 --- /dev/null +++ b/include/caffe/layers/recurrent_layer.hpp @@ -0,0 +1,187 @@ +#ifndef CAFFE_RECURRENT_LAYER_HPP_ +#define CAFFE_RECURRENT_LAYER_HPP_ + +#include +#include +#include + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/layer.hpp" +#include "caffe/net.hpp" +#include "caffe/proto/caffe.pb.h" +#include "caffe/util/format.hpp" + +namespace caffe { + +template class RecurrentLayer; + +/** + * @brief An abstract class for implementing recurrent behavior inside of an + * unrolled network. This Layer type cannot be instantiated -- instead, + * you should use one of its implementations which defines the recurrent + * architecture, such as RNNLayer or LSTMLayer. + */ +template +class RecurrentLayer : public Layer { + public: + explicit RecurrentLayer(const LayerParameter& param) + : Layer(param) {} + virtual void LayerSetUp(const vector*>& bottom, + const vector*>& top); + virtual void Reshape(const vector*>& bottom, + const vector*>& top); + virtual void Reset(); + + virtual inline const char* type() const { return "Recurrent"; } + virtual inline int MinBottomBlobs() const { + int min_bottoms = 2; + if (this->layer_param_.recurrent_param().expose_hidden()) { + vector inputs; + this->RecurrentInputBlobNames(&inputs); + min_bottoms += inputs.size(); + } + return min_bottoms; + } + virtual inline int MaxBottomBlobs() const { return MinBottomBlobs() + 1; } + virtual inline int ExactNumTopBlobs() const { + int num_tops = 1; + if (this->layer_param_.recurrent_param().expose_hidden()) { + vector outputs; + this->RecurrentOutputBlobNames(&outputs); + num_tops += outputs.size(); + } + return num_tops; + } + + virtual inline bool AllowForceBackward(const int bottom_index) const { + // Can't propagate to sequence continuation indicators. + return bottom_index != 1; + } + + protected: + /** + * @brief Fills net_param with the recurrent network architecture. Subclasses + * should define this -- see RNNLayer and LSTMLayer for examples. + */ + virtual void FillUnrolledNet(NetParameter* net_param) const = 0; + + /** + * @brief Fills names with the names of the 0th timestep recurrent input + * Blob&s. Subclasses should define this -- see RNNLayer and LSTMLayer + * for examples. + */ + virtual void RecurrentInputBlobNames(vector* names) const = 0; + + /** + * @brief Fills shapes with the shapes of the recurrent input Blob&s. + * Subclasses should define this -- see RNNLayer and LSTMLayer + * for examples. + */ + virtual void RecurrentInputShapes(vector* shapes) const = 0; + + /** + * @brief Fills names with the names of the Tth timestep recurrent output + * Blob&s. Subclasses should define this -- see RNNLayer and LSTMLayer + * for examples. + */ + virtual void RecurrentOutputBlobNames(vector* names) const = 0; + + /** + * @brief Fills names with the names of the output blobs, concatenated across + * all timesteps. Should return a name for each top Blob. + * Subclasses should define this -- see RNNLayer and LSTMLayer for + * examples. + */ + virtual void OutputBlobNames(vector* names) const = 0; + + /** + * @param bottom input Blob vector (length 2-3) + * + * -# @f$ (T \times N \times ...) @f$ + * the time-varying input @f$ x @f$. After the first two axes, whose + * dimensions must correspond to the number of timesteps @f$ T @f$ and + * the number of independent streams @f$ N @f$, respectively, its + * dimensions may be arbitrary. Note that the ordering of dimensions -- + * @f$ (T \times N \times ...) @f$, rather than + * @f$ (N \times T \times ...) @f$ -- means that the @f$ N @f$ + * independent input streams must be "interleaved". + * + * -# @f$ (T \times N) @f$ + * the sequence continuation indicators @f$ \delta @f$. + * These inputs should be binary (0 or 1) indicators, where + * @f$ \delta_{t,n} = 0 @f$ means that timestep @f$ t @f$ of stream + * @f$ n @f$ is the beginning of a new sequence, and hence the previous + * hidden state @f$ h_{t-1} @f$ is multiplied by @f$ \delta_t = 0 @f$ + * and has no effect on the cell's output at timestep @f$ t @f$, and + * a value of @f$ \delta_{t,n} = 1 @f$ means that timestep @f$ t @f$ of + * stream @f$ n @f$ is a continuation from the previous timestep + * @f$ t-1 @f$, and the previous hidden state @f$ h_{t-1} @f$ affects the + * updated hidden state and output. + * + * -# @f$ (N \times ...) @f$ (optional) + * the static (non-time-varying) input @f$ x_{static} @f$. + * After the first axis, whose dimension must be the number of + * independent streams, its dimensions may be arbitrary. + * This is mathematically equivalent to using a time-varying input of + * @f$ x'_t = [x_t; x_{static}] @f$ -- i.e., tiling the static input + * across the @f$ T @f$ timesteps and concatenating with the time-varying + * input. Note that if this input is used, all timesteps in a single + * batch within a particular one of the @f$ N @f$ streams must share the + * same static input, even if the sequence continuation indicators + * suggest that difference sequences are ending and beginning within a + * single batch. This may require padding and/or truncation for uniform + * length. + * + * @param top output Blob vector (length 1) + * -# @f$ (T \times N \times D) @f$ + * the time-varying output @f$ y @f$, where @f$ D @f$ is + * recurrent_param.num_output(). + * Refer to documentation for particular RecurrentLayer implementations + * (such as RNNLayer and LSTMLayer) for the definition of @f$ y @f$. + */ + virtual void Forward_cpu(const vector*>& bottom, + const vector*>& top); + virtual void Forward_gpu(const vector*>& bottom, + const vector*>& top); + virtual void Backward_cpu(const vector*>& top, + const vector& propagate_down, const vector*>& bottom); + + /// @brief A Net to implement the Recurrent functionality. + shared_ptr > unrolled_net_; + + /// @brief The number of independent streams to process simultaneously. + int N_; + + /** + * @brief The number of timesteps in the layer's input, and the number of + * timesteps over which to backpropagate through time. + */ + int T_; + + /// @brief Whether the layer has a "static" input copied across all timesteps. + bool static_input_; + + /** + * @brief The last layer to run in the network. (Any later layers are losses + * added to force the recurrent net to do backprop.) + */ + int last_layer_index_; + + /** + * @brief Whether the layer's hidden state at the first and last timesteps + * are layer inputs and outputs, respectively. + */ + bool expose_hidden_; + + vector* > recur_input_blobs_; + vector* > recur_output_blobs_; + vector* > output_blobs_; + Blob* x_input_blob_; + Blob* x_static_input_blob_; + Blob* cont_input_blob_; +}; + +} // namespace caffe + +#endif // CAFFE_RECURRENT_LAYER_HPP_ diff --git a/src/caffe/layers/recurrent_layer.cpp b/src/caffe/layers/recurrent_layer.cpp new file mode 100644 index 00000000000..e0c82773392 --- /dev/null +++ b/src/caffe/layers/recurrent_layer.cpp @@ -0,0 +1,295 @@ +#include +#include + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/layer.hpp" +#include "caffe/layers/recurrent_layer.hpp" +#include "caffe/util/math_functions.hpp" + +namespace caffe { + +template +void RecurrentLayer::LayerSetUp(const vector*>& bottom, + const vector*>& top) { + CHECK_GE(bottom[0]->num_axes(), 2) + << "bottom[0] must have at least 2 axes -- (#timesteps, #streams, ...)"; + T_ = bottom[0]->shape(0); + N_ = bottom[0]->shape(1); + LOG(INFO) << "Initializing recurrent layer: assuming input batch contains " + << T_ << " timesteps of " << N_ << " independent streams."; + + CHECK_EQ(bottom[1]->num_axes(), 2) + << "bottom[1] must have exactly 2 axes -- (#timesteps, #streams)"; + CHECK_EQ(T_, bottom[1]->shape(0)); + CHECK_EQ(N_, bottom[1]->shape(1)); + + // If expose_hidden is set, we take as input and produce as output + // the hidden state blobs at the first and last timesteps. + expose_hidden_ = this->layer_param_.recurrent_param().expose_hidden(); + + // Get (recurrent) input/output names. + vector output_names; + OutputBlobNames(&output_names); + vector recur_input_names; + RecurrentInputBlobNames(&recur_input_names); + vector recur_output_names; + RecurrentOutputBlobNames(&recur_output_names); + const int num_recur_blobs = recur_input_names.size(); + CHECK_EQ(num_recur_blobs, recur_output_names.size()); + + // If provided, bottom[2] is a static input to the recurrent net. + const int num_hidden_exposed = expose_hidden_ * num_recur_blobs; + static_input_ = (bottom.size() > 2 + num_hidden_exposed); + if (static_input_) { + CHECK_GE(bottom[2]->num_axes(), 1); + CHECK_EQ(N_, bottom[2]->shape(0)); + } + + // Create a NetParameter; setup the inputs that aren't unique to particular + // recurrent architectures. + NetParameter net_param; + + LayerParameter* input_layer_param = net_param.add_layer(); + input_layer_param->set_type("Input"); + InputParameter* input_param = input_layer_param->mutable_input_param(); + input_layer_param->add_top("x"); + BlobShape input_shape; + for (int i = 0; i < bottom[0]->num_axes(); ++i) { + input_shape.add_dim(bottom[0]->shape(i)); + } + input_param->add_shape()->CopyFrom(input_shape); + + input_shape.Clear(); + for (int i = 0; i < bottom[1]->num_axes(); ++i) { + input_shape.add_dim(bottom[1]->shape(i)); + } + input_layer_param->add_top("cont"); + input_param->add_shape()->CopyFrom(input_shape); + + if (static_input_) { + input_shape.Clear(); + for (int i = 0; i < bottom[2]->num_axes(); ++i) { + input_shape.add_dim(bottom[2]->shape(i)); + } + input_layer_param->add_top("x_static"); + input_param->add_shape()->CopyFrom(input_shape); + } + + // Call the child's FillUnrolledNet implementation to specify the unrolled + // recurrent architecture. + this->FillUnrolledNet(&net_param); + + // Prepend this layer's name to the names of each layer in the unrolled net. + const string& layer_name = this->layer_param_.name(); + if (layer_name.size()) { + for (int i = 0; i < net_param.layer_size(); ++i) { + LayerParameter* layer = net_param.mutable_layer(i); + layer->set_name(layer_name + "_" + layer->name()); + } + } + + // Add "pseudo-losses" to all outputs to force backpropagation. + // (Setting force_backward is too aggressive as we may not need to backprop to + // all inputs, e.g., the sequence continuation indicators.) + vector pseudo_losses(output_names.size()); + for (int i = 0; i < output_names.size(); ++i) { + LayerParameter* layer = net_param.add_layer(); + pseudo_losses[i] = output_names[i] + "_pseudoloss"; + layer->set_name(pseudo_losses[i]); + layer->set_type("Reduction"); + layer->add_bottom(output_names[i]); + layer->add_top(pseudo_losses[i]); + layer->add_loss_weight(1); + } + + // Create the unrolled net. + unrolled_net_.reset(new Net(net_param)); + unrolled_net_->set_debug_info( + this->layer_param_.recurrent_param().debug_info()); + + // Setup pointers to the inputs. + x_input_blob_ = CHECK_NOTNULL(unrolled_net_->blob_by_name("x").get()); + cont_input_blob_ = CHECK_NOTNULL(unrolled_net_->blob_by_name("cont").get()); + if (static_input_) { + x_static_input_blob_ = + CHECK_NOTNULL(unrolled_net_->blob_by_name("x_static").get()); + } + + // Setup pointers to paired recurrent inputs/outputs. + recur_input_blobs_.resize(num_recur_blobs); + recur_output_blobs_.resize(num_recur_blobs); + for (int i = 0; i < recur_input_names.size(); ++i) { + recur_input_blobs_[i] = + CHECK_NOTNULL(unrolled_net_->blob_by_name(recur_input_names[i]).get()); + recur_output_blobs_[i] = + CHECK_NOTNULL(unrolled_net_->blob_by_name(recur_output_names[i]).get()); + } + + // Setup pointers to outputs. + CHECK_EQ(top.size() - num_hidden_exposed, output_names.size()) + << "OutputBlobNames must provide an output blob name for each top."; + output_blobs_.resize(output_names.size()); + for (int i = 0; i < output_names.size(); ++i) { + output_blobs_[i] = + CHECK_NOTNULL(unrolled_net_->blob_by_name(output_names[i]).get()); + } + + // We should have 2 inputs (x and cont), plus a number of recurrent inputs, + // plus maybe a static input. + CHECK_EQ(2 + num_recur_blobs + static_input_, + unrolled_net_->input_blobs().size()); + + // This layer's parameters are any parameters in the layers of the unrolled + // net. We only want one copy of each parameter, so check that the parameter + // is "owned" by the layer, rather than shared with another. + this->blobs_.clear(); + for (int i = 0; i < unrolled_net_->params().size(); ++i) { + if (unrolled_net_->param_owners()[i] == -1) { + LOG(INFO) << "Adding parameter " << i << ": " + << unrolled_net_->param_display_names()[i]; + this->blobs_.push_back(unrolled_net_->params()[i]); + } + } + // Check that param_propagate_down is set for all of the parameters in the + // unrolled net; set param_propagate_down to true in this layer. + for (int i = 0; i < unrolled_net_->layers().size(); ++i) { + for (int j = 0; j < unrolled_net_->layers()[i]->blobs().size(); ++j) { + CHECK(unrolled_net_->layers()[i]->param_propagate_down(j)) + << "param_propagate_down not set for layer " << i << ", param " << j; + } + } + this->param_propagate_down_.clear(); + this->param_propagate_down_.resize(this->blobs_.size(), true); + + // Set the diffs of recurrent outputs to 0 -- we can't backpropagate across + // batches. + for (int i = 0; i < recur_output_blobs_.size(); ++i) { + caffe_set(recur_output_blobs_[i]->count(), Dtype(0), + recur_output_blobs_[i]->mutable_cpu_diff()); + } + + // Check that the last output_names.size() layers are the pseudo-losses; + // set last_layer_index so that we don't actually run these layers. + const vector& layer_names = unrolled_net_->layer_names(); + last_layer_index_ = layer_names.size() - 1 - pseudo_losses.size(); + for (int i = last_layer_index_ + 1, j = 0; i < layer_names.size(); ++i, ++j) { + CHECK_EQ(layer_names[i], pseudo_losses[j]); + } +} + +template +void RecurrentLayer::Reshape(const vector*>& bottom, + const vector*>& top) { + CHECK_GE(bottom[0]->num_axes(), 2) + << "bottom[0] must have at least 2 axes -- (#timesteps, #streams, ...)"; + CHECK_EQ(T_, bottom[0]->shape(0)) << "input number of timesteps changed"; + N_ = bottom[0]->shape(1); + CHECK_EQ(bottom[1]->num_axes(), 2) + << "bottom[1] must have exactly 2 axes -- (#timesteps, #streams)"; + CHECK_EQ(T_, bottom[1]->shape(0)); + CHECK_EQ(N_, bottom[1]->shape(1)); + x_input_blob_->ReshapeLike(*bottom[0]); + vector cont_shape = bottom[1]->shape(); + cont_input_blob_->Reshape(cont_shape); + if (static_input_) { + x_static_input_blob_->ReshapeLike(*bottom[2]); + } + vector recur_input_shapes; + RecurrentInputShapes(&recur_input_shapes); + CHECK_EQ(recur_input_shapes.size(), recur_input_blobs_.size()); + for (int i = 0; i < recur_input_shapes.size(); ++i) { + recur_input_blobs_[i]->Reshape(recur_input_shapes[i]); + } + unrolled_net_->Reshape(); + x_input_blob_->ShareData(*bottom[0]); + x_input_blob_->ShareDiff(*bottom[0]); + cont_input_blob_->ShareData(*bottom[1]); + if (static_input_) { + x_static_input_blob_->ShareData(*bottom[2]); + x_static_input_blob_->ShareDiff(*bottom[2]); + } + if (expose_hidden_) { + const int bottom_offset = 2 + static_input_; + for (int i = bottom_offset, j = 0; i < bottom.size(); ++i, ++j) { + CHECK(recur_input_blobs_[j]->shape() == bottom[i]->shape()) + << "bottom[" << i << "] shape must match hidden state input shape: " + << recur_input_blobs_[j]->shape_string(); + recur_input_blobs_[j]->ShareData(*bottom[i]); + } + } + for (int i = 0; i < output_blobs_.size(); ++i) { + top[i]->ReshapeLike(*output_blobs_[i]); + top[i]->ShareData(*output_blobs_[i]); + top[i]->ShareDiff(*output_blobs_[i]); + } + if (expose_hidden_) { + const int top_offset = output_blobs_.size(); + for (int i = top_offset, j = 0; i < top.size(); ++i, ++j) { + top[i]->ReshapeLike(*recur_output_blobs_[j]); + } + } +} + +template +void RecurrentLayer::Reset() { + // "Reset" the hidden state of the net by zeroing out all recurrent outputs. + for (int i = 0; i < recur_output_blobs_.size(); ++i) { + caffe_set(recur_output_blobs_[i]->count(), Dtype(0), + recur_output_blobs_[i]->mutable_cpu_data()); + } +} + +template +void RecurrentLayer::Forward_cpu(const vector*>& bottom, + const vector*>& top) { + // Hacky fix for test time: reshare all the internal shared blobs, which may + // currently point to a stale owner blob that was dropped when Solver::Test + // called test_net->ShareTrainedLayersWith(net_.get()). + // TODO: somehow make this work non-hackily. + if (this->phase_ == TEST) { + unrolled_net_->ShareWeights(); + } + + DCHECK_EQ(recur_input_blobs_.size(), recur_output_blobs_.size()); + if (!expose_hidden_) { + for (int i = 0; i < recur_input_blobs_.size(); ++i) { + const int count = recur_input_blobs_[i]->count(); + DCHECK_EQ(count, recur_output_blobs_[i]->count()); + const Dtype* timestep_T_data = recur_output_blobs_[i]->cpu_data(); + Dtype* timestep_0_data = recur_input_blobs_[i]->mutable_cpu_data(); + caffe_copy(count, timestep_T_data, timestep_0_data); + } + } + + unrolled_net_->ForwardTo(last_layer_index_); + + if (expose_hidden_) { + const int top_offset = output_blobs_.size(); + for (int i = top_offset, j = 0; i < top.size(); ++i, ++j) { + top[i]->ShareData(*recur_output_blobs_[j]); + } + } +} + +template +void RecurrentLayer::Backward_cpu(const vector*>& top, + const vector& propagate_down, const vector*>& bottom) { + CHECK(!propagate_down[1]) << "Cannot backpropagate to sequence indicators."; + + // TODO: skip backpropagation to inputs and parameters inside the unrolled + // net according to propagate_down[0] and propagate_down[2]. For now just + // backprop to inputs and parameters unconditionally, as either the inputs or + // the parameters do need backward (or Net would have set + // layer_needs_backward_[i] == false for this layer). + unrolled_net_->BackwardFrom(last_layer_index_); +} + +#ifdef CPU_ONLY +STUB_GPU_FORWARD(RecurrentLayer, Forward); +#endif + +INSTANTIATE_CLASS(RecurrentLayer); + +} // namespace caffe diff --git a/src/caffe/layers/recurrent_layer.cu b/src/caffe/layers/recurrent_layer.cu new file mode 100644 index 00000000000..4dd2b0e2165 --- /dev/null +++ b/src/caffe/layers/recurrent_layer.cu @@ -0,0 +1,44 @@ +#include + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/layer.hpp" +#include "caffe/layers/recurrent_layer.hpp" +#include "caffe/util/math_functions.hpp" + +namespace caffe { + +template +void RecurrentLayer::Forward_gpu(const vector*>& bottom, + const vector*>& top) { + // Hacky fix for test time... reshare all the shared blobs. + // TODO: somehow make this work non-hackily. + if (this->phase_ == TEST) { + unrolled_net_->ShareWeights(); + } + + DCHECK_EQ(recur_input_blobs_.size(), recur_output_blobs_.size()); + if (!expose_hidden_) { + for (int i = 0; i < recur_input_blobs_.size(); ++i) { + const int count = recur_input_blobs_[i]->count(); + DCHECK_EQ(count, recur_output_blobs_[i]->count()); + const Dtype* timestep_T_data = recur_output_blobs_[i]->gpu_data(); + Dtype* timestep_0_data = recur_input_blobs_[i]->mutable_gpu_data(); + caffe_copy(count, timestep_T_data, timestep_0_data); + } + } + + unrolled_net_->ForwardTo(last_layer_index_); + + if (expose_hidden_) { + const int top_offset = output_blobs_.size(); + for (int i = top_offset, j = 0; i < top.size(); ++i, ++j) { + top[i]->ShareData(*recur_output_blobs_[j]); + } + } +} + +INSTANTIATE_LAYER_GPU_FORWARD(RecurrentLayer); + +} // namespace caffe diff --git a/src/caffe/proto/caffe.proto b/src/caffe/proto/caffe.proto index 15810718631..1556781cbc2 100644 --- a/src/caffe/proto/caffe.proto +++ b/src/caffe/proto/caffe.proto @@ -306,7 +306,7 @@ message ParamSpec { // NOTE // Update the next available ID when you add a new LayerParameter field. // -// LayerParameter next available layer-specific ID: 146 (last added: parameter_param) +// LayerParameter next available layer-specific ID: 147 (last added: recurrent_param) message LayerParameter { optional string name = 1; // the layer name optional string type = 2; // the layer type @@ -390,6 +390,7 @@ message LayerParameter { optional PowerParameter power_param = 122; optional PReLUParameter prelu_param = 131; optional PythonParameter python_param = 130; + optional RecurrentParameter recurrent_param = 146; optional ReductionParameter reduction_param = 136; optional ReLUParameter relu_param = 123; optional ReshapeParameter reshape_param = 133; @@ -928,6 +929,25 @@ message PythonParameter { optional bool share_in_parallel = 4 [default = false]; } +// Message that stores parameters used by RecurrentLayer +message RecurrentParameter { + // The dimension of the output (and usually hidden state) representation -- + // must be explicitly set to non-zero. + optional uint32 num_output = 1 [default = 0]; + + optional FillerParameter weight_filler = 2; // The filler for the weight + optional FillerParameter bias_filler = 3; // The filler for the bias + + // Whether to enable displaying debug_info in the unrolled recurrent net. + optional bool debug_info = 4 [default = false]; + + // Whether to add as additional inputs (bottoms) the initial hidden state + // blobs, and add as additional outputs (tops) the final timestep hidden state + // blobs. The number of additional bottom/top blobs required depends on the + // recurrent architecture -- e.g., 1 for RNNs, 2 for LSTMs. + optional bool expose_hidden = 5 [default = false]; +} + // Message that stores parameters used by ReductionLayer message ReductionParameter { enum ReductionOp { From cf5f369574dd51045c1c92625c0fe6694a031f2a Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Sun, 15 Feb 2015 14:56:50 -0800 Subject: [PATCH 19/39] Add RNNLayer, with tests --- include/caffe/layers/rnn_layer.hpp | 47 ++++++ src/caffe/layers/rnn_layer.cpp | 236 +++++++++++++++++++++++++++++ src/caffe/test/test_rnn_layer.cpp | 217 ++++++++++++++++++++++++++ 3 files changed, 500 insertions(+) create mode 100644 include/caffe/layers/rnn_layer.hpp create mode 100644 src/caffe/layers/rnn_layer.cpp create mode 100644 src/caffe/test/test_rnn_layer.cpp diff --git a/include/caffe/layers/rnn_layer.hpp b/include/caffe/layers/rnn_layer.hpp new file mode 100644 index 00000000000..6dce238ae17 --- /dev/null +++ b/include/caffe/layers/rnn_layer.hpp @@ -0,0 +1,47 @@ +#ifndef CAFFE_RNN_LAYER_HPP_ +#define CAFFE_RNN_LAYER_HPP_ + +#include +#include +#include + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/layer.hpp" +#include "caffe/layers/recurrent_layer.hpp" +#include "caffe/net.hpp" +#include "caffe/proto/caffe.pb.h" + +namespace caffe { + +template class RecurrentLayer; + +/** + * @brief Processes time-varying inputs using a simple recurrent neural network + * (RNN). Implemented as a network unrolling the RNN computation in time. + * + * Given time-varying inputs @f$ x_t @f$, computes hidden state @f$ + * h_t := \tanh[ W_{hh} h_{t_1} + W_{xh} x_t + b_h ] + * @f$, and outputs @f$ + * o_t := \tanh[ W_{ho} h_t + b_o ] + * @f$. + */ +template +class RNNLayer : public RecurrentLayer { + public: + explicit RNNLayer(const LayerParameter& param) + : RecurrentLayer(param) {} + + virtual inline const char* type() const { return "RNN"; } + + protected: + virtual void FillUnrolledNet(NetParameter* net_param) const; + virtual void RecurrentInputBlobNames(vector* names) const; + virtual void RecurrentOutputBlobNames(vector* names) const; + virtual void RecurrentInputShapes(vector* shapes) const; + virtual void OutputBlobNames(vector* names) const; +}; + +} // namespace caffe + +#endif // CAFFE_RNN_LAYER_HPP_ diff --git a/src/caffe/layers/rnn_layer.cpp b/src/caffe/layers/rnn_layer.cpp new file mode 100644 index 00000000000..f62ae8c77de --- /dev/null +++ b/src/caffe/layers/rnn_layer.cpp @@ -0,0 +1,236 @@ +#include +#include + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/layer.hpp" +#include "caffe/layers/rnn_layer.hpp" +#include "caffe/util/math_functions.hpp" + +namespace caffe { + +template +void RNNLayer::RecurrentInputBlobNames(vector* names) const { + names->resize(1); + (*names)[0] = "h_0"; +} + +template +void RNNLayer::RecurrentOutputBlobNames(vector* names) const { + names->resize(1); + (*names)[0] = "h_" + format_int(this->T_); +} + +template +void RNNLayer::RecurrentInputShapes(vector* shapes) const { + const int num_output = this->layer_param_.recurrent_param().num_output(); + shapes->resize(1); + (*shapes)[0].Clear(); + (*shapes)[0].add_dim(1); // a single timestep + (*shapes)[0].add_dim(this->N_); + (*shapes)[0].add_dim(num_output); +} + +template +void RNNLayer::OutputBlobNames(vector* names) const { + names->resize(1); + (*names)[0] = "o"; +} + +template +void RNNLayer::FillUnrolledNet(NetParameter* net_param) const { + const int num_output = this->layer_param_.recurrent_param().num_output(); + CHECK_GT(num_output, 0) << "num_output must be positive"; + const FillerParameter& weight_filler = + this->layer_param_.recurrent_param().weight_filler(); + const FillerParameter& bias_filler = + this->layer_param_.recurrent_param().bias_filler(); + + // Add generic LayerParameter's (without bottoms/tops) of layer types we'll + // use to save redundant code. + LayerParameter hidden_param; + hidden_param.set_type("InnerProduct"); + hidden_param.mutable_inner_product_param()->set_num_output(num_output); + hidden_param.mutable_inner_product_param()->set_bias_term(false); + hidden_param.mutable_inner_product_param()->set_axis(2); + hidden_param.mutable_inner_product_param()-> + mutable_weight_filler()->CopyFrom(weight_filler); + + LayerParameter biased_hidden_param(hidden_param); + biased_hidden_param.mutable_inner_product_param()->set_bias_term(true); + biased_hidden_param.mutable_inner_product_param()-> + mutable_bias_filler()->CopyFrom(bias_filler); + + LayerParameter sum_param; + sum_param.set_type("Eltwise"); + sum_param.mutable_eltwise_param()->set_operation( + EltwiseParameter_EltwiseOp_SUM); + + LayerParameter tanh_param; + tanh_param.set_type("TanH"); + + LayerParameter scale_param; + scale_param.set_type("Scale"); + scale_param.mutable_scale_param()->set_axis(0); + + LayerParameter slice_param; + slice_param.set_type("Slice"); + slice_param.mutable_slice_param()->set_axis(0); + + vector input_shapes; + RecurrentInputShapes(&input_shapes); + CHECK_EQ(1, input_shapes.size()); + + LayerParameter* input_layer_param = net_param->add_layer(); + input_layer_param->set_type("Input"); + InputParameter* input_param = input_layer_param->mutable_input_param(); + input_layer_param->add_top("h_0"); + input_param->add_shape()->CopyFrom(input_shapes[0]); + + LayerParameter* cont_slice_param = net_param->add_layer(); + cont_slice_param->CopyFrom(slice_param); + cont_slice_param->set_name("cont_slice"); + cont_slice_param->add_bottom("cont"); + cont_slice_param->mutable_slice_param()->set_axis(0); + + // Add layer to transform all timesteps of x to the hidden state dimension. + // W_xh_x = W_xh * x + b_h + { + LayerParameter* x_transform_param = net_param->add_layer(); + x_transform_param->CopyFrom(biased_hidden_param); + x_transform_param->set_name("x_transform"); + x_transform_param->add_param()->set_name("W_xh"); + x_transform_param->add_param()->set_name("b_h"); + x_transform_param->add_bottom("x"); + x_transform_param->add_top("W_xh_x"); + x_transform_param->add_propagate_down(true); + } + + if (this->static_input_) { + // Add layer to transform x_static to the hidden state dimension. + // W_xh_x_static = W_xh_static * x_static + LayerParameter* x_static_transform_param = net_param->add_layer(); + x_static_transform_param->CopyFrom(hidden_param); + x_static_transform_param->mutable_inner_product_param()->set_axis(1); + x_static_transform_param->set_name("W_xh_x_static"); + x_static_transform_param->add_param()->set_name("W_xh_static"); + x_static_transform_param->add_bottom("x_static"); + x_static_transform_param->add_top("W_xh_x_static_preshape"); + x_static_transform_param->add_propagate_down(true); + + LayerParameter* reshape_param = net_param->add_layer(); + reshape_param->set_type("Reshape"); + BlobShape* new_shape = + reshape_param->mutable_reshape_param()->mutable_shape(); + new_shape->add_dim(1); // One timestep. + // Should infer this->N as the dimension so we can reshape on batch size. + new_shape->add_dim(-1); + new_shape->add_dim( + x_static_transform_param->inner_product_param().num_output()); + reshape_param->set_name("W_xh_x_static_reshape"); + reshape_param->add_bottom("W_xh_x_static_preshape"); + reshape_param->add_top("W_xh_x_static"); + } + + LayerParameter* x_slice_param = net_param->add_layer(); + x_slice_param->CopyFrom(slice_param); + x_slice_param->set_name("W_xh_x_slice"); + x_slice_param->add_bottom("W_xh_x"); + + LayerParameter output_concat_layer; + output_concat_layer.set_name("o_concat"); + output_concat_layer.set_type("Concat"); + output_concat_layer.add_top("o"); + output_concat_layer.mutable_concat_param()->set_axis(0); + + for (int t = 1; t <= this->T_; ++t) { + string tm1s = format_int(t - 1); + string ts = format_int(t); + + cont_slice_param->add_top("cont_" + ts); + x_slice_param->add_top("W_xh_x_" + ts); + + // Add layer to flush the hidden state when beginning a new sequence, + // as indicated by cont_t. + // h_conted_{t-1} := cont_t * h_{t-1} + // + // Normally, cont_t is binary (i.e., 0 or 1), so: + // h_conted_{t-1} := h_{t-1} if cont_t == 1 + // 0 otherwise + { + LayerParameter* cont_h_param = net_param->add_layer(); + cont_h_param->CopyFrom(scale_param); + cont_h_param->set_name("h_conted_" + tm1s); + cont_h_param->add_bottom("h_" + tm1s); + cont_h_param->add_bottom("cont_" + ts); + cont_h_param->add_top("h_conted_" + tm1s); + } + + // Add layer to compute + // W_hh_h_{t-1} := W_hh * h_conted_{t-1} + { + LayerParameter* w_param = net_param->add_layer(); + w_param->CopyFrom(hidden_param); + w_param->set_name("W_hh_h_" + tm1s); + w_param->add_param()->set_name("W_hh"); + w_param->add_bottom("h_conted_" + tm1s); + w_param->add_top("W_hh_h_" + tm1s); + w_param->mutable_inner_product_param()->set_axis(2); + } + + // Add layers to compute + // h_t := \tanh( W_hh * h_conted_{t-1} + W_xh * x_t + b_h ) + // = \tanh( W_hh_h_{t-1} + W_xh_t ) + { + LayerParameter* h_input_sum_param = net_param->add_layer(); + h_input_sum_param->CopyFrom(sum_param); + h_input_sum_param->set_name("h_input_sum_" + ts); + h_input_sum_param->add_bottom("W_hh_h_" + tm1s); + h_input_sum_param->add_bottom("W_xh_x_" + ts); + if (this->static_input_) { + h_input_sum_param->add_bottom("W_xh_x_static"); + } + h_input_sum_param->add_top("h_neuron_input_" + ts); + } + { + LayerParameter* h_neuron_param = net_param->add_layer(); + h_neuron_param->CopyFrom(tanh_param); + h_neuron_param->set_name("h_neuron_" + ts); + h_neuron_param->add_bottom("h_neuron_input_" + ts); + h_neuron_param->add_top("h_" + ts); + } + + // Add layer to compute + // W_ho_h_t := W_ho * h_t + b_o + { + LayerParameter* w_param = net_param->add_layer(); + w_param->CopyFrom(biased_hidden_param); + w_param->set_name("W_ho_h_" + ts); + w_param->add_param()->set_name("W_ho"); + w_param->add_param()->set_name("b_o"); + w_param->add_bottom("h_" + ts); + w_param->add_top("W_ho_h_" + ts); + w_param->mutable_inner_product_param()->set_axis(2); + } + + // Add layers to compute + // o_t := \tanh( W_ho h_t + b_o) + // = \tanh( W_ho_h_t ) + { + LayerParameter* o_neuron_param = net_param->add_layer(); + o_neuron_param->CopyFrom(tanh_param); + o_neuron_param->set_name("o_neuron_" + ts); + o_neuron_param->add_bottom("W_ho_h_" + ts); + o_neuron_param->add_top("o_" + ts); + } + output_concat_layer.add_bottom("o_" + ts); + } // for (int t = 1; t <= this->T_; ++t) + + net_param->add_layer()->CopyFrom(output_concat_layer); +} + +INSTANTIATE_CLASS(RNNLayer); +REGISTER_LAYER_CLASS(RNN); + +} // namespace caffe diff --git a/src/caffe/test/test_rnn_layer.cpp b/src/caffe/test/test_rnn_layer.cpp new file mode 100644 index 00000000000..dd8952d62d6 --- /dev/null +++ b/src/caffe/test/test_rnn_layer.cpp @@ -0,0 +1,217 @@ +#include +#include + +#include "gtest/gtest.h" + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/layers/rnn_layer.hpp" + +#include "caffe/test/test_caffe_main.hpp" +#include "caffe/test/test_gradient_check_util.hpp" + +namespace caffe { + +template +class RNNLayerTest : public MultiDeviceTest { + typedef typename TypeParam::Dtype Dtype; + + protected: + RNNLayerTest() : num_output_(7) { + blob_bottom_vec_.push_back(&blob_bottom_); + blob_bottom_vec_.push_back(&blob_bottom_cont_); + blob_top_vec_.push_back(&blob_top_); + + ReshapeBlobs(1, 3); + + layer_param_.mutable_recurrent_param()->set_num_output(num_output_); + FillerParameter* weight_filler = + layer_param_.mutable_recurrent_param()->mutable_weight_filler(); + weight_filler->set_type("gaussian"); + weight_filler->set_std(0.2); + FillerParameter* bias_filler = + layer_param_.mutable_recurrent_param()->mutable_bias_filler(); + bias_filler->set_type("gaussian"); + bias_filler->set_std(0.1); + + layer_param_.set_phase(TEST); + } + + void ReshapeBlobs(int num_timesteps, int num_instances) { + blob_bottom_.Reshape(num_timesteps, num_instances, 3, 2); + blob_bottom_static_.Reshape(num_instances, 2, 3, 4); + vector shape(2); + shape[0] = num_timesteps; + shape[1] = num_instances; + blob_bottom_cont_.Reshape(shape); + + FillerParameter filler_param; + filler_param.set_min(-1); + filler_param.set_max(1); + UniformFiller filler(filler_param); + filler.Fill(&blob_bottom_); + } + + int num_output_; + LayerParameter layer_param_; + Blob blob_bottom_; + Blob blob_bottom_cont_; + Blob blob_bottom_static_; + Blob blob_top_; + vector*> blob_bottom_vec_; + vector*> blob_top_vec_; +}; + +TYPED_TEST_CASE(RNNLayerTest, TestDtypesAndDevices); + +TYPED_TEST(RNNLayerTest, TestSetUp) { + typedef typename TypeParam::Dtype Dtype; + RNNLayer layer(this->layer_param_); + layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + vector expected_top_shape = this->blob_bottom_.shape(); + expected_top_shape.resize(3); + expected_top_shape[2] = this->num_output_; + EXPECT_TRUE(this->blob_top_.shape() == expected_top_shape); +} + +TYPED_TEST(RNNLayerTest, TestForward) { + typedef typename TypeParam::Dtype Dtype; + const int kNumTimesteps = 3; + const int num = this->blob_bottom_.shape(1); + this->ReshapeBlobs(kNumTimesteps, num); + + // Fill the cont blob with <0, 1, 1, ..., 1>, + // indicating a sequence that begins at the first timestep + // then continues for the rest of the sequence. + for (int t = 0; t < kNumTimesteps; ++t) { + for (int n = 0; n < num; ++n) { + this->blob_bottom_cont_.mutable_cpu_data()[t * num + n] = t > 0; + } + } + + // Process the full sequence in a single batch. + FillerParameter filler_param; + filler_param.set_mean(0); + filler_param.set_std(1); + GaussianFiller sequence_filler(filler_param); + sequence_filler.Fill(&this->blob_bottom_); + shared_ptr > layer(new RNNLayer(this->layer_param_)); + Caffe::set_random_seed(1701); + layer->SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + LOG(INFO) << "Calling forward for full sequence RNN"; + layer->Forward(this->blob_bottom_vec_, this->blob_top_vec_); + + // Copy the inputs and outputs to reuse/check them later. + Blob bottom_copy(this->blob_bottom_.shape()); + bottom_copy.CopyFrom(this->blob_bottom_); + Blob top_copy(this->blob_top_.shape()); + top_copy.CopyFrom(this->blob_top_); + + // Process the batch one timestep at a time; + // check that we get the same result. + this->ReshapeBlobs(1, num); + layer.reset(new RNNLayer(this->layer_param_)); + Caffe::set_random_seed(1701); + layer->SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + const int bottom_count = this->blob_bottom_.count(); + const int top_count = this->blob_top_.count(); + const Dtype kEpsilon = 1e-5; + for (int t = 0; t < kNumTimesteps; ++t) { + caffe_copy(bottom_count, bottom_copy.cpu_data() + t * bottom_count, + this->blob_bottom_.mutable_cpu_data()); + for (int n = 0; n < num; ++n) { + this->blob_bottom_cont_.mutable_cpu_data()[n] = t > 0; + } + LOG(INFO) << "Calling forward for RNN timestep " << t; + layer->Forward(this->blob_bottom_vec_, this->blob_top_vec_); + for (int i = 0; i < top_count; ++i) { + ASSERT_LT(t * top_count + i, top_copy.count()); + EXPECT_NEAR(this->blob_top_.cpu_data()[i], + top_copy.cpu_data()[t * top_count + i], kEpsilon) + << "t = " << t << "; i = " << i; + } + } + + // Process the batch one timestep at a time with all cont blobs set to 0. + // Check that we get a different result, except in the first timestep. + Caffe::set_random_seed(1701); + layer.reset(new RNNLayer(this->layer_param_)); + layer->SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + for (int t = 0; t < kNumTimesteps; ++t) { + caffe_copy(bottom_count, bottom_copy.cpu_data() + t * bottom_count, + this->blob_bottom_.mutable_cpu_data()); + for (int n = 0; n < num; ++n) { + this->blob_bottom_cont_.mutable_cpu_data()[n] = 0; + } + LOG(INFO) << "Calling forward for RNN timestep " << t; + layer->Forward(this->blob_bottom_vec_, this->blob_top_vec_); + for (int i = 0; i < top_count; ++i) { + if (t == 0) { + EXPECT_NEAR(this->blob_top_.cpu_data()[i], + top_copy.cpu_data()[t * top_count + i], kEpsilon) + << "t = " << t << "; i = " << i; + } else { + EXPECT_NE(this->blob_top_.cpu_data()[i], + top_copy.cpu_data()[t * top_count + i]) + << "t = " << t << "; i = " << i; + } + } + } +} + +TYPED_TEST(RNNLayerTest, TestGradient) { + typedef typename TypeParam::Dtype Dtype; + RNNLayer layer(this->layer_param_); + GradientChecker checker(1e-2, 1e-3); + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 0); +} + +TYPED_TEST(RNNLayerTest, TestGradientNonZeroCont) { + typedef typename TypeParam::Dtype Dtype; + RNNLayer layer(this->layer_param_); + GradientChecker checker(1e-2, 1e-3); + for (int i = 0; i < this->blob_bottom_cont_.count(); ++i) { + this->blob_bottom_cont_.mutable_cpu_data()[i] = i > 2; + } + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 0); +} + +TYPED_TEST(RNNLayerTest, TestGradientNonZeroContBufferSize2) { + typedef typename TypeParam::Dtype Dtype; + this->ReshapeBlobs(2, 2); + // fill the values + FillerParameter filler_param; + UniformFiller filler(filler_param); + filler.Fill(&this->blob_bottom_); + RNNLayer layer(this->layer_param_); + GradientChecker checker(1e-2, 1e-3); + for (int i = 0; i < this->blob_bottom_cont_.count(); ++i) { + this->blob_bottom_cont_.mutable_cpu_data()[i] = i > 2; + } + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 0); +} + +TYPED_TEST(RNNLayerTest, TestGradientNonZeroContBufferSize2WithStaticInput) { + typedef typename TypeParam::Dtype Dtype; + this->ReshapeBlobs(2, 2); + FillerParameter filler_param; + UniformFiller filler(filler_param); + filler.Fill(&this->blob_bottom_); + filler.Fill(&this->blob_bottom_static_); + this->blob_bottom_vec_.push_back(&this->blob_bottom_static_); + RNNLayer layer(this->layer_param_); + GradientChecker checker(1e-2, 1e-3); + for (int i = 0; i < this->blob_bottom_cont_.count(); ++i) { + this->blob_bottom_cont_.mutable_cpu_data()[i] = i > 2; + } + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 0); + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 2); +} + +} // namespace caffe From 51a68f0a0e9e376597d7cabae709ff969ad30c98 Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Tue, 5 Apr 2016 09:56:04 -0700 Subject: [PATCH 20/39] Add LSTMLayer and LSTMUnitLayer, with tests --- include/caffe/layers/lstm_layer.hpp | 154 ++++++++++++++ src/caffe/layers/lstm_layer.cpp | 244 +++++++++++++++++++++++ src/caffe/layers/lstm_unit_layer.cpp | 131 ++++++++++++ src/caffe/layers/lstm_unit_layer.cu | 154 ++++++++++++++ src/caffe/test/test_lstm_layer.cpp | 288 +++++++++++++++++++++++++++ 5 files changed, 971 insertions(+) create mode 100644 include/caffe/layers/lstm_layer.hpp create mode 100644 src/caffe/layers/lstm_layer.cpp create mode 100644 src/caffe/layers/lstm_unit_layer.cpp create mode 100644 src/caffe/layers/lstm_unit_layer.cu create mode 100644 src/caffe/test/test_lstm_layer.cpp diff --git a/include/caffe/layers/lstm_layer.hpp b/include/caffe/layers/lstm_layer.hpp new file mode 100644 index 00000000000..a0e67c9d432 --- /dev/null +++ b/include/caffe/layers/lstm_layer.hpp @@ -0,0 +1,154 @@ +#ifndef CAFFE_LSTM_LAYER_HPP_ +#define CAFFE_LSTM_LAYER_HPP_ + +#include +#include +#include + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/layer.hpp" +#include "caffe/layers/recurrent_layer.hpp" +#include "caffe/net.hpp" +#include "caffe/proto/caffe.pb.h" + +namespace caffe { + +template class RecurrentLayer; + +/** + * @brief Processes sequential inputs using a "Long Short-Term Memory" (LSTM) + * [1] style recurrent neural network (RNN). Implemented by unrolling + * the LSTM computation through time. + * + * The specific architecture used in this implementation is as described in + * "Learning to Execute" [2], reproduced below: + * i_t := \sigmoid[ W_{hi} * h_{t-1} + W_{xi} * x_t + b_i ] + * f_t := \sigmoid[ W_{hf} * h_{t-1} + W_{xf} * x_t + b_f ] + * o_t := \sigmoid[ W_{ho} * h_{t-1} + W_{xo} * x_t + b_o ] + * g_t := \tanh[ W_{hg} * h_{t-1} + W_{xg} * x_t + b_g ] + * c_t := (f_t .* c_{t-1}) + (i_t .* g_t) + * h_t := o_t .* \tanh[c_t] + * In the implementation, the i, f, o, and g computations are performed as a + * single inner product. + * + * Notably, this implementation lacks the "diagonal" gates, as used in the + * LSTM architectures described by Alex Graves [3] and others. + * + * [1] Hochreiter, Sepp, and Schmidhuber, Jürgen. "Long short-term memory." + * Neural Computation 9, no. 8 (1997): 1735-1780. + * + * [2] Zaremba, Wojciech, and Sutskever, Ilya. "Learning to execute." + * arXiv preprint arXiv:1410.4615 (2014). + * + * [3] Graves, Alex. "Generating sequences with recurrent neural networks." + * arXiv preprint arXiv:1308.0850 (2013). + */ +template +class LSTMLayer : public RecurrentLayer { + public: + explicit LSTMLayer(const LayerParameter& param) + : RecurrentLayer(param) {} + + virtual inline const char* type() const { return "LSTM"; } + + protected: + virtual void FillUnrolledNet(NetParameter* net_param) const; + virtual void RecurrentInputBlobNames(vector* names) const; + virtual void RecurrentOutputBlobNames(vector* names) const; + virtual void RecurrentInputShapes(vector* shapes) const; + virtual void OutputBlobNames(vector* names) const; +}; + +/** + * @brief A helper for LSTMLayer: computes a single timestep of the + * non-linearity of the LSTM, producing the updated cell and hidden + * states. + */ +template +class LSTMUnitLayer : public Layer { + public: + explicit LSTMUnitLayer(const LayerParameter& param) + : Layer(param) {} + virtual void Reshape(const vector*>& bottom, + const vector*>& top); + + virtual inline const char* type() const { return "LSTMUnit"; } + virtual inline int ExactNumBottomBlobs() const { return 3; } + virtual inline int ExactNumTopBlobs() const { return 2; } + + virtual inline bool AllowForceBackward(const int bottom_index) const { + // Can't propagate to sequence continuation indicators. + return bottom_index != 2; + } + + protected: + /** + * @param bottom input Blob vector (length 3) + * -# @f$ (1 \times N \times D) @f$ + * the previous timestep cell state @f$ c_{t-1} @f$ + * -# @f$ (1 \times N \times 4D) @f$ + * the "gate inputs" @f$ [i_t', f_t', o_t', g_t'] @f$ + * -# @f$ (1 \times N) @f$ + * the sequence continuation indicators @f$ \delta_t @f$ + * @param top output Blob vector (length 2) + * -# @f$ (1 \times N \times D) @f$ + * the updated cell state @f$ c_t @f$, computed as: + * i_t := \sigmoid[i_t'] + * f_t := \sigmoid[f_t'] + * o_t := \sigmoid[o_t'] + * g_t := \tanh[g_t'] + * c_t := cont_t * (f_t .* c_{t-1}) + (i_t .* g_t) + * -# @f$ (1 \times N \times D) @f$ + * the updated hidden state @f$ h_t @f$, computed as: + * h_t := o_t .* \tanh[c_t] + */ + virtual void Forward_cpu(const vector*>& bottom, + const vector*>& top); + virtual void Forward_gpu(const vector*>& bottom, + const vector*>& top); + + /** + * @brief Computes the error gradient w.r.t. the LSTMUnit inputs. + * + * @param top output Blob vector (length 2), providing the error gradient with + * respect to the outputs + * -# @f$ (1 \times N \times D) @f$: + * containing error gradients @f$ \frac{\partial E}{\partial c_t} @f$ + * with respect to the updated cell state @f$ c_t @f$ + * -# @f$ (1 \times N \times D) @f$: + * containing error gradients @f$ \frac{\partial E}{\partial h_t} @f$ + * with respect to the updated cell state @f$ h_t @f$ + * @param propagate_down see Layer::Backward. + * @param bottom input Blob vector (length 3), into which the error gradients + * with respect to the LSTMUnit inputs @f$ c_{t-1} @f$ and the gate + * inputs are computed. Computatation of the error gradients w.r.t. + * the sequence indicators is not implemented. + * -# @f$ (1 \times N \times D) @f$ + * the error gradient w.r.t. the previous timestep cell state + * @f$ c_{t-1} @f$ + * -# @f$ (1 \times N \times 4D) @f$ + * the error gradient w.r.t. the "gate inputs" + * @f$ [ + * \frac{\partial E}{\partial i_t} + * \frac{\partial E}{\partial f_t} + * \frac{\partial E}{\partial o_t} + * \frac{\partial E}{\partial g_t} + * ] @f$ + * -# @f$ (1 \times 1 \times N) @f$ + * the gradient w.r.t. the sequence continuation indicators + * @f$ \delta_t @f$ is currently not computed. + */ + virtual void Backward_cpu(const vector*>& top, + const vector& propagate_down, const vector*>& bottom); + virtual void Backward_gpu(const vector*>& top, + const vector& propagate_down, const vector*>& bottom); + + /// @brief The hidden and output dimension. + int hidden_dim_; + Blob X_acts_; +}; + +} // namespace caffe + +#endif // CAFFE_LSTM_LAYER_HPP_ diff --git a/src/caffe/layers/lstm_layer.cpp b/src/caffe/layers/lstm_layer.cpp new file mode 100644 index 00000000000..da48dba4c05 --- /dev/null +++ b/src/caffe/layers/lstm_layer.cpp @@ -0,0 +1,244 @@ +#include +#include + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/layer.hpp" +#include "caffe/layers/lstm_layer.hpp" +#include "caffe/util/math_functions.hpp" + +namespace caffe { + +template +void LSTMLayer::RecurrentInputBlobNames(vector* names) const { + names->resize(2); + (*names)[0] = "h_0"; + (*names)[1] = "c_0"; +} + +template +void LSTMLayer::RecurrentOutputBlobNames(vector* names) const { + names->resize(2); + (*names)[0] = "h_" + format_int(this->T_); + (*names)[1] = "c_T"; +} + +template +void LSTMLayer::RecurrentInputShapes(vector* shapes) const { + const int num_output = this->layer_param_.recurrent_param().num_output(); + const int num_blobs = 2; + shapes->resize(num_blobs); + for (int i = 0; i < num_blobs; ++i) { + (*shapes)[i].Clear(); + (*shapes)[i].add_dim(1); // a single timestep + (*shapes)[i].add_dim(this->N_); + (*shapes)[i].add_dim(num_output); + } +} + +template +void LSTMLayer::OutputBlobNames(vector* names) const { + names->resize(1); + (*names)[0] = "h"; +} + +template +void LSTMLayer::FillUnrolledNet(NetParameter* net_param) const { + const int num_output = this->layer_param_.recurrent_param().num_output(); + CHECK_GT(num_output, 0) << "num_output must be positive"; + const FillerParameter& weight_filler = + this->layer_param_.recurrent_param().weight_filler(); + const FillerParameter& bias_filler = + this->layer_param_.recurrent_param().bias_filler(); + + // Add generic LayerParameter's (without bottoms/tops) of layer types we'll + // use to save redundant code. + LayerParameter hidden_param; + hidden_param.set_type("InnerProduct"); + hidden_param.mutable_inner_product_param()->set_num_output(num_output * 4); + hidden_param.mutable_inner_product_param()->set_bias_term(false); + hidden_param.mutable_inner_product_param()->set_axis(2); + hidden_param.mutable_inner_product_param()-> + mutable_weight_filler()->CopyFrom(weight_filler); + + LayerParameter biased_hidden_param(hidden_param); + biased_hidden_param.mutable_inner_product_param()->set_bias_term(true); + biased_hidden_param.mutable_inner_product_param()-> + mutable_bias_filler()->CopyFrom(bias_filler); + + LayerParameter sum_param; + sum_param.set_type("Eltwise"); + sum_param.mutable_eltwise_param()->set_operation( + EltwiseParameter_EltwiseOp_SUM); + + LayerParameter scale_param; + scale_param.set_type("Scale"); + scale_param.mutable_scale_param()->set_axis(0); + + LayerParameter slice_param; + slice_param.set_type("Slice"); + slice_param.mutable_slice_param()->set_axis(0); + + LayerParameter split_param; + split_param.set_type("Split"); + + vector input_shapes; + RecurrentInputShapes(&input_shapes); + CHECK_EQ(2, input_shapes.size()); + + LayerParameter* input_layer_param = net_param->add_layer(); + input_layer_param->set_type("Input"); + InputParameter* input_param = input_layer_param->mutable_input_param(); + + input_layer_param->add_top("c_0"); + input_param->add_shape()->CopyFrom(input_shapes[0]); + + input_layer_param->add_top("h_0"); + input_param->add_shape()->CopyFrom(input_shapes[1]); + + LayerParameter* cont_slice_param = net_param->add_layer(); + cont_slice_param->CopyFrom(slice_param); + cont_slice_param->set_name("cont_slice"); + cont_slice_param->add_bottom("cont"); + cont_slice_param->mutable_slice_param()->set_axis(0); + + // Add layer to transform all timesteps of x to the hidden state dimension. + // W_xc_x = W_xc * x + b_c + { + LayerParameter* x_transform_param = net_param->add_layer(); + x_transform_param->CopyFrom(biased_hidden_param); + x_transform_param->set_name("x_transform"); + x_transform_param->add_param()->set_name("W_xc"); + x_transform_param->add_param()->set_name("b_c"); + x_transform_param->add_bottom("x"); + x_transform_param->add_top("W_xc_x"); + x_transform_param->add_propagate_down(true); + } + + if (this->static_input_) { + // Add layer to transform x_static to the gate dimension. + // W_xc_x_static = W_xc_static * x_static + LayerParameter* x_static_transform_param = net_param->add_layer(); + x_static_transform_param->CopyFrom(hidden_param); + x_static_transform_param->mutable_inner_product_param()->set_axis(1); + x_static_transform_param->set_name("W_xc_x_static"); + x_static_transform_param->add_param()->set_name("W_xc_static"); + x_static_transform_param->add_bottom("x_static"); + x_static_transform_param->add_top("W_xc_x_static_preshape"); + x_static_transform_param->add_propagate_down(true); + + LayerParameter* reshape_param = net_param->add_layer(); + reshape_param->set_type("Reshape"); + BlobShape* new_shape = + reshape_param->mutable_reshape_param()->mutable_shape(); + new_shape->add_dim(1); // One timestep. + // Should infer this->N as the dimension so we can reshape on batch size. + new_shape->add_dim(-1); + new_shape->add_dim( + x_static_transform_param->inner_product_param().num_output()); + reshape_param->set_name("W_xc_x_static_reshape"); + reshape_param->add_bottom("W_xc_x_static_preshape"); + reshape_param->add_top("W_xc_x_static"); + } + + LayerParameter* x_slice_param = net_param->add_layer(); + x_slice_param->CopyFrom(slice_param); + x_slice_param->add_bottom("W_xc_x"); + x_slice_param->set_name("W_xc_x_slice"); + + LayerParameter output_concat_layer; + output_concat_layer.set_name("h_concat"); + output_concat_layer.set_type("Concat"); + output_concat_layer.add_top("h"); + output_concat_layer.mutable_concat_param()->set_axis(0); + + for (int t = 1; t <= this->T_; ++t) { + string tm1s = format_int(t - 1); + string ts = format_int(t); + + cont_slice_param->add_top("cont_" + ts); + x_slice_param->add_top("W_xc_x_" + ts); + + // Add layers to flush the hidden state when beginning a new + // sequence, as indicated by cont_t. + // h_conted_{t-1} := cont_t * h_{t-1} + // + // Normally, cont_t is binary (i.e., 0 or 1), so: + // h_conted_{t-1} := h_{t-1} if cont_t == 1 + // 0 otherwise + { + LayerParameter* cont_h_param = net_param->add_layer(); + cont_h_param->CopyFrom(scale_param); + cont_h_param->set_name("h_conted_" + tm1s); + cont_h_param->add_bottom("h_" + tm1s); + cont_h_param->add_bottom("cont_" + ts); + cont_h_param->add_top("h_conted_" + tm1s); + } + + // Add layer to compute + // W_hc_h_{t-1} := W_hc * h_conted_{t-1} + { + LayerParameter* w_param = net_param->add_layer(); + w_param->CopyFrom(hidden_param); + w_param->set_name("transform_" + ts); + w_param->add_param()->set_name("W_hc"); + w_param->add_bottom("h_conted_" + tm1s); + w_param->add_top("W_hc_h_" + tm1s); + w_param->mutable_inner_product_param()->set_axis(2); + } + + // Add the outputs of the linear transformations to compute the gate input. + // gate_input_t := W_hc * h_conted_{t-1} + W_xc * x_t + b_c + // = W_hc_h_{t-1} + W_xc_x_t + b_c + { + LayerParameter* input_sum_layer = net_param->add_layer(); + input_sum_layer->CopyFrom(sum_param); + input_sum_layer->set_name("gate_input_" + ts); + input_sum_layer->add_bottom("W_hc_h_" + tm1s); + input_sum_layer->add_bottom("W_xc_x_" + ts); + if (this->static_input_) { + input_sum_layer->add_bottom("W_xc_x_static"); + } + input_sum_layer->add_top("gate_input_" + ts); + } + + // Add LSTMUnit layer to compute the cell & hidden vectors c_t and h_t. + // Inputs: c_{t-1}, gate_input_t = (i_t, f_t, o_t, g_t), cont_t + // Outputs: c_t, h_t + // [ i_t' ] + // [ f_t' ] := gate_input_t + // [ o_t' ] + // [ g_t' ] + // i_t := \sigmoid[i_t'] + // f_t := \sigmoid[f_t'] + // o_t := \sigmoid[o_t'] + // g_t := \tanh[g_t'] + // c_t := cont_t * (f_t .* c_{t-1}) + (i_t .* g_t) + // h_t := o_t .* \tanh[c_t] + { + LayerParameter* lstm_unit_param = net_param->add_layer(); + lstm_unit_param->set_type("LSTMUnit"); + lstm_unit_param->add_bottom("c_" + tm1s); + lstm_unit_param->add_bottom("gate_input_" + ts); + lstm_unit_param->add_bottom("cont_" + ts); + lstm_unit_param->add_top("c_" + ts); + lstm_unit_param->add_top("h_" + ts); + lstm_unit_param->set_name("unit_" + ts); + } + output_concat_layer.add_bottom("h_" + ts); + } // for (int t = 1; t <= this->T_; ++t) + + { + LayerParameter* c_T_copy_param = net_param->add_layer(); + c_T_copy_param->CopyFrom(split_param); + c_T_copy_param->add_bottom("c_" + format_int(this->T_)); + c_T_copy_param->add_top("c_T"); + } + net_param->add_layer()->CopyFrom(output_concat_layer); +} + +INSTANTIATE_CLASS(LSTMLayer); +REGISTER_LAYER_CLASS(LSTM); + +} // namespace caffe diff --git a/src/caffe/layers/lstm_unit_layer.cpp b/src/caffe/layers/lstm_unit_layer.cpp new file mode 100644 index 00000000000..277c031ad15 --- /dev/null +++ b/src/caffe/layers/lstm_unit_layer.cpp @@ -0,0 +1,131 @@ +#include +#include +#include + +#include "caffe/layer.hpp" +#include "caffe/layers/lstm_layer.hpp" + +namespace caffe { + +template +inline Dtype sigmoid(Dtype x) { + return 1. / (1. + exp(-x)); +} + +template +inline Dtype tanh(Dtype x) { + return 2. * sigmoid(2. * x) - 1.; +} + +template +void LSTMUnitLayer::Reshape(const vector*>& bottom, + const vector*>& top) { + const int num_instances = bottom[0]->shape(1); + for (int i = 0; i < bottom.size(); ++i) { + if (i == 2) { + CHECK_EQ(2, bottom[i]->num_axes()); + } else { + CHECK_EQ(3, bottom[i]->num_axes()); + } + CHECK_EQ(1, bottom[i]->shape(0)); + CHECK_EQ(num_instances, bottom[i]->shape(1)); + } + hidden_dim_ = bottom[0]->shape(2); + CHECK_EQ(num_instances, bottom[1]->shape(1)); + CHECK_EQ(4 * hidden_dim_, bottom[1]->shape(2)); + top[0]->ReshapeLike(*bottom[0]); + top[1]->ReshapeLike(*bottom[0]); + X_acts_.ReshapeLike(*bottom[1]); +} + +template +void LSTMUnitLayer::Forward_cpu(const vector*>& bottom, + const vector*>& top) { + const int num = bottom[0]->shape(1); + const int x_dim = hidden_dim_ * 4; + const Dtype* C_prev = bottom[0]->cpu_data(); + const Dtype* X = bottom[1]->cpu_data(); + const Dtype* cont = bottom[2]->cpu_data(); + Dtype* C = top[0]->mutable_cpu_data(); + Dtype* H = top[1]->mutable_cpu_data(); + for (int n = 0; n < num; ++n) { + for (int d = 0; d < hidden_dim_; ++d) { + const Dtype i = sigmoid(X[d]); + const Dtype f = (*cont == 0) ? 0 : + (*cont * sigmoid(X[1 * hidden_dim_ + d])); + const Dtype o = sigmoid(X[2 * hidden_dim_ + d]); + const Dtype g = tanh(X[3 * hidden_dim_ + d]); + const Dtype c_prev = C_prev[d]; + const Dtype c = f * c_prev + i * g; + C[d] = c; + const Dtype tanh_c = tanh(c); + H[d] = o * tanh_c; + } + C_prev += hidden_dim_; + X += x_dim; + C += hidden_dim_; + H += hidden_dim_; + ++cont; + } +} + +template +void LSTMUnitLayer::Backward_cpu(const vector*>& top, + const vector& propagate_down, const vector*>& bottom) { + CHECK(!propagate_down[2]) << "Cannot backpropagate to sequence indicators."; + if (!propagate_down[0] && !propagate_down[1]) { return; } + + const int num = bottom[0]->shape(1); + const int x_dim = hidden_dim_ * 4; + const Dtype* C_prev = bottom[0]->cpu_data(); + const Dtype* X = bottom[1]->cpu_data(); + const Dtype* cont = bottom[2]->cpu_data(); + const Dtype* C = top[0]->cpu_data(); + const Dtype* H = top[1]->cpu_data(); + const Dtype* C_diff = top[0]->cpu_diff(); + const Dtype* H_diff = top[1]->cpu_diff(); + Dtype* C_prev_diff = bottom[0]->mutable_cpu_diff(); + Dtype* X_diff = bottom[1]->mutable_cpu_diff(); + for (int n = 0; n < num; ++n) { + for (int d = 0; d < hidden_dim_; ++d) { + const Dtype i = sigmoid(X[d]); + const Dtype f = (*cont == 0) ? 0 : + (*cont * sigmoid(X[1 * hidden_dim_ + d])); + const Dtype o = sigmoid(X[2 * hidden_dim_ + d]); + const Dtype g = tanh(X[3 * hidden_dim_ + d]); + const Dtype c_prev = C_prev[d]; + const Dtype c = C[d]; + const Dtype tanh_c = tanh(c); + Dtype* c_prev_diff = C_prev_diff + d; + Dtype* i_diff = X_diff + d; + Dtype* f_diff = X_diff + 1 * hidden_dim_ + d; + Dtype* o_diff = X_diff + 2 * hidden_dim_ + d; + Dtype* g_diff = X_diff + 3 * hidden_dim_ + d; + const Dtype c_term_diff = + C_diff[d] + H_diff[d] * o * (1 - tanh_c * tanh_c); + *c_prev_diff = c_term_diff * f; + *i_diff = c_term_diff * g * i * (1 - i); + *f_diff = c_term_diff * c_prev * f * (1 - f); + *o_diff = H_diff[d] * tanh_c * o * (1 - o); + *g_diff = c_term_diff * i * (1 - g * g); + } + C_prev += hidden_dim_; + X += x_dim; + C += hidden_dim_; + H += hidden_dim_; + C_diff += hidden_dim_; + H_diff += hidden_dim_; + X_diff += x_dim; + C_prev_diff += hidden_dim_; + ++cont; + } +} + +#ifdef CPU_ONLY +STUB_GPU(LSTMUnitLayer); +#endif + +INSTANTIATE_CLASS(LSTMUnitLayer); +REGISTER_LAYER_CLASS(LSTMUnit); + +} // namespace caffe diff --git a/src/caffe/layers/lstm_unit_layer.cu b/src/caffe/layers/lstm_unit_layer.cu new file mode 100644 index 00000000000..15bb451d9e0 --- /dev/null +++ b/src/caffe/layers/lstm_unit_layer.cu @@ -0,0 +1,154 @@ +#include +#include +#include + +#include "caffe/layer.hpp" +#include "caffe/layers/lstm_layer.hpp" + +namespace caffe { + +template +__device__ Dtype sigmoid(const Dtype x) { + return Dtype(1) / (Dtype(1) + exp(-x)); +} + +template +__device__ Dtype tanh(const Dtype x) { + return Dtype(2) * sigmoid(Dtype(2) * x) - Dtype(1); +} + +template +__global__ void LSTMActsForward(const int nthreads, const int dim, + const Dtype* X, Dtype* X_acts) { + CUDA_KERNEL_LOOP(index, nthreads) { + const int x_dim = 4 * dim; + const int d = index % x_dim; + if (d < 3 * dim) { + X_acts[index] = sigmoid(X[index]); + } else { + X_acts[index] = tanh(X[index]); + } + } +} + +template +__global__ void LSTMUnitForward(const int nthreads, const int dim, + const Dtype* C_prev, const Dtype* X, const Dtype* cont, + Dtype* C, Dtype* H) { + CUDA_KERNEL_LOOP(index, nthreads) { + const int n = index / dim; + const int d = index % dim; + const Dtype* X_offset = X + 4 * dim * n; + const Dtype i = X_offset[d]; + const Dtype f = X_offset[1 * dim + d]; + const Dtype o = X_offset[2 * dim + d]; + const Dtype g = X_offset[3 * dim + d]; + const Dtype c_prev = C_prev[index]; + const Dtype c = cont[n] * f * c_prev + i * g; + C[index] = c; + const Dtype tanh_c = tanh(c); + H[index] = o * tanh_c; + } +} + +template +void LSTMUnitLayer::Forward_gpu(const vector*>& bottom, + const vector*>& top) { + const int count = top[1]->count(); + const Dtype* C_prev = bottom[0]->gpu_data(); + const Dtype* X = bottom[1]->gpu_data(); + const Dtype* cont = bottom[2]->gpu_data(); + Dtype* X_acts = X_acts_.mutable_gpu_data(); + Dtype* C = top[0]->mutable_gpu_data(); + Dtype* H = top[1]->mutable_gpu_data(); + const int X_count = bottom[1]->count(); + // NOLINT_NEXT_LINE(whitespace/operators) + LSTMActsForward<<>>( + X_count, hidden_dim_, X, X_acts); + CUDA_POST_KERNEL_CHECK; + // NOLINT_NEXT_LINE(whitespace/operators) + LSTMUnitForward<<>>( + count, hidden_dim_, C_prev, X_acts, cont, C, H); + CUDA_POST_KERNEL_CHECK; +} + +template +__global__ void LSTMUnitBackward(const int nthreads, const int dim, + const Dtype* C_prev, const Dtype* X, const Dtype* C, const Dtype* H, + const Dtype* cont, const Dtype* C_diff, const Dtype* H_diff, + Dtype* C_prev_diff, Dtype* X_diff) { + CUDA_KERNEL_LOOP(index, nthreads) { + const int n = index / dim; + const int d = index % dim; + const Dtype* X_offset = X + 4 * dim * n; + const Dtype i = X_offset[d]; + const Dtype f = X_offset[1 * dim + d]; + const Dtype o = X_offset[2 * dim + d]; + const Dtype g = X_offset[3 * dim + d]; + const Dtype c_prev = C_prev[index]; + const Dtype c = C[index]; + const Dtype tanh_c = tanh(c); + Dtype* c_prev_diff = C_prev_diff + index; + Dtype* X_diff_offset = X_diff + 4 * dim * n; + Dtype* i_diff = X_diff_offset + d; + Dtype* f_diff = X_diff_offset + 1 * dim + d; + Dtype* o_diff = X_diff_offset + 2 * dim + d; + Dtype* g_diff = X_diff_offset + 3 * dim + d; + const Dtype c_term_diff = + C_diff[index] + H_diff[index] * o * (1 - tanh_c * tanh_c); + const Dtype cont_n = cont[n]; + *c_prev_diff = cont_n * c_term_diff * f; + *i_diff = c_term_diff * g; + *f_diff = cont_n * c_term_diff * c_prev; + *o_diff = H_diff[index] * tanh_c; + *g_diff = c_term_diff * i; + } +} + +template +__global__ void LSTMActsBackward(const int nthreads, const int dim, + const Dtype* X_acts, const Dtype* X_acts_diff, Dtype* X_diff) { + CUDA_KERNEL_LOOP(index, nthreads) { + const int x_dim = 4 * dim; + const int d = index % x_dim; + const Dtype X_act = X_acts[index]; + if (d < 3 * dim) { + X_diff[index] = X_acts_diff[index] * X_act * (Dtype(1) - X_act); + } else { + X_diff[index] = X_acts_diff[index] * (Dtype(1) - X_act * X_act); + } + } +} + +template +void LSTMUnitLayer::Backward_gpu(const vector*>& top, + const vector& propagate_down, + const vector*>& bottom) { + CHECK(!propagate_down[2]) << "Cannot backpropagate to sequence indicators."; + if (!propagate_down[0] && !propagate_down[1]) { return; } + + const int count = top[1]->count(); + const Dtype* C_prev = bottom[0]->gpu_data(); + const Dtype* X_acts = X_acts_.gpu_data(); + const Dtype* cont = bottom[2]->gpu_data(); + const Dtype* C = top[0]->gpu_data(); + const Dtype* H = top[1]->gpu_data(); + const Dtype* C_diff = top[0]->gpu_diff(); + const Dtype* H_diff = top[1]->gpu_diff(); + Dtype* C_prev_diff = bottom[0]->mutable_gpu_diff(); + Dtype* X_acts_diff = X_acts_.mutable_gpu_diff(); + LSTMUnitBackward // NOLINT_NEXT_LINE(whitespace/operators) + <<>>(count, hidden_dim_, + C_prev, X_acts, C, H, cont, C_diff, H_diff, C_prev_diff, X_acts_diff); + CUDA_POST_KERNEL_CHECK; + const int X_count = bottom[1]->count(); + Dtype* X_diff = bottom[1]->mutable_gpu_diff(); + LSTMActsBackward // NOLINT_NEXT_LINE(whitespace/operators) + <<>>( + X_count, hidden_dim_, X_acts, X_acts_diff, X_diff); + CUDA_POST_KERNEL_CHECK; +} + +INSTANTIATE_LAYER_GPU_FUNCS(LSTMUnitLayer); + +} // namespace caffe diff --git a/src/caffe/test/test_lstm_layer.cpp b/src/caffe/test/test_lstm_layer.cpp new file mode 100644 index 00000000000..51905baafac --- /dev/null +++ b/src/caffe/test/test_lstm_layer.cpp @@ -0,0 +1,288 @@ +#include +#include + +#include "gtest/gtest.h" + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/layers/lstm_layer.hpp" + +#include "caffe/test/test_caffe_main.hpp" +#include "caffe/test/test_gradient_check_util.hpp" + +namespace caffe { + +template +class LSTMLayerTest : public MultiDeviceTest { + typedef typename TypeParam::Dtype Dtype; + + protected: + LSTMLayerTest() : num_output_(7) { + blob_bottom_vec_.push_back(&blob_bottom_); + blob_bottom_vec_.push_back(&blob_bottom_cont_); + blob_top_vec_.push_back(&blob_top_); + unit_blob_bottom_vec_.push_back(&unit_blob_bottom_c_prev_); + unit_blob_bottom_vec_.push_back(&unit_blob_bottom_x_); + unit_blob_bottom_vec_.push_back(&unit_blob_bottom_cont_); + unit_blob_top_vec_.push_back(&unit_blob_top_c_); + unit_blob_top_vec_.push_back(&unit_blob_top_h_); + + ReshapeBlobs(1, 3); + + layer_param_.mutable_recurrent_param()->set_num_output(num_output_); + FillerParameter* weight_filler = + layer_param_.mutable_recurrent_param()->mutable_weight_filler(); + weight_filler->set_type("gaussian"); + weight_filler->set_std(0.2); + FillerParameter* bias_filler = + layer_param_.mutable_recurrent_param()->mutable_bias_filler(); + bias_filler->set_type("gaussian"); + bias_filler->set_std(0.1); + + layer_param_.set_phase(TEST); + } + + void ReshapeBlobs(int num_timesteps, int num_instances) { + blob_bottom_.Reshape(num_timesteps, num_instances, 3, 2); + blob_bottom_static_.Reshape(num_instances, 2, 3, 4); + vector shape(2); + shape[0] = num_timesteps; + shape[1] = num_instances; + blob_bottom_cont_.Reshape(shape); + shape.push_back(num_output_); + + shape[0] = 1; shape[1] = num_instances; shape[2] = 4 * num_output_; + unit_blob_bottom_x_.Reshape(shape); + shape[0] = 1; shape[1] = num_instances; shape[2] = num_output_; + unit_blob_bottom_c_prev_.Reshape(shape); + shape.resize(2); + shape[0] = 1; shape[1] = num_instances; + unit_blob_bottom_cont_.Reshape(shape); + + FillerParameter filler_param; + filler_param.set_min(-1); + filler_param.set_max(1); + UniformFiller filler(filler_param); + filler.Fill(&blob_bottom_); + filler.Fill(&unit_blob_bottom_c_prev_); + filler.Fill(&unit_blob_bottom_x_); + } + + int num_output_; + LayerParameter layer_param_; + Blob blob_bottom_; + Blob blob_bottom_cont_; + Blob blob_bottom_static_; + Blob blob_top_; + vector*> blob_bottom_vec_; + vector*> blob_top_vec_; + + Blob unit_blob_bottom_cont_; + Blob unit_blob_bottom_c_prev_; + Blob unit_blob_bottom_x_; + Blob unit_blob_top_c_; + Blob unit_blob_top_h_; + vector*> unit_blob_bottom_vec_; + vector*> unit_blob_top_vec_; +}; + +TYPED_TEST_CASE(LSTMLayerTest, TestDtypesAndDevices); + +TYPED_TEST(LSTMLayerTest, TestSetUp) { + typedef typename TypeParam::Dtype Dtype; + LSTMLayer layer(this->layer_param_); + layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + vector expected_top_shape = this->blob_bottom_.shape(); + expected_top_shape.resize(3); + expected_top_shape[2] = this->num_output_; + EXPECT_TRUE(this->blob_top_.shape() == expected_top_shape); +} + +TYPED_TEST(LSTMLayerTest, TestForward) { + typedef typename TypeParam::Dtype Dtype; + const int kNumTimesteps = 3; + const int num = this->blob_bottom_.shape(1); + this->ReshapeBlobs(kNumTimesteps, num); + + // Fill the cont blob with <0, 1, 1, ..., 1>, + // indicating a sequence that begins at the first timestep + // then continues for the rest of the sequence. + for (int t = 0; t < kNumTimesteps; ++t) { + for (int n = 0; n < num; ++n) { + this->blob_bottom_cont_.mutable_cpu_data()[t * num + n] = t > 0; + } + } + + // Process the full sequence in a single batch. + FillerParameter filler_param; + filler_param.set_mean(0); + filler_param.set_std(1); + GaussianFiller sequence_filler(filler_param); + Caffe::set_random_seed(1); + sequence_filler.Fill(&this->blob_bottom_); + shared_ptr > layer(new LSTMLayer(this->layer_param_)); + Caffe::set_random_seed(1701); + layer->SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + LOG(INFO) << "Calling forward for full sequence LSTM"; + layer->Forward(this->blob_bottom_vec_, this->blob_top_vec_); + + // Copy the inputs and outputs to reuse/check them later. + Blob bottom_copy(this->blob_bottom_.shape()); + bottom_copy.CopyFrom(this->blob_bottom_); + Blob top_copy(this->blob_top_.shape()); + top_copy.CopyFrom(this->blob_top_); + + // Process the batch one timestep at a time; + // check that we get the same result. + this->ReshapeBlobs(1, num); + layer.reset(new LSTMLayer(this->layer_param_)); + Caffe::set_random_seed(1701); + layer->SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + const int bottom_count = this->blob_bottom_.count(); + const int top_count = this->blob_top_.count(); + const Dtype kEpsilon = 1e-5; + for (int t = 0; t < kNumTimesteps; ++t) { + caffe_copy(bottom_count, bottom_copy.cpu_data() + t * bottom_count, + this->blob_bottom_.mutable_cpu_data()); + for (int n = 0; n < num; ++n) { + this->blob_bottom_cont_.mutable_cpu_data()[n] = t > 0; + } + LOG(INFO) << "Calling forward for LSTM timestep " << t; + layer->Forward(this->blob_bottom_vec_, this->blob_top_vec_); + for (int i = 0; i < top_count; ++i) { + ASSERT_LT(t * top_count + i, top_copy.count()); + EXPECT_NEAR(this->blob_top_.cpu_data()[i], + top_copy.cpu_data()[t * top_count + i], kEpsilon) + << "t = " << t << "; i = " << i; + } + } + + // Process the batch one timestep at a time with all cont blobs set to 0. + // Check that we get a different result, except in the first timestep. + Caffe::set_random_seed(1701); + layer.reset(new LSTMLayer(this->layer_param_)); + layer->SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + for (int t = 0; t < kNumTimesteps; ++t) { + caffe_copy(bottom_count, bottom_copy.cpu_data() + t * bottom_count, + this->blob_bottom_.mutable_cpu_data()); + for (int n = 0; n < num; ++n) { + this->blob_bottom_cont_.mutable_cpu_data()[n] = 0; + } + LOG(INFO) << "Calling forward for LSTM timestep " << t; + layer->Forward(this->blob_bottom_vec_, this->blob_top_vec_); + for (int i = 0; i < top_count; ++i) { + if (t == 0) { + EXPECT_NEAR(this->blob_top_.cpu_data()[i], + top_copy.cpu_data()[t * top_count + i], kEpsilon) + << "t = " << t << "; i = " << i; + } else { + EXPECT_NE(this->blob_top_.cpu_data()[i], + top_copy.cpu_data()[t * top_count + i]) + << "t = " << t << "; i = " << i; + } + } + } +} + +TYPED_TEST(LSTMLayerTest, TestLSTMUnitSetUp) { + typedef typename TypeParam::Dtype Dtype; + LayerParameter layer_param; + LSTMUnitLayer layer(layer_param); + layer.SetUp(this->unit_blob_bottom_vec_, this->unit_blob_top_vec_); + const int num_axes = this->unit_blob_bottom_c_prev_.num_axes(); + ASSERT_EQ(num_axes, this->unit_blob_top_c_.num_axes()); + ASSERT_EQ(num_axes, this->unit_blob_top_h_.num_axes()); + for (int i = 0; i < num_axes; ++i) { + EXPECT_EQ(this->unit_blob_bottom_c_prev_.shape(i), + this->unit_blob_top_c_.shape(i)); + EXPECT_EQ(this->unit_blob_bottom_c_prev_.shape(i), + this->unit_blob_top_h_.shape(i)); + } +} + +TYPED_TEST(LSTMLayerTest, TestLSTMUnitGradient) { + typedef typename TypeParam::Dtype Dtype; + LayerParameter layer_param; + LSTMUnitLayer layer(layer_param); + GradientChecker checker(1e-2, 1e-3); + Dtype* cont_data = this->blob_bottom_cont_.mutable_cpu_data(); + cont_data[0] = 0; + cont_data[1] = 0; + cont_data[2] = 0; + checker.CheckGradientExhaustive(&layer, this->unit_blob_bottom_vec_, + this->unit_blob_top_vec_, 0); + checker.CheckGradientExhaustive(&layer, this->unit_blob_bottom_vec_, + this->unit_blob_top_vec_, 1); +} + +TYPED_TEST(LSTMLayerTest, TestLSTMUnitGradientNonZeroCont) { + typedef typename TypeParam::Dtype Dtype; + LayerParameter layer_param; + LSTMUnitLayer layer(layer_param); + GradientChecker checker(1e-2, 1e-3); + Dtype* cont_data = this->blob_bottom_cont_.mutable_cpu_data(); + cont_data[0] = 1; + cont_data[1] = 0; + cont_data[2] = 1; + checker.CheckGradientExhaustive(&layer, this->unit_blob_bottom_vec_, + this->unit_blob_top_vec_, 0); + checker.CheckGradientExhaustive(&layer, this->unit_blob_bottom_vec_, + this->unit_blob_top_vec_, 1); +} + +TYPED_TEST(LSTMLayerTest, TestGradient) { + typedef typename TypeParam::Dtype Dtype; + LSTMLayer layer(this->layer_param_); + GradientChecker checker(1e-2, 1e-3); + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 0); +} + +TYPED_TEST(LSTMLayerTest, TestGradientNonZeroCont) { + typedef typename TypeParam::Dtype Dtype; + LSTMLayer layer(this->layer_param_); + GradientChecker checker(1e-2, 1e-3); + for (int i = 0; i < this->blob_bottom_cont_.count(); ++i) { + this->blob_bottom_cont_.mutable_cpu_data()[i] = i > 2; + } + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 0); +} + +TYPED_TEST(LSTMLayerTest, TestGradientNonZeroContBufferSize2) { + typedef typename TypeParam::Dtype Dtype; + this->ReshapeBlobs(2, 2); + FillerParameter filler_param; + UniformFiller filler(filler_param); + filler.Fill(&this->blob_bottom_); + LSTMLayer layer(this->layer_param_); + GradientChecker checker(1e-2, 1e-3); + for (int i = 0; i < this->blob_bottom_cont_.count(); ++i) { + this->blob_bottom_cont_.mutable_cpu_data()[i] = i > 2; + } + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 0); +} + +TYPED_TEST(LSTMLayerTest, TestGradientNonZeroContBufferSize2WithStaticInput) { + typedef typename TypeParam::Dtype Dtype; + this->ReshapeBlobs(2, 2); + FillerParameter filler_param; + UniformFiller filler(filler_param); + filler.Fill(&this->blob_bottom_); + filler.Fill(&this->blob_bottom_static_); + this->blob_bottom_vec_.push_back(&this->blob_bottom_static_); + LSTMLayer layer(this->layer_param_); + GradientChecker checker(1e-2, 1e-3); + for (int i = 0; i < this->blob_bottom_cont_.count(); ++i) { + this->blob_bottom_cont_.mutable_cpu_data()[i] = i > 2; + } + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 0); + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 2); +} + + +} // namespace caffe From 7e7631f27e4145b9838b17d46bb1ffc42279b1e4 Mon Sep 17 00:00:00 2001 From: Chuck Cho Date: Thu, 2 Jun 2016 14:35:14 -0400 Subject: [PATCH 21/39] Fixing a typo --- tools/extract_features.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/extract_features.cpp b/tools/extract_features.cpp index 704467250a6..51c791e4021 100644 --- a/tools/extract_features.cpp +++ b/tools/extract_features.cpp @@ -130,7 +130,7 @@ int feature_extraction_pipeline(int argc, char** argv) { txns.push_back(txn); } - LOG(ERROR)<< "Extacting Features"; + LOG(ERROR)<< "Extracting Features"; Datum datum; std::vector image_indices(num_features, 0); From 742c93f31be4c874aa5fd0103f25f8a2f8d4d63d Mon Sep 17 00:00:00 2001 From: philkr Date: Mon, 23 May 2016 20:09:45 -0700 Subject: [PATCH 22/39] Exposing load_hdf5 and save_hdf5 to python --- python/caffe/_caffe.cpp | 12 +++++++++++- python/caffe/test/test_net.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/python/caffe/_caffe.cpp b/python/caffe/_caffe.cpp index 32b5d921094..48a0c8f2e95 100644 --- a/python/caffe/_caffe.cpp +++ b/python/caffe/_caffe.cpp @@ -114,6 +114,14 @@ void Net_Save(const Net& net, string filename) { WriteProtoToBinaryFile(net_param, filename.c_str()); } +void Net_SaveHDF5(const Net& net, string filename) { + net.ToHDF5(filename); +} + +void Net_LoadHDF5(Net* net, string filename) { + net->CopyTrainedLayersFromHDF5(filename.c_str()); +} + void Net_SetInputArrays(Net* net, bp::object data_obj, bp::object labels_obj) { // check that this network has an input MemoryDataLayer @@ -267,7 +275,9 @@ BOOST_PYTHON_MODULE(_caffe) { bp::return_value_policy())) .def("_set_input_arrays", &Net_SetInputArrays, bp::with_custodian_and_ward<1, 2, bp::with_custodian_and_ward<1, 3> >()) - .def("save", &Net_Save); + .def("save", &Net_Save) + .def("save_hdf5", &Net_SaveHDF5) + .def("load_hdf5", &Net_LoadHDF5); BP_REGISTER_SHARED_PTR_TO_PYTHON(Net); bp::class_, shared_ptr >, boost::noncopyable>( diff --git a/python/caffe/test/test_net.py b/python/caffe/test/test_net.py index aad828aa8aa..4cacfcd05bb 100644 --- a/python/caffe/test/test_net.py +++ b/python/caffe/test/test_net.py @@ -79,3 +79,17 @@ def test_save_and_read(self): for i in range(len(self.net.params[name])): self.assertEqual(abs(self.net.params[name][i].data - net2.params[name][i].data).sum(), 0) + + def test_save_hdf5(self): + f = tempfile.NamedTemporaryFile(mode='w+', delete=False) + f.close() + self.net.save_hdf5(f.name) + net_file = simple_net_file(self.num_output) + net2 = caffe.Net(net_file, caffe.TRAIN) + net2.load_hdf5(f.name) + os.remove(net_file) + os.remove(f.name) + for name in self.net.params: + for i in range(len(self.net.params[name])): + self.assertEqual(abs(self.net.params[name][i].data + - net2.params[name][i].data).sum(), 0) From d167e61a23a54de529d51731fbe543ff4cec0d3c Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Wed, 1 Jun 2016 09:50:57 -0700 Subject: [PATCH 23/39] Add level and stages to Net constructor This internal functionality will be exposed through the various interfaces in subsequent commits Also adds C++ tests for all-in-one nets --- include/caffe/net.hpp | 1 + src/caffe/net.cpp | 11 +++- src/caffe/test/test_net.cpp | 128 ++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 1 deletion(-) diff --git a/include/caffe/net.hpp b/include/caffe/net.hpp index 0addb3c2a6d..493bdf294e2 100644 --- a/include/caffe/net.hpp +++ b/include/caffe/net.hpp @@ -25,6 +25,7 @@ class Net { public: explicit Net(const NetParameter& param, const Net* root_net = NULL); explicit Net(const string& param_file, Phase phase, + const int level = 0, const vector* stages = NULL, const Net* root_net = NULL); virtual ~Net() {} diff --git a/src/caffe/net.cpp b/src/caffe/net.cpp index f0bf594936c..644cb7e97ee 100644 --- a/src/caffe/net.cpp +++ b/src/caffe/net.cpp @@ -28,11 +28,20 @@ Net::Net(const NetParameter& param, const Net* root_net) } template -Net::Net(const string& param_file, Phase phase, const Net* root_net) +Net::Net(const string& param_file, Phase phase, + const int level, const vector* stages, + const Net* root_net) : root_net_(root_net) { NetParameter param; ReadNetParamsFromTextFileOrDie(param_file, ¶m); + // Set phase, stages and level param.mutable_state()->set_phase(phase); + if (stages != NULL) { + for (int i = 0; i < stages->size(); i++) { + param.mutable_state()->add_stage((*stages)[i]); + } + } + param.mutable_state()->set_level(level); Init(param); } diff --git a/src/caffe/test/test_net.cpp b/src/caffe/test/test_net.cpp index 92fd317fee8..24b957f2acc 100644 --- a/src/caffe/test/test_net.cpp +++ b/src/caffe/test/test_net.cpp @@ -9,6 +9,7 @@ #include "caffe/common.hpp" #include "caffe/filler.hpp" #include "caffe/net.hpp" +#include "caffe/util/io.hpp" #include "caffe/util/math_functions.hpp" #include "caffe/test/test_caffe_main.hpp" @@ -29,6 +30,17 @@ class NetTest : public MultiDeviceTest { net_.reset(new Net(param)); } + virtual void InitNetFromProtoFileWithState(const string& proto, + Phase phase = caffe::TRAIN, const int level = 0, + const vector* stages = NULL) { + NetParameter param; + CHECK(google::protobuf::TextFormat::ParseFromString(proto, ¶m)); + string param_file; + MakeTempFilename(¶m_file); + WriteProtoToTextFile(param, param_file); + net_.reset(new Net(param_file, phase, level, stages)); + } + virtual void CopyNetBlobs(const bool copy_diff, vector > >* blobs_copy) { CHECK(net_); @@ -771,6 +783,62 @@ class NetTest : public MultiDeviceTest { InitNetFromProtoString(proto); } + virtual void InitAllInOneNet(Phase phase = caffe::TRAIN, + const int level = 0, const vector* stages = NULL) { + string proto = + "name: 'All-in-one Network'" + "layer { " + " name: 'train-data' " + " type: 'DummyData' " + " top: 'data' " + " top: 'label' " + " dummy_data_param { " + " shape { dim: 1 dim: 10 } " + " shape { dim: 1 dim: 1 } " + " } " + " include { phase: TRAIN stage: 'train' } " + "} " + "layer { " + " name: 'val-data' " + " type: 'DummyData' " + " top: 'data' " + " top: 'label' " + " dummy_data_param { " + " shape { dim: 1 dim: 10 } " + " shape { dim: 1 dim: 1 } " + " } " + " include { phase: TEST stage: 'val' } " + "} " + "layer { " + " name: 'deploy-data' " + " type: 'Input' " + " top: 'data' " + " input_param { " + " shape { dim: 1 dim: 10 } " + " } " + " include { phase: TEST stage: 'deploy' } " + "} " + "layer { " + " name: 'ip' " + " type: 'InnerProduct' " + " bottom: 'data' " + " top: 'ip' " + " inner_product_param { " + " num_output: 2 " + " } " + "} " + "layer { " + " name: 'loss' " + " type: 'SoftmaxWithLoss' " + " bottom: 'ip' " + " bottom: 'label' " + " top: 'loss' " + " include { phase: TRAIN stage: 'train' } " + " include { phase: TEST stage: 'val' } " + "} "; + InitNetFromProtoFileWithState(proto, phase, level, stages); + } + int seed_; shared_ptr > net_; }; @@ -2473,4 +2541,64 @@ TYPED_TEST(NetTest, TestForcePropagateDown) { } } +TYPED_TEST(NetTest, TestAllInOneNetTrain) { + vector stages; + stages.push_back("train"); + this->InitAllInOneNet(caffe::TRAIN, 0, &stages); + bool found_data = false; + bool found_loss = false; + for (int i = 0; i < this->net_->layers().size(); ++i) { + const string& layer_name = this->net_->layer_names()[i]; + if (layer_name == "train-data") { + found_data = true; + } else if (layer_name == "loss") { + found_loss = true; + } else { + ASSERT_NE(layer_name, "val-data"); + ASSERT_NE(layer_name, "deploy-data"); + } + } + ASSERT_TRUE(found_data); + ASSERT_TRUE(found_loss); +} + +TYPED_TEST(NetTest, TestAllInOneNetVal) { + vector stages; + stages.push_back("val"); + this->InitAllInOneNet(caffe::TEST, 0, &stages); + bool found_data = false; + bool found_loss = false; + for (int i = 0; i < this->net_->layers().size(); ++i) { + const string& layer_name = this->net_->layer_names()[i]; + if (layer_name == "val-data") { + found_data = true; + } else if (layer_name == "loss") { + found_loss = true; + } else { + ASSERT_NE(layer_name, "train-data"); + ASSERT_NE(layer_name, "deploy-data"); + } + } + ASSERT_TRUE(found_data); + ASSERT_TRUE(found_loss); +} + +TYPED_TEST(NetTest, TestAllInOneNetDeploy) { + vector stages; + stages.push_back("deploy"); + this->InitAllInOneNet(caffe::TEST, 0, &stages); + bool found_data = false; + for (int i = 0; i < this->net_->layers().size(); ++i) { + const string& layer_name = this->net_->layer_names()[i]; + if (layer_name == "deploy-data") { + found_data = true; + } else { + ASSERT_NE(layer_name, "train-data"); + ASSERT_NE(layer_name, "val-data"); + ASSERT_NE(layer_name, "loss"); + } + } + ASSERT_TRUE(found_data); +} + } // namespace caffe From 66e84d785a72d66511bffe30c0f016af9103deb8 Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Wed, 1 Jun 2016 09:56:51 -0700 Subject: [PATCH 24/39] Add phase, level and stages to tools/caffe Adds command-line flags for phase, level and stage train -- override level and stages for test_state from solver test -- set level and stages time -- set phase, level and stages --- tools/caffe.cpp | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/tools/caffe.cpp b/tools/caffe.cpp index 5bb60eb161d..9bf4214ad93 100644 --- a/tools/caffe.cpp +++ b/tools/caffe.cpp @@ -34,6 +34,13 @@ DEFINE_string(solver, "", "The solver definition protocol buffer text file."); DEFINE_string(model, "", "The model definition protocol buffer text file."); +DEFINE_string(phase, "", + "Optional; network phase (TRAIN or TEST). Only used for 'time'."); +DEFINE_int32(level, 0, + "Optional; network level."); +DEFINE_string(stage, "", + "Optional; network stages (not to be confused with phase), " + "separated by ','."); DEFINE_string(snapshot, "", "Optional; the snapshot solver state to resume training."); DEFINE_string(weights, "", @@ -101,6 +108,25 @@ static void get_gpus(vector* gpus) { } } +// Parse phase from flags +caffe::Phase get_phase_from_flags(caffe::Phase default_value) { + if (FLAGS_phase == "") + return default_value; + if (FLAGS_phase == "TRAIN") + return caffe::TRAIN; + if (FLAGS_phase == "TEST") + return caffe::TEST; + LOG(FATAL) << "phase must be \"TRAIN\" or \"TEST\""; + return caffe::TRAIN; // Avoid warning +} + +// Parse stages from flags +vector get_stages_from_flags() { + vector stages; + boost::split(stages, FLAGS_stage, boost::is_any_of(",")); + return stages; +} + // caffe commands to call by // caffe // @@ -156,10 +182,16 @@ int train() { CHECK(!FLAGS_snapshot.size() || !FLAGS_weights.size()) << "Give a snapshot to resume training or weights to finetune " "but not both."; + vector stages = get_stages_from_flags(); caffe::SolverParameter solver_param; caffe::ReadSolverParamsFromTextFileOrDie(FLAGS_solver, &solver_param); + solver_param.mutable_train_state()->set_level(FLAGS_level); + for (int i = 0; i < stages.size(); i++) { + solver_param.mutable_train_state()->add_stage(stages[i]); + } + // If the gpus flag is not provided, allow the mode and device to be set // in the solver prototxt. if (FLAGS_gpu.size() == 0 @@ -229,6 +261,7 @@ RegisterBrewFunction(train); int test() { CHECK_GT(FLAGS_model.size(), 0) << "Need a model definition to score."; CHECK_GT(FLAGS_weights.size(), 0) << "Need model weights to score."; + vector stages = get_stages_from_flags(); // Set device id and mode vector gpus; @@ -247,7 +280,7 @@ int test() { Caffe::set_mode(Caffe::CPU); } // Instantiate the caffe net. - Net caffe_net(FLAGS_model, caffe::TEST); + Net caffe_net(FLAGS_model, caffe::TEST, FLAGS_level, &stages); caffe_net.CopyTrainedLayersFrom(FLAGS_weights); LOG(INFO) << "Running for " << FLAGS_iterations << " iterations."; @@ -300,6 +333,8 @@ RegisterBrewFunction(test); // Time: benchmark the execution time of a model. int time() { CHECK_GT(FLAGS_model.size(), 0) << "Need a model definition to time."; + caffe::Phase phase = get_phase_from_flags(caffe::TRAIN); + vector stages = get_stages_from_flags(); // Set device id and mode vector gpus; @@ -313,7 +348,7 @@ int time() { Caffe::set_mode(Caffe::CPU); } // Instantiate the caffe net. - Net caffe_net(FLAGS_model, caffe::TRAIN); + Net caffe_net(FLAGS_model, phase, FLAGS_level, &stages); // Do a clean forward and backward pass, so that memory allocation are done // and future iterations will be more stable. From 19adc7a79e3acacc777076143357cc0569781cd3 Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Wed, 1 Jun 2016 10:02:41 -0700 Subject: [PATCH 25/39] Add level and stages to pycaffe Uses Boost.Python's pattern matching to differentiate between constructors Also adds Python tests for all-in-one nets --- python/caffe/_caffe.cpp | 44 +++++-- python/caffe/test/test_net.py | 228 +++++++++++++++++++++++++++++++++- 2 files changed, 263 insertions(+), 9 deletions(-) diff --git a/python/caffe/_caffe.cpp b/python/caffe/_caffe.cpp index 48a0c8f2e95..e2726286dfb 100644 --- a/python/caffe/_caffe.cpp +++ b/python/caffe/_caffe.cpp @@ -86,19 +86,42 @@ void CheckContiguousArray(PyArrayObject* arr, string name, } } -// Net constructor for passing phase as int -shared_ptr > Net_Init( - string param_file, int phase) { - CheckFile(param_file); +// Net constructor +shared_ptr > Net_Init(string network_file, int phase, + const int level, const bp::object& stages, + const bp::object& weights) { + CheckFile(network_file); + + // Convert stages from list to vector + vector stages_vector; + if (!stages.is_none()) { + for (int i = 0; i < len(stages); i++) { + stages_vector.push_back(bp::extract(stages[i])); + } + } + + // Initialize net + shared_ptr > net(new Net(network_file, + static_cast(phase), level, &stages_vector)); + + // Load weights + if (!weights.is_none()) { + std::string weights_file_str = bp::extract(weights); + CheckFile(weights_file_str); + net->CopyTrainedLayersFrom(weights_file_str); + } - shared_ptr > net(new Net(param_file, - static_cast(phase))); return net; } -// Net construct-and-load convenience constructor +// Legacy Net construct-and-load convenience constructor shared_ptr > Net_Init_Load( string param_file, string pretrained_param_file, int phase) { + LOG(WARNING) << "DEPRECATION WARNING - deprecated use of Python interface"; + LOG(WARNING) << "Use this instead (with the named \"weights\"" + << " parameter):"; + LOG(WARNING) << "Net('" << param_file << "', " << phase + << ", weights='" << pretrained_param_file << "')"; CheckFile(param_file); CheckFile(pretrained_param_file); @@ -245,7 +268,12 @@ BOOST_PYTHON_MODULE(_caffe) { bp::class_, shared_ptr >, boost::noncopyable >("Net", bp::no_init) - .def("__init__", bp::make_constructor(&Net_Init)) + // Constructor + .def("__init__", bp::make_constructor(&Net_Init, + bp::default_call_policies(), (bp::arg("network_file"), "phase", + bp::arg("level")=0, bp::arg("stages")=bp::object(), + bp::arg("weights")=bp::object()))) + // Legacy constructor .def("__init__", bp::make_constructor(&Net_Init_Load)) .def("_forward", &Net::ForwardFromTo) .def("_backward", &Net::BackwardFromTo) diff --git a/python/caffe/test/test_net.py b/python/caffe/test/test_net.py index 4cacfcd05bb..300aabdeea5 100644 --- a/python/caffe/test/test_net.py +++ b/python/caffe/test/test_net.py @@ -72,7 +72,11 @@ def test_save_and_read(self): f.close() self.net.save(f.name) net_file = simple_net_file(self.num_output) - net2 = caffe.Net(net_file, f.name, caffe.TRAIN) + # Test legacy constructor + # should print deprecation warning + caffe.Net(net_file, f.name, caffe.TRAIN) + # Test named constructor + net2 = caffe.Net(net_file, caffe.TRAIN, weights=f.name) os.remove(net_file) os.remove(f.name) for name in self.net.params: @@ -93,3 +97,225 @@ def test_save_hdf5(self): for i in range(len(self.net.params[name])): self.assertEqual(abs(self.net.params[name][i].data - net2.params[name][i].data).sum(), 0) + +class TestLevels(unittest.TestCase): + + TEST_NET = """ +layer { + name: "data" + type: "DummyData" + top: "data" + dummy_data_param { shape { dim: 1 dim: 1 dim: 10 dim: 10 } } +} +layer { + name: "NoLevel" + type: "InnerProduct" + bottom: "data" + top: "NoLevel" + inner_product_param { num_output: 1 } +} +layer { + name: "Level0Only" + type: "InnerProduct" + bottom: "data" + top: "Level0Only" + include { min_level: 0 max_level: 0 } + inner_product_param { num_output: 1 } +} +layer { + name: "Level1Only" + type: "InnerProduct" + bottom: "data" + top: "Level1Only" + include { min_level: 1 max_level: 1 } + inner_product_param { num_output: 1 } +} +layer { + name: "Level>=0" + type: "InnerProduct" + bottom: "data" + top: "Level>=0" + include { min_level: 0 } + inner_product_param { num_output: 1 } +} +layer { + name: "Level>=1" + type: "InnerProduct" + bottom: "data" + top: "Level>=1" + include { min_level: 1 } + inner_product_param { num_output: 1 } +} +""" + + def setUp(self): + self.f = tempfile.NamedTemporaryFile(mode='w+') + self.f.write(self.TEST_NET) + self.f.flush() + + def tearDown(self): + self.f.close() + + def check_net(self, net, blobs): + net_blobs = [b for b in net.blobs.keys() if 'data' not in b] + self.assertEqual(net_blobs, blobs) + + def test_0(self): + net = caffe.Net(self.f.name, caffe.TEST) + self.check_net(net, ['NoLevel', 'Level0Only', 'Level>=0']) + + def test_1(self): + net = caffe.Net(self.f.name, caffe.TEST, level=1) + self.check_net(net, ['NoLevel', 'Level1Only', 'Level>=0', 'Level>=1']) + + +class TestStages(unittest.TestCase): + + TEST_NET = """ +layer { + name: "data" + type: "DummyData" + top: "data" + dummy_data_param { shape { dim: 1 dim: 1 dim: 10 dim: 10 } } +} +layer { + name: "A" + type: "InnerProduct" + bottom: "data" + top: "A" + include { stage: "A" } + inner_product_param { num_output: 1 } +} +layer { + name: "B" + type: "InnerProduct" + bottom: "data" + top: "B" + include { stage: "B" } + inner_product_param { num_output: 1 } +} +layer { + name: "AorB" + type: "InnerProduct" + bottom: "data" + top: "AorB" + include { stage: "A" } + include { stage: "B" } + inner_product_param { num_output: 1 } +} +layer { + name: "AandB" + type: "InnerProduct" + bottom: "data" + top: "AandB" + include { stage: "A" stage: "B" } + inner_product_param { num_output: 1 } +} +""" + + def setUp(self): + self.f = tempfile.NamedTemporaryFile(mode='w+') + self.f.write(self.TEST_NET) + self.f.flush() + + def tearDown(self): + self.f.close() + + def check_net(self, net, blobs): + net_blobs = [b for b in net.blobs.keys() if 'data' not in b] + self.assertEqual(net_blobs, blobs) + + def test_A(self): + net = caffe.Net(self.f.name, caffe.TEST, stages=['A']) + self.check_net(net, ['A', 'AorB']) + + def test_B(self): + net = caffe.Net(self.f.name, caffe.TEST, stages=['B']) + self.check_net(net, ['B', 'AorB']) + + def test_AandB(self): + net = caffe.Net(self.f.name, caffe.TEST, stages=['A', 'B']) + self.check_net(net, ['A', 'B', 'AorB', 'AandB']) + + +class TestAllInOne(unittest.TestCase): + + TEST_NET = """ +layer { + name: "train_data" + type: "DummyData" + top: "data" + top: "label" + dummy_data_param { + shape { dim: 1 dim: 1 dim: 10 dim: 10 } + shape { dim: 1 dim: 1 dim: 1 dim: 1 } + } + include { phase: TRAIN stage: "train" } +} +layer { + name: "val_data" + type: "DummyData" + top: "data" + top: "label" + dummy_data_param { + shape { dim: 1 dim: 1 dim: 10 dim: 10 } + shape { dim: 1 dim: 1 dim: 1 dim: 1 } + } + include { phase: TEST stage: "val" } +} +layer { + name: "deploy_data" + type: "Input" + top: "data" + input_param { shape { dim: 1 dim: 1 dim: 10 dim: 10 } } + include { phase: TEST stage: "deploy" } +} +layer { + name: "ip" + type: "InnerProduct" + bottom: "data" + top: "ip" + inner_product_param { num_output: 2 } +} +layer { + name: "loss" + type: "SoftmaxWithLoss" + bottom: "ip" + bottom: "label" + top: "loss" + include: { phase: TRAIN stage: "train" } + include: { phase: TEST stage: "val" } +} +layer { + name: "pred" + type: "Softmax" + bottom: "ip" + top: "pred" + include: { phase: TEST stage: "deploy" } +} +""" + + def setUp(self): + self.f = tempfile.NamedTemporaryFile(mode='w+') + self.f.write(self.TEST_NET) + self.f.flush() + + def tearDown(self): + self.f.close() + + def check_net(self, net, outputs): + self.assertEqual(list(net.blobs['data'].shape), [1,1,10,10]) + self.assertEqual(net.outputs, outputs) + + def test_train(self): + net = caffe.Net(self.f.name, caffe.TRAIN, stages=['train']) + self.check_net(net, ['loss']) + + def test_val(self): + net = caffe.Net(self.f.name, caffe.TEST, stages=['val']) + self.check_net(net, ['loss']) + + def test_deploy(self): + net = caffe.Net(self.f.name, caffe.TEST, stages=['deploy']) + self.check_net(net, ['pred']) + From dec2381cc8d6465f0997cd29b143b3c6e13416ef Mon Sep 17 00:00:00 2001 From: philkr Date: Thu, 3 Sep 2015 14:28:55 -0700 Subject: [PATCH 26/39] Exposing solver callbacks to python --- python/caffe/_caffe.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/python/caffe/_caffe.cpp b/python/caffe/_caffe.cpp index 48a0c8f2e95..334088e8a57 100644 --- a/python/caffe/_caffe.cpp +++ b/python/caffe/_caffe.cpp @@ -228,6 +228,27 @@ bp::object BlobVec_add_blob(bp::tuple args, bp::dict kwargs) { return bp::object(); } +template +class PythonCallback: public Solver::Callback { + protected: + bp::object on_start_, on_gradients_ready_; + + public: + PythonCallback(bp::object on_start, bp::object on_gradients_ready) + : on_start_(on_start), on_gradients_ready_(on_gradients_ready) { } + virtual void on_gradients_ready() { + on_gradients_ready_(); + } + virtual void on_start() { + on_start_(); + } +}; +template +void Solver_add_callback(Solver * solver, bp::object on_start, + bp::object on_gradients_ready) { + solver->add_callback(new PythonCallback(on_start, on_gradients_ready)); +} + BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS(SolveOverloads, Solve, 0, 1); BOOST_PYTHON_MODULE(_caffe) { @@ -317,6 +338,7 @@ BOOST_PYTHON_MODULE(_caffe) { .add_property("test_nets", bp::make_function(&Solver::test_nets, bp::return_internal_reference<>())) .add_property("iter", &Solver::iter) + .def("add_callback", &Solver_add_callback) .def("solve", static_cast::*)(const char*)>( &Solver::Solve), SolveOverloads()) .def("step", &Solver::Step) From 118c97ff5890e92b9aa603d925d947d45086b330 Mon Sep 17 00:00:00 2001 From: Valentin Tolmer Date: Tue, 21 Jun 2016 17:37:55 -0700 Subject: [PATCH 27/39] add clear_param_diffs to the python net interface --- python/caffe/_caffe.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/python/caffe/_caffe.cpp b/python/caffe/_caffe.cpp index 334088e8a57..a7fb886aa06 100644 --- a/python/caffe/_caffe.cpp +++ b/python/caffe/_caffe.cpp @@ -271,6 +271,7 @@ BOOST_PYTHON_MODULE(_caffe) { .def("_forward", &Net::ForwardFromTo) .def("_backward", &Net::BackwardFromTo) .def("reshape", &Net::Reshape) + .def("clear_param_diffs", &Net::ClearParamDiffs) // The cast is to select a particular overload. .def("copy_from", static_cast::*)(const string)>( &Net::CopyTrainedLayersFrom)) From 892c78dd7833f1818a76d4025076b34946200fa0 Mon Sep 17 00:00:00 2001 From: Valentin Tolmer Date: Tue, 21 Jun 2016 17:42:31 -0700 Subject: [PATCH 28/39] add unit test for clear_param_diffs --- python/caffe/test/test_net.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/caffe/test/test_net.py b/python/caffe/test/test_net.py index 4cacfcd05bb..7fb9f475d43 100644 --- a/python/caffe/test/test_net.py +++ b/python/caffe/test/test_net.py @@ -63,6 +63,17 @@ def test_forward_backward(self): self.net.forward() self.net.backward() + def test_clear_param_diffs(self): + # Run a forward/backward step to have non-zero diffs + self.net.forward() + self.net.backward() + diff = self.net.params["conv"][0].diff + # Check that we have non-zero diffs + self.assertTrue(diff.max() > 0) + self.net.clear_param_diffs() + # Check that the diffs are now 0 + self.assertTrue((diff == 0).all()) + def test_inputs_outputs(self): self.assertEqual(self.net.inputs, []) self.assertEqual(self.net.outputs, ['loss']) From f0b1a9e770594f93fecda9e876faaafaede2b496 Mon Sep 17 00:00:00 2001 From: Carl Doersch Date: Sun, 3 Jul 2016 12:32:19 -0700 Subject: [PATCH 29/39] Add phase support for draw net --- python/caffe/draw.py | 32 +++++++++++++++++++++++++++----- python/draw_net.py | 15 ++++++++++++++- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/python/caffe/draw.py b/python/caffe/draw.py index 61205ca9f37..9eecf6d7b46 100644 --- a/python/caffe/draw.py +++ b/python/caffe/draw.py @@ -127,7 +127,7 @@ def choose_color_by_layertype(layertype): return color -def get_pydot_graph(caffe_net, rankdir, label_edges=True): +def get_pydot_graph(caffe_net, rankdir, label_edges=True, phase=None): """Create a data structure which represents the `caffe_net`. Parameters @@ -137,6 +137,9 @@ def get_pydot_graph(caffe_net, rankdir, label_edges=True): Direction of graph layout. label_edges : boolean, optional Label the edges (default is True). + phase : {caffe_pb2.Phase.TRAIN, caffe_pb2.Phase.TEST, None} optional + Include layers from this network phase. If None, include all layers. + (the default is None) Returns ------- @@ -148,6 +151,19 @@ def get_pydot_graph(caffe_net, rankdir, label_edges=True): pydot_nodes = {} pydot_edges = [] for layer in caffe_net.layer: + if phase is not None: + included = False + if len(layer.include) == 0: + included = True + if len(layer.include) > 0 and len(layer.exclude) > 0: + raise ValueError('layer ' + layer.name + ' has both include ' + 'and exclude specified.') + for layer_phase in layer.include: + included = included or layer_phase.phase == phase + for layer_phase in layer.exclude: + included = included and not layer_phase.phase == phase + if not included: + continue node_label = get_layer_label(layer, rankdir) node_name = "%s_%s" % (layer.name, layer.type) if (len(layer.bottom) == 1 and len(layer.top) == 1 and @@ -186,7 +202,7 @@ def get_pydot_graph(caffe_net, rankdir, label_edges=True): return pydot_graph -def draw_net(caffe_net, rankdir, ext='png'): +def draw_net(caffe_net, rankdir, ext='png', phase=None): """Draws a caffe net and returns the image string encoded using the given extension. @@ -195,16 +211,19 @@ def draw_net(caffe_net, rankdir, ext='png'): caffe_net : a caffe.proto.caffe_pb2.NetParameter protocol buffer. ext : string, optional The image extension (the default is 'png'). + phase : {caffe_pb2.Phase.TRAIN, caffe_pb2.Phase.TEST, None} optional + Include layers from this network phase. If None, include all layers. + (the default is None) Returns ------- string : Postscript representation of the graph. """ - return get_pydot_graph(caffe_net, rankdir).create(format=ext) + return get_pydot_graph(caffe_net, rankdir, phase=phase).create(format=ext) -def draw_net_to_file(caffe_net, filename, rankdir='LR'): +def draw_net_to_file(caffe_net, filename, rankdir='LR', phase=None): """Draws a caffe net, and saves it to file using the format given as the file extension. Use '.raw' to output raw text that you can manually feed to graphviz to draw graphs. @@ -216,7 +235,10 @@ def draw_net_to_file(caffe_net, filename, rankdir='LR'): The path to a file where the networks visualization will be stored. rankdir : {'LR', 'TB', 'BT'} Direction of graph layout. + phase : {caffe_pb2.Phase.TRAIN, caffe_pb2.Phase.TEST, None} optional + Include layers from this network phase. If None, include all layers. + (the default is None) """ ext = filename[filename.rfind('.')+1:] with open(filename, 'wb') as fid: - fid.write(draw_net(caffe_net, rankdir, ext)) + fid.write(draw_net(caffe_net, rankdir, ext, phase)) diff --git a/python/draw_net.py b/python/draw_net.py index ec76a744da3..dfe70d26a71 100755 --- a/python/draw_net.py +++ b/python/draw_net.py @@ -28,6 +28,11 @@ def parse_args(): 'http://www.graphviz.org/doc/info/' 'attrs.html#k:rankdir'), default='LR') + parser.add_argument('--phase', + help=('Which network phase to draw: can be TRAIN, ' + 'TEST, or ALL. If ALL, then all layers are drawn ' + 'regardless of phase.'), + default="ALL") args = parser.parse_args() return args @@ -38,7 +43,15 @@ def main(): net = caffe_pb2.NetParameter() text_format.Merge(open(args.input_net_proto_file).read(), net) print('Drawing net to %s' % args.output_image_file) - caffe.draw.draw_net_to_file(net, args.output_image_file, args.rankdir) + phase=None; + if args.phase == "TRAIN": + phase = caffe.TRAIN + elif args.phase == "TEST": + phase = caffe.TEST + elif args.phase != "ALL": + raise ValueError("Unknown phase: " + args.phase) + caffe.draw.draw_net_to_file(net, args.output_image_file, args.rankdir, + phase) if __name__ == '__main__': From f9fd20ea3893c515b19cae6fa3693b1649fb9487 Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Fri, 8 Jul 2016 12:05:17 -0700 Subject: [PATCH 30/39] Fix Python installation with CMake install target --- python/CMakeLists.txt | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index a22641401f0..bf492a24b1c 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -22,13 +22,19 @@ if(UNIX OR APPLE) endif() # ---[ Install -file(GLOB files1 *.py requirements.txt) -install(FILES ${files1} DESTINATION python) - -file(GLOB files2 caffe/*.py) -install(FILES ${files2} DESTINATION python/caffe) +# scripts +file(GLOB python_files *.py requirements.txt) +install(FILES ${python_files} DESTINATION python) + +# module +install(DIRECTORY caffe + DESTINATION python + FILES_MATCHING + PATTERN "*.py" + PATTERN "ilsvrc_2012_mean.npy" + PATTERN "test" EXCLUDE + ) + +# _caffe.so install(TARGETS pycaffe DESTINATION python/caffe) -install(DIRECTORY caffe/imagenet caffe/proto caffe/test DESTINATION python/caffe) - - From f1a8470aa21e35a5b2bb83007f8fb7680a354815 Mon Sep 17 00:00:00 2001 From: Nishidha Panpaliya Date: Tue, 17 May 2016 01:14:53 -0500 Subject: [PATCH 31/39] Fix for a random failure in this test due to floating point comparison. So, instead of exact match, used EXPECT_FLOAT_EQ that tolerates some precision while comparing two floats --- src/caffe/test/test_embed_layer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/caffe/test/test_embed_layer.cpp b/src/caffe/test/test_embed_layer.cpp index dc7f5c4aa47..13f13a878d3 100644 --- a/src/caffe/test/test_embed_layer.cpp +++ b/src/caffe/test/test_embed_layer.cpp @@ -124,7 +124,7 @@ TYPED_TEST(EmbedLayerTest, TestForwardWithBias) { top_offset[4] = 0; bias_offset[0] = 0; for (int j = 0; j < kNumOutput; ++j) { - EXPECT_EQ(layer->blobs()[0]->data_at(weight_offset) + + EXPECT_FLOAT_EQ(layer->blobs()[0]->data_at(weight_offset) + layer->blobs()[1]->data_at(bias_offset), this->blob_top_->data_at(top_offset)); ++top_offset[4]; From 35a9a075cdc65c86021dde4d11e3b1c05e27971b Mon Sep 17 00:00:00 2001 From: Valentin Tolmer Date: Wed, 22 Jun 2016 15:13:54 -0700 Subject: [PATCH 32/39] add set_random_seed to the python interface --- python/caffe/__init__.py | 2 +- python/caffe/_caffe.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/python/caffe/__init__.py b/python/caffe/__init__.py index e2881b89c1b..35868a403a3 100644 --- a/python/caffe/__init__.py +++ b/python/caffe/__init__.py @@ -1,5 +1,5 @@ from .pycaffe import Net, SGDSolver, NesterovSolver, AdaGradSolver, RMSPropSolver, AdaDeltaSolver, AdamSolver -from ._caffe import set_mode_cpu, set_mode_gpu, set_device, Layer, get_solver, layer_type_list +from ._caffe import set_mode_cpu, set_mode_gpu, set_device, Layer, get_solver, layer_type_list, set_random_seed from ._caffe import __version__ from .proto.caffe_pb2 import TRAIN, TEST from .classifier import Classifier diff --git a/python/caffe/_caffe.cpp b/python/caffe/_caffe.cpp index 334088e8a57..3db55ea4340 100644 --- a/python/caffe/_caffe.cpp +++ b/python/caffe/_caffe.cpp @@ -51,6 +51,8 @@ const int NPY_DTYPE = NPY_FLOAT32; void set_mode_cpu() { Caffe::set_mode(Caffe::CPU); } void set_mode_gpu() { Caffe::set_mode(Caffe::GPU); } +void set_random_seed(unsigned int seed) { Caffe::set_random_seed(seed); } + // For convenience, check that input files can be opened, and raise an // exception that boost will send to Python if not (caffe could still crash // later if the input files are disturbed before they are actually used, but @@ -260,6 +262,7 @@ BOOST_PYTHON_MODULE(_caffe) { // Caffe utility functions bp::def("set_mode_cpu", &set_mode_cpu); bp::def("set_mode_gpu", &set_mode_gpu); + bp::def("set_random_seed", &set_random_seed); bp::def("set_device", &Caffe::SetDevice); bp::def("layer_type_list", &LayerRegistry::LayerTypeList); From a64cfbd08591db0b061ad7ad39c54cd45c0e252a Mon Sep 17 00:00:00 2001 From: Alessandro Giusti Date: Mon, 11 Jul 2016 20:33:16 +0200 Subject: [PATCH 33/39] Update parse_log.py Aligned output description in docstring with actual output returned by parse_log --- tools/extra/parse_log.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/extra/parse_log.py b/tools/extra/parse_log.py index bb9b65ad615..375b0db73b3 100755 --- a/tools/extra/parse_log.py +++ b/tools/extra/parse_log.py @@ -16,13 +16,10 @@ def parse_log(path_to_log): """Parse log file - Returns (train_dict_list, train_dict_names, test_dict_list, test_dict_names) + Returns (train_dict_list, test_dict_list) train_dict_list and test_dict_list are lists of dicts that define the table rows - - train_dict_names and test_dict_names are ordered tuples of the column names - for the two dict_lists """ regex_iteration = re.compile('Iteration (\d+)') From 12c74460d3e7c416b869e6b4afa0e5c2e84ec29b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Malte=20St=C3=A6r=20Nissen?= Date: Tue, 12 Jul 2016 13:17:52 +0200 Subject: [PATCH 34/39] Support for spaces in directories when downloading cifar10 --- data/cifar10/get_cifar10.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/cifar10/get_cifar10.sh b/data/cifar10/get_cifar10.sh index 623c848513e..423f10989c4 100755 --- a/data/cifar10/get_cifar10.sh +++ b/data/cifar10/get_cifar10.sh @@ -2,7 +2,7 @@ # This scripts downloads the CIFAR10 (binary version) data and unzips it. DIR="$( cd "$(dirname "$0")" ; pwd -P )" -cd $DIR +cd "$DIR" echo "Downloading..." From e14b7f7ea597afe532bf1c4d4013f2c63494d7a6 Mon Sep 17 00:00:00 2001 From: Valentin Tolmer Date: Tue, 21 Jun 2016 14:58:43 -0700 Subject: [PATCH 35/39] improve top_names and bottom_names in pycaffe --- python/caffe/pycaffe.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/python/caffe/pycaffe.py b/python/caffe/pycaffe.py index ca6d050e2bd..5bae18d9a4d 100644 --- a/python/caffe/pycaffe.py +++ b/python/caffe/pycaffe.py @@ -292,21 +292,31 @@ def _Net_batch(self, blobs): padding]) yield padded_batch - -class _Net_IdNameWrapper: - """ - A simple wrapper that allows the ids propery to be accessed as a dict - indexed by names. Used for top and bottom names +def _Net_get_id_name(func, field): """ - def __init__(self, net, func): - self.net, self.func = net, func + Generic property that maps func to the layer names into an OrderedDict. + + Used for top_names and bottom_names. - def __getitem__(self, name): - # Map the layer name to id - ids = self.func(self.net, list(self.net._layer_names).index(name)) - # Map the blob id to name - id_to_name = list(self.net.blobs) - return [id_to_name[i] for i in ids] + Parameters + ---------- + func: function id -> [id] + field: implementation field name (cache) + + Returns + ------ + A one-parameter function that can be set as a property. + """ + @property + def get_id_name(self): + if not hasattr(self, field): + id_to_name = list(self.blobs) + res = OrderedDict([(self._layer_names[i], + [id_to_name[j] for j in func(self, i)]) + for i in range(len(self.layers))]) + setattr(self, field, res) + return getattr(self, field) + return get_id_name # Attach methods to Net. Net.blobs = _Net_blobs @@ -320,5 +330,5 @@ def __getitem__(self, name): Net._batch = _Net_batch Net.inputs = _Net_inputs Net.outputs = _Net_outputs -Net.top_names = property(lambda n: _Net_IdNameWrapper(n, Net._top_ids)) -Net.bottom_names = property(lambda n: _Net_IdNameWrapper(n, Net._bottom_ids)) +Net.top_names = _Net_get_id_name(Net._top_ids, "_top_names") +Net.bottom_names = _Net_get_id_name(Net._bottom_ids, "_bottom_names") From 7c50a2cb87c6b044f85ced87273d302fb21394f7 Mon Sep 17 00:00:00 2001 From: Valentin Tolmer Date: Tue, 21 Jun 2016 17:17:05 -0700 Subject: [PATCH 36/39] add test for top/bottom names --- python/caffe/test/test_net.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/caffe/test/test_net.py b/python/caffe/test/test_net.py index 4cacfcd05bb..96821e40c1d 100644 --- a/python/caffe/test/test_net.py +++ b/python/caffe/test/test_net.py @@ -3,6 +3,7 @@ import os import numpy as np import six +from collections import OrderedDict import caffe @@ -67,6 +68,18 @@ def test_inputs_outputs(self): self.assertEqual(self.net.inputs, []) self.assertEqual(self.net.outputs, ['loss']) + def test_top_bottom_names(self): + self.assertEqual(self.net.top_names, + OrderedDict([('data', ['data', 'label']), + ('conv', ['conv']), + ('ip', ['ip']), + ('loss', ['loss'])])) + self.assertEqual(self.net.bottom_names, + OrderedDict([('data', []), + ('conv', ['data']), + ('ip', ['conv']), + ('loss', ['ip', 'label'])])) + def test_save_and_read(self): f = tempfile.NamedTemporaryFile(mode='w+', delete=False) f.close() From d9ad2ef90a1cbaa2b22b229539a14341efe59ee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Malte=20St=C3=A6r=20Nissen?= Date: Wed, 13 Jul 2016 11:17:54 +0200 Subject: [PATCH 37/39] Support spaces in path when downloading ILSVRC12 and MNIST --- data/ilsvrc12/get_ilsvrc_aux.sh | 2 +- data/mnist/get_mnist.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/ilsvrc12/get_ilsvrc_aux.sh b/data/ilsvrc12/get_ilsvrc_aux.sh index 90935f25099..dc0d0a72790 100755 --- a/data/ilsvrc12/get_ilsvrc_aux.sh +++ b/data/ilsvrc12/get_ilsvrc_aux.sh @@ -8,7 +8,7 @@ # - the training splits with labels DIR="$( cd "$(dirname "$0")" ; pwd -P )" -cd $DIR +cd "$DIR" echo "Downloading..." diff --git a/data/mnist/get_mnist.sh b/data/mnist/get_mnist.sh index 6d875219489..ecadffa44f7 100755 --- a/data/mnist/get_mnist.sh +++ b/data/mnist/get_mnist.sh @@ -2,7 +2,7 @@ # This scripts downloads the mnist data and unzips it. DIR="$( cd "$(dirname "$0")" ; pwd -P )" -cd $DIR +cd "$DIR" echo "Downloading..." From 93d321227f0681165b126d9ca47b211f5d2c1909 Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Wed, 13 Jul 2016 15:58:29 -0700 Subject: [PATCH 38/39] Add "set -e" and $@ to example scripts --- examples/cifar10/create_cifar10.sh | 1 + examples/cifar10/train_full.sh | 7 ++++--- examples/cifar10/train_full_sigmoid.sh | 3 ++- examples/cifar10/train_full_sigmoid_bn.sh | 3 ++- examples/cifar10/train_quick.sh | 5 +++-- examples/imagenet/create_imagenet.sh | 1 + examples/imagenet/resume_training.sh | 4 +++- examples/imagenet/train_caffenet.sh | 3 ++- examples/mnist/create_mnist.sh | 1 + examples/mnist/train_lenet.sh | 3 ++- examples/mnist/train_lenet_adam.sh | 3 ++- examples/mnist/train_lenet_consolidated.sh | 3 ++- examples/mnist/train_lenet_rmsprop.sh | 4 +++- examples/mnist/train_mnist_autoencoder.sh | 3 ++- examples/mnist/train_mnist_autoencoder_adadelta.sh | 3 ++- examples/mnist/train_mnist_autoencoder_adagrad.sh | 3 ++- examples/mnist/train_mnist_autoencoder_nesterov.sh | 3 ++- examples/siamese/create_mnist_siamese.sh | 1 + examples/siamese/train_mnist_siamese.sh | 3 ++- 19 files changed, 39 insertions(+), 18 deletions(-) diff --git a/examples/cifar10/create_cifar10.sh b/examples/cifar10/create_cifar10.sh index a42725cb610..7ee1d6ad0a0 100755 --- a/examples/cifar10/create_cifar10.sh +++ b/examples/cifar10/create_cifar10.sh @@ -1,5 +1,6 @@ #!/usr/bin/env sh # This script converts the cifar data into leveldb format. +set -e EXAMPLE=examples/cifar10 DATA=data/cifar10 diff --git a/examples/cifar10/train_full.sh b/examples/cifar10/train_full.sh index ef112e1f6db..06ecc2dccb0 100755 --- a/examples/cifar10/train_full.sh +++ b/examples/cifar10/train_full.sh @@ -1,16 +1,17 @@ #!/usr/bin/env sh +set -e TOOLS=./build/tools $TOOLS/caffe train \ - --solver=examples/cifar10/cifar10_full_solver.prototxt + --solver=examples/cifar10/cifar10_full_solver.prototxt $@ # reduce learning rate by factor of 10 $TOOLS/caffe train \ --solver=examples/cifar10/cifar10_full_solver_lr1.prototxt \ - --snapshot=examples/cifar10/cifar10_full_iter_60000.solverstate.h5 + --snapshot=examples/cifar10/cifar10_full_iter_60000.solverstate.h5 $@ # reduce learning rate by factor of 10 $TOOLS/caffe train \ --solver=examples/cifar10/cifar10_full_solver_lr2.prototxt \ - --snapshot=examples/cifar10/cifar10_full_iter_65000.solverstate.h5 + --snapshot=examples/cifar10/cifar10_full_iter_65000.solverstate.h5 $@ diff --git a/examples/cifar10/train_full_sigmoid.sh b/examples/cifar10/train_full_sigmoid.sh index 9cff06d3e34..9b5d5213b2a 100755 --- a/examples/cifar10/train_full_sigmoid.sh +++ b/examples/cifar10/train_full_sigmoid.sh @@ -1,7 +1,8 @@ #!/usr/bin/env sh +set -e TOOLS=./build/tools $TOOLS/caffe train \ - --solver=examples/cifar10/cifar10_full_sigmoid_solver.prototxt + --solver=examples/cifar10/cifar10_full_sigmoid_solver.prototxt $@ diff --git a/examples/cifar10/train_full_sigmoid_bn.sh b/examples/cifar10/train_full_sigmoid_bn.sh index 011387c996e..05547f3a104 100755 --- a/examples/cifar10/train_full_sigmoid_bn.sh +++ b/examples/cifar10/train_full_sigmoid_bn.sh @@ -1,7 +1,8 @@ #!/usr/bin/env sh +set -e TOOLS=./build/tools $TOOLS/caffe train \ - --solver=examples/cifar10/cifar10_full_sigmoid_solver_bn.prototxt + --solver=examples/cifar10/cifar10_full_sigmoid_solver_bn.prototxt $@ diff --git a/examples/cifar10/train_quick.sh b/examples/cifar10/train_quick.sh index 6b7d228879b..d2b875340ee 100755 --- a/examples/cifar10/train_quick.sh +++ b/examples/cifar10/train_quick.sh @@ -1,11 +1,12 @@ #!/usr/bin/env sh +set -e TOOLS=./build/tools $TOOLS/caffe train \ - --solver=examples/cifar10/cifar10_quick_solver.prototxt + --solver=examples/cifar10/cifar10_quick_solver.prototxt $@ # reduce learning rate by factor of 10 after 8 epochs $TOOLS/caffe train \ --solver=examples/cifar10/cifar10_quick_solver_lr1.prototxt \ - --snapshot=examples/cifar10/cifar10_quick_iter_4000.solverstate.h5 + --snapshot=examples/cifar10/cifar10_quick_iter_4000.solverstate.h5 $@ diff --git a/examples/imagenet/create_imagenet.sh b/examples/imagenet/create_imagenet.sh index e912ac43cd7..1bf08b1aa8f 100755 --- a/examples/imagenet/create_imagenet.sh +++ b/examples/imagenet/create_imagenet.sh @@ -1,6 +1,7 @@ #!/usr/bin/env sh # Create the imagenet lmdb inputs # N.B. set the path to the imagenet train + val data dirs +set -e EXAMPLE=examples/imagenet DATA=data/ilsvrc12 diff --git a/examples/imagenet/resume_training.sh b/examples/imagenet/resume_training.sh index bf7945c0fd0..4aef204368e 100755 --- a/examples/imagenet/resume_training.sh +++ b/examples/imagenet/resume_training.sh @@ -1,5 +1,7 @@ #!/usr/bin/env sh +set -e ./build/tools/caffe train \ --solver=models/bvlc_reference_caffenet/solver.prototxt \ - --snapshot=models/bvlc_reference_caffenet/caffenet_train_10000.solverstate.h5 + --snapshot=models/bvlc_reference_caffenet/caffenet_train_10000.solverstate.h5 \ + $@ diff --git a/examples/imagenet/train_caffenet.sh b/examples/imagenet/train_caffenet.sh index 94558ec5466..a5094d44ae0 100755 --- a/examples/imagenet/train_caffenet.sh +++ b/examples/imagenet/train_caffenet.sh @@ -1,4 +1,5 @@ #!/usr/bin/env sh +set -e ./build/tools/caffe train \ - --solver=models/bvlc_reference_caffenet/solver.prototxt + --solver=models/bvlc_reference_caffenet/solver.prototxt $@ diff --git a/examples/mnist/create_mnist.sh b/examples/mnist/create_mnist.sh index 06ecc27de63..f5e2e7960c5 100755 --- a/examples/mnist/create_mnist.sh +++ b/examples/mnist/create_mnist.sh @@ -1,6 +1,7 @@ #!/usr/bin/env sh # This script converts the mnist data into lmdb/leveldb format, # depending on the value assigned to $BACKEND. +set -e EXAMPLE=examples/mnist DATA=data/mnist diff --git a/examples/mnist/train_lenet.sh b/examples/mnist/train_lenet.sh index 1b6bf7d978d..f7f9b86198d 100755 --- a/examples/mnist/train_lenet.sh +++ b/examples/mnist/train_lenet.sh @@ -1,3 +1,4 @@ #!/usr/bin/env sh +set -e -./build/tools/caffe train --solver=examples/mnist/lenet_solver.prototxt +./build/tools/caffe train --solver=examples/mnist/lenet_solver.prototxt $@ diff --git a/examples/mnist/train_lenet_adam.sh b/examples/mnist/train_lenet_adam.sh index a32ecf2d9c2..7b4e905681b 100755 --- a/examples/mnist/train_lenet_adam.sh +++ b/examples/mnist/train_lenet_adam.sh @@ -1,3 +1,4 @@ #!/usr/bin/env sh +set -e -./build/tools/caffe train --solver=examples/mnist/lenet_solver_adam.prototxt +./build/tools/caffe train --solver=examples/mnist/lenet_solver_adam.prototxt $@ diff --git a/examples/mnist/train_lenet_consolidated.sh b/examples/mnist/train_lenet_consolidated.sh index c855467897e..c5f02666822 100755 --- a/examples/mnist/train_lenet_consolidated.sh +++ b/examples/mnist/train_lenet_consolidated.sh @@ -1,4 +1,5 @@ #!/usr/bin/env sh +set -e ./build/tools/caffe train \ - --solver=examples/mnist/lenet_consolidated_solver.prototxt + --solver=examples/mnist/lenet_consolidated_solver.prototxt $@ diff --git a/examples/mnist/train_lenet_rmsprop.sh b/examples/mnist/train_lenet_rmsprop.sh index 621cab238bf..adfa7ab0fca 100755 --- a/examples/mnist/train_lenet_rmsprop.sh +++ b/examples/mnist/train_lenet_rmsprop.sh @@ -1,3 +1,5 @@ #!/usr/bin/env sh +set -e -./build/tools/caffe train --solver=examples/mnist/lenet_solver_rmsprop.prototxt +./build/tools/caffe train \ + --solver=examples/mnist/lenet_solver_rmsprop.prototxt $@ diff --git a/examples/mnist/train_mnist_autoencoder.sh b/examples/mnist/train_mnist_autoencoder.sh index cfd67e82fda..724a0f14a49 100755 --- a/examples/mnist/train_mnist_autoencoder.sh +++ b/examples/mnist/train_mnist_autoencoder.sh @@ -1,4 +1,5 @@ #!/usr/bin/env sh +set -e ./build/tools/caffe train \ - --solver=examples/mnist/mnist_autoencoder_solver.prototxt + --solver=examples/mnist/mnist_autoencoder_solver.prototxt $@ diff --git a/examples/mnist/train_mnist_autoencoder_adadelta.sh b/examples/mnist/train_mnist_autoencoder_adadelta.sh index 4be0ebddedc..a660dbb9ed2 100755 --- a/examples/mnist/train_mnist_autoencoder_adadelta.sh +++ b/examples/mnist/train_mnist_autoencoder_adadelta.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e ./build/tools/caffe train \ - --solver=examples/mnist/mnist_autoencoder_solver_adadelta.prototxt + --solver=examples/mnist/mnist_autoencoder_solver_adadelta.prototxt $@ diff --git a/examples/mnist/train_mnist_autoencoder_adagrad.sh b/examples/mnist/train_mnist_autoencoder_adagrad.sh index 95fe1b17bd5..4c11dfa67ac 100755 --- a/examples/mnist/train_mnist_autoencoder_adagrad.sh +++ b/examples/mnist/train_mnist_autoencoder_adagrad.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e ./build/tools/caffe train \ - --solver=examples/mnist/mnist_autoencoder_solver_adagrad.prototxt + --solver=examples/mnist/mnist_autoencoder_solver_adagrad.prototxt $@ diff --git a/examples/mnist/train_mnist_autoencoder_nesterov.sh b/examples/mnist/train_mnist_autoencoder_nesterov.sh index cf19ea749b3..fd0559d2488 100755 --- a/examples/mnist/train_mnist_autoencoder_nesterov.sh +++ b/examples/mnist/train_mnist_autoencoder_nesterov.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e ./build/tools/caffe train \ - --solver=examples/mnist/mnist_autoencoder_solver_nesterov.prototxt + --solver=examples/mnist/mnist_autoencoder_solver_nesterov.prototxt $@ diff --git a/examples/siamese/create_mnist_siamese.sh b/examples/siamese/create_mnist_siamese.sh index 43ad6b184a7..03adce54d9b 100755 --- a/examples/siamese/create_mnist_siamese.sh +++ b/examples/siamese/create_mnist_siamese.sh @@ -1,5 +1,6 @@ #!/usr/bin/env sh # This script converts the mnist data into leveldb format. +set -e EXAMPLES=./build/examples/siamese DATA=./data/mnist diff --git a/examples/siamese/train_mnist_siamese.sh b/examples/siamese/train_mnist_siamese.sh index 84a30a8ac44..e01ac2ceefd 100755 --- a/examples/siamese/train_mnist_siamese.sh +++ b/examples/siamese/train_mnist_siamese.sh @@ -1,5 +1,6 @@ #!/usr/bin/env sh +set -e TOOLS=./build/tools -$TOOLS/caffe train --solver=examples/siamese/mnist_siamese_solver.prototxt +$TOOLS/caffe train --solver=examples/siamese/mnist_siamese_solver.prototxt $@ From 9663905ee52c554ed455ec3959086735e5768f7f Mon Sep 17 00:00:00 2001 From: Kenneth Tran Date: Thu, 14 Jul 2016 15:54:08 -0700 Subject: [PATCH 39/39] Sync with BVLC master --- .gitignore | 3 ++- README.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 15165e7f0db..e68c272f49a 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,7 @@ models/* # build, distribute, and bins (+ python proto bindings) build +windows/packages .build_debug/* .build_release/* distribute/* @@ -100,4 +101,4 @@ MANIFEST-* *.sdf *.opensdf *.pdb -*.props \ No newline at end of file +*.props diff --git a/README.md b/README.md index 41fcf8cc298..cc57f170f23 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ If you don't have CUDA installed, you can experiment with CPU_ONLY build. In `.\windows\CommonSettings.props` set `CpuOnlyBuild` to `true` and set `UseCuDNN` to `false`. ### cuDNN -Download `cuDNN v3` or `cuDNN v4` [from nVidia website](https://developer.nvidia.com/cudnn). +Download `cuDNN v4` or `cuDNN v5` [from nVidia website](https://developer.nvidia.com/cudnn). Unpack downloaded zip to %CUDA_PATH% (environment variable set by CUDA installer). Alternatively, you can unpack zip to any location and set `CuDnnPath` to point to this location in `.\windows\CommonSettings.props`. `CuDnnPath` defined in `.\windows\CommonSettings.props`.