Release V0.3.0 [1. Added PrivateTextLineDataset, PrivateInput, Secure…

…LogicalAnd, SecureLogicalOr, SecureLogicalXor, SecureLogicalNot secure operations; 2. Speedup some backend operations; 3. Uses related python classes such as PrivateTextLineDataset and iterators to load large data sets, thereby reducing memory usage; 4. Some known bugs are fixed.]

.gitignore

@@ -56,4 +56,4 @@ core
 !core/
 *.log.*
 *.log
-
+*yyl*

RELEASE.md

@@ -3,6 +3,7 @@
 
 - [Release Notes](#release-notes)
   - [Introduction](#introduction)
+  - [Rosetta v0.3.0](#rosetta-v030)
   - [Rosetta v0.2.1](#rosetta-v021)
   - [Rosetta v0.2.0](#rosetta-v020)
   - [Rosetta v0.1.1](#rosetta-v011)
@@ -19,6 +20,18 @@
 
 This document will maintain and continually update the release notes of each version of Rosetta. If you have questions or comments, please contact us via rosetta@latticex.foundation.
 
+## Rosetta v0.3.0
+
+1. Added `PrivateTextLineDataset`, `PrivateInput` secure operations.
+
+2. Added `SecureLogicalAnd`, `SecureLogicalOr`,  `SecureLogicalXor`, `SecureLogicalNot` secure operations.
+
+3. Speedup some backend operations.
+
+4. Uses related python classes such as PrivateTextLineDataset and iterators to load large data sets, thereby reducing memory usage.
+
+5. Some known bugs are fixed.
+
 ## Rosetta v0.2.1
 
 1. Support 128-bit integer data type, which enables big integer and high precision float-point operations.

cc/modules/common/include/utils/file_directory.h

@@ -26,6 +26,13 @@
 #include <random>
 #include <regex>
 #include <string>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include "cc/modules/common/include/utils/logger.h"
+
+
 using namespace std;
 
 /**
@@ -134,3 +141,77 @@ static void fromfile(vector<vector<T>>& v, const string& filename) {
   ifile.close();
 }
 
+// Get line or fields count of `csv` file
+//
+// Arguments:
+// * file: file path to analyse.
+// * delimiter: the delimiter of fields.
+// * lines: output of count of lines
+// * fields: output of count of fields in a line
+// * ignore_blank_line: whether skip or ignore blank lines
+//
+// Returns an error if the while loop could not be fully constructed.
+//
+// TODO(kelvin): 
+int get_file_lines_fields(const string& file, char delimiter, int& lines, int& fields, bool ignore_blank_line=false)
+{
+  fields = 1;
+  lines = 0;
+  const int buf_size = 16*1024;
+  char buf[buf_size] = {0};
+  int fd = open(file.data(), O_RDONLY);
+  if (fd == -1)
+  {
+    log_error << "open file: "  << file << "failed!" << endl;
+    return -1;
+  }
+
+  // read the first line
+  ssize_t bytes = 0;
+  if (0 < (bytes = read(fd, buf, 4096))) {
+    char* p = buf;
+    while (*p != '\n' && p != buf+bytes) {
+      if (*(p++) == ',')
+        fields++;
+    }
+  }
+  //reset file to begin
+  lseek(fd, 0, SEEK_SET);
+
+  int index = 0;
+  while(0 < (bytes=read(fd, buf, buf_size))){
+    //count lines for the buffer
+    char* p = buf;
+    if (ignore_blank_line) {
+      char* pre = p;
+      while (p != buf+bytes)
+      {
+        if(*(p++) == '\n')
+        {
+          if (p - pre == 1)
+          {
+            pre = p;
+            continue;
+          }
+
+          lines++;
+          pre = p;
+        }
+      }
+    } else {
+      while (p != buf+bytes)
+      {
+        if(*(p++) == '\n')
+        {
+          lines++;
+        }
+      }
+    }
+
+    if (bytes >=1 && bytes < buf_size &&  buf[bytes-1] !='\n')
+      ++lines;
+  }
+
+  log_debug << "lines: "<< lines << ", fields: " << fields << endl;
+  return 0;
+}

cc/modules/protocol/mpc/snn/include/snn_opsets.h

@@ -667,7 +667,7 @@ class ReconstructBit2PC : public OpBase {
 
 /*
   @note: this Polynomial is mostly for internal usage, 
-    especially for complex functionalities, such as Log and Log1p, that 
+    especially for complex funtionalities, such as Log and Log1p, that 
     are implemented with polynomial interpolation.
   
   @attention: for now DO NOT use this directly if you are not sure.
@@ -809,7 +809,7 @@ class Sigmoid : public OpBase {
   }
 
   int Run(const vector<mpc_t>& a, vector<mpc_t>& b, size_t size) {
-#define USE_SIGMOID_VERSION 1
+#define USE_SIGMOID_VERSION 2
 #if USE_SIGMOID_VERSION == 0
     return RunChebyshevPoly(a, b, size);
 #elif USE_SIGMOID_VERSION == 1

cc/modules/protocol/mpc/snn/src/snn_protocol_ops.cpp

@@ -186,7 +186,7 @@ int SnnProtocolOps::PrivateInput(
   vector<mpc_t> out_vec;
   std::make_shared<rosetta::snn::PrivateInput>(_op_msg_id, net_io_)->Run(party_id, in_vec, out_vec);
 
-  snn_encode_(out_vec, out_str_vec);
+  snn_encode(out_vec, out_str_vec);
   return 0;
 }
 

cc/modules/protocol/mpc/snn/tests/snn_truncation.cpp

@@ -0,0 +1,50 @@
+#include "snn__test.h"
+
+void run(int partyid) {
+  SNN_PROTOCOL_TEST_INIT(partyid);
+  // mimic 8^3 * 8^2
+  vector<double> X(100000, 8.0 * 8.0 * 8.0);
+  vector<double> Y(100000, 8.0 * 8.0);
+
+  vector<double> EXPECT(100000, 32768.0);
+  size_t size = X.size();
+
+  // vector<double> X = { 8.0, -8.0, 5, -5, 2, -2};
+  // vector<double> EXPECT = {0.9997, 0.00033, 0.9933, 0.00669, 0.8808, 0.1192};
+  // size_t size = X.size();
+  print_vec(X, 10, "X");
+  print_vec(Y, 10, "Y");
+
+  string msgid("Mul OP(s) (share,share)");
+  cout << __FUNCTION__ << " " << msgid << endl;
+
+  vector<string> strX, strY, strZ;
+  vector<string> zZ(strZ.size());
+  // In theory, the probability of this truncation error is about 1/(2^{64-(15+16+16)}, 
+  // i.e. 1/(2^17) = 0.000008
+  // we do this 10-timers to make this more likely to happen.
+  int wrong_cnt = 0;
+  int ITER = 10;
+  for(int kkk = 0;  kkk < ITER; ++kkk) {
+      snn0.GetOps(msgid)->PrivateInput(0, X, strX);
+      snn0.GetOps(msgid)->PrivateInput(0, Y, strY);
+      snn0.GetOps(msgid)->Mul(strX, strY, strZ);
+      snn0.GetOps(msgid)->Reveal(strZ, zZ);
+      print_vec(zZ, 10, "SNN Mul plaintext:");
+      print_vec(EXPECT, 10, "Mul expected:");
+      for(int i = 0; i < size; ++i) {
+        auto inner = stol(zZ[i]);
+        auto expected = long(EXPECT[i]);
+        if (inner != expected) {
+          wrong_cnt++;
+          cout << i << "-th item wrong in iter: " << kkk << "!!!" << endl;
+          cout << "inner: " << inner << " <-> expected: " << expected << endl;
+        }
+      }
+  }
+  cout << "error num:" << wrong_cnt << endl;
+  cout << "probability in this case:" << (wrong_cnt * 1.0) / (ITER * size) << endl; 
+  SNN_PROTOCOL_TEST_UNINIT(partyid);
+}
+
+RUN_MPC_TEST(run);

cc/tf/secureops/data/secure_dataset_ops.cc

@@ -0,0 +1,35 @@
+// ==============================================================================
+// Copyright 2020 The LatticeX Foundation
+// This file is part of the Rosetta library.
+//
+// The Rosetta library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The Rosetta library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the Rosetta library. If not, see <http://www.gnu.org/licenses/>.
+// ==============================================================================
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_def_builder.h"
+#include "tensorflow/core/framework/shape_inference.h"
+
+namespace tensorflow {
+
+REGISTER_OP("PrivateTextLineDataset")
+    .Input("filenames: string")
+    .Input("compression_type: string")
+    .Input("buffer_size: int64")
+    .Input("data_owner: int64")
+    .Output("handle: variant")
+    .SetIsStateful();
+    // shape function will depressed at present !!!!
+
+}