Initial attempt at getting protobuf.js loading etcd protos.

package.json

@@ -0,0 +1,20 @@
+{
+	"name": "etcd-pump",
+	"version": "1.0.0",
+	"description": "",
+	"main": "etcd-pump.js",
+	"scripts": {
+		"test": "echo \"Error: no test specified\" && exit 1"
+	},
+	"repository": {
+		"type": "git",
+		"url": "https://archive.voodoowarez.com/etcd-pump.git"
+	},
+	"author": "",
+	"license": "ISC",
+	"dependencies": {
+		"es6-promisify": "^3.0.0",
+		"grpc": "^0.11.1",
+		"protobufjs": "^4.1.3"
+	}
+}

proto/etcdserver/etcdserverpb/rpc.proto

@@ -0,0 +1,196 @@
+syntax = "proto3";
+package etcdserverpb;
+
+import "gogoproto/gogo.proto";
+import "etcd/storage/storagepb/kv.proto";
+
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+
+service KV {
+  // Range gets the keys in the range from the store.
+  rpc Range(RangeRequest) returns (RangeResponse) {}
+
+  // Put puts the given key into the store.
+  // A put request increases the revision of the store,
+  // and generates one event in the event history.
+  rpc Put(PutRequest) returns (PutResponse) {}
+
+  // Delete deletes the given range from the store.
+  // A delete request increase the revision of the store,
+  // and generates one event in the event history.
+  rpc DeleteRange(DeleteRangeRequest) returns (DeleteRangeResponse) {}
+
+  // Txn processes all the requests in one transaction.
+  // A txn request increases the revision of the store,
+  // and generates events with the same revision in the event history.
+  rpc Txn(TxnRequest) returns (TxnResponse) {}
+
+  // Compact compacts the event history in etcd. User should compact the
+  // event history periodically, or it will grow infinitely.
+  rpc Compact(CompactionRequest) returns (CompactionResponse) {}
+}
+
+service watch {
+  // Watch watches the events happening or happened. Both input and output
+  // are stream. One watch rpc can watch for multiple keys or prefixs and
+  // get a stream of events. The whole events history can be watched unless
+  // compacted.
+  rpc Watch(stream WatchRequest) returns (stream WatchResponse) {}
+}
+
+message ResponseHeader {
+  uint64 cluster_id = 1;
+  uint64 member_id = 2;
+  // revision of the store when the request was applied.
+  int64 revision = 3;
+  // term of raft when the request was applied.
+  uint64 raft_term = 4;
+}
+
+message RangeRequest {
+  // if the range_end is not given, the request returns the key.
+  bytes key = 1;
+  // if the range_end is given, it gets the keys in range [key, range_end).
+  bytes range_end = 2;
+  // limit the number of keys returned.
+  int64 limit = 3;
+  // range over the store at the given revision.
+  // if revision is less or equal to zero, range over the newest store.
+  // if the revision has been compacted, ErrCompaction will be returned in
+  // response.
+  int64 revision = 4;
+}
+
+message RangeResponse {
+  ResponseHeader header = 1;
+  repeated storagepb.KeyValue kvs = 2;
+  // more indicates if there are more keys to return in the requested range.
+  bool more = 3;
+}
+
+message PutRequest {
+  bytes key = 1;
+  bytes value = 2;
+}
+
+message PutResponse {
+  ResponseHeader header = 1;
+}
+
+message DeleteRangeRequest {
+  // if the range_end is not given, the request deletes the key.
+  bytes key = 1;
+  // if the range_end is given, it deletes the keys in range [key, range_end).
+  bytes range_end = 2;
+}
+
+message DeleteRangeResponse {
+  ResponseHeader header = 1;
+}
+
+message RequestUnion {
+  oneof request {
+    RangeRequest request_range = 1;
+    PutRequest request_put = 2;
+    DeleteRangeRequest request_delete_range = 3;
+  }
+}
+
+message ResponseUnion {
+  oneof response {
+    RangeResponse response_range = 1;
+    PutResponse response_put = 2;
+    DeleteRangeResponse response_delete_range = 3;
+  }
+}
+
+message Compare {
+  enum CompareResult {
+    EQUAL = 0;
+    GREATER = 1;
+    LESS = 2;
+  }
+  enum CompareTarget {
+    VERSION = 0;
+    CREATE = 1;
+    MOD = 2;
+    VALUE= 3;
+  }
+  CompareResult result = 1;
+  CompareTarget target = 2;
+  // key path
+  bytes key = 3;
+  oneof target_union {
+    // version of the given key
+    int64 version = 4;
+    // create revision of the given key
+    int64 create_revision = 5;
+    // last modified revision of the given key
+    int64 mod_revision = 6;
+    // value of the given key
+    bytes value = 7;
+  }
+}
+
+// If the comparisons succeed, then the success requests will be processed in order,
+// and the response will contain their respective responses in order.
+// If the comparisons fail, then the failure requests will be processed in order,
+// and the response will contain their respective responses in order.
+
+// From google paxosdb paper:
+// Our implementation hinges around a powerful primitive which we call MultiOp. All other database
+// operations except for iteration are implemented as a single call to MultiOp. A MultiOp is applied atomically
+// and consists of three components:
+// 1. A list of tests called guard. Each test in guard checks a single entry in the database. It may check
+// for the absence or presence of a value, or compare with a given value. Two different tests in the guard
+// may apply to the same or different entries in the database. All tests in the guard are applied and
+// MultiOp returns the results. If all tests are true, MultiOp executes t op (see item 2 below), otherwise
+// it executes f op (see item 3 below).
+// 2. A list of database operations called t op. Each operation in the list is either an insert, delete, or
+// lookup operation, and applies to a single database entry. Two different operations in the list may apply
+// to the same or different entries in the database. These operations are executed
+// if guard evaluates to
+// true.
+// 3. A list of database operations called f op. Like t op, but executed if guard evaluates to false.
+message TxnRequest {
+  repeated Compare compare = 1;
+  repeated RequestUnion success = 2;
+  repeated RequestUnion failure = 3;
+}
+
+message TxnResponse {
+  ResponseHeader header = 1;
+  bool succeeded = 2;
+  repeated ResponseUnion responses = 3;
+}
+
+// Compaction compacts the kv store upto the given revision (including).
+// It removes the old versions of a key. It keeps the newest version of
+// the key even if its latest modification revision is smaller than the given
+// revision.
+message CompactionRequest {
+  int64 revision = 1;
+}
+
+message CompactionResponse {
+  ResponseHeader header = 1;
+}
+
+message WatchRequest {
+  // the key to be watched
+  bytes key = 1;
+  // the prefix to be watched.
+  bytes prefix = 2;
+  // start_revision is an optional revision (including) to watch from. No start_revision is "now".
+  int64 start_revision = 3;
+  // TODO: support Range watch?
+  // TODO: support notification every time interval or revision increase?
+  // TODO: support cancel watch if the server cannot reach with majority?
+}
+
+message WatchResponse {
+  ResponseHeader header = 1;
+  // TODO: support batched events response?
+  storagepb.Event event = 2;
+}

proto/etcdserver/etcdserverpb/rpc.proto.url

@@ -0,0 +1 @@
+https://github.com/raw/coreos/etcd/master/etcdserver/etcdserverpb/rpc.proto

proto/gogoproto/gogo.proto

@@ -0,0 +1,107 @@
+// Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved.
+// http://github.com/gogo/protobuf/gogoproto
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto2";
+package gogoproto;
+
+import "google/protobuf/descriptor.proto";
+
+option java_package = "com.google.protobuf";
+option java_outer_classname = "GoGoProtos";
+
+extend google.protobuf.EnumOptions {
+	optional bool goproto_enum_prefix = 62001;
+	optional bool goproto_enum_stringer = 62021;
+	optional bool enum_stringer = 62022;
+}
+
+extend google.protobuf.FileOptions {
+	optional bool goproto_getters_all = 63001;
+	optional bool goproto_enum_prefix_all = 63002;
+	optional bool goproto_stringer_all = 63003;
+	optional bool verbose_equal_all = 63004;
+	optional bool face_all = 63005;
+	optional bool gostring_all = 63006;
+	optional bool populate_all = 63007;
+	optional bool stringer_all = 63008;
+	optional bool onlyone_all = 63009;
+
+	optional bool equal_all = 63013;
+	optional bool description_all = 63014;
+	optional bool testgen_all = 63015;
+	optional bool benchgen_all = 63016;
+	optional bool marshaler_all = 63017;
+	optional bool unmarshaler_all = 63018;
+
+	optional bool sizer_all = 63020;
+
+	optional bool goproto_enum_stringer_all = 63021;
+	optional bool enum_stringer_all = 63022;
+
+	optional bool unsafe_marshaler_all = 63023;
+	optional bool unsafe_unmarshaler_all = 63024;
+
+	optional bool goproto_extensions_map_all = 63025;
+	optional bool goproto_unrecognized_all = 63026;
+	optional bool gogoproto_import = 63027;
+}
+
+extend google.protobuf.MessageOptions {
+	optional bool goproto_getters = 64001;
+	optional bool goproto_stringer = 64003;
+	optional bool verbose_equal = 64004;
+	optional bool face = 64005;
+	optional bool gostring = 64006;
+	optional bool populate = 64007;
+  	optional bool stringer = 67008;
+	optional bool onlyone = 64009;
+
+	optional bool equal = 64013;
+	optional bool description = 64014;
+	optional bool testgen = 64015;
+	optional bool benchgen = 64016;
+	optional bool marshaler = 64017;
+	optional bool unmarshaler = 64018;
+
+	optional bool sizer = 64020;
+
+	optional bool unsafe_marshaler = 64023;
+	optional bool unsafe_unmarshaler = 64024;
+
+	optional bool goproto_extensions_map = 64025;
+	optional bool goproto_unrecognized = 64026;
+}
+
+extend google.protobuf.FieldOptions {
+  optional bool nullable = 65001;
+  optional bool embed = 65002;
+  optional string customtype = 65003;
+  optional string customname = 65004;
+  optional string jsontag = 65005;
+  optional string moretags = 65006;
+  optional string casttype = 65007;
+}
+

proto/gogoproto/gogo.proto.url

@@ -0,0 +1 @@
+https://github.com/raw/gogo/protobuf/master/gogoproto/gogo.proto

proto/storage/storagepb/kv.proto

@@ -0,0 +1,37 @@
+syntax = "proto3";
+package storagepb;
+
+import "gogoproto/gogo.proto";
+
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.sizer_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_enum_prefix_all) = false;
+
+message KeyValue {
+  bytes key = 1;
+  int64 create_revision = 2;
+  // mod_revision is the last modified revision of the key.
+  int64 mod_revision = 3;
+  // version is the version of the key. A deletion resets
+  // the version to zero and any modification of the key
+  // increases its version.
+  int64 version = 4;
+  bytes value = 5;
+}
+
+message Event {
+  enum EventType {
+    PUT = 0;
+    DELETE = 1;
+    EXPIRE = 2;
+  }
+  EventType type = 1;
+  // a put event contains the current key-value
+  // a delete/expire event contains the previous
+  // key-value
+  KeyValue kv = 2;
+  // watchID is the ID of watching this event is sent to.
+  int64 watchID = 3;
+}

proto/storage/storagepb/kv.proto.url

@@ -0,0 +1 @@
+https://github.com/raw/coreos/etcd/master/storage/storagepb/kv.proto