From 2ddb76b6e93174787a68f68fb28d26b8ece7cc56 Mon Sep 17 00:00:00 2001 From: dcodeIO Date: Sat, 4 Feb 2017 05:36:57 +0100 Subject: [PATCH] CLI: Added an experimental --sparse option to limit pbjs output to actually referenced types within main files; Other: Added a few more common google types from google/api, see #433 --- .travis.yml | 12 ++-- cli/pbjs.js | 116 +++++++++++++++++++++++++++++----- cli/util.js | 32 ++++++++++ google/api/annotations.json | 83 ++++++++++++++++++++++++ google/api/annotations.proto | 11 ++++ google/api/http.json | 86 +++++++++++++++++++++++++ google/api/http.proto | 31 +++++++++ google/protobuf/api.json | 118 +++++++++++++++++++++++++++++++++++ google/protobuf/api.proto | 34 ++++++++++ google/protobuf/type.json | 8 +++ package.json | 1 - scripts/gencommons.js | 26 ++++++++ src/field.js | 7 ++- src/object.js | 6 ++ src/parse.js | 19 ++++-- 15 files changed, 561 insertions(+), 29 deletions(-) create mode 100644 google/api/annotations.json create mode 100644 google/api/annotations.proto create mode 100644 google/api/http.json create mode 100644 google/api/http.proto create mode 100644 google/protobuf/api.json create mode 100644 google/protobuf/api.proto create mode 100644 scripts/gencommons.js diff --git a/.travis.yml b/.travis.yml index ae55efe9b..80709b0b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,17 +5,19 @@ node_js: - 4.3.2 - 6 - 7 +branches: + only: master +script: npm test && npm run bench + env: CXX=g++-4.8 addons: apt: sources: ubuntu-toolchain-r-test packages: g++-4.8 -script: npm test && npm run bench -branches: - only: master + matrix: include: - node_js: 6 - script: set -e; if [ -n "$SAUCE_USERNAME" ]; then npm install zuul@3.11.1 zuul-ngrok@4.0.0; travis_wait npm run zuul; sleep 3; fi + script: set -e; if [ -n "$SAUCE_USERNAME" ]; then npm install zuul@^3.11.1 zuul-ngrok@^4.0.0; travis_wait npm run zuul; sleep 3; fi - node_js: 6 - script: npm run coverage-ci + script: npm install coveralls@^2.11.15; npm run coverage-ci diff --git a/cli/pbjs.js b/cli/pbjs.js index 9e1c01ca0..e72c3ec0e 100644 --- a/cli/pbjs.js +++ b/cli/pbjs.js @@ -31,7 +31,7 @@ exports.main = function(args, callback) { lint : "l" }, string: [ "target", "out", "path", "wrap", "root", "lint" ], - boolean: [ "keep-case", "create", "encode", "decode", "verify", "convert", "delimited", "beautify", "comments", "es6" ], + boolean: [ "keep-case", "create", "encode", "decode", "verify", "convert", "delimited", "beautify", "comments", "es6", "sparse" ], default: { target : "json", create : true, @@ -51,6 +51,9 @@ exports.main = function(args, callback) { files = argv._, paths = typeof argv.path === "string" ? [ argv.path ] : argv.path || []; + // protobuf.js package directory contains additional, otherwise non-bundled google types + paths.push(path.relative(process.cwd(), path.join(__dirname, "..")) || "."); + if (!files.length) { var descs = Object.keys(targets).filter(function(key) { return !targets[key].private; }).map(function(key) { return " " + util.pad(key, 14, true) + targets[key].description; @@ -71,6 +74,8 @@ exports.main = function(args, callback) { "", " -o, --out Saves to a file instead of writing to stdout.", "", + " --sparse Exports only those types referenced from a main file (experimental).", + "", chalk.bold.gray(" Module targets only:"), "", " -w, --wrap Specifies the wrapper to use. Also accepts a path to require a custom wrapper.", @@ -124,17 +129,33 @@ exports.main = function(args, callback) { var root = new protobuf.Root(); + var mainFiles = []; + // Search include paths when resolving imports root.resolvePath = function pbjsResolvePath(origin, target) { - var filepath = protobuf.util.path.resolve(origin, target); - if (fs.existsSync(filepath)) - return filepath; + var normOrigin = protobuf.util.path.normalize(origin), + normTarget = protobuf.util.path.normalize(target); + if (!normOrigin) + mainFiles.push(normTarget); + + var resolved = protobuf.util.path.resolve(normOrigin, normTarget, true); + var idx = resolved.lastIndexOf("google/protobuf/"); + if (idx > -1) { + var altname = resolved.substring(idx); + if (altname in protobuf.common) + resolved = altname; + } + + if (fs.existsSync(resolved)) + return resolved; + for (var i = 0; i < paths.length; ++i) { - var ifilepath = protobuf.util.path.resolve(paths[i] + "/", target); - if (fs.existsSync(ifilepath)) - return ifilepath; + var iresolved = protobuf.util.path.resolve(paths[i] + "/", target); + if (fs.existsSync(iresolved)) + return iresolved; } - return filepath; + + return resolved; }; // Use es6 syntax if not explicitly specified on the command line and the es6 wrapper is used @@ -153,19 +174,28 @@ exports.main = function(args, callback) { }); process.stdin.on("end", function() { var source = Buffer.concat(data).toString("utf8"); - if (source.charAt(0) !== "{") { - protobuf.parse(source, root, parseOptions); - } else { - var json = JSON.parse(source); - root.setOptions(json.options).addJSON(json); + try { + if (source.charAt(0) !== "{") { + protobuf.parse.filename = "-"; + protobuf.parse(source, root, parseOptions); + } else { + var json = JSON.parse(source); + root.setOptions(json.options).addJSON(json); + } + callTarget(); + } catch (err) { + if (callback) + return callback(err); + throw err; } - callTarget(); }); // Load from disk } else { try { - root.loadSync(files, parseOptions); // sync is deterministic while async is not + root.loadSync(files, parseOptions).resolveAll(); // sync is deterministic while async is not + if (argv.sparse) + sparsify(root); callTarget(); } catch (err) { if (callback) { @@ -176,6 +206,62 @@ exports.main = function(args, callback) { } } + function markReferenced(tobj) { + tobj.referenced = true; + // also mark a type's fields and oneofs + if (tobj.fieldsArray) + tobj.fieldsArray.forEach(function(fobj) { + fobj.referenced = true; + }); + if (tobj.oneofsArray) + tobj.oneofsArray.forEach(function(oobj) { + oobj.referenced = true; + }); + // also mark an extension field's extended type, but not its (other) fields + if (tobj.extensionField) + tobj.extensionField.parent.referenced = true; + } + + function sparsify(root) { + + // 1. mark directly or indirectly referenced objects + util.traverse(root, function(obj) { + if (!obj.filename) + return; + if (mainFiles.indexOf(obj.filename) > -1) + util.traverseResolved(obj, markReferenced); + }); + + // 2. empty unreferenced objects + util.traverse(root, function(obj) { + var parent = obj.parent; + if (!parent || obj.referenced) // root or referenced + return; + // remove unreferenced namespaces + if (obj instanceof protobuf.Namespace) { + var hasReferenced = false; + util.traverse(obj, function(iobj) { + if (iobj.referenced) + hasReferenced = true; + }); + if (hasReferenced) { // replace with plain namespace if a namespace subclass + if (obj instanceof protobuf.Type || obj instanceof protobuf.Service) { + var robj = new protobuf.Namespace(obj.name, obj.options); + robj.nested = obj.nested; + parent.add(robj); + } + } else // remove completely if nothing inside is referenced + parent.remove(obj); + + // remove everything else unreferenced + } else if (!(obj instanceof protobuf.Namespace)) + parent.remove(obj); + }); + + // 3. validate that everything is fine + root.resolveAll(); + } + function callTarget() { target(root, argv, function targetCallback(err, output) { if (err) { diff --git a/cli/util.js b/cli/util.js index f8077b5b1..d752d29d5 100644 --- a/cli/util.js +++ b/cli/util.js @@ -33,6 +33,38 @@ exports.requireAll = function requireAll(dirname) { return all; }; +exports.traverse = function traverse(current, fn) { + fn(current); + if (current.fieldsArray) + current.fieldsArray.forEach(function(field) { + traverse(field, fn); + }); + if (current.oneofsArray) + current.oneofsArray.forEach(function(oneof) { + traverse(oneof, fn); + }); + if (current.methodsArray) + current.methodsArray.forEach(function(method) { + traverse(method, fn); + }); + if (current.nestedArray) + current.nestedArray.forEach(function(nested) { + traverse(nested, fn); + }); +}; + +exports.traverseResolved = function traverseResolved(current, fn) { + fn(current); + if (current.resolvedType) + traverseResolved(current.resolvedType, fn); + if (current.resolvedKeyType) + traverseResolved(current.resolvedKeyType, fn); + if (current.resolvedRequestType) + traverseResolved(current.resolvedRequestType, fn); + if (current.resolvedResponseType) + traverseResolved(current.resolvedResponseType, fn); +}; + exports.inspect = function inspect(object, indent) { if (!object) return ""; diff --git a/google/api/annotations.json b/google/api/annotations.json new file mode 100644 index 000000000..3f13a7338 --- /dev/null +++ b/google/api/annotations.json @@ -0,0 +1,83 @@ +{ + "nested": { + "google": { + "nested": { + "api": { + "nested": { + "http": { + "type": "HttpRule", + "id": 72295728, + "extend": "google.protobuf.MethodOptions" + }, + "HttpRule": { + "oneofs": { + "pattern": { + "oneof": [ + "get", + "put", + "post", + "delete", + "patch", + "custom" + ] + } + }, + "fields": { + "get": { + "type": "string", + "id": 2 + }, + "put": { + "type": "string", + "id": 3 + }, + "post": { + "type": "string", + "id": 4 + }, + "delete": { + "type": "string", + "id": 5 + }, + "patch": { + "type": "string", + "id": 6 + }, + "custom": { + "type": "CustomHttpPattern", + "id": 8 + }, + "selector": { + "type": "string", + "id": 1 + }, + "body": { + "type": "string", + "id": 7 + }, + "additionalBindings": { + "rule": "repeated", + "type": "HttpRule", + "id": 11 + } + } + } + } + }, + "protobuf": { + "nested": { + "MethodOptions": { + "fields": {}, + "extensions": [ + [ + 1000, + 536870911 + ] + ] + } + } + } + } + } + } +} \ No newline at end of file diff --git a/google/api/annotations.proto b/google/api/annotations.proto new file mode 100644 index 000000000..63a8eefda --- /dev/null +++ b/google/api/annotations.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package google.api; + +import "google/api/http.proto"; +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.MethodOptions { + + HttpRule http = 72295728; +} \ No newline at end of file diff --git a/google/api/http.json b/google/api/http.json new file mode 100644 index 000000000..e3a0f4f90 --- /dev/null +++ b/google/api/http.json @@ -0,0 +1,86 @@ +{ + "nested": { + "google": { + "nested": { + "api": { + "nested": { + "Http": { + "fields": { + "rules": { + "rule": "repeated", + "type": "HttpRule", + "id": 1 + } + } + }, + "HttpRule": { + "oneofs": { + "pattern": { + "oneof": [ + "get", + "put", + "post", + "delete", + "patch", + "custom" + ] + } + }, + "fields": { + "get": { + "type": "string", + "id": 2 + }, + "put": { + "type": "string", + "id": 3 + }, + "post": { + "type": "string", + "id": 4 + }, + "delete": { + "type": "string", + "id": 5 + }, + "patch": { + "type": "string", + "id": 6 + }, + "custom": { + "type": "CustomHttpPattern", + "id": 8 + }, + "selector": { + "type": "string", + "id": 1 + }, + "body": { + "type": "string", + "id": 7 + }, + "additionalBindings": { + "rule": "repeated", + "type": "HttpRule", + "id": 11 + } + } + }, + "CustomHttpPattern": { + "fields": { + "kind": { + "type": "string", + "id": 1 + }, + "path": { + "type": "string", + "id": 2 + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/google/api/http.proto b/google/api/http.proto new file mode 100644 index 000000000..e9a7e9de5 --- /dev/null +++ b/google/api/http.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package google.api; + +message Http { + + repeated HttpRule rules = 1; +} + +message HttpRule { + + oneof pattern { + + string get = 2; + string put = 3; + string post = 4; + string delete = 5; + string patch = 6; + CustomHttpPattern custom = 8; + } + + string selector = 1; + string body = 7; + repeated HttpRule additional_bindings = 11; +} + +message CustomHttpPattern { + + string kind = 1; + string path = 2; +} \ No newline at end of file diff --git a/google/protobuf/api.json b/google/protobuf/api.json new file mode 100644 index 000000000..5460612f4 --- /dev/null +++ b/google/protobuf/api.json @@ -0,0 +1,118 @@ +{ + "nested": { + "google": { + "nested": { + "protobuf": { + "nested": { + "Api": { + "fields": { + "name": { + "type": "string", + "id": 1 + }, + "methods": { + "rule": "repeated", + "type": "Method", + "id": 2 + }, + "options": { + "rule": "repeated", + "type": "Option", + "id": 3 + }, + "version": { + "type": "string", + "id": 4 + }, + "sourceContext": { + "type": "SourceContext", + "id": 5 + }, + "mixins": { + "rule": "repeated", + "type": "Mixin", + "id": 6 + }, + "syntax": { + "type": "Syntax", + "id": 7 + } + } + }, + "Method": { + "fields": { + "name": { + "type": "string", + "id": 1 + }, + "requestTypeUrl": { + "type": "string", + "id": 2 + }, + "requestStreaming": { + "type": "bool", + "id": 3 + }, + "responseTypeUrl": { + "type": "string", + "id": 4 + }, + "responseStreaming": { + "type": "bool", + "id": 5 + }, + "options": { + "rule": "repeated", + "type": "Option", + "id": 6 + }, + "syntax": { + "type": "Syntax", + "id": 7 + } + } + }, + "Mixin": { + "fields": { + "name": { + "type": "string", + "id": 1 + }, + "root": { + "type": "string", + "id": 2 + } + } + }, + "SourceContext": { + "fields": { + "fileName": { + "type": "string", + "id": 1 + } + } + }, + "Option": { + "fields": { + "name": { + "type": "string", + "id": 1 + }, + "value": { + "type": "Any", + "id": 2 + } + } + }, + "Syntax": { + "values": { + "SYNTAX_PROTO2": 0, + "SYNTAX_PROTO3": 1 + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/google/protobuf/api.proto b/google/protobuf/api.proto new file mode 100644 index 000000000..cf6ae3f33 --- /dev/null +++ b/google/protobuf/api.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/source_context.proto"; +import "google/protobuf/type.proto"; + +message Api { + + string name = 1; + repeated Method methods = 2; + repeated Option options = 3; + string version = 4; + SourceContext source_context = 5; + repeated Mixin mixins = 6; + Syntax syntax = 7; +} + +message Method { + + string name = 1; + string request_type_url = 2; + bool request_streaming = 3; + string response_type_url = 4; + bool response_streaming = 5; + repeated Option options = 6; + Syntax syntax = 7; +} + +message Mixin { + + string name = 1; + string root = 2; +} \ No newline at end of file diff --git a/google/protobuf/type.json b/google/protobuf/type.json index e18ea703a..fffa70d98 100644 --- a/google/protobuf/type.json +++ b/google/protobuf/type.json @@ -185,6 +185,14 @@ "id": 2 } } + }, + "SourceContext": { + "fields": { + "fileName": { + "type": "string", + "id": 1 + } + } } } } diff --git a/package.json b/package.json index 95bdfd750..b442bfc16 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,6 @@ "browserify-wrap": "^1.0.2", "bundle-collapser": "^1.2.1", "chalk": "^1.1.3", - "coveralls": "^2.11.15", "escodegen": "^1.8.1", "eslint": "^3.14.1", "espree": "^3.1.3", diff --git a/scripts/gencommons.js b/scripts/gencommons.js new file mode 100644 index 000000000..79e3faaa2 --- /dev/null +++ b/scripts/gencommons.js @@ -0,0 +1,26 @@ +"use strict"; +var pbjs = require("../cli/pbjs"); + +[ + "google/protobuf/api.proto", + "google/protobuf/descriptor.proto", + "google/protobuf/field_mask.proto", + "google/protobuf/source_context.proto", + "google/protobuf/type.proto", + + "google/api/annotations.proto", + "google/api/http.proto" +] +.forEach(function(file) { + var out = file.replace(/\.proto$/, ".json"); + pbjs.main([ + "--target", "json", + "--sparse", + "--out", out, + file + ], function(err) { + if (err) + throw err; + process.stdout.write("pbjs: " + file + " -> " + out + "\n"); + }); +}); \ No newline at end of file diff --git a/src/field.js b/src/field.js index 5539dd75b..abbc04c6d 100644 --- a/src/field.js +++ b/src/field.js @@ -221,12 +221,13 @@ Field.prototype.resolve = function resolve() { if (!Type) Type = require("./type"); - if (this.resolvedType = this.parent.lookup(this.type, Type)) + var scope = this.declaringField ? this.declaringField.parent : this.parent; + if (this.resolvedType = scope.lookup(this.type, Type)) this.typeDefault = null; - else if (this.resolvedType = this.parent.lookup(this.type, Enum)) + else if (this.resolvedType = scope.lookup(this.type, Enum)) this.typeDefault = this.resolvedType.values[Object.keys(this.resolvedType.values)[0]]; // first defined else - throw Error("unresolvable field type: " + this.type); + throw Error("unresolvable field type: " + this.type + " in " + scope); } // use explicitly set default value if present diff --git a/src/object.js b/src/object.js index 2414fe3bf..82869974b 100644 --- a/src/object.js +++ b/src/object.js @@ -52,6 +52,12 @@ function ReflectionObject(name, options) { * @type {?string} */ this.comment = null; + + /** + * Defining file name. + * @type {?string} + */ + this.filename = null; } Object.defineProperties(ReflectionObject.prototype, { diff --git a/src/parse.js b/src/parse.js index 2c0042870..4ec497229 100644 --- a/src/parse.js +++ b/src/parse.js @@ -94,9 +94,10 @@ function parse(source, root, options) { var applyCase = options.keepCase ? function(name) { return name; } : camelCase; /* istanbul ignore next */ - function illegal(token, name) { + function illegal(token, name, insideTryCatch) { var filename = parse.filename; - parse.filename = null; + if (!insideTryCatch) + parse.filename = null; return Error("illegal " + (name || "token") + " '" + token + "' (" + (filename ? filename + ", " : "") + "line " + tn.line() + ")"); } @@ -127,7 +128,7 @@ function parse(source, root, options) { return false; } try { - return parseNumber(token); + return parseNumber(token, /* insideTryCatch */ true); } catch (e) { /* istanbul ignore else */ if (acceptTypeRef && isTypeRef(token)) @@ -146,7 +147,7 @@ function parse(source, root, options) { return [ start, end ]; } - function parseNumber(token) { + function parseNumber(token, insideTryCatch) { var sign = 1; if (token.charAt(0) === "-") { sign = -1; @@ -167,7 +168,7 @@ function parse(source, root, options) { if (/^(?!e)[0-9]*(?:\.[0-9]*)?(?:[e][+-]?[0-9]+)?$/.test(tokenLower)) return sign * parseFloat(token); /* istanbul ignore next */ - throw illegal(token, "number"); + throw illegal(token, "number", insideTryCatch); } function parseId(token, acceptNegative) { @@ -266,6 +267,7 @@ function parse(source, root, options) { throw illegal(name, "type name"); var type = new Type(name); type.comment = cmnt(); + type.filename = parse.filename; if (skip("{", true)) { while ((token = next()) !== "}") { var tokenLower = lower(token); @@ -328,6 +330,7 @@ function parse(source, root, options) { var field = new Field(name, parseId(next()), type, rule, extend), trailingLine = tn.line(); field.comment = cmnt(); + field.filename = parse.filename; parseInlineOptions(field); if (!field.comment) field.comment = cmnt(trailingLine); @@ -352,6 +355,7 @@ function parse(source, root, options) { type.group = true; type.comment = cmnt(); var field = new Field(fieldName, id, name, rule); + type.filename = field.filename = parse.filename; skip("{"); while ((token = next()) !== "}") { switch (token = lower(token)) { @@ -397,6 +401,7 @@ function parse(source, root, options) { var field = new MapField(name, parseId(next()), keyType, valueType), trailingLine = tn.line(); field.comment = cmnt(); + field.filename = parse.filename; parseInlineOptions(field); if (!field.comment) field.comment = cmnt(trailingLine); @@ -414,6 +419,7 @@ function parse(source, root, options) { var oneof = new OneOf(name), trailingLine = tn.line(); oneof.comment = cmnt(); + oneof.filename = parse.filename; if (skip("{", true)) { while ((token = next()) !== "}") { if (token === "option") { @@ -442,6 +448,7 @@ function parse(source, root, options) { var enm = new Enum(name); enm.comment = cmnt(); + enm.filename = parse.filename; if (skip("{", true)) { while ((token = next()) !== "}") { if (lower(token) === "option") { @@ -537,6 +544,7 @@ function parse(source, root, options) { var name = token; var service = new Service(name); service.comment = cmnt(); + service.filename = parse.filename; if (skip("{", true)) { while ((token = next()) !== "}") { var tokenLower = lower(token); @@ -588,6 +596,7 @@ function parse(source, root, options) { var method = new Method(name, type, requestType, responseType, requestStream, responseStream), trailingLine = tn.line(); method.comment = cmnt(); + method.filename = parse.filename; if (skip("{", true)) { while ((token = next()) !== "}") { var tokenLower = lower(token);