Skip to content

Commit

Permalink
Fixed: Hardened tokenize/parse, esp. comment parsing, see #713
Browse files Browse the repository at this point in the history
  • Loading branch information
@dcodeIO
dcodeIO committed Mar 22, 2017
1 parent 47bb95a commit b01bb58
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 86 deletions.
140 changes: 69 additions & 71 deletions src/parse.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,18 +257,34 @@ function parse(source, root, options) {
return false;
}

function ifBlock(obj, fn) {
var trailingLine = tn.line();
obj.comment = cmnt(); // try block-type comment
obj.filename = parse.filename;
if (skip("{", true)) {
fn();
skip(";", true);
return true;
}
skip(";");
if (typeof obj.comment !== 'string')
obj.comment = cmnt(trailingLine); // try line-type comment if no block
return false;
}

function parseType(parent, token) {
var name = next();

/* istanbul ignore next */
if (!nameRe.test(name))
throw illegal(name, "type name");
var type = new Type(name);
type.comment = cmnt();
type.filename = parse.filename;
if (skip("{", true)) {
if (!nameRe.test(token = next()))
throw illegal(token, "type name");

var type = new Type(token);
ifBlock(type, function() {
while ((token = next()) !== "}") {

if (parseCommon(type, token))
continue;

switch (token) {

case "map":
Expand Down Expand Up @@ -302,9 +318,7 @@ function parse(source, root, options) {
break;
}
}
skip(";", true);
} else
skip(";");
});
parent.add(type);
}

Expand All @@ -314,22 +328,26 @@ function parse(source, root, options) {
parseGroup(parent, rule);
return;
}

/* istanbul ignore next */
if (!typeRefRe.test(type))
throw illegal(type, "type");

var name = next();

/* istanbul ignore next */
if (!nameRe.test(name))
throw illegal(name, "name");

name = applyCase(name);
skip("=");
var field = new Field(name, parseId(next()), type, rule, extend),
trailingLine = tn.line();
field.comment = cmnt();
field.comment = cmnt(); // try block-type
field.filename = parse.filename;
parseInlineOptions(field);
if (!field.comment)
field.comment = cmnt(trailingLine);
field.comment = cmnt(trailingLine); // try line-type
// JSON defaults to packed=true if not set so we have to set packed=false explicity when
// parsing proto2 descriptors without the option, where applicable. This must be done for
// any type (not just packable types) because enums also use varint encoding and it is not
Expand All @@ -341,39 +359,43 @@ function parse(source, root, options) {

function parseGroup(parent, rule) {
var name = next();

/* istanbul ignore next */
if (!nameRe.test(name))
throw illegal(name, "name");

var fieldName = util.lcFirst(name);
if (name === fieldName)
name = util.ucFirst(name);
skip("=");
var id = parseId(next());
var type = new Type(name);
type.group = true;
type.comment = cmnt();
var field = new Field(fieldName, id, name, rule);
type.filename = field.filename = parse.filename;
skip("{");
while ((token = next()) !== "}") {
switch (token) {
case "option":
parseOption(type, token);
skip(";");
break;
case "required":
case "optional":
case "repeated":
parseField(type, token);
break;
field.filename = parse.filename;
ifBlock(type, function() {
while ((token = next()) !== "}") {
switch (token) {

/* istanbul ignore next */
default:
throw illegal(token); // there are no groups with proto3 semantics
case "option":
parseOption(type, token);
skip(";");
break;

case "required":
case "optional":
case "repeated":
parseField(type, token);
break;

/* istanbul ignore next */
default:
throw illegal(token); // there are no groups with proto3 semantics
}
}
}
skip(";", true);
parent.add(type).add(field);
});
parent.add(type)
.add(field);
}

function parseMapField(parent) {
Expand All @@ -398,11 +420,11 @@ function parse(source, root, options) {
skip("=");
var field = new MapField(name, parseId(next()), keyType, valueType),
trailingLine = tn.line();
field.comment = cmnt();
field.comment = cmnt(); // try block-type
field.filename = parse.filename;
parseInlineOptions(field);
if (!field.comment)
field.comment = cmnt(trailingLine);
field.comment = cmnt(trailingLine); // try line-type
parent.add(field);
}

Expand All @@ -414,11 +436,8 @@ function parse(source, root, options) {
throw illegal(name, "name");

name = applyCase(name);
var oneof = new OneOf(name),
trailingLine = tn.line();
oneof.comment = cmnt();
oneof.filename = parse.filename;
if (skip("{", true)) {
var oneof = new OneOf(name);
ifBlock(oneof, function() {
while ((token = next()) !== "}") {
if (token === "option") {
parseOption(oneof, token);
Expand All @@ -428,12 +447,7 @@ function parse(source, root, options) {
parseField(oneof, "optional");
}
}
skip(";", true);
} else {
skip(";");
if (!oneof.comment)
oneof.comment = cmnt(trailingLine);
}
});
parent.add(oneof);
}

Expand All @@ -445,19 +459,15 @@ function parse(source, root, options) {
throw illegal(name, "name");

var enm = new Enum(name);
enm.comment = cmnt();
enm.filename = parse.filename;
if (skip("{", true)) {
ifBlock(enm, function() {
while ((token = next()) !== "}") {
if (token === "option") {
parseOption(enm, token);
skip(";");
} else
parseEnumValue(enm, token);
}
skip(";", true);
} else
skip(";");
});
parent.add(enm);
}

Expand All @@ -471,7 +481,7 @@ function parse(source, root, options) {
skip("=");
var value = parseId(next(), true),
trailingLine = tn.line();
parent.add(name, value, cmnt());
parent.add(name, value, cmnt()); // block-type only
parseInlineOptions({}); // skips enum value options
if (!parent.comments[name])
parent.comments[name] = cmnt(trailingLine);
Expand Down Expand Up @@ -539,11 +549,8 @@ function parse(source, root, options) {
if (!nameRe.test(token))
throw illegal(token, "service name");

var name = token;
var service = new Service(name);
service.comment = cmnt();
service.filename = parse.filename;
if (skip("{", true)) {
var service = new Service(token);
ifBlock(service, function() {
while ((token = next()) !== "}") {
switch (token) {
case "option":
Expand All @@ -559,9 +566,7 @@ function parse(source, root, options) {
throw illegal(token);
}
}
skip(";", true);
} else
skip(";");
});
parent.add(service);
}

Expand Down Expand Up @@ -590,13 +595,11 @@ function parse(source, root, options) {

responseType = token;
skip(")");
var method = new Method(name, type, requestType, responseType, requestStream, responseStream),
trailingLine = tn.line();
method.comment = cmnt();
method.filename = parse.filename;
if (skip("{", true)) {
var method = new Method(name, type, requestType, responseType, requestStream, responseStream);
ifBlock(method, function() {
while ((token = next()) !== "}") {
switch (token) {

case "option":
parseOption(method, token);
skip(";");
Expand All @@ -607,12 +610,7 @@ function parse(source, root, options) {
throw illegal(token);
}
}
skip(";", true);
} else {
skip(";");
if (!method.comment)
method.comment = cmnt(trailingLine);
}
});
parent.add(method);
}

Expand Down
36 changes: 21 additions & 15 deletions src/tokenize.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,26 @@ function tokenize(source) {
return false;
}

/**
* Gets a comment.
* @param {number=} trailingLine Trailing line number if applicable
* @returns {?string} Comment text
* @inner
*/
function cmnt(trailingLine) {
var ret;
if (trailingLine === undefined)
ret = commentLine === line - 1 && commentText || null;
else {
if (!commentText)
peek();
ret = commentLine === trailingLine && commentType === "/" && commentText || null;
}
commentType = commentText = null;
commentLine = 0;
return ret;
}

return {
next: next,
peek: peek,
Expand All @@ -252,21 +272,7 @@ function tokenize(source) {
line: function() {
return line;
},
cmnt: function(trailingLine) {
var ret;
if (trailingLine === undefined)
ret = commentLine === line - 1 && commentText || null;
else {
if (!commentText)
peek();
ret = commentLine === trailingLine && commentType === "/" && commentText || null;
}
if (ret) {
commentType = commentText = null;
commentLine = 0;
}
return ret;
}
cmnt: cmnt
};
/* eslint-enable callback-return */
}

0 comments on commit b01bb58

Please sign in to comment.