Merge pull request #1095 from zeitgeist87/FixGreedy

Fix corner cases for the greedy flag

components/prism-core.js

@@ -255,26 +255,18 @@ var _ = _self.Prism = {
 		return Token.stringify(_.util.encode(tokens), language);
 	},
 
-	tokenize: function(text, grammar, language) {
+	matchGrammar: function (text, strarr, grammar, index, startPos, oneshot, target) {
 		var Token = _.Token;
 
-		var strarr = [text];
-
-		var rest = grammar.rest;
-
-		if (rest) {
-			for (var token in rest) {
-				grammar[token] = rest[token];
-			}
-
-			delete grammar.rest;
-		}
-
-		tokenloop: for (var token in grammar) {
+		for (var token in grammar) {
 			if(!grammar.hasOwnProperty(token) || !grammar[token]) {
 				continue;
 			}
 
+			if (token == target) {
+				return;
+			}
+
 			var patterns = grammar[token];
 			patterns = (_.util.type(patterns) === "Array") ? patterns : [patterns];
 
@@ -295,13 +287,13 @@ var _ = _self.Prism = {
 				pattern = pattern.pattern || pattern;
 
 				// Don’t cache length as it changes during the loop
-				for (var i=0, pos = 0; i<strarr.length; pos += strarr[i].length, ++i) {
+				for (var i = index, pos = startPos; i < strarr.length; pos += strarr[i].length, ++i) {
 
 					var str = strarr[i];
 
 					if (strarr.length > text.length) {
 						// Something went terribly wrong, ABORT, ABORT!
-						break tokenloop;
+						return;
 					}
 
 					if (str instanceof Token) {
@@ -326,7 +318,7 @@ var _ = _self.Prism = {
 						    k = i,
 						    p = pos;
 
-						for (var len = strarr.length; k < len && p < to; ++k) {
+						for (var len = strarr.length; k < len && (p < to || (!strarr[k].type && !strarr[k - 1].greedy)); ++k) {
 							p += strarr[k].length;
 							// Move the index i to the element in strarr that is closest to from
 							if (from >= p) {
@@ -350,6 +342,10 @@ var _ = _self.Prism = {
 					}
 
 					if (!match) {
+						if (oneshot) {
+							break;
+						}
+
 						continue;
 					}
 
@@ -366,6 +362,8 @@ var _ = _self.Prism = {
 					var args = [i, delNum];
 
 					if (before) {
+						++i;
+						pos += before.length;
 						args.push(before);
 					}
 
@@ -378,9 +376,31 @@ var _ = _self.Prism = {
 					}
 
 					Array.prototype.splice.apply(strarr, args);
+
+					if (delNum != 1)
+						_.matchGrammar(text, strarr, grammar, i, pos, true, token);
+
+					if (oneshot)
+						break;
 				}
 			}
 		}
+	},
+
+	tokenize: function(text, grammar, language) {
+		var strarr = [text];
+
+		var rest = grammar.rest;
+
+		if (rest) {
+			for (var token in rest) {
+				grammar[token] = rest[token];
+			}
+
+			delete grammar.rest;
+		}
+
+		_.matchGrammar(text, strarr, grammar, 0, 0, false);
 
 		return strarr;
 	},

prism.js

@@ -260,26 +260,18 @@ var _ = _self.Prism = {
 		return Token.stringify(_.util.encode(tokens), language);
 	},
 
-	tokenize: function(text, grammar, language) {
+	matchGrammar: function (text, strarr, grammar, index, startPos, oneshot, target) {
 		var Token = _.Token;
 
-		var strarr = [text];
-
-		var rest = grammar.rest;
-
-		if (rest) {
-			for (var token in rest) {
-				grammar[token] = rest[token];
-			}
-
-			delete grammar.rest;
-		}
-
-		tokenloop: for (var token in grammar) {
+		for (var token in grammar) {
 			if(!grammar.hasOwnProperty(token) || !grammar[token]) {
 				continue;
 			}
 
+			if (token == target) {
+				return;
+			}
+
 			var patterns = grammar[token];
 			patterns = (_.util.type(patterns) === "Array") ? patterns : [patterns];
 
@@ -300,13 +292,13 @@ var _ = _self.Prism = {
 				pattern = pattern.pattern || pattern;
 
 				// Don’t cache length as it changes during the loop
-				for (var i=0, pos = 0; i<strarr.length; pos += strarr[i].length, ++i) {
+				for (var i = index, pos = startPos; i < strarr.length; pos += strarr[i].length, ++i) {
 
 					var str = strarr[i];
 
 					if (strarr.length > text.length) {
 						// Something went terribly wrong, ABORT, ABORT!
-						break tokenloop;
+						return;
 					}
 
 					if (str instanceof Token) {
@@ -331,7 +323,7 @@ var _ = _self.Prism = {
 						    k = i,
 						    p = pos;
 
-						for (var len = strarr.length; k < len && p < to; ++k) {
+						for (var len = strarr.length; k < len && (p < to || (!strarr[k].type && !strarr[k - 1].greedy)); ++k) {
 							p += strarr[k].length;
 							// Move the index i to the element in strarr that is closest to from
 							if (from >= p) {
@@ -355,6 +347,10 @@ var _ = _self.Prism = {
 					}
 
 					if (!match) {
+						if (oneshot) {
+							break;
+						}
+
 						continue;
 					}
 
@@ -371,6 +367,8 @@ var _ = _self.Prism = {
 					var args = [i, delNum];
 
 					if (before) {
+						++i;
+						pos += before.length;
 						args.push(before);
 					}
 
@@ -383,9 +381,31 @@ var _ = _self.Prism = {
 					}
 
 					Array.prototype.splice.apply(strarr, args);
+
+					if (delNum != 1)
+						_.matchGrammar(text, strarr, grammar, i, pos, true, token);
+
+					if (oneshot)
+						break;
 				}
 			}
 		}
+	},
+
+	tokenize: function(text, grammar, language) {
+		var strarr = [text];
+
+		var rest = grammar.rest;
+
+		if (rest) {
+			for (var token in rest) {
+				grammar[token] = rest[token];
+			}
+
+			delete grammar.rest;
+		}
+
+		_.matchGrammar(text, strarr, grammar, 0, 0, false);
 
 		return strarr;
 	},

tests/languages/clike/string_feature.test

@@ -8,6 +8,7 @@ bar"
 bar'
 "foo /* comment */ bar"
 'foo // bar'
+'foo // bar' //comment
 
 ----------------------------------------------------
 
@@ -19,7 +20,9 @@ bar'
 	["string", "\"foo\\\r\nbar\""],
 	["string", "'foo\\\r\nbar'"],
 	["string", "\"foo /* comment */ bar\""],
-	["string", "'foo // bar'"]
+	["string", "'foo // bar'"],
+	["string", "'foo // bar'"],
+	["comment", "//comment"]
 ]
 
 ----------------------------------------------------

tests/languages/css/string_feature.test

@@ -4,14 +4,17 @@
 bar"
 'foo\
 bar'
+"foo /* bar" /* and out */
 
 ----------------------------------------------------
 
 [
 	["string", "\"f\\\"oo\""],
 	["string", "'f\\'oo'"],
 	["string", "\"foo\\\r\nbar\""],
-    ["string", "'foo\\\r\nbar'"]
+    ["string", "'foo\\\r\nbar'"],
+    ["string", "\"foo /* bar\""],
+    ["comment", "/* and out */"]
 ]
 
 ----------------------------------------------------

tests/languages/javascript/template-string_feature.test

@@ -4,6 +4,7 @@ bar`
 `40+2=${40+2}`
 `${foo()}`
 "foo `a` `b` `c` `d` bar"
+"test // test" `template`
 
 ----------------------------------------------------
 
@@ -36,7 +37,11 @@ bar`
         ]],
         ["string", "`"]
     ]],
-    ["string", "\"foo `a` `b` `c` `d` bar\""]
+    ["string", "\"foo `a` `b` `c` `d` bar\""],
+    ["string", "\"test // test\""],
+    ["template-string", [
+        ["string", "`template`"]
+    ]]
 ]
 
 ----------------------------------------------------

tests/languages/lolcode/string_feature.test

@@ -3,6 +3,7 @@
 "fo:"o"
 "foo:)bar:>baz"
 "foo:{bar}baz"
+"foo BTW bar" BTW and out
 
 ----------------------------------------------------
 
@@ -25,7 +26,9 @@
 		"\"foo",
 		["variable", ":{bar}"],
 		"baz\""
-	]]
+	]],
+	["string", ["\"foo BTW bar\""]],
+	["comment", "BTW and out"]
 ]
 
 ----------------------------------------------------