Partial solution for the "Comment-like substrings"-problem

This patch introduces a new attribute called `greedy`. The
attribute is a simple boolean flag. If there is no match
for a greedy pattern it can concatenate the next two tokens
into a single string and try to match on this string again.
If a match is found on the second attempt, then the old tokens are
deleted and replaced by the new match.

This solves the "Comment-like substrings"-problem for exactly one
comment at very little cost. With this patch the following code is
highlighted correctly:

"foo /* bar */ baz"; "foo // bar";
/lala"test"sdf/;

This approach fails if there are more than one comments inside the
string:

"foo /* bar */ baz /* bar */ baz";

Signed-off-by: Andreas Rohner <andreas.rohner@gmx.net>

components/prism-clike.js

@@ -9,7 +9,10 @@ Prism.languages.clike = {
 			lookbehind: true
 		}
 	],
-	'string': /(["'])(\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,
+	'string': {
+		pattern: /(["'])(\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,
+		greedy: true
+	},
 	'class-name': {
 		pattern: /((?:\b(?:class|interface|extends|implements|trait|instanceof|new)\s+)|(?:catch\s+\())[a-z0-9_\.\\]+/i,
 		lookbehind: true,

components/prism-core.js

@@ -274,6 +274,7 @@ var _ = _self.Prism = {
 				var pattern = patterns[j],
 					inside = pattern.inside,
 					lookbehind = !!pattern.lookbehind,
+					greedy = !!pattern.greedy,
 					lookbehindLength = 0,
 					alias = pattern.alias;
 
@@ -294,7 +295,38 @@ var _ = _self.Prism = {
 
 					pattern.lastIndex = 0;
 
-					var match = pattern.exec(str);
+					var match = pattern.exec(str),
+					    delNum = 1;
+
+					if (!match && greedy && i != strarr.length - 1) {
+						var nextToken = strarr[i + 1].matchedStr || strarr[i + 1],
+						    combStr = str + nextToken;
+
+						if (i < strarr.length - 2) {
+							combStr += strarr[i + 2].matchedStr || strarr[i + 2];
+						}
+
+						pattern.lastIndex = 0;
+						match = pattern.exec(combStr);
+						if (!match) {
+							continue;
+						}
+
+						var from = match.index + (lookbehind ? match[1].length : 0);
+						if (from >= str.length) {
+							continue;
+						}
+						var to = match.index + match[0].length,
+						    len = str.length + nextToken.length;
+
+						delNum = 3;
+
+						if (to <= len) {
+							delNum = 2;
+							combStr = combStr.slice(0, len);
+						}
+						str = combStr;
+					}
 
 					if (match) {
 						if(lookbehind) {
@@ -308,13 +340,13 @@ var _ = _self.Prism = {
 							before = str.slice(0, from + 1),
 							after = str.slice(to + 1);
 
-						var args = [i, 1];
+						var args = [i, delNum];
 
 						if (before) {
 							args.push(before);
 						}
 
-						var wrapped = new Token(token, inside? _.tokenize(match, inside) : match, alias);
+						var wrapped = new Token(token, inside? _.tokenize(match, inside) : match, alias, match);
 
 						args.push(wrapped);
 
@@ -356,10 +388,11 @@ var _ = _self.Prism = {
 	}
 };
 
-var Token = _.Token = function(type, content, alias) {
+var Token = _.Token = function(type, content, alias, matchedStr) {
 	this.type = type;
 	this.content = content;
 	this.alias = alias;
+	this.matchedStr = matchedStr || null;
 };
 
 Token.stringify = function(o, language, parent) {

components/prism-javascript.js

@@ -8,7 +8,8 @@ Prism.languages.javascript = Prism.languages.extend('clike', {
 Prism.languages.insertBefore('javascript', 'keyword', {
 	'regex': {
 		pattern: /(^|[^/])\/(?!\/)(\[.+?]|\\.|[^/\\\r\n])+\/[gimyu]{0,5}(?=\s*($|[\r\n,.;})]))/,
-		lookbehind: true
+		lookbehind: true,
+		greedy: true
 	}
 });
 

prism.js

@@ -279,6 +279,7 @@ var _ = _self.Prism = {
 				var pattern = patterns[j],
 					inside = pattern.inside,
 					lookbehind = !!pattern.lookbehind,
+					greedy = !!pattern.greedy,
 					lookbehindLength = 0,
 					alias = pattern.alias;
 
@@ -299,7 +300,38 @@ var _ = _self.Prism = {
 
 					pattern.lastIndex = 0;
 
-					var match = pattern.exec(str);
+					var match = pattern.exec(str),
+					    delNum = 1;
+
+					if (!match && greedy && i != strarr.length - 1) {
+						var nextToken = strarr[i + 1].matchedStr || strarr[i + 1],
+						    combStr = str + nextToken;
+
+						if (i < strarr.length - 2) {
+							combStr += strarr[i + 2].matchedStr || strarr[i + 2];
+						}
+
+						pattern.lastIndex = 0;
+						match = pattern.exec(combStr);
+						if (!match) {
+							continue;
+						}
+
+						var from = match.index + (lookbehind ? match[1].length : 0);
+						if (from >= str.length) {
+							continue;
+						}
+						var to = match.index + match[0].length,
+						    len = str.length + nextToken.length;
+
+						delNum = 3;
+
+						if (to <= len) {
+							delNum = 2;
+							combStr = combStr.slice(0, len);
+						}
+						str = combStr;
+					}
 
 					if (match) {
 						if(lookbehind) {
@@ -313,13 +345,13 @@ var _ = _self.Prism = {
 							before = str.slice(0, from + 1),
 							after = str.slice(to + 1);
 
-						var args = [i, 1];
+						var args = [i, delNum];
 
 						if (before) {
 							args.push(before);
 						}
 
-						var wrapped = new Token(token, inside? _.tokenize(match, inside) : match, alias);
+						var wrapped = new Token(token, inside? _.tokenize(match, inside) : match, alias, match);
 
 						args.push(wrapped);
 
@@ -361,10 +393,11 @@ var _ = _self.Prism = {
 	}
 };
 
-var Token = _.Token = function(type, content, alias) {
+var Token = _.Token = function(type, content, alias, matchedStr) {
 	this.type = type;
 	this.content = content;
 	this.alias = alias;
+	this.matchedStr = matchedStr || null;
 };
 
 Token.stringify = function(o, language, parent) {
@@ -575,7 +608,10 @@ Prism.languages.clike = {
 			lookbehind: true
 		}
 	],
-	'string': /(["'])(\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,
+	'string': {
+		pattern: /(["'])(\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,
+		greedy: true
+	},
 	'class-name': {
 		pattern: /((?:\b(?:class|interface|extends|implements|trait|instanceof|new)\s+)|(?:catch\s+\())[a-z0-9_\.\\]+/i,
 		lookbehind: true,
@@ -606,7 +642,8 @@ Prism.languages.javascript = Prism.languages.extend('clike', {
 Prism.languages.insertBefore('javascript', 'keyword', {
 	'regex': {
 		pattern: /(^|[^/])\/(?!\/)(\[.+?]|\\.|[^/\\\r\n])+\/[gimyu]{0,5}(?=\s*($|[\r\n,.;})]))/,
-		lookbehind: true
+		lookbehind: true,
+		greedy: true
 	}
 });