diff --git a/components/prism-core.js b/components/prism-core.js index 9f82910a8f..8b8343b485 100644 --- a/components/prism-core.js +++ b/components/prism-core.js @@ -890,7 +890,7 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, rematch) { var removeCount = 1; // this is the to parameter of removeBetween - if (greedy && currentNode != tokenList.tail.prev) { + if (greedy) { pattern.lastIndex = pos; var match = pattern.exec(text); if (!match) { diff --git a/components/prism-core.min.js b/components/prism-core.min.js index 267167ed8b..1589f00289 100644 --- a/components/prism-core.min.js +++ b/components/prism-core.min.js @@ -1 +1 @@ -var _self="undefined"!=typeof window?window:"undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?self:{},Prism=function(u){var c=/\blang(?:uage)?-([\w-]+)\b/i,n=0,M={manual:u.Prism&&u.Prism.manual,disableWorkerMessageHandler:u.Prism&&u.Prism.disableWorkerMessageHandler,util:{encode:function e(n){return n instanceof W?new W(n.type,e(n.content),n.alias):Array.isArray(n)?n.map(e):n.replace(/&/g,"&").replace(/=l.reach);k+=y.value.length,y=y.next){var b=y.value;if(t.length>n.length)return;if(!(b instanceof W)){var x=1;if(h&&y!=t.tail.prev){m.lastIndex=k;var w=m.exec(n);if(!w)break;var A=w.index+(f&&w[1]?w[1].length:0),P=w.index+w[0].length,S=k;for(S+=y.value.length;S<=A;)y=y.next,S+=y.value.length;if(S-=y.value.length,k=S,y.value instanceof W)continue;for(var E=y;E!==t.tail&&(Sl.reach&&(l.reach=j);var C=y.prev;L&&(C=I(t,C,L),k+=L.length),z(t,C,x);var _=new W(o,g?M.tokenize(O,g):O,v,O);y=I(t,C,_),N&&I(t,y,N),1"+a.content+""},!u.document)return u.addEventListener&&(M.disableWorkerMessageHandler||u.addEventListener("message",function(e){var n=JSON.parse(e.data),t=n.language,r=n.code,a=n.immediateClose;u.postMessage(M.highlight(r,M.languages[t],t)),a&&u.close()},!1)),M;var e=M.util.currentScript();function t(){M.manual||M.highlightAll()}if(e&&(M.filename=e.src,e.hasAttribute("data-manual")&&(M.manual=!0)),!M.manual){var r=document.readyState;"loading"===r||"interactive"===r&&e&&e.defer?document.addEventListener("DOMContentLoaded",t):window.requestAnimationFrame?window.requestAnimationFrame(t):window.setTimeout(t,16)}return M}(_self);"undefined"!=typeof module&&module.exports&&(module.exports=Prism),"undefined"!=typeof global&&(global.Prism=Prism); \ No newline at end of file +var _self="undefined"!=typeof window?window:"undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?self:{},Prism=function(u){var c=/\blang(?:uage)?-([\w-]+)\b/i,n=0,M={manual:u.Prism&&u.Prism.manual,disableWorkerMessageHandler:u.Prism&&u.Prism.disableWorkerMessageHandler,util:{encode:function e(n){return n instanceof W?new W(n.type,e(n.content),n.alias):Array.isArray(n)?n.map(e):n.replace(/&/g,"&").replace(/=l.reach);k+=y.value.length,y=y.next){var b=y.value;if(t.length>n.length)return;if(!(b instanceof W)){var x=1;if(h){m.lastIndex=k;var w=m.exec(n);if(!w)break;var A=w.index+(f&&w[1]?w[1].length:0),P=w.index+w[0].length,S=k;for(S+=y.value.length;S<=A;)y=y.next,S+=y.value.length;if(S-=y.value.length,k=S,y.value instanceof W)continue;for(var E=y;E!==t.tail&&(Sl.reach&&(l.reach=j);var C=y.prev;L&&(C=I(t,C,L),k+=L.length),z(t,C,x);var _=new W(o,g?M.tokenize(O,g):O,v,O);y=I(t,C,_),N&&I(t,y,N),1"+a.content+""},!u.document)return u.addEventListener&&(M.disableWorkerMessageHandler||u.addEventListener("message",function(e){var n=JSON.parse(e.data),t=n.language,r=n.code,a=n.immediateClose;u.postMessage(M.highlight(r,M.languages[t],t)),a&&u.close()},!1)),M;var e=M.util.currentScript();function t(){M.manual||M.highlightAll()}if(e&&(M.filename=e.src,e.hasAttribute("data-manual")&&(M.manual=!0)),!M.manual){var r=document.readyState;"loading"===r||"interactive"===r&&e&&e.defer?document.addEventListener("DOMContentLoaded",t):window.requestAnimationFrame?window.requestAnimationFrame(t):window.setTimeout(t,16)}return M}(_self);"undefined"!=typeof module&&module.exports&&(module.exports=Prism),"undefined"!=typeof global&&(global.Prism=Prism); \ No newline at end of file diff --git a/docs/prism-core.js.html b/docs/prism-core.js.html index fed9633629..f8a5edcd1e 100644 --- a/docs/prism-core.js.html +++ b/docs/prism-core.js.html @@ -943,7 +943,7 @@

prism-core.js

var removeCount = 1; // this is the to parameter of removeBetween - if (greedy && currentNode != tokenList.tail.prev) { + if (greedy) { pattern.lastIndex = pos; var match = pattern.exec(text); if (!match) { diff --git a/prism.js b/prism.js index c25249c8e8..02af74b4c8 100644 --- a/prism.js +++ b/prism.js @@ -895,7 +895,7 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, rematch) { var removeCount = 1; // this is the to parameter of removeBetween - if (greedy && currentNode != tokenList.tail.prev) { + if (greedy) { pattern.lastIndex = pos; var match = pattern.exec(text); if (!match) { diff --git a/tests/core/greedy.js b/tests/core/greedy.js index f21f87d1e3..d7fc9fbcc7 100644 --- a/tests/core/greedy.js +++ b/tests/core/greedy.js @@ -84,4 +84,25 @@ describe('Greedy matching', function () { ] }); }); + + it('should always match tokens against the whole text', function () { + // this is to test for a bug where greedy tokens where matched like non-greedy ones if the token stream ended on + // a string + testTokens({ + grammar: { + 'a': /a/, + 'b': { + pattern: /^b/, + greedy: true + } + }, + code: 'bab', + expected: [ + ["b", "b"], + ["a", "a"], + "b" + ] + }); + }); + });