From 8747e069119b24bff2703374d7961e3446a74034 Mon Sep 17 00:00:00 2001
From: Sebastian Pfitzner <pfitzseb@gmail.com>
Date: Tue, 12 Dec 2023 14:32:35 +0100
Subject: [PATCH] fix: handle escaped chars in raw/var string macros

---
 grammars/julia.cson        | 20 ++++++++++++++
 grammars/julia.json        | 30 +++++++++++++++++----
 grammars/julia_vscode.json | 28 +++++++++++++++++---
 test/test.js               | 54 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 123 insertions(+), 9 deletions(-)

diff --git a/grammars/julia.cson b/grammars/julia.cson
index b29075b..d4398b5 100644
--- a/grammars/julia.cson
+++ b/grammars/julia.cson
@@ -542,6 +542,11 @@ repository:
         endCaptures:
           '0':
             name: 'punctuation.definition.string.end.julia'
+        patterns: [
+          {
+            include: '#string_escaped_char'
+          }
+        ]
       }
       {
         begin: '(raw)(")'
@@ -555,6 +560,11 @@ repository:
         endCaptures:
           '0':
             name: 'punctuation.definition.string.end.julia'
+        patterns: [
+          {
+            include: '#string_escaped_char'
+          }
+        ]
       }
       {
         begin: '(sql)(""")'
@@ -582,11 +592,21 @@ repository:
         begin: 'var"""'
         end: '"""'
         name: 'constant.other.symbol.julia'
+        patterns: [
+          {
+            include: '#string_escaped_char'
+          }
+        ]
       }
       {
         begin: 'var"'
         end: '"'
         name: 'constant.other.symbol.julia'
+        patterns: [
+          {
+            include: '#string_escaped_char'
+          }
+        ]
       }
       {
         begin: '^\\s?(doc)?(""")\\s?$'
diff --git a/grammars/julia.json b/grammars/julia.json
index e00ecb5..948fef0 100644
--- a/grammars/julia.json
+++ b/grammars/julia.json
@@ -114,7 +114,7 @@
           "begin": "\\{",
           "beginCaptures": {
             "0": {
-          "name": "meta.bracket.julia"
+              "name": "meta.bracket.julia"
             }
           },
           "end": "(\\})((?:\\.)?'*)",
@@ -622,7 +622,12 @@
             "0": {
               "name": "punctuation.definition.string.end.julia"
             }
-          }
+          },
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "(raw)(\")",
@@ -640,7 +645,12 @@
             "0": {
               "name": "punctuation.definition.string.end.julia"
             }
-          }
+          },
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "(sql)(\"\"\")",
@@ -672,12 +682,22 @@
         {
           "begin": "var\"\"\"",
           "end": "\"\"\"",
-          "name": "constant.other.symbol.julia"
+          "name": "constant.other.symbol.julia",
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "var\"",
           "end": "\"",
-          "name": "constant.other.symbol.julia"
+          "name": "constant.other.symbol.julia",
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "^\\s?(doc)?(\"\"\")\\s?$",
diff --git a/grammars/julia_vscode.json b/grammars/julia_vscode.json
index 927db06..cf93a3f 100644
--- a/grammars/julia_vscode.json
+++ b/grammars/julia_vscode.json
@@ -622,7 +622,12 @@
             "0": {
               "name": "punctuation.definition.string.end.julia"
             }
-          }
+          },
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "(raw)(\")",
@@ -640,7 +645,12 @@
             "0": {
               "name": "punctuation.definition.string.end.julia"
             }
-          }
+          },
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "(sql)(\"\"\")",
@@ -672,12 +682,22 @@
         {
           "begin": "var\"\"\"",
           "end": "\"\"\"",
-          "name": "constant.other.symbol.julia"
+          "name": "constant.other.symbol.julia",
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "var\"",
           "end": "\"",
-          "name": "constant.other.symbol.julia"
+          "name": "constant.other.symbol.julia",
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "^\\s?(doc)?(\"\"\")\\s?$",
diff --git a/test/test.js b/test/test.js
index 18f6bb5..2677884 100644
--- a/test/test.js
+++ b/test/test.js
@@ -3484,4 +3484,58 @@ describe('Julia grammar', function () {
             },
         ])
     })
+    it("tokenizes escape codes in raw strings", function () {
+        const tokens = tokenize(grammar, 'raw"a\\"b"')
+        compareTokens(tokens, [
+            {
+                value: 'raw',
+                scopes: ["string.quoted.other.julia", "support.function.macro.julia"]
+            },
+            {
+                value: '"',
+                scopes: ["string.quoted.other.julia", "punctuation.definition.string.begin.julia"]
+            },
+            {
+                value: 'a',
+                scopes: ["string.quoted.other.julia"]
+            },
+            {
+                value: '\\"',
+                scopes: ["string.quoted.other.julia", "constant.character.escape.julia"]
+            },
+            {
+                value: 'b',
+                scopes: ["string.quoted.other.julia"]
+            },
+            {
+                value: '"',
+                scopes: ["string.quoted.other.julia", "punctuation.definition.string.end.julia"]
+            },
+        ])
+    })
+    it("tokenizes escape codes in var strings", function () {
+        const tokens = tokenize(grammar, 'var"a\\"b"')
+        compareTokens(tokens, [
+            {
+                value: 'var"',
+                scopes: ["constant.other.symbol.julia"]
+            },
+            {
+                value: 'a',
+                scopes: ["constant.other.symbol.julia"]
+            },
+            {
+                value: '\\"',
+                scopes: ["constant.other.symbol.julia", "constant.character.escape.julia"]
+            },
+            {
+                value: 'b',
+                scopes: ["constant.other.symbol.julia"]
+            },
+            {
+                value: '"',
+                scopes: ["constant.other.symbol.julia"]
+            },
+        ])
+    })
 })