Merge pull request #270 from JuliaEditorSupport/sp/fix-raw-var-escape

fix: handle escaped chars in raw/var string macros
JuliaEditorSupport · Dec 12, 2023 · 5be44fc · 5be44fc
2 parents 9e79abe + 8747e06
commit 5be44fc
Show file tree

Hide file tree

Showing 4 changed files with 123 additions and 9 deletions.
diff --git a/grammars/julia.cson b/grammars/julia.cson
@@ -542,6 +542,11 @@ repository:
         endCaptures:
           '0':
             name: 'punctuation.definition.string.end.julia'
+        patterns: [
+          {
+            include: '#string_escaped_char'
+          }
+        ]
       }
       {
         begin: '(raw)(")'
@@ -555,6 +560,11 @@ repository:
         endCaptures:
           '0':
             name: 'punctuation.definition.string.end.julia'
+        patterns: [
+          {
+            include: '#string_escaped_char'
+          }
+        ]
       }
       {
         begin: '(sql)(""")'
@@ -582,11 +592,21 @@ repository:
         begin: 'var"""'
         end: '"""'
         name: 'constant.other.symbol.julia'
+        patterns: [
+          {
+            include: '#string_escaped_char'
+          }
+        ]
       }
       {
         begin: 'var"'
         end: '"'
         name: 'constant.other.symbol.julia'
+        patterns: [
+          {
+            include: '#string_escaped_char'
+          }
+        ]
       }
       {
         begin: '^\\s?(doc)?(""")\\s?$'

diff --git a/grammars/julia.json b/grammars/julia.json
@@ -114,7 +114,7 @@
           "begin": "\\{",
           "beginCaptures": {
             "0": {
-          "name": "meta.bracket.julia"
+              "name": "meta.bracket.julia"
             }
           },
           "end": "(\\})((?:\\.)?'*)",
@@ -622,7 +622,12 @@
             "0": {
               "name": "punctuation.definition.string.end.julia"
             }
-          }
+          },
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "(raw)(\")",
@@ -640,7 +645,12 @@
             "0": {
               "name": "punctuation.definition.string.end.julia"
             }
-          }
+          },
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "(sql)(\"\"\")",
@@ -672,12 +682,22 @@
         {
           "begin": "var\"\"\"",
           "end": "\"\"\"",
-          "name": "constant.other.symbol.julia"
+          "name": "constant.other.symbol.julia",
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "var\"",
           "end": "\"",
-          "name": "constant.other.symbol.julia"
+          "name": "constant.other.symbol.julia",
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "^\\s?(doc)?(\"\"\")\\s?$",

diff --git a/grammars/julia_vscode.json b/grammars/julia_vscode.json
@@ -622,7 +622,12 @@
             "0": {
               "name": "punctuation.definition.string.end.julia"
             }
-          }
+          },
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "(raw)(\")",
@@ -640,7 +645,12 @@
             "0": {
               "name": "punctuation.definition.string.end.julia"
             }
-          }
+          },
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "(sql)(\"\"\")",
@@ -672,12 +682,22 @@
         {
           "begin": "var\"\"\"",
           "end": "\"\"\"",
-          "name": "constant.other.symbol.julia"
+          "name": "constant.other.symbol.julia",
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "var\"",
           "end": "\"",
-          "name": "constant.other.symbol.julia"
+          "name": "constant.other.symbol.julia",
+          "patterns": [
+            {
+              "include": "#string_escaped_char"
+            }
+          ]
         },
         {
           "begin": "^\\s?(doc)?(\"\"\")\\s?$",

diff --git a/test/test.js b/test/test.js
@@ -3484,4 +3484,58 @@ describe('Julia grammar', function () {
             },
         ])
     })
+    it("tokenizes escape codes in raw strings", function () {
+        const tokens = tokenize(grammar, 'raw"a\\"b"')
+        compareTokens(tokens, [
+            {
+                value: 'raw',
+                scopes: ["string.quoted.other.julia", "support.function.macro.julia"]
+            },
+            {
+                value: '"',
+                scopes: ["string.quoted.other.julia", "punctuation.definition.string.begin.julia"]
+            },
+            {
+                value: 'a',
+                scopes: ["string.quoted.other.julia"]
+            },
+            {
+                value: '\\"',
+                scopes: ["string.quoted.other.julia", "constant.character.escape.julia"]
+            },
+            {
+                value: 'b',
+                scopes: ["string.quoted.other.julia"]
+            },
+            {
+                value: '"',
+                scopes: ["string.quoted.other.julia", "punctuation.definition.string.end.julia"]
+            },
+        ])
+    })
+    it("tokenizes escape codes in var strings", function () {
+        const tokens = tokenize(grammar, 'var"a\\"b"')
+        compareTokens(tokens, [
+            {
+                value: 'var"',
+                scopes: ["constant.other.symbol.julia"]
+            },
+            {
+                value: 'a',
+                scopes: ["constant.other.symbol.julia"]
+            },
+            {
+                value: '\\"',
+                scopes: ["constant.other.symbol.julia", "constant.character.escape.julia"]
+            },
+            {
+                value: 'b',
+                scopes: ["constant.other.symbol.julia"]
+            },
+            {
+                value: '"',
+                scopes: ["constant.other.symbol.julia"]
+            },
+        ])
+    })
 })