Improvements to Python F-strings and string prefixes (#1642)

This PR adds support for [string interpolation](https://www.python.org/dev/peps/pep-0498/) (aka. f-strings) and makes the [string prefixes](https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals) part of the string.

Resolves #1636.

### Known issues

Assumes that strings inside the interpolation expression are 'nice'. So strings
with unfortunate numbers of curley braces will cause incorrect highlighting: E.g.: `f"{'}'}"`.

components/prism-python.js

@@ -3,13 +3,36 @@ Prism.languages.python = {
 		pattern: /(^|[^\\])#.*/,
 		lookbehind: true
 	},
+	'string-interpolation': {
+		pattern: /(?:f|rf|fr)(?:("""|''')[\s\S]+?\1|("|')(?:\\.|(?!\2)[^\\\r\n])*\2)/i,
+		greedy: true,
+		inside: {
+			'interpolation': {
+				// "{" <expression> <optional "!s", "!r", or "!a"> <optional ":" format specifier> "}"
+				pattern: /((?:^|[^{])(?:{{)*){(?!{)(?:[^{}]|{(?!{)(?:[^{}]|{(?!{)(?:[^{}])+})+})+}/,
+				lookbehind: true,
+				inside: {
+					'format-spec': {
+						pattern: /(:)[^:(){}]+(?=}$)/,
+						lookbehind: true
+					},
+					'conversion-option': {
+						pattern: /![sra](?=[:}]$)/,
+						alias: 'punctuation'
+					},
+					rest: null
+				}
+			},
+			'string': /[\s\S]+/
+		}
+	},
 	'triple-quoted-string': {
-		pattern: /("""|''')[\s\S]+?\1/,
+		pattern: /(?:[rub]|rb|br)?("""|''')[\s\S]+?\1/i,
 		greedy: true,
 		alias: 'string'
 	},
 	'string': {
-		pattern: /("|')(?:\\.|(?!\1)[^\\\r\n])*\1/,
+		pattern: /(?:[rub]|rb|br)?("|')(?:\\.|(?!\1)[^\\\r\n])*\1/i,
 		greedy: true
 	},
 	'function': {
@@ -35,3 +58,5 @@ Prism.languages.python = {
 	'operator': /[-+%=]=?|!=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~]/,
 	'punctuation': /[{}[\];(),.:]/
 };
+
+Prism.languages.python['string-interpolation'].inside['interpolation'].inside.rest = Prism.languages.python;

examples/prism-python.html

@@ -57,8 +57,5 @@ <h2>Known failures</h2>
 	If a failure is listed here, it doesn’t mean it will never be fixed. This is more of a “known bugs” list, just with a certain type of bug.
 </p>
 
-<h3>Triple-quoted strings with what look like strings inside</h3>
-<pre><code>def antique(string):
-    """Replace anachronistic Latin "j" with "i"."""
-    return string.replace("j", "i").replace("J", "I")
-    </code></pre>
+<h3>Interpolation expressions containing strings with <code>{</code> or <code>}</code></h3>
+<pre><code>f"{'}'}"</code></pre>

tests/languages/python/string-interpolation_feature.test

@@ -0,0 +1,147 @@
+f'The value is {value}.'
+
+f"The value is {'4'}."
+
+f'input={value!s:#06x}'
+
+f'{{{4*10}}}'
+
+fr'x={4*10}\n'
+
+f'''{x
++1}'''
+
+f'mapping is { {a:b for (a, b) in ((1, 2), (3, 4))} }'
+
+f'{(lambda x: x*2)(3)}'
+
+----------------------------------------------------
+
+[
+	["string-interpolation", [
+		["string", "f'The value is "],
+		["interpolation", [
+			["punctuation", "{"],
+			"value",
+			["punctuation", "}"]
+		]],
+		["string", ".'"]
+	]],
+
+	["string-interpolation", [
+		["string", "f\"The value is "],
+		["interpolation", [
+			["punctuation", "{"],
+			["string", "'4'"],
+			["punctuation", "}"]
+		]],
+		["string", ".\""]
+	]],
+
+	["string-interpolation", [
+		["string", "f'input="],
+		["interpolation", [
+			["punctuation", "{"],
+			"value",
+			["conversion-option", "!s"],
+			["punctuation", ":"],
+			["format-spec", "#06x"],
+			["punctuation", "}"]
+		]],
+		["string", "'"]
+	]],
+
+	["string-interpolation", [
+		["string", "f'{{"],
+		["interpolation", [
+			["punctuation", "{"],
+			["number", "4"],
+			["operator", "*"],
+			["number", "10"],
+			["punctuation", "}"]
+		]],
+		["string", "}}'"]
+	]],
+
+	["string-interpolation", [
+		["string", "fr'x="],
+		["interpolation", [
+			["punctuation", "{"],
+			["number", "4"],
+			["operator", "*"],
+			["number", "10"],
+			["punctuation", "}"]
+		]],
+		["string", "\\n'"]
+	]],
+
+	["string-interpolation", [
+		["string", "f'''"],
+		["interpolation", [
+			["punctuation", "{"],
+			"x\r\n",
+			["operator", "+"],
+			["number", "1"],
+			["punctuation", "}"]
+		]],
+		["string", "'''"]
+	]],
+
+	["string-interpolation", [
+		["string", "f'mapping is "],
+		["interpolation", [
+			["punctuation", "{"],
+			["punctuation", "{"],
+			"a",
+			["punctuation", ":"],
+			"b ",
+			["keyword", "for"],
+			["punctuation", "("],
+			"a",
+			["punctuation", ","],
+			" b",
+			["punctuation", ")"],
+			["keyword", "in"],
+			["punctuation", "("],
+			["punctuation", "("],
+			["number", "1"],
+			["punctuation", ","],
+			["number", "2"],
+			["punctuation", ")"],
+			["punctuation", ","],
+			["punctuation", "("],
+			["number", "3"],
+			["punctuation", ","],
+			["number", "4"],
+			["punctuation", ")"],
+			["punctuation", ")"],
+			["punctuation", "}"],
+			["punctuation", "}"]
+		]],
+		["string", "'"]
+	]],
+
+	["string-interpolation", [
+		["string", "f'"],
+		["interpolation", [
+			["punctuation", "{"],
+			["punctuation", "("],
+			["keyword", "lambda"],
+			" x",
+			["punctuation", ":"],
+			" x",
+			["operator", "*"],
+			["number", "2"],
+			["punctuation", ")"],
+			["punctuation", "("],
+			["number", "3"],
+			["punctuation", ")"],
+			["punctuation", "}"]
+		]],
+		["string", "'"]
+	]]
+]
+
+----------------------------------------------------
+
+Checks for string interpolation.

tests/languages/python/string_feature.test

@@ -4,14 +4,24 @@
 'fo\'obar'
 "fo\" # comment obar"
 
+r"\n"
+b'foo'
+rb"foo\n"
+u"foo"
+
 ----------------------------------------------------
 
 [
 	["string", "\"\""],
 	["string", "\"fo\\\"obar\""],
 	["string", "''"],
 	["string", "'fo\\'obar'"],
-	["string", "\"fo\\\" # comment obar\""]
+	["string", "\"fo\\\" # comment obar\""],
+
+	["string", "r\"\\n\""],
+	["string", "b'foo'"],
+	["string", "rb\"foo\\n\""],
+	["string", "u\"foo\""]
 ]
 
 ----------------------------------------------------