Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for PEP 701 #7376

Merged
merged 45 commits into from
Sep 29, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
0ce342b
Add support for the new f-string tokens per PEP 701 (#6659)
dhruvmanila Sep 14, 2023
2e9ea6f
Add support for parsing f-string as per PEP 701 (#7041)
dhruvmanila Sep 14, 2023
d9876fc
Use narrow type for string parsing patterns (#7211)
dhruvmanila Sep 14, 2023
e249840
Disallow non-parenthesized lambda expr in f-string (#7263)
dhruvmanila Sep 14, 2023
fc28174
Fix curly brace escape handling in f-strings (#7331)
dhruvmanila Sep 14, 2023
3f81cb1
Allow `NUL` character in f-strings (#7378)
dhruvmanila Sep 14, 2023
deee2df
Update `Stylist` quote detection with new f-string token (#7328)
dhruvmanila Sep 15, 2023
37b8a93
Update `PLE2510`, `PLE2512-2515` to check in f-strings (#7387)
dhruvmanila Sep 16, 2023
049c8d5
Update `F541` to use new f-string tokens (#7327)
dhruvmanila Sep 18, 2023
57fac75
Update `Indexer` to use new f-string tokens (#7325)
dhruvmanila Sep 19, 2023
21ca907
Update `RUF001`, `RUF003` to check in f-strings (#7477)
dhruvmanila Sep 19, 2023
19dc285
Update `W605` to check in f-strings (#7329)
dhruvmanila Sep 19, 2023
363aeb9
Update `ISC001`, `ISC002` to check in f-strings (#7515)
dhruvmanila Sep 20, 2023
3aceb6b
Detect `noqa` directives for multi-line f-strings (#7326)
dhruvmanila Sep 21, 2023
97a5e35
Use the new f-string tokens in string formatting (#7586)
dhruvmanila Sep 22, 2023
01123d5
Ignore quote escapes in expression part of f-string (#7597)
dhruvmanila Sep 25, 2023
e9a2595
Remove `StringKind::FString` and `StringKind::RawFString` (#7667)
dhruvmanila Sep 26, 2023
2ba96ee
Separate `Q003` to accomodate f-string context (#7588)
dhruvmanila Sep 27, 2023
2d8270f
Update `Q000`, `Q001` with the new f-string tokens (#7589)
dhruvmanila Sep 27, 2023
0563dcb
Fix clippy, cargo fmt
dhruvmanila Sep 27, 2023
6114f61
Ignore complete f-string if any part should be ignored
dhruvmanila Sep 28, 2023
57501e2
Fix rebase
dhruvmanila Sep 29, 2023
3839819
Add support for the new f-string tokens per PEP 701 (#6659)
dhruvmanila Sep 14, 2023
94b0b52
Add support for parsing f-string as per PEP 701 (#7041)
dhruvmanila Sep 14, 2023
4521198
Use narrow type for string parsing patterns (#7211)
dhruvmanila Sep 14, 2023
3cc5455
Disallow non-parenthesized lambda expr in f-string (#7263)
dhruvmanila Sep 14, 2023
63b03b7
Fix curly brace escape handling in f-strings (#7331)
dhruvmanila Sep 14, 2023
846948f
Allow `NUL` character in f-strings (#7378)
dhruvmanila Sep 14, 2023
658435e
Update `Stylist` quote detection with new f-string token (#7328)
dhruvmanila Sep 15, 2023
402ac49
Update `PLE2510`, `PLE2512-2515` to check in f-strings (#7387)
dhruvmanila Sep 16, 2023
4481558
Update `F541` to use new f-string tokens (#7327)
dhruvmanila Sep 18, 2023
124cd4a
Update `Indexer` to use new f-string tokens (#7325)
dhruvmanila Sep 19, 2023
49ea2e5
Update `RUF001`, `RUF003` to check in f-strings (#7477)
dhruvmanila Sep 19, 2023
26d5daf
Update `W605` to check in f-strings (#7329)
dhruvmanila Sep 19, 2023
fac0974
Update `ISC001`, `ISC002` to check in f-strings (#7515)
dhruvmanila Sep 20, 2023
5077513
Detect `noqa` directives for multi-line f-strings (#7326)
dhruvmanila Sep 21, 2023
e032429
Use the new f-string tokens in string formatting (#7586)
dhruvmanila Sep 22, 2023
e4e2b45
Ignore quote escapes in expression part of f-string (#7597)
dhruvmanila Sep 25, 2023
c61d134
Remove `StringKind::FString` and `StringKind::RawFString` (#7667)
dhruvmanila Sep 26, 2023
48d4f23
Separate `Q003` to accomodate f-string context (#7588)
dhruvmanila Sep 27, 2023
a04d4c1
Update `Q000`, `Q001` with the new f-string tokens (#7589)
dhruvmanila Sep 27, 2023
0b676cb
Fix clippy, cargo fmt
dhruvmanila Sep 27, 2023
68e7018
Ignore complete f-string if any part should be ignored
dhruvmanila Sep 28, 2023
b8b5131
Update logical line rules for the new f-string tokens
dhruvmanila Sep 28, 2023
204a62c
Merge branch 'dhruv/fstring-logical-line-rules' into dhruv/pep-701
dhruvmanila Sep 29, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion crates/ruff_benchmark/benches/formatter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ fn benchmark_formatter(criterion: &mut Criterion) {
let comment_ranges = comment_ranges.finish();

// Parse the AST.
let module = parse_tokens(tokens, Mode::Module, "<filename>")
let module = parse_tokens(tokens, case.code(), Mode::Module, "<filename>")
.expect("Input to be a valid python program");

b.iter(|| {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,3 +59,23 @@
_ = foo + bar + "abc"
_ = "abc" + foo + bar
_ = foo + "abc" + bar

# Multiple strings nested inside a f-string
_ = f"a {'b' 'c' 'd'} e"
_ = f"""abc {"def" "ghi"} jkl"""
_ = f"""abc {
"def"
"ghi"
} jkl"""

# Nested f-strings
_ = "a" f"b {f"c" f"d"} e" "f"
_ = f"b {f"c" f"d {f"e" f"f"} g"} h"
_ = f"b {f"abc" \
f"def"} g"

# Explicitly concatenated nested f-strings
_ = f"a {f"first"
+ f"second"} d"
_ = f"a {f"first {f"middle"}"
+ f"second"} d"
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,33 @@
'This is a'
'\'string\''
)

# Same as above, but with f-strings
f'This is a \'string\'' # Q003
f'This is \\ a \\\'string\'' # Q003
f'"This" is a \'string\''
f"This is a 'string'"
f"\"This\" is a 'string'"
fr'This is a \'string\''
fR'This is a \'string\''
foo = (
f'This is a'
f'\'string\'' # Q003
)

# Nested f-strings (Python 3.12+)
#
# The first one is interesting because the fix for it is valid pre 3.12:
#
# f"'foo' {'nested'}"
#
# but as the actual string itself is invalid pre 3.12, we don't catch it.
f'\'foo\' {'nested'}' # Q003
f'\'foo\' {f'nested'}' # Q003
f'\'foo\' {f'\'nested\''} \'\'' # Q003

f'normal {f'nested'} normal'
f'\'normal\' {f'nested'} normal' # Q003
f'\'normal\' {f'nested'} "double quotes"'
f'\'normal\' {f'\'nested\' {'other'} normal'} "double quotes"' # Q003
f'\'normal\' {f'\'nested\' {'other'} "double quotes"'} normal' # Q00l
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,32 @@
"This is a"
"\"string\""
)

# Same as above, but with f-strings
f"This is a \"string\""
f"'This' is a \"string\""
f'This is a "string"'
f'\'This\' is a "string"'
fr"This is a \"string\""
fR"This is a \"string\""
foo = (
f"This is a"
f"\"string\""
)

# Nested f-strings (Python 3.12+)
#
# The first one is interesting because the fix for it is valid pre 3.12:
#
# f'"foo" {"nested"}'
#
# but as the actual string itself is invalid pre 3.12, we don't catch it.
f"\"foo\" {"foo"}" # Q003
f"\"foo\" {f"foo"}" # Q003
f"\"foo\" {f"\"foo\""} \"\"" # Q003

f"normal {f"nested"} normal"
f"\"normal\" {f"nested"} normal" # Q003
f"\"normal\" {f"nested"} 'single quotes'"
f"\"normal\" {f"\"nested\" {"other"} normal"} 'single quotes'" # Q003
f"\"normal\" {f"\"nested\" {"other"} 'single quotes'"} normal" # Q003
5 changes: 5 additions & 0 deletions crates/ruff_linter/resources/test/fixtures/pycodestyle/E20.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,3 +84,8 @@
x = [ #
'some value',
]

# F-strings
f"{ {'a': 1} }"
f"{[ { {'a': 1} } ]}"
f"normal { {f"{ { [1, 2] } }" } } normal"
11 changes: 11 additions & 0 deletions crates/ruff_linter/resources/test/fixtures/pycodestyle/E23.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,16 @@ def foo() -> None:
'tag_smalldata':[('byte_count_mdtype', 'u4'), ('data', 'S4')],
}

# E231
f"{(a,b)}"

# Okay because it's hard to differentiate between the usages of a colon in a f-string
f"{a:=1}"
f"{ {'a':1} }"
f"{a:.3f}"
f"{(a:=1)}"
f"{(lambda x:x)}"
f"normal{f"{a:.3f}"}normal"

#: Okay
a = (1,
6 changes: 6 additions & 0 deletions crates/ruff_linter/resources/test/fixtures/pycodestyle/E25.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,9 @@ def add(a: int=0, b: int =0, c: int= 0) -> int:
#: Okay
def add(a: int = _default(name='f')):
return a

# F-strings
f"{a=}"
f"{a:=1}"
f"{foo(a=1)}"
f"normal {f"{a=}"} normal"
8 changes: 8 additions & 0 deletions crates/ruff_linter/resources/test/fixtures/pycodestyle/W19.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,3 +152,11 @@ def test_keys(self):
multiline string with tab in it, different lines
'''
" single line string with tab in it"

f"test{
tab_indented_should_be_flagged
} <- this tab is fine"

f"""test{
tab_indented_should_be_flagged
} <- this tab is fine"""
54 changes: 54 additions & 0 deletions crates/ruff_linter/resources/test/fixtures/pycodestyle/W605_2.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# Same as `W605_0.py` but using f-strings instead.

#: W605:1:10
regex = f'\.png$'

#: W605:2:1
regex = f'''
\.png$
'''

#: W605:2:6
f(
f'\_'
)

#: W605:4:6
f"""
multi-line
literal
with \_ somewhere
in the middle
"""

#: W605:1:38
value = f'new line\nand invalid escape \_ here'


#: Okay
regex = fr'\.png$'
regex = f'\\.png$'
regex = fr'''
\.png$
'''
regex = fr'''
\\.png$
'''
s = f'\\'
regex = f'\w' # noqa
regex = f'''
\w
''' # noqa

regex = f'\\\_'
value = f'\{{1}}'
value = f'\{1}'
value = f'{1:\}'
value = f"{f"\{1}"}"
value = rf"{f"\{1}"}"

# Okay
value = rf'\{{1}}'
value = rf'\{1}'
value = rf'{1:\}'
value = f"{rf"\{1}"}"
6 changes: 2 additions & 4 deletions crates/ruff_linter/resources/test/fixtures/pyflakes/F541.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,5 @@
""f""
''f""
(""f""r"")

# To be fixed
# Error: f-string: single '}' is not allowed at line 41 column 8
# f"\{{x}}"
f"{v:{f"0.2f"}}"
f"\{{x}}"
Binary file not shown.
16 changes: 16 additions & 0 deletions crates/ruff_linter/resources/test/fixtures/ruff/confusables.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,19 @@ def f():
# consisting of a single ambiguous character, while the second character is a "word
# boundary" (whitespace) that it itself ambiguous.
x = "Р усский"

# Same test cases as above but using f-strings instead:
x = f"𝐁ad string"
x = f"−"
x = f"Русский"
x = f"βα Bαd"
x = f"Р усский"

# Nested f-strings
x = f"𝐁ad string {f" {f"Р усский"}"}"

# Comments inside f-strings
x = f"string { # And here's a comment with an unusual parenthesis: )
# And here's a comment with a greek alpha: ∗
foo # And here's a comment with an unusual punctuation mark: ᜵
}"
4 changes: 2 additions & 2 deletions crates/ruff_linter/src/checkers/ast/analyze/expression.rs
Original file line number Diff line number Diff line change
Expand Up @@ -963,9 +963,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
pylint::rules::await_outside_async(checker, expr);
}
}
Expr::FString(ast::ExprFString { values, .. }) => {
Expr::FString(fstring @ ast::ExprFString { values, .. }) => {
if checker.enabled(Rule::FStringMissingPlaceholders) {
pyflakes::rules::f_string_missing_placeholders(expr, values, checker);
pyflakes::rules::f_string_missing_placeholders(fstring, checker);
}
if checker.enabled(Rule::HardcodedSQLExpression) {
flake8_bandit::rules::hardcoded_sql_expression(checker, expr);
Expand Down
2 changes: 1 addition & 1 deletion crates/ruff_linter/src/checkers/ast/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ impl<'a> Checker<'a> {

// Find the quote character used to start the containing f-string.
let expr = self.semantic.current_expression()?;
let string_range = self.indexer.f_string_range(expr.start())?;
let string_range = self.indexer.fstring_ranges().innermost(expr.start())?;
let trailing_quote = trailing_quote(self.locator.slice(string_range))?;

// Invert the quote character, if it's a single quote.
Expand Down
62 changes: 33 additions & 29 deletions crates/ruff_linter/src/checkers/tokens.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,23 +45,25 @@ pub(crate) fn check_tokens(
let mut state_machine = StateMachine::default();
for &(ref tok, range) in tokens.iter().flatten() {
let is_docstring = state_machine.consume(tok);
if matches!(tok, Tok::String { .. } | Tok::Comment(_)) {
ruff::rules::ambiguous_unicode_character(
&mut diagnostics,
locator,
range,
if tok.is_string() {
if is_docstring {
Context::Docstring
} else {
Context::String
}
let context = match tok {
Tok::String { .. } => {
if is_docstring {
Context::Docstring
} else {
Context::Comment
},
settings,
);
}
Context::String
}
}
Tok::FStringMiddle { .. } => Context::String,
Tok::Comment(_) => Context::Comment,
_ => continue,
};
ruff::rules::ambiguous_unicode_character(
&mut diagnostics,
locator,
range,
context,
settings,
);
}
}

Expand All @@ -75,14 +77,14 @@ pub(crate) fn check_tokens(

if settings.rules.enabled(Rule::InvalidEscapeSequence) {
for (tok, range) in tokens.iter().flatten() {
if tok.is_string() {
pycodestyle::rules::invalid_escape_sequence(
&mut diagnostics,
locator,
*range,
settings.rules.should_fix(Rule::InvalidEscapeSequence),
);
}
pycodestyle::rules::invalid_escape_sequence(
&mut diagnostics,
locator,
indexer,
tok,
*range,
settings.rules.should_fix(Rule::InvalidEscapeSequence),
);
}
}

Expand All @@ -98,9 +100,7 @@ pub(crate) fn check_tokens(
Rule::InvalidCharacterZeroWidthSpace,
]) {
for (tok, range) in tokens.iter().flatten() {
if tok.is_string() {
pylint::rules::invalid_string_characters(&mut diagnostics, *range, locator);
}
pylint::rules::invalid_string_characters(&mut diagnostics, tok, *range, locator);
}
}

Expand All @@ -118,13 +118,16 @@ pub(crate) fn check_tokens(
);
}

if settings.rules.enabled(Rule::AvoidableEscapedQuote) && settings.flake8_quotes.avoid_escape {
flake8_quotes::rules::avoidable_escaped_quote(&mut diagnostics, tokens, locator, settings);
}

if settings.rules.any_enabled(&[
Rule::BadQuotesInlineString,
Rule::BadQuotesMultilineString,
Rule::BadQuotesDocstring,
Rule::AvoidableEscapedQuote,
]) {
flake8_quotes::rules::from_tokens(&mut diagnostics, tokens, locator, settings);
flake8_quotes::rules::check_string_quotes(&mut diagnostics, tokens, locator, settings);
}

if settings.rules.any_enabled(&[
Expand All @@ -136,6 +139,7 @@ pub(crate) fn check_tokens(
tokens,
&settings.flake8_implicit_str_concat,
locator,
indexer,
);
}

Expand Down
Loading
Loading