Skip to content

Commit 764aa0e

Browse files
ntBreAlexWaygood
ntBre
and
AlexWaygood
authored
Allow passing ParseOptions to inline tests (#16357)
## Summary This PR adds support for a pragma-style header for inline parser tests containing JSON-serialized `ParseOptions`. For example, ```python # parse_options: { "target-version": "3.9" } match 2: case 1: pass ``` The line must start with `# parse_options: ` and then the rest of the (trimmed) line is deserialized into `ParseOptions` used for parsing the the test. ## Test Plan Existing inline tests, plus two new inline tests for `match-before-py310`. --------- Co-authored-by: Alex Waygood <alex.waygood@gmail.com>
1 parent 568cf88 commit 764aa0e

8 files changed

Lines changed: 238 additions & 7 deletions

File tree

‎Cargo.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/ruff_python_parser/Cargo.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,13 @@ unicode-normalization = { workspace = true }
2929

3030
[dev-dependencies]
3131
ruff_annotate_snippets = { workspace = true }
32+
ruff_python_ast = { workspace = true, features = ["serde"] }
3233
ruff_source_file = { workspace = true }
3334

3435
anyhow = { workspace = true }
3536
insta = { workspace = true, features = ["glob"] }
37+
serde = { workspace = true }
38+
serde_json = { workspace = true }
3639
walkdir = { workspace = true }
3740

3841
[lints]

‎crates/ruff_python_parser/resources/inline/err/match_before_py310.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# parse_options: { "target-version": "3.9" }
2+
match 2:
3+
case 1:
4+
pass

‎crates/ruff_python_parser/resources/inline/ok/match_after_py310.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# parse_options: { "target-version": "3.10" }
2+
match 2:
3+
case 1:
4+
pass

‎crates/ruff_python_parser/src/parser/statement.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2265,6 +2265,18 @@ impl<'src> Parser<'src> {
22652265

22662266
let cases = self.parse_match_body();
22672267

2268+
// test_err match_before_py310
2269+
// # parse_options: { "target-version": "3.9" }
2270+
// match 2:
2271+
// case 1:
2272+
// pass
2273+
2274+
// test_ok match_after_py310
2275+
// # parse_options: { "target-version": "3.10" }
2276+
// match 2:
2277+
// case 1:
2278+
// pass
2279+
22682280
if self.options.target_version < PythonVersion::PY310 {
22692281
self.unsupported_syntax_errors.push(UnsupportedSyntaxError {
22702282
kind: UnsupportedSyntaxErrorKind::MatchBeforePy310,

‎crates/ruff_python_parser/tests/fixtures.rs‎

Lines changed: 96 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use std::path::Path;
55

66
use ruff_annotate_snippets::{Level, Renderer, Snippet};
77
use ruff_python_ast::visitor::source_order::{walk_module, SourceOrderVisitor, TraversalSignal};
8-
use ruff_python_ast::{AnyNodeRef, Mod};
8+
use ruff_python_ast::{AnyNodeRef, Mod, PythonVersion};
99
use ruff_python_parser::{parse_unchecked, Mode, ParseErrorType, ParseOptions, Token};
1010
use ruff_source_file::{LineIndex, OneIndexed, SourceCode};
1111
use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
@@ -34,9 +34,14 @@ fn inline_err() {
3434
/// Snapshots the AST.
3535
fn test_valid_syntax(input_path: &Path) {
3636
let source = fs::read_to_string(input_path).expect("Expected test file to exist");
37-
let parsed = parse_unchecked(&source, ParseOptions::from(Mode::Module));
37+
let options = extract_options(&source).unwrap_or_else(|| {
38+
ParseOptions::from(Mode::Module).with_target_version(PythonVersion::latest())
39+
});
40+
let parsed = parse_unchecked(&source, options);
3841

39-
if !parsed.is_valid() {
42+
let is_valid = parsed.is_valid() && parsed.unsupported_syntax_errors().is_empty();
43+
44+
if !is_valid {
4045
let line_index = LineIndex::from_source_text(&source);
4146
let source_code = SourceCode::new(&source, &line_index);
4247

@@ -55,6 +60,19 @@ fn test_valid_syntax(input_path: &Path) {
5560
.unwrap();
5661
}
5762

63+
for error in parsed.unsupported_syntax_errors() {
64+
writeln!(
65+
&mut message,
66+
"{}\n",
67+
CodeFrame {
68+
range: error.range,
69+
error: &ParseErrorType::OtherError(error.to_string()),
70+
source_code: &source_code,
71+
}
72+
)
73+
.unwrap();
74+
}
75+
5876
panic!("{input_path:?}: {message}");
5977
}
6078

@@ -78,10 +96,15 @@ fn test_valid_syntax(input_path: &Path) {
7896
/// Snapshots the AST and the error messages.
7997
fn test_invalid_syntax(input_path: &Path) {
8098
let source = fs::read_to_string(input_path).expect("Expected test file to exist");
81-
let parsed = parse_unchecked(&source, ParseOptions::from(Mode::Module));
99+
let options = extract_options(&source).unwrap_or_else(|| {
100+
ParseOptions::from(Mode::Module).with_target_version(PythonVersion::latest())
101+
});
102+
let parsed = parse_unchecked(&source, options);
103+
104+
let is_valid = parsed.is_valid() && parsed.unsupported_syntax_errors().is_empty();
82105

83106
assert!(
84-
!parsed.is_valid(),
107+
!is_valid,
85108
"{input_path:?}: Expected parser to generate at least one syntax error for a program containing syntax errors."
86109
);
87110

@@ -92,11 +115,13 @@ fn test_invalid_syntax(input_path: &Path) {
92115
writeln!(&mut output, "## AST").unwrap();
93116
writeln!(&mut output, "\n```\n{:#?}\n```", parsed.syntax()).unwrap();
94117

95-
writeln!(&mut output, "## Errors\n").unwrap();
96-
97118
let line_index = LineIndex::from_source_text(&source);
98119
let source_code = SourceCode::new(&source, &line_index);
99120

121+
if !parsed.errors().is_empty() {
122+
writeln!(&mut output, "## Errors\n").unwrap();
123+
}
124+
100125
for error in parsed.errors() {
101126
writeln!(
102127
&mut output,
@@ -110,6 +135,23 @@ fn test_invalid_syntax(input_path: &Path) {
110135
.unwrap();
111136
}
112137

138+
if !parsed.unsupported_syntax_errors().is_empty() {
139+
writeln!(&mut output, "## Unsupported Syntax Errors\n").unwrap();
140+
}
141+
142+
for error in parsed.unsupported_syntax_errors() {
143+
writeln!(
144+
&mut output,
145+
"{}\n",
146+
CodeFrame {
147+
range: error.range,
148+
error: &ParseErrorType::OtherError(error.to_string()),
149+
source_code: &source_code,
150+
}
151+
)
152+
.unwrap();
153+
}
154+
113155
insta::with_settings!({
114156
omit_expression => true,
115157
input_file => input_path,
@@ -119,6 +161,53 @@ fn test_invalid_syntax(input_path: &Path) {
119161
});
120162
}
121163

164+
/// Copy of [`ParseOptions`] for deriving [`Deserialize`] with serde as a dev-dependency.
165+
#[derive(serde::Deserialize)]
166+
#[serde(rename_all = "kebab-case")]
167+
struct JsonParseOptions {
168+
#[serde(default)]
169+
mode: JsonMode,
170+
#[serde(default)]
171+
target_version: PythonVersion,
172+
}
173+
174+
/// Copy of [`Mode`] for deserialization.
175+
#[derive(Default, serde::Deserialize)]
176+
#[serde(rename_all = "kebab-case")]
177+
enum JsonMode {
178+
#[default]
179+
Module,
180+
Expression,
181+
ParenthesizedExpression,
182+
Ipython,
183+
}
184+
185+
impl From<JsonParseOptions> for ParseOptions {
186+
fn from(value: JsonParseOptions) -> Self {
187+
let mode = match value.mode {
188+
JsonMode::Module => Mode::Module,
189+
JsonMode::Expression => Mode::Expression,
190+
JsonMode::ParenthesizedExpression => Mode::ParenthesizedExpression,
191+
JsonMode::Ipython => Mode::Ipython,
192+
};
193+
Self::from(mode).with_target_version(value.target_version)
194+
}
195+
}
196+
197+
/// Extract [`ParseOptions`] from an initial pragma line, if present.
198+
///
199+
/// For example,
200+
///
201+
/// ```python
202+
/// # parse_options: { "target-version": "3.10" }
203+
/// def f(): ...
204+
fn extract_options(source: &str) -> Option<ParseOptions> {
205+
let header = source.lines().next()?;
206+
let (_label, options) = header.split_once("# parse_options: ")?;
207+
let options: Option<JsonParseOptions> = serde_json::from_str(options.trim()).ok();
208+
options.map(ParseOptions::from)
209+
}
210+
122211
// Test that is intentionally ignored by default.
123212
// Use it for quickly debugging a parser issue.
124213
#[test]

‎crates/ruff_python_parser/tests/snapshots/invalid_syntax@match_before_py310.py.snap‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
---
2+
source: crates/ruff_python_parser/tests/fixtures.rs
3+
input_file: crates/ruff_python_parser/resources/inline/err/match_before_py310.py
4+
---
5+
## AST
6+
7+
```
8+
Module(
9+
ModModule {
10+
range: 0..79,
11+
body: [
12+
Match(
13+
StmtMatch {
14+
range: 45..78,
15+
subject: NumberLiteral(
16+
ExprNumberLiteral {
17+
range: 51..52,
18+
value: Int(
19+
2,
20+
),
21+
},
22+
),
23+
cases: [
24+
MatchCase {
25+
range: 58..78,
26+
pattern: MatchValue(
27+
PatternMatchValue {
28+
range: 63..64,
29+
value: NumberLiteral(
30+
ExprNumberLiteral {
31+
range: 63..64,
32+
value: Int(
33+
1,
34+
),
35+
},
36+
),
37+
},
38+
),
39+
guard: None,
40+
body: [
41+
Pass(
42+
StmtPass {
43+
range: 74..78,
44+
},
45+
),
46+
],
47+
},
48+
],
49+
},
50+
),
51+
],
52+
},
53+
)
54+
```
55+
## Unsupported Syntax Errors
56+
57+
|
58+
1 | # parse_options: { "target-version": "3.9" }
59+
2 | match 2:
60+
| ^^^^^ Syntax Error: Cannot use `match` statement on Python 3.9 (syntax was added in Python 3.10)
61+
3 | case 1:
62+
4 | pass
63+
|

‎crates/ruff_python_parser/tests/snapshots/valid_syntax@match_after_py310.py.snap‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
---
2+
source: crates/ruff_python_parser/tests/fixtures.rs
3+
input_file: crates/ruff_python_parser/resources/inline/ok/match_after_py310.py
4+
---
5+
## AST
6+
7+
```
8+
Module(
9+
ModModule {
10+
range: 0..80,
11+
body: [
12+
Match(
13+
StmtMatch {
14+
range: 46..79,
15+
subject: NumberLiteral(
16+
ExprNumberLiteral {
17+
range: 52..53,
18+
value: Int(
19+
2,
20+
),
21+
},
22+
),
23+
cases: [
24+
MatchCase {
25+
range: 59..79,
26+
pattern: MatchValue(
27+
PatternMatchValue {
28+
range: 64..65,
29+
value: NumberLiteral(
30+
ExprNumberLiteral {
31+
range: 64..65,
32+
value: Int(
33+
1,
34+
),
35+
},
36+
),
37+
},
38+
),
39+
guard: None,
40+
body: [
41+
Pass(
42+
StmtPass {
43+
range: 75..79,
44+
},
45+
),
46+
],
47+
},
48+
],
49+
},
50+
),
51+
],
52+
},
53+
)
54+
```

0 commit comments

Comments
 (0)