diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b0538abad20..ac1c9306df0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -104,7 +104,6 @@ jobs: env: RUST_BACKTRACE: full name: Run rust tests - needs: lalrpop runs-on: ${{ matrix.os }} strategy: matrix: @@ -112,11 +111,6 @@ jobs: fail-fast: false steps: - uses: actions/checkout@v3 - - name: Cache generated parser - uses: actions/cache@v2 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - uses: dtolnay/rust-toolchain@stable with: components: clippy @@ -160,16 +154,9 @@ jobs: exotic_targets: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Ensure compilation on various targets - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Cache generated parser - uses: actions/cache@v2 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - - uses: dtolnay/rust-toolchain@stable with: target: i686-unknown-linux-gnu @@ -224,7 +211,6 @@ jobs: snippets_cpython: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} - needs: lalrpop env: RUST_BACKTRACE: full name: Run snippets and cpython tests @@ -235,12 +221,6 @@ jobs: fail-fast: false steps: - uses: actions/checkout@v3 - - name: Cache generated parser - uses: actions/cache@v2 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - - uses: dtolnay/rust-toolchain@stable - uses: actions/setup-python@v2 with: @@ -290,50 +270,11 @@ jobs: mkdir site-packages target/release/rustpython --install-pip ensurepip --user - lalrpop: - if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} - name: Generate parser with lalrpop - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - - name: Cache generated parser - uses: actions/cache@v2 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - - name: Check if cached generated parser exists - id: generated_parser - uses: andstor/file-existence-action@v1 - with: - files: "compiler/parser/python.rs" - - if: runner.os == 'Windows' - name: Force python.lalrpop to be lf # actions@checkout ignore .gitattributes - run: | - set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file - - name: Install lalrpop - if: steps.generated_parser.outputs.files_exists == 'false' - uses: baptiste0928/cargo-install@v1 - with: - crate: lalrpop - version: "0.19.8" - - name: Run lalrpop - if: steps.generated_parser.outputs.files_exists == 'false' - run: lalrpop compiler/parser/python.lalrpop - lint: name: Check Rust code with rustfmt and clippy - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Cache generated parser - uses: actions/cache@v2 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy @@ -361,15 +302,9 @@ jobs: miri: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Run tests under miri - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Cache generated parser - uses: actions/cache@v2 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - uses: dtolnay/rust-toolchain@master with: toolchain: nightly @@ -384,15 +319,9 @@ jobs: wasm: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Check the WASM package and demo - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Cache generated parser - uses: actions/cache@v2 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 @@ -434,15 +363,9 @@ jobs: wasm-wasi: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Run snippets and cpython tests on wasm-wasi - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Cache generated parser - uses: actions/cache@v2 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasi diff --git a/Cargo.lock b/Cargo.lock index a021269788d..7fa826f12e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,15 +91,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" -[[package]] -name = "ascii-canvas" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6" -dependencies = [ - "term", -] - [[package]] name = "atomic" version = "0.5.1" @@ -141,21 +132,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bit-set" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bitflags" version = "1.3.2" @@ -650,12 +626,6 @@ dependencies = [ "syn", ] -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - [[package]] name = "digest" version = "0.10.6" @@ -706,15 +676,6 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" -[[package]] -name = "ena" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3" -dependencies = [ - "log", -] - [[package]] name = "encode_unicode" version = "0.3.6" @@ -786,12 +747,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "flame" version = "0.2.2" @@ -1056,38 +1011,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "lalrpop" -version = "0.19.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b30455341b0e18f276fa64540aff54deafb54c589de6aca68659c63dd2d5d823" -dependencies = [ - "ascii-canvas", - "atty", - "bit-set", - "diff", - "ena", - "itertools", - "lalrpop-util", - "petgraph", - "pico-args", - "regex", - "regex-syntax", - "string_cache", - "term", - "tiny-keccak", - "unicode-xid", -] - -[[package]] -name = "lalrpop-util" -version = "0.19.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf796c978e9b4d983414f4caedc9273aa33ee214c5b887bd55fde84c85d2dc4" -dependencies = [ - "regex", -] - [[package]] name = "lazy_static" version = "0.2.11" @@ -1319,12 +1242,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "new_debug_unreachable" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" - [[package]] name = "nibble_vec" version = "0.1.0" @@ -1556,22 +1473,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" [[package]] -name = "petgraph" -version = "0.6.2" +name = "peg" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +checksum = "a07f2cafdc3babeebc087e499118343442b742cc7c31b4d054682cc598508554" dependencies = [ - "fixedbitset", - "indexmap", + "peg-macros", + "peg-runtime", +] + +[[package]] +name = "peg-macros" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a90084dc05cf0428428e3d12399f39faad19b0909f64fb9170c9fdd6d9cd49b" +dependencies = [ + "peg-runtime", + "proc-macro2", + "quote", ] +[[package]] +name = "peg-runtime" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa00462b37ead6d11a82c9d568b26682d78e0477dc02d1966c013af80969739" + [[package]] name = "phf" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" dependencies = [ - "phf_shared 0.11.1", + "phf_shared", ] [[package]] @@ -1581,7 +1515,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" dependencies = [ "phf_generator", - "phf_shared 0.11.1", + "phf_shared", ] [[package]] @@ -1590,19 +1524,10 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" dependencies = [ - "phf_shared 0.11.1", + "phf_shared", "rand", ] -[[package]] -name = "phf_shared" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" -dependencies = [ - "siphasher", -] - [[package]] name = "phf_shared" version = "0.11.1" @@ -1612,12 +1537,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "pico-args" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468" - [[package]] name = "pkg-config" version = "0.3.26" @@ -1669,12 +1588,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "precomputed-hash" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" - [[package]] name = "proc-macro-crate" version = "1.2.1" @@ -2073,11 +1986,10 @@ dependencies = [ "anyhow", "insta", "itertools", - "lalrpop", - "lalrpop-util", "log", "num-bigint", "num-traits", + "peg", "phf", "phf_codegen", "rustc-hash", @@ -2480,19 +2392,6 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" -[[package]] -name = "string_cache" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08" -dependencies = [ - "new_debug_unreachable", - "once_cell", - "parking_lot", - "phf_shared 0.10.0", - "precomputed-hash", -] - [[package]] name = "strsim" version = "0.8.0" @@ -2571,17 +2470,6 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9410d0f6853b1d94f0e519fb95df60f29d2c1eff2d921ffdf01a4c8a3b54f12d" -[[package]] -name = "term" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" -dependencies = [ - "dirs-next", - "rustversion", - "winapi", -] - [[package]] name = "termcolor" version = "1.1.3" @@ -2899,12 +2787,6 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" -[[package]] -name = "unicode-xid" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" - [[package]] name = "unicode_names2" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 11df8b91f4f..b9872d8a83e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ members = [ ] [features] -default = ["threading", "stdlib", "zlib", "importlib", "encodings", "rustpython-parser/lalrpop"] +default = ["threading", "stdlib", "zlib", "importlib", "encodings"] importlib = ["rustpython-vm/importlib"] encodings = ["rustpython-vm/encodings"] stdlib = ["rustpython-stdlib", "rustpython-pylib"] diff --git a/compiler/parser/Cargo.toml b/compiler/parser/Cargo.toml index c7b8af64dd1..7d3bbd426bc 100644 --- a/compiler/parser/Cargo.toml +++ b/compiler/parser/Cargo.toml @@ -8,12 +8,8 @@ repository = "https://github.com/RustPython/RustPython" license = "MIT" edition = "2021" -[features] -default = ["lalrpop"] # removing this causes potential build failure - [build-dependencies] anyhow = "1.0.45" -lalrpop = { version = "0.19.8", optional = true } phf_codegen = "0.11.1" tiny-keccak = { version = "2", features = ["sha3"] } @@ -23,7 +19,6 @@ rustpython-compiler-core = { path = "../core", version = "0.2.0" } ahash = "0.7.6" itertools = "0.10.3" -lalrpop-util = "0.19.8" log = "0.4.16" num-bigint = "0.4.3" num-traits = "0.2.14" @@ -33,6 +28,7 @@ thiserror = "1.0" unic-emoji-char = "0.9.0" unic-ucd-ident = "0.9.0" unicode_names2 = "0.5.0" +peg = "0.8" [dev-dependencies] insta = "1.14.0" diff --git a/compiler/parser/build.rs b/compiler/parser/build.rs index f9a3439b3d6..8151328ef58 100644 --- a/compiler/parser/build.rs +++ b/compiler/parser/build.rs @@ -1,107 +1,10 @@ -use std::fmt::Write as _; use std::fs::File; -use std::io::{BufRead, BufReader, BufWriter, Write}; +use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; -use tiny_keccak::{Hasher, Sha3}; -fn main() -> anyhow::Result<()> { - const SOURCE: &str = "python.lalrpop"; +fn main() { let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()); - - println!("cargo:rerun-if-changed={SOURCE}"); - - try_lalrpop(SOURCE, &out_dir.join("python.rs"))?; gen_phf(&out_dir); - - Ok(()) -} - -fn requires_lalrpop(source: &str, target: &Path) -> Option { - let Ok(target) = File::open(target) else { - return Some("python.rs doesn't exist. regenerate.".to_owned()); - }; - - let sha_prefix = "// sha3: "; - let sha3_line = if let Some(sha3_line) = - BufReader::with_capacity(128, target) - .lines() - .find_map(|line| { - let line = line.unwrap(); - line.starts_with(sha_prefix).then_some(line) - }) { - sha3_line - } else { - // no sha3 line - maybe old version of lalrpop installed - return Some("python.rs doesn't include sha3 hash. regenerate.".to_owned()); - }; - let expected_sha3_str = sha3_line.strip_prefix(sha_prefix).unwrap(); - - let actual_sha3 = { - let mut hasher = Sha3::v256(); - let mut f = BufReader::new(File::open(source).unwrap()); - let mut line = String::new(); - while f.read_line(&mut line).unwrap() != 0 { - if line.ends_with('\n') { - line.pop(); - if line.ends_with('\r') { - line.pop(); - } - } - hasher.update(line.as_bytes()); - hasher.update(b"\n"); - line.clear(); - } - let mut hash = [0u8; 32]; - hasher.finalize(&mut hash); - hash - }; - let eq = sha_equal(expected_sha3_str, &actual_sha3); - if !eq { - let mut actual_sha3_str = String::new(); - for byte in actual_sha3 { - write!(actual_sha3_str, "{byte:02x}").unwrap(); - } - return Some(format!( - "python.rs hash expected: {expected_sha3_str} but actual: {actual_sha3_str}" - )); - } - None -} - -fn try_lalrpop(source: &str, target: &Path) -> anyhow::Result<()> { - let Some(_message) = requires_lalrpop(source, target) else { - return Ok(()); - }; - - #[cfg(feature = "lalrpop")] - // We are not using lalrpop::process_root() or Configuration::process_current_dir() - // because of https://github.com/lalrpop/lalrpop/issues/699. - lalrpop::Configuration::new() - .use_cargo_dir_conventions() - .set_in_dir(Path::new(".")) - .process() - .unwrap_or_else(|e| { - println!("cargo:warning={_message}"); - panic!("running lalrpop failed. {e:?}"); - }); - - #[cfg(not(feature = "lalrpop"))] - { - println!("cargo:warning=try: cargo build --manifest-path=compiler/parser/Cargo.toml --features=lalrpop"); - } - Ok(()) -} - -fn sha_equal(expected_sha3_str: &str, actual_sha3: &[u8; 32]) -> bool { - if expected_sha3_str.len() != 64 { - panic!("lalrpop version? hash bug is fixed in 0.19.8"); - } - - let mut expected_sha3 = [0u8; 32]; - for (i, b) in expected_sha3.iter_mut().enumerate() { - *b = u8::from_str_radix(&expected_sha3_str[i * 2..][..2], 16).unwrap(); - } - *actual_sha3 == expected_sha3 } fn gen_phf(out_dir: &Path) { diff --git a/compiler/parser/python.lalrpop b/compiler/parser/python.lalrpop deleted file mode 100644 index 719cef464bd..00000000000 --- a/compiler/parser/python.lalrpop +++ /dev/null @@ -1,1435 +0,0 @@ -// See also: file:///usr/share/doc/python/html/reference/grammar.html?highlight=grammar -// See also: https://github.com/antlr/grammars-v4/blob/master/python3/Python3.g4 -// See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions -// See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword - -use crate::{ - ast, - error::{LexicalError, LexicalErrorType}, - function::{ArgumentList, parse_args, parse_params, validate_arguments}, - lexer, - context::set_context, - string::parse_strings, - token::StringKind, -}; -use num_bigint::BigInt; - -grammar; - -// This is a hack to reduce the amount of lalrpop tables generated: -// For each public entry point, a full parse table is generated. -// By having only a single pub function, we reduce this to one. -pub Top: ast::Mod = { - StartModule => ast::Mod::Module { body, type_ignores: vec![] }, - StartInteractive => ast::Mod::Interactive { body }, - StartExpression ("\n")* => ast::Mod::Expression { body: Box::new(body) }, -}; - -Program: ast::Suite = { - => { - lines.into_iter().flatten().collect() - }, -}; - -// A file line either has a declaration, or an empty newline: -FileLine: ast::Suite = { - Statement, - "\n" => vec![], -}; - -Suite: ast::Suite = { - SimpleStatement, - "\n" Indent Dedent => s.into_iter().flatten().collect(), -}; - -Statement: ast::Suite = { - SimpleStatement, - => vec![s], -}; - -SimpleStatement: ast::Suite = { - ";"? "\n" => { - let mut statements = vec![s1]; - statements.extend(s2.into_iter().map(|e| e.1)); - statements - } -}; - -SmallStatement: ast::Stmt = { - ExpressionStatement, - PassStatement, - DelStatement, - FlowStatement, - ImportStatement, - GlobalStatement, - NonlocalStatement, - AssertStatement, -}; - -PassStatement: ast::Stmt = { - "pass" => { - ast::Stmt { - location, - end_location: Some(end_location), - custom: (), - node: ast::StmtKind::Pass, - } - }, -}; - -DelStatement: ast::Stmt = { - "del" => { - ast::Stmt { - location, - end_location: Some(end_location), - custom: (), - node: ast::StmtKind::Delete { targets: targets.into_iter().map(|expr| set_context(expr, ast::ExprContext::Del)).collect() }, - } - }, -}; - -ExpressionStatement: ast::Stmt = { - => { - // Just an expression, no assignment: - if suffix.is_empty() { - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::Expr { value: Box::new(expression) } - } - } else { - let mut targets = vec![set_context(expression, ast::ExprContext::Store)]; - let mut values = suffix; - - while values.len() > 1 { - targets.push(set_context(values.remove(0), ast::ExprContext::Store)); - } - - let value = Box::new(values.into_iter().next().unwrap()); - - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::Assign { targets, value, type_comment: None }, - } - } - }, - => { - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::AugAssign { - target: Box::new(set_context(target, ast::ExprContext::Store)), - op, - value: Box::new(rhs) - }, - } - }, - > ":" > => { - let simple = matches!(target.node, ast::ExprKind::Name { .. }); - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::AnnAssign { - target: Box::new(set_context(target, ast::ExprContext::Store)), - annotation: Box::new(annotation), - value: rhs.map(Box::new), - simple: if simple { 1 } else { 0 }, - }, - } - }, -}; - -AssignSuffix: ast::Expr = { - "=" => e -}; - -TestListOrYieldExpr: ast::Expr = { - TestList, - YieldExpr -} - -#[inline] -TestOrStarExprList: ast::Expr = { - // as far as I can tell, these were the same - TestList -}; - -TestOrStarExpr: ast::Expr = { - Test<"all">, - StarExpr, -}; - -NamedOrStarExpr: ast::Expr = { - NamedExpression, - StarExpr, -}; - -TestOrStarNamedExpr: ast::Expr = { - NamedExpressionTest, - StarExpr, -}; - -AugAssign: ast::Operator = { - "+=" => ast::Operator::Add, - "-=" => ast::Operator::Sub, - "*=" => ast::Operator::Mult, - "@=" => ast::Operator::MatMult, - "/=" => ast::Operator::Div, - "%=" => ast::Operator::Mod, - "&=" => ast::Operator::BitAnd, - "|=" => ast::Operator::BitOr, - "^=" => ast::Operator::BitXor, - "<<=" => ast::Operator::LShift, - ">>=" => ast::Operator::RShift, - "**=" => ast::Operator::Pow, - "//=" => ast::Operator::FloorDiv, -}; - -FlowStatement: ast::Stmt = { - "break" => { - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::Break, - } - }, - "continue" => { - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::Continue, - } - }, - "return" => { - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::Return { value: value.map(Box::new) }, - } - }, - => { - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::Expr { value: Box::new(expression) }, - } - }, - RaiseStatement, -}; - -RaiseStatement: ast::Stmt = { - "raise" => { - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::Raise { exc: None, cause: None }, - } - }, - "raise" > )?> => { - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) }, - } - }, -}; - -ImportStatement: ast::Stmt = { - "import" >> => { - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::Import { names }, - } - }, - "from" "import" => { - let (level, module) = source; - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::ImportFrom { - level, - module, - names - }, - } - }, -}; - -ImportFromLocation: (Option, Option) = { - => { - (Some(dots.iter().sum()), Some(name)) - }, - => { - (Some(dots.iter().sum()), None) - }, -}; - -ImportDots: usize = { - "..." => 3, - "." => 1, -}; - -ImportAsNames: Vec = { - >> => i, - "(" >> ","? ")" => i, - "*" => { - // Star import all - vec![ast::Alias::new(location, end_location, ast::AliasData { name: "*".to_string(), asname: None })] - }, -}; - - -#[inline] -ImportAsAlias: ast::Alias = { - => ast::Alias::new(location, end_location, ast::AliasData { name, asname: a.map(|a| a.1) }), -} - -// A name like abc or abc.def.ghi -DottedName: String = { - => n, - => { - let mut r = n.to_string(); - for x in n2 { - r.push_str("."); - r.push_str(&x.1); - } - r - }, -}; - -GlobalStatement: ast::Stmt = { - "global" > => { - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::Global { names } - } - }, -}; - -NonlocalStatement: ast::Stmt = { - "nonlocal" > => { - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::Nonlocal { names } - } - }, -}; - -AssertStatement: ast::Stmt = { - "assert" > )?> => { - ast::Stmt { - custom: (), - location, - end_location: Some(end_location), - node: ast::StmtKind::Assert { - test: Box::new(test), - msg: msg.map(|e| Box::new(e.1)) - } - } - }, -}; - -CompoundStatement: ast::Stmt = { - IfStatement, - WhileStatement, - ForStatement, - TryStatement, - WithStatement, - FuncDef, - ClassDef, -}; - -IfStatement: ast::Stmt = { - "if" ":" => { - // Determine last else: - let mut last = s3.map(|s| s.2).unwrap_or_default(); - let end_location = last - .last() - .or_else(|| s2.last().and_then(|last| last.4.last())) - .or_else(|| body.last()) - .unwrap() - .end_location; - // handle elif: - for i in s2.into_iter().rev() { - let x = ast::Stmt { - custom: (), - location: i.0, - end_location, - node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last }, - }; - last = vec![x]; - } - - ast::Stmt { - custom: (), - location, - end_location, - node: ast::StmtKind::If { test: Box::new(test), body, orelse: last } - } - }, -}; - -WhileStatement: ast::Stmt = { - "while" ":" => { - let orelse = s2.map(|s| s.2).unwrap_or_default(); - let end_location = orelse - .last() - .or_else(|| body.last()) - .unwrap() - .end_location; - ast::Stmt { - custom: (), - location, - end_location, - node: ast::StmtKind::While { - test: Box::new(test), - body, - orelse - }, - } - }, -}; - -ForStatement: ast::Stmt = { - "for" "in" ":" => { - let orelse = s2.map(|s| s.2).unwrap_or_default(); - let end_location = orelse - .last() - .or_else(|| body.last()) - .unwrap() - .end_location - .unwrap(); - let target = Box::new(set_context(target, ast::ExprContext::Store)); - let iter = Box::new(iter); - let type_comment = None; - let node = if is_async.is_some() { - ast::StmtKind::AsyncFor { target, iter, body, orelse, type_comment } - } else { - ast::StmtKind::For { target, iter, body, orelse, type_comment } - }; - ast::Stmt::new(location, end_location, node) - }, -}; - -TryStatement: ast::Stmt = { - "try" ":" => { - let orelse = else_suite.map(|s| s.2).unwrap_or_default(); - let finalbody = finally.map(|s| s.2).unwrap_or_default(); - let end_location = finalbody - .last() - .map(|last| last.end_location) - .or_else(|| orelse.last().map(|last| last.end_location)) - .or_else(|| handlers.last().map(|last| last.end_location)) - .unwrap(); - ast::Stmt { - custom: (), - location, - end_location, - node: ast::StmtKind::Try { - body, - handlers, - orelse, - finalbody, - }, - } - }, - "try" ":" => { - let handlers = vec![]; - let orelse = vec![]; - let finalbody = finally.2; - let end_location = finalbody.last().unwrap().end_location; - ast::Stmt { - custom: (), - location, - end_location, - node: ast::StmtKind::Try { - body, - handlers, - orelse, - finalbody, - }, - } - }, -}; - -ExceptClause: ast::Excepthandler = { - "except" ?> ":" => { - let end_location = body.last().unwrap().end_location.unwrap(); - ast::Excepthandler::new( - location, - end_location, - ast::ExcepthandlerKind::ExceptHandler { - type_: typ.map(Box::new), - name: None, - body, - }, - ) - }, - "except" "as" Identifier)> ":" => { - let end_location = body.last().unwrap().end_location.unwrap(); - ast::Excepthandler::new( - location, - end_location, - ast::ExcepthandlerKind::ExceptHandler { - type_: Some(Box::new(x.0)), - name: Some(x.2), - body, - }, - ) - }, -}; - -WithStatement: ast::Stmt = { - "with" ":" => { - let end_location = body.last().unwrap().end_location.unwrap(); - let type_comment = None; - let node = if is_async.is_some() { - ast::StmtKind::AsyncWith { items, body, type_comment } - } else { - ast::StmtKind::With { items, body, type_comment } - }; - ast::Stmt::new(location, end_location, node) - }, -}; - -WithItems: Vec = { - "(" ","? ")", - "(" ",")?> > >)*> ","? ")" => { - left.into_iter().flatten().chain([mid]).chain(right).collect() - }, - > => vec![<>], - > >)+> => { - [item].into_iter().chain(items).collect() - } -}; - -#[inline] -WithItemsNoAs: Vec = { - >> => { - <>.into_iter().map(|context_expr| ast::Withitem { context_expr, optional_vars: None }).collect() - }, -} - -WithItem: ast::Withitem = { - > if Goal != "as" => ast::Withitem { context_expr: <>, optional_vars: None }, - > "as" > => { - let optional_vars = Some(Box::new(set_context(vars, ast::ExprContext::Store))); - ast::Withitem { context_expr, optional_vars } - }, -}; - -FuncDef: ast::Stmt = { - "def" " Test<"all">)?> ":" => { - let args = Box::new(args); - let returns = r.map(|x| Box::new(x.1)); - let end_location = body.last().unwrap().end_location.unwrap(); - let type_comment = None; - let node = if is_async.is_some() { - ast::StmtKind::AsyncFunctionDef { name, args, body, decorator_list, returns, type_comment } - } else { - ast::StmtKind::FunctionDef { name, args, body, decorator_list, returns, type_comment } - }; - ast::Stmt::new(location, end_location, node) - }, -}; - -Parameters: ast::Arguments = { - "(" )?> ")" =>? { - let args = validate_arguments( - a.unwrap_or_else(|| ast::Arguments { - posonlyargs: vec![], - args: vec![], - vararg: None, - kwonlyargs: vec![], - kw_defaults: vec![], - kwarg: None, - defaults: vec![] - }) - )?; - - Ok(args) - } -}; - -// Note that this is a macro which is used once for function defs, and -// once for lambda defs. -ParameterList: ast::Arguments = { - > )?> ","? =>? { - let (posonlyargs, args, defaults) = parse_params(param1)?; - - // Now gather rest of parameters: - let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1); - - Ok(ast::Arguments { - posonlyargs, - args, - kwonlyargs, - vararg, - kwarg, - defaults, - kw_defaults, - }) - }, - > )> ","? =>? { - let (posonlyargs, args, defaults) = parse_params(param1)?; - - // Now gather rest of parameters: - let vararg = None; - let kwonlyargs = vec![]; - let kw_defaults = vec![]; - let kwarg = kw.1; - - Ok(ast::Arguments { - posonlyargs, - args, - kwonlyargs, - vararg, - kwarg, - defaults, - kw_defaults, - }) - }, - > ","? => { - let (vararg, kwonlyargs, kw_defaults, kwarg) = params; - ast::Arguments { - posonlyargs: vec![], - args: vec![], - kwonlyargs, - vararg, - kwarg, - defaults: vec![], - kw_defaults, - } - }, - > ","? => { - ast::Arguments { - posonlyargs: vec![], - args: vec![], - kwonlyargs: vec![], - vararg: None, - kwarg, - defaults: vec![], - kw_defaults: vec![], - } - }, -}; - -// Use inline here to make sure the "," is not creating an ambiguity. -#[inline] -ParameterDefs: (Vec<(ast::Arg, Option)>, Vec<(ast::Arg, Option)>) = { - >> => { - (vec![], args) - }, - >> "," "/" )*> => { - (pos_args, args.into_iter().map(|e| e.1).collect()) - }, -}; - -ParameterDef: (ast::Arg, Option) = { - => (i, None), - "=" > => (i, Some(e)), -}; - -UntypedParameter: ast::Arg = { - => ast::Arg::new( - location, - end_location, - ast::ArgData { arg, annotation: None, type_comment: None }, - ), -}; - -TypedParameter: ast::Arg = { - )?> => { - let annotation = a.map(|x| Box::new(x.1)); - ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None }) - }, -}; - -// Use inline here to make sure the "," is not creating an ambiguity. -// TODO: figure out another grammar that makes this inline no longer required. -#[inline] -ParameterListStarArgs: (Option>, Vec, Vec, Option>) = { - "*" )*> )?> =>? { - // Extract keyword arguments: - let mut kwonlyargs = Vec::new(); - let mut kw_defaults = Vec::new(); - let mut kwargs = Vec::new(); - for (name, value) in kw.into_iter().map(|x| x.1) { - if let Some(value) = value { - kwonlyargs.push(name); - kw_defaults.push(value); - } else { - kwargs.push(name); - } - } - kwargs.extend(kwonlyargs.into_iter()); - - if va.is_none() && kwargs.is_empty() && kwarg.is_none() { - Err(LexicalError { - error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), - location: location, - })? - } - - let kwarg = kwarg.map(|n| n.1).flatten(); - let va = va.map(Box::new); - - Ok((va, kwargs, kw_defaults, kwarg)) - } -}; - -KwargParameter: Option> = { - "**" => { - kwarg.map(Box::new) - } -}; - -ClassDef: ast::Stmt = { - "class" ":" => { - let (bases, keywords) = match a { - Some((_, arg, _)) => (arg.args, arg.keywords), - None => (vec![], vec![]), - }; - let end_location = body.last().unwrap().end_location; - ast::Stmt { - custom: (), - location, - end_location, - node: ast::StmtKind::ClassDef { - name, - bases, - keywords, - body, - decorator_list, - }, - } - }, -}; - -// Decorators: -Decorator: ast::Expr = { - "@" "\n" => { - p - }, -}; - -YieldExpr: ast::Expr = { - "yield" => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Yield { value: value.map(Box::new) } - }, - "yield" "from" > => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::YieldFrom { value: Box::new(e) } - }, -}; - -Test: ast::Expr = { - > "if" > "else" > => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::IfExp { - test: Box::new(test), - body: Box::new(body), - orelse: Box::new(orelse), - } - }, - OrTest, - LambdaDef, -}; - -NamedExpressionTest: ast::Expr = { - NamedExpression, - Test<"all">, -} - -NamedExpression: ast::Expr = { - ":=" > => { - ast::Expr { - location, - end_location: value.end_location, - custom: (), - node: ast::ExprKind::NamedExpr { - target: Box::new(ast::Expr::new( - location, - end_location, - ast::ExprKind::Name { id, ctx: ast::ExprContext::Store }, - )), - value: Box::new(value), - } - } - }, -}; - -LambdaDef: ast::Expr = { - "lambda" ?> ":" > =>? { - let p = validate_arguments( - p.unwrap_or_else(|| { - ast::Arguments { - posonlyargs: vec![], - args: vec![], - vararg: None, - kwonlyargs: vec![], - kw_defaults: vec![], - kwarg: None, - defaults: vec![] - } - } - ))?; - - Ok(ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Lambda { - args: Box::new(p), - body: Box::new(body) - } - }) - } -} - -OrTest: ast::Expr = { - > )+> => { - let mut values = vec![e1]; - values.extend(e2.into_iter().map(|e| e.1)); - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values } - } - }, - AndTest, -}; - -AndTest: ast::Expr = { - > )+> => { - let mut values = vec![e1]; - values.extend(e2.into_iter().map(|e| e.1)); - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values } - } - }, - NotTest, -}; - -NotTest: ast::Expr = { - "not" > => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not } - }, - Comparison, -}; - -Comparison: ast::Expr = { - > )+> => { - let (ops, comparators) = comparisons.into_iter().unzip(); - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Compare { left: Box::new(left), ops, comparators } - } - }, - Expression, -}; - -CompOp: ast::Cmpop = { - "==" => ast::Cmpop::Eq, - "!=" => ast::Cmpop::NotEq, - "<" => ast::Cmpop::Lt, - "<=" => ast::Cmpop::LtE, - ">" => ast::Cmpop::Gt, - ">=" => ast::Cmpop::GtE, - "in" => ast::Cmpop::In, - "not" "in" => ast::Cmpop::NotIn, - "is" => ast::Cmpop::Is, - "is" "not" => ast::Cmpop::IsNot, -}; - -Expression: ast::Expr = { - > "|" > => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) } - }, - XorExpression, -}; - -XorExpression: ast::Expr = { - > "^" > => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) } - }, - AndExpression, -}; - -AndExpression: ast::Expr = { - > "&" > => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) } - }, - ShiftExpression, -}; - -ShiftExpression: ast::Expr = { - > > => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) } - }, - ArithmeticExpression, -}; - -ShiftOp: ast::Operator = { - "<<" => ast::Operator::LShift, - ">>" => ast::Operator::RShift, -}; - -ArithmeticExpression: ast::Expr = { - > > => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) } - }, - Term, -}; - -AddOp: ast::Operator = { - "+" => ast::Operator::Add, - "-" => ast::Operator::Sub, -}; - -Term: ast::Expr = { - > > => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) } - }, - Factor, -}; - -MulOp: ast::Operator = { - "*" => ast::Operator::Mult, - "/" => ast::Operator::Div, - "//" => ast::Operator::FloorDiv, - "%" => ast::Operator::Mod, - "@" => ast::Operator::MatMult, -}; - -Factor: ast::Expr = { - > => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::UnaryOp { operand: Box::new(e), op } - }, - Power, -}; - -UnaryOp: ast::Unaryop = { - "+" => ast::Unaryop::UAdd, - "-" => ast::Unaryop::USub, - "~" => ast::Unaryop::Invert, -}; - -Power: ast::Expr = { - > "**" > => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) } - }, - AtomExpr, -}; - -AtomExpr: ast::Expr = { - "await" > => { - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Await { value: Box::new(atom) } - } - }, - AtomExpr2, -} - -AtomExpr2: ast::Expr = { - Atom, - > "(" ")" => { - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Call { func: Box::new(f), args: a.args, keywords: a.keywords } - } - }, - > "[" "]" => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Subscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load } - }, - > "." => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Attribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load } - }, -}; - -SubscriptList: ast::Expr = { - => { - if s2.is_empty() && trailing_comma.is_none() { - s1 - } else { - let mut dims = vec![s1]; - for x in s2 { - dims.push(x.1) - } - - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Tuple { elts: dims, ctx: ast::ExprContext::Load }, - } - } - } -}; - -Subscript: ast::Expr = { - NamedExpressionTest, - ?> ":" ?> => { - let lower = e1.map(Box::new); - let upper = e2.map(Box::new); - let step = e3.flatten().map(Box::new); - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Slice { lower, upper, step } - } - } -}; - -SliceOp: Option = { - ":" ?> => e, -} - -Atom: ast::Expr = { - =>? Ok(parse_strings(s)?), - => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Constant { value, kind: None } - }, - => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load } - }, - "[" "]" => { - let elts = e.unwrap_or_default(); - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::List { elts, ctx: ast::ExprContext::Load } - } - }, - "[" "]" => { - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::ListComp { elt: Box::new(elt), generators } - } - }, - "(" >> ")" if Goal != "no-withitems" => { - if elts.len() == 1 && trailing_comma.is_none() { - elts.into_iter().next().unwrap() - } else { - ast::Expr::new( - location, - end_location, - ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load }, - ) - } - }, - "(" >> ",")?> )*> ")" =>? { - if left.is_none() && right.is_empty() && trailing_comma.is_none() { - if matches!(mid.node, ast::ExprKind::Starred { .. }) { - Err(LexicalError{ - error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), - location: mid.location, - })? - } - Ok(mid) - } else { - let elts = left.into_iter().flatten().chain([mid]).chain(right).collect(); - Ok(ast::Expr::new( - location, - end_location, - ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load }, - )) - } - }, - "(" ")" => ast::Expr::new( - location, - end_location, - ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load } - ), - "(" ")" => e, - "(" ")" => { - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::GeneratorExp { elt: Box::new(elt), generators } - } - }, - "(" "**" > ")" =>? { - Err(LexicalError{ - error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()), - location: location, - }.into()) - }, - "{" "}" => { - let (keys, values) = e - .unwrap_or_default() - .into_iter() - .map(|(k, v)| (k.map(|x| *x), v)) - .unzip(); - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Dict { keys, values } - } - }, - "{" "}" => { - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::DictComp { - key: Box::new(e1.0), - value: Box::new(e1.1), - generators, - } - } - }, - "{" "}" => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Set { elts } - }, - "{" "}" => { - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::SetComp { elt: Box::new(elt), generators } - } - }, - "True" => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: true.into(), kind: None }), - "False" => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: false.into(), kind: None }), - "None" => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }), - "..." => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }), -}; - -ListLiteralValues: Vec = { - > ","? => e, -}; - -DictLiteralValues: Vec<(Option>, ast::Expr)> = { - > ","? => elements, -}; - -DictEntry: (ast::Expr, ast::Expr) = { - > ":" > => (e1, e2), -}; - -DictElement: (Option>, ast::Expr) = { - => (Some(Box::new(e.0)), e.1), - "**" > => (None, e), -}; - -SetLiteralValues: Vec = { - > ","? => e1 -}; - -ExpressionOrStarExpression = { - Expression<"all">, - StarExpr -}; - -ExpressionList: ast::Expr = { - GenericList -}; - -ExpressionList2: Vec = { - > ","? => elements, -}; - -// A test list is one of: -// - a list of expressions -// - a single expression -// - a single expression followed by a trailing comma -#[inline] -TestList: ast::Expr = { - GenericList -}; - -GenericList: ast::Expr = { - > => { - if elts.len() == 1 && trailing_comma.is_none() { - elts.into_iter().next().unwrap() - } else { - ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load } - } - } - } -} - -// Test -StarExpr: ast::Expr = { - "*" > => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Starred { value: Box::new(e), ctx: ast::ExprContext::Load }, - } -}; - -// Comprehensions: -CompFor: Vec = => c; - -SingleForComprehension: ast::Comprehension = { - "for" "in" > => { - let is_async = is_async.is_some(); - ast::Comprehension { - target: set_context(target, ast::ExprContext::Store), - iter, - ifs, - is_async: if is_async { 1 } else { 0 }, - } - } -}; - -ExpressionNoCond: ast::Expr = OrTest<"all">; -ComprehensionIf: ast::Expr = "if" => c; - -ArgumentList: ArgumentList = { - > =>? { - let arg_list = parse_args(e)?; - Ok(arg_list) - } -}; - -FunctionArgument: (Option<(ast::Location, ast::Location, Option)>, ast::Expr) = { - => { - let expr = match c { - Some(c) => ast::Expr { - location: e.location, - end_location: e.end_location, - custom: (), - node: ast::ExprKind::GeneratorExp { - elt: Box::new(e), - generators: c, - } - }, - None => e, - }; - (None, expr) - }, - "=" > => (Some((location, end_location, Some(i))), e), - "*" > => { - let expr = ast::Expr::new( - location, - end_location, - ast::ExprKind::Starred { value: Box::new(e), ctx: ast::ExprContext::Load }, - ); - (None, expr) - }, - "**" > => (Some((location, end_location, None)), e), -}; - -#[inline] -Comma: Vec = { - ",")*> => { - let mut items = items; - items.extend(last); - items - } -}; - -#[inline] -OneOrMore: Vec = { - => { - let mut items = vec![i1]; - items.extend(i2.into_iter().map(|e| e.1)); - items - } -}; - -Constant: ast::Constant = { - => ast::Constant::Int(value), - => ast::Constant::Float(value), - => ast::Constant::Complex { real: s.0, imag: s.1 }, -}; - -Identifier: String = => s; - -// Hook external lexer: -extern { - type Location = ast::Location; - type Error = LexicalError; - - enum lexer::Tok { - Indent => lexer::Tok::Indent, - Dedent => lexer::Tok::Dedent, - StartModule => lexer::Tok::StartModule, - StartInteractive => lexer::Tok::StartInteractive, - StartExpression => lexer::Tok::StartExpression, - "+" => lexer::Tok::Plus, - "-" => lexer::Tok::Minus, - "~" => lexer::Tok::Tilde, - ":" => lexer::Tok::Colon, - "." => lexer::Tok::Dot, - "..." => lexer::Tok::Ellipsis, - "," => lexer::Tok::Comma, - "*" => lexer::Tok::Star, - "**" => lexer::Tok::DoubleStar, - "&" => lexer::Tok::Amper, - "@" => lexer::Tok::At, - "%" => lexer::Tok::Percent, - "//" => lexer::Tok::DoubleSlash, - "^" => lexer::Tok::CircumFlex, - "|" => lexer::Tok::Vbar, - "<<" => lexer::Tok::LeftShift, - ">>" => lexer::Tok::RightShift, - "/" => lexer::Tok::Slash, - "(" => lexer::Tok::Lpar, - ")" => lexer::Tok::Rpar, - "[" => lexer::Tok::Lsqb, - "]" => lexer::Tok::Rsqb, - "{" => lexer::Tok::Lbrace, - "}" => lexer::Tok::Rbrace, - "=" => lexer::Tok::Equal, - "+=" => lexer::Tok::PlusEqual, - "-=" => lexer::Tok::MinusEqual, - "*=" => lexer::Tok::StarEqual, - "@=" => lexer::Tok::AtEqual, - "/=" => lexer::Tok::SlashEqual, - "%=" => lexer::Tok::PercentEqual, - "&=" => lexer::Tok::AmperEqual, - "|=" => lexer::Tok::VbarEqual, - "^=" => lexer::Tok::CircumflexEqual, - "<<=" => lexer::Tok::LeftShiftEqual, - ">>=" => lexer::Tok::RightShiftEqual, - "**=" => lexer::Tok::DoubleStarEqual, - "//=" => lexer::Tok::DoubleSlashEqual, - ":=" => lexer::Tok::ColonEqual, - "==" => lexer::Tok::EqEqual, - "!=" => lexer::Tok::NotEqual, - "<" => lexer::Tok::Less, - "<=" => lexer::Tok::LessEqual, - ">" => lexer::Tok::Greater, - ">=" => lexer::Tok::GreaterEqual, - "->" => lexer::Tok::Rarrow, - "and" => lexer::Tok::And, - "as" => lexer::Tok::As, - "assert" => lexer::Tok::Assert, - "async" => lexer::Tok::Async, - "await" => lexer::Tok::Await, - "break" => lexer::Tok::Break, - "class" => lexer::Tok::Class, - "continue" => lexer::Tok::Continue, - "def" => lexer::Tok::Def, - "del" => lexer::Tok::Del, - "elif" => lexer::Tok::Elif, - "else" => lexer::Tok::Else, - "except" => lexer::Tok::Except, - "finally" => lexer::Tok::Finally, - "for" => lexer::Tok::For, - "from" => lexer::Tok::From, - "global" => lexer::Tok::Global, - "if" => lexer::Tok::If, - "import" => lexer::Tok::Import, - "in" => lexer::Tok::In, - "is" => lexer::Tok::Is, - "lambda" => lexer::Tok::Lambda, - "nonlocal" => lexer::Tok::Nonlocal, - "not" => lexer::Tok::Not, - "or" => lexer::Tok::Or, - "pass" => lexer::Tok::Pass, - "raise" => lexer::Tok::Raise, - "return" => lexer::Tok::Return, - "try" => lexer::Tok::Try, - "while" => lexer::Tok::While, - "with" => lexer::Tok::With, - "yield" => lexer::Tok::Yield, - "True" => lexer::Tok::True, - "False" => lexer::Tok::False, - "None" => lexer::Tok::None, - int => lexer::Tok::Int { value: }, - float => lexer::Tok::Float { value: }, - complex => lexer::Tok::Complex { real: , imag: }, - string => lexer::Tok::String { - value: , - kind: , - triple_quoted: - }, - name => lexer::Tok::Name { name: }, - "\n" => lexer::Tok::Newline, - ";" => lexer::Tok::Semi, - "#" => lexer::Tok::Comment(_), - } -} diff --git a/compiler/parser/src/context.rs b/compiler/parser/src/context.rs deleted file mode 100644 index f10e1054357..00000000000 --- a/compiler/parser/src/context.rs +++ /dev/null @@ -1,177 +0,0 @@ -use rustpython_ast::{Expr, ExprContext, ExprKind}; - -pub fn set_context(expr: Expr, ctx: ExprContext) -> Expr { - match expr.node { - ExprKind::Name { id, .. } => Expr { - node: ExprKind::Name { id, ctx }, - ..expr - }, - ExprKind::Tuple { elts, .. } => Expr { - node: ExprKind::Tuple { - elts: elts - .into_iter() - .map(|elt| set_context(elt, ctx.clone())) - .collect(), - ctx, - }, - ..expr - }, - ExprKind::List { elts, .. } => Expr { - node: ExprKind::List { - elts: elts - .into_iter() - .map(|elt| set_context(elt, ctx.clone())) - .collect(), - ctx, - }, - ..expr - }, - ExprKind::Attribute { value, attr, .. } => Expr { - node: ExprKind::Attribute { value, attr, ctx }, - ..expr - }, - ExprKind::Subscript { value, slice, .. } => Expr { - node: ExprKind::Subscript { value, slice, ctx }, - ..expr - }, - ExprKind::Starred { value, .. } => Expr { - node: ExprKind::Starred { - value: Box::new(set_context(*value, ctx.clone())), - ctx, - }, - ..expr - }, - _ => expr, - } -} - -#[cfg(test)] -mod tests { - use crate::parser::parse_program; - - #[test] - fn test_assign_name() { - let source = "x = (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_assign_tuple() { - let source = "(x, y) = (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_assign_list() { - let source = "[x, y] = (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_assign_attribute() { - let source = "x.y = (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_assign_subscript() { - let source = "x[y] = (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_assign_starred() { - let source = "(x, *y) = (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_assign_for() { - let source = "for x in (1, 2, 3): pass"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_assign_list_comp() { - let source = "x = [y for y in (1, 2, 3)]"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_assign_set_comp() { - let source = "x = {y for y in (1, 2, 3)}"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_assign_with() { - let source = "with 1 as x: pass"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_assign_named_expr() { - let source = "if x:= 1: pass"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_ann_assign_name() { - let source = "x: int = 1"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_aug_assign_name() { - let source = "x += 1"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_aug_assign_attribute() { - let source = "x.y += (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_aug_assign_subscript() { - let source = "x[y] += (1, 2, 3)"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_del_name() { - let source = "del x"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_del_attribute() { - let source = "del x.y"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_del_subscript() { - let source = "del x[y]"; - let parse_ast = parse_program(source, "").unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } -} diff --git a/compiler/parser/src/error.rs b/compiler/parser/src/error.rs index 4edc02eaee8..6bdeec77e30 100644 --- a/compiler/parser/src/error.rs +++ b/compiler/parser/src/error.rs @@ -2,7 +2,6 @@ //! The goal is to provide a matching and a safe error API, maksing errors from LALR use crate::{ast::Location, token::Tok}; -use lalrpop_util::ParseError as LalrpopError; use std::fmt; /// Represents an error during lexical scanning. @@ -156,17 +155,6 @@ impl fmt::Display for FStringErrorType { } } -impl From for LalrpopError { - fn from(err: FStringError) -> Self { - lalrpop_util::ParseError::User { - error: LexicalError { - error: LexicalErrorType::FStringError(err.error), - location: err.location, - }, - } - } -} - /// Represents an error during parsing pub type ParseError = rustpython_compiler_core::BaseError; @@ -184,59 +172,6 @@ pub enum ParseErrorType { Lexical(LexicalErrorType), } -/// Convert `lalrpop_util::ParseError` to our internal type -pub(crate) fn parse_error_from_lalrpop( - err: LalrpopError, - source_path: &str, -) -> ParseError { - let source_path = source_path.to_owned(); - match err { - // TODO: Are there cases where this isn't an EOF? - LalrpopError::InvalidToken { location } => ParseError { - error: ParseErrorType::Eof, - location, - source_path, - }, - LalrpopError::ExtraToken { token } => ParseError { - error: ParseErrorType::ExtraToken(token.1), - location: token.0, - source_path, - }, - LalrpopError::User { error } => ParseError { - error: ParseErrorType::Lexical(error.error), - location: error.location, - source_path, - }, - LalrpopError::UnrecognizedToken { token, expected } => { - // Hacky, but it's how CPython does it. See PyParser_AddToken, - // in particular "Only one possible expected token" comment. - let expected = (expected.len() == 1).then(|| expected[0].clone()); - ParseError { - error: ParseErrorType::UnrecognizedToken(token.1, expected), - location: token.0.with_col_offset(1), - source_path, - } - } - LalrpopError::UnrecognizedEOF { location, expected } => { - // This could be an initial indentation error that we should ignore - let indent_error = expected == ["Indent"]; - if indent_error { - ParseError { - error: ParseErrorType::Lexical(LexicalErrorType::IndentationError), - location, - source_path, - } - } else { - ParseError { - error: ParseErrorType::Eof, - location, - source_path, - } - } - } - } -} - impl fmt::Display for ParseErrorType { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { diff --git a/compiler/parser/src/function.rs b/compiler/parser/src/function.rs deleted file mode 100644 index 21ccf013b8e..00000000000 --- a/compiler/parser/src/function.rs +++ /dev/null @@ -1,137 +0,0 @@ -use crate::ast; -use crate::error::{LexicalError, LexicalErrorType}; -use rustc_hash::FxHashSet; - -pub struct ArgumentList { - pub args: Vec, - pub keywords: Vec, -} - -type ParameterDefs = (Vec, Vec, Vec); -type ParameterDef = (ast::Arg, Option); - -pub fn validate_arguments(arguments: ast::Arguments) -> Result { - let mut all_args: Vec<&ast::Located> = vec![]; - - all_args.extend(arguments.posonlyargs.iter()); - all_args.extend(arguments.args.iter()); - - if let Some(a) = &arguments.vararg { - all_args.push(a); - } - - all_args.extend(arguments.kwonlyargs.iter()); - - if let Some(a) = &arguments.kwarg { - all_args.push(a); - } - - let mut all_arg_names = FxHashSet::with_hasher(Default::default()); - for arg in all_args { - let arg_name = &arg.node.arg; - if !all_arg_names.insert(arg_name) { - return Err(LexicalError { - error: LexicalErrorType::DuplicateArgumentError(arg_name.to_string()), - location: arg.location, - }); - } - } - - Ok(arguments) -} - -pub fn parse_params( - params: (Vec, Vec), -) -> Result { - let mut pos_only = Vec::with_capacity(params.0.len()); - let mut names = Vec::with_capacity(params.1.len()); - let mut defaults = vec![]; - - let mut try_default = |name: &ast::Arg, default| { - if let Some(default) = default { - defaults.push(default); - } else if !defaults.is_empty() { - // Once we have started with defaults, all remaining arguments must - // have defaults - return Err(LexicalError { - error: LexicalErrorType::DefaultArgumentError, - location: name.location, - }); - } - Ok(()) - }; - - for (name, default) in params.0 { - try_default(&name, default)?; - pos_only.push(name); - } - - for (name, default) in params.1 { - try_default(&name, default)?; - names.push(name); - } - - Ok((pos_only, names, defaults)) -} - -type FunctionArgument = ( - Option<(ast::Location, ast::Location, Option)>, - ast::Expr, -); - -pub fn parse_args(func_args: Vec) -> Result { - let mut args = vec![]; - let mut keywords = vec![]; - - let mut keyword_names = - FxHashSet::with_capacity_and_hasher(func_args.len(), Default::default()); - let mut double_starred = false; - for (name, value) in func_args { - match name { - Some((start, end, name)) => { - if let Some(keyword_name) = &name { - if keyword_names.contains(keyword_name) { - return Err(LexicalError { - error: LexicalErrorType::DuplicateKeywordArgumentError( - keyword_name.to_string(), - ), - location: start, - }); - } - - keyword_names.insert(keyword_name.clone()); - } else { - double_starred = true; - } - - keywords.push(ast::Keyword::new( - start, - end, - ast::KeywordData { arg: name, value }, - )); - } - None => { - // Allow starred arguments after keyword arguments but - // not after double-starred arguments. - if !keywords.is_empty() && !is_starred(&value) { - return Err(LexicalError { - error: LexicalErrorType::PositionalArgumentError, - location: value.location, - }); - } else if double_starred { - return Err(LexicalError { - error: LexicalErrorType::UnpackedArgumentError, - location: value.location, - }); - } - - args.push(value); - } - } - } - Ok(ArgumentList { args, keywords }) -} - -fn is_starred(exp: &ast::Expr) -> bool { - matches!(exp.node, ast::ExprKind::Starred { .. }) -} diff --git a/compiler/parser/src/lib.rs b/compiler/parser/src/lib.rs index ce9dde1d129..c3a6365fac2 100644 --- a/compiler/parser/src/lib.rs +++ b/compiler/parser/src/lib.rs @@ -23,13 +23,10 @@ extern crate log; pub use rustpython_ast as ast; pub mod error; -mod function; pub mod lexer; pub mod mode; pub mod parser; mod string_parser; -#[rustfmt::skip] -mod python; -mod context; mod string; pub mod token; +pub mod peg_parser; diff --git a/compiler/parser/src/mode.rs b/compiler/parser/src/mode.rs index cd84a098ace..38a7341893d 100644 --- a/compiler/parser/src/mode.rs +++ b/compiler/parser/src/mode.rs @@ -1,22 +1,10 @@ -use crate::token::Tok; - -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy)] pub enum Mode { Module, Interactive, Expression, } -impl Mode { - pub(crate) fn to_marker(self) -> Tok { - match self { - Self::Module => Tok::StartModule, - Self::Interactive => Tok::StartInteractive, - Self::Expression => Tok::StartExpression, - } - } -} - impl From for Mode { fn from(mode: rustpython_compiler_core::Mode) -> Self { use rustpython_compiler_core::Mode as CompileMode; diff --git a/compiler/parser/src/parser.rs b/compiler/parser/src/parser.rs index 8abbcacf71c..c075a0d6015 100644 --- a/compiler/parser/src/parser.rs +++ b/compiler/parser/src/parser.rs @@ -5,12 +5,11 @@ //! parse a whole program, a single statement, or a single //! expression. -use crate::lexer::{LexResult, Tok}; +use crate::error::ParseErrorType; pub use crate::mode::Mode; -use crate::{ast, error::ParseError, lexer, python}; +use crate::{ast, error::ParseError, lexer}; +use crate::{lexer::LexResult, peg_parser}; use ast::Location; -use itertools::Itertools; -use std::iter; /* * Parse python code. @@ -102,14 +101,12 @@ pub fn parse_tokens( mode: Mode, source_path: &str, ) -> Result { - let marker_token = (Default::default(), mode.to_marker(), Default::default()); - let tokenizer = iter::once(Ok(marker_token)) - .chain(lxr) - .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)); - - python::TopParser::new() - .parse(tokenizer) - .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path)) + let parser = peg_parser::Parser::from(lxr).map_err(|e| ParseError { + error: ParseErrorType::Lexical(e.error), + location: e.location, + source_path: source_path.to_owned(), + })?; + parser.parse(mode, source_path) } #[cfg(test)] @@ -122,6 +119,12 @@ mod tests { insta::assert_debug_snapshot!(parse_ast); } + #[test] + fn test_parse_pass() { + let parse_ast = parse_program("pass", "").unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + #[test] fn test_parse_string() { let source = "'Hello world'"; @@ -157,6 +160,20 @@ mod tests { insta::assert_debug_snapshot!(parse_ast); } + #[test] + fn test_parse_kwds() { + let source = "func(token, serializer, incref=incref, **kwds)"; + let parse_ast = parse_program(source, "").unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_assigment() { + let source = "[] = gen_b"; + let parse_ast = parse_program(source, "").unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + #[test] fn test_parse_if_elif_else() { let source = "if 1: 10\nelif 2: 20\nelse: 30"; @@ -315,4 +332,12 @@ with (0 as a, 1 as b,): pass let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "").unwrap(); insta::assert_debug_snapshot!(parse_ast); } + + #[test] + fn test_pyfile() { + let fname = "/home/skz/Projects/cpython/Tools/c-analyzer/c_parser/parser/_delim.py"; + let source = std::fs::read_to_string(fname).unwrap(); + let parse_ast = parse_program(&source, fname).unwrap(); + dbg!(parse_ast); + } } diff --git a/compiler/parser/src/peg_parser.rs b/compiler/parser/src/peg_parser.rs new file mode 100644 index 00000000000..3fd0466d515 --- /dev/null +++ b/compiler/parser/src/peg_parser.rs @@ -0,0 +1,1475 @@ +use crate::{ + ast::{self, Located, Location}, + error::{LexicalError, ParseErrorType}, + lexer::LexResult, + mode::Mode, + token::{StringKind, Tok}, +}; +use num_bigint::BigInt; + +#[derive(Debug, Clone)] +pub struct Parser { + tokens: Vec, + locations: Vec<(Location, Location)>, + names: Vec, + ints: Vec, + floats: Vec, + complexes: Vec<(f64, f64)>, + strings: Vec<(String, StringKind, bool)>, + // comments: Vec, +} + +impl Parser { + pub fn from(lexer: impl IntoIterator) -> Result { + let mut tokens = vec![]; + let mut locations = vec![]; + let mut names = vec![]; + let mut ints = vec![]; + let mut floats = vec![]; + let mut complexes = vec![]; + let mut strings = vec![]; + // let mut comments = vec![]; + + for tok in lexer { + let (begin, tok, end) = tok?; + let tok = match tok { + Tok::Name { name } => { + names.push(name); + PegTok::Name((names.len() - 1) as u32) + } + Tok::Int { value } => { + ints.push(value); + PegTok::Int((ints.len() - 1) as u32) + } + Tok::Float { value } => { + floats.push(value); + PegTok::Float((floats.len() - 1) as u32) + } + Tok::Complex { real, imag } => { + complexes.push((real, imag)); + PegTok::Complex((complexes.len() - 1) as u32) + } + Tok::String { + value, + kind, + triple_quoted, + } => { + strings.push((value, kind, triple_quoted)); + PegTok::String((strings.len() - 1) as u32) + } + Tok::Newline => PegTok::Newline, + Tok::Indent => PegTok::Indent, + Tok::Dedent => PegTok::Dedent, + Tok::EndOfFile => PegTok::EndOfFile, + Tok::Lpar => PegTok::Lpar, + Tok::Rpar => PegTok::Rpar, + Tok::Lsqb => PegTok::Lsqb, + Tok::Rsqb => PegTok::Rsqb, + Tok::Colon => PegTok::Colon, + Tok::Comma => PegTok::Comma, + Tok::Semi => PegTok::Semi, + Tok::Plus => PegTok::Plus, + Tok::Minus => PegTok::Minus, + Tok::Star => PegTok::Star, + Tok::Slash => PegTok::Slash, + Tok::Vbar => PegTok::Vbar, + Tok::Amper => PegTok::Amper, + Tok::Less => PegTok::Less, + Tok::Greater => PegTok::Greater, + Tok::Equal => PegTok::Equal, + Tok::Dot => PegTok::Dot, + Tok::Percent => PegTok::Percent, + Tok::Lbrace => PegTok::Lbrace, + Tok::Rbrace => PegTok::Rbrace, + Tok::EqEqual => PegTok::EqEqual, + Tok::NotEqual => PegTok::NotEqual, + Tok::LessEqual => PegTok::LessEqual, + Tok::GreaterEqual => PegTok::GreaterEqual, + Tok::Tilde => PegTok::Tilde, + Tok::CircumFlex => PegTok::CircumFlex, + Tok::LeftShift => PegTok::LeftShift, + Tok::RightShift => PegTok::RightShift, + Tok::DoubleStar => PegTok::DoubleStar, + Tok::DoubleStarEqual => PegTok::DoubleStarEqual, + Tok::PlusEqual => PegTok::PlusEqual, + Tok::MinusEqual => PegTok::MinusEqual, + Tok::StarEqual => PegTok::StarEqual, + Tok::SlashEqual => PegTok::SlashEqual, + Tok::PercentEqual => PegTok::PercentEqual, + Tok::AmperEqual => PegTok::AmperEqual, + Tok::VbarEqual => PegTok::VbarEqual, + Tok::CircumflexEqual => PegTok::CircumflexEqual, + Tok::LeftShiftEqual => PegTok::LeftShiftEqual, + Tok::RightShiftEqual => PegTok::RightShiftEqual, + Tok::DoubleSlash => PegTok::DoubleSlash, + Tok::DoubleSlashEqual => PegTok::DoubleSlashEqual, + Tok::ColonEqual => PegTok::ColonEqual, + Tok::At => PegTok::At, + Tok::AtEqual => PegTok::AtEqual, + Tok::Rarrow => PegTok::Rarrow, + Tok::Ellipsis => PegTok::Ellipsis, + Tok::False => PegTok::False, + Tok::None => PegTok::None, + Tok::True => PegTok::True, + Tok::And => PegTok::And, + Tok::As => PegTok::As, + Tok::Assert => PegTok::Assert, + Tok::Async => PegTok::Async, + Tok::Await => PegTok::Await, + Tok::Break => PegTok::Break, + Tok::Class => PegTok::Class, + Tok::Continue => PegTok::Continue, + Tok::Def => PegTok::Def, + Tok::Del => PegTok::Del, + Tok::Elif => PegTok::Elif, + Tok::Else => PegTok::Else, + Tok::Except => PegTok::Except, + Tok::Finally => PegTok::Finally, + Tok::For => PegTok::For, + Tok::From => PegTok::From, + Tok::Global => PegTok::Global, + Tok::If => PegTok::If, + Tok::Import => PegTok::Import, + Tok::In => PegTok::In, + Tok::Is => PegTok::Is, + Tok::Lambda => PegTok::Lambda, + Tok::Nonlocal => PegTok::Nonlocal, + Tok::Not => PegTok::Not, + Tok::Or => PegTok::Or, + Tok::Pass => PegTok::Pass, + Tok::Raise => PegTok::Raise, + Tok::Return => PegTok::Return, + Tok::Try => PegTok::Try, + Tok::While => PegTok::While, + Tok::With => PegTok::With, + Tok::Yield => PegTok::Yield, + _ => continue, + }; + tokens.push(tok); + locations.push((begin, end)); + } + + Ok(Self { + tokens, + locations, + names, + ints, + floats, + complexes, + strings, + // comments, + }) + } + + fn _parse(&self, mode: Mode) -> Result> { + match mode { + Mode::Module => python_parser::file(self, self), + Mode::Interactive => python_parser::interactive(self, self), + Mode::Expression => python_parser::eval(self, self), + } + } + + pub fn parse( + &self, + mode: Mode, + source_path: &str, + ) -> Result { + self._parse(mode).map_err(|e| crate::error::ParseError { + error: e.location.error, + location: e.location.location, + source_path: source_path.to_owned(), + }) + } + + fn new_located(&self, begin: usize, end: usize, node: T) -> Located { + assert!(begin < end); + let location = self.locations[begin].0; + let end_location = self.locations[end - 1].1; + Located::new(location, end_location, node) + } + + fn new_located_single(&self, tok_pos: usize, node: T) -> Located { + let loc = self.locations[tok_pos]; + Located::new(loc.0, loc.1, node) + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum PegTok { + Name(u32), + Int(u32), + Float(u32), + Complex(u32), + String(u32), + Newline, + // NonLogicalNewline, + Indent, + Dedent, + // StartModule, + // StartInteractive, + // StartExpression, + EndOfFile, + Lpar, + Rpar, + Lsqb, + Rsqb, + Colon, + Comma, + // Comment(u32), + Semi, + Plus, + Minus, + Star, + Slash, + Vbar, // '|' + Amper, // '&' + Less, + Greater, + Equal, + Dot, + Percent, + Lbrace, + Rbrace, + EqEqual, + NotEqual, + LessEqual, + GreaterEqual, + Tilde, + CircumFlex, + LeftShift, + RightShift, + DoubleStar, + DoubleStarEqual, // '**=' + PlusEqual, + MinusEqual, + StarEqual, + SlashEqual, + PercentEqual, + AmperEqual, // '&=' + VbarEqual, + CircumflexEqual, // '^=' + LeftShiftEqual, + RightShiftEqual, + DoubleSlash, // '//' + DoubleSlashEqual, + ColonEqual, + At, + AtEqual, + Rarrow, + Ellipsis, + + // Keywords (alphabetically): + False, + None, + True, + + And, + As, + Assert, + Async, + Await, + Break, + Class, + Continue, + Def, + Del, + Elif, + Else, + Except, + Finally, + For, + From, + Global, + If, + Import, + In, + Is, + Lambda, + Nonlocal, + Not, + Or, + Pass, + Raise, + Return, + Try, + While, + With, + Yield, +} + +pub struct PegParseError { + location: Location, + error: ParseErrorType, +} + +impl std::fmt::Display for PegParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.location.fmt_with(f, &self.error) + } +} + +impl peg::Parse for Parser { + type PositionRepr = PegParseError; + + fn start<'input>(&'input self) -> usize { + 0 + } + + fn is_eof<'input>(&'input self, p: usize) -> bool { + p >= self.tokens.len() + } + + fn position_repr<'input>(&'input self, p: usize) -> Self::PositionRepr { + // format!("source: {}, p: {}, loc: {:?}, curr: {}", &self.source_path, p, self.locations[p], self.tokens[p]) + if self.is_eof(p) { + PegParseError { + location: Default::default(), + error: ParseErrorType::Eof, + } + } else { + // TODO: UnrecognizedToken + // ParseErrorType::InvalidToken + let tok = self.tokens[p]; + let tok = match tok { + PegTok::Name(id) => Tok::Name { + name: self.names[id as usize].clone(), + }, + PegTok::Int(id) => Tok::Int { + value: self.ints[id as usize].clone(), + }, + PegTok::Float(id) => Tok::Float { + value: self.floats[id as usize].clone(), + }, + PegTok::Complex(id) => { + let value = self.complexes[id as usize]; + Tok::Complex { + real: value.0, + imag: value.1, + } + } + PegTok::String(id) => { + let value = self.strings[id as usize].clone(); + Tok::String { + value: value.0, + kind: value.1, + triple_quoted: value.2, + } + } + PegTok::Newline => Tok::Newline, + PegTok::Indent => Tok::Indent, + PegTok::Dedent => Tok::Dedent, + PegTok::EndOfFile => Tok::EndOfFile, + PegTok::Lpar => Tok::Lpar, + PegTok::Rpar => Tok::Rpar, + PegTok::Lsqb => Tok::Lsqb, + PegTok::Rsqb => Tok::Rsqb, + PegTok::Colon => Tok::Colon, + PegTok::Comma => Tok::Comma, + PegTok::Semi => Tok::Semi, + PegTok::Plus => Tok::Plus, + PegTok::Minus => Tok::Minus, + PegTok::Star => Tok::Star, + PegTok::Slash => Tok::Slash, + PegTok::Vbar => Tok::Vbar, + PegTok::Amper => Tok::Amper, + PegTok::Less => Tok::Less, + PegTok::Greater => Tok::Greater, + PegTok::Equal => Tok::Equal, + PegTok::Dot => Tok::Dot, + PegTok::Percent => Tok::Percent, + PegTok::Lbrace => Tok::Lbrace, + PegTok::Rbrace => Tok::Rbrace, + PegTok::EqEqual => Tok::EqEqual, + PegTok::NotEqual => Tok::NotEqual, + PegTok::LessEqual => Tok::LessEqual, + PegTok::GreaterEqual => Tok::GreaterEqual, + PegTok::Tilde => Tok::Tilde, + PegTok::CircumFlex => Tok::CircumFlex, + PegTok::LeftShift => Tok::LeftShift, + PegTok::RightShift => Tok::RightShift, + PegTok::DoubleStar => Tok::DoubleStar, + PegTok::DoubleStarEqual => Tok::DoubleStarEqual, + PegTok::PlusEqual => Tok::PlusEqual, + PegTok::MinusEqual => Tok::MinusEqual, + PegTok::StarEqual => Tok::StarEqual, + PegTok::SlashEqual => Tok::SlashEqual, + PegTok::PercentEqual => Tok::PercentEqual, + PegTok::AmperEqual => Tok::AmperEqual, + PegTok::VbarEqual => Tok::VbarEqual, + PegTok::CircumflexEqual => Tok::CircumflexEqual, + PegTok::LeftShiftEqual => Tok::LeftShiftEqual, + PegTok::RightShiftEqual => Tok::RightShiftEqual, + PegTok::DoubleSlash => Tok::DoubleSlash, + PegTok::DoubleSlashEqual => Tok::DoubleSlashEqual, + PegTok::ColonEqual => Tok::ColonEqual, + PegTok::At => Tok::At, + PegTok::AtEqual => Tok::AtEqual, + PegTok::Rarrow => Tok::Rarrow, + PegTok::Ellipsis => Tok::Ellipsis, + PegTok::False => Tok::False, + PegTok::None => Tok::None, + PegTok::True => Tok::True, + PegTok::And => Tok::And, + PegTok::As => Tok::As, + PegTok::Assert => Tok::Assert, + PegTok::Async => Tok::Async, + PegTok::Await => Tok::Await, + PegTok::Break => Tok::Break, + PegTok::Class => Tok::Class, + PegTok::Continue => Tok::Continue, + PegTok::Def => Tok::Def, + PegTok::Del => Tok::Del, + PegTok::Elif => Tok::Elif, + PegTok::Else => Tok::Else, + PegTok::Except => Tok::Except, + PegTok::Finally => Tok::Finally, + PegTok::For => Tok::For, + PegTok::From => Tok::From, + PegTok::Global => Tok::Global, + PegTok::If => Tok::If, + PegTok::Import => Tok::Import, + PegTok::In => Tok::In, + PegTok::Is => Tok::Is, + PegTok::Lambda => Tok::Lambda, + PegTok::Nonlocal => Tok::Nonlocal, + PegTok::Not => Tok::Not, + PegTok::Or => Tok::Or, + PegTok::Pass => Tok::Pass, + PegTok::Raise => Tok::Raise, + PegTok::Return => Tok::Return, + PegTok::Try => Tok::Try, + PegTok::While => Tok::While, + PegTok::With => Tok::With, + PegTok::Yield => Tok::Yield, + }; + PegParseError { + location: self.locations[p].0, + error: ParseErrorType::UnrecognizedToken(tok, None), + } + } + } +} + +impl<'input> peg::ParseElem<'input> for Parser { + type Element = PegTok; + + fn parse_elem(&'input self, pos: usize) -> peg::RuleResult { + match self.tokens.get(pos) { + Some(tok) => peg::RuleResult::Matched(pos + 1, *tok), + None => peg::RuleResult::Failed, + } + } +} + +impl<'input> peg::ParseSlice<'input> for Parser { + type Slice = &'input [PegTok]; + + fn parse_slice(&'input self, p1: usize, p2: usize) -> Self::Slice { + &self.tokens[p1..p2] + } +} + +peg::parser! { grammar python_parser(zelf: &Parser) for Parser { + +use PegTok::*; +use crate::token::StringKind; +use ast::{ + Expr, Stmt, ExprKind, StmtKind, ExprContext, Withitem, Cmpop, Keyword, KeywordData, Comprehension, + Operator, Excepthandler, ExcepthandlerKind, Arguments, Arg, ArgData +}; +use std::option::Option::{Some, None}; +use std::string::String; + +pub rule file() -> ast::Mod + = a:(statements() / {vec![]}) { ast::Mod::Module { body: a, type_ignores: vec![] } } + +pub rule interactive() -> ast::Mod + = a:statement() { ast::Mod::Interactive { body: a } } + +pub rule eval() -> ast::Mod + = a:expressions() [Newline]* { ast::Mod::Expression { body: Box::new(a) } } + +// TODO: +// pub rule func_type() -> ast::Mod +// = [Lpar] a:type_expressions() + +pub rule fstring() -> Expr = star_expressions() + +rule statements() -> Vec = a:statement()+ { a.into_iter().flatten().collect() } + +rule statement() -> Vec + = a:compound_stmt() { vec![a] } + / simple_stmts() + +rule statement_newline() -> Vec + = a:compound_stmt() [Newline] { vec![a] } + / simple_stmts() + / begin:position!() [Newline] { vec![zelf.new_located_single(begin, StmtKind::Pass)] } + / [EndOfFile] {? Err("unexpected EOF") } + +rule simple_stmts() -> Vec + = a:simple_stmt() ![Semi] [Newline] { vec![a] } + / a:simple_stmt() ++ [Semi] [Semi]? [Newline] {a} + +#[cache] +rule simple_stmt() -> Stmt + = assignment() + / loc() + / &[Return] a:return_stmt() {a} + / &[Import | From] a:import_stmt() {a} + / &[Raise] a:raise_stmt() {a} + / loc(<[Pass] { StmtKind::Pass }>) + / &[Del] a:del_stmt() {a} + / &[Yield] a:yield_stmt() {a} + / &[Assert] a:assert_stmt() {a} + / loc(<[Break] { StmtKind::Break }>) + / loc(<[Continue] { StmtKind::Continue }>) + / &[Global] a:global_stmt() {a} + / &[Nonlocal] a:nonlocal_stmt() {a} + +rule compound_stmt() -> Stmt = + &[Def | At | Async] a:function_def() {a} / + &[If] a:if_stmt() {a} / + &[Class | At] a:class_def() {a} / + &[With | Async] a:with_stmt() {a} / + &[For | Async] a:for_stmt() {a} / + &[Try] a:try_stmt() {a} / + &[While] a:while_stmt() {a} + // TODO: + // match_stmt() + +rule assignment() -> Stmt = + loc() / + loc() / single_subscript_attribute_target(ExprContext::Store)) + [Colon] b:expression() c:([Equal] z:annotated_rhs() {z})? { + StmtKind::AnnAssign { target: Box::new(a), annotation: Box::new(b), value: option_box(c), simple: 0 } + }>) / + loc() / + loc() + +rule annotated_rhs() -> Expr = yield_expr() / star_expressions() + +rule augassign() -> Operator = + [PlusEqual] { Operator::Add } / + [MinusEqual] { Operator::Sub } / + [StarEqual] { Operator::Mult } / + [AtEqual] { Operator::MatMult } / + [SlashEqual] { Operator::Div } / + [PercentEqual] { Operator::Mod } / + [AmperEqual] { Operator::BitAnd } / + [VbarEqual] { Operator::BitOr } / + [CircumflexEqual] { Operator::BitXor } / + [LeftShiftEqual] { Operator::LShift } / + [RightShiftEqual] { Operator::RShift } / + [DoubleStarEqual] { Operator::Pow } / + [DoubleSlashEqual] { Operator::FloorDiv } + +rule return_stmt() -> Stmt = loc(<[Return] a:star_expressions()? { + StmtKind::Return { value: option_box(a) } +}>) + +rule raise_stmt() -> Stmt = + loc(<[Raise] a:expression() b:([From] z:expression() {z})? { + StmtKind::Raise { exc: Some(Box::new(a)), cause: option_box(b) } + }>) / + loc(<[Raise] { + StmtKind::Raise { exc: None, cause: None } + }>) + +rule global_stmt() -> Stmt = loc(<[Global] names:name() ++ [Comma] { + StmtKind::Global { names } +}>) + +rule nonlocal_stmt() -> Stmt = loc(<[Nonlocal] names:name() ++ [Comma] { + StmtKind::Nonlocal { names } +}>) + +rule del_stmt() -> Stmt = loc(<[Del] a:del_targets() &[Semi | Newline] { + StmtKind::Delete { targets: a } +}>) + +rule yield_stmt() -> Stmt = loc() + +rule assert_stmt() -> Stmt = loc(<[Assert] a:expression() b:([Comma] z:expression() {z})? { + StmtKind::Assert { test: Box::new(a), msg: option_box(b) } +}>) + +rule import_stmt() -> Stmt = import_name() / import_from() + +rule import_name() -> Stmt = loc(<[Import] a:dotted_as_names() { + StmtKind::Import { names: a } +}>) +rule import_from() -> Stmt = + loc(<[From] a:[Dot | Ellipsis]* b:dotted_name() [Import] c:import_from_targets() { + StmtKind::ImportFrom { module: Some(b), names: c, level: count_dots(a) } + }>) / + loc(<[From] a:[Dot | Ellipsis]+ [Import] b:import_from_targets() { + StmtKind::ImportFrom { module: None, names: b, level: count_dots(a) } + }>) +rule import_from_targets() -> Vec = + par() / + a:import_from_as_names() ![Comma] {a} / + a:loc(<[Star] { + ast::AliasData { name: "*".to_owned(), asname: None } + }>) { vec![a] } +rule import_from_as_names() -> Vec = import_from_as_name() ++ [Comma] +rule import_from_as_name() -> ast::Alias = loc() +rule dotted_as_names() -> Vec = dotted_as_name() ++ [Comma] +rule dotted_as_name() -> ast::Alias = loc() + +#[cache_left_rec] +rule dotted_name() -> String = + a:dotted_name() [Dot] b:name() { + format!("{}.{}", a, b) + } / + name() + +#[cache] +rule block() -> Vec + = [Newline] [Indent] a:statements() [Dedent] {a} + / simple_stmts() + +rule decorator() -> Expr = [At] f:named_expression() [Newline] {f} + +rule class_def() -> Stmt = + dec:decorator()* begin:position!() [Class] name:name() arg:par()? [Colon] b:block() end:block_end() { + let (bases, keywords) = arg.flatten().unwrap_or_default(); + let stmt = StmtKind::ClassDef { name, bases, keywords, body: b, decorator_list: dec }; + zelf.new_located(begin, end, stmt) + } + +rule function_def() -> Stmt = + dec:decorator()* begin:position!() is_async:[Async]? [Def] name:name() p:par() + r:([Rarrow] z:expression() {z})? [Colon] tc:func_type_comment() b:block() end:block_end() { + let stmt = if is_async.is_none() { + StmtKind::FunctionDef { name, args: Box::new(p.unwrap_or_else(make_empty_arguments)), body: b, decorator_list: dec, returns: option_box(r), type_comment: tc } + } else { + StmtKind::AsyncFunctionDef { name, args: Box::new(p.unwrap_or_else(make_empty_arguments)), body: b, decorator_list: dec, returns: option_box(r), type_comment: tc } + }; + zelf.new_located(begin, end, stmt) + } + +rule params() -> Arguments = parameters() + +rule parameters() -> Arguments = + a:slash_no_default() c:param_no_default()* d:param_with_default()* e:star_etc()? { + make_arguments(a, Default::default(), c, d, e) + } / + b:slash_with_default() d:param_with_default()* e:star_etc()? { + make_arguments(vec![], b, vec![], d, e) + } / + c:param_no_default()+ d:param_with_default()* e:star_etc()? { + make_arguments(vec![], Default::default(), c, d, e) + } / + d:param_with_default()+ e:star_etc()? { + make_arguments(vec![], Default::default(), vec![], d, e) + } / + e:star_etc() { + make_arguments(vec![], Default::default(), vec![], vec![], Some(e)) + } + +rule slash_no_default() -> Vec = a:param_no_default()+ [Slash] param_split() {a} +rule slash_with_default() -> (Vec, Vec<(Arg, Expr)>) = + a:param_no_default()* b:param_with_default()+ [Slash] param_split() {(a, b)} + +rule star_etc() -> (Option, Vec<(Arg, Option)>, Option) = + [Star] a:param_no_default() b:param_maybe_default()* c:kwds()? { + (Some(a), b, c) + } / + [Star] a:param_no_default_star_annotation() b:param_maybe_default()* c:kwds()? { + (Some(a), b, c) + } / + [Star] [Comma] b:param_maybe_default()+ c:kwds()? { + (None, b, c) + } / + c:kwds() { + (None, vec![], Some(c)) + } + +rule kwds() -> Arg = [DoubleStar] a:param_no_default() {a} + +// TODO: type_comment +rule param_no_default() -> Arg = a:param() param_split() {a} +rule param_no_default_star_annotation() -> Arg = a:param_star_annotation() param_split() {a} +rule param_with_default() -> (Arg, Expr) = a:param() c:default() param_split() {(a, c)} +rule param_maybe_default() -> (Arg, Option) = a:param() c:default()? param_split() {(a, c)} +rule param() -> Arg = + loc() +rule param_star_annotation() -> Arg = + loc() +rule annotation() -> Expr = [Colon] a:expression() {a} +rule star_annotation() -> Expr = [Colon] a:star_annotation() {a} +rule default() -> Expr = [Equal] a:expression() {a} +rule param_split() = [Comma] / &[Rpar] + +rule if_stmt() -> Stmt + = begin:position!() [If] a:named_expression() [Colon] b:block() c:( + z:elif_stmt() {vec![z]} / else_block() / {vec![]} + ) end:block_end() { + zelf.new_located(begin, end, StmtKind::If { test: Box::new(a), body: b, orelse: c }) + } + +rule elif_stmt() -> Stmt + = begin:position!() [Elif] a:named_expression() [Colon] b:block() end:block_end() c:( + z:elif_stmt() {vec![z]} / else_block() / {vec![]} + ) { + zelf.new_located(begin, end, StmtKind::If { test: Box::new(a), body: b, orelse: c }) + } + +rule else_block() -> Vec = [Else] [Colon] b:block() {b} +rule else_block_opt() -> Vec = else_block() / {vec![]} + +rule while_stmt() -> Stmt = + loc_block_end(<[While] a:named_expression() [Colon] b:block() c:else_block_opt() { + StmtKind::While { test: Box::new(a), body: b, orelse: c } + }>) + +rule for_stmt() -> Stmt = + loc_block_end() + +rule with_stmt() -> Stmt = + loc_block_end() [Colon] b:block() { + if is_async.is_none() { + StmtKind::With { items: a, body: b, type_comment: None } + } else { + StmtKind::AsyncWith { items: a, body: b, type_comment: None } + } + }>) / + loc_block_end() + +rule with_item() -> Withitem = + e:expression() vars:([As] t:star_target() &[Comma | Rpar | Colon] {t})? { + Withitem { context_expr: e, optional_vars: option_box(vars) } + } + +rule try_stmt() -> Stmt + = loc_block_end(<[Try] [Colon] b:block() x:( + f:finally_block() {(vec![], vec![], f)} + / ex:except_block()+ el:else_block_opt() f:(finally_block() / {vec![]}) {(ex, el, f)} + ) { + StmtKind::Try { body: b, handlers: x.0, orelse: x.1, finalbody: x.2 } + }>) + // TODO: except star + // loc(<[Try] [Colon] b:block() ex:except_star_block()+ el:else_block_opt() f:finally_block() { + // StmtKind::{ body: b, handlers: ex, orelse: el, finalbody: f } + // }>) + +rule except_block() -> Excepthandler = + loc_block_end(<[Except] e:expression() t:([As] z:name() {z})? [Colon] b:block() { + ExcepthandlerKind::ExceptHandler { type_: Some(Box::new(e)), name: t, body: b } + }>) / + loc_block_end(<[Except] [Colon] b:block() { + ExcepthandlerKind::ExceptHandler { type_: None, name: None, body: b } + }>) +rule except_star_block() -> Excepthandler = + loc_block_end(<[Except] [Star] e:expression() t:([As] z:name() {z})? [Colon] b:block() { + ExcepthandlerKind::ExceptHandler { type_: Some(Box::new(e)), name: t, body: b } + }>) +rule finally_block() -> Vec = [Finally] [Colon] b:block() {b} + +// rule match_stmt() -> Stmt = +// [Match] + +rule expressions() -> Expr = pack_tuple_expr(, ExprContext::Load) + +rule expression() -> Expr = + loc() / + disjunction() / + lambdef() + +rule yield_expr() -> Expr = + loc(<[Yield] [From] a:expression() { + ExprKind::YieldFrom { value: Box::new(a) } + }>) / + loc(<[Yield] a:star_expressions()? { + ExprKind::Yield { value: option_box(a) } + }>) + +rule star_expressions() -> Expr = pack_tuple_expr(, ExprContext::Load) + +rule star_expression() -> Expr = + loc(<[Star] a:bitwise_or() { + ExprKind::Starred { value: Box::new(a), ctx: ExprContext::Load } + }>) / + expression() + +rule star_named_expressions() -> Vec = + a:star_named_expression() ++ [Comma] [Comma]? {a} + +rule star_named_expression() -> Expr = + loc(<[Star] a:bitwise_or() { + ExprKind::Starred { value: Box::new(a), ctx: ExprContext::Load } + }>) / + named_expression() + +rule assignment_expression() -> Expr = + loc() + +rule named_expression() -> Expr = + assignment_expression() / + a:expression() ![ColonEqual] {a} + +// #[cache] +// rule disjunction() -> Expr = +// loc( [Or] { +// ExprKind::BoolOp { op: ast::Boolop::Or, values: a } +// }>) / +// conjunction() +#[cache] +rule disjunction() -> Expr + = begin:position!() a:conjunction() v:([Or] z:conjunction() ++ [Or] {z})? end:position!() { + if let Some(v) = v { + zelf.new_located(begin, end, ExprKind::BoolOp { op: ast::Boolop::Or, values: insert_front(v, a) }) + } else { a } + } + +#[cache] +rule conjunction() -> Expr + = begin:position!() a:inversion() v:([And] z:inversion() ++ [And] {z})? end:position!() { + if let Some(v) = v { + zelf.new_located(begin, end, ExprKind::BoolOp { op: ast::Boolop::And, values: insert_front(v, a) }) + } else { a } + } + +#[cache] +rule inversion() -> Expr = + loc(<[Not] a:inversion() { + ExprKind::UnaryOp { op: ast::Unaryop::Not, operand: Box::new(a) } + }>) / + comparison() + +#[cache] +rule comparison() -> Expr + = begin:position!() a:bitwise_or() b:compare_op_bitwise_or_pair()* end:position!() { + if b.is_empty() { return a; } + let (ops, comparators) = comparison_ops_comparators(b); + zelf.new_located(begin, end, ExprKind::Compare { left: Box::new(a), ops, comparators }) + } + +// TODO: simplify +#[cache] +rule compare_op_bitwise_or_pair() -> (Cmpop, Expr) = + eq_bitwise_or() / + noteq_bitwise_or() / + lte_bitwise_or() / + lt_bitwise_or() / + gte_bitwise_or() / + gt_bitwise_or() / + notin_bitwise_or() / + in_bitwise_or() / + isnot_bitwise_or() / + is_bitwise_or() + +rule eq_bitwise_or() -> (Cmpop, Expr) = [EqEqual] a:bitwise_or() { (Cmpop::Eq, a) } +rule noteq_bitwise_or() -> (Cmpop, Expr) = [NotEqual] a:bitwise_or() { (Cmpop::NotEq, a) } +rule lte_bitwise_or() -> (Cmpop, Expr) = [LessEqual] a:bitwise_or() { (Cmpop::LtE, a) } +rule lt_bitwise_or() -> (Cmpop, Expr) = [Less] a:bitwise_or() { (Cmpop::Lt, a) } +rule gte_bitwise_or() -> (Cmpop, Expr) = [GreaterEqual] a:bitwise_or() { (Cmpop::GtE, a) } +rule gt_bitwise_or() -> (Cmpop, Expr) = [Greater] a:bitwise_or() { (Cmpop::Gt, a) } +rule notin_bitwise_or() -> (Cmpop, Expr) = [Not] [In] a:bitwise_or() { (Cmpop::NotIn, a) } +rule in_bitwise_or() -> (Cmpop, Expr) = [In] a:bitwise_or() { (Cmpop::In, a) } +rule isnot_bitwise_or() -> (Cmpop, Expr) = [Is] [Not] a:bitwise_or() { (Cmpop::IsNot, a) } +rule is_bitwise_or() -> (Cmpop, Expr) = [Is] a:bitwise_or() { (Cmpop::Is, a) } + +#[cache_left_rec] +rule bitwise_or() -> Expr = + loc() / + bitwise_xor() + +#[cache_left_rec] +rule bitwise_xor() -> Expr = + loc() / + bitwise_and() + +#[cache_left_rec] +rule bitwise_and() -> Expr = + loc() / + shift_expr() + +#[cache_left_rec] +rule shift_expr() -> Expr + = loc() + / sum() +rule shift_op() -> Operator + = [LeftShift] { Operator::LShift } + / [RightShift] { Operator::RShift } + +#[cache_left_rec] +rule sum() -> Expr = + loc() + / term() +rule sum_op() -> Operator + = [Plus] { Operator::Add } + / [Minus] { Operator::Sub } + +#[cache_left_rec] +rule term() -> Expr + = loc() + / factor() +rule term_op() -> Operator + = [Star] { Operator::Mult } + / [Slash] { Operator::Div } + / [DoubleSlash] { Operator::FloorDiv } + / [Percent] { Operator::Mod } + / [At] { Operator::MatMult } + +#[cache] +rule factor() -> Expr = + loc() / + power() +rule factor_op() -> ast::Unaryop + = [Plus] { ast::Unaryop::UAdd } + / [Minus] { ast::Unaryop::USub } + / [Tilde] { ast::Unaryop::Invert } + +rule power() -> Expr + = begin:position!() a:await_primary() b:([DoubleStar] z:factor() {z})? end:position!() { + if let Some(b) = b { + zelf.new_located(begin, end, ExprKind::BinOp { left: Box::new(a), op: Operator::Pow, right: Box::new(b) }) + } else { a } + } + +#[cache] +rule await_primary() -> Expr + = loc(<[Await] a:primary() { + ExprKind::Await { value: Box::new(a) } + }>) + / primary() + +#[cache_left_rec] +rule primary() -> Expr + = loc() + / loc() + / loc() { + ExprKind::Call { func: Box::new(a), args: b.0, keywords: b.1 } + }>) + / loc() { + ExprKind::Subscript { value: Box::new(a), slice: Box::new(b), ctx: ExprContext::Load } + }>) + / atom() + +rule slices() -> Expr = + a:slice() ![Comma] {a} / + loc() + +rule slice() -> Expr = + loc() / + named_expression() + +rule atom() -> Expr + = name_expr(ExprContext::Load) + / loc(< + [True] { ExprKind::Constant { value: ast::Constant::Bool(true), kind: None } } + / [False] { ExprKind::Constant { value: ast::Constant::Bool(false), kind: None } } + / [PegTok::None] { ExprKind::Constant { value: ast::Constant::None, kind: None } } + / [Int(id)] { ExprKind::Constant { value: ast::Constant::Int(zelf.ints[id as usize].clone()), kind: None } } + / [Float(id)] { ExprKind::Constant { value: ast::Constant::Float(zelf.floats[id as usize]), kind: None } } + / [Complex(id)] { + let (real, imag) = zelf.complexes[id as usize]; + ExprKind::Constant { value: ast::Constant::Complex { real, imag }, kind: None } + } + / [Ellipsis] { ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None } } + >) + / strings() + / &[Lpar] a:(tuple() / group() / genexp()) {a} + / &[Lsqb] a:(list() / listcomp()) {a} + / &[Lbrace] a:(dict() / set() / dictcomp() / setcomp()) {a} + +rule group() -> Expr = par() + +rule lambdef() -> Expr = + loc(<[Lambda] a:lambda_params()? [Colon] b:expression() { + ExprKind::Lambda { args: Box::new(a.unwrap_or_else(make_empty_arguments)), body: Box::new(b) } + }>) + +rule lambda_params() -> Arguments = lambda_parameters() + +rule lambda_parameters() -> Arguments = + a:lambda_slash_no_default() c:lambda_param_no_default()* d:lambda_param_with_default()* e:lambda_star_etc()? { + make_arguments(a, Default::default(), c, d, e) + } / + b:lambda_slash_with_default() d:lambda_param_with_default()* e:lambda_star_etc()? { + make_arguments(vec![], b, vec![], d, e) + } / + c:lambda_param_no_default()+ d:lambda_param_with_default()* e:lambda_star_etc()? { + make_arguments(vec![], Default::default(), c, d, e) + } / + d:lambda_param_with_default()+ e:lambda_star_etc()? { + make_arguments(vec![], Default::default(), vec![], d, e) + } / + e:lambda_star_etc() { + make_arguments(vec![], Default::default(), vec![], vec![], Some(e)) + } + +rule lambda_slash_no_default() -> Vec = + a:lambda_param_no_default()+ [Slash] lambda_param_split() {a} + +rule lambda_slash_with_default() -> (Vec, Vec<(Arg, Expr)>) = + a:lambda_param_no_default()* b:lambda_param_with_default()+ [Slash] lambda_param_split() {(a, b)} + +rule lambda_star_etc() -> (Option, Vec<(Arg, Option)>, Option) = + [Star] a:lambda_param_no_default() b:lambda_param_maybe_default()* c:lambda_kwds()? { + (Some(a), b, c) + } / + [Star] [Comma] b:lambda_param_maybe_default()+ c:lambda_kwds()? { + (None, b, c) + } / + c:lambda_kwds() { + (None, vec![], Some(c)) + } + +rule lambda_kwds() -> Arg = + [DoubleStar] a:lambda_param_no_default() {a} + +rule lambda_param_no_default() -> Arg = a:lambda_param() lambda_param_split() {a} +rule lambda_param_with_default() -> (Arg, Expr) = a:lambda_param() c:default() lambda_param_split() {(a, c)} +rule lambda_param_maybe_default() -> (Arg, Option) = a:lambda_param() c:default()? lambda_param_split() {(a, c)} +rule lambda_param() -> Arg = + loc() +rule lambda_param_split() = [Comma] / &[Colon] + +#[cache] +rule strings() -> Expr = a:string()+ {? + // TODO: error handling + crate::string::parse_strings(a).map_err(|_| "string format error") +} + +rule string() -> (Location, (String, StringKind, bool), Location) = + begin:position!() [PegTok::String(id)] end:position!() { + (zelf.locations[begin].0, zelf.strings[id as usize].clone(), zelf.locations[end - 1].1) + } + +rule list() -> Expr = + loc() { + ExprKind::List { elts: a, ctx: ExprContext::Load } + }>) + +rule tuple() -> Expr = + loc() { + ExprKind::Tuple { elts: a, ctx: ExprContext::Load } + }>) + +rule set() -> Expr = + loc() { + ExprKind::Set { elts: a } + }>) + +rule dict() -> Expr = + loc() { + let (keys, values) = dict_kvpairs(a); + ExprKind::Dict { keys, values } + }>) + +rule double_starred_kvpairs() -> Vec<(Option, Expr)> = + a:double_starred_kvpair() ++ [Comma] [Comma]? {a} + +rule double_starred_kvpair() -> (Option, Expr) = + [DoubleStar] a:bitwise_or() { (None, a) } / + a:kvpair() { (Some(a.0), a.1) } + +rule kvpair() -> (Expr, Expr) = + a:expression() [Colon] b:expression() { (a, b) } + +rule for_if_clauses() -> Vec = for_if_clause()+ + +rule for_if_clause() -> Comprehension = + is_async:[Async]? [For] a:star_targets() [In] b:disjunction() c:([If] z:disjunction() { z })* { + Comprehension { target: a, iter: b, ifs: c, is_async: if is_async.is_some() {1} else {0} } + } + +rule listcomp() -> Expr = + loc()>) + +rule setcomp() -> Expr = + loc()>) + +rule genexp() -> Expr = + loc()>) + +rule dictcomp() -> Expr = + loc()>) + +#[cache] +rule arguments() -> (Vec, Vec) = a:args() [Comma]? &[Rpar] {a} + +rule args() -> (Vec, Vec) = + a:(starred_expression() / z:(assignment_expression() / z:expression() ![ColonEqual] {z}) ![Equal] {z}) ++ [Comma] + b:([Comma] k:kwargs() {k} / {vec![]}) { + let (mut ex, kw) = keyword_or_starred_partition(b); + let mut a = a; + a.append(&mut ex); + (a, kw) + } / + a:kwargs() { + keyword_or_starred_partition(a) + } + +rule kwargs() -> Vec = + a:kwarg_or_starred() ++ [Comma] [Comma] b:kwarg_or_double_starred() ++ [Comma] { + let mut a = a; + let mut b = b; + a.append(&mut b); + a + } / + kwarg_or_starred() ++ [Comma] / + kwarg_or_double_starred() ++ [Comma] + +rule starred_expression() -> Expr = + loc(<[Star] a:expression() { + ExprKind::Starred { value: Box::new(a), ctx: ExprContext::Load } + }>) + +rule kwarg_or_starred() -> KeywordOrStarred = + a:loc() { KeywordOrStarred::Keyword(a) } / + a:starred_expression() { + KeywordOrStarred::Starred(a) + } + +rule kwarg_or_double_starred() -> KeywordOrStarred = + a:loc() { KeywordOrStarred::Keyword(a) } / + a:loc(<[DoubleStar] a:expression() { + KeywordData { arg: None, value: a } + }>) { KeywordOrStarred::Keyword(a) } + +rule star_targets() -> Expr = + a:star_target() ![Comma] {a} / + loc() + +rule star_targets_list() -> Vec = a:star_target() ++ [Comma] [Comma]? {a} + +rule star_targets_tuple() -> Vec + = a:star_target() [Comma] v:star_target() ** [Comma] tail:[Comma]? {? + if tail.is_some() && v.is_empty() { + Err("invalid token ','") + } else { + Ok(insert_front(v, a)) + } + } + // a:star_target() ([Comma] v:star_target())+ [Comma]? {} + // begin:position!() a:star_target() v:([Comma z:star_target() {z}])+ [Comma]? {a} / + // a:star_target() [Comma] { vec![a] } + +#[cache] +rule star_target() -> Expr = + loc(<[Star] ![Star] a:star_target() { + ExprKind::Starred { value: Box::new(a), ctx: ExprContext::Store } + }>) / + target_with_star_atom() + +#[cache] +rule target_with_star_atom() -> Expr = + single_subscript_attribute_target(ExprContext::Store) / + star_atom() + +rule star_atom() -> Expr + = name_expr(ExprContext::Store) + / par() + / loc() { + ExprKind::Tuple { elts: a, ctx: ExprContext::Store } + }>) + / loc() { + ExprKind::List { elts: a, ctx: ExprContext::Store } + }>) + +rule single_target() -> Expr = + single_subscript_attribute_target(ExprContext::Store) / + name_expr(ExprContext::Store) / + par() + +rule single_subscript_attribute_target(ctx: ExprContext) -> Expr = loc(< + a:t_primary() [Dot] attr:name() !t_lookahead() { + ExprKind::Attribute { value: Box::new(a), attr, ctx: ctx.clone() } + } + / a:t_primary() b:sqb() !t_lookahead() { + ExprKind::Subscript { value: Box::new(a), slice: Box::new(b), ctx: ctx.clone() } + }>) + +#[cache_left_rec] +rule t_primary() -> Expr = + loc() / + loc() &t_lookahead() { + ExprKind::Subscript { value: Box::new(a), slice: Box::new(b), ctx: ExprContext::Load } + }>) / + loc() / + loc() &t_lookahead() { + let (ex, kw) = b.unwrap_or_default(); + ExprKind::Call { func: Box::new(a), args: ex, keywords: kw } + }>) / + a:atom() &t_lookahead() {a} + +rule t_lookahead() = [Lpar] / [Lsqb] / [Dot] + +rule del_targets() -> Vec = a:del_target() ++ [Comma] [Comma]? {a} + +#[cache] +rule del_target() -> Expr = + single_subscript_attribute_target(ExprContext::Del) / + del_t_atom() + +rule del_t_atom() -> Expr = + name_expr(ExprContext::Del) / + par() / + loc() { + ExprKind::Tuple { elts: a, ctx: ExprContext::Del } + }>) / + loc() { + ExprKind::List { elts: a, ctx: ExprContext::Del } + }>) + +rule loc(r: rule) -> Located = begin:position!() z:r() end:position!() { + zelf.new_located(begin, end, z) +} + +rule loc_block_end(r: rule) -> Located = begin:position!() z:r() end:block_end() { + zelf.new_located(begin, end, z) +} + +rule block_end() -> usize = p:position!() { + let mut p = p; + while zelf.tokens[p - 1] == Newline || zelf.tokens[p - 1] == Dedent { + p -= 1; + } + p +} + +rule name() -> String = [Name(id)] { zelf.names[id as usize].clone() } +rule name_expr(ctx: ExprContext) -> Expr = + loc() + +rule par(r: rule) -> T = [Lpar] z:r() [Rpar] {z} +rule sqb(r: rule) -> T = [Lsqb] z:r() [Rsqb] {z} +rule brace(r: rule) -> T = [Lbrace] z:r() [Rbrace] {z} + +// not yet supported by lexer +rule type_comment() -> Option = { None } +// not yet supported by lexer +rule func_type_comment() -> Option = { None } + +// TODO: optimize +rule pack_tuple_expr(r: rule, ctx: ExprContext) -> Expr + = begin:position!() a:r() v:([Comma] z:r() {z})* tail:[Comma]? end:position!() { + if tail.is_none() && v.is_empty() { + a + } else { + zelf.new_located(begin, end, ExprKind::Tuple { elts: insert_front(v, a), ctx: ctx.clone() }) + } + } + // loc( [Comma] [Comma]? { + // ExprKind::Tuple { elts: z, ctx: ctx.clone() } + // }>) / + // loc() / + // r() + +}} + +#[cold] +#[inline(always)] +fn insert_front(mut v: Vec, a: T) -> Vec { + v.insert(0, a); + v +} + +fn count_dots(toks: Vec) -> Option { + if toks.is_empty() { + return None; + } + + let mut count = 0; + for tok in toks { + count += match tok { + PegTok::Dot => 1, + PegTok::Ellipsis => 3, + _ => unreachable!(), + }; + } + Some(count) +} + +fn option_box(val: Option) -> Option> { + val.map(|x| Box::new(x)) +} + +enum KeywordOrStarred { + Keyword(ast::Keyword), + Starred(ast::Expr), +} + +fn keyword_or_starred_partition(v: Vec) -> (Vec, Vec) { + let mut ex_vec = vec![]; + let mut kw_vec = vec![]; + for x in v { + match x { + KeywordOrStarred::Keyword(kw) => kw_vec.push(kw), + KeywordOrStarred::Starred(ex) => ex_vec.push(ex), + } + } + (ex_vec, kw_vec) +} + +fn dict_kvpairs( + v: Vec<(Option, ast::Expr)>, +) -> (Vec>, Vec) { + let mut keys = Vec::with_capacity(v.len()); + let mut values = Vec::with_capacity(v.len()); + + for (key, value) in v { + keys.push(key); + values.push(value); + } + + // let (packed, unpacked) = v.into_iter().partition::, _>(|x| x.0.is_some()); + // for x in packed { + // keys.push(x.0.unwrap()); + // values.push(x.1); + // } + // for x in unpacked { + // values.push(x.1); + // } + (keys, values) +} + +fn comparison_ops_comparators( + v: Vec<(ast::Cmpop, ast::Expr)>, +) -> (Vec, Vec) { + let mut ops = Vec::with_capacity(v.len()); + let mut comparators = Vec::with_capacity(v.len()); + + for x in v { + ops.push(x.0); + comparators.push(x.1); + } + (ops, comparators) +} + +fn make_arguments( + slash_no_default: Vec, + slash_with_default: (Vec, Vec<(ast::Arg, ast::Expr)>), + param_no_default: Vec, + param_with_default: Vec<(ast::Arg, ast::Expr)>, + star_etc: Option<( + Option, + Vec<(ast::Arg, Option)>, + Option, + )>, +) -> ast::Arguments { + let mut posonlyargs = slash_no_default; + posonlyargs.extend(slash_with_default.0.iter().cloned()); + posonlyargs.extend(slash_with_default.1.iter().map(|x| x.0.clone())); + + let mut posargs = param_no_default; + posargs.extend(param_with_default.iter().map(|x| x.0.clone())); + + let posdefaults: Vec = slash_with_default + .1 + .iter() + .map(|x| x.1.clone()) + .chain(param_with_default.iter().map(|x| x.1.clone())) + .collect(); + + // TODO: refactor remove option wrap for star_etc + let (vararg, kwonly, kwarg) = star_etc.unwrap_or_default(); + let kwonlyargs: Vec = kwonly.iter().map(|x| x.0.clone()).collect(); + let kw_defaults: Vec = kwonly.iter().filter_map(|x| x.1.clone()).collect(); + + ast::Arguments { + posonlyargs, + args: posargs, + vararg: option_box(vararg), + kwonlyargs, + kw_defaults, + kwarg: option_box(kwarg), + defaults: posdefaults, + } +} + +fn make_empty_arguments() -> ast::Arguments { + ast::Arguments { + posonlyargs: Default::default(), + args: Default::default(), + vararg: Default::default(), + kwonlyargs: Default::default(), + kw_defaults: Default::default(), + kwarg: Default::default(), + defaults: Default::default(), + } +} diff --git a/compiler/parser/src/python.rs b/compiler/parser/src/python.rs deleted file mode 100644 index a00274d6c46..00000000000 --- a/compiler/parser/src/python.rs +++ /dev/null @@ -1,3 +0,0 @@ -#![allow(clippy::all)] -#![allow(unused)] -include!(concat!(env!("OUT_DIR"), "/python.rs")); diff --git a/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_assigment.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_assigment.snap new file mode 100644 index 00000000000..a307f52c0fe --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_assigment.snap @@ -0,0 +1,58 @@ +--- +source: compiler/parser/src/parser.rs +expression: parse_ast +--- +[ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), + custom: (), + node: Assign { + targets: [ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 2, + }, + ), + custom: (), + node: List { + elts: [], + ctx: Store, + }, + }, + ], + value: Located { + location: Location { + row: 1, + column: 5, + }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), + custom: (), + node: Name { + id: "gen_b", + ctx: Load, + }, + }, + type_comment: None, + }, + }, +] diff --git a/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwds.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwds.snap new file mode 100644 index 00000000000..3513e83641f --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwds.snap @@ -0,0 +1,159 @@ +--- +source: compiler/parser/src/parser.rs +expression: parse_ast +--- +[ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 46, + }, + ), + custom: (), + node: Expr { + value: Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 46, + }, + ), + custom: (), + node: Call { + func: Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 4, + }, + ), + custom: (), + node: Name { + id: "func", + ctx: Load, + }, + }, + args: [ + Located { + location: Location { + row: 1, + column: 5, + }, + end_location: Some( + Location { + row: 1, + column: 10, + }, + ), + custom: (), + node: Name { + id: "token", + ctx: Load, + }, + }, + Located { + location: Location { + row: 1, + column: 12, + }, + end_location: Some( + Location { + row: 1, + column: 22, + }, + ), + custom: (), + node: Name { + id: "serializer", + ctx: Load, + }, + }, + ], + keywords: [ + Located { + location: Location { + row: 1, + column: 24, + }, + end_location: Some( + Location { + row: 1, + column: 37, + }, + ), + custom: (), + node: KeywordData { + arg: Some( + "incref", + ), + value: Located { + location: Location { + row: 1, + column: 31, + }, + end_location: Some( + Location { + row: 1, + column: 37, + }, + ), + custom: (), + node: Name { + id: "incref", + ctx: Load, + }, + }, + }, + }, + Located { + location: Location { + row: 1, + column: 39, + }, + end_location: Some( + Location { + row: 1, + column: 45, + }, + ), + custom: (), + node: KeywordData { + arg: None, + value: Located { + location: Location { + row: 1, + column: 41, + }, + end_location: Some( + Location { + row: 1, + column: 45, + }, + ), + custom: (), + node: Name { + id: "kwds", + ctx: Load, + }, + }, + }, + }, + ], + }, + }, + }, + }, +] diff --git a/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_pass.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_pass.snap new file mode 100644 index 00000000000..856d92d7205 --- /dev/null +++ b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_pass.snap @@ -0,0 +1,20 @@ +--- +source: compiler/parser/src/parser.rs +expression: parse_ast +--- +[ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 1, + column: 4, + }, + ), + custom: (), + node: Pass, + }, +]