Skip to content

Commit 48966a7

Browse files
coolreader18coolreader18
committed
Make py_compile_bytecode!() an expression
1 parent 7e68f99 commit 48966a7

File tree

8 files changed

+142
-91
lines changed

8 files changed

+142
-91
lines changed

‎Cargo.lock‎

Lines changed: 13 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎derive/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,4 @@ quote = "0.6.11"
1313
proc-macro2 = "0.4.27"
1414
rustpython_compiler = { path = "../compiler" }
1515
bincode = "1.1"
16+
proc-macro-hack = "0.5"

‎derive/src/compile_bytecode.rs‎

Lines changed: 103 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,76 @@
1-
use super::Diagnostic;
1+
use crate::{extract_spans, Diagnostic};
22
use bincode;
33
use proc_macro2::{Span, TokenStream as TokenStream2};
44
use quote::quote;
55
use rustpython_compiler::{bytecode::CodeObject, compile};
66
use std::env;
77
use std::fs;
8-
use std::path::{Path, PathBuf};
8+
use std::path::PathBuf;
99
use syn::parse::{Parse, ParseStream, Result as ParseResult};
10-
use syn::{self, parse2, Ident, Lit, LitByteStr, Meta, MetaList, NestedMeta, Token};
10+
use syn::{self, parse2, Lit, LitByteStr, Meta, Token};
1111

12-
struct BytecodeConst {
13-
ident: Ident,
14-
meta: MetaList,
12+
enum CompilationSourceKind {
13+
File(PathBuf),
14+
SourceCode(String),
1515
}
1616

17-
impl BytecodeConst {
18-
fn compile(&self, manifest_dir: &Path) -> Result<CodeObject, Diagnostic> {
19-
let meta = &self.meta;
17+
struct CompilationSource {
18+
kind: CompilationSourceKind,
19+
span: (Span, Span),
20+
}
21+
22+
impl CompilationSource {
23+
fn compile(self, mode: &compile::Mode, source_path: String) -> Result<CodeObject, Diagnostic> {
24+
let compile = |source| {
25+
compile::compile(source, mode, source_path).map_err(|err| {
26+
Diagnostic::spans_error(self.span, format!("Compile error: {}", err))
27+
})
28+
};
29+
30+
match &self.kind {
31+
CompilationSourceKind::File(rel_path) => {
32+
let mut path = PathBuf::from(
33+
env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present"),
34+
);
35+
path.push(rel_path);
36+
let source = fs::read_to_string(&path).map_err(|err| {
37+
Diagnostic::spans_error(
38+
self.span,
39+
format!("Error reading file {:?}: {}", path, err),
40+
)
41+
})?;
42+
compile(&source)
43+
}
44+
CompilationSourceKind::SourceCode(code) => compile(code),
45+
}
46+
}
47+
}
48+
49+
struct PyCompileInput {
50+
span: Span,
51+
metas: Vec<Meta>,
52+
}
2053

54+
impl PyCompileInput {
55+
fn compile(&self) -> Result<CodeObject, Diagnostic> {
2156
let mut source_path = None;
2257
let mut mode = None;
23-
let mut source_lit = None;
58+
let mut source: Option<CompilationSource> = None;
59+
60+
fn assert_source_empty(source: &Option<CompilationSource>) -> Result<(), Diagnostic> {
61+
if let Some(source) = source {
62+
Err(Diagnostic::spans_error(
63+
source.span.clone(),
64+
"Cannot have more than one source",
65+
))
66+
} else {
67+
Ok(())
68+
}
69+
}
2470

25-
for meta in &meta.nested {
71+
for meta in &self.metas {
2672
match meta {
27-
NestedMeta::Literal(lit) => source_lit = Some(lit),
28-
NestedMeta::Meta(Meta::NameValue(name_value)) => {
73+
Meta::NameValue(name_value) => {
2974
if name_value.ident == "mode" {
3075
mode = Some(match &name_value.lit {
3176
Lit::Str(s) => match s.value().as_str() {
@@ -41,94 +86,69 @@ impl BytecodeConst {
4186
Lit::Str(s) => s.value(),
4287
_ => bail_span!(name_value.lit, "source_path must be string"),
4388
})
89+
} else if name_value.ident == "source" {
90+
assert_source_empty(&source)?;
91+
let code = match &name_value.lit {
92+
Lit::Str(s) => s.value(),
93+
_ => bail_span!(name_value.lit, "source must be a string"),
94+
};
95+
source = Some(CompilationSource {
96+
kind: CompilationSourceKind::SourceCode(code),
97+
span: extract_spans(&name_value).unwrap(),
98+
});
99+
} else if name_value.ident == "file" {
100+
assert_source_empty(&source)?;
101+
let path = match &name_value.lit {
102+
Lit::Str(s) => PathBuf::from(s.value()),
103+
_ => bail_span!(name_value.lit, "source must be a string"),
104+
};
105+
source = Some(CompilationSource {
106+
kind: CompilationSourceKind::File(path),
107+
span: extract_spans(&name_value).unwrap(),
108+
});
44109
}
45110
}
46111
_ => {}
47112
}
48113
}
49114

50-
let source = if meta.ident == "file" {
51-
let path = match source_lit {
52-
Some(Lit::Str(s)) => s.value(),
53-
_ => bail_span!(source_lit, "Expected string literal for path to file()"),
54-
};
55-
let path = manifest_dir.join(path);
56-
fs::read_to_string(&path)
57-
.map_err(|err| err_span!(source_lit, "Error reading file {:?}: {}", path, err))?
58-
} else if meta.ident == "source" {
59-
match source_lit {
60-
Some(Lit::Str(s)) => s.value(),
61-
_ => bail_span!(source_lit, "Expected string literal for source()"),
62-
}
63-
} else {
64-
bail_span!(meta.ident, "Expected either 'file' or 'source'")
65-
};
66-
67-
compile::compile(
68-
&source,
69-
&mode.unwrap_or(compile::Mode::Exec),
70-
source_path.unwrap_or_else(|| "frozen".to_string()),
71-
)
72-
.map_err(|err| err_span!(source_lit, "Compile error: {}", err))
73-
}
74-
}
75-
76-
impl Parse for BytecodeConst {
77-
/// Parse the form `static ref IDENT = metalist(...);`
78-
fn parse(input: ParseStream) -> ParseResult<Self> {
79-
input.parse::<Token![static]>()?;
80-
input.parse::<Token![ref]>()?;
81-
let ident = input.parse()?;
82-
input.parse::<Token![=]>()?;
83-
let meta = input.parse()?;
84-
input.parse::<Token![;]>()?;
85-
Ok(BytecodeConst { ident, meta })
115+
source
116+
.ok_or_else(|| {
117+
Diagnostic::span_error(
118+
self.span.clone(),
119+
"Must have either file or source in py_compile_bytecode!()",
120+
)
121+
})?
122+
.compile(
123+
&mode.unwrap_or(compile::Mode::Exec),
124+
source_path.unwrap_or_else(|| "frozen".to_string()),
125+
)
86126
}
87127
}
88128

89-
struct PyCompileInput(Vec<BytecodeConst>);
90-
91129
impl Parse for PyCompileInput {
92130
fn parse(input: ParseStream) -> ParseResult<Self> {
93-
std::iter::from_fn(|| {
94-
if input.is_empty() {
95-
None
96-
} else {
97-
Some(input.parse())
98-
}
99-
})
100-
.collect::<ParseResult<_>>()
101-
.map(PyCompileInput)
131+
let span = input.cursor().span();
132+
let metas = input
133+
.parse_terminated::<Meta, Token![,]>(Meta::parse)?
134+
.into_iter()
135+
.collect();
136+
Ok(PyCompileInput { span, metas })
102137
}
103138
}
104139

105140
pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result<TokenStream2, Diagnostic> {
106-
let PyCompileInput(consts) = parse2(input)?;
107-
108-
let manifest_dir = PathBuf::from(
109-
env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present"),
110-
);
141+
let input: PyCompileInput = parse2(input)?;
111142

112-
let consts = consts
113-
.into_iter()
114-
.map(|bytecode_const| -> Result<_, Diagnostic> {
115-
let code_obj = bytecode_const.compile(&manifest_dir)?;
116-
let ident = bytecode_const.ident;
117-
let bytes = bincode::serialize(&code_obj).expect("Failed to serialize");
118-
let bytes = LitByteStr::new(&bytes, Span::call_site());
119-
Ok(quote! {
120-
static ref #ident: ::rustpython_vm::bytecode::CodeObject = {
121-
use bincode;
122-
bincode::deserialize(#bytes).expect("Deserializing CodeObject failed")
123-
};
124-
})
125-
})
126-
.collect::<Result<Vec<_>, _>>()?;
143+
let code_obj = input.compile()?;
144+
let bytes = bincode::serialize(&code_obj).expect("Failed to serialize");
145+
let bytes = LitByteStr::new(&bytes, Span::call_site());
127146

128147
let output = quote! {
129-
lazy_static! {
130-
#(#consts)*
131-
}
148+
({
149+
use bincode;
150+
bincode::deserialize(#bytes).expect("Deserializing CodeObject failed")
151+
})
132152
};
133153

134154
Ok(output)

‎derive/src/error.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,15 @@ impl Diagnostic {
8585
}
8686
}
8787

88+
pub fn spans_error<T: Into<String>>(spans: (Span, Span), text: T) -> Diagnostic {
89+
Diagnostic {
90+
inner: Repr::Single {
91+
text: text.into(),
92+
span: Some(spans),
93+
},
94+
}
95+
}
96+
8897
pub fn spanned_error<T: Into<String>>(node: &ToTokens, text: T) -> Diagnostic {
8998
Diagnostic {
9099
inner: Repr::Single {
@@ -122,7 +131,7 @@ impl From<Error> for Diagnostic {
122131
}
123132
}
124133

125-
fn extract_spans(node: &ToTokens) -> Option<(Span, Span)> {
134+
pub fn extract_spans(node: &dyn ToTokens) -> Option<(Span, Span)> {
126135
let mut t = TokenStream::new();
127136
node.to_tokens(&mut t);
128137
let mut tokens = t.into_iter();

‎derive/src/lib.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@ mod compile_bytecode;
88
mod from_args;
99
mod pyclass;
1010

11-
use error::Diagnostic;
11+
use error::{extract_spans, Diagnostic};
1212
use proc_macro::TokenStream;
1313
use proc_macro2::TokenStream as TokenStream2;
14+
use proc_macro_hack::proc_macro_hack;
1415
use quote::ToTokens;
1516
use syn::{parse_macro_input, AttributeArgs, DeriveInput, Item};
1617

@@ -49,7 +50,7 @@ pub fn pystruct_sequence(attr: TokenStream, item: TokenStream) -> TokenStream {
4950
result_to_tokens(pyclass::impl_pystruct_sequence(attr, item))
5051
}
5152

52-
#[proc_macro]
53+
#[proc_macro_hack]
5354
pub fn py_compile_bytecode(input: TokenStream) -> TokenStream {
5455
result_to_tokens(compile_bytecode::impl_py_compile_bytecode(input.into()))
5556
}

‎vm/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ crc = "^1.0.0"
3333
bincode = "1.1.4"
3434
unicode_categories = "0.1.1"
3535
maplit = "1.0"
36+
proc-macro-hack = "0.5"
3637

3738

3839
# TODO: release and publish to crates.io

‎vm/src/frozen.rs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
11
use crate::bytecode::CodeObject;
22
use std::collections::HashMap;
33

4-
py_compile_bytecode! {
5-
static ref HELLO = source(
6-
"initialized = True
4+
lazy_static! {
5+
static ref HELLO: CodeObject = py_compile_bytecode!(
6+
source = "initialized = True
77
print(\"Hello world!\")
88
",
99
);
10-
static ref IMPORTLIB_BOOTSTRAP = file("../Lib/importlib/_bootstrap.py");
11-
static ref IMPORTLIB_BOOTSTRAP_EXTERNAL = file("../Lib/importlib/_bootstrap_external.py");
10+
static ref IMPORTLIB_BOOTSTRAP: CodeObject =
11+
py_compile_bytecode!(file = "../Lib/importlib/_bootstrap.py");
12+
static ref IMPORTLIB_BOOTSTRAP_EXTERNAL: CodeObject =
13+
py_compile_bytecode!(file = "../Lib/importlib/_bootstrap_external.py");
1214
}
1315

1416
pub fn get_module_inits() -> HashMap<&'static str, &'static CodeObject> {

‎vm/src/lib.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ extern crate self as rustpython_vm;
2929

3030
pub use rustpython_derive::*;
3131

32+
use proc_macro_hack::proc_macro_hack;
33+
#[proc_macro_hack]
34+
pub use rustpython_derive::py_compile_bytecode;
35+
3236
//extern crate eval; use eval::eval::*;
3337
// use py_code_object::{Function, NativeType, PyCodeObject};
3438

0 commit comments

Comments
 (0)