Compress bytecode using LZ4

windelbouwman · windelbouwman · commit 64ab147f64e5 · 2019-10-07T20:54:22.000+02:00
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/bytecode/Cargo.toml b/bytecode/Cargo.toml
@@ -9,7 +9,9 @@ license = "MIT"
 
 
 [dependencies]
+bincode =  "1.1"
 bitflags = "1.1"
+lz4-compress = "0.1.1"
 num-bigint = { version = "0.2", features = ["serde"] }
 num-complex = { version = "0.2", features = ["serde"] }
 serde = { version = "1.0", features = ["derive"] }
diff --git a/bytecode/src/bytecode.rs b/bytecode/src/bytecode.rs
@@ -378,6 +378,18 @@ impl CodeObject {
         }
     }
 
+    /// Load a code object from bytes
+    pub fn from_bytes(data: &[u8]) -> Result<Self, Box<dyn std::error::Error>> {
+        let data = lz4_compress::decompress(data)?;
+        bincode::deserialize::<Self>(&data).map_err(|e| e.into())
+    }
+
+    /// Serialize this bytecode to bytes.
+    pub fn to_bytes(&self) -> Vec<u8> {
+        let data = bincode::serialize(&self).expect("Code object must be serializable");
+        lz4_compress::compress(&data)
+    }
+
     pub fn get_constants(&self) -> impl Iterator<Item = &Constant> {
         self.instructions.iter().filter_map(|x| {
             if let Instruction::LoadConst { value } = x {
diff --git a/derive/Cargo.toml b/derive/Cargo.toml
@@ -19,6 +19,5 @@ quote = "0.6.11"
 proc-macro2 = "0.4.27"
 rustpython-compiler = { path = "../compiler", version = "0.1.1" }
 rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
-bincode =  "1.1"
 proc-macro-hack = { version = "0.5", optional = true }
 maplit = "1.0"
diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs
@@ -14,7 +14,6 @@
 //! ```
 
 use crate::{extract_spans, Diagnostic};
-use bincode;
 use proc_macro2::{Span, TokenStream as TokenStream2};
 use quote::quote;
 use rustpython_bytecode::bytecode::{CodeObject, FrozenModule};
@@ -249,11 +248,11 @@ pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result<TokenStream2, Dia
         .into_iter()
         .map(|(module_name, FrozenModule { code, package })| {
             let module_name = LitStr::new(&module_name, Span::call_site());
-            let bytes = bincode::serialize(&code).expect("Failed to serialize");
+            let bytes = code.to_bytes();
             let bytes = LitByteStr::new(&bytes, Span::call_site());
             quote! {
                 #module_name.into() => ::rustpython_vm::bytecode::FrozenModule {
-                    code: bincode::deserialize::<::rustpython_vm::bytecode::CodeObject>(
+                    code: ::rustpython_vm::bytecode::CodeObject::from_bytes(
                         #bytes
                     ).expect("Deserializing CodeObject failed"),
                     package: #package,
@@ -263,7 +262,6 @@ pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result<TokenStream2, Dia
 
     let output = quote! {
         ({
-            use ::rustpython_vm::__exports::bincode;
             use ::rustpython_vm::__exports::hashmap;
             hashmap! {
                 #(#modules),*
diff --git a/vm/Cargo.toml b/vm/Cargo.toml
@@ -52,7 +52,6 @@ hex = "0.3.2"
 hexf-parse = "0.1.0"
 indexmap = "1.0.2"
 crc = "^1.0.0"
-bincode = "1.1.4"
 unicode_categories = "0.1.1"
 unicode_names2 = "0.2.2"
 unicode-casing = "0.1.0"
diff --git a/vm/src/lib.rs b/vm/src/lib.rs
@@ -79,6 +79,5 @@ pub use rustpython_bytecode::*;
 
 #[doc(hidden)]
 pub mod __exports {
-    pub use bincode;
     pub use maplit::hashmap;
 }
diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs
@@ -5,11 +5,11 @@ use crate::pyobject::{PyObjectRef, PyResult};
 use crate::vm::VirtualMachine;
 
 fn marshal_dumps(co: PyCodeRef, _vm: &VirtualMachine) -> PyBytes {
-    PyBytes::new(bincode::serialize(&co.code).unwrap())
+    PyBytes::new(co.code.to_bytes())
 }
 
 fn marshal_loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult<PyCode> {
-    let code = bincode::deserialize::<bytecode::CodeObject>(&code_bytes)
+    let code = bytecode::CodeObject::from_bytes(&code_bytes)
         .map_err(|_| vm.new_value_error("Couldn't deserialize python bytecode".to_owned()))?;
     Ok(PyCode { code })
 }

Original file line number	Diff line number	Diff line change
`@@ -79,6 +79,5 @@ pub use rustpython_bytecode::*;`
`79`	`79`
`80`	`80`	`#[doc(hidden)]`
`81`	`81`	`pub mod __exports {`
`82`		`- pub use bincode;`
`83`	`82`	`pub use maplit::hashmap;`
`84`	`83`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,11 +5,11 @@ use crate::pyobject::{PyObjectRef, PyResult};`
`5`	`5`	`use crate::vm::VirtualMachine;`
`6`	`6`
`7`	`7`	`fn marshal_dumps(co: PyCodeRef, _vm: &VirtualMachine) -> PyBytes {`
`8`		`- PyBytes::new(bincode::serialize(&co.code).unwrap())`
	`8`	`+ PyBytes::new(co.code.to_bytes())`
`9`	`9`	`}`
`10`	`10`
`11`	`11`	`fn marshal_loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult<PyCode> {`
`12`		`- let code = bincode::deserialize::<bytecode::CodeObject>(&code_bytes)`
	`12`	`+ let code = bytecode::CodeObject::from_bytes(&code_bytes)`
`13`	`13`	`.map_err(\|_\| vm.new_value_error("Couldn't deserialize python bytecode".to_owned()))?;`
`14`	`14`	`Ok(PyCode { code })`
`15`	`15`	`}`