Skip to content

Commit bdcc764

Browse files
jimbakerjimbaker
committed
Added _marshal, a Java implementation of the Python module, and made
current. test_marshal now passes (with minor modifications around GC and floating point accuracy on to/from string representation), except for support of code objects. That's next. Pulled IOFile (now PyIOFile) from cPickle to share its duck typing with _marshal A historical note: from what I can discern in the log, the original marshal.py was written by Guido van Rossum specifically for "JPython" in 1997, one of the few bits he apparently did. Now it survives only as a shell, to import in _marshal.
1 parent 54ba682 commit bdcc764

File tree

9 files changed

+770
-510
lines changed

9 files changed

+770
-510
lines changed

‎Lib/marshal.py‎

Lines changed: 17 additions & 325 deletions
Original file line numberDiff line numberDiff line change
@@ -5,332 +5,24 @@
55
66
"""
77

8-
import StringIO
9-
import string
10-
from types import *
11-
try:
12-
import new
13-
except ImportError:
14-
new = None
15-
16-
TYPE_NULL = '0'
17-
TYPE_NONE = 'N'
18-
TYPE_FALSE = 'F'
19-
TYPE_TRUE = 'T'
20-
TYPE_ELLIPSIS = '.'
21-
TYPE_INT = 'i'
22-
TYPE_INT64 = 'I'
23-
TYPE_FLOAT = 'f'
24-
TYPE_COMPLEX = 'x'
25-
TYPE_LONG = 'l'
26-
TYPE_STRING = 's'
27-
TYPE_TUPLE = '('
28-
TYPE_LIST = '['
29-
TYPE_DICT = '{'
30-
TYPE_CODE = 'c'
31-
TYPE_UNKNOWN = '?'
32-
33-
34-
class Marshaller:
35-
36-
dispatch = {}
37-
38-
def __init__(self, f):
39-
self.f = f
40-
41-
def dump(self, x):
42-
self.dispatch[type(x)](self, x)
43-
44-
def w_long64(self, x):
45-
self.w_long(x)
46-
self.w_long(x>>32)
47-
48-
def w_long(self, x):
49-
write = self.f.write
50-
write(chr((x) & 0xff))
51-
write(chr((x>> 8) & 0xff))
52-
write(chr((x>>16) & 0xff))
53-
write(chr((x>>24) & 0xff))
54-
55-
def w_short(self, x):
56-
write = self.f.write
57-
write(chr((x) & 0xff))
58-
write(chr((x>> 8) & 0xff))
59-
60-
def dump_none(self, x):
61-
self.f.write(TYPE_NONE)
62-
dispatch[NoneType] = dump_none
63-
64-
def dump_bool(self, x):
65-
if x:
66-
self.f.write(TYPE_TRUE)
67-
else:
68-
self.f.write(TYPE_FALSE)
69-
dispatch[BooleanType] = dump_bool
70-
71-
def dump_ellipsis(self, x):
72-
self.f.write(TYPE_ELLIPSIS)
73-
try:
74-
dispatch[EllipsisType] = dump_ellipsis
75-
except NameError:
76-
pass
77-
78-
def dump_int(self, x):
79-
y = x>>31
80-
if y and y != -1:
81-
self.f.write(TYPE_INT64)
82-
self.w_long64(x)
83-
else:
84-
self.f.write(TYPE_INT)
85-
self.w_long(x)
86-
dispatch[IntType] = dump_int
87-
88-
def dump_long(self, x):
89-
self.f.write(TYPE_LONG)
90-
sign = 1
91-
if x < 0:
92-
sign = -1
93-
x = -x
94-
digits = []
95-
while x:
96-
digits.append(x & 0x7FFF)
97-
x = x>>15
98-
self.w_long(len(digits) * sign)
99-
for d in digits:
100-
self.w_short(d)
101-
dispatch[LongType] = dump_long
102-
103-
def dump_float(self, x):
104-
write = self.f.write
105-
write(TYPE_FLOAT)
106-
s = `x`
107-
write(chr(len(s)))
108-
write(s)
109-
dispatch[FloatType] = dump_float
110-
111-
def dump_complex(self, x):
112-
write = self.f.write
113-
write(TYPE_COMPLEX)
114-
s = `x.real`
115-
write(chr(len(s)))
116-
write(s)
117-
s = `x.imag`
118-
write(chr(len(s)))
119-
write(s)
120-
try:
121-
dispatch[ComplexType] = dump_complex
122-
except NameError:
123-
pass
124-
125-
def dump_string(self, x):
126-
self.f.write(TYPE_STRING)
127-
self.w_long(len(x))
128-
self.f.write(x)
129-
dispatch[StringType] = dump_string
130-
131-
def dump_tuple(self, x):
132-
self.f.write(TYPE_TUPLE)
133-
self.w_long(len(x))
134-
for item in x:
135-
self.dump(item)
136-
dispatch[TupleType] = dump_tuple
137-
138-
def dump_list(self, x):
139-
self.f.write(TYPE_LIST)
140-
self.w_long(len(x))
141-
for item in x:
142-
self.dump(item)
143-
dispatch[ListType] = dump_list
144-
145-
def dump_dict(self, x):
146-
self.f.write(TYPE_DICT)
147-
for key, value in x.items():
148-
self.dump(key)
149-
self.dump(value)
150-
self.f.write(TYPE_NULL)
151-
dispatch[DictionaryType] = dump_dict
152-
153-
def dump_code(self, x):
154-
self.f.write(TYPE_CODE)
155-
self.w_short(x.co_argcount)
156-
self.w_short(x.co_nlocals)
157-
self.w_short(x.co_stacksize)
158-
self.w_short(x.co_flags)
159-
self.dump(x.co_code)
160-
self.dump(x.co_consts)
161-
self.dump(x.co_names)
162-
self.dump(x.co_varnames)
163-
self.dump(x.co_filename)
164-
self.dump(x.co_name)
165-
self.w_short(x.co_firstlineno)
166-
self.dump(x.co_lnotab)
167-
try:
168-
dispatch[CodeType] = dump_code
169-
except NameError:
170-
pass
171-
172-
173-
class NULL:
174-
pass
175-
176-
class Unmarshaller:
177-
178-
dispatch = {}
179-
180-
def __init__(self, f):
181-
self.f = f
182-
183-
def load(self):
184-
c = self.f.read(1)
185-
if not c:
186-
raise EOFError
187-
return self.dispatch[c](self)
188-
189-
def r_short(self):
190-
read = self.f.read
191-
lo = ord(read(1))
192-
hi = ord(read(1))
193-
x = lo | (hi<<8)
194-
if x & 0x8000:
195-
x = x - 0x10000
196-
return x
197-
198-
def r_long(self):
199-
read = self.f.read
200-
a = ord(read(1))
201-
b = ord(read(1))
202-
c = ord(read(1))
203-
d = ord(read(1))
204-
# convert unsigned d to signed to avoid d << 24 possibly
205-
# overflowing into a long value
206-
if d > 127:
207-
d -= 256
208-
x = a | (b<<8) | (c<<16) | (d<<24)
209-
if x & 0x80000000 and x > 0:
210-
x = string.atoi(x - 0x100000000L)
211-
return x
212-
213-
def r_long64(self):
214-
a = self.r_long()
215-
b = self.r_long()
216-
return a | (b<<32)
217-
218-
def load_null(self):
219-
return NULL
220-
dispatch[TYPE_NULL] = load_null
221-
222-
def load_none(self):
223-
return None
224-
dispatch[TYPE_NONE] = load_none
225-
226-
def load_False(self):
227-
return False
228-
dispatch[TYPE_FALSE] = load_False
229-
230-
def load_True(self):
231-
return True
232-
dispatch[TYPE_TRUE] = load_True
233-
234-
def load_ellipsis(self):
235-
return EllipsisType
236-
dispatch[TYPE_ELLIPSIS] = load_ellipsis
237-
238-
def load_int(self):
239-
return self.r_long()
240-
dispatch[TYPE_INT] = load_int
241-
242-
def load_int64(self):
243-
return self.r_long64()
244-
dispatch[TYPE_INT64] = load_int64
245-
246-
def load_long(self):
247-
size = self.r_long()
248-
sign = 1
249-
if size < 0:
250-
sign = -1
251-
size = -size
252-
x = 0L
253-
for i in range(size):
254-
d = self.r_short()
255-
x = x | (d<<(i*15L))
256-
return x * sign
257-
dispatch[TYPE_LONG] = load_long
258-
259-
def load_float(self):
260-
n = ord(self.f.read(1))
261-
s = self.f.read(n)
262-
return string.atof(s)
263-
dispatch[TYPE_FLOAT] = load_float
264-
265-
def load_complex(self):
266-
n = ord(self.f.read(1))
267-
s = self.f.read(n)
268-
real = float(s)
269-
n = ord(self.f.read(1))
270-
s = self.f.read(n)
271-
imag = float(s)
272-
return complex(real, imag)
273-
dispatch[TYPE_COMPLEX] = load_complex
274-
275-
def load_string(self):
276-
n = self.r_long()
277-
return self.f.read(n)
278-
dispatch[TYPE_STRING] = load_string
279-
280-
def load_tuple(self):
281-
return tuple(self.load_list())
282-
dispatch[TYPE_TUPLE] = load_tuple
283-
284-
def load_list(self):
285-
n = self.r_long()
286-
list = []
287-
for i in range(n):
288-
list.append(self.load())
289-
return list
290-
dispatch[TYPE_LIST] = load_list
291-
292-
def load_dict(self):
293-
d = {}
294-
while 1:
295-
key = self.load()
296-
if key is NULL:
297-
break
298-
value = self.load()
299-
d[key] = value
300-
return d
301-
dispatch[TYPE_DICT] = load_dict
302-
303-
def load_code(self):
304-
argcount = self.r_short()
305-
nlocals = self.r_short()
306-
stacksize = self.r_short()
307-
flags = self.r_short()
308-
code = self.load()
309-
consts = self.load()
310-
names = self.load()
311-
varnames = self.load()
312-
filename = self.load()
313-
name = self.load()
314-
firstlineno = self.r_short()
315-
lnotab = self.load()
316-
if not new:
317-
raise RuntimeError, "can't unmarshal code objects; no 'new' module"
318-
return new.code(argcount, nlocals, stacksize, flags, code, consts,
319-
names, varnames, filename, name, firstlineno, lnotab)
320-
dispatch[TYPE_CODE] = load_code
321-
322-
323-
def dump(x, f):
324-
Marshaller(f).dump(x)
325-
326-
def load(f):
327-
return Unmarshaller(f).load()
328-
329-
def dumps(x):
330-
f = StringIO.StringIO()
331-
dump(x, f)
8+
import cStringIO
9+
from _marshal import Marshaller, Unmarshaller
10+
11+
def dump(x, f, version=2):
12+
Marshaller(f, version).dump(x)
13+
14+
# XXX - added just for debugging. remove!
15+
def load(f, debug=False):
16+
u = Unmarshaller(f)
17+
if debug:
18+
u._debug()
19+
return u.load()
20+
21+
def dumps(x, version=2):
22+
f = cStringIO.StringIO()
23+
dump(x, f, version)
33224
return f.getvalue()
33325

33426
def loads(s):
335-
f = StringIO.StringIO(s)
27+
f = cStringIO.StringIO(s)
33628
return load(f)

‎Lib/new.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,5 @@
2020
# from types import CodeType as code
2121
#except ImportError:
2222
# pass
23+
24+
from org.python.core import PyBytecode as code

0 commit comments

Comments
 (0)