Skip to content

Commit 7f50efa

Browse files
author
gvanrossum
committed
Implement appropriate __getnewargs__ for all immutable subclassable builtin
types. The special handling for these can now be removed from save_newobj(). Add some testing for this. Also add support for setting the 'fast' flag on the Python Pickler class, which suppresses use of the memo. git-svn-id: http://svn.python.org/projects/python/trunk@31050 6015fed2-1504-0410-9fe1-9d1591cc4771
1 parent 9453d36 commit 7f50efa

File tree

10 files changed

+133
-19
lines changed

10 files changed

+133
-19
lines changed

‎Lib/pickle.py‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ def __init__(self, file, proto=1):
191191
self.memo = {}
192192
self.proto = int(proto)
193193
self.bin = proto >= 1
194+
self.fast = 0
194195

195196
def clear_memo(self):
196197
"""Clears the pickler's "memo".
@@ -230,6 +231,8 @@ def memoize(self, obj):
230231
# But there appears no advantage to any other scheme, and this
231232
# scheme allows the Unpickler memo to be implemented as a plain (but
232233
# growable) array, indexed by memo key.
234+
if self.fast:
235+
return
233236
memo_len = len(self.memo)
234237
self.write(self.put(memo_len))
235238
self.memo[id(obj)] = memo_len, obj
@@ -378,14 +381,7 @@ def save_newobj(self, obj):
378381
if getnewargs:
379382
args = getnewargs() # This bette not reference obj
380383
else:
381-
# XXX These types should each grow a __getnewargs__
382-
# implementation so this special-casing is unnecessary.
383-
for cls in int, long, float, complex, str, UnicodeType, tuple:
384-
if cls and isinstance(obj, cls):
385-
args = (cls(obj),)
386-
break
387-
else:
388-
args = ()
384+
args = ()
389385

390386
save = self.save
391387
write = self.write

‎Lib/test/pickletester.py‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,21 @@ def test_newobj_list(self):
324324
## print
325325
## pickletools.dis(s)
326326

327+
def test_newobj_generic(self):
328+
for proto in [0, 1, 2]:
329+
for C in myclasses:
330+
B = C.__base__
331+
x = C(C.sample)
332+
x.foo = 42
333+
s = self.dumps(x, proto)
334+
## import pickletools
335+
## print
336+
## pickletools.dis(s)
337+
y = self.loads(s)
338+
detail = (proto, C, B, x, y, type(y))
339+
self.assertEqual(B(x), B(y), detail)
340+
self.assertEqual(x.__dict__, y.__dict__, detail)
341+
327342
# XXX Temporary hack, so long as the C implementation of pickle protocol
328343
# XXX 2 isn't ready. When it is, move the methods in TempAbstractPickleTests
329344
# XXX into AbstractPickleTests above, and get rid of TempAbstractPickleTests
@@ -405,11 +420,38 @@ def test_global_ext4(self):
405420
finally:
406421
copy_reg.remove_extension(__name__, "MyList", 0xfffff0)
407422

423+
class MyInt(int):
424+
sample = 1
425+
426+
class MyLong(long):
427+
sample = 1L
428+
429+
class MyFloat(float):
430+
sample = 1.0
431+
432+
class MyComplex(complex):
433+
sample = 1.0 + 0.0j
434+
435+
class MyStr(str):
436+
sample = "hello"
437+
438+
class MyUnicode(unicode):
439+
sample = u"hello \u1234"
440+
408441
class MyTuple(tuple):
409-
pass
442+
sample = (1, 2, 3)
410443

411444
class MyList(list):
412-
pass
445+
sample = [1, 2, 3]
446+
447+
class MyDict(dict):
448+
sample = {"a": 1, "b": 2}
449+
450+
myclasses = [MyInt, MyLong, MyFloat,
451+
# MyComplex, # XXX complex somehow doesn't work here :-(
452+
MyStr, MyUnicode,
453+
MyTuple, MyList, MyDict]
454+
413455

414456
class SlotList(MyList):
415457
__slots__ = ["foo"]

‎Lib/test/test_pickle.py‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,13 @@
1111

1212
class PickleTests(AbstractPickleTests, AbstractPickleModuleTests, XXXTemp):
1313

14-
def setUp(self):
15-
self.dumps = pickle.dumps
16-
self.loads = pickle.loads
14+
def dumps(self, arg, proto=0, fast=0):
15+
# Ignore fast
16+
return pickle.dumps(arg, proto)
17+
18+
def loads(self, buf):
19+
# Ignore fast
20+
return pickle.loads(buf)
1721

1822
module = pickle
1923
error = KeyError
@@ -22,9 +26,11 @@ class PicklerTests(AbstractPickleTests):
2226

2327
error = KeyError
2428

25-
def dumps(self, arg, proto=0):
29+
def dumps(self, arg, proto=0, fast=0):
2630
f = StringIO()
2731
p = pickle.Pickler(f, proto)
32+
if fast:
33+
p.fast = fast
2834
p.dump(arg)
2935
f.seek(0)
3036
return f.read()
@@ -36,12 +42,14 @@ def loads(self, buf):
3642

3743
class PersPicklerTests(AbstractPersistentPicklerTests):
3844

39-
def dumps(self, arg, proto=0):
45+
def dumps(self, arg, proto=0, fast=0):
4046
class PersPickler(pickle.Pickler):
4147
def persistent_id(subself, obj):
4248
return self.persistent_id(obj)
4349
f = StringIO()
4450
p = PersPickler(f, proto)
51+
if fast:
52+
p.fast = fast
4553
p.dump(arg)
4654
f.seek(0)
4755
return f.read()

‎Objects/complexobject.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -639,8 +639,15 @@ complex_conjugate(PyObject *self)
639639
return PyComplex_FromCComplex(c);
640640
}
641641

642+
static PyObject *
643+
complex_getnewargs(PyComplexObject *v)
644+
{
645+
return Py_BuildValue("(D)", v->cval);
646+
}
647+
642648
static PyMethodDef complex_methods[] = {
643649
{"conjugate", (PyCFunction)complex_conjugate, METH_NOARGS},
650+
{"__getnewargs__", (PyCFunction)complex_getnewargs, METH_NOARGS},
644651
{NULL, NULL} /* sentinel */
645652
};
646653

‎Objects/floatobject.c‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -726,6 +726,17 @@ float_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
726726
return new;
727727
}
728728

729+
static PyObject *
730+
float_getnewargs(PyFloatObject *v)
731+
{
732+
return Py_BuildValue("(d)", v->ob_fval);
733+
}
734+
735+
static PyMethodDef float_methods[] = {
736+
{"__getnewargs__", (PyCFunction)float_getnewargs, METH_NOARGS},
737+
{NULL, NULL} /* sentinel */
738+
};
739+
729740
PyDoc_STRVAR(float_doc,
730741
"float(x) -> floating point number\n\
731742
\n\
@@ -803,7 +814,7 @@ PyTypeObject PyFloat_Type = {
803814
0, /* tp_weaklistoffset */
804815
0, /* tp_iter */
805816
0, /* tp_iternext */
806-
0, /* tp_methods */
817+
float_methods, /* tp_methods */
807818
0, /* tp_members */
808819
0, /* tp_getset */
809820
0, /* tp_base */

‎Objects/intobject.c‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -850,6 +850,17 @@ int_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
850850
return new;
851851
}
852852

853+
static PyObject *
854+
int_getnewargs(PyIntObject *v)
855+
{
856+
return Py_BuildValue("(l)", v->ob_ival);
857+
}
858+
859+
static PyMethodDef int_methods[] = {
860+
{"__getnewargs__", (PyCFunction)int_getnewargs, METH_NOARGS},
861+
{NULL, NULL} /* sentinel */
862+
};
863+
853864
PyDoc_STRVAR(int_doc,
854865
"int(x[, base]) -> integer\n\
855866
\n\
@@ -931,7 +942,7 @@ PyTypeObject PyInt_Type = {
931942
0, /* tp_weaklistoffset */
932943
0, /* tp_iter */
933944
0, /* tp_iternext */
934-
0, /* tp_methods */
945+
int_methods, /* tp_methods */
935946
0, /* tp_members */
936947
0, /* tp_getset */
937948
0, /* tp_base */

‎Objects/longobject.c‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2646,6 +2646,17 @@ long_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
26462646
return (PyObject *)new;
26472647
}
26482648

2649+
static PyObject *
2650+
long_getnewargs(PyLongObject *v)
2651+
{
2652+
return Py_BuildValue("(N)", _PyLong_Copy(v));
2653+
}
2654+
2655+
static PyMethodDef long_methods[] = {
2656+
{"__getnewargs__", (PyCFunction)long_getnewargs, METH_NOARGS},
2657+
{NULL, NULL} /* sentinel */
2658+
};
2659+
26492660
PyDoc_STRVAR(long_doc,
26502661
"long(x[, base]) -> integer\n\
26512662
\n\
@@ -2726,7 +2737,7 @@ PyTypeObject PyLong_Type = {
27262737
0, /* tp_weaklistoffset */
27272738
0, /* tp_iter */
27282739
0, /* tp_iternext */
2729-
0, /* tp_methods */
2740+
long_methods, /* tp_methods */
27302741
0, /* tp_members */
27312742
0, /* tp_getset */
27322743
0, /* tp_base */

‎Objects/stringobject.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3045,6 +3045,12 @@ string_splitlines(PyStringObject *self, PyObject *args)
30453045

30463046
#undef SPLIT_APPEND
30473047

3048+
static PyObject *
3049+
string_getnewargs(PyStringObject *v)
3050+
{
3051+
return Py_BuildValue("(s#)", v->ob_sval, v->ob_size);
3052+
}
3053+
30483054

30493055
static PyMethodDef
30503056
string_methods[] = {
@@ -3091,6 +3097,7 @@ string_methods[] = {
30913097
expandtabs__doc__},
30923098
{"splitlines", (PyCFunction)string_splitlines, METH_VARARGS,
30933099
splitlines__doc__},
3100+
{"__getnewargs__", (PyCFunction)string_getnewargs, METH_NOARGS},
30943101
{NULL, NULL} /* sentinel */
30953102
};
30963103

‎Objects/tupleobject.c‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -587,6 +587,18 @@ tuplesubscript(PyTupleObject* self, PyObject* item)
587587
}
588588
}
589589

590+
static PyObject *
591+
tuple_getnewargs(PyTupleObject *v)
592+
{
593+
return Py_BuildValue("(N)", tupleslice(v, 0, v->ob_size));
594+
595+
}
596+
597+
static PyMethodDef tuple_methods[] = {
598+
{"__getnewargs__", (PyCFunction)tuple_getnewargs, METH_NOARGS},
599+
{NULL, NULL} /* sentinel */
600+
};
601+
590602
static PyMappingMethods tuple_as_mapping = {
591603
(inquiry)tuplelength,
592604
(binaryfunc)tuplesubscript,
@@ -625,7 +637,7 @@ PyTypeObject PyTuple_Type = {
625637
0, /* tp_weaklistoffset */
626638
tuple_iter, /* tp_iter */
627639
0, /* tp_iternext */
628-
0, /* tp_methods */
640+
tuple_methods, /* tp_methods */
629641
0, /* tp_members */
630642
0, /* tp_getset */
631643
0, /* tp_base */

‎Objects/unicodeobject.c‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5741,6 +5741,14 @@ unicode_endswith(PyUnicodeObject *self,
57415741
}
57425742

57435743

5744+
5745+
static PyObject *
5746+
unicode_getnewargs(PyUnicodeObject *v)
5747+
{
5748+
return Py_BuildValue("(u#)", v->str, v->length);
5749+
}
5750+
5751+
57445752
static PyMethodDef unicode_methods[] = {
57455753

57465754
/* Order is according to common usage: often used methods should
@@ -5791,6 +5799,7 @@ static PyMethodDef unicode_methods[] = {
57915799
{"freelistsize", (PyCFunction) unicode_freelistsize, METH_NOARGS},
57925800
#endif
57935801

5802+
{"__getnewargs__", (PyCFunction)unicode_getnewargs, METH_NOARGS},
57945803
{NULL, NULL}
57955804
};
57965805

0 commit comments

Comments
 (0)