ARROW-1053: [Python] Remove unnecessary Py_INCREF in PyBuffer causing memory leak

wesm · wesm · commit ff72951f0262 · 2017-05-19T09:06:14.000-04:00
cc @BryanCutler Author: Wes McKinney <wes.mckinney@twosigma.com> Closes #704 from wesm/ARROW-1053 and squashes the following commits: 2f90337 [Wes McKinney] Remove unnecessary Py_INCREF in PyBuffer causing memory leak
diff --git a/cpp/src/arrow/python/common.cc b/cpp/src/arrow/python/common.cc
@@ -55,7 +55,6 @@ PyBuffer::PyBuffer(PyObject* obj) : Buffer(nullptr, 0), obj_(nullptr) {
     size_ = buffer->len;
     capacity_ = buffer->len;
     is_mutable_ = false;
-    Py_INCREF(obj_);
   }
 }
 
diff --git a/python/scripts/test_leak.py b/python/scripts/test_leak.py
@@ -21,6 +21,7 @@
 import numpy as np
 import memory_profiler
 import gc
+import io
 
 
 def leak():
@@ -32,4 +33,28 @@ def leak():
         table.to_pandas()
         gc.collect()
 
-leak()
+# leak()
+
+
+def leak2():
+    data = [pa.array(np.concatenate([np.random.randn(100000)] * 10))]
+    table = pa.Table.from_arrays(data, ['foo'])
+    while True:
+        print('calling to_pandas')
+        print('memory_usage: {0}'.format(memory_profiler.memory_usage()))
+        df = table.to_pandas()
+
+        batch = pa.RecordBatch.from_pandas(df)
+
+        sink = io.BytesIO()
+        writer = pa.RecordBatchFileWriter(sink, batch.schema)
+        writer.write_batch(batch)
+        writer.close()
+
+        buf_reader = pa.BufferReader(sink.getvalue())
+        reader = pa.open_file(buf_reader)
+        reader.read_all()
+
+        gc.collect()
+
+leak2()

Original file line number	Diff line number	Diff line change
`@@ -55,7 +55,6 @@ PyBuffer::PyBuffer(PyObject* obj) : Buffer(nullptr, 0), obj_(nullptr) {`
`55`	`55`	`size_ = buffer->len;`
`56`	`56`	`capacity_ = buffer->len;`
`57`	`57`	`is_mutable_ = false;`
`58`		`- Py_INCREF(obj_);`
`59`	`58`	`}`
`60`	`59`	`}`
`61`	`60`