Skip to content

Commit 9470336

Browse files
jeff5jeff5
committed
Add implementations of PyBuffer based on java.nio.ByteBuffer
Implementations and a test using the recent refactoring of the array- based buffer implementation. More refactoring and API change is pending, see comments for a start.
1 parent 5e8b9a6 commit 9470336

File tree

7 files changed

+1097
-12
lines changed

7 files changed

+1097
-12
lines changed

‎src/org/python/core/buffer/BaseArrayBuffer.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,7 @@
99
/**
1010
* Base implementation of the Buffer API for when the storage implementation is <code>byte[]</code>.
1111
* The description of {@link BaseBuffer} mostly applies. Methods provided or overridden here are
12-
* appropriate to 1-dimensional arrays backed by <code>byte[]</code>.
13-
*
12+
* appropriate to 1-dimensional arrays, of any item size, backed by <code>byte[]</code>.
1413
*/
1514
public abstract class BaseArrayBuffer extends BaseBuffer implements PyBuffer {
1615

@@ -275,5 +274,4 @@ public boolean isContiguous(char order) {
275274
}
276275
}
277276
}
278-
279277
}

‎src/org/python/core/buffer/BaseNIOBuffer.java‎

Lines changed: 345 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,345 @@
1+
package org.python.core.buffer;
2+
3+
import java.nio.BufferOverflowException;
4+
import java.nio.ByteBuffer;
5+
import java.nio.ReadOnlyBufferException;
6+
7+
import org.python.core.PyBUF;
8+
import org.python.core.PyBuffer;
9+
import org.python.core.PyException;
10+
11+
/**
12+
* Base implementation of the Buffer API for when the storage implementation is
13+
* <code>java.nio.ByteBuffer</code>. The description of {@link BaseBuffer} mostly applies. Methods
14+
* provided or overridden here are appropriate to 1-dimensional arrays, of any item size, backed by
15+
* a <code>ByteBuffer</code>.
16+
*/
17+
public abstract class BaseNIOBuffer extends BaseBuffer implements PyBuffer {
18+
19+
/**
20+
* A {@link java.nio.ByteBuffer} (possibly a direct buffer) wrapping the storage that the
21+
* exporter is sharing with the consumer. The data to be exposed may be only a subset of the
22+
* bytes in the buffer, defined by the navigation information <code>index0</code>,
23+
* <code>shape</code>, <code>strides</code>, etc., usually defined in the constructor.
24+
* <p>
25+
* Implementations must not adjust the position and limit of <code>storage</code> after
26+
* construction. It will generally be a duplicate of (not a reference to) a ByteBuffer held by
27+
* the client code. The capacity and backing store are fixed in construction, and the position
28+
* will always be {@link #index0}. The limit is always higher than any valid data, and in the
29+
* case of a contiguous buffer (with positive stride), is exactly just beyond the last item, so
30+
* that a series of ByteBuffer.get operations will yield the data.
31+
*/
32+
protected ByteBuffer storage;
33+
34+
/**
35+
* Partially construct an instance of <code>BaseNIOBuffer</code> in support of a sub-class,
36+
* specifying the 'feature flags', or at least a starting set to be adjusted later. These are
37+
* the features of the buffer exported, not the flags that form the consumer's request. The
38+
* buffer will be read-only and/or backed by a (heap) array according to the properties of the
39+
* <code>ByteBuffer</code> passed in. {@link PyBUF#FORMAT} is implicitly added to the feature
40+
* flags. To complete initialisation, the sub-class normally must assign: {@link #index0}) and
41+
* the navigation arrays ({@link #shape}, {@link #strides}), and call
42+
* {@link #checkRequestFlags(int)} passing the consumer's request flags.
43+
*
44+
* @param storage the <code>ByteBuffer</code> wrapping the exported object state. NOTE: this
45+
* <code>PyBuffer</code> keeps a reference and may manipulate the position, mark and
46+
* limit hereafter. Use {@link ByteBuffer#duplicate()} to give it an isolated copy.
47+
* @param featureFlags bit pattern that specifies the actual features allowed/required
48+
*/
49+
protected BaseNIOBuffer(ByteBuffer storage, int featureFlags) {
50+
super(featureFlags & ~(WRITABLE | AS_ARRAY));
51+
this.storage = storage;
52+
53+
// Deduce other feature flags from the client's ByteBuffer
54+
if (!storage.isReadOnly()) {
55+
addFeatureFlags(WRITABLE);
56+
}
57+
if (storage.hasArray()) {
58+
addFeatureFlags(AS_ARRAY);
59+
}
60+
}
61+
62+
@Override
63+
protected int getSize() {
64+
return shape[0];
65+
}
66+
67+
@Override
68+
public int getLen() {
69+
return shape[0] * getItemsize();
70+
}
71+
72+
@Override
73+
protected byte byteAtImpl(int byteIndex) throws IndexOutOfBoundsException {
74+
return storage.get(byteIndex);
75+
}
76+
77+
@Override
78+
protected void storeAtImpl(byte value, int byteIndex) throws IndexOutOfBoundsException,
79+
PyException {
80+
// XXX consider catching ReadonlyBufferException instead of checking (and others: index?)
81+
checkWritable();
82+
storage.put(byteIndex, value);
83+
}
84+
85+
@Override
86+
protected int byteIndex(int... indices) throws IndexOutOfBoundsException {
87+
// BaseBuffer implementation can be simplified since if indices.length!=1 we error.
88+
checkDimension(indices.length); // throws if != 1
89+
return byteIndex(indices[0]);
90+
}
91+
92+
/**
93+
* {@inheritDoc}
94+
* <p>
95+
* Specialised to one-dimensional, possibly strided buffer.
96+
*/
97+
@Override
98+
protected int calcGreatestIndex() {
99+
int stride = strides[0];
100+
if (stride == 1) {
101+
return index0 + shape[0] - 1;
102+
} else if (stride > 0) {
103+
return index0 + (shape[0] - 1) * stride;
104+
} else {
105+
return index0 - 1;
106+
}
107+
}
108+
109+
/**
110+
* {@inheritDoc}
111+
* <p>
112+
* Specialised to one-dimensional, possibly strided buffer.
113+
*/
114+
@Override
115+
protected int calcLeastIndex() {
116+
int stride = strides[0];
117+
if (stride < 0) {
118+
return index0 + (shape[0] - 1) * stride;
119+
} else {
120+
return index0;
121+
}
122+
}
123+
124+
/**
125+
* {@inheritDoc}
126+
* <p>
127+
* The default implementation in <code>BaseNIOBuffer</code> deals with the general
128+
* one-dimensional case of arbitrary item size and stride.
129+
*/
130+
@Override
131+
public void copyTo(int srcIndex, byte[] dest, int destPos, int count)
132+
throws IndexOutOfBoundsException {
133+
// Wrap the destination, taking care to reflect the necessary range we shall write.
134+
ByteBuffer destBuf = ByteBuffer.wrap(dest, destPos, count * getItemsize());
135+
copyTo(srcIndex, destBuf, count);
136+
}
137+
138+
/**
139+
* {@inheritDoc}
140+
* <p>
141+
* The default implementation in <code>BaseBuffer</code> deals with the general one-dimensional
142+
* case of arbitrary item size and stride.
143+
*/
144+
// XXX Should this become part of the PyBUffer interface?
145+
public void copyTo(ByteBuffer dest) throws BufferOverflowException, ReadOnlyBufferException,
146+
PyException {
147+
// Note shape[0] is the number of items in the buffer
148+
copyTo(0, dest, shape[0]);
149+
}
150+
151+
/**
152+
* {@inheritDoc}
153+
* <p>
154+
* The default implementation in <code>BaseNIOBuffer</code> deals with the general
155+
* one-dimensional case of arbitrary item size and stride.
156+
*/
157+
// XXX Should this become part of the PyBuffer interface?
158+
protected void copyTo(int srcIndex, ByteBuffer dest, int count) throws BufferOverflowException,
159+
ReadOnlyBufferException, IndexOutOfBoundsException, PyException {
160+
161+
if (count > 0) {
162+
163+
ByteBuffer src = getNIOByteBuffer(srcIndex);
164+
165+
// Pick up attributes necessary to choose an efficient copy strategy
166+
int itemsize = getItemsize();
167+
int stride = getStrides()[0];
168+
169+
// Strategy depends on whether items are laid end-to-end contiguously or there are gaps
170+
if (stride == itemsize) {
171+
// stride == itemsize: straight copy of contiguous bytes
172+
src.limit(src.position() + count * itemsize);
173+
dest.put(src);
174+
175+
} else if (itemsize == 1) {
176+
// Non-contiguous copy: single byte items
177+
int pos = src.position();
178+
for (int i = 0; i < count; i++) {
179+
src.position(pos);
180+
dest.put(src.get());
181+
pos += stride;
182+
}
183+
184+
} else {
185+
// Non-contiguous copy: each time, copy itemsize bytes then skip
186+
int pos = src.position();
187+
for (int i = 0; i < count; i++) {
188+
src.limit(pos + itemsize).position(pos);
189+
dest.put(src);
190+
pos += stride;
191+
}
192+
}
193+
}
194+
}
195+
196+
/**
197+
* {@inheritDoc}
198+
* <p>
199+
* The default implementation in <code>BaseNIOBuffer</code> deals with the general
200+
* one-dimensional case of arbitrary item size and stride.
201+
*/
202+
@Override
203+
public void copyFrom(byte[] src, int srcPos, int destIndex, int count)
204+
throws IndexOutOfBoundsException, PyException {
205+
// Wrap the source, taking care to reflect the range we shall read.
206+
ByteBuffer srcBuf = ByteBuffer.wrap(src, srcPos, count * getItemsize());
207+
copyFrom(srcBuf, destIndex, count);
208+
}
209+
210+
/**
211+
* {@inheritDoc}
212+
* <p>
213+
* The default implementation in <code>BaseNIOBuffer</code> deals with the general
214+
* one-dimensional case of arbitrary item size and stride.
215+
*/
216+
// XXX Should this become part of the PyBUffer interface?
217+
protected void copyFrom(ByteBuffer src, int dstIndex, int count)
218+
throws IndexOutOfBoundsException, PyException {
219+
220+
checkWritable();
221+
222+
if (count > 0) {
223+
224+
ByteBuffer dst = getNIOByteBuffer(dstIndex);
225+
226+
// Pick up attributes necessary to choose an efficient copy strategy
227+
int itemsize = getItemsize();
228+
int stride = getStrides()[0];
229+
int skip = stride - itemsize;
230+
231+
// Strategy depends on whether items are laid end-to-end or there are gaps
232+
if (skip == 0) {
233+
// Straight copy of contiguous bytes
234+
dst.put(src);
235+
236+
} else if (itemsize == 1) {
237+
// Non-contiguous copy: single byte items
238+
int pos = dst.position();
239+
for (int i = 0; i < count; i++) {
240+
dst.position(pos);
241+
dst.put(src.get());
242+
// Next byte written will be here
243+
pos += stride;
244+
}
245+
246+
} else {
247+
// Non-contiguous copy: each time, copy itemsize bytes at a time
248+
int pos = dst.position();
249+
for (int i = 0; i < count; i++) {
250+
dst.position(pos);
251+
// Delineate the next itemsize bytes in the src
252+
src.limit(src.position() + itemsize);
253+
dst.put(src);
254+
// Next byte written will be here
255+
pos += stride;
256+
}
257+
}
258+
}
259+
}
260+
261+
/**
262+
* {@inheritDoc}
263+
* <p>
264+
* The default implementation in <code>BaseNIOBuffer</code> deals with the general
265+
* one-dimensional case.
266+
*/
267+
@Override
268+
public void copyFrom(PyBuffer src) throws IndexOutOfBoundsException, PyException {
269+
270+
int length = getLen();
271+
int itemsize = getItemsize();
272+
273+
// Valid operation only if writable and same length and itemsize
274+
checkWritable();
275+
if (src.getLen() != length || src.getItemsize() != itemsize) {
276+
throw differentStructure();
277+
}
278+
279+
if (length > 0) {
280+
// Pick up attributes necessary to choose an efficient copy strategy
281+
int stride = getStrides()[0];
282+
int skip = stride - itemsize;
283+
284+
ByteBuffer dst = getNIOByteBuffer();
285+
286+
// Strategy depends on whether destination items are laid end-to-end or there are gaps
287+
if (skip == 0) {
288+
// Straight copy to contiguous bytes
289+
for (int i = 0; i < length; i++) {
290+
dst.put(src.byteAt(i));
291+
}
292+
293+
} else if (itemsize == 1) {
294+
// Non-contiguous copy: single byte items
295+
int pos = dst.position();
296+
for (int i = 0; i < length; i++) {
297+
dst.put(pos, src.byteAt(i));
298+
pos += stride;
299+
}
300+
301+
} else {
302+
// Non-contiguous copy: each time, and itemsize > 1
303+
int pos = dst.position();
304+
int s = 0;
305+
for (int i = 0; i < length; i++) {
306+
for (int j = 0; j < itemsize; j++) {
307+
dst.put(pos++, src.byteAt(s++));
308+
}
309+
pos += skip;
310+
}
311+
}
312+
}
313+
314+
}
315+
316+
@Override
317+
protected ByteBuffer getNIOByteBufferImpl() {
318+
return storage.duplicate();
319+
}
320+
321+
@SuppressWarnings("deprecation")
322+
@Override
323+
public Pointer getBuf() {
324+
checkHasArray();
325+
return new Pointer(storage.array(), index0);
326+
}
327+
328+
/**
329+
* {@inheritDoc}
330+
* <p>
331+
* Specialised in <code>BaseArrayBuffer</code> to one dimension.
332+
*/
333+
@Override
334+
public boolean isContiguous(char order) {
335+
if ("CFA".indexOf(order) < 0) {
336+
return false;
337+
} else {
338+
if (getShape()[0] < 2) {
339+
return true;
340+
} else {
341+
return getStrides()[0] == getItemsize();
342+
}
343+
}
344+
}
345+
}

‎src/org/python/core/buffer/SimpleBuffer.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,8 @@ protected int byteIndex(int index) throws IndexOutOfBoundsException {
131131
return index0 + index;
132132
}
133133

134+
// XXX Consider moving to clauses in getBufferSlice(int, int, int, int)
135+
// to avoid delegation loop where that delegates to this but in BaseBuffer the reverse.
134136
@Override
135137
public PyBuffer getBufferSlice(int flags, int start, int count) {
136138
if (count > 0) {

0 commit comments

Comments
 (0)