Skip to content

Commit 8881beb

Browse files
ludochludoch
committed
Merge pull request #75 from aozarov/temp
make chunk size configurable
2 parents b0220b5 + a771c12 commit 8881beb

File tree

6 files changed

+36
-12
lines changed

6 files changed

+36
-12
lines changed

‎gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -479,7 +479,7 @@ public String params() {
479479
}
480480

481481
public static void printUsage() {
482-
StringBuilder actionAndParams = new StringBuilder("");
482+
StringBuilder actionAndParams = new StringBuilder();
483483
for (Map.Entry<String, StorageAction> entry : ACTIONS.entrySet()) {
484484
actionAndParams.append("\n\t").append(entry.getKey());
485485

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannel.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,11 @@ public interface BlobReadChannel extends ReadableByteChannel, Serializable, Clos
3939
void close();
4040

4141
void seek(int position) throws IOException;
42+
43+
/**
44+
* Sets the minimum size that will be read by a single RPC.
45+
* Read data will be locally buffered until consumed.
46+
*/
47+
void chunkSize(int chunkSize);
48+
4249
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
*/
3434
class BlobReadChannelImpl implements BlobReadChannel {
3535

36-
private static final int MIN_BUFFER_SIZE = 2 * 1024 * 1024;
36+
private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024;
3737
private static final long serialVersionUID = 4821762590742862669L;
3838

3939
private final StorageServiceOptions serviceOptions;
@@ -42,6 +42,7 @@ class BlobReadChannelImpl implements BlobReadChannel {
4242
private int position;
4343
private boolean isOpen;
4444
private boolean endOfStream;
45+
private int chunkSize = DEFAULT_CHUNK_SIZE;
4546

4647
private transient StorageRpc storageRpc;
4748
private transient StorageObject storageObject;
@@ -105,14 +106,19 @@ public void seek(int position) throws IOException {
105106
endOfStream = false;
106107
}
107108

109+
@Override
110+
public void chunkSize(int chunkSize) {
111+
this.chunkSize = chunkSize <= 0 ? DEFAULT_CHUNK_SIZE : chunkSize;
112+
}
113+
108114
@Override
109115
public int read(ByteBuffer byteBuffer) throws IOException {
110116
validateOpen();
111117
if (buffer == null) {
112118
if (endOfStream) {
113119
return -1;
114120
}
115-
final int toRead = Math.max(byteBuffer.remaining(), MIN_BUFFER_SIZE);
121+
final int toRead = Math.max(byteBuffer.remaining(), chunkSize);
116122
buffer = runWithRetries(new Callable<byte[]>() {
117123
@Override
118124
public byte[] call() {

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,9 @@
2929
*/
3030
public interface BlobWriteChannel extends WritableByteChannel, Serializable, Closeable {
3131

32+
/**
33+
* Sets the minimum size that will be written by a single RPC.
34+
* Written data will be buffered and only flushed upon reaching this size or closing the channel.
35+
*/
36+
void chunkSize(int chunkSize);
3237
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@
3535
class BlobWriterChannelImpl implements BlobWriteChannel {
3636

3737
private static final long serialVersionUID = 8675286882724938737L;
38-
private static final int CHUNK_SIZE = 256 * 1024;
39-
private static final int MIN_BUFFER_SIZE = 8 * CHUNK_SIZE;
38+
private static final int MIN_CHUNK_SIZE = 256 * 1024;
39+
private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE;
4040

4141
private final StorageServiceOptions options;
4242
private final Blob blob;
@@ -45,6 +45,7 @@ class BlobWriterChannelImpl implements BlobWriteChannel {
4545
private byte[] buffer = new byte[0];
4646
private int limit;
4747
private boolean isOpen = true;
48+
private int chunkSize = DEFAULT_CHUNK_SIZE;
4849

4950
private transient StorageRpc storageRpc;
5051
private transient StorageObject storageObject;
@@ -65,8 +66,8 @@ private void writeObject(ObjectOutputStream out) throws IOException {
6566
}
6667

6768
private void flush(boolean compact) {
68-
if (limit >= MIN_BUFFER_SIZE || compact && limit >= CHUNK_SIZE) {
69-
final int length = limit - limit % CHUNK_SIZE;
69+
if (limit >= chunkSize || compact && limit >= MIN_CHUNK_SIZE) {
70+
final int length = limit - limit % MIN_CHUNK_SIZE;
7071
runWithRetries(callable(new Runnable() {
7172
@Override
7273
public void run() {
@@ -75,7 +76,7 @@ public void run() {
7576
}), options.retryParams(), StorageServiceImpl.EXCEPTION_HANDLER);
7677
position += length;
7778
limit -= length;
78-
byte[] temp = new byte[compact ? limit : MIN_BUFFER_SIZE];
79+
byte[] temp = new byte[compact ? limit : chunkSize];
7980
System.arraycopy(buffer, length, temp, 0, limit);
8081
buffer = temp;
8182
}
@@ -107,8 +108,7 @@ public int write(ByteBuffer byteBuffer) throws IOException {
107108
if (spaceInBuffer >= toWrite) {
108109
byteBuffer.get(buffer, limit, toWrite);
109110
} else {
110-
buffer = Arrays.copyOf(buffer,
111-
Math.max(MIN_BUFFER_SIZE, buffer.length + toWrite - spaceInBuffer));
111+
buffer = Arrays.copyOf(buffer, Math.max(chunkSize, buffer.length + toWrite - spaceInBuffer));
112112
byteBuffer.get(buffer, limit, toWrite);
113113
}
114114
limit += toWrite;
@@ -135,4 +135,10 @@ public void run() {
135135
buffer = null;
136136
}
137137
}
138+
139+
@Override
140+
public void chunkSize(int chunkSize) {
141+
chunkSize = (chunkSize / MIN_CHUNK_SIZE) * MIN_CHUNK_SIZE;
142+
this.chunkSize = Math.max(MIN_CHUNK_SIZE, chunkSize);
143+
}
138144
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,8 @@ public static BlobTargetOption predefinedAcl(PredefinedAcl acl) {
121121
return new BlobTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry());
122122
}
123123

124-
public static BlobTargetOption doesNotExists() {
125-
return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0);
124+
public static BlobTargetOption doesNotExist() {
125+
return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0L);
126126
}
127127

128128
public static BlobTargetOption generationMatch() {

0 commit comments

Comments
 (0)