Skip to content

Commit e12d2ad

Browse files
BenWhiteheadBenWhitehead
committed
chore: refactor Storage#appendableBlobUpload to take a config object rather than buffer size as a parameter
This avoids needing N new methods in the future if new features are added, and also makes it easier for defaults to be defined.
1 parent d6044a1 commit e12d2ad

9 files changed

Lines changed: 310 additions & 34 deletions

File tree

‎google-cloud-storage/clirr-ignored-differences.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@
130130
<difference>
131131
<differenceType>7012</differenceType>
132132
<className>com/google/cloud/storage/Storage</className>
133-
<method>com.google.cloud.storage.AppendableBlobUpload appendableBlobUpload(com.google.cloud.storage.BlobInfo, int, com.google.cloud.storage.Storage$BlobWriteOption[])</method>
133+
<method>com.google.cloud.storage.AppendableBlobUpload appendableBlobUpload(com.google.cloud.storage.BlobInfo, com.google.cloud.storage.AppendableBlobUploadConfig, com.google.cloud.storage.Storage$BlobWriteOption[])</method>
134134
</difference>
135135
<difference>
136136
<differenceType>7005</differenceType>

‎google-cloud-storage/src/main/java/com/google/cloud/storage/AppendableBlobUploadConfig.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import static com.google.cloud.storage.ByteSizeConstants._2MiB;
20+
import static java.util.Objects.requireNonNull;
21+
22+
import com.google.api.core.BetaApi;
23+
import com.google.cloud.storage.Storage.BlobWriteOption;
24+
import com.google.cloud.storage.TransportCompatibility.Transport;
25+
import javax.annotation.concurrent.Immutable;
26+
27+
/**
28+
* Configuration parameters for an appendable uploads channel.
29+
*
30+
* <p>Instances of this class are immutable and thread safe.
31+
*
32+
* @see Storage#appendableBlobUpload(BlobInfo, AppendableBlobUploadConfig, BlobWriteOption...)
33+
* @since 2.51.0 This new api is in preview and is subject to breaking changes.
34+
*/
35+
@Immutable
36+
@BetaApi
37+
@TransportCompatibility({Transport.GRPC})
38+
public final class AppendableBlobUploadConfig {
39+
40+
private static final AppendableBlobUploadConfig INSTANCE =
41+
new AppendableBlobUploadConfig(FlushPolicy.maxFlushSize(_2MiB));
42+
43+
private final FlushPolicy flushPolicy;
44+
45+
private AppendableBlobUploadConfig(FlushPolicy flushPolicy) {
46+
this.flushPolicy = flushPolicy;
47+
}
48+
49+
/**
50+
* The {@link FlushPolicy} which will be used to determine when and how many bytes to flush to
51+
* GCS.
52+
*
53+
* <p><i>Default:</i> {@link FlushPolicy#maxFlushSize(int) FlushPolicy.maxFlushSize(2 * 1024 *
54+
* 1024)}
55+
*
56+
* @see #withFlushPolicy(FlushPolicy)
57+
* @since 2.51.0 This new api is in preview and is subject to breaking changes.
58+
*/
59+
@BetaApi
60+
public FlushPolicy getFlushPolicy() {
61+
return flushPolicy;
62+
}
63+
64+
/**
65+
* Return an instance with the {@code FlushPolicy} set to be the specified value.
66+
*
67+
* @see #getFlushPolicy()
68+
* @since 2.51.0 This new api is in preview and is subject to breaking changes.
69+
*/
70+
@BetaApi
71+
public AppendableBlobUploadConfig withFlushPolicy(FlushPolicy flushPolicy) {
72+
requireNonNull(flushPolicy, "flushPolicy must be non null");
73+
if (this.flushPolicy.equals(flushPolicy)) {
74+
return this;
75+
}
76+
return new AppendableBlobUploadConfig(flushPolicy);
77+
}
78+
79+
/**
80+
* Default instance factory method.
81+
*
82+
* <p>The {@link FlushPolicy} of this instance is equivalent to the following:
83+
*
84+
* <pre>{@code
85+
* AppendableBlobUploadConfig.of()
86+
* .withFlushPolicy(FlushPolicy.maxFlushSize(2 * 1024 * 1024))
87+
* }</pre>
88+
*
89+
* @since 2.51.0 This new api is in preview and is subject to breaking changes.
90+
*/
91+
@BetaApi
92+
public static AppendableBlobUploadConfig of() {
93+
return INSTANCE;
94+
}
95+
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/FlushPolicy.java‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
/*
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import static com.google.cloud.storage.ByteSizeConstants._2MiB;
20+
21+
import com.google.api.core.BetaApi;
22+
import com.google.api.core.InternalExtensionOnly;
23+
import com.google.cloud.storage.GapicBidiWritableByteChannelSessionBuilder.AppendableUploadBuilder;
24+
import com.google.cloud.storage.GapicBidiWritableByteChannelSessionBuilder.AppendableUploadBuilder.BufferedAppendableUploadBuilder;
25+
import com.google.common.base.MoreObjects;
26+
import com.google.common.base.Preconditions;
27+
import java.util.Objects;
28+
import javax.annotation.concurrent.Immutable;
29+
30+
/**
31+
* Base class used for flush policies which are responsible for configuring an upload channel's
32+
* behavior with regard to flushes.
33+
*
34+
* <p>Instances of this class and all its subclasses are immutable and thread safe.
35+
*
36+
* @since 2.51.0 This new api is in preview and is subject to breaking changes.
37+
*/
38+
@BetaApi
39+
@Immutable
40+
@InternalExtensionOnly
41+
public abstract class FlushPolicy {
42+
43+
private FlushPolicy() {}
44+
45+
/**
46+
* Default instance factory method for {@link MaxFlushSizeFlushPolicy}.
47+
*
48+
* @since 2.51.0 This new api is in preview and is subject to breaking changes.
49+
*/
50+
@BetaApi
51+
public static MaxFlushSizeFlushPolicy maxFlushSize() {
52+
return MaxFlushSizeFlushPolicy.INSTANCE;
53+
}
54+
55+
/**
56+
* Alias for {@link FlushPolicy#maxFlushSize() FlushPolicy.maxFlushSize()}{@code .}{@link
57+
* MaxFlushSizeFlushPolicy#withMaxFlushSize(int) withMaxFlushSize(int)}
58+
*
59+
* @since 2.51.0 This new api is in preview and is subject to breaking changes.
60+
*/
61+
@BetaApi
62+
public static MaxFlushSizeFlushPolicy maxFlushSize(int maxFlushSize) {
63+
return maxFlushSize().withMaxFlushSize(maxFlushSize);
64+
}
65+
66+
abstract BufferedAppendableUploadBuilder apply(AppendableUploadBuilder builder);
67+
68+
@Override
69+
public abstract boolean equals(Object obj);
70+
71+
@Override
72+
public abstract int hashCode();
73+
74+
@Override
75+
public abstract String toString();
76+
77+
/**
78+
* Define a {@link FlushPolicy} where a max number of bytes will be flushed to GCS per flush.
79+
*
80+
* <p>If there are not enough bytes to trigger a flush, they will be held in memory until there
81+
* are enough bytes, or an explicit flush is performed by closing the channel. If more bytes are
82+
* provided than the configured {@code maxFlushSize}, multiple flushes will be performed.
83+
*
84+
* <p>Instances of this class are immutable and thread safe.
85+
*
86+
* @since 2.51.0 This new api is in preview and is subject to breaking changes.
87+
*/
88+
@Immutable
89+
@BetaApi
90+
public static final class MaxFlushSizeFlushPolicy extends FlushPolicy {
91+
private static final MaxFlushSizeFlushPolicy INSTANCE = new MaxFlushSizeFlushPolicy(_2MiB);
92+
93+
private final int maxFlushSize;
94+
95+
public MaxFlushSizeFlushPolicy(int maxFlushSize) {
96+
this.maxFlushSize = maxFlushSize;
97+
}
98+
99+
/**
100+
* The maximum number of bytes to include in each automatic flush
101+
*
102+
* <p><i>Default:</i> {@code 2097152 (2 MiB)}
103+
*
104+
* @see #withMaxFlushSize(int)
105+
*/
106+
@BetaApi
107+
public int getMaxFlushSize() {
108+
return maxFlushSize;
109+
}
110+
111+
/**
112+
* Return an instance with the {@code maxFlushSize} set to the specified value.
113+
*
114+
* <p><i>Default:</i> {@code 2097152 (2 MiB)}
115+
*
116+
* @param maxFlushSize The number of bytes to buffer before flushing.
117+
* @return The new instance
118+
* @see #getMaxFlushSize()
119+
*/
120+
@BetaApi
121+
public MaxFlushSizeFlushPolicy withMaxFlushSize(int maxFlushSize) {
122+
Preconditions.checkArgument(maxFlushSize >= 0, "maxFlushSize >= 0 (%s >= 0)", maxFlushSize);
123+
if (this.maxFlushSize == maxFlushSize) {
124+
return this;
125+
}
126+
return new MaxFlushSizeFlushPolicy(maxFlushSize);
127+
}
128+
129+
@Override
130+
BufferedAppendableUploadBuilder apply(AppendableUploadBuilder builder) {
131+
return builder.buffered(BufferHandle.allocate(maxFlushSize));
132+
}
133+
134+
@Override
135+
public boolean equals(Object o) {
136+
if (this == o) {
137+
return true;
138+
}
139+
if (!(o instanceof MaxFlushSizeFlushPolicy)) {
140+
return false;
141+
}
142+
MaxFlushSizeFlushPolicy that = (MaxFlushSizeFlushPolicy) o;
143+
return maxFlushSize == that.maxFlushSize;
144+
}
145+
146+
@Override
147+
public int hashCode() {
148+
return Objects.hashCode(maxFlushSize);
149+
}
150+
151+
@Override
152+
public String toString() {
153+
return MoreObjects.toStringHelper(this).add("maxFlushSize", maxFlushSize).toString();
154+
}
155+
}
156+
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import com.google.cloud.storage.BlobWriteSessionConfig.WriterFactory;
5252
import com.google.cloud.storage.BufferedWritableByteChannelSession.BufferedWritableByteChannel;
5353
import com.google.cloud.storage.Conversions.Decoder;
54+
import com.google.cloud.storage.GapicBidiWritableByteChannelSessionBuilder.AppendableUploadBuilder;
5455
import com.google.cloud.storage.GrpcUtils.ZeroCopyServerStreamingCallable;
5556
import com.google.cloud.storage.HmacKey.HmacKeyMetadata;
5657
import com.google.cloud.storage.HmacKey.HmacKeyState;
@@ -1424,22 +1425,17 @@ public BlobWriteSession blobWriteSession(BlobInfo info, BlobWriteOption... optio
14241425
@BetaApi
14251426
@Override
14261427
public AppendableBlobUpload appendableBlobUpload(
1427-
BlobInfo blob, int bufferSize, BlobWriteOption... options) throws IOException {
1428-
boolean takeOver = blob.getGeneration() != null;
1429-
return getAppendableBlobUpload(blob, bufferSize, takeOver, options);
1430-
}
1431-
1432-
private AppendableBlobUpload getAppendableBlobUpload(
1433-
BlobInfo blob, int bufferSize, boolean takeOver, BlobWriteOption... options)
1428+
BlobInfo blob, AppendableBlobUploadConfig uploadConfig, BlobWriteOption... options)
14341429
throws IOException {
1430+
boolean takeOver = blob.getGeneration() != null;
14351431
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blob);
14361432
BidiWriteObjectRequest req =
14371433
takeOver
14381434
? getBidiWriteObjectRequestForTakeover(blob, opts)
14391435
: getBidiWriteObjectRequest(blob, opts);
14401436
BidiAppendableWrite baw = new BidiAppendableWrite(req, takeOver);
14411437
ApiFuture<BidiAppendableWrite> startAppendableWrite = ApiFutures.immediateFuture(baw);
1442-
WritableByteChannelSession<BufferedWritableByteChannel, BidiWriteObjectResponse> build =
1438+
AppendableUploadBuilder appendableUploadBuilder =
14431439
ResumableMedia.gapic()
14441440
.write()
14451441
.bidiByteChannel(storageClient.bidiWriteObjectCallable())
@@ -1469,8 +1465,11 @@ public boolean shouldRetry(
14691465
.idempotent()
14701466
.shouldRetry(previousThrowable, null);
14711467
}
1472-
}))
1473-
.buffered(BufferHandle.allocate(bufferSize))
1468+
}));
1469+
WritableByteChannelSession<BufferedWritableByteChannel, BidiWriteObjectResponse> build =
1470+
uploadConfig
1471+
.getFlushPolicy()
1472+
.apply(appendableUploadBuilder)
14741473
.setStartAsync(startAppendableWrite)
14751474
.setGetCallable(storageClient.getObjectCallable())
14761475
.build();

‎google-cloud-storage/src/main/java/com/google/cloud/storage/OtelStorageDecorator.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1508,8 +1508,9 @@ public ApiFuture<BlobReadSession> blobReadSession(BlobId id, BlobSourceOption...
15081508
@TransportCompatibility({Transport.GRPC})
15091509
@BetaApi
15101510
public AppendableBlobUpload appendableBlobUpload(
1511-
BlobInfo blob, int bufferSize, BlobWriteOption... options) throws IOException {
1512-
return delegate.appendableBlobUpload(blob, bufferSize, options);
1511+
BlobInfo blob, AppendableBlobUploadConfig uploadConfig, BlobWriteOption... options)
1512+
throws IOException {
1513+
return delegate.appendableBlobUpload(blob, uploadConfig, options);
15131514
}
15141515

15151516
@Override

‎google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5888,7 +5888,8 @@ default ApiFuture<BlobReadSession> blobReadSession(BlobId id, BlobSourceOption..
58885888
@BetaApi
58895889
@TransportCompatibility({Transport.GRPC})
58905890
default AppendableBlobUpload appendableBlobUpload(
5891-
BlobInfo blob, int bufferSize, BlobWriteOption... options) throws IOException {
5891+
BlobInfo blob, AppendableBlobUploadConfig uploadConfig, BlobWriteOption... options)
5892+
throws IOException {
58925893
return throwGrpcOnly(
58935894
fmtMethodName("appendableBlobUpload", BlobId.class, BlobWriteOption.class));
58945895
}

0 commit comments

Comments
 (0)