Skip to content

fix: update BlobAppendableUploadConfig and FlushPolicy.MinFlushSizeFlushPolicy to default to 4MiB minFlushSize and 16MiB maxPendingBytes #3249

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
BenWhitehead wants to merge 1 commit into from
Closed

fix: update BlobAppendableUploadConfig and FlushPolicy.MinFlushSizeFlushPolicy to default to 4MiB minFlushSize and 16MiB maxPendingBytes #3249

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions google-cloud-storage/src/main/java/com/google/cloud/storage/BlobAppendableUploadConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

package com.google.cloud.storage;

import static com.google.cloud.storage.ByteSizeConstants._256KiB;
import static java.util.Objects.requireNonNull;

import com.google.api.core.ApiFuture;
Expand Down Expand Up @@ -54,7 +53,7 @@ public final class BlobAppendableUploadConfig {

private static final BlobAppendableUploadConfig INSTANCE =
new BlobAppendableUploadConfig(
FlushPolicy.minFlushSize(_256KiB), CloseAction.CLOSE_WITHOUT_FINALIZING, 3);
FlushPolicy.minFlushSize(), CloseAction.CLOSE_WITHOUT_FINALIZING, 3);

private final FlushPolicy flushPolicy;
private final CloseAction closeAction;
Expand All @@ -71,7 +70,7 @@ private BlobAppendableUploadConfig(
* The {@link FlushPolicy} which will be used to determine when and how many bytes to flush to
* GCS.
*
* <p><i>Default:</i> {@link FlushPolicy#minFlushSize(int) FlushPolicy.minFlushSize(256 * 1024)}
* <p><i>Default:</i> {@link FlushPolicy#minFlushSize()}
*
* @see #withFlushPolicy(FlushPolicy)
* @since 2.51.0 This new api is in preview and is subject to breaking changes.
Expand All @@ -84,7 +83,7 @@ public FlushPolicy getFlushPolicy() {
/**
* Return an instance with the {@code FlushPolicy} set to be the specified value.
*
* <p><i>Default:</i> {@link FlushPolicy#minFlushSize(int) FlushPolicy.minFlushSize(256 * 1024)}
* <p><i>Default:</i> {@link FlushPolicy#minFlushSize()}
*
* @see #getFlushPolicy()
* @since 2.51.0 This new api is in preview and is subject to breaking changes.
Expand Down
1 change: 1 addition & 0 deletions google-cloud-storage/src/main/java/com/google/cloud/storage/ByteSizeConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ final class ByteSizeConstants {
static final int _768KiB = 768 * _1KiB;
static final int _1MiB = 1024 * _1KiB;
static final int _2MiB = 2 * _1MiB;
static final int _4MiB = 4 * _1MiB;
static final int _16MiB = 16 * _1MiB;
static final int _32MiB = 32 * _1MiB;
static final long _1GiB = 1024 * _1MiB;
Expand Down
14 changes: 11 additions & 3 deletions google-cloud-storage/src/main/java/com/google/cloud/storage/FlushPolicy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import static com.google.cloud.storage.ByteSizeConstants._16MiB;
import static com.google.cloud.storage.ByteSizeConstants._2MiB;
import static com.google.cloud.storage.ByteSizeConstants._4MiB;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalExtensionOnly;
Expand Down Expand Up @@ -67,6 +68,13 @@ public static MaxFlushSizeFlushPolicy maxFlushSize(int maxFlushSize) {
/**
* Default instance factory method for {@link MinFlushSizeFlushPolicy}.
*
* <p><i>Default:</i> logically equivalent to the following:
*
* <pre>
* {@link #minFlushSize(int) FlushPolicy.minFlushSize}(4 * 1024 * 1024)
* .{@link MinFlushSizeFlushPolicy#withMaxPendingBytes(long) withMaxPendingBytes}(16 * 1024 * 1024)
* </pre>
*
* @since 2.51.0 This new api is in preview and is subject to breaking changes.
*/
@BetaApi
Expand Down Expand Up @@ -204,7 +212,7 @@ public String toString() {
@BetaApi
public static final class MinFlushSizeFlushPolicy extends FlushPolicy {
private static final MinFlushSizeFlushPolicy INSTANCE =
new MinFlushSizeFlushPolicy(_2MiB, _16MiB);
new MinFlushSizeFlushPolicy(_4MiB, _16MiB);

private final int minFlushSize;
private final long maxPendingBytes;
Expand All @@ -217,7 +225,7 @@ private MinFlushSizeFlushPolicy(int minFlushSize, long maxPendingBytes) {
/**
* The minimum number of bytes to include in each automatic flush
*
* <p><i>Default:</i> {@code 2097152 (2 MiB)}
* <p><i>Default:</i> {@code 4194304 (4 MiB)}
*
* @see #withMinFlushSize(int)
*/
Expand All @@ -229,7 +237,7 @@ public int getMinFlushSize() {
/**
* Return an instance with the {@code minFlushSize} set to the specified value.
*
* <p><i>Default:</i> {@code 2097152 (2 MiB)}
* <p><i>Default:</i> {@code 4194304 (4 MiB)}
*
* @param minFlushSize The number of bytes to buffer before flushing.
* @return The new instance
Expand Down
5 changes: 3 additions & 2 deletions google-cloud-storage/src/test/java/com/google/cloud/storage/ITAppendableUploadTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,8 +269,9 @@ public static final class UploadConfigParameters implements ParametersProvider {
ImmutableList.of(
FlushPolicy.minFlushSize(1_000),
FlushPolicy.minFlushSize(1_000).withMaxPendingBytes(5_000),
FlushPolicy.maxFlushSize(1_000),
FlushPolicy.maxFlushSize(500_000));
FlushPolicy.maxFlushSize(500_000),
FlushPolicy.minFlushSize(),
FlushPolicy.maxFlushSize());
private static final ImmutableList<CloseAction> closeActions =
ImmutableList.copyOf(CloseAction.values());
public static final ImmutableList<Integer> objectSizes =
Expand Down
13 changes: 11 additions & 2 deletions google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import com.google.api.gax.rpc.OutOfRangeException;
import com.google.cloud.storage.BlobAppendableUpload.AppendableUploadWriteableByteChannel;
import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown;
import com.google.cloud.storage.FlushPolicy.MinFlushSizeFlushPolicy;
import com.google.cloud.storage.Storage.BlobWriteOption;
import com.google.cloud.storage.TransportCompatibility.Transport;
import com.google.cloud.storage.ZeroCopySupport.DisposableByteString;
Expand Down Expand Up @@ -77,6 +78,8 @@ public final class ITObjectReadSessionTest {

@Inject public Generator generator;

@Inject public Backend backend;

@Test
public void bytes()
throws ExecutionException, InterruptedException, TimeoutException, IOException {
Expand Down Expand Up @@ -355,9 +358,15 @@ private BlobInfo create(ChecksummedTestContent content)
throws IOException, ExecutionException, InterruptedException, TimeoutException {
BlobInfo info = BlobInfo.newBuilder(bucket, generator.randomObjectName()).build();

BlobAppendableUploadConfig config = BlobAppendableUploadConfig.of();
if (backend == Backend.TEST_BENCH) {
// workaround for https://github.com/googleapis/storage-testbench/issues/733
MinFlushSizeFlushPolicy flushPolicy =
FlushPolicy.minFlushSize(256 * 1024).withMaxPendingBytes(4 * 1024 * 1024);
config = config.withFlushPolicy(flushPolicy);
}
BlobAppendableUpload upload =
storage.blobAppendableUpload(
info, BlobAppendableUploadConfig.of(), BlobWriteOption.doesNotExist());
storage.blobAppendableUpload(info, config, BlobWriteOption.doesNotExist());
try (AppendableUploadWriteableByteChannel channel = upload.open()) {
Buffers.emptyTo(ByteBuffer.wrap(content.getBytes()), channel);
channel.finalizeAndClose();
Expand Down
Loading