lazyWriteChannel;
+ private BufferHandle bufferHandle;
+
+ /**
+ * This is tracked for compatibility with BlobWriteChannel, such that simply creating a writer
+ * will create an object.
+ *
+ * In the future we should move away from this behavior, and only create an object if write is
+ * called.
+ */
+ protected boolean writeCalledAtLeastOnce;
+
+ protected BaseStorageWriteChannel(Decoder objectDecoder) {
+ this.objectDecoder = objectDecoder;
+ this.result = SettableApiFuture.create();
+ this.open = true;
+ this.chunkSize = _16MiB;
+ this.writeCalledAtLeastOnce = false;
+ }
+
+ @Override
+ public final synchronized void setChunkSize(int chunkSize) {
+ Preconditions.checkArgument(chunkSize > 0, "chunkSize must be > 0, received %d", chunkSize);
+ Preconditions.checkState(
+ bufferHandle == null || bufferHandle.position() == 0,
+ "unable to change chunk size with data buffered");
+ this.chunkSize = chunkSize;
+ }
+
+ @Override
+ public final synchronized boolean isOpen() {
+ return open;
+ }
+
+ @Override
+ public final synchronized void close() throws IOException {
+ try {
+ if (open && !writeCalledAtLeastOnce) {
+ this.write(ByteBuffer.allocate(0));
+ }
+ if (internalGetLazyChannel().isOpen()) {
+ StorageException.wrapIOException(internalGetLazyChannel().getChannel()::close);
+ }
+ } finally {
+ open = false;
+ }
+ }
+
+ @Override
+ public final synchronized int write(ByteBuffer src) throws IOException {
+ if (!open) {
+ throw new ClosedChannelException();
+ }
+ writeCalledAtLeastOnce = true;
+ try {
+ BufferedWritableByteChannel tmp = internalGetLazyChannel().getChannel();
+ if (!tmp.isOpen()) {
+ return 0;
+ }
+ int write = tmp.write(src);
+ return write;
+ } catch (StorageException e) {
+ throw new IOException(e);
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException(StorageException.coalesce(e));
+ }
+ }
+
+ @Override
+ public final ApiFuture getObject() {
+ return ApiFutures.transform(result, objectDecoder::decode, MoreExecutors.directExecutor());
+ }
+
+ protected final BufferHandle getBufferHandle() {
+ if (bufferHandle == null) {
+ bufferHandle = BufferHandle.allocate(Buffers.alignSize(getChunkSize(), _256KiB));
+ }
+ return bufferHandle;
+ }
+
+ protected final int getChunkSize() {
+ return chunkSize;
+ }
+
+ @Nullable
+ protected final T getResolvedObject() {
+ if (result.isDone()) {
+ return StorageException.wrapFutureGet(result);
+ } else {
+ return null;
+ }
+ }
+
+ protected final long getCommittedPosition() {
+ return position;
+ }
+
+ protected final void setCommittedPosition(long l) {
+ position = l;
+ }
+
+ protected final void setOpen(boolean isOpen) {
+ this.open = isOpen;
+ }
+
+ protected abstract LazyWriteChannel newLazyWriteChannel();
+
+ private LazyWriteChannel internalGetLazyChannel() {
+ if (lazyWriteChannel == null) {
+ LazyWriteChannel tmp = newLazyWriteChannel();
+ ApiFuture future = tmp.getSession().getResult();
+ ApiFutures.addCallback(
+ future,
+ new ApiFutureCallback() {
+ @Override
+ public void onFailure(Throwable t) {
+ if (!result.isDone()) {
+ result.setException(t);
+ }
+ }
+
+ @Override
+ public void onSuccess(T t) {
+ if (!result.isDone()) {
+ result.set(t);
+ }
+ }
+ },
+ MoreExecutors.directExecutor());
+ lazyWriteChannel = tmp;
+ }
+ return lazyWriteChannel;
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobReadChannelV2.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobReadChannelV2.java
index 1f3c5b52cc..82aba8b136 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobReadChannelV2.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobReadChannelV2.java
@@ -143,15 +143,18 @@ public String toString() {
static final class BlobReadChannelContext {
private final HttpStorageOptions storageOptions;
private final HttpRetryAlgorithmManager retryAlgorithmManager;
+ private final HttpClientContext httpClientContext;
private final Storage apiaryClient;
private BlobReadChannelContext(
HttpStorageOptions storageOptions,
- Storage apiaryClient,
- HttpRetryAlgorithmManager retryAlgorithmManager) {
+ HttpRetryAlgorithmManager retryAlgorithmManager,
+ HttpClientContext httpClientContext,
+ Storage apiaryClient) {
this.storageOptions = storageOptions;
- this.apiaryClient = apiaryClient;
this.retryAlgorithmManager = retryAlgorithmManager;
+ this.httpClientContext = httpClientContext;
+ this.apiaryClient = apiaryClient;
}
public HttpStorageOptions getStorageOptions() {
@@ -162,13 +165,20 @@ public HttpRetryAlgorithmManager getRetryAlgorithmManager() {
return retryAlgorithmManager;
}
+ public HttpClientContext getHttpClientContext() {
+ return httpClientContext;
+ }
+
public Storage getApiaryClient() {
return apiaryClient;
}
static BlobReadChannelContext from(HttpStorageOptions options) {
return new BlobReadChannelContext(
- options, options.getStorageRpcV1().getStorage(), options.getRetryAlgorithmManager());
+ options,
+ options.getRetryAlgorithmManager(),
+ HttpClientContext.from(options.getStorageRpcV1()),
+ options.getStorageRpcV1().getStorage());
}
static BlobReadChannelContext from(com.google.cloud.storage.Storage s) {
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java
index 7b19258790..c74c94f4cd 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java
@@ -16,371 +16,78 @@
package com.google.cloud.storage;
-import static java.util.Objects.requireNonNull;
-import static java.util.concurrent.Executors.callable;
-
-import com.google.api.core.InternalApi;
+import com.google.api.core.ApiFuture;
import com.google.api.gax.retrying.ResultRetryAlgorithm;
import com.google.api.services.storage.model.StorageObject;
import com.google.cloud.BaseWriteChannel;
import com.google.cloud.RestorableState;
-import com.google.cloud.RetryHelper;
import com.google.cloud.WriteChannel;
-import java.math.BigInteger;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import org.checkerframework.checker.nullness.qual.NonNull;
-
-/** Write channel implementation to upload Google Cloud Storage blobs. */
-class BlobWriteChannel extends BaseWriteChannel {
-
- private final ResultRetryAlgorithm algorithmForWrite;
- // Detect if flushBuffer() is being retried or not.
- // TODO: I don't think this is thread safe, and there's probably a better way to detect a retry
- // occuring.
- private boolean retrying = false;
- private boolean checkingForLastChunk = false;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
- // Contains metadata of the updated object or null if upload is not completed.
- private StorageObject storageObject;
-
- BlobWriteChannel(
- HttpStorageOptions storageOptions,
- BlobInfo blobInfo,
- String uploadId,
- ResultRetryAlgorithm algorithmForWrite) {
- super(storageOptions, blobInfo, uploadId);
- this.algorithmForWrite = algorithmForWrite;
- }
+/**
+ * Hierarchy retained for {@link RestorableState#restore()}. Will be removed in next major version!
+ */
+@Deprecated
+class BlobWriteChannel extends BaseWriteChannel
+ implements StorageWriteChannel {
- boolean isRetrying() {
- return retrying;
+ private BlobWriteChannel() {
+ super(null, null, null);
+ throw new IllegalStateException("Illegal method access");
}
- StorageObject getStorageObject() {
- return storageObject;
+ @Override
+ public ApiFuture getObject() {
+ throw new IllegalStateException("Illegal method access");
}
@Override
protected HttpStorageOptions getOptions() {
- return (HttpStorageOptions) super.getOptions();
- }
-
- private StorageObject transmitChunk(
- int chunkOffset, int chunkLength, long position, boolean last) {
- return getOptions()
- .getStorageRpcV1()
- .writeWithResponse(getUploadId(), getBuffer(), chunkOffset, position, chunkLength, last);
+ throw new IllegalStateException("Illegal method access");
}
- private long getRemotePosition() {
- return getOptions().getStorageRpcV1().getCurrentUploadOffset(getUploadId());
- }
-
- private static StorageException unrecoverableState(
- String uploadId,
- int chunkOffset,
- int chunkLength,
- long localPosition,
- long remotePosition,
- boolean last) {
- return unrecoverableState(
- uploadId,
- chunkOffset,
- chunkLength,
- localPosition,
- remotePosition,
- last,
- "Unable to recover in upload.\nThis may be a symptom of multiple clients uploading to the same upload session.");
- }
-
- private static StorageException errorResolvingMetadataLastChunk(
- String uploadId,
- int chunkOffset,
- int chunkLength,
- long localPosition,
- long remotePosition,
- boolean last) {
- return unrecoverableState(
- uploadId,
- chunkOffset,
- chunkLength,
- localPosition,
- remotePosition,
- last,
- "Unable to load object metadata to determine if last chunk was successfully written");
- }
-
- private static StorageException unrecoverableState(
- String uploadId,
- int chunkOffset,
- int chunkLength,
- long localPosition,
- long remotePosition,
- boolean last,
- String message) {
- StringBuilder sb = new StringBuilder();
- sb.append(message).append("\n\n");
- sb.append("For debugging purposes:\n");
- sb.append("uploadId: ").append(uploadId).append('\n');
- sb.append("chunkOffset: ").append(chunkOffset).append('\n');
- sb.append("chunkLength: ").append(chunkLength).append('\n');
- sb.append("localOffset: ").append(localPosition).append('\n');
- sb.append("remoteOffset: ").append(remotePosition).append('\n');
- sb.append("lastChunk: ").append(last).append("\n\n");
- return new StorageException(0, sb.toString());
- }
-
- // Retriable interruption occurred.
- // Variables:
- // chunk = getBuffer()
- // localNextByteOffset == getPosition()
- // chunkSize = getChunkSize()
- //
- // Case 1: localNextByteOffset == remoteNextByteOffset:
- // Retrying the entire chunk
- //
- // Case 2: localNextByteOffset < remoteNextByteOffset
- // && driftOffset < chunkSize:
- // Upload progressed and localNextByteOffset is not in-sync with
- // remoteNextByteOffset and driftOffset is less than chunkSize.
- // driftOffset must be less than chunkSize for it to retry using
- // chunk maintained in memory.
- // Find the driftOffset by subtracting localNextByteOffset from
- // remoteNextByteOffset.
- // Use driftOffset to determine where to restart from using the chunk in
- // memory.
- //
- // Case 3: localNextByteOffset < remoteNextByteOffset
- // && driftOffset == chunkSize:
- // Special case of Case 2.
- // If chunkSize is equal to driftOffset then remoteNextByteOffset has moved on
- // to the next chunk.
- //
- // Case 4: localNextByteOffset < remoteNextByteOffset
- // && driftOffset > chunkSize:
- // Throw exception as remoteNextByteOffset has drifted beyond the retriable
- // chunk maintained in memory. This is not possible unless there's multiple
- // clients uploading to the same resumable upload session.
- //
- // Case 5: localNextByteOffset > remoteNextByteOffset:
- // For completeness, this case is not possible because it would require retrying
- // a 400 status code which is not allowed.
- //
- // Case 6: remoteNextByteOffset==-1 && last == true
- // Upload is complete and retry occurred in the "last" chunk. Data sent was
- // received by the service.
- //
- // Case 7: remoteNextByteOffset==-1 && last == false && !checkingForLastChunk
- // Not last chunk and are not checkingForLastChunk, allow for the client to
- // catch up to final chunk which meets
- // Case 6.
- //
- // Case 8: remoteNextByteOffset==-1 && last == false && checkingForLastChunk
- // Not last chunk and checkingForLastChunk means this is the second time we
- // hit this case, meaning the upload was completed by a different client.
- //
- // Case 9: Only possible if the client local offset continues beyond the remote
- // offset which is not possible.
- //
@Override
protected void flushBuffer(final int length, final boolean lastChunk) {
- try {
- Retrying.run(
- getOptions(),
- algorithmForWrite,
- callable(
- new Runnable() {
- @Override
- public void run() {
- // Get remote offset from API
- final long localPosition = getPosition();
- // For each request it should be possible to retry from its location in this code
- final long remotePosition = isRetrying() ? getRemotePosition() : localPosition;
- final int chunkOffset = (int) (remotePosition - localPosition);
- final int chunkLength = length - chunkOffset;
- final boolean uploadAlreadyComplete = remotePosition == -1;
- // Enable isRetrying state to reduce number of calls to getRemotePosition()
- if (!isRetrying()) {
- retrying = true;
- }
- if (uploadAlreadyComplete && lastChunk) {
- // Case 6
- // Request object metadata if not available
- long totalBytes = getPosition() + length;
- if (storageObject == null) {
- storageObject =
- getOptions()
- .getStorageRpcV1()
- .queryCompletedResumableUpload(getUploadId(), totalBytes);
- }
- // the following checks are defined here explicitly to provide a more
- // informative if either storageObject is unable to be resolved or it's size is
- // unable to be determined. This scenario is a very rare case of failure that
- // can arise when packets are lost.
- if (storageObject == null) {
- throw errorResolvingMetadataLastChunk(
- getUploadId(),
- chunkOffset,
- chunkLength,
- localPosition,
- remotePosition,
- lastChunk);
- }
- // Verify that with the final chunk we match the blob length
- BigInteger size = storageObject.getSize();
- if (size == null) {
- throw errorResolvingMetadataLastChunk(
- getUploadId(),
- chunkOffset,
- chunkLength,
- localPosition,
- remotePosition,
- lastChunk);
- }
- if (size.longValue() != totalBytes) {
- throw unrecoverableState(
- getUploadId(),
- chunkOffset,
- chunkLength,
- localPosition,
- remotePosition,
- lastChunk);
- }
- retrying = false;
- } else if (uploadAlreadyComplete && !lastChunk && !checkingForLastChunk) {
- // Case 7
- // Make sure this is the second to last chunk.
- checkingForLastChunk = true;
- // Continue onto next chunk in case this is the last chunk
- } else if (localPosition <= remotePosition && chunkOffset < getChunkSize()) {
- // Case 1 && Case 2
- // We are in a position to send a chunk
- storageObject =
- transmitChunk(chunkOffset, chunkLength, remotePosition, lastChunk);
- retrying = false;
- } else if (localPosition < remotePosition && chunkOffset == getChunkSize()) {
- // Case 3
- // Continue to next chunk to catch up with remotePosition we are one chunk
- // behind
- retrying = false;
- } else {
- // Case 4 && Case 8 && Case 9
- throw unrecoverableState(
- getUploadId(),
- chunkOffset,
- chunkLength,
- localPosition,
- remotePosition,
- lastChunk);
- }
- }
- }),
- Function.identity());
- } catch (RetryHelper.RetryHelperException e) {
- throw StorageException.translateAndThrow(e);
- }
+ throw new IllegalStateException("Illegal method access");
}
protected StateImpl.Builder stateBuilder() {
- return StateImpl.builder(getOptions(), getEntity(), getUploadId())
- .setResultRetryAlgorithm(algorithmForWrite);
+ throw new IllegalStateException("Illegal method access");
}
- static Builder newBuilder() {
- return new Builder();
- }
-
- static final class Builder {
- private HttpStorageOptions storageOptions;
- private BlobInfo blobInfo;
- private Supplier<@NonNull String> uploadIdSupplier;
- private ResultRetryAlgorithm algorithmForWrite;
-
- public Builder setStorageOptions(HttpStorageOptions storageOptions) {
- this.storageOptions = storageOptions;
- return this;
- }
-
- public Builder setBlobInfo(BlobInfo blobInfo) {
- this.blobInfo = blobInfo;
- return this;
- }
-
- public Builder setUploadIdSupplier(Supplier uploadIdSupplier) {
- this.uploadIdSupplier = uploadIdSupplier;
- return this;
- }
-
- public Builder setAlgorithmForWrite(ResultRetryAlgorithm algorithmForWrite) {
- this.algorithmForWrite = algorithmForWrite;
- return this;
- }
-
- BlobWriteChannel build() {
- String uploadId = requireNonNull(uploadIdSupplier, "uploadId must be non null").get();
- return new BlobWriteChannel(
- requireNonNull(storageOptions, "storageOptions must be non null"),
- blobInfo,
- requireNonNull(uploadId, "uploadId must be non null"),
- requireNonNull(algorithmForWrite, "algorithmForWrite must be non null"));
- }
+ @Override
+ public RestorableState capture() {
+ throw new IllegalStateException("Illegal method access");
}
+ /** Retained for binary compatibility. Will be removed at next major version! */
+ @SuppressWarnings("unused")
+ @Deprecated
+ @VisibleForTesting
static class StateImpl extends BaseWriteChannel.BaseState {
private static final long serialVersionUID = -6700378962714601115L;
- private final ResultRetryAlgorithm algorithmForWrite;
+ private ResultRetryAlgorithm algorithmForWrite;
- StateImpl(Builder builder) {
+ private StateImpl(Builder builder) {
super(builder);
- this.algorithmForWrite = builder.algorithmForWrite;
- }
-
- @InternalApi
- Builder toBuilder(StorageOptions options) {
- Builder builder = new Builder(options, entity, uploadId);
- builder.setPosition(position).setBuffer(buffer).setIsOpen(isOpen).setChunkSize(chunkSize);
- return builder;
- }
-
- static class Builder extends BaseWriteChannel.BaseState.Builder {
- private ResultRetryAlgorithm algorithmForWrite;
-
- private Builder(StorageOptions options, BlobInfo blobInfo, String uploadId) {
- super(options, blobInfo, uploadId);
- }
-
- public Builder setResultRetryAlgorithm(ResultRetryAlgorithm algorithmForWrite) {
- this.algorithmForWrite = algorithmForWrite;
- return this;
- }
-
- @Override
- public RestorableState build() {
- return new StateImpl(this);
- }
- }
-
- static Builder builder(StorageOptions options, BlobInfo blobInfo, String uploadId) {
- return new Builder(options, blobInfo, uploadId);
}
@Override
public WriteChannel restore() {
try {
- BlobWriteChannel channel =
- BlobWriteChannel.newBuilder()
- // should be okay, as the value always comes from an instance we gate to have
- // HttpStorageOptions
- .setStorageOptions((HttpStorageOptions) serviceOptions)
- .setBlobInfo(entity)
- .setUploadIdSupplier(() -> uploadId)
- .setAlgorithmForWrite(algorithmForWrite)
- .build();
- channel.restore(this);
- return channel;
+ StorageObject encode =
+ entity != null ? Conversions.apiary().blobInfo().encode(entity) : null;
+ return new BlobWriteChannelV2.BlobWriteChannelV2State(
+ (HttpStorageOptions) serviceOptions,
+ JsonResumableWrite.of(encode, ImmutableMap.of(), uploadId, position),
+ position,
+ isOpen,
+ chunkSize,
+ buffer)
+ .restore();
} catch (Exception e) {
throw StorageException.coalesce(e);
}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannelV2.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannelV2.java
new file mode 100644
index 0000000000..48bad4de6c
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannelV2.java
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.api.core.ApiFutures;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.cloud.RestorableState;
+import com.google.cloud.WriteChannel;
+import com.google.cloud.storage.BlobReadChannelV2.BlobReadChannelContext;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Objects;
+
+final class BlobWriteChannelV2 extends BaseStorageWriteChannel {
+
+ private final BlobReadChannelContext blobChannelContext;
+ private final JsonResumableWrite start;
+
+ BlobWriteChannelV2(BlobReadChannelContext blobChannelContext, JsonResumableWrite start) {
+ super(Conversions.apiary().blobInfo());
+ this.start = start;
+ this.blobChannelContext = blobChannelContext;
+ }
+
+ @Override
+ public synchronized RestorableState capture() {
+ final byte[] bufferSnapshot;
+ BufferHandle handle = getBufferHandle();
+ if (handle.position() > 0) {
+ ByteBuffer byteBuffer = handle.get();
+ // duplicate so we don't actually modify the existing instance
+ ByteBuffer dup = byteBuffer.duplicate();
+ dup.flip();
+ int remaining = dup.remaining();
+ bufferSnapshot = new byte[remaining];
+ dup.get(bufferSnapshot);
+ } else {
+ bufferSnapshot = new byte[0];
+ }
+ return new BlobWriteChannelV2State(
+ blobChannelContext.getStorageOptions(),
+ start,
+ getCommittedPosition(),
+ isOpen(),
+ getChunkSize(),
+ bufferSnapshot);
+ }
+
+ @Override
+ protected LazyWriteChannel newLazyWriteChannel() {
+ return new LazyWriteChannel<>(
+ () ->
+ ResumableMedia.http()
+ .write()
+ .byteChannel(blobChannelContext.getHttpClientContext())
+ .resumable()
+ .setCommittedBytesCallback(this::setCommittedPosition)
+ .withRetryConfig(
+ blobChannelContext.getStorageOptions().asRetryDependencies(),
+ blobChannelContext.getRetryAlgorithmManager().idempotent())
+ .buffered(getBufferHandle())
+ .setStartAsync(ApiFutures.immediateFuture(start))
+ .build());
+ }
+
+ static final class BlobWriteChannelV2State
+ implements RestorableState, Serializable {
+ private static final long serialVersionUID = -1901664719924133474L;
+
+ private final HttpStorageOptions options;
+ private final JsonResumableWrite resumableWrite;
+
+ private final Long position;
+ private final Boolean isOpen;
+ private final Integer chunkSize;
+ private final byte[] bufferSnapshot;
+
+ BlobWriteChannelV2State(
+ HttpStorageOptions options,
+ JsonResumableWrite resumableWrite,
+ Long position,
+ Boolean isOpen,
+ Integer chunkSize,
+ byte[] bufferSnapshot) {
+ this.options = options;
+ this.resumableWrite = resumableWrite;
+ this.position = position;
+ this.isOpen = isOpen;
+ this.chunkSize = chunkSize;
+ this.bufferSnapshot = bufferSnapshot;
+ }
+
+ @Override
+ public WriteChannel restore() {
+ JsonResumableWrite resumableWrite = this.resumableWrite;
+ if (position != null) {
+ resumableWrite = resumableWrite.withBeginOffset(position);
+ }
+ BlobWriteChannelV2 channel =
+ new BlobWriteChannelV2(BlobReadChannelContext.from(options), resumableWrite);
+ if (chunkSize != null) {
+ channel.setChunkSize(chunkSize);
+ }
+ if (bufferSnapshot != null && bufferSnapshot.length > 0) {
+ BufferHandle handle = channel.getBufferHandle();
+ ByteBuffer byteBuffer = handle.get();
+ byteBuffer.put(bufferSnapshot);
+ }
+ if (position != null) {
+ channel.setCommittedPosition(position);
+ }
+ if (isOpen != null) {
+ channel.setOpen(isOpen);
+ }
+ return channel;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof BlobWriteChannelV2State)) {
+ return false;
+ }
+ BlobWriteChannelV2State that = (BlobWriteChannelV2State) o;
+ return Objects.equals(options, that.options)
+ && Objects.equals(resumableWrite, that.resumableWrite)
+ && Objects.equals(position, that.position)
+ && Objects.equals(isOpen, that.isOpen)
+ && Objects.equals(chunkSize, that.chunkSize)
+ && Arrays.equals(bufferSnapshot, that.bufferSnapshot);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hash(options, resumableWrite, position, isOpen, chunkSize);
+ result = 31 * result + Arrays.hashCode(bufferSnapshot);
+ return result;
+ }
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Buffers.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Buffers.java
index 1e397608ac..a828523cab 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Buffers.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Buffers.java
@@ -54,6 +54,10 @@ static void position(Buffer b, int position) {
b.position(position);
}
+ static int position(Buffer b) {
+ return b.position();
+ }
+
/**
* attempt to drain all of {@code content} into {@code dsts} starting from {@code dsts[0]} through
* {@code dsts[dsts.length - 1]}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ByteRangeSpec.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ByteRangeSpec.java
index ca4cd64d71..524bfe3c2e 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ByteRangeSpec.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ByteRangeSpec.java
@@ -16,6 +16,9 @@
package com.google.cloud.storage;
+import static com.google.api.client.util.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkArgument;
+
import com.google.api.core.InternalApi;
import com.google.common.base.MoreObjects;
import com.google.common.base.MoreObjects.ToStringHelper;
@@ -124,6 +127,20 @@ static ByteRangeSpec explicitClosed(
return create(beginOffset, endOffsetInclusive, LeftClosedRightClosedByteRangeSpec::new);
}
+ static ByteRangeSpec parse(String string) {
+ checkNotNull(string, "Range header is null");
+ checkArgument(string.startsWith("bytes="), "malformed Range header value: %s", string);
+
+ int i = string.indexOf('-');
+ String minS = string.substring(6, i);
+ String maxS = string.substring(i + 1);
+
+ long min = Long.parseLong(minS);
+ long max = Long.parseLong(maxS);
+
+ return explicitClosed(min, max);
+ }
+
private static ByteRangeSpec create(
@Nullable Long beginOffset,
@Nullable Long length,
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ByteSizeConstants.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ByteSizeConstants.java
index cac56927fa..cbdbd94d67 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ByteSizeConstants.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ByteSizeConstants.java
@@ -23,10 +23,19 @@ final class ByteSizeConstants {
static final int _256KiB = 256 * _1KiB;
static final int _384KiB = 384 * _1KiB;
static final int _512KiB = 512 * _1KiB;
+ static final int _768KiB = 768 * _1KiB;
static final int _1MiB = 1024 * _1KiB;
static final int _2MiB = 2 * _1MiB;
static final int _16MiB = 16 * _1MiB;
static final int _32MiB = 32 * _1MiB;
+ static final long _1GiB = 1024 * _1MiB;
+ static final long _1TiB = 1024 * _1GiB;
+ static final long _5TiB = 5 * _1TiB;
+
+ static final long _128KiBL = 131072L;
+ static final long _256KiBL = 262144L;
+ static final long _512KiBL = 524288L;
+ static final long _768KiBL = 786432L;
private ByteSizeConstants() {}
}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Conversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Conversions.java
index 680dd1aec4..3b81639a08 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Conversions.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Conversions.java
@@ -46,6 +46,10 @@ default Decoder andThen(Decoder d) {
static Decoder identity() {
return (x) -> x;
}
+
+ default Decoder compose(Decoder before) {
+ return in -> this.decode(before.decode(in));
+ }
}
interface Codec extends Encoder, Decoder {
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedWritableByteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedWritableByteChannel.java
index 007e48021d..951a52edbc 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedWritableByteChannel.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedWritableByteChannel.java
@@ -62,16 +62,6 @@ final class GapicUnbufferedWritableByteChannel<
requestFactory.bucketName(), writeCtx.getConfirmedBytes()::set, resultFuture::set);
}
- @Override
- public int write(ByteBuffer src) throws IOException {
- return Math.toIntExact(write(new ByteBuffer[] {src}));
- }
-
- @Override
- public long write(ByteBuffer[] srcs) throws IOException {
- return write(srcs, 0, srcs.length);
- }
-
@Override
public long write(ByteBuffer[] srcs, int srcsOffset, int srcLength) throws IOException {
if (!open) {
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicWritableByteChannelSessionBuilder.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicWritableByteChannelSessionBuilder.java
index e39bf85d33..80bca1c572 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicWritableByteChannelSessionBuilder.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicWritableByteChannelSessionBuilder.java
@@ -20,6 +20,7 @@
import com.google.api.core.ApiFuture;
import com.google.api.core.ApiFutures;
+import com.google.api.core.InternalApi;
import com.google.api.core.SettableApiFuture;
import com.google.api.gax.retrying.ResultRetryAlgorithm;
import com.google.api.gax.rpc.ClientStreamingCallable;
@@ -219,10 +220,12 @@ final class ResumableUploadBuilder {
private RetryingDependencies deps;
private ResultRetryAlgorithm alg;
+ private boolean fsyncEvery;
ResumableUploadBuilder() {
this.deps = RetryingDependencies.attemptOnce();
this.alg = Retrying.neverRetry();
+ this.fsyncEvery = true;
}
ResumableUploadBuilder withRetryConfig(RetryingDependencies deps, ResultRetryAlgorithm alg) {
@@ -231,6 +234,12 @@ ResumableUploadBuilder withRetryConfig(RetryingDependencies deps, ResultRetryAlg
return this;
}
+ @InternalApi
+ ResumableUploadBuilder setFsyncEvery(boolean fsyncEvery) {
+ this.fsyncEvery = fsyncEvery;
+ return this;
+ }
+
/**
* Do not apply any intermediate buffering. Any call to {@link
* java.nio.channels.WritableByteChannel#write(ByteBuffer)} will be segmented as is and sent to
@@ -281,7 +290,10 @@ UnbufferedWritableByteChannelSession build() {
return new UnbufferedWriteSession<>(
requireNonNull(start, "start must be non null"),
bindFunction(
- WriteFlushStrategy.fsyncEveryFlush(write, deps, alg, Retrying::newCallContext),
+ fsyncEvery
+ ? WriteFlushStrategy.fsyncEveryFlush(
+ write, deps, alg, Retrying::newCallContext)
+ : WriteFlushStrategy.fsyncOnClose(write),
ResumableWrite::identity)
.andThen(StorageByteChannels.writable()::createSynchronized));
}
@@ -310,7 +322,10 @@ BufferedWritableByteChannelSession build() {
return new BufferedWriteSession<>(
requireNonNull(start, "start must be non null"),
bindFunction(
- WriteFlushStrategy.fsyncEveryFlush(write, deps, alg, Retrying::newCallContext),
+ fsyncEvery
+ ? WriteFlushStrategy.fsyncEveryFlush(
+ write, deps, alg, Retrying::newCallContext)
+ : WriteFlushStrategy.fsyncOnClose(write),
ResumableWrite::identity)
.andThen(c -> new DefaultBufferedWritableByteChannel(bufferHandle, c))
.andThen(StorageByteChannels.writable()::createSynchronized));
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcBlobWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcBlobWriteChannel.java
index 7eaae03397..f3520180b3 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcBlobWriteChannel.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcBlobWriteChannel.java
@@ -16,38 +16,23 @@
package com.google.cloud.storage;
-import static com.google.cloud.storage.ByteSizeConstants._16MiB;
-import static com.google.cloud.storage.ByteSizeConstants._256KiB;
-
import com.google.api.core.ApiFuture;
import com.google.api.gax.retrying.ResultRetryAlgorithm;
import com.google.api.gax.rpc.ClientStreamingCallable;
import com.google.cloud.RestorableState;
import com.google.cloud.WriteChannel;
-import com.google.cloud.storage.BufferedWritableByteChannelSession.BufferedWritableByteChannel;
import com.google.cloud.storage.Retrying.RetryingDependencies;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Suppliers;
import com.google.storage.v2.WriteObjectRequest;
import com.google.storage.v2.WriteObjectResponse;
-import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.function.Supplier;
-final class GrpcBlobWriteChannel implements WriteChannel {
-
- private final LazyWriteChannel lazyWriteChannel;
-
- private int chunkSize = _16MiB;
+final class GrpcBlobWriteChannel extends BaseStorageWriteChannel {
- /**
- * This is tracked for compatibility with BlobWriteChannel, such that simply creating a writer
- * will create an object.
- *
- * In the future we should move away from this behavior, and only create an object if write is
- * called.
- */
- private boolean writeCalledAtLeastOnce = false;
+ private final ClientStreamingCallable write;
+ private final RetryingDependencies deps;
+ private final ResultRetryAlgorithm alg;
+ private final Supplier> start;
+ private final Hasher hasher;
GrpcBlobWriteChannel(
ClientStreamingCallable write,
@@ -55,27 +40,12 @@ final class GrpcBlobWriteChannel implements WriteChannel {
ResultRetryAlgorithm alg,
Supplier> start,
Hasher hasher) {
- lazyWriteChannel =
- new LazyWriteChannel(
- Suppliers.memoize(
- () ->
- ResumableMedia.gapic()
- .write()
- .byteChannel(write)
- .setHasher(hasher)
- .setByteStringStrategy(ByteStringStrategy.copy())
- .resumable()
- .withRetryConfig(deps, alg)
- .buffered(BufferHandle.allocate(Buffers.alignSize(chunkSize, _256KiB)))
- .setStartAsync(start.get())
- .build()));
- }
-
- @Override
- public void setChunkSize(int chunkSize) {
- Preconditions.checkState(
- !lazyWriteChannel.isOpened(), "Unable to change chunkSize after write");
- this.chunkSize = chunkSize;
+ super(Conversions.grpc().blobInfo().compose(WriteObjectResponse::getResource));
+ this.write = write;
+ this.deps = deps;
+ this.alg = alg;
+ this.start = start;
+ this.hasher = hasher;
}
@Override
@@ -84,57 +54,18 @@ public RestorableState capture() {
}
@Override
- public int write(ByteBuffer src) throws IOException {
- writeCalledAtLeastOnce = true;
- return lazyWriteChannel.getChannel().write(src);
- }
-
- @Override
- public boolean isOpen() {
- if (!writeCalledAtLeastOnce) {
- return true;
- } else {
- return lazyWriteChannel.isOpened() && lazyWriteChannel.getChannel().isOpen();
- }
- }
-
- @Override
- public void close() throws IOException {
- if (!writeCalledAtLeastOnce) {
- lazyWriteChannel.getChannel().write(ByteBuffer.allocate(0));
- }
- if (isOpen()) {
- lazyWriteChannel.getChannel().close();
- }
- }
-
- ApiFuture getResults() {
- return lazyWriteChannel.session.get().getResult();
- }
-
- private static final class LazyWriteChannel {
- private final Supplier> session;
- private final Supplier channel;
-
- private boolean opened = false;
-
- public LazyWriteChannel(
- Supplier> session) {
- this.session = session;
- this.channel =
- Suppliers.memoize(
- () -> {
- opened = true;
- return session.get().open();
- });
- }
-
- public BufferedWritableByteChannel getChannel() {
- return channel.get();
- }
-
- public boolean isOpened() {
- return opened;
- }
+ protected LazyWriteChannel newLazyWriteChannel() {
+ return new LazyWriteChannel<>(
+ () ->
+ ResumableMedia.gapic()
+ .write()
+ .byteChannel(write)
+ .setHasher(hasher)
+ .setByteStringStrategy(ByteStringStrategy.copy())
+ .resumable()
+ .withRetryConfig(deps, alg)
+ .buffered(getBufferHandle())
+ .setStartAsync(start.get())
+ .build());
}
}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcResumableSession.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcResumableSession.java
new file mode 100644
index 0000000000..8de44fb654
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcResumableSession.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.api.core.ApiFutures;
+import com.google.api.gax.grpc.GrpcCallContext;
+import com.google.api.gax.retrying.ResultRetryAlgorithm;
+import com.google.api.gax.rpc.ClientStreamingCallable;
+import com.google.api.gax.rpc.UnaryCallable;
+import com.google.cloud.storage.BufferedWritableByteChannelSession.BufferedWritableByteChannel;
+import com.google.cloud.storage.Conversions.Decoder;
+import com.google.cloud.storage.Retrying.RetryingDependencies;
+import com.google.storage.v2.Object;
+import com.google.storage.v2.QueryWriteStatusRequest;
+import com.google.storage.v2.QueryWriteStatusResponse;
+import com.google.storage.v2.WriteObjectRequest;
+import com.google.storage.v2.WriteObjectResponse;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+final class GrpcResumableSession {
+
+ private final RetryingDependencies deps;
+ private final ResultRetryAlgorithm alg;
+ private final ClientStreamingCallable writeCallable;
+ private final UnaryCallable
+ queryWriteStatusCallable;
+ private final ResumableWrite resumableWrite;
+ private final Hasher hasher;
+
+ GrpcResumableSession(
+ RetryingDependencies deps,
+ ResultRetryAlgorithm alg,
+ ClientStreamingCallable writeCallable,
+ UnaryCallable queryWriteStatusCallable,
+ ResumableWrite resumableWrite,
+ Hasher hasher) {
+ this.deps = deps;
+ this.alg = alg;
+ this.writeCallable = writeCallable;
+ this.queryWriteStatusCallable = queryWriteStatusCallable;
+ this.resumableWrite = resumableWrite;
+ this.hasher = hasher;
+ }
+
+ ResumableOperationResult<@Nullable Object> query() {
+ QueryWriteStatusRequest.Builder b =
+ QueryWriteStatusRequest.newBuilder().setUploadId(resumableWrite.getRes().getUploadId());
+ if (resumableWrite.getReq().hasCommonObjectRequestParams()) {
+ b.setCommonObjectRequestParams(resumableWrite.getReq().getCommonObjectRequestParams());
+ }
+ QueryWriteStatusRequest req = b.build();
+ try {
+ QueryWriteStatusResponse response = queryWriteStatusCallable.call(req);
+ if (response.hasResource()) {
+ return ResumableOperationResult.complete(
+ response.getResource(), response.getResource().getSize());
+ } else {
+ return ResumableOperationResult.incremental(response.getPersistedSize());
+ }
+ } catch (Exception e) {
+ throw StorageException.coalesce(e);
+ }
+ }
+
+ ResumableOperationResult<@Nullable Object> put(RewindableContent content) {
+ AtomicBoolean dirty = new AtomicBoolean(false);
+ GrpcCallContext retryingCallContext = Retrying.newCallContext();
+ BufferHandle handle = BufferHandle.allocate(ByteSizeConstants._2MiB);
+
+ return Retrying.run(
+ deps,
+ alg,
+ () -> {
+ if (dirty.getAndSet(true)) {
+ ResumableOperationResult<@Nullable Object> query = query();
+ if (query.getObject() != null) {
+ return query;
+ } else {
+ content.rewindTo(query.getPersistedSize());
+ }
+ }
+ WritableByteChannelSession session =
+ ResumableMedia.gapic()
+ .write()
+ .byteChannel(writeCallable.withDefaultCallContext(retryingCallContext))
+ .setByteStringStrategy(ByteStringStrategy.copy())
+ .setHasher(hasher)
+ .resumable()
+ .setFsyncEvery(false)
+ .buffered(handle)
+ .setStartAsync(ApiFutures.immediateFuture(resumableWrite))
+ .build();
+
+ try (BufferedWritableByteChannel channel = session.open()) {
+ content.writeTo(channel);
+ }
+
+ WriteObjectResponse response = session.getResult().get();
+ if (response.hasResource()) {
+ return ResumableOperationResult.complete(
+ response.getResource(), response.getResource().getSize());
+ } else {
+ return ResumableOperationResult.incremental(response.getPersistedSize());
+ }
+ },
+ Decoder.identity());
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java
index 14b4789901..d7d4059196 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java
@@ -28,6 +28,7 @@
import static java.util.Objects.requireNonNull;
import com.google.api.core.ApiFuture;
+import com.google.api.core.ApiFutures;
import com.google.api.core.BetaApi;
import com.google.api.gax.grpc.GrpcCallContext;
import com.google.api.gax.paging.AbstractPage;
@@ -35,6 +36,7 @@
import com.google.api.gax.retrying.ResultRetryAlgorithm;
import com.google.api.gax.rpc.ApiException;
import com.google.api.gax.rpc.ApiExceptions;
+import com.google.api.gax.rpc.ClientStreamingCallable;
import com.google.api.gax.rpc.NotFoundException;
import com.google.api.gax.rpc.StatusCode;
import com.google.api.gax.rpc.UnaryCallable;
@@ -72,6 +74,7 @@
import com.google.common.collect.Streams;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
+import com.google.common.util.concurrent.MoreExecutors;
import com.google.iam.v1.GetIamPolicyRequest;
import com.google.iam.v1.SetIamPolicyRequest;
import com.google.iam.v1.TestIamPermissionsRequest;
@@ -123,7 +126,6 @@
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
-import java.nio.channels.SeekableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.Files;
import java.nio.file.OpenOption;
@@ -138,6 +140,7 @@
import java.util.Spliterator;
import java.util.Spliterators.AbstractSpliterator;
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Predicate;
@@ -285,55 +288,34 @@ public Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOp
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
WriteObjectRequest req = getWriteObjectRequest(blobInfo, opts);
- long size = Files.size(path);
- if (size < bufferSize) {
- // ignore the bufferSize argument if the file is smaller than it
- GrpcCallContext merge = Utils.merge(grpcCallContext, Retrying.newCallContext());
- return Retrying.run(
- getOptions(),
- retryAlgorithmManager.getFor(req),
- () -> {
- BufferedWritableByteChannelSession session =
- ResumableMedia.gapic()
- .write()
- .byteChannel(storageClient.writeObjectCallable().withDefaultCallContext(merge))
- .setHasher(Hasher.enabled())
- .setByteStringStrategy(ByteStringStrategy.noCopy())
- .direct()
- .buffered(Buffers.allocate(size))
- .setRequest(req)
- .build();
-
- try (SeekableByteChannel src = Files.newByteChannel(path, READ_OPS);
- BufferedWritableByteChannel dst = session.open()) {
- ByteStreams.copy(src, dst);
- } catch (Exception e) {
- throw StorageException.coalesce(e);
- }
- return session.getResult();
- },
- this::getBlob);
- } else {
- ApiFuture start = startResumableWrite(grpcCallContext, req);
- BufferedWritableByteChannelSession session =
- ResumableMedia.gapic()
- .write()
- .byteChannel(
- storageClient.writeObjectCallable().withDefaultCallContext(grpcCallContext))
- .setHasher(Hasher.noop())
- .setByteStringStrategy(ByteStringStrategy.noCopy())
- .resumable()
- .withRetryConfig(getOptions(), retryAlgorithmManager.idempotent())
- .buffered(Buffers.allocateAligned(bufferSize, _256KiB))
- .setStartAsync(start)
- .build();
- try (SeekableByteChannel src = Files.newByteChannel(path, READ_OPS);
- BufferedWritableByteChannel dst = session.open()) {
- ByteStreams.copy(src, dst);
- } catch (Exception e) {
- throw StorageException.coalesce(e);
+ ClientStreamingCallable write =
+ storageClient.writeObjectCallable().withDefaultCallContext(grpcCallContext);
+
+ ApiFuture start = startResumableWrite(grpcCallContext, req);
+ ApiFuture session2 =
+ ApiFutures.transform(
+ start,
+ rw ->
+ ResumableSession.grpc(
+ getOptions(),
+ retryAlgorithmManager.idempotent(),
+ write,
+ storageClient.queryWriteStatusCallable(),
+ rw,
+ Hasher.noop()),
+ MoreExecutors.directExecutor());
+ try {
+ GrpcResumableSession got = session2.get();
+ ResumableOperationResult<@Nullable Object> put = got.put(RewindableContent.of(path));
+ Object object = put.getObject();
+ if (object == null) {
+ // if by some odd chance the put didn't get the Object, query for it
+ ResumableOperationResult<@Nullable Object> query = got.query();
+ object = query.getObject();
}
- return getBlob(session.getResult());
+ return codecs.blobInfo().decode(object).asBlob(this);
+ } catch (InterruptedException | ExecutionException e) {
+ throw StorageException.coalesce(e);
}
}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpClientContext.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpClientContext.java
new file mode 100644
index 0000000000..35c93c02f5
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpClientContext.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpRequestFactory;
+import com.google.api.client.json.JsonObjectParser;
+import com.google.api.client.util.ObjectParser;
+import com.google.cloud.storage.spi.v1.StorageRpc;
+import io.opencensus.trace.Span;
+import io.opencensus.trace.Tracer;
+import io.opencensus.trace.Tracing;
+import java.util.List;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+final class HttpClientContext {
+
+ private final HttpRequestFactory requestFactory;
+ private final ObjectParser objectParser;
+ private final Tracer tracer;
+
+ private HttpClientContext(
+ HttpRequestFactory requestFactory, ObjectParser objectParser, Tracer tracer) {
+ this.requestFactory = requestFactory;
+ this.objectParser = objectParser;
+ this.tracer = tracer;
+ }
+
+ @SuppressWarnings({"unchecked", "SameParameterValue"})
+ static @Nullable String firstHeaderValue(
+ @NonNull HttpHeaders headers, @NonNull String headerName) {
+ Object v = headers.get(headerName);
+ // HttpHeaders doesn't type its get method, so we have to jump through hoops here
+ if (v instanceof List) {
+ List list = (List) v;
+ return list.get(0);
+ } else {
+ return null;
+ }
+ }
+
+ public HttpRequestFactory getRequestFactory() {
+ return requestFactory;
+ }
+
+ public ObjectParser getObjectParser() {
+ return objectParser;
+ }
+
+ public Tracer getTracer() {
+ return tracer;
+ }
+
+ public Span startSpan(String name) {
+ // record events is hardcoded to true in HttpStorageRpc, preserve it here
+ return tracer.spanBuilder(name).setRecordEvents(true).startSpan();
+ }
+
+ static HttpClientContext from(StorageRpc storageRpc) {
+ return new HttpClientContext(
+ storageRpc.getStorage().getRequestFactory(),
+ storageRpc.getStorage().getObjectParser(),
+ Tracing.getTracer());
+ }
+
+ public static HttpClientContext of(
+ HttpRequestFactory requestFactory, JsonObjectParser jsonObjectParser) {
+ return new HttpClientContext(requestFactory, jsonObjectParser, Tracing.getTracer());
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpContentRange.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpContentRange.java
new file mode 100644
index 0000000000..49fc5a1949
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpContentRange.java
@@ -0,0 +1,271 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.base.MoreObjects;
+import java.util.Objects;
+import java.util.function.UnaryOperator;
+
+abstract class HttpContentRange {
+
+ private final boolean finalizing;
+
+ private HttpContentRange(boolean finalizing) {
+ this.finalizing = finalizing;
+ }
+
+ public abstract String getHeaderValue();
+
+ public abstract boolean endOffsetEquals(long e);
+
+ public boolean isFinalizing() {
+ return finalizing;
+ }
+
+ static Total of(ByteRangeSpec spec, long size) {
+ checkArgument(size >= 0, "size must be >= 0");
+ checkArgument(size >= spec.endOffsetInclusive(), "size must be >= end");
+ return new Total(spec, size);
+ }
+
+ static Incomplete of(ByteRangeSpec spec) {
+ return new Incomplete(spec);
+ }
+
+ static Size of(long size) {
+ checkArgument(size >= 0, "size must be >= 0");
+ return new Size(size);
+ }
+
+ static Query query() {
+ return Query.INSTANCE;
+ }
+
+ static HttpContentRange parse(String string) {
+ if ("bytes */*".equals(string)) {
+ return HttpContentRange.query();
+ } else if (string.startsWith("bytes */")) {
+ return HttpContentRange.of(Long.parseLong(string.substring(8)));
+ } else {
+ int idxDash = string.indexOf('-');
+ int idxSlash = string.indexOf('/');
+
+ String beginS = string.substring(6, idxDash);
+ String endS = string.substring(idxDash + 1, idxSlash);
+ long begin = Long.parseLong(beginS);
+ long end = Long.parseLong(endS);
+ if (string.endsWith("/*")) {
+ return HttpContentRange.of(ByteRangeSpec.explicitClosed(begin, end));
+ } else {
+ String sizeS = string.substring(idxSlash + 1);
+ long size = Long.parseLong(sizeS);
+ return HttpContentRange.of(ByteRangeSpec.explicitClosed(begin, end), size);
+ }
+ }
+ }
+
+ static final class Incomplete extends HttpContentRange implements HasRange {
+
+ private final ByteRangeSpec spec;
+
+ private Incomplete(ByteRangeSpec spec) {
+ super(false);
+ this.spec = spec;
+ }
+
+ @Override
+ public String getHeaderValue() {
+ return String.format("bytes %d-%d/*", spec.beginOffset(), spec.endOffsetInclusive());
+ }
+
+ @Override
+ public boolean endOffsetEquals(long e) {
+ return e == spec.endOffset();
+ }
+
+ @Override
+ public ByteRangeSpec range() {
+ return spec;
+ }
+
+ @Override
+ public Incomplete map(UnaryOperator f) {
+ return new Incomplete(f.apply(spec));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof Incomplete)) {
+ return false;
+ }
+ Incomplete that = (Incomplete) o;
+ return Objects.equals(spec, that.spec);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(spec);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this).add("spec", spec).toString();
+ }
+ }
+
+ static final class Total extends HttpContentRange implements HasRange, HasSize {
+
+ private final ByteRangeSpec spec;
+ private final long size;
+
+ private Total(ByteRangeSpec spec, long size) {
+ super(true);
+ this.spec = spec;
+ this.size = size;
+ }
+
+ @Override
+ public String getHeaderValue() {
+ return String.format("bytes %d-%d/%d", spec.beginOffset(), spec.endOffsetInclusive(), size);
+ }
+
+ @Override
+ public boolean endOffsetEquals(long e) {
+ return e == spec.endOffset();
+ }
+
+ @Override
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ public ByteRangeSpec range() {
+ return spec;
+ }
+
+ @Override
+ public Total map(UnaryOperator f) {
+ return new Total(f.apply(spec), size);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof Total)) {
+ return false;
+ }
+ Total total = (Total) o;
+ return size == total.size && Objects.equals(spec, total.spec);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(spec, size);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this).add("spec", spec).add("size", size).toString();
+ }
+ }
+
+ static final class Size extends HttpContentRange implements HasSize {
+
+ private final long size;
+
+ private Size(long size) {
+ super(true);
+ this.size = size;
+ }
+
+ @Override
+ public String getHeaderValue() {
+ return String.format("bytes */%d", size);
+ }
+
+ @Override
+ public boolean endOffsetEquals(long e) {
+ return false;
+ }
+
+ @Override
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof Size)) {
+ return false;
+ }
+ Size size1 = (Size) o;
+ return size == size1.size;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(size);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this).add("size", size).toString();
+ }
+ }
+
+ static final class Query extends HttpContentRange {
+
+ private static final Query INSTANCE = new Query();
+
+ private Query() {
+ super(false);
+ }
+
+ @Override
+ public boolean endOffsetEquals(long e) {
+ return false;
+ }
+
+ @Override
+ public String getHeaderValue() {
+ return "bytes */*";
+ }
+ }
+
+ interface HasRange {
+
+ ByteRangeSpec range();
+
+ T map(UnaryOperator f);
+ }
+
+ interface HasSize {
+
+ long getSize();
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpDownloadSessionBuilder.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpDownloadSessionBuilder.java
index 370c9d6cea..2cf21d767b 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpDownloadSessionBuilder.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpDownloadSessionBuilder.java
@@ -38,7 +38,7 @@ final class HttpDownloadSessionBuilder {
private HttpDownloadSessionBuilder() {}
- public static HttpDownloadSessionBuilder create() {
+ static HttpDownloadSessionBuilder create() {
return INSTANCE;
}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java
index 3483dd9074..b525cf75b8 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java
@@ -29,6 +29,7 @@
import com.google.cloud.TransportOptions;
import com.google.cloud.http.HttpTransportOptions;
import com.google.cloud.spi.ServiceRpcFactory;
+import com.google.cloud.storage.Retrying.RetryingDependencies;
import com.google.cloud.storage.TransportCompatibility.Transport;
import com.google.cloud.storage.spi.StorageRpcFactory;
import com.google.cloud.storage.spi.v1.HttpStorageRpc;
@@ -51,6 +52,7 @@ public class HttpStorageOptions extends StorageOptions {
private static final String DEFAULT_HOST = "https://storage.googleapis.com";
private final HttpRetryAlgorithmManager retryAlgorithmManager;
+ private final transient RetryDependenciesAdapter retryDepsAdapter;
private HttpStorageOptions(Builder builder, StorageDefaults serviceDefaults) {
super(builder, serviceDefaults);
@@ -58,6 +60,7 @@ private HttpStorageOptions(Builder builder, StorageDefaults serviceDefaults) {
new HttpRetryAlgorithmManager(
MoreObjects.firstNonNull(
builder.storageRetryStrategy, defaults().getStorageRetryStrategy()));
+ retryDepsAdapter = new RetryDependenciesAdapter();
}
@Override
@@ -102,6 +105,11 @@ public static HttpStorageDefaults defaults() {
return HttpStorageDefaults.INSTANCE;
}
+ @InternalApi
+ RetryingDependencies asRetryDependencies() {
+ return retryDepsAdapter;
+ }
+
public static class Builder extends StorageOptions.Builder {
private StorageRetryStrategy storageRetryStrategy;
@@ -321,4 +329,23 @@ public ServiceRpc create(StorageOptions options) {
}
}
}
+
+ /**
+ * We don't yet want to make HttpStorageOptions itself implement {@link RetryingDependencies} but
+ * we do need use it in a couple places, for those we create this adapter.
+ */
+ private final class RetryDependenciesAdapter implements RetryingDependencies {
+
+ private RetryDependenciesAdapter() {}
+
+ @Override
+ public RetrySettings getRetrySettings() {
+ return HttpStorageOptions.this.getRetrySettings();
+ }
+
+ @Override
+ public ApiClock getClock() {
+ return HttpStorageOptions.this.getClock();
+ }
+ }
}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpUploadSessionBuilder.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpUploadSessionBuilder.java
new file mode 100644
index 0000000000..a673c1f9b4
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpUploadSessionBuilder.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import org.checkerframework.checker.nullness.qual.NonNull;
+
+final class HttpUploadSessionBuilder {
+ private static final HttpUploadSessionBuilder INSTANCE = new HttpUploadSessionBuilder();
+
+ private HttpUploadSessionBuilder() {}
+
+ static HttpUploadSessionBuilder create() {
+ return INSTANCE;
+ }
+
+ @NonNull
+ HttpWritableByteChannelSessionBuilder byteChannel(@NonNull HttpClientContext httpClientContext) {
+ return new HttpWritableByteChannelSessionBuilder(httpClientContext);
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpWritableByteChannelSessionBuilder.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpWritableByteChannelSessionBuilder.java
new file mode 100644
index 0000000000..19abf0928b
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpWritableByteChannelSessionBuilder.java
@@ -0,0 +1,190 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import static java.util.Objects.requireNonNull;
+
+import com.google.api.core.ApiFuture;
+import com.google.api.core.SettableApiFuture;
+import com.google.api.gax.retrying.ResultRetryAlgorithm;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.cloud.storage.ChannelSession.BufferedWriteSession;
+import com.google.cloud.storage.ChannelSession.UnbufferedWriteSession;
+import com.google.cloud.storage.Retrying.RetryingDependencies;
+import com.google.cloud.storage.UnbufferedWritableByteChannelSession.UnbufferedWritableByteChannel;
+import java.nio.ByteBuffer;
+import java.util.function.BiFunction;
+import java.util.function.LongConsumer;
+import org.checkerframework.checker.nullness.qual.NonNull;
+
+final class HttpWritableByteChannelSessionBuilder {
+
+ private static final int DEFAULT_BUFFER_CAPACITY = ByteSizeConstants._16MiB;
+ @NonNull private final HttpClientContext httpClientContext;
+
+ HttpWritableByteChannelSessionBuilder(@NonNull HttpClientContext httpClientContext) {
+ this.httpClientContext =
+ requireNonNull(httpClientContext, "httpClientContext must be non null");
+ }
+
+ /**
+ * The build {@link WritableByteChannelSession} will perform a "Resumable" upload.
+ *
+ * A "Resumable" upload will sync the transmitted data with GCS upon each individual flush and
+ * when the channel is closed.
+ *
+ *
If an error is returned the individual flush can be transparently retried.
+ */
+ ResumableUploadBuilder resumable() {
+ return new ResumableUploadBuilder(httpClientContext);
+ }
+
+ static final class ResumableUploadBuilder {
+
+ @NonNull private final HttpClientContext httpClientContext;
+ private RetryingDependencies deps;
+ private ResultRetryAlgorithm alg;
+ private LongConsumer committedBytesCallback;
+
+ ResumableUploadBuilder(@NonNull HttpClientContext httpClientContext) {
+ this.httpClientContext = httpClientContext;
+ this.deps = RetryingDependencies.attemptOnce();
+ this.alg = Retrying.neverRetry();
+ this.committedBytesCallback = l -> {};
+ }
+
+ ResumableUploadBuilder setCommittedBytesCallback(@NonNull LongConsumer committedBytesCallback) {
+ this.committedBytesCallback =
+ requireNonNull(committedBytesCallback, "committedBytesCallback must be non null");
+ return this;
+ }
+
+ ResumableUploadBuilder withRetryConfig(
+ @NonNull RetryingDependencies deps, @NonNull ResultRetryAlgorithm alg) {
+ this.deps = requireNonNull(deps, "deps must be non null");
+ this.alg = requireNonNull(alg, "alg must be non null");
+ return this;
+ }
+
+ /**
+ * Do not apply any intermediate buffering. Any call to {@link
+ * java.nio.channels.WritableByteChannel#write(ByteBuffer)} will be segmented as is and sent to
+ * GCS.
+ *
+ *
Note: this is considered an advanced API, and should not be used in circumstances in which
+ * control of {@link ByteBuffer}s sent to {@code write} is not self-contained.
+ */
+ UnbufferedResumableUploadBuilder unbuffered() {
+ return new UnbufferedResumableUploadBuilder();
+ }
+
+ /** Buffer up to {@link #DEFAULT_BUFFER_CAPACITY} worth of bytes before attempting to flush */
+ BufferedResumableUploadBuilder buffered() {
+ return buffered(BufferHandle.allocate(DEFAULT_BUFFER_CAPACITY));
+ }
+
+ /**
+ * Buffer using {@code byteBuffer} worth of space before attempting to flush.
+ *
+ *
The provided {@link ByteBuffer} should be aligned with GCSs block size of 256
+ * KiB.
+ */
+ BufferedResumableUploadBuilder buffered(ByteBuffer byteBuffer) {
+ return buffered(BufferHandle.handleOf(byteBuffer));
+ }
+
+ BufferedResumableUploadBuilder buffered(BufferHandle bufferHandle) {
+ return new BufferedResumableUploadBuilder(bufferHandle);
+ }
+
+ /**
+ * When constructing any of our channel sessions, there is always a {@link
+ * GapicUnbufferedWritableByteChannel} at the bottom of it. This method creates a BiFunction
+ * which will instantiate the {@link GapicUnbufferedWritableByteChannel} when provided with a
+ * {@code StartT} value and a {@code SettableApiFuture}.
+ *
+ * As part of providing the function, the provided parameters {@code FlusherFactory} and
+ * {@code f} are "bound" into the returned function. In conjunction with the configured fields
+ * of this class a new instance of {@link GapicUnbufferedWritableByteChannel} can be
+ * constructed.
+ */
+ private BiFunction<
+ JsonResumableWrite, SettableApiFuture, UnbufferedWritableByteChannel>
+ bindFunction() {
+ // it is theoretically possible that the setter methods for the following variables could
+ // be called again between when this method is invoked and the resulting function is invoked.
+ // To ensure we are using the specified values at the point in time they are bound to the
+ // function read them into local variables which will be closed over rather than the class
+ // fields.
+ RetryingDependencies boundDeps = deps;
+ ResultRetryAlgorithm boundAlg = alg;
+ return (start, resultFuture) ->
+ new ApiaryUnbufferedWritableByteChannel(
+ httpClientContext, boundDeps, boundAlg, start, resultFuture, committedBytesCallback);
+ }
+
+ final class UnbufferedResumableUploadBuilder {
+
+ private ApiFuture start;
+
+ /**
+ * Set the Future which will contain the ResumableWrite information necessary to open the
+ * Write stream.
+ */
+ UnbufferedResumableUploadBuilder setStartAsync(ApiFuture start) {
+ this.start = requireNonNull(start, "start must be non null");
+ return this;
+ }
+
+ UnbufferedWritableByteChannelSession build() {
+ return new UnbufferedWriteSession<>(
+ requireNonNull(start, "start must be non null"),
+ bindFunction().andThen(StorageByteChannels.writable()::createSynchronized));
+ }
+ }
+
+ final class BufferedResumableUploadBuilder {
+
+ private final BufferHandle bufferHandle;
+
+ private ApiFuture start;
+
+ BufferedResumableUploadBuilder(BufferHandle bufferHandle) {
+ this.bufferHandle = bufferHandle;
+ }
+
+ /**
+ * Set the Future which will contain the ResumableWrite information necessary to open the
+ * Write stream.
+ */
+ BufferedResumableUploadBuilder setStartAsync(ApiFuture start) {
+ this.start = requireNonNull(start, "start must be non null");
+ return this;
+ }
+
+ BufferedWritableByteChannelSession build() {
+ return new BufferedWriteSession<>(
+ requireNonNull(start, "start must be non null"),
+ bindFunction()
+ .andThen(c -> new DefaultBufferedWritableByteChannel(bufferHandle, c))
+ .andThen(StorageByteChannels.writable()::createSynchronized));
+ }
+ }
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableSession.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableSession.java
new file mode 100644
index 0000000000..7cc2f74e89
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableSession.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.api.gax.retrying.ResultRetryAlgorithm;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.cloud.storage.Conversions.Decoder;
+import com.google.cloud.storage.Retrying.RetryingDependencies;
+import com.google.cloud.storage.spi.v1.HttpRpcContext;
+import com.google.cloud.storage.spi.v1.HttpStorageRpc;
+import io.opencensus.trace.EndSpanOptions;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+final class JsonResumableSession {
+
+ static final String SPAN_NAME_WRITE =
+ String.format("Sent.%s.write", HttpStorageRpc.class.getName());
+ static final EndSpanOptions END_SPAN_OPTIONS =
+ EndSpanOptions.builder().setSampleToLocalSpanStore(true).build();
+
+ private final HttpClientContext context;
+ private final RetryingDependencies deps;
+ private final ResultRetryAlgorithm alg;
+ private final JsonResumableWrite resumableWrite;
+
+ JsonResumableSession(
+ HttpClientContext context,
+ RetryingDependencies deps,
+ ResultRetryAlgorithm alg,
+ JsonResumableWrite resumableWrite) {
+ this.context = context;
+ this.deps = deps;
+ this.alg = alg;
+ this.resumableWrite = resumableWrite;
+ }
+
+ /**
+ * Not automatically retried. Usually called from within another retrying context. We don't yet
+ * have the concept of nested retry handling.
+ */
+ ResumableOperationResult<@Nullable StorageObject> query() {
+ return new JsonResumableSessionQueryTask(context, resumableWrite.getUploadId()).call();
+ }
+
+ ResumableOperationResult<@Nullable StorageObject> put(
+ RewindableContent content, HttpContentRange contentRange) {
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ context, resumableWrite.getUploadId(), content, contentRange);
+ HttpRpcContext httpRpcContext = HttpRpcContext.getInstance();
+ httpRpcContext.newInvocationId();
+ AtomicBoolean dirty = new AtomicBoolean(false);
+ return Retrying.run(
+ deps,
+ alg,
+ () -> {
+ if (dirty.getAndSet(true)) {
+ ResumableOperationResult<@Nullable StorageObject> query = query();
+ long persistedSize = query.getPersistedSize();
+ if (contentRange.endOffsetEquals(persistedSize) || query.getObject() != null) {
+ return query;
+ } else {
+ task.rewindTo(persistedSize);
+ }
+ }
+ return task.call();
+ },
+ Decoder.identity());
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableSessionFailureScenario.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableSessionFailureScenario.java
new file mode 100644
index 0000000000..2b6e8d569c
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableSessionFailureScenario.java
@@ -0,0 +1,233 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.storage.StorageException.IOExceptionCallable;
+import com.google.common.io.CharStreams;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.function.Predicate;
+import javax.annotation.ParametersAreNonnullByDefault;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+@ParametersAreNonnullByDefault
+enum JsonResumableSessionFailureScenario {
+ // TODO: send more bytes than are in the Content-Range header
+ SCENARIO_0(BaseServiceException.UNKNOWN_CODE, null, "Unknown Error"),
+ SCENARIO_0_1(BaseServiceException.UNKNOWN_CODE, null, "Response not application/json."),
+ SCENARIO_1(
+ BaseServiceException.UNKNOWN_CODE,
+ "invalid",
+ "Attempt to append to already finalized resumable session."),
+ SCENARIO_2(
+ BaseServiceException.UNKNOWN_CODE,
+ "invalid",
+ "Attempt to finalize resumable session with fewer bytes than the backend has received."),
+ SCENARIO_3(
+ BaseServiceException.UNKNOWN_CODE,
+ "dataLoss",
+ "Attempt to finalize resumable session with more bytes than the backend has received."),
+ SCENARIO_4(200, "ok", "Attempt to finalize an already finalized session with same object size"),
+ SCENARIO_4_1(
+ BaseServiceException.UNKNOWN_CODE,
+ "dataLoss",
+ "Finalized resumable session, but object size less than expected."),
+ SCENARIO_4_2(
+ BaseServiceException.UNKNOWN_CODE,
+ "dataLoss",
+ "Finalized resumable session, but object size greater than expected."),
+ SCENARIO_5(
+ BaseServiceException.UNKNOWN_CODE,
+ "dataLoss",
+ "Client side data loss detected. Attempt to append to a resumable session with an offset higher than the backend has"),
+ SCENARIO_7(
+ BaseServiceException.UNKNOWN_CODE,
+ "dataLoss",
+ "Client side data loss detected. Bytes acked is more than client sent."),
+ SCENARIO_9(503, "backendNotConnected", "Ack less than bytes sent"),
+ QUERY_SCENARIO_1(503, "", "Missing Range header in response");
+
+ private static final String PREFIX_I = "\t|< ";
+ private static final String PREFIX_O = "\t|> ";
+ private static final String PREFIX_X = "\t| ";
+
+ private static final Predicate includedHeaders =
+ matches("Content-Length")
+ .or(matches("Content-Encoding"))
+ .or(matches("Content-Range"))
+ .or(matches("Content-Type"))
+ .or(matches("Range"))
+ .or(startsWith("X-Goog-Stored-"))
+ .or(matches("X-GUploader-UploadID"));
+
+ private static final Predicate> includeHeader =
+ e -> includedHeaders.test(e.getKey());
+
+ private final int code;
+ @Nullable private final String reason;
+ private final String message;
+
+ JsonResumableSessionFailureScenario(int code, @Nullable String reason, String message) {
+ this.code = code;
+ this.reason = reason;
+ this.message = message;
+ }
+
+ StorageException toStorageException(String uploadId, HttpResponse resp) {
+ return toStorageException(
+ uploadId, resp, null, () -> CharStreams.toString(new InputStreamReader(resp.getContent())));
+ }
+
+ StorageException toStorageException(
+ String uploadId, @Nullable HttpResponse resp, @Nullable Throwable cause) {
+ if (resp != null) {
+ // an exception caused this, do not try to read the content from the response.
+ return toStorageException(uploadId, resp, cause, () -> null);
+ } else {
+ return new StorageException(code, message, reason, cause);
+ }
+ }
+
+ StorageException toStorageException(
+ String uploadId,
+ HttpResponse resp,
+ @Nullable Throwable cause,
+ IOExceptionCallable<@Nullable String> contentCallable) {
+ return toStorageException(code, message, reason, uploadId, resp, cause, contentCallable);
+ }
+
+ static StorageException toStorageException(
+ HttpResponse response, HttpResponseException cause, String uploadId) {
+ String statusMessage = cause.getStatusMessage();
+ StorageException se =
+ JsonResumableSessionFailureScenario.toStorageException(
+ cause.getStatusCode(),
+ String.format(
+ "%d %s", cause.getStatusCode(), statusMessage == null ? "" : statusMessage),
+ "",
+ uploadId,
+ response,
+ cause,
+ () -> null);
+ return se;
+ }
+
+ static StorageException toStorageException(
+ int overrideCode,
+ String message,
+ @Nullable String reason,
+ String uploadId,
+ HttpResponse resp,
+ @Nullable Throwable cause,
+ IOExceptionCallable<@Nullable String> contentCallable) {
+ Throwable suppress = null;
+ StringBuilder sb = new StringBuilder();
+ sb.append(message);
+ // add request context
+ sb.append("\n").append(PREFIX_O).append("PUT ").append(uploadId);
+ recordHeaderTo(resp.getRequest().getHeaders(), PREFIX_O, sb);
+
+ sb.append("\n").append(PREFIX_X);
+ // add response context
+ {
+ int code = resp.getStatusCode();
+ sb.append("\n").append(PREFIX_I).append("HTTP/1.1 ").append(code);
+ if (resp.getStatusMessage() != null) {
+ sb.append(" ").append(resp.getStatusMessage());
+ }
+
+ recordHeaderTo(resp.getHeaders(), PREFIX_I, sb);
+ // try to include any body that we can handle
+ if (isOk(code) || code == 503 || code == 400) {
+ try {
+ String content = contentCallable.call();
+ if (content != null) {
+ sb.append("\n").append(PREFIX_I);
+ if (content.contains("\n") || content.contains("\r\n")) {
+ sb.append("\n").append(PREFIX_I).append(content.replaceAll("\r?\n", "\n" + PREFIX_I));
+ } else {
+ sb.append("\n").append(PREFIX_I).append(content);
+ }
+ }
+ } catch (IOException e) {
+ // com.google.api.client.http.HttpResponseException.Builder.Builder
+ // prints an exception which might occur while attempting to resolve the content
+ // this can lose the context about the request it was for, instead we register it
+ // as a suppressed exception
+ suppress = new StorageException(0, "Error reading response content for diagnostics.", e);
+ }
+ }
+
+ sb.append("\n").append(PREFIX_X);
+ }
+ StorageException storageException =
+ new StorageException(overrideCode, sb.toString(), reason, cause);
+ if (suppress != null) {
+ storageException.addSuppressed(suppress);
+ }
+ return storageException;
+ }
+
+ static boolean isOk(int code) {
+ return code == 200 || code == 201;
+ }
+
+ static boolean isContinue(int code) {
+ return code == 308;
+ }
+
+ // The header names from HttpHeaders are lower cased, define some utility methods to create
+ // predicates where we can specify values ignoring case
+ private static Predicate matches(String expected) {
+ String lower = expected.toLowerCase(Locale.US);
+ return lower::equals;
+ }
+
+ private static Predicate startsWith(String prefix) {
+ String lower = prefix.toLowerCase(Locale.US);
+ return s -> s.startsWith(lower);
+ }
+
+ private static void recordHeaderTo(HttpHeaders h, String prefix, StringBuilder sb) {
+ h.entrySet().stream()
+ .filter(includeHeader)
+ .forEach(
+ e -> {
+ String key = e.getKey();
+ String value = headerValueToString(e.getValue());
+ sb.append("\n").append(prefix).append(key).append(": ").append(value);
+ });
+ }
+
+ private static String headerValueToString(Object o) {
+ if (o instanceof List) {
+ List l = (List) o;
+ if (l.size() == 1) {
+ return l.get(0).toString();
+ }
+ }
+
+ return o.toString();
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableSessionPutTask.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableSessionPutTask.java
new file mode 100644
index 0000000000..5a4864996f
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableSessionPutTask.java
@@ -0,0 +1,239 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.cloud.storage.HttpContentRange.HasRange;
+import com.google.cloud.storage.StorageException.IOExceptionCallable;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import io.opencensus.common.Scope;
+import io.opencensus.trace.Span;
+import io.opencensus.trace.Status;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.util.Locale;
+import java.util.concurrent.Callable;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+final class JsonResumableSessionPutTask
+ implements Callable> {
+
+ private final HttpClientContext context;
+ private final String uploadId;
+ private final RewindableContent content;
+ private final HttpContentRange originalContentRange;
+
+ private HttpContentRange contentRange;
+
+ @VisibleForTesting
+ JsonResumableSessionPutTask(
+ HttpClientContext httpClientContext,
+ String uploadId,
+ RewindableContent content,
+ HttpContentRange originalContentRange) {
+ this.context = httpClientContext;
+ this.uploadId = uploadId;
+ this.content = content;
+ this.originalContentRange = originalContentRange;
+ this.contentRange = originalContentRange;
+ }
+
+ public void rewindTo(long offset) {
+ if (originalContentRange instanceof HasRange) {
+ HasRange hasRange = (HasRange) originalContentRange;
+ ByteRangeSpec range = hasRange.range();
+ long originalBegin = range.beginOffset();
+ long contentOffset = offset - originalBegin;
+ Preconditions.checkArgument(
+ 0 <= contentOffset && contentOffset < range.length(),
+ "Rewind offset is out of bounds. (%s <= %s < %s)",
+ originalBegin,
+ offset,
+ range.endOffset());
+ content.rewindTo(contentOffset);
+ } else {
+ content.rewindTo(0);
+ }
+
+ if (contentRange instanceof HttpContentRange.HasRange) {
+ HttpContentRange.HasRange range = (HttpContentRange.HasRange) contentRange;
+ contentRange = range.map(s -> s.withNewBeginOffset(offset));
+ }
+ }
+
+ public ResumableOperationResult<@Nullable StorageObject> call() throws IOException {
+ Span span = context.startSpan(JsonResumableSession.SPAN_NAME_WRITE);
+ Scope scope = context.getTracer().withSpan(span);
+
+ boolean success = false;
+ boolean finalizing = originalContentRange.isFinalizing();
+
+ HttpRequest req =
+ context
+ .getRequestFactory()
+ .buildPutRequest(new GenericUrl(uploadId), content)
+ .setParser(context.getObjectParser());
+ req.setThrowExceptionOnExecuteError(false);
+ req.getHeaders().setContentRange(contentRange.getHeaderValue());
+
+ HttpResponse response = null;
+ try {
+ response = req.execute();
+
+ int code = response.getStatusCode();
+
+ if (!finalizing && JsonResumableSessionFailureScenario.isContinue(code)) {
+ long effectiveEnd = ((HttpContentRange.HasRange) contentRange).range().endOffset();
+ @Nullable String range = response.getHeaders().getRange();
+ ByteRangeSpec ackRange = ByteRangeSpec.parse(range);
+ if (ackRange.endOffset() == effectiveEnd) {
+ success = true;
+ return ResumableOperationResult.incremental(ackRange.endOffset());
+ } else if (ackRange.endOffset() < effectiveEnd) {
+ StorageException se =
+ JsonResumableSessionFailureScenario.SCENARIO_9.toStorageException(uploadId, response);
+ span.setStatus(Status.UNKNOWN.withDescription(se.getMessage()));
+ throw se;
+ } else {
+ StorageException se =
+ JsonResumableSessionFailureScenario.SCENARIO_7.toStorageException(uploadId, response);
+ span.setStatus(Status.UNKNOWN.withDescription(se.getMessage()));
+ throw se;
+ }
+ } else if (finalizing && JsonResumableSessionFailureScenario.isOk(code)) {
+ @Nullable StorageObject storageObject;
+ @Nullable BigInteger actualSize;
+
+ Long contentLength = response.getHeaders().getContentLength();
+ String contentType = response.getHeaders().getContentType();
+ String storedContentLength =
+ HttpClientContext.firstHeaderValue(
+ response.getHeaders(), "x-goog-stored-content-length");
+ boolean isJson = contentType != null && contentType.startsWith("application/json");
+ if (isJson) {
+ storageObject = response.parseAs(StorageObject.class);
+ actualSize = storageObject != null ? storageObject.getSize() : null;
+ } else if ((contentLength == null || contentLength == 0) && storedContentLength != null) {
+ // when a signed url is used, the finalize response is empty
+ response.ignore();
+ actualSize = new BigInteger(storedContentLength, 10);
+ success = true;
+ storageObject = null;
+ } else {
+ response.ignore();
+ StorageException se =
+ JsonResumableSessionFailureScenario.SCENARIO_0_1.toStorageException(
+ uploadId, response, null, () -> null);
+ span.setStatus(Status.UNKNOWN.withDescription(se.getMessage()));
+ throw se;
+ }
+ BigInteger expectedSize =
+ BigInteger.valueOf(((HttpContentRange.HasSize) contentRange).getSize());
+ int compare = expectedSize.compareTo(actualSize);
+ if (compare == 0) {
+ success = true;
+ //noinspection DataFlowIssue compareTo result will filter out actualSize == null
+ return ResumableOperationResult.complete(storageObject, actualSize.longValue());
+ } else if (compare > 0) {
+ StorageException se =
+ JsonResumableSessionFailureScenario.SCENARIO_4_1.toStorageException(
+ uploadId, response, null, toString(storageObject));
+ span.setStatus(Status.UNKNOWN.withDescription(se.getMessage()));
+ throw se;
+ } else {
+ StorageException se =
+ JsonResumableSessionFailureScenario.SCENARIO_4_2.toStorageException(
+ uploadId, response, null, toString(storageObject));
+ span.setStatus(Status.UNKNOWN.withDescription(se.getMessage()));
+ throw se;
+ }
+ } else if (!finalizing && JsonResumableSessionFailureScenario.isOk(code)) {
+ StorageException se =
+ JsonResumableSessionFailureScenario.SCENARIO_1.toStorageException(uploadId, response);
+ span.setStatus(Status.UNKNOWN.withDescription(se.getMessage()));
+ throw se;
+ } else if (finalizing && JsonResumableSessionFailureScenario.isContinue(code)) {
+ // in order to finalize the content range must have a size, cast down to read it
+ HttpContentRange.HasSize size = (HttpContentRange.HasSize) contentRange;
+
+ ByteRangeSpec range = ByteRangeSpec.parse(response.getHeaders().getRange());
+ if (range.endOffsetInclusive() < size.getSize()) {
+ StorageException se =
+ JsonResumableSessionFailureScenario.SCENARIO_3.toStorageException(uploadId, response);
+ span.setStatus(Status.UNKNOWN.withDescription(se.getMessage()));
+ throw se;
+ } else {
+ StorageException se =
+ JsonResumableSessionFailureScenario.SCENARIO_2.toStorageException(uploadId, response);
+ span.setStatus(Status.UNKNOWN.withDescription(se.getMessage()));
+ throw se;
+ }
+ } else {
+ HttpResponseException cause = new HttpResponseException(response);
+ String contentType = response.getHeaders().getContentType();
+ // If the content-range header value has run ahead of the backend, it will respond with
+ // a 503 with plain text content
+ // Attempt to detect this very loosely as to minimize impact of modified error message
+ // This is accurate circa 2023-06
+ if ((!JsonResumableSessionFailureScenario.isOk(code)
+ && !JsonResumableSessionFailureScenario.isContinue(code))
+ && contentType != null
+ && contentType.startsWith("text/plain")) {
+ String errorMessage = cause.getContent().toLowerCase(Locale.US);
+ if (errorMessage.contains("content-range")) {
+ StorageException se =
+ JsonResumableSessionFailureScenario.SCENARIO_5.toStorageException(
+ uploadId, response, cause, cause::getContent);
+ span.setStatus(Status.UNKNOWN.withDescription(se.getMessage()));
+ throw se;
+ }
+ }
+ StorageException se =
+ JsonResumableSessionFailureScenario.toStorageException(response, cause, uploadId);
+ span.setStatus(Status.UNKNOWN.withDescription(se.getMessage()));
+ throw se;
+ }
+ } catch (StorageException | IllegalArgumentException e) {
+ // IllegalArgumentException can happen if there is no json in the body and we try to parse it
+ // Our retry algorithms have special case for this, so in an effort to keep compatibility
+ // with those existing behaviors, explicitly rethrow an IllegalArgumentException that may have
+ // happened
+ span.setStatus(Status.UNKNOWN.withDescription(e.getMessage()));
+ throw e;
+ } catch (Exception e) {
+ StorageException se =
+ JsonResumableSessionFailureScenario.SCENARIO_0.toStorageException(uploadId, response, e);
+ span.setStatus(Status.UNKNOWN.withDescription(se.getMessage()));
+ throw se;
+ } finally {
+ if (success && !finalizing && response != null) {
+ response.ignore();
+ }
+ scope.close();
+ span.end(JsonResumableSession.END_SPAN_OPTIONS);
+ }
+ }
+
+ static IOExceptionCallable<@Nullable String> toString(@Nullable Object o) {
+ return () -> o != null ? o.toString() : null;
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableSessionQueryTask.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableSessionQueryTask.java
new file mode 100644
index 0000000000..b37d2396d3
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableSessionQueryTask.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import static com.google.cloud.storage.HttpClientContext.firstHeaderValue;
+
+import com.google.api.client.http.EmptyContent;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.services.storage.model.StorageObject;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.util.Locale;
+import java.util.concurrent.Callable;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+final class JsonResumableSessionQueryTask
+ implements Callable> {
+
+ private final HttpClientContext context;
+ private final String uploadId;
+
+ JsonResumableSessionQueryTask(HttpClientContext context, String uploadId) {
+ this.context = context;
+ this.uploadId = uploadId;
+ }
+
+ public ResumableOperationResult<@Nullable StorageObject> call() {
+ HttpResponse response = null;
+ try {
+ HttpRequest req =
+ context
+ .getRequestFactory()
+ .buildPutRequest(new GenericUrl(uploadId), new EmptyContent())
+ .setParser(context.getObjectParser());
+ req.setThrowExceptionOnExecuteError(false);
+ req.getHeaders().setContentRange(HttpContentRange.query().getHeaderValue());
+
+ response = req.execute();
+
+ int code = response.getStatusCode();
+ if (JsonResumableSessionFailureScenario.isOk(code)) {
+ @Nullable StorageObject storageObject;
+ @Nullable BigInteger actualSize;
+
+ Long contentLength = response.getHeaders().getContentLength();
+ String contentType = response.getHeaders().getContentType();
+ String storedContentLength =
+ firstHeaderValue(response.getHeaders(), "x-goog-stored-content-length");
+ boolean isJson = contentType != null && contentType.startsWith("application/json");
+ if (isJson) {
+ storageObject = response.parseAs(StorageObject.class);
+ actualSize = storageObject != null ? storageObject.getSize() : null;
+ } else if ((contentLength == null || contentLength == 0) && storedContentLength != null) {
+ // when a signed url is used, the finalize response is empty
+ response.ignore();
+ actualSize = new BigInteger(storedContentLength, 10);
+ storageObject = null;
+ } else {
+ response.ignore();
+ throw JsonResumableSessionFailureScenario.SCENARIO_0_1.toStorageException(
+ uploadId, response, null, () -> null);
+ }
+ if (actualSize != null) {
+ if (storageObject != null) {
+ return ResumableOperationResult.complete(storageObject, actualSize.longValue());
+ } else {
+ return ResumableOperationResult.incremental(actualSize.longValue());
+ }
+ } else {
+ throw JsonResumableSessionFailureScenario.SCENARIO_0.toStorageException(
+ uploadId,
+ response,
+ null,
+ () -> storageObject != null ? storageObject.toString() : null);
+ }
+ } else if (JsonResumableSessionFailureScenario.isContinue(code)) {
+ String range1 = response.getHeaders().getRange();
+ if (range1 != null) {
+ ByteRangeSpec range = ByteRangeSpec.parse(range1);
+ long endOffset = range.endOffset();
+ return ResumableOperationResult.incremental(endOffset);
+ } else {
+ throw JsonResumableSessionFailureScenario.QUERY_SCENARIO_1.toStorageException(
+ uploadId, response);
+ }
+ } else {
+ HttpResponseException cause = new HttpResponseException(response);
+ String contentType = response.getHeaders().getContentType();
+ // If the content-range header value has run ahead of the backend, it will respond with
+ // a 503 with plain text content
+ // Attempt to detect this very loosely as to minimize impact of modified error message
+ // This is accurate circa 2023-06
+ if ((!JsonResumableSessionFailureScenario.isOk(code)
+ && !JsonResumableSessionFailureScenario.isContinue(code))
+ && contentType != null
+ && contentType.startsWith("text/plain")) {
+ String errorMessage = cause.getContent().toLowerCase(Locale.US);
+ if (errorMessage.contains("content-range")) {
+ throw JsonResumableSessionFailureScenario.SCENARIO_5.toStorageException(
+ uploadId, response, cause, cause::getContent);
+ }
+ }
+ throw JsonResumableSessionFailureScenario.toStorageException(response, cause, uploadId);
+ }
+ } catch (StorageException se) {
+ throw se;
+ } catch (Exception e) {
+ throw JsonResumableSessionFailureScenario.SCENARIO_0.toStorageException(
+ uploadId, response, e);
+ } finally {
+ if (response != null) {
+ try {
+ response.ignore();
+ } catch (IOException ignore) {
+ }
+ }
+ }
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableWrite.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableWrite.java
new file mode 100644
index 0000000000..336ce0e477
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonResumableWrite.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.api.services.storage.model.StorageObject;
+import com.google.cloud.storage.spi.v1.StorageRpc;
+import com.google.common.base.MoreObjects;
+import com.google.gson.Gson;
+import com.google.gson.stream.JsonReader;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.io.StringReader;
+import java.util.Map;
+import java.util.Objects;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.checkerframework.checker.nullness.qual.NonNull;
+
+final class JsonResumableWrite implements Serializable {
+ private static final long serialVersionUID = 7934407897802252292L;
+ private static final Gson gson = new Gson();
+
+ @MonotonicNonNull private transient StorageObject object;
+ @MonotonicNonNull private final Map options;
+
+ @MonotonicNonNull private final String signedUrl;
+
+ @NonNull private final String uploadId;
+ private final long beginOffset;
+
+ private volatile String objectJson;
+
+ private JsonResumableWrite(
+ StorageObject object,
+ Map options,
+ String signedUrl,
+ @NonNull String uploadId,
+ long beginOffset) {
+ this.object = object;
+ this.options = options;
+ this.signedUrl = signedUrl;
+ this.uploadId = uploadId;
+ this.beginOffset = beginOffset;
+ }
+
+ public @NonNull String getUploadId() {
+ return uploadId;
+ }
+
+ public long getBeginOffset() {
+ return beginOffset;
+ }
+
+ public JsonResumableWrite withBeginOffset(long newBeginOffset) {
+ checkArgument(
+ newBeginOffset >= beginOffset,
+ "New beginOffset must be >= existing beginOffset (%s >= %s)",
+ newBeginOffset,
+ beginOffset);
+ return new JsonResumableWrite(object, options, signedUrl, uploadId, newBeginOffset);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof JsonResumableWrite)) {
+ return false;
+ }
+ JsonResumableWrite that = (JsonResumableWrite) o;
+ return beginOffset == that.beginOffset
+ && Objects.equals(object, that.object)
+ && Objects.equals(options, that.options)
+ && Objects.equals(signedUrl, that.signedUrl)
+ && Objects.equals(uploadId, that.uploadId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(object, options, signedUrl, uploadId, beginOffset);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("object", object)
+ .add("options", options)
+ .add("signedUrl", signedUrl)
+ .add("uploadId", uploadId)
+ .add("beginOffset", beginOffset)
+ .toString();
+ }
+
+ private String getObjectJson() {
+ if (objectJson == null) {
+ synchronized (this) {
+ if (objectJson == null) {
+ objectJson = gson.toJson(object);
+ }
+ }
+ }
+ return objectJson;
+ }
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ String ignore = getObjectJson();
+ out.defaultWriteObject();
+ }
+
+ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+ in.defaultReadObject();
+ JsonReader jsonReader = gson.newJsonReader(new StringReader(this.objectJson));
+ this.object = gson.fromJson(jsonReader, StorageObject.class);
+ }
+
+ static JsonResumableWrite of(
+ StorageObject req, Map options, String uploadId, long beginOffset) {
+ return new JsonResumableWrite(req, options, null, uploadId, beginOffset);
+ }
+
+ static JsonResumableWrite of(String signedUrl, String uploadId, long beginOffset) {
+ return new JsonResumableWrite(null, null, signedUrl, uploadId, beginOffset);
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/LazyWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/LazyWriteChannel.java
new file mode 100644
index 0000000000..1c14eda853
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/LazyWriteChannel.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.cloud.storage.BufferedWritableByteChannelSession.BufferedWritableByteChannel;
+import java.util.function.Supplier;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.checkerframework.checker.nullness.qual.NonNull;
+
+final class LazyWriteChannel {
+
+ private final Supplier> sessionSupplier;
+
+ @MonotonicNonNull private volatile BufferedWritableByteChannelSession session;
+ @MonotonicNonNull private volatile BufferedWritableByteChannel channel;
+
+ private boolean open = false;
+
+ LazyWriteChannel(Supplier> sessionSupplier) {
+ this.sessionSupplier = sessionSupplier;
+ }
+
+ @NonNull
+ BufferedWritableByteChannel getChannel() {
+ if (channel != null) {
+ return channel;
+ } else {
+ synchronized (this) {
+ if (channel == null) {
+ open = true;
+ channel = getSession().open();
+ }
+ return channel;
+ }
+ }
+ }
+
+ @NonNull
+ BufferedWritableByteChannelSession getSession() {
+ if (session != null) {
+ return session;
+ } else {
+ synchronized (this) {
+ if (session == null) {
+ session = sessionSupplier.get();
+ }
+ return session;
+ }
+ }
+ }
+
+ boolean isOpen() {
+ return open && getChannel().isOpen();
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableMedia.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableMedia.java
index 85c96bd8d5..2d3fbf939a 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableMedia.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableMedia.java
@@ -103,6 +103,10 @@ static final class HttpMediaSession {
private HttpMediaSession() {}
+ HttpUploadSessionBuilder write() {
+ return HttpUploadSessionBuilder.create();
+ }
+
HttpDownloadSessionBuilder read() {
return HttpDownloadSessionBuilder.create();
}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableOperationResult.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableOperationResult.java
new file mode 100644
index 0000000000..88b5b7565e
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableOperationResult.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.common.base.MoreObjects;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+abstract class ResumableOperationResult<@Nullable T> {
+
+ private ResumableOperationResult() {}
+
+ abstract @Nullable T getObject();
+
+ abstract long getPersistedSize();
+
+ static ResumableOperationResult complete(T t, long persistedSize) {
+ return new CompletedResult<>(t, persistedSize);
+ }
+
+ static <@Nullable T> ResumableOperationResult incremental(long persistedSize) {
+ return new IncrementalResult<>(persistedSize);
+ }
+
+ private static final class CompletedResult extends ResumableOperationResult {
+
+ private final long persistedSize;
+ private final T entity;
+
+ private CompletedResult(T entity, long persistedSize) {
+ this.entity = entity;
+ this.persistedSize = persistedSize;
+ }
+
+ @Override
+ public @Nullable T getObject() {
+ return entity;
+ }
+
+ @Override
+ public long getPersistedSize() {
+ return persistedSize;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("persistedSize", persistedSize)
+ .add("entity", entity)
+ .toString();
+ }
+ }
+
+ private static final class IncrementalResult<@Nullable T> extends ResumableOperationResult {
+
+ private final long persistedSize;
+
+ private IncrementalResult(long persistedSize) {
+ this.persistedSize = persistedSize;
+ }
+
+ @Override
+ public @Nullable T getObject() {
+ return null;
+ }
+
+ @Override
+ public long getPersistedSize() {
+ return persistedSize;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this).add("persistedSize", persistedSize).toString();
+ }
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableSession.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableSession.java
new file mode 100644
index 0000000000..5c308f7fb9
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableSession.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.api.gax.retrying.ResultRetryAlgorithm;
+import com.google.api.gax.rpc.ClientStreamingCallable;
+import com.google.api.gax.rpc.UnaryCallable;
+import com.google.cloud.storage.Retrying.RetryingDependencies;
+import com.google.storage.v2.QueryWriteStatusRequest;
+import com.google.storage.v2.QueryWriteStatusResponse;
+import com.google.storage.v2.WriteObjectRequest;
+import com.google.storage.v2.WriteObjectResponse;
+
+final class ResumableSession {
+
+ private ResumableSession() {}
+
+ static JsonResumableSession json(
+ HttpClientContext context,
+ RetryingDependencies deps,
+ ResultRetryAlgorithm alg,
+ JsonResumableWrite resumableWrite) {
+ return new JsonResumableSession(context, deps, alg, resumableWrite);
+ }
+
+ static GrpcResumableSession grpc(
+ RetryingDependencies deps,
+ ResultRetryAlgorithm alg,
+ ClientStreamingCallable writeCallable,
+ UnaryCallable queryWriteStatusCallable,
+ ResumableWrite resumableWrite,
+ Hasher hasher) {
+ return new GrpcResumableSession(
+ deps, alg, writeCallable, queryWriteStatusCallable, resumableWrite, hasher);
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java
index f625b58b95..0b5f66ce35 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java
@@ -103,11 +103,16 @@ static U run(
} catch (StorageException se) {
// we hope for this case
throw se;
+ } catch (IllegalArgumentException iae) {
+ // IllegalArgumentException can happen if there is no json in the body and we try
+ // to parse it Our retry algorithms have special case for this, so in an effort to
+ // keep compatibility with those existing behaviors, explicitly rethrow an
+ // IllegalArgumentException that may have happened
+ throw iae;
} catch (Exception e) {
- // but wire in this fall through just in case.
+ // Wire in this fall through just in case.
// all of our retry algorithms are centered around StorageException so this helps
- // those
- // be more effective
+ // those be more effective
throw StorageException.coalesce(e);
}
},
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java
new file mode 100644
index 0000000000..166ccbc97d
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java
@@ -0,0 +1,235 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.api.client.http.AbstractHttpContent;
+import com.google.api.client.http.HttpMediaType;
+import com.google.common.base.Preconditions;
+import com.google.common.io.ByteStreams;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.GatheringByteChannel;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+
+abstract class RewindableContent extends AbstractHttpContent {
+
+ private RewindableContent() {
+ super((HttpMediaType) null);
+ }
+
+ @Override
+ public abstract long getLength();
+
+ abstract void rewindTo(long offset);
+
+ abstract long writeTo(WritableByteChannel gbc) throws IOException;
+
+ abstract long writeTo(GatheringByteChannel gbc) throws IOException;
+
+ @Override
+ public final boolean retrySupported() {
+ return false;
+ }
+
+ static RewindableContent empty() {
+ return EmptyRewindableContent.INSTANCE;
+ }
+
+ static RewindableContent of(ByteBuffer... buffers) {
+ return new ByteBufferContent(buffers);
+ }
+
+ static RewindableContent of(Path path) throws IOException {
+ return new PathRewindableContent(path);
+ }
+
+ private static final class EmptyRewindableContent extends RewindableContent {
+ private static final EmptyRewindableContent INSTANCE = new EmptyRewindableContent();
+
+ @Override
+ public long getLength() {
+ return 0L;
+ }
+
+ @Override
+ public void writeTo(OutputStream out) throws IOException {
+ out.flush();
+ }
+
+ @Override
+ long writeTo(WritableByteChannel gbc) {
+ return 0;
+ }
+
+ @Override
+ long writeTo(GatheringByteChannel gbc) {
+ return 0;
+ }
+
+ @Override
+ protected void rewindTo(long offset) {}
+ }
+
+ private static final class PathRewindableContent extends RewindableContent {
+
+ private final Path path;
+ private final long size;
+
+ private long readOffset;
+
+ private PathRewindableContent(Path path) throws IOException {
+ this.path = path;
+ this.size = Files.size(path);
+ this.readOffset = 0;
+ }
+
+ @Override
+ public long getLength() {
+ return size - readOffset;
+ }
+
+ @Override
+ void rewindTo(long offset) {
+ Preconditions.checkArgument(
+ offset < size, "provided offset must be less than size (%d < %d)", offset, size);
+ this.readOffset = offset;
+ }
+
+ @Override
+ public void writeTo(OutputStream out) throws IOException {
+ try (SeekableByteChannel in = Files.newByteChannel(path, StandardOpenOption.READ)) {
+ in.position(readOffset);
+ ByteStreams.copy(in, Channels.newChannel(out));
+ out.flush();
+ }
+ }
+
+ @Override
+ long writeTo(WritableByteChannel gbc) throws IOException {
+ try (SeekableByteChannel in = Files.newByteChannel(path, StandardOpenOption.READ)) {
+ in.position(readOffset);
+ return ByteStreams.copy(in, gbc);
+ }
+ }
+
+ @Override
+ long writeTo(GatheringByteChannel gbc) throws IOException {
+ try (SeekableByteChannel in = Files.newByteChannel(path, StandardOpenOption.READ)) {
+ in.position(readOffset);
+ return ByteStreams.copy(in, gbc);
+ }
+ }
+ }
+
+ private static final class ByteBufferContent extends RewindableContent {
+
+ private final ByteBuffer[] buffers;
+ // keep an array of the positions in case we need to rewind them for retries
+ // doing this is simpler than duplicating the buffers and using marks, as we don't need to
+ // advance the position of the original buffers upon success.
+ // We generally expect success, and in this case are planning in case of failure.
+ private final int[] positions;
+ private final long totalLength;
+ // track whether we have changed any state
+ private boolean dirty;
+
+ private long offset;
+
+ private ByteBufferContent(ByteBuffer[] buffers) {
+ this.buffers = buffers;
+ this.positions = Arrays.stream(buffers).mapToInt(Buffers::position).toArray();
+ this.totalLength = Arrays.stream(buffers).mapToLong(Buffer::remaining).sum();
+ this.dirty = false;
+ }
+
+ @Override
+ public long getLength() {
+ return totalLength - offset;
+ }
+
+ @Override
+ public void writeTo(OutputStream out) throws IOException {
+ dirty = true;
+ WritableByteChannel c = Channels.newChannel(out);
+ for (ByteBuffer buffer : buffers) {
+ c.write(buffer);
+ }
+ out.flush();
+ }
+
+ @Override
+ long writeTo(WritableByteChannel gbc) throws IOException {
+ dirty = true;
+ int retVal = 0;
+ for (ByteBuffer buffer : buffers) {
+ retVal += gbc.write(buffer);
+ }
+ return retVal;
+ }
+
+ @Override
+ long writeTo(GatheringByteChannel gbc) throws IOException {
+ dirty = true;
+ return gbc.write(buffers);
+ }
+
+ @Override
+ void rewindTo(long offset) {
+ Preconditions.checkArgument(
+ offset < totalLength,
+ "provided offset must be less than totalLength (%s < %s)",
+ offset,
+ totalLength);
+ if (dirty || offset != this.offset) {
+ // starting from the end of our data, walk back the buffers updating their position
+ // to coincide with the rewind of the overall content
+ int idx = buffers.length - 1;
+ for (long currentOffset = totalLength; currentOffset > 0; ) {
+ int position = positions[idx];
+ ByteBuffer buf = buffers[idx];
+
+ int origRemaining = buf.limit() - position;
+
+ long begin = currentOffset - origRemaining;
+
+ if (begin <= offset && offset < currentOffset) {
+ long diff = offset - begin;
+ Buffers.position(buf, position + Math.toIntExact(diff));
+ } else if (offset >= currentOffset) {
+ // the desired offset is after this buf
+ // ensure it does not have any available
+ Buffers.position(buf, buf.limit());
+ } else {
+ Buffers.position(buf, position);
+ }
+
+ currentOffset = begin;
+ idx -= 1;
+ }
+ }
+ this.offset = offset;
+ }
+ }
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java
index 8c0768b9de..d4ad26015f 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java
@@ -23,6 +23,7 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.concurrent.Executors.callable;
+import com.google.api.core.ApiFuture;
import com.google.api.gax.paging.Page;
import com.google.api.gax.retrying.ResultRetryAlgorithm;
import com.google.api.services.storage.model.BucketAccessControl;
@@ -84,8 +85,12 @@
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.function.Function;
+import java.util.function.Supplier;
+import org.checkerframework.checker.nullness.qual.Nullable;
final class StorageImpl extends BaseService implements Storage {
@@ -225,9 +230,40 @@ public Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOp
if (Files.isDirectory(path)) {
throw new StorageException(0, path + " is a directory");
}
- try (InputStream input = Files.newInputStream(path)) {
- return createFrom(blobInfo, input, bufferSize, options);
+ Opts opts = Opts.unwrap(options).resolveFrom(blobInfo);
+ final Map optionsMap = opts.getRpcOptions();
+ BlobInfo.Builder builder = blobInfo.toBuilder().setMd5(null).setCrc32c(null);
+ BlobInfo updated = opts.blobInfoMapper().apply(builder).build();
+ StorageObject encode = codecs.blobInfo().encode(updated);
+
+ Supplier uploadIdSupplier =
+ ResumableMedia.startUploadForBlobInfo(
+ getOptions(),
+ updated,
+ optionsMap,
+ retryAlgorithmManager.getForResumableUploadSessionCreate(optionsMap));
+ JsonResumableWrite jsonResumableWrite =
+ JsonResumableWrite.of(encode, optionsMap, uploadIdSupplier.get(), 0);
+
+ JsonResumableSession session =
+ ResumableSession.json(
+ HttpClientContext.from(storageRpc),
+ getOptions().asRetryDependencies(),
+ retryAlgorithmManager.idempotent(),
+ jsonResumableWrite);
+ long size = Files.size(path);
+ HttpContentRange contentRange =
+ HttpContentRange.of(ByteRangeSpec.relativeLength(0L, size), size);
+ ResumableOperationResult put =
+ session.put(RewindableContent.of(path), contentRange);
+ // all exception translation is taken care of down in the JsonResumableSession
+ StorageObject object = put.getObject();
+ if (object == null) {
+ // if by some odd chance the put didn't get the StorageObject, query for it
+ ResumableOperationResult<@Nullable StorageObject> query = session.query();
+ object = query.getObject();
}
+ return codecs.blobInfo().decode(object).asBlob(this);
}
@Override
@@ -241,14 +277,20 @@ public Blob createFrom(
BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options)
throws IOException {
- BlobWriteChannel blobWriteChannel;
- try (WriteChannel writer = writer(blobInfo, options)) {
- blobWriteChannel = (BlobWriteChannel) writer;
+ ApiFuture objectFuture;
+ try (StorageWriteChannel writer = writer(blobInfo, options)) {
+ objectFuture = writer.getObject();
uploadHelper(Channels.newChannel(content), writer, bufferSize);
}
- StorageObject objectProto = blobWriteChannel.getStorageObject();
- BlobInfo info = Conversions.apiary().blobInfo().decode(objectProto);
- return info.asBlob(this);
+ // keep these two try blocks separate for the time being
+ // leaving the above will cause the writer to close writing and finalizing the session and
+ // (hopefully, on successful finalization) resolve our future
+ try {
+ BlobInfo info = objectFuture.get(10, TimeUnit.SECONDS);
+ return info.asBlob(this);
+ } catch (ExecutionException | InterruptedException | TimeoutException e) {
+ throw StorageException.coalesce(e);
+ }
}
/*
@@ -613,38 +655,41 @@ public void downloadTo(BlobId blob, OutputStream outputStream, BlobSourceOption.
}
@Override
- public BlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) {
+ public StorageWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) {
Opts opts = Opts.unwrap(options).resolveFrom(blobInfo);
final Map optionsMap = opts.getRpcOptions();
BlobInfo.Builder builder = blobInfo.toBuilder().setMd5(null).setCrc32c(null);
BlobInfo updated = opts.blobInfoMapper().apply(builder).build();
- return BlobWriteChannel.newBuilder()
- .setStorageOptions(getOptions())
- .setUploadIdSupplier(
- ResumableMedia.startUploadForBlobInfo(
- getOptions(),
- updated,
- optionsMap,
- retryAlgorithmManager.getForResumableUploadSessionCreate(optionsMap)))
- .setAlgorithmForWrite(retryAlgorithmManager.getForResumableUploadSessionWrite(optionsMap))
- .build();
+
+ StorageObject encode = codecs.blobInfo().encode(updated);
+ // open the resumable session outside the write channel
+ // the exception behavior of open is different from #write(ByteBuffer)
+ Supplier uploadIdSupplier =
+ ResumableMedia.startUploadForBlobInfo(
+ getOptions(),
+ updated,
+ optionsMap,
+ retryAlgorithmManager.getForResumableUploadSessionCreate(optionsMap));
+ JsonResumableWrite jsonResumableWrite =
+ JsonResumableWrite.of(encode, optionsMap, uploadIdSupplier.get(), 0);
+ return new BlobWriteChannelV2(BlobReadChannelContext.from(getOptions()), jsonResumableWrite);
}
@Override
- public BlobWriteChannel writer(URL signedURL) {
+ public StorageWriteChannel writer(URL signedURL) {
+ // TODO: is it possible to know if a signed url is configured to have a constraint which makes
+ // it idempotent?
ResultRetryAlgorithm forResumableUploadSessionCreate =
- retryAlgorithmManager.getForResumableUploadSessionCreate(
- Collections
- .emptyMap()); // TODO: is it possible to know if a signed url is configured to have
- // a constraint which makes it idempotent?
- return BlobWriteChannel.newBuilder()
- .setStorageOptions(getOptions())
- .setUploadIdSupplier(
- ResumableMedia.startUploadForSignedUrl(
- getOptions(), signedURL, forResumableUploadSessionCreate))
- .setAlgorithmForWrite(
- retryAlgorithmManager.getForResumableUploadSessionWrite(Collections.emptyMap()))
- .build();
+ retryAlgorithmManager.getForResumableUploadSessionCreate(Collections.emptyMap());
+ // open the resumable session outside the write channel
+ // the exception behavior of open is different from #write(ByteBuffer)
+ String signedUrlString = signedURL.toString();
+ Supplier uploadIdSupplier =
+ ResumableMedia.startUploadForSignedUrl(
+ getOptions(), signedURL, forResumableUploadSessionCreate);
+ JsonResumableWrite jsonResumableWrite =
+ JsonResumableWrite.of(signedUrlString, uploadIdSupplier.get(), 0);
+ return new BlobWriteChannelV2(BlobReadChannelContext.from(getOptions()), jsonResumableWrite);
}
@Override
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageWriteChannel.java
new file mode 100644
index 0000000000..d1badc0b17
--- /dev/null
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageWriteChannel.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.api.core.ApiFuture;
+import com.google.cloud.WriteChannel;
+
+interface StorageWriteChannel extends WriteChannel {
+ ApiFuture getObject();
+}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnbufferedWritableByteChannelSession.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnbufferedWritableByteChannelSession.java
index 1df74aba76..8affde6b59 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnbufferedWritableByteChannelSession.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnbufferedWritableByteChannelSession.java
@@ -17,11 +17,23 @@
package com.google.cloud.storage;
import com.google.cloud.storage.UnbufferedWritableByteChannelSession.UnbufferedWritableByteChannel;
+import java.io.IOException;
+import java.nio.ByteBuffer;
import java.nio.channels.GatheringByteChannel;
import java.nio.channels.WritableByteChannel;
interface UnbufferedWritableByteChannelSession
extends WritableByteChannelSession {
- interface UnbufferedWritableByteChannel extends WritableByteChannel, GatheringByteChannel {}
+ interface UnbufferedWritableByteChannel extends WritableByteChannel, GatheringByteChannel {
+ @Override
+ default int write(ByteBuffer src) throws IOException {
+ return Math.toIntExact(write(new ByteBuffer[] {src}));
+ }
+
+ @Override
+ default long write(ByteBuffer[] srcs) throws IOException {
+ return write(srcs, 0, srcs.length);
+ }
+ }
}
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java
index 67bafab86f..c24a68d4d6 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java
@@ -37,6 +37,7 @@
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
@@ -283,6 +284,14 @@ static void diffMaps(
keys.map(NamedField::literal).map(k -> NamedField.nested(parent, k)).forEach(sink);
}
+ static T[] subArray(T[] ts, int offset, int length) {
+ if (offset == 0 && length == ts.length) {
+ return ts;
+ } else {
+ return Arrays.copyOfRange(ts, offset, length);
+ }
+ }
+
private static int crc32cDecode(String from) {
byte[] decodeCrc32c = BaseEncoding.base64().decode(from);
return Ints.fromByteArray(decodeCrc32c);
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java
index 100f5b6ce5..98d9476f89 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java
@@ -102,11 +102,11 @@ public class HttpStorageRpc implements StorageRpc {
// declare this HttpStatus code here as it's not included in java.net.HttpURLConnection
private static final int SC_REQUESTED_RANGE_NOT_SATISFIABLE = 416;
+ private static final boolean IS_RECORD_EVENTS = true;
private final StorageOptions options;
private final Storage storage;
private final Tracer tracer = Tracing.getTracer();
- private final CensusHttpModule censusHttpModule;
private final HttpRequestInitializer batchRequestInitializer;
private static final long MEGABYTE = 1024L * 1024L;
@@ -123,7 +123,7 @@ public HttpStorageRpc(StorageOptions options, JsonFactory jsonFactory) {
this.options = options;
// Open Census initialization
- censusHttpModule = new CensusHttpModule(tracer, true);
+ CensusHttpModule censusHttpModule = new CensusHttpModule(tracer, IS_RECORD_EVENTS);
initializer = censusHttpModule.getHttpRequestInitializer(initializer);
initializer = new InvocationIdInitializer(initializer);
batchRequestInitializer = censusHttpModule.getHttpRequestInitializer(null);
@@ -318,10 +318,7 @@ private static void setEncryptionHeaders(
/** Helper method to start a span. */
private Span startSpan(String spanName) {
- return tracer
- .spanBuilder(spanName)
- .setRecordEvents(censusHttpModule.isRecordEvents())
- .startSpan();
+ return tracer.spanBuilder(spanName).setRecordEvents(IS_RECORD_EVENTS).startSpan();
}
@Override
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/package-info.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/package-info.java
index 096e189de1..ae3f262846 100644
--- a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/package-info.java
+++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/package-info.java
@@ -34,8 +34,7 @@
* RemoteStorageHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS);
* }
*
- * @see
- * Google Cloud Java tools for testing
+ * @see Google Cloud
+ * Storage testing
*/
package com.google.cloud.storage.testing;
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java
index 68ac4683be..4cb1ee86ae 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java
@@ -40,7 +40,6 @@
import com.google.cloud.storage.Acl.User;
import com.google.cloud.storage.Blob.BlobSourceOption;
import com.google.cloud.storage.BlobInfo.BuilderImpl;
-import com.google.cloud.storage.Storage.BlobWriteOption;
import com.google.cloud.storage.Storage.CopyRequest;
import com.google.cloud.storage.spi.v1.HttpStorageRpc;
import com.google.cloud.storage.spi.v1.StorageRpc;
@@ -387,43 +386,6 @@ public void testReaderWithDecryptionKey() throws Exception {
assertSame(channel, blob.reader(BlobSourceOption.decryptionKey(KEY)));
}
- @Test
- public void testWriter() throws Exception {
- initializeExpectedBlob();
- BlobWriteChannel channel = createMock(BlobWriteChannel.class);
- expect(storage.getOptions()).andReturn(mockOptions).anyTimes();
- expect(storage.writer(eq(expectedBlob))).andReturn(channel);
- replay(storage);
- initializeBlob();
- assertSame(channel, blob.writer());
- }
-
- @Test
- public void testWriterWithEncryptionKey() throws Exception {
- initializeExpectedBlob();
- BlobWriteChannel channel = createMock(BlobWriteChannel.class);
- expect(storage.getOptions()).andReturn(mockOptions).anyTimes();
- expect(storage.writer(eq(expectedBlob), eq(BlobWriteOption.encryptionKey(BASE64_KEY))))
- .andReturn(channel)
- .times(2);
- replay(storage);
- initializeBlob();
- assertSame(channel, blob.writer(BlobWriteOption.encryptionKey(BASE64_KEY)));
- assertSame(channel, blob.writer(BlobWriteOption.encryptionKey(KEY)));
- }
-
- @Test
- public void testWriterWithKmsKeyName() throws Exception {
- initializeExpectedBlob();
- BlobWriteChannel channel = createMock(BlobWriteChannel.class);
- expect(storage.getOptions()).andReturn(mockOptions).anyTimes();
- expect(storage.writer(eq(expectedBlob), eq(BlobWriteOption.kmsKeyName(KMS_KEY_NAME))))
- .andReturn(channel);
- replay(storage);
- initializeBlob();
- assertSame(channel, blob.writer(BlobWriteOption.kmsKeyName(KMS_KEY_NAME)));
- }
-
@Test
public void testSignUrl() throws Exception {
initializeExpectedBlob();
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java
deleted file mode 100644
index 2898064aaa..0000000000
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java
+++ /dev/null
@@ -1,961 +0,0 @@
-/*
- * Copyright 2015 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.google.cloud.storage;
-
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.capture;
-import static org.easymock.EasyMock.captureLong;
-import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.eq;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import com.google.api.gax.retrying.ResultRetryAlgorithm;
-import com.google.api.services.storage.model.StorageObject;
-import com.google.cloud.NoCredentials;
-import com.google.cloud.RestorableState;
-import com.google.cloud.WriteChannel;
-import com.google.cloud.storage.spi.StorageRpcFactory;
-import com.google.cloud.storage.spi.v1.StorageRpc;
-import com.google.common.collect.ImmutableMap;
-import java.io.IOException;
-import java.math.BigInteger;
-import java.net.MalformedURLException;
-import java.net.SocketException;
-import java.net.URL;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Random;
-import org.easymock.Capture;
-import org.easymock.CaptureType;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class BlobWriteChannelTest {
-
- private static final String BUCKET_NAME = "b";
- private static final String BLOB_NAME = "n";
- private static final String UPLOAD_ID = "uploadid";
- private static final BlobInfo BLOB_INFO = BlobInfo.newBuilder(BUCKET_NAME, BLOB_NAME).build();
- private static final BlobInfo BLOB_INFO_WITH_GENERATION =
- BlobInfo.newBuilder(BUCKET_NAME, BLOB_NAME, 1L).build();
- private static final StorageObject UPDATED_BLOB = new StorageObject();
- private static final Map EMPTY_RPC_OPTIONS = ImmutableMap.of();
- private static final Map RPC_OPTIONS_GENERATION =
- ImmutableMap.of(StorageRpc.Option.IF_GENERATION_MATCH, 1L);
- private static final int MIN_CHUNK_SIZE = 256 * 1024;
- private static final int DEFAULT_CHUNK_SIZE = 60 * MIN_CHUNK_SIZE; // 15MiB
- private static final int CUSTOM_CHUNK_SIZE = 4 * MIN_CHUNK_SIZE;
- private static final Random RANDOM = new Random();
- private static final String SIGNED_URL =
- "http://www.test.com/test-bucket/test1.txt?GoogleAccessId=testClient-test@test.com&Expires=1553839761&Signature=MJUBXAZ7";
- private static final StorageException socketClosedException =
- new StorageException(new SocketException("Socket closed"));
- private HttpStorageOptions options;
- private StorageRpcFactory rpcFactoryMock;
- private StorageRpc storageRpcMock;
- private BlobWriteChannel writer;
- private HttpRetryAlgorithmManager retryAlgorithmManager;
-
- @Before
- public void setUp() {
- rpcFactoryMock = createMock(StorageRpcFactory.class);
- storageRpcMock = createMock(StorageRpc.class);
- expect(rpcFactoryMock.create(anyObject(StorageOptions.class))).andReturn(storageRpcMock);
- replay(rpcFactoryMock);
- options =
- HttpStorageOptions.newBuilder()
- .setProjectId("projectid")
- .setServiceRpcFactory(rpcFactoryMock)
- .setCredentials(NoCredentials.getInstance())
- .build();
- retryAlgorithmManager = options.getRetryAlgorithmManager();
- }
-
- @After
- public void tearDown() throws Exception {
- verify(rpcFactoryMock, storageRpcMock);
- }
-
- @Test
- public void testCreate() {
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO), EMPTY_RPC_OPTIONS))
- .andReturn(UPLOAD_ID);
- replay(storageRpcMock);
- writer = newWriter();
- assertTrue(writer.isOpen());
- assertNull(writer.getStorageObject());
- }
-
- @Test
- public void testCreateRetryableError() {
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO_WITH_GENERATION),
- RPC_OPTIONS_GENERATION))
- .andThrow(socketClosedException);
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO_WITH_GENERATION),
- RPC_OPTIONS_GENERATION))
- .andReturn(UPLOAD_ID);
- replay(storageRpcMock);
- writer = newWriter(true);
- assertTrue(writer.isOpen());
- assertNull(writer.getStorageObject());
- }
-
- @Test
- public void testCreateNonRetryableError() {
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO), EMPTY_RPC_OPTIONS))
- .andThrow(new RuntimeException());
- replay(storageRpcMock);
- try {
- newWriter();
- Assert.fail();
- } catch (RuntimeException ex) {
- assertNotNull(ex.getMessage());
- }
- }
-
- @Test
- public void testWriteWithoutFlush() throws Exception {
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO), EMPTY_RPC_OPTIONS))
- .andReturn(UPLOAD_ID);
- replay(storageRpcMock);
- writer = newWriter();
- assertEquals(MIN_CHUNK_SIZE, writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE)));
- }
-
- @Test
- public void testWriteWithFlushRetryChunk() throws Exception {
- ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO_WITH_GENERATION),
- RPC_OPTIONS_GENERATION))
- .andReturn(UPLOAD_ID);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andThrow(socketClosedException);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(0L);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andReturn(null);
- replay(storageRpcMock);
- writer = newWriter(true);
- writer.setChunkSize(MIN_CHUNK_SIZE);
- assertEquals(MIN_CHUNK_SIZE, writer.write(buffer));
- assertTrue(writer.isOpen());
- assertNull(writer.getStorageObject());
- assertArrayEquals(buffer.array(), capturedBuffer.getValue());
- }
-
- @Test
- public void testWriteWithRetryFullChunk() throws Exception {
- ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO_WITH_GENERATION),
- RPC_OPTIONS_GENERATION))
- .andReturn(UPLOAD_ID);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID), (byte[]) anyObject(), eq(0), eq(0L), eq(MIN_CHUNK_SIZE), eq(false)))
- .andThrow(socketClosedException);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(0L);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andReturn(null);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- (byte[]) anyObject(),
- eq(0),
- eq((long) MIN_CHUNK_SIZE),
- eq(0),
- eq(true)))
- .andReturn(Conversions.apiary().blobInfo().encode(BLOB_INFO));
- replay(storageRpcMock);
- writer = newWriter(true);
- writer.setChunkSize(MIN_CHUNK_SIZE);
- assertEquals(MIN_CHUNK_SIZE, writer.write(buffer));
- writer.close();
- assertFalse(writer.isOpen());
- assertNotNull(writer.getStorageObject());
- assertArrayEquals(buffer.array(), capturedBuffer.getValue());
- }
-
- @Test
- public void testWriteWithRemoteProgressMade() throws Exception {
- ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO_WITH_GENERATION),
- RPC_OPTIONS_GENERATION))
- .andReturn(UPLOAD_ID);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andThrow(socketClosedException);
- // Simulate GCS received 10 bytes but not the rest of the chunk
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(10L);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(10),
- eq(10L),
- eq(MIN_CHUNK_SIZE - 10),
- eq(false)))
- .andReturn(null);
- replay(storageRpcMock);
- writer = newWriter(true);
- writer.setChunkSize(MIN_CHUNK_SIZE);
- assertEquals(MIN_CHUNK_SIZE, writer.write(buffer));
- assertTrue(writer.isOpen());
- assertNull(writer.getStorageObject());
- assertArrayEquals(buffer.array(), capturedBuffer.getValue());
- }
-
- @Test
- public void testWriteWithDriftRetryCase4() throws Exception {
- ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO_WITH_GENERATION),
- RPC_OPTIONS_GENERATION))
- .andReturn(UPLOAD_ID);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andThrow(socketClosedException);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn((long) MIN_CHUNK_SIZE);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq((long) MIN_CHUNK_SIZE),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andReturn(null);
- replay(storageRpcMock);
- writer = newWriter(true);
- writer.setChunkSize(MIN_CHUNK_SIZE);
- assertEquals(MIN_CHUNK_SIZE, writer.write(buffer));
- assertArrayEquals(buffer.array(), capturedBuffer.getValue());
- capturedBuffer.reset();
- buffer.rewind();
- assertEquals(MIN_CHUNK_SIZE, writer.write(buffer));
- assertArrayEquals(buffer.array(), capturedBuffer.getValue());
- assertTrue(writer.isOpen());
- assertNull(writer.getStorageObject());
- }
-
- @Test
- public void testWriteWithUnreachableRemoteOffset() throws Exception {
- ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO_WITH_GENERATION),
- RPC_OPTIONS_GENERATION))
- .andReturn(UPLOAD_ID);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andThrow(socketClosedException);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(MIN_CHUNK_SIZE + 10L);
- replay(storageRpcMock);
- writer = newWriter(true);
- writer.setChunkSize(MIN_CHUNK_SIZE);
- try {
- writer.write(buffer);
- fail("Expected StorageException");
- } catch (StorageException storageException) {
- // expected storageException
- }
- assertTrue(writer.isOpen());
- assertNull(writer.getStorageObject());
- assertArrayEquals(buffer.array(), capturedBuffer.getValue());
- }
-
- @Test
- public void testWriteWithRetryAndObjectMetadata() throws Exception {
- ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO_WITH_GENERATION),
- RPC_OPTIONS_GENERATION))
- .andReturn(UPLOAD_ID);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andThrow(socketClosedException);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(10L);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(10),
- eq(10L),
- eq(MIN_CHUNK_SIZE - 10),
- eq(false)))
- .andThrow(socketClosedException);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(-1L);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(-1L);
- expect(storageRpcMock.queryCompletedResumableUpload(eq(UPLOAD_ID), eq((long) MIN_CHUNK_SIZE)))
- .andThrow(socketClosedException);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(-1L);
- expect(storageRpcMock.queryCompletedResumableUpload(eq(UPLOAD_ID), eq((long) MIN_CHUNK_SIZE)))
- .andReturn(
- Conversions.apiary()
- .blobInfo()
- .encode(BLOB_INFO)
- .setSize(BigInteger.valueOf(MIN_CHUNK_SIZE)));
- replay(storageRpcMock);
- writer = newWriter(true);
- writer.setChunkSize(MIN_CHUNK_SIZE);
- assertEquals(MIN_CHUNK_SIZE, writer.write(buffer));
- writer.close();
- assertFalse(writer.isOpen());
- assertNotNull(writer.getStorageObject());
- assertArrayEquals(buffer.array(), capturedBuffer.getValue());
- }
-
- @Test
- public void testWriteWithUploadCompletedByAnotherClient() throws Exception {
- ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO_WITH_GENERATION),
- RPC_OPTIONS_GENERATION))
- .andReturn(UPLOAD_ID);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andReturn(null);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq((long) MIN_CHUNK_SIZE),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andThrow(socketClosedException);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(-1L);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(-1L);
- replay(storageRpcMock);
- writer = newWriter(true);
- writer.setChunkSize(MIN_CHUNK_SIZE);
- try {
- writer.write(buffer);
- buffer.rewind();
- writer.write(buffer);
- buffer.rewind();
- writer.write(buffer);
- fail("Expected completed exception.");
- } catch (StorageException ex) {
-
- }
- assertTrue(writer.isOpen());
- }
-
- @Test
- public void testWriteWithLocalOffsetGoingBeyondRemoteOffset() throws Exception {
- ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO_WITH_GENERATION),
- RPC_OPTIONS_GENERATION))
- .andReturn(UPLOAD_ID);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andReturn(null);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq((long) MIN_CHUNK_SIZE),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andThrow(socketClosedException);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(0L);
- replay(storageRpcMock);
- writer = newWriter(true);
- writer.setChunkSize(MIN_CHUNK_SIZE);
- try {
- writer.write(buffer);
- buffer.rewind();
- writer.write(buffer);
- writer.close();
- fail("Expected completed exception.");
- } catch (StorageException ex) {
- }
- assertTrue(writer.isOpen());
- }
-
- @Test
- public void testGetCurrentUploadOffset() throws Exception {
- ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO_WITH_GENERATION),
- RPC_OPTIONS_GENERATION))
- .andReturn(UPLOAD_ID);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andThrow(socketClosedException);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andThrow(socketClosedException);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(0L);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(false)))
- .andReturn(null);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- (byte[]) anyObject(),
- eq(0),
- eq((long) MIN_CHUNK_SIZE),
- eq(0),
- eq(true)))
- .andReturn(Conversions.apiary().blobInfo().encode(BLOB_INFO));
- replay(storageRpcMock);
- writer = newWriter(true);
- writer.setChunkSize(MIN_CHUNK_SIZE);
- assertEquals(MIN_CHUNK_SIZE, writer.write(buffer));
- writer.close();
- assertFalse(writer.isOpen());
- assertNotNull(writer.getStorageObject());
- assertArrayEquals(buffer.array(), capturedBuffer.getValue());
- }
-
- @Test
- public void testWriteWithLastFlushRetryChunkButCompleted() throws Exception {
- ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO_WITH_GENERATION),
- RPC_OPTIONS_GENERATION))
- .andReturn(UPLOAD_ID);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(true)))
- .andThrow(socketClosedException);
- expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(-1L);
- expect(storageRpcMock.queryCompletedResumableUpload(eq(UPLOAD_ID), eq((long) MIN_CHUNK_SIZE)))
- .andReturn(
- Conversions.apiary()
- .blobInfo()
- .encode(BLOB_INFO)
- .setSize(BigInteger.valueOf(MIN_CHUNK_SIZE)));
- replay(storageRpcMock);
- writer = newWriter(true);
- assertEquals(MIN_CHUNK_SIZE, writer.write(buffer));
- writer.close();
- assertFalse(writer.isRetrying());
- assertFalse(writer.isOpen());
- assertNotNull(writer.getStorageObject());
- // Capture captures entire buffer of a chunk even when not completely used.
- // Making assert selective up to the size of MIN_CHUNK_SIZE
- assertArrayEquals(Arrays.copyOf(capturedBuffer.getValue(), MIN_CHUNK_SIZE), buffer.array());
- }
-
- @Test
- public void testWriteWithFlush() throws Exception {
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO), EMPTY_RPC_OPTIONS))
- .andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(CUSTOM_CHUNK_SIZE),
- eq(false)))
- .andReturn(null);
- replay(storageRpcMock);
- writer = newWriter();
- writer.setChunkSize(CUSTOM_CHUNK_SIZE);
- ByteBuffer buffer = randomBuffer(CUSTOM_CHUNK_SIZE);
- assertEquals(CUSTOM_CHUNK_SIZE, writer.write(buffer));
- assertArrayEquals(buffer.array(), capturedBuffer.getValue());
- assertNull(writer.getStorageObject());
- }
-
- @Test
- public void testWritesAndFlush() throws Exception {
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO), EMPTY_RPC_OPTIONS))
- .andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(DEFAULT_CHUNK_SIZE),
- eq(false)))
- .andReturn(null);
- replay(storageRpcMock);
- writer = newWriter();
- ByteBuffer[] buffers = new ByteBuffer[DEFAULT_CHUNK_SIZE / MIN_CHUNK_SIZE];
- for (int i = 0; i < buffers.length; i++) {
- buffers[i] = randomBuffer(MIN_CHUNK_SIZE);
- assertEquals(MIN_CHUNK_SIZE, writer.write(buffers[i]));
- assertNull(writer.getStorageObject());
- }
- for (int i = 0; i < buffers.length; i++) {
- assertArrayEquals(
- buffers[i].array(),
- Arrays.copyOfRange(
- capturedBuffer.getValue(), MIN_CHUNK_SIZE * i, MIN_CHUNK_SIZE * (i + 1)));
- }
- }
-
- @Test
- public void testCloseWithoutFlush() throws Exception {
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO), EMPTY_RPC_OPTIONS))
- .andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)))
- .andReturn(UPDATED_BLOB);
- replay(storageRpcMock);
- writer = newWriter();
- assertTrue(writer.isOpen());
- writer.close();
- assertArrayEquals(new byte[0], capturedBuffer.getValue());
- assertFalse(writer.isOpen());
- assertSame(UPDATED_BLOB, writer.getStorageObject());
- }
-
- @Test
- public void testCloseWithFlush() throws Exception {
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO), EMPTY_RPC_OPTIONS))
- .andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance();
- ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(true)))
- .andReturn(UPDATED_BLOB);
- replay(storageRpcMock);
- writer = newWriter();
- assertTrue(writer.isOpen());
- writer.write(buffer);
- writer.close();
- assertEquals(DEFAULT_CHUNK_SIZE, capturedBuffer.getValue().length);
- assertArrayEquals(buffer.array(), Arrays.copyOf(capturedBuffer.getValue(), MIN_CHUNK_SIZE));
- assertFalse(writer.isOpen());
- assertSame(UPDATED_BLOB, writer.getStorageObject());
- }
-
- @Test
- public void testWriteClosed() throws Exception {
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO), EMPTY_RPC_OPTIONS))
- .andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)))
- .andReturn(UPDATED_BLOB);
- replay(storageRpcMock);
- writer = newWriter();
- writer.close();
- try {
- writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE));
- fail("Expected BlobWriteChannel write to throw IOException");
- } catch (IOException ex) {
- // expected
- }
- assertSame(UPDATED_BLOB, writer.getStorageObject());
- }
-
- @Test
- public void testSaveAndRestore() throws Exception {
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO), EMPTY_RPC_OPTIONS))
- .andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance(CaptureType.ALL);
- Capture capturedPosition = Capture.newInstance(CaptureType.ALL);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- captureLong(capturedPosition),
- eq(DEFAULT_CHUNK_SIZE),
- eq(false)))
- .andReturn(null);
- expectLastCall().times(2);
- replay(storageRpcMock);
- ByteBuffer buffer1 = randomBuffer(DEFAULT_CHUNK_SIZE);
- ByteBuffer buffer2 = randomBuffer(DEFAULT_CHUNK_SIZE);
- writer = newWriter();
- assertEquals(DEFAULT_CHUNK_SIZE, writer.write(buffer1));
- assertArrayEquals(buffer1.array(), capturedBuffer.getValues().get(0));
- assertEquals(new Long(0L), capturedPosition.getValues().get(0));
- RestorableState writerState = writer.capture();
- WriteChannel restoredWriter = writerState.restore();
- assertEquals(DEFAULT_CHUNK_SIZE, restoredWriter.write(buffer2));
- assertArrayEquals(buffer2.array(), capturedBuffer.getValues().get(1));
- assertEquals(new Long(DEFAULT_CHUNK_SIZE), capturedPosition.getValues().get(1));
- }
-
- @Test
- public void testSaveAndRestoreClosed() throws Exception {
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO), EMPTY_RPC_OPTIONS))
- .andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)))
- .andReturn(UPDATED_BLOB);
- replay(storageRpcMock);
- writer = newWriter();
- writer.close();
- RestorableState writerState = writer.capture();
- RestorableState expectedWriterState =
- BlobWriteChannel.StateImpl.builder(options, BLOB_INFO, UPLOAD_ID)
- .setBuffer(null)
- .setChunkSize(DEFAULT_CHUNK_SIZE)
- .setIsOpen(false)
- .setPosition(0)
- .build();
- WriteChannel restoredWriter = writerState.restore();
- assertArrayEquals(new byte[0], capturedBuffer.getValue());
- assertEquals(expectedWriterState, restoredWriter.capture());
- }
-
- @Test
- public void testStateEquals() {
- expect(
- storageRpcMock.open(
- Conversions.apiary().blobInfo().encode(BLOB_INFO), EMPTY_RPC_OPTIONS))
- .andReturn(UPLOAD_ID)
- .times(2);
- replay(storageRpcMock);
- writer = newWriter();
- // avoid closing when you don't want partial writes to GCS upon failure
- @SuppressWarnings("resource")
- WriteChannel writer2 = newWriter();
- RestorableState state = writer.capture();
- RestorableState state2 = writer2.capture();
- assertEquals(state, state2);
- assertEquals(state.hashCode(), state2.hashCode());
- assertEquals(state.toString(), state2.toString());
- }
-
- @Test
- public void testWriteWithSignedURLAndWithFlush() throws Exception {
- expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(CUSTOM_CHUNK_SIZE),
- eq(false)))
- .andReturn(null);
- replay(storageRpcMock);
- writer = newWriterForSignedUrl();
- writer.setChunkSize(CUSTOM_CHUNK_SIZE);
- ByteBuffer buffer = randomBuffer(CUSTOM_CHUNK_SIZE);
- assertEquals(CUSTOM_CHUNK_SIZE, writer.write(buffer));
- assertArrayEquals(buffer.array(), capturedBuffer.getValue());
- }
-
- @Test
- public void testWriteWithSignedURLAndFlush() throws Exception {
- expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(DEFAULT_CHUNK_SIZE),
- eq(false)))
- .andReturn(null);
- replay(storageRpcMock);
- writer = newWriterForSignedUrl();
- ByteBuffer[] buffers = new ByteBuffer[DEFAULT_CHUNK_SIZE / MIN_CHUNK_SIZE];
- for (int i = 0; i < buffers.length; i++) {
- buffers[i] = randomBuffer(MIN_CHUNK_SIZE);
- assertEquals(MIN_CHUNK_SIZE, writer.write(buffers[i]));
- }
- for (int i = 0; i < buffers.length; i++) {
- assertArrayEquals(
- buffers[i].array(),
- Arrays.copyOfRange(
- capturedBuffer.getValue(), MIN_CHUNK_SIZE * i, MIN_CHUNK_SIZE * (i + 1)));
- }
- }
-
- @Test
- public void testCloseWithSignedURLWithoutFlush() throws Exception {
- expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)))
- .andReturn(UPDATED_BLOB);
- replay(storageRpcMock);
- writer = newWriterForSignedUrl();
- assertTrue(writer.isOpen());
- writer.close();
- assertArrayEquals(new byte[0], capturedBuffer.getValue());
- assertTrue(!writer.isOpen());
- }
-
- @Test
- public void testCloseWithSignedURLWithFlush() throws Exception {
- expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance();
- ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- eq(0L),
- eq(MIN_CHUNK_SIZE),
- eq(true)))
- .andReturn(UPDATED_BLOB);
- replay(storageRpcMock);
- writer = newWriterForSignedUrl();
- assertTrue(writer.isOpen());
- writer.write(buffer);
- writer.close();
- assertEquals(DEFAULT_CHUNK_SIZE, capturedBuffer.getValue().length);
- assertArrayEquals(buffer.array(), Arrays.copyOf(capturedBuffer.getValue(), MIN_CHUNK_SIZE));
- assertTrue(!writer.isOpen());
- }
-
- @Test
- public void testWriteWithSignedURLClosed() throws Exception {
- expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance();
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)))
- .andReturn(UPDATED_BLOB);
- replay(storageRpcMock);
- writer = newWriterForSignedUrl();
- writer.close();
- try {
- writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE));
- fail("Expected BlobWriteChannel write to throw IOException");
- } catch (IOException ex) {
- // expected
- }
- }
-
- @Test
- public void testSaveAndRestoreWithSignedURL() throws Exception {
- expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID);
- Capture capturedBuffer = Capture.newInstance(CaptureType.ALL);
- Capture capturedPosition = Capture.newInstance(CaptureType.ALL);
- expect(
- storageRpcMock.writeWithResponse(
- eq(UPLOAD_ID),
- capture(capturedBuffer),
- eq(0),
- captureLong(capturedPosition),
- eq(DEFAULT_CHUNK_SIZE),
- eq(false)))
- .andReturn(null);
- expectLastCall().times(2);
- replay(storageRpcMock);
- ByteBuffer buffer1 = randomBuffer(DEFAULT_CHUNK_SIZE);
- ByteBuffer buffer2 = randomBuffer(DEFAULT_CHUNK_SIZE);
- writer = newWriterForSignedUrl();
- assertEquals(DEFAULT_CHUNK_SIZE, writer.write(buffer1));
- assertArrayEquals(buffer1.array(), capturedBuffer.getValues().get(0));
- assertEquals(new Long(0L), capturedPosition.getValues().get(0));
- RestorableState writerState = writer.capture();
- WriteChannel restoredWriter = writerState.restore();
- assertEquals(DEFAULT_CHUNK_SIZE, restoredWriter.write(buffer2));
- assertArrayEquals(buffer2.array(), capturedBuffer.getValues().get(1));
- assertEquals(new Long(DEFAULT_CHUNK_SIZE), capturedPosition.getValues().get(1));
- }
-
- private BlobWriteChannel newWriter() {
- return newWriter(false);
- }
-
- private BlobWriteChannel newWriter(boolean withGeneration) {
- Map optionsMap =
- withGeneration ? RPC_OPTIONS_GENERATION : EMPTY_RPC_OPTIONS;
- ResultRetryAlgorithm createResultAlgorithm =
- retryAlgorithmManager.getForResumableUploadSessionCreate(optionsMap);
- ResultRetryAlgorithm writeResultAlgorithm =
- retryAlgorithmManager.getForResumableUploadSessionWrite(optionsMap);
- final BlobInfo blobInfo = withGeneration ? BLOB_INFO_WITH_GENERATION : BLOB_INFO;
- return BlobWriteChannel.newBuilder()
- .setStorageOptions(options)
- .setBlobInfo(blobInfo)
- .setUploadIdSupplier(
- ResumableMedia.startUploadForBlobInfo(
- options, blobInfo, optionsMap, createResultAlgorithm))
- .setAlgorithmForWrite(writeResultAlgorithm)
- .build();
- }
-
- private BlobWriteChannel newWriterForSignedUrl() throws MalformedURLException {
- Map optionsMap = Collections.emptyMap();
- ResultRetryAlgorithm createResultAlgorithm =
- retryAlgorithmManager.getForResumableUploadSessionCreate(optionsMap);
- ResultRetryAlgorithm writeResultAlgorithm =
- retryAlgorithmManager.getForResumableUploadSessionWrite(optionsMap);
- return BlobWriteChannel.newBuilder()
- .setStorageOptions(options)
- .setUploadIdSupplier(
- ResumableMedia.startUploadForSignedUrl(
- options, new URL(SIGNED_URL), createResultAlgorithm))
- .setAlgorithmForWrite(writeResultAlgorithm)
- .build();
- }
-
- private static ByteBuffer randomBuffer(int size) {
- byte[] byteArray = new byte[size];
- RANDOM.nextBytes(byteArray);
- return ByteBuffer.wrap(byteArray);
- }
-}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/FakeHttpServer.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/FakeHttpServer.java
new file mode 100644
index 0000000000..cff9dc4696
--- /dev/null
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/FakeHttpServer.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
+import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
+import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpHeaderValues.CLOSE;
+
+import io.grpc.netty.shaded.io.netty.bootstrap.ServerBootstrap;
+import io.grpc.netty.shaded.io.netty.buffer.ByteBuf;
+import io.grpc.netty.shaded.io.netty.channel.Channel;
+import io.grpc.netty.shaded.io.netty.channel.ChannelFuture;
+import io.grpc.netty.shaded.io.netty.channel.ChannelFutureListener;
+import io.grpc.netty.shaded.io.netty.channel.ChannelHandlerContext;
+import io.grpc.netty.shaded.io.netty.channel.ChannelInitializer;
+import io.grpc.netty.shaded.io.netty.channel.ChannelOption;
+import io.grpc.netty.shaded.io.netty.channel.ChannelPipeline;
+import io.grpc.netty.shaded.io.netty.channel.EventLoopGroup;
+import io.grpc.netty.shaded.io.netty.channel.SimpleChannelInboundHandler;
+import io.grpc.netty.shaded.io.netty.channel.nio.NioEventLoopGroup;
+import io.grpc.netty.shaded.io.netty.channel.socket.SocketChannel;
+import io.grpc.netty.shaded.io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.FullHttpResponse;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.HttpHeaders;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.HttpObjectAggregator;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.HttpRequest;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.HttpServerCodec;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.HttpServerExpectContinueHandler;
+import io.grpc.netty.shaded.io.netty.handler.logging.LogLevel;
+import io.grpc.netty.shaded.io.netty.handler.logging.LoggingHandler;
+import java.net.InetSocketAddress;
+import java.net.URI;
+
+final class FakeHttpServer implements AutoCloseable {
+
+ private final URI endpoint;
+ private final Channel channel;
+ private final Runnable shutdown;
+
+ private FakeHttpServer(URI endpoint, Channel channel, Runnable shutdown) {
+ this.endpoint = endpoint;
+ this.channel = channel;
+ this.shutdown = shutdown;
+ }
+
+ public URI getEndpoint() {
+ return endpoint;
+ }
+
+ @Override
+ public void close() throws Exception {
+ shutdown.run();
+ channel.closeFuture().syncUninterruptibly();
+ }
+
+ static FakeHttpServer of(HttpRequestHandler server) {
+ // based on
+ // https://github.com/netty/netty/blob/59aa6e635b9996cf21cd946e64353270679adc73/example/src/main/java/io/netty/example/http/helloworld/HttpHelloWorldServer.java
+ InetSocketAddress address = new InetSocketAddress("localhost", 0);
+ // Configure the server.
+ EventLoopGroup bossGroup = new NioEventLoopGroup(1);
+ EventLoopGroup workerGroup = new NioEventLoopGroup();
+ ServerBootstrap b = new ServerBootstrap();
+ b.option(ChannelOption.SO_BACKLOG, 1024);
+ b.group(bossGroup, workerGroup)
+ .channel(NioServerSocketChannel.class)
+ .handler(new LoggingHandler(LogLevel.DEBUG))
+ .childHandler(
+ new ChannelInitializer() {
+ @Override
+ protected void initChannel(SocketChannel ch) {
+ ChannelPipeline p = ch.pipeline();
+ p.addLast(new HttpServerCodec());
+ // Accept a request and content up to 100 MiB
+ // If we don't do this, sometimes the ordering on the wire will result in the server
+ // rejecting the request before the client has finished sending.
+ // While our client can handle this scenario and retry, it makes assertions more
+ // difficult due to the variability of request counts.
+ p.addLast(new HttpObjectAggregator(100 * 1024 * 1024));
+ p.addLast(new HttpServerExpectContinueHandler());
+ p.addLast(new Handler(server));
+ }
+ });
+
+ Channel channel = b.bind(address).syncUninterruptibly().channel();
+
+ InetSocketAddress socketAddress = (InetSocketAddress) channel.localAddress();
+ return new FakeHttpServer(
+ URI.create("http://localhost:" + socketAddress.getPort()),
+ channel,
+ () -> {
+ bossGroup.shutdownGracefully();
+ workerGroup.shutdownGracefully();
+ });
+ }
+
+ interface HttpRequestHandler {
+ FullHttpResponse apply(HttpRequest req) throws Exception;
+ }
+
+ /**
+ * Based on
+ * https://github.com/netty/netty/blob/59aa6e635b9996cf21cd946e64353270679adc73/example/src/main/java/io/netty/example/http/helloworld/HttpHelloWorldServerHandler.java
+ */
+ private static final class Handler extends SimpleChannelInboundHandler {
+
+ private final HttpRequestHandler server;
+
+ private Handler(HttpRequestHandler server) {
+ this.server = server;
+ }
+
+ @Override
+ public void channelReadComplete(ChannelHandlerContext ctx) {
+ ctx.flush();
+ }
+
+ @Override
+ protected void channelRead0(ChannelHandlerContext ctx, HttpRequest req) throws Exception {
+ FullHttpResponse resp = server.apply(req);
+ HttpHeaders headers = resp.headers();
+ if (!headers.contains(CONTENT_LENGTH)) {
+ ByteBuf content = resp.content();
+ headers.setInt(CONTENT_LENGTH, content.readableBytes());
+ }
+ headers.set(CONNECTION, CLOSE);
+ ChannelFuture future = ctx.writeAndFlush(resp);
+ future.addListener(ChannelFutureListener.CLOSE);
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+ cause.printStackTrace();
+ ctx.close();
+ }
+ }
+}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcStorageImplUploadRetryTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcStorageImplUploadRetryTest.java
index ac192a1ff5..12f1bd6b9a 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcStorageImplUploadRetryTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcStorageImplUploadRetryTest.java
@@ -29,6 +29,8 @@
import com.google.storage.v2.ChecksummedData;
import com.google.storage.v2.Object;
import com.google.storage.v2.ObjectChecksums;
+import com.google.storage.v2.QueryWriteStatusRequest;
+import com.google.storage.v2.QueryWriteStatusResponse;
import com.google.storage.v2.StartResumableWriteRequest;
import com.google.storage.v2.StartResumableWriteResponse;
import com.google.storage.v2.StorageGrpc.StorageImplBase;
@@ -93,7 +95,7 @@ public void create_inputStream() throws Exception {
@Test
public void createFrom_path_smallerThanBufferSize() throws Exception {
- Direct.FakeService service = Direct.FakeService.create();
+ Resumable.FakeService service = Resumable.FakeService.create();
try (TmpFile tmpFile = DataGenerator.base64Characters().tempFile(baseDir, objectContentSize);
FakeServer server = FakeServer.of(service);
@@ -287,8 +289,16 @@ public void startResumableWrite(
}
}
+ @Override
+ public void queryWriteStatus(
+ QueryWriteStatusRequest request,
+ StreamObserver responseObserver) {
+ responseObserver.onNext(QueryWriteStatusResponse.newBuilder().setPersistedSize(0).build());
+ responseObserver.onCompleted();
+ }
+
// a bit of constructor lifecycle hackery to appease the compiler
- // Even though the thing past to super() is a lazy function, the closing over of the outer
+ // Even though the thing passed to super() is a lazy function, the closing over of the outer
// fields happens earlier than they are available. To side step this fact, we provide the
// AtomicBoolean as a constructor argument which can be closed over without issue, and then
// bind it to the class field after super().
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITJsonResumableSessionPutTaskTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITJsonResumableSessionPutTaskTest.java
new file mode 100644
index 0000000000..b7d9d6c74a
--- /dev/null
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITJsonResumableSessionPutTaskTest.java
@@ -0,0 +1,874 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import static com.google.cloud.storage.ByteSizeConstants._128KiBL;
+import static com.google.cloud.storage.ByteSizeConstants._256KiB;
+import static com.google.cloud.storage.ByteSizeConstants._256KiBL;
+import static com.google.cloud.storage.ByteSizeConstants._512KiBL;
+import static com.google.cloud.storage.ByteSizeConstants._768KiBL;
+import static com.google.common.truth.Truth.assertThat;
+import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpHeaderNames.CONTENT_RANGE;
+import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
+import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpResponseStatus.OK;
+import static org.junit.Assert.assertThrows;
+
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.JsonObjectParser;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.gax.retrying.ResultRetryAlgorithm;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.cloud.storage.FakeHttpServer.HttpRequestHandler;
+import com.google.cloud.storage.it.runner.StorageITRunner;
+import com.google.cloud.storage.it.runner.annotations.Backend;
+import com.google.cloud.storage.it.runner.annotations.ParallelFriendly;
+import com.google.cloud.storage.it.runner.annotations.SingleBackend;
+import com.google.common.collect.ImmutableMap;
+import io.grpc.netty.shaded.io.netty.buffer.ByteBuf;
+import io.grpc.netty.shaded.io.netty.buffer.Unpooled;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.FullHttpResponse;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.HttpHeaderNames;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.HttpResponseStatus;
+import java.math.BigInteger;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+
+@RunWith(StorageITRunner.class)
+@SingleBackend(Backend.PROD)
+@ParallelFriendly
+public final class ITJsonResumableSessionPutTaskTest {
+ private static final GsonFactory gson = GsonFactory.getDefaultInstance();
+ private static final NetHttpTransport transport = new NetHttpTransport.Builder().build();
+ private static final HttpResponseStatus RESUME_INCOMPLETE =
+ HttpResponseStatus.valueOf(308, "Resume Incomplete");
+ private static final HttpResponseStatus APPEND_GREATER_THAN_CURRENT_SIZE =
+ HttpResponseStatus.valueOf(503, "");
+ private HttpClientContext httpClientContext;
+
+ @Rule public final TemporaryFolder temp = new TemporaryFolder();
+
+ @Before
+ public void setUp() throws Exception {
+ httpClientContext =
+ HttpClientContext.of(transport.createRequestFactory(), new JsonObjectParser(gson));
+ }
+
+ @Test
+ public void emptyObjectHappyPath() throws Exception {
+
+ HttpRequestHandler handler =
+ req -> {
+ StorageObject so = new StorageObject();
+ so.setName("object-name").setSize(BigInteger.ZERO);
+ ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(so));
+
+ DefaultFullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), OK, buf);
+ resp.headers().set(CONTENT_TYPE, "application/json; charset=utf-8");
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ httpClientContext,
+ uploadUrl,
+ RewindableContent.empty(),
+ HttpContentRange.of(ByteRangeSpec.explicitClosed(0L, 0L), 0));
+
+ ResumableOperationResult<@Nullable StorageObject> operationResult = task.call();
+ StorageObject object = operationResult.getObject();
+ assertThat(object).isNotNull();
+ assertThat(operationResult.getPersistedSize()).isEqualTo(0L);
+ }
+ }
+
+ /**
+ *
+ *
+ * S.9
+ *
+ * Partial successful append to session
+ *
+ * The client has sent N bytes, the server confirmed N bytes as committed. The client sends K
+ * bytes starting at offset N. The server responds with only N + L with 0 <= L < K bytes as
+ * committed.
+ */
+ @Test
+ public void scenario9() throws Exception {
+
+ HttpRequestHandler handler =
+ req -> {
+ String contentRangeString = req.headers().get(CONTENT_RANGE);
+ HttpContentRange parse = HttpContentRange.parse(contentRangeString);
+ long endInclusive = ((HttpContentRange.HasRange) parse).range().endOffsetInclusive();
+ FullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), RESUME_INCOMPLETE);
+ ByteRangeSpec range = ByteRangeSpec.explicitClosed(0L, endInclusive - 1);
+ resp.headers().set(HttpHeaderNames.RANGE, range.getHttpRangeHeader());
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ AtomicLong confirmedBytes = new AtomicLong(-1L);
+
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ httpClientContext,
+ uploadUrl,
+ RewindableContent.empty(),
+ HttpContentRange.of(ByteRangeSpec.explicitClosed(0L, 10L)));
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ assertThat(se.getCode()).isEqualTo(503);
+ assertThat(confirmedBytes.get()).isEqualTo(-1L);
+ }
+ }
+
+ /**
+ *
+ *
+ *
S.7
+ *
+ * GCS Acknowledges more bytes than were sent in the PUT
+ *
+ * The client believes the server offset is N, it sends K bytes and the server responds that N
+ * + 2K bytes are now committed.
+ *
+ *
The client has detected data loss and should raise an error and prevent sending of more
+ * bytes.
+ */
+ @Test
+ public void scenario7() throws Exception {
+
+ HttpRequestHandler handler =
+ req -> {
+ String contentRangeString = req.headers().get(CONTENT_RANGE);
+ HttpContentRange parse = HttpContentRange.parse(contentRangeString);
+ long endInclusive = ((HttpContentRange.HasRange) parse).range().endOffsetInclusive();
+ FullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), RESUME_INCOMPLETE);
+ ByteRangeSpec range = ByteRangeSpec.explicitClosed(0L, endInclusive + 1);
+ resp.headers().set(HttpHeaderNames.RANGE, range.getHttpRangeHeader());
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ AtomicLong confirmedBytes = new AtomicLong(-1L);
+
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ httpClientContext,
+ uploadUrl,
+ RewindableContent.empty(),
+ HttpContentRange.of(ByteRangeSpec.explicitClosed(0L, 10L)));
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ assertThat(se.getCode()).isEqualTo(0);
+ assertThat(se.getReason()).isEqualTo("dataLoss");
+ assertThat(confirmedBytes.get()).isEqualTo(-1L);
+ }
+ }
+
+ /**
+ *
+ *
+ *
S.1
+ *
+ * Attempting to append to a session which has already been finalized should raise an error
+ *
+ *
+ *
+ *
+ * | server state |
+ *
+ * resource = { name= obj, persisted_size = 524288 }
+ * |
+ *
+ *
+ * | client state |
+ *
+ * write_offset = 0, data = [0:262144]
+ * |
+ *
+ *
+ * | request |
+ *
+ * PUT $UPLOAD_ID
+ * Content-Range: bytes 0-262143/*
+ * |
+ *
+ *
+ * | response |
+ *
+ * 200 OK
+ * Content-Type: application/json; charset=utf-8
+ *
+ * {"name": "obj", "size": 524288}
+ * |
+ *
+ *
+ */
+ @Test
+ public void scenario1() throws Exception {
+ HttpRequestHandler handler =
+ req -> {
+ StorageObject so = new StorageObject();
+ URI uri = URI.create(req.uri());
+ so.setName("object")
+ .setBucket("bucket")
+ .setGeneration(1L)
+ .setMetageneration(1L)
+ .setSize(BigInteger.valueOf(_512KiBL))
+ .setMetadata(ImmutableMap.of("upload_id", uri.toString()));
+
+ ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(so));
+ DefaultFullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), OK, buf);
+ resp.headers().set(CONTENT_TYPE, "application/json; charset=utf-8");
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler);
+ TmpFile tmpFile =
+ DataGenerator.base64Characters().tempFile(temp.newFolder().toPath(), _256KiBL)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ AtomicLong confirmedBytes = new AtomicLong(-1L);
+
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ httpClientContext,
+ uploadUrl,
+ RewindableContent.of(tmpFile.getPath()),
+ HttpContentRange.of(ByteRangeSpec.explicit(0L, _256KiBL)));
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ assertThat(se.getCode()).isEqualTo(0);
+ assertThat(se.getReason()).isEqualTo("invalid");
+ assertThat(confirmedBytes.get()).isEqualTo(-1L);
+ }
+ }
+
+ /**
+ *
+ *
+ * S.2
+ *
+ * Attempting to finalize a session with fewer bytes than GCS acknowledges.
+ *
+ *
+ *
+ *
+ * | server state |
+ *
+ * persisted_size = 524288
+ * |
+ *
+ *
+ * | client state |
+ *
+ * write_offset = 262144, finish = true
+ * |
+ *
+ *
+ * | request |
+ *
+ * PUT $UPLOAD_ID
+ * Content-Range: bytes */262144
+ * |
+ *
+ *
+ * | response |
+ *
+ * 308 Resume Incomplete
+ * Range: bytes=0-524287
+ * |
+ *
+ *
+ */
+ @Test
+ public void scenario2() throws Exception {
+
+ HttpRequestHandler handler =
+ req -> {
+ FullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), RESUME_INCOMPLETE);
+ ByteRangeSpec range = ByteRangeSpec.explicit(0L, _512KiBL);
+ resp.headers().set(HttpHeaderNames.RANGE, range.getHttpRangeHeader());
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ AtomicLong confirmedBytes = new AtomicLong(-1L);
+
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ httpClientContext,
+ uploadUrl,
+ RewindableContent.empty(),
+ HttpContentRange.of(_256KiBL));
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ assertThat(se.getCode()).isEqualTo(0);
+ assertThat(se.getReason()).isEqualTo("invalid");
+ assertThat(confirmedBytes.get()).isEqualTo(-1L);
+ }
+ }
+
+ /**
+ *
+ *
+ * S.3
+ *
+ * Attempting to finalize a session with more bytes than GCS acknowledges.
+ *
+ *
+ *
+ *
+ * | server state |
+ *
+ * persisted_size = 262144
+ * |
+ *
+ *
+ * | client state |
+ *
+ * write_offset = 524288, finish = true
+ * |
+ *
+ *
+ * | request |
+ *
+ * PUT $UPLOAD_ID
+ * Content-Range: bytes */524288
+ * |
+ *
+ *
+ * | response |
+ *
+ * 308 Resume Incomplete
+ * Range: bytes=0-262143
+ * |
+ *
+ *
+ */
+ @Test
+ public void scenario3() throws Exception {
+
+ HttpRequestHandler handler =
+ req -> {
+ FullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), RESUME_INCOMPLETE);
+ ByteRangeSpec range = ByteRangeSpec.explicit(0L, _256KiBL);
+ resp.headers().set(HttpHeaderNames.RANGE, range.getHttpRangeHeader());
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ AtomicLong confirmedBytes = new AtomicLong(-1L);
+
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ httpClientContext,
+ uploadUrl,
+ RewindableContent.empty(),
+ HttpContentRange.of(_512KiBL));
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ assertThat(se.getCode()).isEqualTo(0);
+ assertThat(se.getReason()).isEqualTo("dataLoss");
+ assertThat(confirmedBytes.get()).isEqualTo(-1L);
+ }
+ }
+
+ /**
+ *
+ *
+ * S.4
+ *
+ * Attempting to finalize an already finalized session
+ *
+ *
+ *
+ *
+ * | server state |
+ *
+ * resource = {name = obj1, size = 262114}
+ * |
+ *
+ *
+ * | client state |
+ *
+ * write_offset = 262114, finish = true
+ * |
+ *
+ *
+ * | request |
+ *
+ * PUT $UPLOAD_ID
+ * Content-Range: bytes */262114
+ * |
+ *
+ *
+ * | response |
+ *
+ * 200 Ok
+ * Content-Type: application/json; charset=utf-8
+ *
+ * {"name": "obj", "size": 262114}
+ * |
+ *
+ *
+ */
+ @Test
+ public void scenario4() throws Exception {
+
+ HttpRequestHandler handler =
+ req -> {
+ StorageObject so = new StorageObject();
+ URI uri = URI.create(req.uri());
+ so.setName("object")
+ .setBucket("bucket")
+ .setGeneration(1L)
+ .setMetageneration(1L)
+ .setSize(BigInteger.valueOf(_256KiBL))
+ .setMetadata(ImmutableMap.of("upload_id", uri.toString()));
+
+ ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(so));
+ DefaultFullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), OK, buf);
+ resp.headers().set(CONTENT_TYPE, "application/json; charset=utf-8");
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ httpClientContext,
+ uploadUrl,
+ RewindableContent.empty(),
+ HttpContentRange.of(_256KiBL));
+
+ ResumableOperationResult<@Nullable StorageObject> operationResult = task.call();
+ StorageObject call = operationResult.getObject();
+ assertThat(call).isNotNull();
+ assertThat(call.getMetadata())
+ .containsEntry("upload_id", uploadUrl.substring(endpoint.toString().length()));
+ assertThat(operationResult.getPersistedSize()).isEqualTo(_256KiBL);
+ }
+ }
+
+ /**
+ *
+ *
+ * S.4.1
+ *
+ * Attempting to finalize an already finalized session (ack < expected)
+ *
+ *
+ *
+ *
+ * | server state |
+ *
+ * resource = {name = obj1, size = 262114}
+ * |
+ *
+ *
+ * | client state |
+ *
+ * write_offset = 524288, finish = true
+ * |
+ *
+ *
+ * | request |
+ *
+ * PUT $UPLOAD_ID
+ * Content-Range: bytes */524288
+ * |
+ *
+ *
+ * | response |
+ *
+ * 200 Ok
+ * Content-Type: application/json; charset=utf-8
+ *
+ * {"name": "obj", "size": 262114}
+ * |
+ *
+ *
+ */
+ @Test
+ public void scenario4_1() throws Exception {
+
+ HttpRequestHandler handler =
+ req -> {
+ StorageObject so = new StorageObject();
+ URI uri = URI.create(req.uri());
+ so.setName("object")
+ .setBucket("bucket")
+ .setGeneration(1L)
+ .setMetageneration(1L)
+ .setSize(BigInteger.valueOf(_256KiBL))
+ .setMetadata(ImmutableMap.of("upload_id", uri.toString()));
+
+ ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(so));
+ DefaultFullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), OK, buf);
+ resp.headers().set(CONTENT_TYPE, "application/json; charset=utf-8");
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ AtomicLong confirmedBytes = new AtomicLong(-1L);
+
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ httpClientContext,
+ uploadUrl,
+ RewindableContent.empty(),
+ HttpContentRange.of(_512KiBL));
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ assertThat(se.getCode()).isEqualTo(0);
+ assertThat(se.getReason()).isEqualTo("dataLoss");
+ assertThat(confirmedBytes.get()).isEqualTo(-1);
+ }
+ }
+
+ /**
+ *
+ *
+ * S.4.2
+ *
+ * Attempting to finalize an already finalized session (ack > expected)
+ *
+ *
+ *
+ *
+ * | server state |
+ *
+ * resource = {name = obj1, size = 262114}
+ * |
+ *
+ *
+ * | client state |
+ *
+ * write_offset = 524288, finish = true
+ * |
+ *
+ *
+ * | request |
+ *
+ * PUT $UPLOAD_ID
+ * Content-Range: bytes */131072
+ * |
+ *
+ *
+ * | response |
+ *
+ * 200 Ok
+ * Content-Type: application/json; charset=utf-8
+ *
+ * {"name": "obj", "size": 262114}
+ * |
+ *
+ *
+ */
+ @Test
+ public void scenario4_2() throws Exception {
+
+ HttpRequestHandler handler =
+ req -> {
+ StorageObject so = new StorageObject();
+ URI uri = URI.create(req.uri());
+ so.setName("object")
+ .setBucket("bucket")
+ .setGeneration(1L)
+ .setMetageneration(1L)
+ .setSize(BigInteger.valueOf(_256KiBL))
+ .setMetadata(ImmutableMap.of("upload_id", uri.toString()));
+
+ ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(so));
+ DefaultFullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), OK, buf);
+ resp.headers().set(CONTENT_TYPE, "application/json; charset=utf-8");
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ AtomicLong confirmedBytes = new AtomicLong(-1L);
+
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ httpClientContext,
+ uploadUrl,
+ RewindableContent.empty(),
+ HttpContentRange.of(_128KiBL));
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ assertThat(se.getCode()).isEqualTo(0);
+ assertThat(se.getReason()).isEqualTo("dataLoss");
+ assertThat(confirmedBytes.get()).isEqualTo(-1);
+ }
+ }
+
+ /**
+ *
+ *
+ * S.5
+ *
+ * Attempt to append to a resumable session with an offset higher than GCS expects
+ *
+ *
+ *
+ *
+ * | server state |
+ *
+ * persisted_size = 262144
+ * |
+ *
+ *
+ * | client state |
+ *
+ * write_offset = 524288, data = [524288:786432]
+ * |
+ *
+ *
+ * | request |
+ *
+ * PUT $UPLOAD_ID
+ * Content-Range: bytes 524288-786431/*
+ * |
+ *
+ *
+ * | response |
+ *
+ * 503
+ * Content-Type: text/plain; charset=utf-8
+ *
+ * Invalid request. According to the Content-Range header, the upload offset is 524288 byte(s), which exceeds already uploaded size of 262144 byte(s).
+ * |
+ *
+ *
+ */
+ @Test
+ public void scenario5() throws Exception {
+
+ HttpRequestHandler handler =
+ req -> {
+ // error message from GCS circa 2023-02
+ ByteBuf buf =
+ Unpooled.wrappedBuffer(
+ "Invalid request. According to the Content-Range header, the upload offset is 524288 byte(s), which exceeds already uploaded size of 262144 byte(s)."
+ .getBytes(StandardCharsets.UTF_8));
+ FullHttpResponse resp =
+ new DefaultFullHttpResponse(
+ req.protocolVersion(), APPEND_GREATER_THAN_CURRENT_SIZE, buf);
+ resp.headers().set(CONTENT_TYPE, "text/plain; charset=utf-8");
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler);
+ TmpFile tmpFile =
+ DataGenerator.base64Characters().tempFile(temp.newFolder().toPath(), _256KiBL)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ AtomicLong confirmedBytes = new AtomicLong(-1L);
+
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ httpClientContext,
+ uploadUrl,
+ RewindableContent.of(tmpFile.getPath()),
+ HttpContentRange.of(ByteRangeSpec.explicit(_512KiBL, _768KiBL)));
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ assertThat(se.getCode()).isEqualTo(0);
+ assertThat(se.getReason()).isEqualTo("dataLoss");
+ assertThat(confirmedBytes.get()).isEqualTo(-1);
+ }
+ }
+
+ @Test
+ public void jsonParseFailure() throws Exception {
+
+ HttpRequestHandler handler =
+ req -> {
+ StorageObject so = new StorageObject();
+ URI uri = URI.create(req.uri());
+ so.setName("object")
+ .setBucket("bucket")
+ .setGeneration(1L)
+ .setMetageneration(1L)
+ .setSize(BigInteger.ZERO)
+ .setMetadata(ImmutableMap.of("upload_id", uri.toString()));
+
+ byte[] bytes = gson.toByteArray(so);
+ ByteBuf buf = Unpooled.wrappedBuffer(bytes, 0, bytes.length / 2);
+ DefaultFullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), OK, buf);
+ resp.headers().set(CONTENT_TYPE, "application/json; charset=utf-8");
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ AtomicLong confirmedBytes = new AtomicLong(-1L);
+
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ httpClientContext, uploadUrl, RewindableContent.empty(), HttpContentRange.of(0));
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ // the parse error happens while trying to read the success object, make sure we raise it as
+ // a client side retryable exception
+ assertThat(se.getCode()).isEqualTo(0);
+ assertThat(se.getReason()).isEqualTo(null);
+ // Finalization was successful, but we can't confirm the number of bytes due to the parse
+ // error
+ assertThat(confirmedBytes.get()).isEqualTo(-1);
+
+ ResultRetryAlgorithm idempotentHandler =
+ StorageRetryStrategy.getDefaultStorageRetryStrategy().getIdempotentHandler();
+ boolean shouldRetry = idempotentHandler.shouldRetry(se, null);
+ assertThat(shouldRetry).isTrue();
+ }
+ }
+
+ @Test
+ public void jsonDeserializationOnlyAttemptedWhenContentPresent() throws Exception {
+
+ HttpRequestHandler handler =
+ req -> {
+ DefaultFullHttpResponse resp = new DefaultFullHttpResponse(req.protocolVersion(), OK);
+ resp.headers().set(CONTENT_TYPE, "text/html; charset=UTF-8");
+ resp.headers().set("x-goog-stored-content-length", "0");
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ httpClientContext, uploadUrl, RewindableContent.empty(), HttpContentRange.of(0));
+
+ ResumableOperationResult<@Nullable StorageObject> operationResult = task.call();
+ StorageObject call = operationResult.getObject();
+ assertThat(call).isNull();
+ assertThat(operationResult.getPersistedSize()).isEqualTo(0L);
+ }
+ }
+
+ @Test
+ public void attemptToRewindOutOfBoundsThrows_lower() {
+ RewindableContent content = RewindableContent.of();
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ null, null, content, HttpContentRange.of(ByteRangeSpec.relativeLength(10L, 10L)));
+
+ IllegalArgumentException iae =
+ assertThrows(IllegalArgumentException.class, () -> task.rewindTo(9));
+ assertThat(iae).hasMessageThat().isEqualTo("Rewind offset is out of bounds. (10 <= 9 < 20)");
+ }
+
+ @Test
+ public void attemptToRewindOutOfBoundsThrows_upper() {
+ RewindableContent content = RewindableContent.of();
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ null, null, content, HttpContentRange.of(ByteRangeSpec.relativeLength(10L, 10L)));
+
+ IllegalArgumentException iae =
+ assertThrows(IllegalArgumentException.class, () -> task.rewindTo(20));
+ assertThat(iae).hasMessageThat().isEqualTo("Rewind offset is out of bounds. (10 <= 20 < 20)");
+ }
+
+ @Test
+ public void repeatedRewindsToTheSameLocationWork() {
+ ByteBuffer buf1 = DataGenerator.base64Characters().genByteBuffer(_256KiB);
+ ByteBuffer buf2 = DataGenerator.base64Characters().genByteBuffer(_256KiB);
+ RewindableContent content = RewindableContent.of(buf1, buf2);
+ JsonResumableSessionPutTask task =
+ new JsonResumableSessionPutTask(
+ null, null, content, HttpContentRange.of(ByteRangeSpec.relativeLength(0L, _512KiBL)));
+
+ task.rewindTo(0);
+ assertThat(buf1.position()).isEqualTo(0);
+ assertThat(buf2.position()).isEqualTo(0);
+
+ int last = buf1.capacity();
+ buf1.position(last);
+ buf2.position(last);
+
+ task.rewindTo(_256KiBL);
+ assertThat(buf1.remaining()).isEqualTo(0);
+ assertThat(buf2.position()).isEqualTo(0);
+
+ task.rewindTo(_256KiBL);
+ assertThat(buf1.remaining()).isEqualTo(0);
+ assertThat(buf2.position()).isEqualTo(0);
+
+ task.rewindTo(_256KiBL + 13);
+ assertThat(buf1.remaining()).isEqualTo(0);
+ assertThat(buf2.position()).isEqualTo(13);
+
+ task.rewindTo(_256KiBL + 13);
+ assertThat(buf1.remaining()).isEqualTo(0);
+ assertThat(buf2.position()).isEqualTo(13);
+ }
+}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITJsonResumableSessionQueryTaskTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITJsonResumableSessionQueryTaskTest.java
new file mode 100644
index 0000000000..07a04ed61e
--- /dev/null
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITJsonResumableSessionQueryTaskTest.java
@@ -0,0 +1,235 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import static com.google.cloud.storage.ByteSizeConstants._256KiBL;
+import static com.google.common.truth.Truth.assertThat;
+import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
+import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpResponseStatus.OK;
+import static org.junit.Assert.assertThrows;
+
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.JsonObjectParser;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.cloud.storage.FakeHttpServer.HttpRequestHandler;
+import com.google.cloud.storage.it.runner.StorageITRunner;
+import com.google.cloud.storage.it.runner.annotations.Backend;
+import com.google.cloud.storage.it.runner.annotations.ParallelFriendly;
+import com.google.cloud.storage.it.runner.annotations.SingleBackend;
+import io.grpc.netty.shaded.io.netty.buffer.ByteBuf;
+import io.grpc.netty.shaded.io.netty.buffer.Unpooled;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.FullHttpResponse;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.HttpHeaderNames;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.HttpResponseStatus;
+import java.math.BigInteger;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.UUID;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@RunWith(StorageITRunner.class)
+@SingleBackend(Backend.PROD)
+@ParallelFriendly
+public final class ITJsonResumableSessionQueryTaskTest {
+ private static final GsonFactory gson = GsonFactory.getDefaultInstance();
+ private static final NetHttpTransport transport = new NetHttpTransport.Builder().build();
+ private static final HttpResponseStatus RESUME_INCOMPLETE =
+ HttpResponseStatus.valueOf(308, "Resume Incomplete");
+ private static final HttpResponseStatus APPEND_GREATER_THAN_CURRENT_SIZE =
+ HttpResponseStatus.valueOf(503, "");
+
+ private HttpClientContext httpClientContext;
+
+ @Before
+ public void setUp() throws Exception {
+ httpClientContext =
+ HttpClientContext.of(transport.createRequestFactory(), new JsonObjectParser(gson));
+ }
+
+ @Test
+ public void successfulSession() throws Exception {
+ HttpRequestHandler handler =
+ req -> {
+ StorageObject so = new StorageObject();
+ so.setName("object-name").setSize(BigInteger.ZERO);
+ ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(so));
+
+ DefaultFullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), OK, buf);
+ resp.headers().set(CONTENT_TYPE, "application/json; charset=utf-8");
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableSessionQueryTask task =
+ new JsonResumableSessionQueryTask(httpClientContext, uploadUrl);
+
+ ResumableOperationResult<@Nullable StorageObject> result = task.call();
+ StorageObject object = result.getObject();
+ assertThat(object).isNotNull();
+ assertThat(result.getPersistedSize()).isEqualTo(0L);
+ }
+ }
+
+ @Test
+ public void successfulSession_noObject() throws Exception {
+ HttpRequestHandler handler =
+ req -> {
+ DefaultFullHttpResponse response = new DefaultFullHttpResponse(req.protocolVersion(), OK);
+ response.headers().set("X-Goog-Stored-Content-Length", 0);
+ return response;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableSessionQueryTask task =
+ new JsonResumableSessionQueryTask(httpClientContext, uploadUrl);
+
+ ResumableOperationResult<@Nullable StorageObject> result = task.call();
+ StorageObject object = result.getObject();
+ assertThat(object).isNull();
+ assertThat(result.getPersistedSize()).isEqualTo(0L);
+ }
+ }
+
+ @Test
+ public void incompleteSession() throws Exception {
+ HttpRequestHandler handler =
+ req -> {
+ DefaultFullHttpResponse response =
+ new DefaultFullHttpResponse(req.protocolVersion(), RESUME_INCOMPLETE);
+ response
+ .headers()
+ .set(
+ HttpHeaderNames.RANGE,
+ ByteRangeSpec.relativeLength(0L, _256KiBL).getHttpRangeHeader());
+ return response;
+ };
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableSessionQueryTask task =
+ new JsonResumableSessionQueryTask(httpClientContext, uploadUrl);
+
+ ResumableOperationResult<@Nullable StorageObject> result = task.call();
+ assertThat(result.getPersistedSize()).isEqualTo(_256KiBL);
+ }
+ }
+
+ /**
+ * This is a hard failure from the perspective of GCS as a range header is a required header to be
+ * included in the response to a query upload request.
+ */
+ @Test
+ public void incompleteSession_missingRangeHeader() throws Exception {
+ HttpRequestHandler handler =
+ req -> new DefaultFullHttpResponse(req.protocolVersion(), RESUME_INCOMPLETE);
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableSessionQueryTask task =
+ new JsonResumableSessionQueryTask(httpClientContext, uploadUrl);
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ assertThat(se.getCode()).isEqualTo(503);
+ assertThat(se).hasMessageThat().contains("Range");
+ }
+ }
+
+ @Test
+ public void successfulSession_noJson_noStoredContentLength() throws Exception {
+ HttpRequestHandler handler = req -> new DefaultFullHttpResponse(req.protocolVersion(), OK);
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableSessionQueryTask task =
+ new JsonResumableSessionQueryTask(httpClientContext, uploadUrl);
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ assertThat(se.getCode()).isEqualTo(0);
+ }
+ }
+
+ @Test
+ public void successfulSession_noSize() throws Exception {
+ HttpRequestHandler handler =
+ req -> {
+ StorageObject so = new StorageObject();
+ so.setName("object-name");
+ ByteBuf buf = Unpooled.wrappedBuffer(gson.toByteArray(so));
+
+ DefaultFullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), OK, buf);
+ resp.headers().set(CONTENT_TYPE, "application/json; charset=utf-8");
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableSessionQueryTask task =
+ new JsonResumableSessionQueryTask(httpClientContext, uploadUrl);
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ assertThat(se.getCode()).isEqualTo(0);
+ }
+ }
+
+ @Test
+ public void query_badOffset() throws Exception {
+ HttpRequestHandler handler =
+ req -> {
+ // error message from GCS circa 2023-02
+ ByteBuf buf =
+ Unpooled.wrappedBuffer(
+ "Invalid request. According to the Content-Range header, the upload offset is 524288 byte(s), which exceeds already uploaded size of 262144 byte(s)."
+ .getBytes(StandardCharsets.UTF_8));
+ FullHttpResponse resp =
+ new DefaultFullHttpResponse(
+ req.protocolVersion(), APPEND_GREATER_THAN_CURRENT_SIZE, buf);
+ resp.headers().set(CONTENT_TYPE, "text/plain; charset=utf-8");
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableSessionQueryTask task =
+ new JsonResumableSessionQueryTask(httpClientContext, uploadUrl);
+
+ StorageException se = assertThrows(StorageException.class, task::call);
+ assertThat(se.getCode()).isEqualTo(0);
+ assertThat(se.getReason()).isEqualTo("dataLoss");
+ }
+ }
+}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITJsonResumableSessionTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITJsonResumableSessionTest.java
new file mode 100644
index 0000000000..7336749dbb
--- /dev/null
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITJsonResumableSessionTest.java
@@ -0,0 +1,270 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import static com.google.cloud.storage.ByteSizeConstants._256KiB;
+import static com.google.cloud.storage.ByteSizeConstants._256KiBL;
+import static com.google.cloud.storage.ByteSizeConstants._512KiB;
+import static com.google.cloud.storage.ByteSizeConstants._512KiBL;
+import static com.google.cloud.storage.ByteSizeConstants._768KiBL;
+import static com.google.common.truth.Truth.assertThat;
+import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpHeaderNames.CONTENT_RANGE;
+import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpHeaderNames.RANGE;
+import static io.grpc.netty.shaded.io.netty.handler.codec.http.HttpResponseStatus.SERVICE_UNAVAILABLE;
+
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.JsonObjectParser;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.core.ApiClock;
+import com.google.api.core.NanoClock;
+import com.google.api.gax.retrying.ResultRetryAlgorithm;
+import com.google.api.gax.retrying.RetrySettings;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.cloud.storage.FakeHttpServer.HttpRequestHandler;
+import com.google.cloud.storage.Retrying.RetryingDependencies;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.HttpRequest;
+import io.grpc.netty.shaded.io.netty.handler.codec.http.HttpResponseStatus;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public final class ITJsonResumableSessionTest {
+ private static final GsonFactory gson = GsonFactory.getDefaultInstance();
+ private static final NetHttpTransport transport = new NetHttpTransport.Builder().build();
+ private static final HttpResponseStatus RESUME_INCOMPLETE =
+ HttpResponseStatus.valueOf(308, "Resume Incomplete");
+ private static final RetryingDependencies RETRYING_DEPENDENCIES =
+ new RetryingDependencies() {
+ @Override
+ public RetrySettings getRetrySettings() {
+ return RetrySettings.newBuilder().setMaxAttempts(3).build();
+ }
+
+ @Override
+ public ApiClock getClock() {
+ return NanoClock.getDefaultClock();
+ }
+ };
+ private static final ResultRetryAlgorithm RETRY_ALGORITHM =
+ StorageRetryStrategy.getUniformStorageRetryStrategy().getIdempotentHandler();
+ private HttpClientContext httpClientContext;
+
+ @Rule public final TemporaryFolder temp = new TemporaryFolder();
+
+ @Before
+ public void setUp() throws Exception {
+ httpClientContext =
+ HttpClientContext.of(transport.createRequestFactory(), new JsonObjectParser(gson));
+ }
+
+ @Test
+ public void rewindWillQueryStatusOnlyWhenDirty() throws Exception {
+ HttpContentRange range1 = HttpContentRange.of(ByteRangeSpec.explicit(0L, _512KiBL));
+ HttpContentRange range2 = HttpContentRange.query();
+ HttpContentRange range3 = HttpContentRange.of(ByteRangeSpec.explicit(_256KiBL, _512KiBL));
+
+ final List requests = Collections.synchronizedList(new ArrayList<>());
+ HttpRequestHandler handler =
+ req -> {
+ requests.add(req);
+ String contentRange = req.headers().get(CONTENT_RANGE);
+ DefaultFullHttpResponse resp =
+ new DefaultFullHttpResponse(req.protocolVersion(), RESUME_INCOMPLETE);
+ if (range1.getHeaderValue().equals(contentRange)) {
+ resp.headers().set(RANGE, ByteRangeSpec.explicit(0L, _256KiBL).getHttpRangeHeader());
+ } else if (range2.getHeaderValue().equals(contentRange)) {
+ resp.headers().set(RANGE, ByteRangeSpec.explicit(0L, _256KiBL).getHttpRangeHeader());
+ } else {
+ resp.headers().set(RANGE, ByteRangeSpec.explicit(0L, _512KiBL).getHttpRangeHeader());
+ }
+ return resp;
+ };
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler);
+ TmpFile tmpFile =
+ DataGenerator.base64Characters().tempFile(temp.newFolder().toPath(), _512KiBL)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableWrite resumableWrite =
+ JsonResumableWrite.of(null, ImmutableMap.of(), uploadUrl, 0);
+ JsonResumableSession session =
+ new JsonResumableSession(
+ httpClientContext, RETRYING_DEPENDENCIES, RETRY_ALGORITHM, resumableWrite);
+
+ ResumableOperationResult<@Nullable StorageObject> operationResult =
+ session.put(RewindableContent.of(tmpFile.getPath()), range1);
+ StorageObject call = operationResult.getObject();
+ assertThat(call).isNull();
+ assertThat(operationResult.getPersistedSize()).isEqualTo(_512KiBL);
+ }
+
+ List actual =
+ requests.stream().map(r -> r.headers().get(CONTENT_RANGE)).collect(Collectors.toList());
+
+ List expected =
+ ImmutableList.of(range1.getHeaderValue(), range2.getHeaderValue(), range3.getHeaderValue());
+
+ assertThat(actual).isEqualTo(expected);
+ }
+
+ @Test
+ public void retryAttemptWillReturnQueryResultIfPersistedSizeMatchesSpecifiedEndOffset()
+ throws Exception {
+ HttpContentRange range1 = HttpContentRange.of(ByteRangeSpec.explicit(0L, _512KiBL));
+ HttpContentRange range2 = HttpContentRange.query();
+ HttpContentRange range3 = HttpContentRange.of(ByteRangeSpec.explicit(_512KiBL, _768KiBL));
+
+ final List requests = Collections.synchronizedList(new ArrayList<>());
+ HttpRequestHandler handler =
+ req -> {
+ requests.add(req);
+ String contentRange = req.headers().get(CONTENT_RANGE);
+ DefaultFullHttpResponse resp;
+ if (range1.getHeaderValue().equals(contentRange)) {
+ resp = new DefaultFullHttpResponse(req.protocolVersion(), SERVICE_UNAVAILABLE);
+ } else if (range2.getHeaderValue().equals(contentRange)) {
+ resp = new DefaultFullHttpResponse(req.protocolVersion(), RESUME_INCOMPLETE);
+ resp.headers().set(RANGE, ByteRangeSpec.explicit(0L, _512KiBL).getHttpRangeHeader());
+ } else {
+ resp = new DefaultFullHttpResponse(req.protocolVersion(), RESUME_INCOMPLETE);
+ resp.headers()
+ .set(RANGE, ByteRangeSpec.explicit(_512KiBL, _768KiBL).getHttpRangeHeader());
+ }
+ return resp;
+ };
+
+ ByteBuffer buf1 = DataGenerator.base64Characters().genByteBuffer(_512KiB);
+ ByteBuffer buf2 = DataGenerator.base64Characters().genByteBuffer(_256KiB);
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableWrite resumableWrite =
+ JsonResumableWrite.of(null, ImmutableMap.of(), uploadUrl, 0);
+ JsonResumableSession session =
+ new JsonResumableSession(
+ httpClientContext, RETRYING_DEPENDENCIES, RETRY_ALGORITHM, resumableWrite);
+
+ ResumableOperationResult<@Nullable StorageObject> operationResult1 =
+ session.put(RewindableContent.of(buf1), range1);
+ StorageObject call1 = operationResult1.getObject();
+ assertThat(call1).isNull();
+ assertThat(operationResult1.getPersistedSize()).isEqualTo(_512KiBL);
+
+ ResumableOperationResult<@Nullable StorageObject> operationResult2 =
+ session.put(RewindableContent.of(buf2), range3);
+ StorageObject call2 = operationResult2.getObject();
+ assertThat(call2).isNull();
+ assertThat(operationResult2.getPersistedSize()).isEqualTo(_768KiBL);
+ }
+
+ List actual =
+ requests.stream().map(r -> r.headers().get(CONTENT_RANGE)).collect(Collectors.toList());
+
+ List expected =
+ ImmutableList.of(range1.getHeaderValue(), range2.getHeaderValue(), range3.getHeaderValue());
+
+ assertThat(actual).isEqualTo(expected);
+ }
+
+ @Test
+ public void rewindOfContentIsRelativeToItsBeginOffsetOfTheOverallObject() throws Exception {
+ HttpContentRange range1 = HttpContentRange.of(ByteRangeSpec.explicit(0L, _512KiBL));
+ HttpContentRange range2 = HttpContentRange.of(ByteRangeSpec.explicit(_512KiBL, _768KiBL));
+ HttpContentRange range3 = HttpContentRange.query();
+
+ final AtomicBoolean fail = new AtomicBoolean(true);
+ final List requests = Collections.synchronizedList(new ArrayList<>());
+ HttpRequestHandler handler =
+ req -> {
+ requests.add(req);
+ String contentRange = req.headers().get(CONTENT_RANGE);
+ DefaultFullHttpResponse resp;
+ if (range1.getHeaderValue().equals(contentRange)
+ || range3.getHeaderValue().equals(contentRange)) {
+ resp = new DefaultFullHttpResponse(req.protocolVersion(), RESUME_INCOMPLETE);
+ resp.headers().set(RANGE, ByteRangeSpec.explicit(0L, _512KiBL).getHttpRangeHeader());
+ } else if (range2.getHeaderValue().equals(contentRange)) {
+ if (fail.getAndSet(false)) {
+ resp = new DefaultFullHttpResponse(req.protocolVersion(), SERVICE_UNAVAILABLE);
+ } else {
+ resp = new DefaultFullHttpResponse(req.protocolVersion(), RESUME_INCOMPLETE);
+ resp.headers()
+ .set(RANGE, ByteRangeSpec.explicit(_512KiBL, _768KiBL).getHttpRangeHeader());
+ }
+ } else {
+ resp = new DefaultFullHttpResponse(req.protocolVersion(), RESUME_INCOMPLETE);
+ resp.headers()
+ .set(RANGE, ByteRangeSpec.explicit(_512KiBL, _768KiBL).getHttpRangeHeader());
+ }
+ return resp;
+ };
+
+ ByteBuffer buf1 = DataGenerator.base64Characters().genByteBuffer(_512KiB);
+ ByteBuffer buf2 = DataGenerator.base64Characters().genByteBuffer(_256KiB);
+
+ try (FakeHttpServer fakeHttpServer = FakeHttpServer.of(handler)) {
+ URI endpoint = fakeHttpServer.getEndpoint();
+ String uploadUrl = String.format("%s/upload/%s", endpoint.toString(), UUID.randomUUID());
+
+ JsonResumableWrite resumableWrite =
+ JsonResumableWrite.of(null, ImmutableMap.of(), uploadUrl, 0);
+ JsonResumableSession session =
+ new JsonResumableSession(
+ httpClientContext, RETRYING_DEPENDENCIES, RETRY_ALGORITHM, resumableWrite);
+
+ ResumableOperationResult<@Nullable StorageObject> operationResult1 =
+ session.put(RewindableContent.of(buf1), range1);
+ StorageObject call1 = operationResult1.getObject();
+ assertThat(call1).isNull();
+ assertThat(operationResult1.getPersistedSize()).isEqualTo(_512KiBL);
+
+ ResumableOperationResult<@Nullable StorageObject> operationResult2 =
+ session.put(RewindableContent.of(buf2), range2);
+ StorageObject call2 = operationResult2.getObject();
+ assertThat(call2).isNull();
+ assertThat(operationResult2.getPersistedSize()).isEqualTo(_768KiBL);
+ }
+
+ List actual =
+ requests.stream().map(r -> r.headers().get(CONTENT_RANGE)).collect(Collectors.toList());
+
+ List expected =
+ ImmutableList.of(
+ range1.getHeaderValue(),
+ range2.getHeaderValue(),
+ range3.getHeaderValue(),
+ range2.getHeaderValue());
+
+ assertThat(actual).isEqualTo(expected);
+ }
+}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/JsonResumableSessionFailureScenarioTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/JsonResumableSessionFailureScenarioTest.java
new file mode 100644
index 0000000000..7f5c7c7ac7
--- /dev/null
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/JsonResumableSessionFailureScenarioTest.java
@@ -0,0 +1,155 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import static com.google.cloud.storage.JsonResumableSessionFailureScenario.isContinue;
+import static com.google.cloud.storage.JsonResumableSessionFailureScenario.isOk;
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.api.client.http.EmptyContent;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.testing.http.MockHttpTransport;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.common.collect.ImmutableMap;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import org.junit.Test;
+
+public final class JsonResumableSessionFailureScenarioTest {
+ private static final GsonFactory gson = GsonFactory.getDefaultInstance();
+
+ @Test
+ public void isOk_200() {
+ assertThat(isOk(200)).isTrue();
+ }
+
+ @Test
+ public void isOk_201() {
+ assertThat(isOk(201)).isTrue();
+ }
+
+ @Test
+ public void isContinue_308() {
+ assertThat(isContinue(308)).isTrue();
+ }
+
+ @Test
+ public void toStorageException_ioExceptionDuringContentResolutionAddedAsSuppressed()
+ throws IOException {
+ HttpRequest req =
+ new MockHttpTransport()
+ .createRequestFactory()
+ .buildPutRequest(new GenericUrl("http://localhost:80980"), new EmptyContent());
+ req.getHeaders().setContentLength(0L).setContentRange(HttpContentRange.of(0).getHeaderValue());
+
+ HttpResponse resp = req.execute();
+ resp.getHeaders().setContentType("text/plain; charset=utf-8").setContentLength(5L);
+
+ StorageException storageException =
+ JsonResumableSessionFailureScenario.SCENARIO_1.toStorageException(
+ "uploadId",
+ resp,
+ new Cause(),
+ () -> {
+ throw new Kaboom();
+ });
+
+ assertThat(storageException.getCode()).isEqualTo(0);
+ assertThat(storageException).hasCauseThat().isInstanceOf(Cause.class);
+ assertThat(storageException.getSuppressed()).isNotEmpty();
+ assertThat(storageException.getSuppressed()[0]).isInstanceOf(StorageException.class);
+ assertThat(storageException.getSuppressed()[0]).hasCauseThat().isInstanceOf(Kaboom.class);
+ }
+
+ @Test
+ public void multilineResponseBodyIsProperlyPrefixed() throws Exception {
+ StorageObject so = new StorageObject();
+ so.setName("object-name")
+ .setSize(BigInteger.ZERO)
+ .setGeneration(1L)
+ .setMetageneration(2L)
+ .setMetadata(
+ ImmutableMap.of(
+ "k1", "v1",
+ "k2", "v2"));
+ final String json = gson.toPrettyString(so);
+
+ byte[] bytes = json.getBytes(StandardCharsets.UTF_8);
+ HttpRequest req =
+ new MockHttpTransport()
+ .createRequestFactory()
+ .buildPutRequest(new GenericUrl("http://localhost:80980"), new EmptyContent());
+ req.getHeaders().setContentLength(0L);
+
+ HttpResponse resp = req.execute();
+ resp.getHeaders()
+ .setContentType("application/json; charset=utf-8")
+ .setContentLength((long) bytes.length);
+
+ StorageException storageException =
+ JsonResumableSessionFailureScenario.SCENARIO_0.toStorageException(
+ "uploadId", resp, null, () -> json);
+
+ assertThat(storageException.getCode()).isEqualTo(0);
+ assertThat(storageException).hasMessageThat().contains("\t|< \"generation\": \"1\",\n");
+ }
+
+ @Test
+ public void xGoogStoredHeadersIncludedIfPresent() throws IOException {
+ HttpRequest req =
+ new MockHttpTransport()
+ .createRequestFactory()
+ .buildPutRequest(new GenericUrl("http://localhost:80980"), new EmptyContent());
+ req.getHeaders().setContentLength(0L);
+
+ HttpResponse resp = req.execute();
+ resp.getHeaders()
+ .set("X-Goog-Stored-Content-Length", "5")
+ .set("x-goog-stored-content-encoding", "identity")
+ .set("X-GOOG-STORED-SOMETHING", "blah")
+ .setContentLength(0L);
+
+ StorageException storageException =
+ JsonResumableSessionFailureScenario.SCENARIO_0.toStorageException(
+ "uploadId", resp, null, () -> null);
+
+ assertThat(storageException.getCode()).isEqualTo(0);
+ assertThat(storageException).hasMessageThat().contains("|< x-goog-stored-content-length: 5");
+ assertThat(storageException)
+ .hasMessageThat()
+ .contains("|< x-goog-stored-content-encoding: identity");
+ assertThat(storageException).hasMessageThat().contains("|< x-goog-stored-something: blah");
+ }
+
+ private static final class Cause extends RuntimeException {
+
+ private Cause() {
+ super("Cause");
+ }
+ }
+
+ private static final class Kaboom extends IOException {
+
+ private Kaboom() {
+ super("Kaboom!!!");
+ }
+ }
+}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java
index da5f5310f1..d6c5ad0afc 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java
@@ -22,7 +22,6 @@
import com.google.cloud.WriteChannel;
import com.google.cloud.storage.BucketInfo.BuilderImpl;
import com.google.common.collect.ImmutableList;
-import com.google.storage.v2.WriteObjectResponse;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.function.Consumer;
@@ -62,14 +61,13 @@ public static Blob blobCopyWithStorage(Blob b, Storage s) {
public static Function> maybeGetBlobInfoFunction() {
return (w) -> {
- BlobWriteChannel blobWriteChannel;
- if (w instanceof BlobWriteChannel) {
- blobWriteChannel = (BlobWriteChannel) w;
- return Optional.of(blobWriteChannel.getStorageObject())
+ if (w instanceof BlobWriteChannelV2) {
+ BlobWriteChannelV2 blobWriteChannel = (BlobWriteChannelV2) w;
+ return Optional.ofNullable(blobWriteChannel.getResolvedObject())
.map(Conversions.apiary().blobInfo()::decode);
} else if (w instanceof GrpcBlobWriteChannel) {
GrpcBlobWriteChannel grpcBlobWriteChannel = (GrpcBlobWriteChannel) w;
- return Optional.of(grpcBlobWriteChannel.getResults())
+ return Optional.of(grpcBlobWriteChannel.getObject())
.map(
f -> {
try {
@@ -77,9 +75,7 @@ public static Function> maybeGetBlobInfoFunctio
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
- })
- .map(WriteObjectResponse::getResource)
- .map(Conversions.grpc().blobInfo()::decode);
+ });
} else {
return Optional.empty();
}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/RewindableByteBufferContentTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/RewindableByteBufferContentTest.java
new file mode 100644
index 0000000000..fe867e6f2f
--- /dev/null
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/RewindableByteBufferContentTest.java
@@ -0,0 +1,160 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import static com.google.cloud.storage.TestUtils.assertAll;
+import static com.google.cloud.storage.TestUtils.xxd;
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
+import com.google.cloud.storage.RewindableContentPropertyTest.ErroringOutputStream;
+import com.google.protobuf.ByteString;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.junit.Before;
+import org.junit.Test;
+
+public final class RewindableByteBufferContentTest {
+
+ private long total;
+ private ByteBuffer[] buffers;
+ private String fullXxd;
+
+ @Before
+ public void setUp() throws Exception {
+ // full buffer
+ ByteBuffer bufFull = DataGenerator.base64Characters().genByteBuffer(16);
+ // limited buffer
+ ByteBuffer bufLimit = DataGenerator.base64Characters().genByteBuffer(16);
+ bufLimit.limit(15);
+ // offset buffer
+ ByteBuffer bufOffset = DataGenerator.base64Characters().genByteBuffer(16);
+ bufOffset.position(3);
+ // offset and limited buffer
+ ByteBuffer bufLimitAndOffset = DataGenerator.base64Characters().genByteBuffer(16);
+ bufLimitAndOffset.position(9).limit(12);
+
+ total =
+ bufFull.remaining()
+ + bufLimit.remaining()
+ + bufOffset.remaining()
+ + bufLimitAndOffset.remaining();
+ buffers = new ByteBuffer[] {bufFull, bufLimit, bufOffset, bufLimitAndOffset};
+ fullXxd = xxd(false, buffers);
+ }
+
+ @Test
+ public void getLength() {
+ RewindableContent content = RewindableContent.of(buffers);
+
+ assertThat(content.getLength()).isEqualTo(total);
+ }
+
+ @Test
+ public void writeTo() throws IOException {
+
+ RewindableContent content = RewindableContent.of(buffers);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ content.writeTo(baos);
+
+ String actual = xxd(baos.toByteArray());
+ assertThat(actual).isEqualTo(fullXxd);
+ }
+
+ @Test
+ public void rewind() throws IOException {
+
+ RewindableContent content = RewindableContent.of(buffers);
+
+ assertThrows(
+ IOException.class,
+ () -> {
+ try (ErroringOutputStream erroringOutputStream = new ErroringOutputStream(25)) {
+ content.writeTo(erroringOutputStream);
+ }
+ });
+ content.rewindTo(0L);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ content.writeTo(baos);
+
+ String actual = xxd(baos.toByteArray());
+ assertThat(actual).isEqualTo(fullXxd);
+ }
+
+ @Test
+ public void rewindTo() throws Exception {
+ RewindableContent content = RewindableContent.of(buffers);
+
+ ByteString reduce =
+ Arrays.stream(buffers)
+ .map(ByteBuffer::duplicate)
+ .map(ByteStringStrategy.noCopy())
+ .reduce(ByteString.empty(), ByteString::concat, (l, r) -> r);
+
+ assertThat(content.getLength()).isEqualTo(total);
+
+ int readOffset = 37;
+ ByteString substring = reduce.substring(readOffset);
+ ByteBuffer readOnlyByteBuffer = substring.asReadOnlyByteBuffer();
+ String expected = xxd(false, readOnlyByteBuffer);
+ long value = total - readOffset;
+ content.rewindTo(readOffset);
+ assertThat(content.getLength()).isEqualTo(value);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ content.writeTo(baos);
+
+ String actual = xxd(baos.toByteArray());
+ assertAll(
+ () -> assertThat(baos.toByteArray()).hasLength(Math.toIntExact(value)),
+ () -> assertThat(actual).isEqualTo(expected));
+ }
+
+ @Test
+ public void rewind_dirtyAware() throws IOException {
+
+ ByteBuffer buf = DataGenerator.base64Characters().genByteBuffer(10);
+ buf.position(3).limit(7);
+
+ int position = buf.position();
+ int limit = buf.limit();
+
+ RewindableContent content = RewindableContent.of(buf);
+ int hackPosition = 2;
+ // after content has initialized, mutate the position underneath it. We're doing this to detect
+ // if rewind is actually modifying things. It shouldn't until the content is dirtied by calling
+ // writeTo
+ buf.position(hackPosition);
+
+ // invoke rewind, and expect it to not do anything
+ content.rewindTo(0L);
+ assertThat(buf.position()).isEqualTo(hackPosition);
+ assertThat(buf.limit()).isEqualTo(limit);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ content.writeTo(baos);
+
+ assertThat(buf.position()).isEqualTo(limit);
+ assertThat(buf.limit()).isEqualTo(limit);
+
+ content.rewindTo(0L);
+ assertThat(buf.position()).isEqualTo(position);
+ assertThat(buf.limit()).isEqualTo(limit);
+ }
+}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/RewindableContentPropertyTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/RewindableContentPropertyTest.java
new file mode 100644
index 0000000000..48d29bc8c8
--- /dev/null
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/RewindableContentPropertyTest.java
@@ -0,0 +1,363 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import static com.google.cloud.storage.TestUtils.xxd;
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
+import com.google.common.base.MoreObjects;
+import com.google.protobuf.ByteString;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+import net.jqwik.api.Arbitraries;
+import net.jqwik.api.Arbitrary;
+import net.jqwik.api.Combinators;
+import net.jqwik.api.ForAll;
+import net.jqwik.api.Property;
+import net.jqwik.api.Provide;
+import net.jqwik.api.RandomDistribution;
+import org.checkerframework.checker.nullness.qual.NonNull;
+
+final class RewindableContentPropertyTest {
+
+ @Property
+ void path(@ForAll("PathScenario") PathScenario pathScenario) throws Exception {
+ try (PathScenario s = pathScenario) {
+ RewindableContent content = RewindableContent.of(s.getPath());
+ assertThrows(
+ IOException.class,
+ () -> {
+ try (ErroringOutputStream erroringOutputStream =
+ new ErroringOutputStream(s.getErrorAtOffset())) {
+ content.writeTo(erroringOutputStream);
+ }
+ });
+ content.rewindTo(s.getRewindOffset());
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ content.writeTo(baos);
+
+ String actual = xxd(baos.toByteArray());
+
+ assertThat(actual).isEqualTo(s.getExpectedXxd());
+ }
+ }
+
+ @Property
+ void byteBuffers(@ForAll("ByteBuffersScenario") ByteBuffersScenario s) throws IOException {
+ RewindableContent content = RewindableContent.of(s.getBuffers());
+ assertThat(content.getLength()).isEqualTo(s.getFullLength());
+ assertThrows(
+ IOException.class,
+ () -> {
+ try (ErroringOutputStream erroringOutputStream =
+ new ErroringOutputStream(s.getErrorAtOffset())) {
+ content.writeTo(erroringOutputStream);
+ }
+ });
+ content.rewindTo(s.getRewindOffset());
+ assertThat(content.getLength()).isEqualTo(s.getPostRewindLength());
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ content.writeTo(baos);
+
+ String actual = xxd(baos.toByteArray());
+
+ assertThat(actual).isEqualTo(s.getExpectedXxd());
+ }
+
+ @Provide("PathScenario")
+ static Arbitrary pathScenario() {
+ return Arbitraries.lazyOf(
+ () ->
+ Arbitraries.oneOf(
+ bytes(1, 10),
+ bytes(10, 100),
+ bytes(100, 1_000),
+ bytes(1_000, 10_000),
+ bytes(10_000, 100_000),
+ bytes(100_000, 1_000_000),
+ bytes(1_000_000, 10_000_000))
+ .flatMap(
+ bytes ->
+ Combinators.combine(
+ Arbitraries.integers().between(0, bytes.length - 1),
+ Arbitraries.integers().between(0, bytes.length - 1),
+ Arbitraries.just(bytes))
+ .as(PathScenario::of)));
+ }
+
+ @Provide("ByteBuffersScenario")
+ static Arbitrary byteBuffersScenarioArbitrary() {
+ return Arbitraries.lazyOf(
+ () ->
+ Arbitraries.oneOf(
+ byteBuffers(1, 10),
+ byteBuffers(10, 100),
+ byteBuffers(100, 1_000),
+ byteBuffers(1_000, 10_000),
+ byteBuffers(10_000, 100_000),
+ byteBuffers(100_000, 1_000_000)))
+ .flatMap(
+ buffers -> {
+ long totalAvailable = Arrays.stream(buffers).mapToLong(ByteBuffer::remaining).sum();
+
+ return Combinators.combine(
+ Arbitraries.longs().between(0, Math.max(0L, totalAvailable - 1)),
+ Arbitraries.longs().between(0, Math.max(0L, totalAvailable - 1)),
+ Arbitraries.just(buffers))
+ .as(ByteBuffersScenario::of);
+ })
+ .filter(bbs -> bbs.getFullLength() > 0);
+ }
+
+ @NonNull
+ private static Arbitrary bytes(int minFileSize, int maxFileSize) {
+ return Arbitraries.integers()
+ .between(minFileSize, maxFileSize)
+ .withDistribution(RandomDistribution.uniform())
+ .map(DataGenerator.base64Characters()::genBytes);
+ }
+
+ @NonNull
+ private static Arbitrary byteBuffers(int perBufferMinSize, int perBufferMaxSize) {
+ return byteBuffer(perBufferMinSize, perBufferMaxSize)
+ .array(ByteBuffer[].class)
+ .ofMinSize(1)
+ .ofMaxSize(10);
+ }
+
+ /**
+ * Generate a ByteBuffer with size between minSize, maxSize with a random position and random
+ * limit
+ */
+ @NonNull
+ private static Arbitrary byteBuffer(int minSize, int maxSize) {
+ return Arbitraries.integers()
+ .between(minSize, maxSize)
+ .withDistribution(RandomDistribution.uniform())
+ .withoutEdgeCases()
+ .map(DataGenerator.base64Characters()::genByteBuffer)
+ .flatMap(
+ buf ->
+ Arbitraries.integers()
+ .between(0, Math.max(0, buf.capacity() - 1))
+ .withoutEdgeCases()
+ .flatMap(
+ limit ->
+ Arbitraries.integers()
+ .between(0, limit)
+ .withoutEdgeCases()
+ .flatMap(
+ position -> {
+ buf.limit(limit);
+ buf.position(position);
+ return Arbitraries.of(buf);
+ })));
+ }
+
+ private static final class PathScenario implements AutoCloseable {
+
+ private static final Path TMP_DIR = Paths.get(System.getProperty("java.io.tmpdir"));
+
+ private final int rewindOffset;
+ private final int errorAtOffset;
+ private final TmpFile tmpFile;
+ private final byte[] expectedBytes;
+ private final String expectedXxd;
+
+ private PathScenario(
+ int rewindOffset, int errorAtOffset, TmpFile tmpFile, byte[] expectedBytes) {
+ this.rewindOffset = rewindOffset;
+ this.errorAtOffset = errorAtOffset;
+ this.tmpFile = tmpFile;
+ this.expectedBytes = expectedBytes;
+ this.expectedXxd = xxd(expectedBytes);
+ }
+
+ public int getRewindOffset() {
+ return rewindOffset;
+ }
+
+ public int getErrorAtOffset() {
+ return errorAtOffset;
+ }
+
+ public Path getPath() {
+ return tmpFile.getPath();
+ }
+
+ public String getExpectedXxd() {
+ return expectedXxd;
+ }
+
+ public long getFullLength() throws IOException {
+ return Files.size(tmpFile.getPath());
+ }
+
+ @Override
+ public void close() throws IOException {
+ tmpFile.close();
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("expectedXxd", "\n" + expectedXxd)
+ .add("expectedBytes.length", expectedBytes.length)
+ .add("rewindOffset", rewindOffset)
+ .add("errorAtOffset", errorAtOffset)
+ .add("tmpFile", tmpFile)
+ .toString();
+ }
+
+ private static PathScenario of(int rewindOffset, int errorAtOffset, byte[] bytes) {
+ try {
+ TmpFile tmpFile1 = TmpFile.of(TMP_DIR, "PathScenario", ".bin");
+ try (SeekableByteChannel writer = tmpFile1.writer()) {
+ writer.write(ByteBuffer.wrap(bytes));
+ }
+ byte[] expectedBytes =
+ Arrays.copyOfRange(bytes, Math.min(rewindOffset, bytes.length), bytes.length);
+ return new PathScenario(rewindOffset, errorAtOffset, tmpFile1, expectedBytes);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static class ByteBuffersScenario {
+
+ private final long rewindOffset;
+ private final long errorAtOffset;
+ private final ByteBuffer[] buffers;
+ private final long fullLength;
+ private final String expectedXxd;
+
+ private ByteBuffersScenario(
+ long rewindOffset,
+ long errorAtOffset,
+ ByteBuffer[] buffers,
+ byte[] expectedBytes,
+ long fullLength) {
+ this.rewindOffset = rewindOffset;
+ this.errorAtOffset = errorAtOffset;
+ this.buffers = buffers;
+ this.fullLength = fullLength;
+ this.expectedXxd = xxd(expectedBytes);
+ }
+
+ public long getRewindOffset() {
+ return rewindOffset;
+ }
+
+ public long getErrorAtOffset() {
+ return errorAtOffset;
+ }
+
+ public ByteBuffer[] getBuffers() {
+ // duplicate the buffer so we have stable toString
+ return Arrays.stream(buffers).map(ByteBuffer::duplicate).toArray(ByteBuffer[]::new);
+ }
+
+ public String getExpectedXxd() {
+ return expectedXxd;
+ }
+
+ public long getFullLength() {
+ return fullLength;
+ }
+
+ public long getPostRewindLength() {
+ return fullLength - rewindOffset;
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("\nexpectedXxd", "\n" + expectedXxd)
+ .add(
+ "\nbuffers",
+ Arrays.stream(buffers)
+ .map(Object::toString)
+ .collect(Collectors.joining("\n\t", "[\n\t", "\n]")))
+ .add("\nrewindOffset", rewindOffset)
+ .add("\nerrorAtOffset", errorAtOffset)
+ .toString();
+ }
+
+ public static ByteBuffersScenario of(
+ long rewindOffset, long errorAtOffset, ByteBuffer[] buffers) {
+
+ ByteString reduce =
+ Arrays.stream(buffers)
+ .map(ByteBuffer::duplicate)
+ .map(ByteStringStrategy.noCopy())
+ .reduce(ByteString.empty(), ByteString::concat, (l, r) -> r);
+
+ byte[] byteArray = reduce.substring(Math.toIntExact(rewindOffset)).toByteArray();
+ return new ByteBuffersScenario(
+ rewindOffset, errorAtOffset, buffers, byteArray, reduce.size());
+ }
+ }
+
+ static final class ErroringOutputStream extends OutputStream {
+ private final long errorAt;
+ private long totalWritten;
+
+ ErroringOutputStream(long errorAt) {
+ this.errorAt = errorAt;
+ this.totalWritten = 0;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ if (totalWritten++ >= errorAt) {
+ throw new IOException("Reached errorAt limit");
+ }
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ if (totalWritten + b.length >= errorAt) {
+ throw new IOException("Reached errorAt limit");
+ } else {
+ totalWritten += b.length;
+ }
+ }
+
+ @Override
+ public void write(@SuppressWarnings("NullableProblems") byte[] b, int off, int len)
+ throws IOException {
+ int diff = len - off;
+ if (totalWritten + diff >= errorAt) {
+ throw new IOException("Reached errorAt limit");
+ } else {
+ totalWritten += diff;
+ }
+ }
+ }
+}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/SerializationTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/SerializationTest.java
index 9fc4b16f7c..87b88a78b8 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/SerializationTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/SerializationTest.java
@@ -20,7 +20,6 @@
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertEquals;
-import com.google.api.gax.retrying.ResultRetryAlgorithm;
import com.google.api.services.storage.model.StorageObject;
import com.google.cloud.BaseSerializationTest;
import com.google.cloud.NoCredentials;
@@ -28,10 +27,11 @@
import com.google.cloud.ReadChannel;
import com.google.cloud.Restorable;
import com.google.cloud.RestorableState;
+import com.google.cloud.WriteChannel;
import com.google.cloud.storage.Acl.Project.ProjectRole;
-import com.google.cloud.storage.BlobReadChannel.StateImpl;
import com.google.cloud.storage.BlobReadChannelV2.BlobReadChannelContext;
import com.google.cloud.storage.BlobReadChannelV2.BlobReadChannelV2State;
+import com.google.cloud.storage.BlobWriteChannelV2.BlobWriteChannelV2State;
import com.google.cloud.storage.Storage.BucketField;
import com.google.cloud.storage.Storage.PredefinedAcl;
import com.google.cloud.storage.UnifiedOpts.Opt;
@@ -205,16 +205,19 @@ protected Serializable[] serializableObjects() {
@SuppressWarnings("resource")
protected Restorable[] restorableObjects() {
HttpStorageOptions options = HttpStorageOptions.newBuilder().setProjectId("p2").build();
- ResultRetryAlgorithm algorithm =
- options.getRetryAlgorithmManager().getForResumableUploadSessionWrite(EMPTY_RPC_OPTIONS);
ReadChannel readerV2 =
new BlobReadChannelV2(
new StorageObject().setBucket("b").setName("n"),
EMPTY_RPC_OPTIONS,
BlobReadChannelContext.from(options));
- BlobWriteChannel writer =
- new BlobWriteChannel(
- options, BlobInfo.newBuilder(BlobId.of("b", "n")).build(), "upload-id", algorithm);
+ WriteChannel writer =
+ new BlobWriteChannelV2(
+ BlobReadChannelContext.from(options),
+ JsonResumableWrite.of(
+ Conversions.apiary().blobInfo().encode(BlobInfo.newBuilder("b", "n").build()),
+ ImmutableMap.of(),
+ "upload-id",
+ 0));
return new Restorable[] {readerV2, writer};
}
@@ -227,7 +230,7 @@ public void restoreOfV1BlobReadChannelShouldReturnV2Channel()
try (InputStream is =
SerializationTest.class
.getClassLoader()
- .getResourceAsStream("com/google/cloud/storage/blobWriteChannel.ser.properties")) {
+ .getResourceAsStream("com/google/cloud/storage/blobReadChannel.ser.properties")) {
properties.load(is);
}
String b64bytes = properties.getProperty("b64bytes");
@@ -239,8 +242,8 @@ public void restoreOfV1BlobReadChannelShouldReturnV2Channel()
Object o = ois.readObject();
assertThat(o).isInstanceOf(RestorableState.class);
RestorableState restorableState = (RestorableState) o;
- assertThat(o).isInstanceOf(StateImpl.class);
- StateImpl state = (StateImpl) restorableState;
+ assertThat(o).isInstanceOf(BlobReadChannel.StateImpl.class);
+ BlobReadChannel.StateImpl state = (BlobReadChannel.StateImpl) restorableState;
ReadChannel restore = state.restore();
assertThat(restore).isInstanceOf(BlobReadChannelV2.class);
RestorableState capture = restore.capture();
@@ -248,6 +251,36 @@ public void restoreOfV1BlobReadChannelShouldReturnV2Channel()
}
}
+ @SuppressWarnings({"deprecation", "rawtypes"})
+ @Test
+ public void restoreOfV1BlobWriteChannelShouldReturnV2Channel()
+ throws IOException, ClassNotFoundException {
+
+ Properties properties = new Properties();
+ try (InputStream is =
+ SerializationTest.class
+ .getClassLoader()
+ .getResourceAsStream("com/google/cloud/storage/blobWriteChannel.ser.properties")) {
+ properties.load(is);
+ }
+ String b64bytes = properties.getProperty("b64bytes");
+ assertThat(b64bytes).isNotEmpty();
+
+ byte[] decode = Base64.getDecoder().decode(b64bytes);
+ try (ByteArrayInputStream bais = new ByteArrayInputStream(decode);
+ ObjectInputStream ois = new ObjectInputStream(bais)) {
+ Object o = ois.readObject();
+ assertThat(o).isInstanceOf(RestorableState.class);
+ RestorableState restorableState = (RestorableState) o;
+ assertThat(o).isInstanceOf(BlobWriteChannel.StateImpl.class);
+ BlobWriteChannel.StateImpl state = (BlobWriteChannel.StateImpl) restorableState;
+ WriteChannel restore = state.restore();
+ assertThat(restore).isInstanceOf(BlobWriteChannelV2.class);
+ RestorableState capture = restore.capture();
+ assertThat(capture).isInstanceOf(BlobWriteChannelV2State.class);
+ }
+ }
+
/**
* Here we override the super classes implementation to remove the "assertNotSame".
*
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java
index ce750154e3..1d1453402d 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java
@@ -18,7 +18,6 @@
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
@@ -32,7 +31,6 @@
import com.google.api.services.storage.model.StorageObject;
import com.google.cloud.ServiceOptions;
import com.google.cloud.Tuple;
-import com.google.cloud.WriteChannel;
import com.google.cloud.storage.Storage.BlobTargetOption;
import com.google.cloud.storage.spi.StorageRpcFactory;
import com.google.cloud.storage.spi.v1.StorageRpc;
@@ -42,7 +40,6 @@
import com.google.common.io.BaseEncoding;
import java.io.ByteArrayInputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.math.BigInteger;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -983,141 +980,6 @@ private BlobInfo initializeUpload(
return blobInfo;
}
- @Test
- public void testCreateFromFile() throws Exception {
- byte[] dataToSend = {1, 2, 3, 4};
- Path tempFile = Files.createTempFile("testCreateFrom", ".tmp");
- Files.write(tempFile, dataToSend);
-
- BlobInfo blobInfo = initializeUpload(dataToSend);
- Blob blob = storage.createFrom(blobInfo, tempFile);
- assertEquals(expectedUpdated, blob);
- }
-
- @Test
- public void testCreateFromStream() throws Exception {
- byte[] dataToSend = {1, 2, 3, 4, 5};
- ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend);
-
- BlobInfo blobInfo = initializeUpload(dataToSend);
- Blob blob = storage.createFrom(blobInfo, stream);
- assertEquals(expectedUpdated, blob);
- }
-
- @Test
- public void testCreateFromWithOptions() throws Exception {
- byte[] dataToSend = {1, 2, 3, 4, 5, 6};
- ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend);
-
- BlobInfo blobInfo = initializeUpload(dataToSend, DEFAULT_BUFFER_SIZE, KMS_KEY_NAME_OPTIONS);
- Blob blob =
- storage.createFrom(blobInfo, stream, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME));
- assertEquals(expectedUpdated, blob);
- }
-
- @Test
- public void testCreateFromWithBufferSize() throws Exception {
- byte[] dataToSend = {1, 2, 3, 4, 5, 6};
- ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend);
- int bufferSize = MIN_BUFFER_SIZE * 2;
-
- BlobInfo blobInfo = initializeUpload(dataToSend, bufferSize);
- Blob blob = storage.createFrom(blobInfo, stream, bufferSize);
- assertEquals(expectedUpdated, blob);
- }
-
- @Test
- public void testCreateFromWithBufferSizeAndOptions() throws Exception {
- byte[] dataToSend = {1, 2, 3, 4, 5, 6};
- ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend);
- int bufferSize = MIN_BUFFER_SIZE * 2;
-
- BlobInfo blobInfo = initializeUpload(dataToSend, bufferSize, KMS_KEY_NAME_OPTIONS);
- Blob blob =
- storage.createFrom(
- blobInfo, stream, bufferSize, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME));
- assertEquals(expectedUpdated, blob);
- }
-
- @Test
- public void testCreateFromWithSmallBufferSize() throws Exception {
- byte[] dataToSend = new byte[100_000];
- ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend);
- int smallBufferSize = 100;
-
- BlobInfo blobInfo = initializeUpload(dataToSend, MIN_BUFFER_SIZE);
- Blob blob = storage.createFrom(blobInfo, stream, smallBufferSize);
- assertEquals(expectedUpdated, blob);
- }
-
- @Test
- public void testCreateFromWithException() throws Exception {
- initializeService();
- String uploadId = "id-exception";
- byte[] bytes = new byte[10];
- byte[] buffer = new byte[MIN_BUFFER_SIZE];
- System.arraycopy(bytes, 0, buffer, 0, bytes.length);
- BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build();
- doReturn(uploadId)
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .open(Conversions.apiary().blobInfo().encode(info), EMPTY_RPC_OPTIONS);
-
- Exception runtimeException = new RuntimeException("message");
- doThrow(runtimeException)
- .when(storageRpcMock)
- .writeWithResponse(uploadId, buffer, 0, 0L, bytes.length, true);
-
- InputStream input = new ByteArrayInputStream(bytes);
- try {
- storage.createFrom(info, input, MIN_BUFFER_SIZE);
- fail();
- } catch (StorageException e) {
- assertSame(runtimeException, e.getCause());
- }
- }
-
- @Test
- public void testCreateFromMultipleParts() throws Exception {
- initializeService();
- String uploadId = "id-multiple-parts";
- int extraBytes = 10;
- int totalSize = MIN_BUFFER_SIZE + extraBytes;
- byte[] dataToSend = new byte[totalSize];
- dataToSend[0] = 42;
- dataToSend[MIN_BUFFER_SIZE + 1] = 43;
-
- StorageObject storageObject = new StorageObject();
- storageObject.setBucket(BLOB_INFO1.getBucket());
- storageObject.setName(BLOB_INFO1.getName());
- storageObject.setSize(BigInteger.valueOf(totalSize));
-
- BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build();
- doReturn(uploadId)
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .open(Conversions.apiary().blobInfo().encode(info), EMPTY_RPC_OPTIONS);
-
- byte[] buffer1 = new byte[MIN_BUFFER_SIZE];
- System.arraycopy(dataToSend, 0, buffer1, 0, MIN_BUFFER_SIZE);
- doReturn(null)
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .writeWithResponse(uploadId, buffer1, 0, 0L, MIN_BUFFER_SIZE, false);
-
- byte[] buffer2 = new byte[MIN_BUFFER_SIZE];
- System.arraycopy(dataToSend, MIN_BUFFER_SIZE, buffer2, 0, extraBytes);
- doReturn(storageObject)
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .writeWithResponse(uploadId, buffer2, 0, (long) MIN_BUFFER_SIZE, extraBytes, true);
-
- InputStream input = new ByteArrayInputStream(dataToSend);
- Blob blob = storage.createFrom(info, input, MIN_BUFFER_SIZE);
- BlobInfo info1 = Conversions.apiary().blobInfo().decode(storageObject);
- assertEquals(info1.asBlob(storage), blob);
- }
-
@Test
public void testListBuckets() {
String cursor = "cursor";
@@ -1341,85 +1203,6 @@ public void testListBlobsWithException() {
}
}
- @Test
- public void testWriter() {
- // verify that md5 and crc32c are cleared if present when calling create
- doReturn("upload-id")
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .open(Conversions.apiary().blobInfo().encode(BLOB_INFO_WITHOUT_HASHES), EMPTY_RPC_OPTIONS);
- initializeService();
- WriteChannel channel = storage.writer(BLOB_INFO_WITH_HASHES);
- assertNotNull(channel);
- assertTrue(channel.isOpen());
- }
-
- @Test
- public void testWriterWithOptions() {
- BlobInfo info = BLOB_INFO1.toBuilder().setMd5(CONTENT_MD5).setCrc32c(CONTENT_CRC32C).build();
- doReturn("upload-id")
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .open(Conversions.apiary().blobInfo().encode(info), BLOB_TARGET_OPTIONS_CREATE);
- initializeService();
- WriteChannel channel =
- storage.writer(
- info,
- BLOB_WRITE_METAGENERATION,
- BLOB_WRITE_NOT_EXIST,
- BLOB_WRITE_PREDEFINED_ACL,
- BLOB_WRITE_CRC2C,
- BLOB_WRITE_MD5_HASH);
- assertNotNull(channel);
- assertTrue(channel.isOpen());
- }
-
- @Test
- public void testWriterWithEncryptionKey() {
- BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build();
- doReturn("upload-id-1", "upload-id-2")
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .open(Conversions.apiary().blobInfo().encode(info), ENCRYPTION_KEY_OPTIONS);
- initializeService();
- WriteChannel channel = storage.writer(info, Storage.BlobWriteOption.encryptionKey(KEY));
- assertNotNull(channel);
- assertTrue(channel.isOpen());
- channel = storage.writer(info, Storage.BlobWriteOption.encryptionKey(BASE64_KEY));
- assertNotNull(channel);
- assertTrue(channel.isOpen());
- }
-
- @Test
- public void testWriterWithKmsKeyName() {
- BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build();
- doReturn("upload-id-1", "upload-id-2")
- .doThrow(UNEXPECTED_CALL_EXCEPTION)
- .when(storageRpcMock)
- .open(Conversions.apiary().blobInfo().encode(info), KMS_KEY_NAME_OPTIONS);
- initializeService();
- WriteChannel channel = storage.writer(info, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME));
- assertNotNull(channel);
- assertTrue(channel.isOpen());
- channel = storage.writer(info, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME));
- assertNotNull(channel);
- assertTrue(channel.isOpen());
- }
-
- @Test
- public void testWriterFailure() {
- doThrow(STORAGE_FAILURE)
- .when(storageRpcMock)
- .open(Conversions.apiary().blobInfo().encode(BLOB_INFO_WITHOUT_HASHES), EMPTY_RPC_OPTIONS);
- initializeService();
- try {
- storage.writer(BLOB_INFO_WITH_HASHES);
- fail();
- } catch (StorageException e) {
- assertSame(STORAGE_FAILURE, e.getCause());
- }
- }
-
@Test
public void testCreateNotification() {
doReturn(Conversions.apiary().notificationInfo().encode(NOTIFICATION_INFO_01))
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java
index 875a3b0d85..af92080a7d 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/TestUtils.java
@@ -226,11 +226,27 @@ public static String xxd(byte[] bytes) {
}
public static String xxd(ByteBuffer bytes) {
+ return xxd(true, bytes);
+ }
+
+ public static String xxd(boolean flip, ByteBuffer bytes) {
ByteBuffer dup = bytes.duplicate();
- dup.flip();
+ if (flip) dup.flip();
return ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(dup));
}
+ public static String xxd(boolean flip, ByteBuffer[] buffers) {
+ ByteBuffer[] dups =
+ Arrays.stream(buffers)
+ .map(ByteBuffer::duplicate)
+ .peek(
+ byteBuffer -> {
+ if (flip) byteBuffer.flip();
+ })
+ .toArray(ByteBuffer[]::new);
+ return ByteBufUtil.prettyHexDump(Unpooled.wrappedBuffer(dups));
+ }
+
public static void assertAll(ThrowingRunnable... trs) throws Exception {
List x =
Arrays.stream(trs)
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/TmpFile.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/TmpFile.java
index a8c846c6c1..eef1b087d6 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/TmpFile.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/TmpFile.java
@@ -16,6 +16,7 @@
package com.google.cloud.storage;
+import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import java.io.IOException;
import java.nio.channels.SeekableByteChannel;
@@ -54,6 +55,11 @@ public void close() throws IOException {
Files.delete(path);
}
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this).add("path", path).toString();
+ }
+
/**
* Create a temporary file, which will be deleted when close is called on the returned {@link
* TmpFile}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/Functions.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/Functions.java
index 46d5643e0b..47ab7a8251 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/Functions.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/Functions.java
@@ -37,6 +37,10 @@ default CtxFunction andThen(CtxFunction f) {
return (Ctx ctx, TestRetryConformance trc) -> f.apply(apply(ctx, trc), trc);
}
+ default CtxFunction compose(CtxFunction f) {
+ return (Ctx ctx, TestRetryConformance trc) -> apply(f.apply(ctx, trc), trc);
+ }
+
static CtxFunction identity() {
return (ctx, c) -> ctx;
}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java
index 8aa55955a2..612879c4fc 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java
@@ -60,6 +60,7 @@
import java.util.Random;
import java.util.Set;
import java.util.function.BiPredicate;
+import java.util.function.Predicate;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import org.junit.After;
@@ -209,7 +210,7 @@ private static CtxFunction getReplaceStorageInObjectsFromCtx() {
* each defined scenario from google-cloud-conformance-tests and our defined {@link
* RpcMethodMappings}.
*/
- private static final class RetryTestCaseResolver {
+ static final class RetryTestCaseResolver {
private static final String HEX_SHUFFLE_SEED_OVERRIDE =
System.getProperty("HEX_SHUFFLE_SEED_OVERRIDE");
@@ -220,7 +221,7 @@ private static final class RetryTestCaseResolver {
private final String host;
private final String projectId;
- RetryTestCaseResolver(
+ private RetryTestCaseResolver(
String retryTestsJsonResourcePath,
RpcMethodMappings mappings,
BiPredicate testAllowFilter,
@@ -383,8 +384,12 @@ static BiPredicate specificMappings(int... mapp
return (m, c) -> set.contains(c.getMappingId());
}
- static BiPredicate instructionsAre(String... instructions) {
- return (m, trc) ->
+ static BiPredicate lift(Predicate p) {
+ return (m, trc) -> p.test(trc);
+ }
+
+ static Predicate instructionsAre(String... instructions) {
+ return trc ->
trc.getInstruction().getInstructionsList().equals(ImmutableList.copyOf(instructions));
}
@@ -392,6 +397,11 @@ static BiPredicate scenarioIdIs(int scenarioId)
return (m, trc) -> trc.getScenarioId() == scenarioId;
}
+ static BiPredicate mappingIdIn(Integer... mappingIds) {
+ ImmutableSet ids = ImmutableSet.copyOf(mappingIds);
+ return (m, trc) -> ids.contains(trc.getMappingId());
+ }
+
static final class Builder {
private String retryTestsJsonResourcePath;
private RpcMethodMappings mappings;
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java
index 67b8159b44..cc14b21e37 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java
@@ -24,6 +24,7 @@
import static com.google.common.base.Predicates.not;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeFalse;
import com.google.cloud.BaseServiceException;
import com.google.cloud.Binding;
@@ -54,6 +55,7 @@
import com.google.cloud.storage.conformance.retry.CtxFunctions.Local;
import com.google.cloud.storage.conformance.retry.CtxFunctions.ResourceSetup;
import com.google.cloud.storage.conformance.retry.CtxFunctions.Rpc;
+import com.google.cloud.storage.conformance.retry.Functions.CtxFunction;
import com.google.cloud.storage.conformance.retry.Functions.EConsumer;
import com.google.cloud.storage.conformance.retry.RpcMethod.storage.bucket_acl;
import com.google.cloud.storage.conformance.retry.RpcMethod.storage.buckets;
@@ -2099,4 +2101,12 @@ private static void put(ArrayList a) {}
private static Predicate methodGroupIs(String s) {
return (c) -> s.equals(c.getMethod().getGroup());
}
+
+ private static CtxFunction temporarilySkipMapping(
+ String message, java.util.function.Predicate p) {
+ return (ctx, trc) -> {
+ assumeFalse(message, p.test(trc));
+ return ctx;
+ };
+ }
}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobWriteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobWriteChannelTest.java
index e81f63e3c9..338f2d9be7 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobWriteChannelTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBlobWriteChannelTest.java
@@ -16,16 +16,19 @@
package com.google.cloud.storage.it;
+import static com.google.cloud.storage.TestUtils.xxd;
import static com.google.common.truth.Truth.assertThat;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import com.google.api.client.json.JsonParser;
import com.google.api.gax.rpc.FixedHeaderProvider;
import com.google.cloud.NoCredentials;
+import com.google.cloud.RestorableState;
import com.google.cloud.WriteChannel;
import com.google.cloud.conformance.storage.v1.InstructionList;
import com.google.cloud.conformance.storage.v1.Method;
@@ -38,26 +41,22 @@
import com.google.cloud.storage.PackagePrivateMethodWorkarounds;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.Storage.BlobWriteOption;
-import com.google.cloud.storage.StorageException;
import com.google.cloud.storage.StorageOptions;
+import com.google.cloud.storage.TransportCompatibility.Transport;
import com.google.cloud.storage.it.runner.StorageITRunner;
import com.google.cloud.storage.it.runner.annotations.Backend;
import com.google.cloud.storage.it.runner.annotations.Inject;
import com.google.cloud.storage.it.runner.annotations.SingleBackend;
+import com.google.cloud.storage.it.runner.annotations.StorageFixture;
import com.google.cloud.storage.it.runner.registry.Generator;
import com.google.cloud.storage.it.runner.registry.TestBench;
import com.google.cloud.storage.it.runner.registry.TestBench.RetryTestResource;
-import com.google.cloud.storage.spi.StorageRpcFactory;
-import com.google.cloud.storage.spi.v1.StorageRpc;
-import com.google.cloud.storage.spi.v1.StorageRpc.Option;
import com.google.common.collect.ImmutableMap;
-import com.google.common.reflect.AbstractInvocationHandler;
-import com.google.common.reflect.Reflection;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.Optional;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Logger;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -84,6 +83,11 @@ public final class ITBlobWriteChannelTest {
@Inject public TestBench testBench;
+ @Inject
+ @StorageFixture(Transport.HTTP)
+ public Storage storage;
+
+ @Inject public BucketInfo bucket;
@Inject public Generator generator;
/**
@@ -110,24 +114,6 @@ public void testJsonEOF_10B() throws IOException {
doJsonUnexpectedEOFTest(contentSize, cappedByteCount);
}
- @Test
- public void blobWriteChannel_handlesRecoveryOnLastChunkWhenGenerationIsPresent_multipleChunks()
- throws IOException {
- int _2MiB = 256 * 1024;
- int contentSize = 292_617;
-
- blobWriteChannel_handlesRecoveryOnLastChunkWhenGenerationIsPresent(_2MiB, contentSize);
- }
-
- @Test
- public void blobWriteChannel_handlesRecoveryOnLastChunkWhenGenerationIsPresent_singleChunk()
- throws IOException {
- int _4MiB = 256 * 1024 * 2;
- int contentSize = 292_617;
-
- blobWriteChannel_handlesRecoveryOnLastChunkWhenGenerationIsPresent(_4MiB, contentSize);
- }
-
@Test
public void testWriteChannelExistingBlob() throws IOException {
HttpStorageOptions baseStorageOptions =
@@ -156,6 +142,53 @@ public void testWriteChannelExistingBlob() throws IOException {
assertTrue(storage.delete(bucketInfo.getName(), blobInfo.getName()));
}
+ @Test
+ public void changeChunkSizeAfterWrite() throws IOException {
+ BlobInfo info = BlobInfo.newBuilder(bucket, generator.randomObjectName()).build();
+ System.out.println("info = " + info);
+
+ int _512KiB = 512 * 1024;
+ byte[] bytes = DataGenerator.base64Characters().genBytes(_512KiB + 13);
+ try (WriteChannel writer = storage.writer(info, BlobWriteOption.doesNotExist())) {
+ writer.setChunkSize(2 * 1024 * 1024);
+ writer.write(ByteBuffer.wrap(bytes, 0, _512KiB));
+ assertThrows(IllegalStateException.class, () -> writer.setChunkSize(768 * 1024));
+ }
+ }
+
+ @Test
+ public void restoreProperlyPlumbsBeginOffset() throws IOException {
+ BlobInfo info = BlobInfo.newBuilder(bucket, generator.randomObjectName()).build();
+ int _256KiB = 256 * 1024;
+
+ byte[] bytes1 = DataGenerator.base64Characters().genBytes(_256KiB);
+ byte[] bytes2 = DataGenerator.base64Characters().genBytes(73);
+
+ int allLength = bytes1.length + bytes2.length;
+ byte[] expected = Arrays.copyOf(bytes1, allLength);
+ System.arraycopy(bytes2, 0, expected, bytes1.length, bytes2.length);
+ String xxdExpected = xxd(expected);
+
+ RestorableState capture;
+ {
+ WriteChannel writer = storage.writer(info, BlobWriteOption.doesNotExist());
+ writer.setChunkSize(_256KiB);
+ writer.write(ByteBuffer.wrap(bytes1));
+ // explicitly do not close writer, it will finalize the session
+ capture = writer.capture();
+ }
+
+ assertThat(capture).isNotNull();
+ WriteChannel restored = capture.restore();
+ restored.write(ByteBuffer.wrap(bytes2));
+ restored.close();
+
+ byte[] readAllBytes = storage.readAllBytes(info.getBlobId());
+ assertThat(readAllBytes).hasLength(expected.length);
+ String xxdActual = xxd(readAllBytes);
+ assertThat(xxdActual).isEqualTo(xxdExpected);
+ }
+
private void doJsonUnexpectedEOFTest(int contentSize, int cappedByteCount) throws IOException {
String blobPath = String.format("%s/%s/blob", generator.randomObjectName(), NOW_STRING);
@@ -176,47 +209,11 @@ private void doJsonUnexpectedEOFTest(int contentSize, int cappedByteCount) throw
.setCredentials(NoCredentials.getInstance())
.setHost(testBench.getBaseUri())
.setProjectId("project-id")
- .build();
- StorageRpc noHeader = (StorageRpc) baseOptions.getRpc();
- StorageRpc yesHeader =
- (StorageRpc)
- baseOptions
- .toBuilder()
- .setHeaderProvider(
- FixedHeaderProvider.create(ImmutableMap.of("x-retry-test-id", retryTest.id)))
- .build()
- .getRpc();
-
- StorageOptions storageOptions =
- baseOptions
- .toBuilder()
- .setServiceRpcFactory(
- options ->
- Reflection.newProxy(
- StorageRpc.class,
- (proxy, method, args) -> {
- try {
- if ("writeWithResponse".equals(method.getName())) {
- boolean lastChunk = (boolean) args[5];
- LOGGER.fine(
- String.format(
- "writeWithResponse called. (lastChunk = %b)", lastChunk));
- if (lastChunk) {
- return method.invoke(yesHeader, args);
- }
- }
- return method.invoke(noHeader, args);
- } catch (Exception e) {
- if (e.getCause() != null) {
- throw e.getCause();
- } else {
- throw e;
- }
- }
- }))
+ .setHeaderProvider(
+ FixedHeaderProvider.create(ImmutableMap.of("x-retry-test-id", retryTest.id)))
.build();
- Storage testStorage = storageOptions.getService();
+ Storage testStorage = baseOptions.getService();
testStorage.create(bucketInfo);
@@ -234,7 +231,7 @@ private void doJsonUnexpectedEOFTest(int contentSize, int cappedByteCount) throw
Optional optionalStorageObject =
PackagePrivateMethodWorkarounds.maybeGetBlobInfoFunction().apply(w);
- assertTrue(optionalStorageObject.isPresent());
+ assertThat(optionalStorageObject.isPresent()).isTrue();
BlobInfo internalInfo = optionalStorageObject.get();
assertThat(internalInfo.getName()).isEqualTo(blobInfoGen0.getName());
@@ -248,100 +245,4 @@ private void doJsonUnexpectedEOFTest(int contentSize, int cappedByteCount) throw
ByteBuffer actual = ByteBuffer.wrap(actualData.toByteArray());
assertEquals(expected, actual);
}
-
- private void blobWriteChannel_handlesRecoveryOnLastChunkWhenGenerationIsPresent(
- int chunkSize, int contentSize) throws IOException {
- Instant now = Clock.systemUTC().instant();
- DateTimeFormatter formatter =
- DateTimeFormatter.ISO_LOCAL_DATE_TIME.withZone(ZoneId.from(ZoneOffset.UTC));
- String nowString = formatter.format(now);
- BucketInfo bucketInfo = BucketInfo.of(generator.randomBucketName());
- String blobPath = String.format("%s/%s/blob", generator.randomObjectName(), nowString);
- BlobId blobId = BlobId.of(bucketInfo.getName(), blobPath);
- BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
-
- ByteBuffer contentGen1 = DataGenerator.base64Characters().genByteBuffer(contentSize);
- ByteBuffer contentGen2 = DataGenerator.base64Characters().genByteBuffer(contentSize);
- ByteBuffer contentGen2Expected = contentGen2.duplicate();
- HttpStorageOptions baseStorageOptions =
- StorageOptions.http()
- .setCredentials(NoCredentials.getInstance())
- .setHost(testBench.getBaseUri())
- .setProjectId("test-project-id")
- .build();
- Storage storage = baseStorageOptions.getService();
- storage.create(bucketInfo);
- WriteChannel ww = storage.writer(blobInfo);
- ww.setChunkSize(chunkSize);
- ww.write(contentGen1);
- ww.close();
-
- Blob blobGen1 = storage.get(blobId);
-
- final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
-
- Storage testStorage =
- baseStorageOptions
- .toBuilder()
- .setServiceRpcFactory(
- new StorageRpcFactory() {
- /**
- * Here we're creating a proxy of StorageRpc where we can delegate all calls to
- * the normal implementation, except in the case of {@link
- * StorageRpc#writeWithResponse(String, byte[], int, long, int, boolean)} where
- * {@code lastChunk == true}. We allow the call to execute, but instead of
- * returning the result we throw an IOException to simulate a prematurely close
- * connection. This behavior is to ensure appropriate handling of a completed
- * upload where the ACK wasn't received. In particular, if an upload is initiated
- * against an object where an {@link Option#IF_GENERATION_MATCH} simply calling
- * get on an object can result in a 404 because the object that is created while
- * the BlobWriteChannel is executing will be a new generation.
- */
- @Override
- public StorageRpc create(final StorageOptions options) {
- return Reflection.newProxy(
- StorageRpc.class,
- new AbstractInvocationHandler() {
- final StorageRpc delegate = (StorageRpc) baseStorageOptions.getRpc();
-
- @Override
- protected Object handleInvocation(
- Object proxy, java.lang.reflect.Method method, Object[] args)
- throws Throwable {
- if ("writeWithResponse".equals(method.getName())) {
- Object result = method.invoke(delegate, args);
- boolean lastChunk = (boolean) args[5];
- // if we're on the lastChunk simulate a connection failure which
- // happens after the request was processed but before response could
- // be received by the client.
- if (lastChunk) {
- exceptionThrown.set(true);
- throw StorageException.translate(
- new IOException("simulated Connection closed prematurely"));
- } else {
- return result;
- }
- }
- return method.invoke(delegate, args);
- }
- });
- }
- })
- .build()
- .getService();
- try (WriteChannel w = testStorage.writer(blobGen1, BlobWriteOption.generationMatch())) {
- w.setChunkSize(chunkSize);
-
- w.write(contentGen2);
- }
-
- assertTrue("Expected an exception to be thrown for the last chunk", exceptionThrown.get());
-
- Blob blobGen2 = storage.get(blobId);
- assertEquals(contentSize, (long) blobGen2.getSize());
- assertNotEquals(blobInfo.getGeneration(), blobGen2.getGeneration());
- ByteArrayOutputStream actualData = new ByteArrayOutputStream();
- blobGen2.downloadTo(actualData);
- assertEquals(contentGen2Expected, ByteBuffer.wrap(actualData.toByteArray()));
- }
}
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITObjectChecksumSupportTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITObjectChecksumSupportTest.java
index 2bcb78e640..12bbf3df5d 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITObjectChecksumSupportTest.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITObjectChecksumSupportTest.java
@@ -28,6 +28,7 @@
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.Storage.BlobWriteOption;
import com.google.cloud.storage.StorageException;
+import com.google.cloud.storage.TmpFile;
import com.google.cloud.storage.TransportCompatibility.Transport;
import com.google.cloud.storage.it.ITObjectChecksumSupportTest.ChecksummedTestContentProvider;
import com.google.cloud.storage.it.runner.StorageITRunner;
@@ -42,8 +43,12 @@
import com.google.common.io.ByteStreams;
import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -54,6 +59,8 @@
@Parameterized(ChecksummedTestContentProvider.class)
public final class ITObjectChecksumSupportTest {
+ private static final Path tmpDir = Paths.get(System.getProperty("java.io.tmpdir"));
+
@Inject public Generator generator;
@Inject public Storage storage;
@@ -116,6 +123,50 @@ public void testCrc32cValidated_createFrom_expectSuccess() throws IOException {
assertThat(blob.getCrc32c()).isEqualTo(content.getCrc32cBase64());
}
+ @Test
+ public void testCrc32cValidated_createFrom_path_expectFailure() throws IOException {
+ String blobName = generator.randomObjectName();
+ BlobId blobId = BlobId.of(bucket.getName(), blobName);
+ BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setCrc32c(content.getCrc32cBase64()).build();
+
+ try (TmpFile tmpFile = TmpFile.of(tmpDir, "prefix", "bin")) {
+ try (SeekableByteChannel writer = tmpFile.writer()) {
+ writer.write(ByteBuffer.wrap(content.concat('x')));
+ }
+ StorageException expected =
+ assertThrows(
+ StorageException.class,
+ () ->
+ storage.createFrom(
+ blobInfo,
+ tmpFile.getPath(),
+ BlobWriteOption.doesNotExist(),
+ BlobWriteOption.crc32cMatch()));
+ assertThat(expected.getCode()).isEqualTo(400);
+ }
+ }
+
+ @Test
+ public void testCrc32cValidated_createFrom_path_expectSuccess() throws IOException {
+ String blobName = generator.randomObjectName();
+ BlobId blobId = BlobId.of(bucket.getName(), blobName);
+ BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setCrc32c(content.getCrc32cBase64()).build();
+
+ try (TmpFile tmpFile = TmpFile.of(tmpDir, "prefix", "bin")) {
+ try (SeekableByteChannel writer = tmpFile.writer()) {
+ writer.write(ByteBuffer.wrap(content.getBytes()));
+ }
+
+ Blob blob =
+ storage.createFrom(
+ blobInfo,
+ tmpFile.getPath(),
+ BlobWriteOption.doesNotExist(),
+ BlobWriteOption.crc32cMatch());
+ assertThat(blob.getCrc32c()).isEqualTo(content.getCrc32cBase64());
+ }
+ }
+
@Test
public void testCrc32cValidated_writer_expectFailure() {
String blobName = generator.randomObjectName();
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/TemporaryBucket.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/TemporaryBucket.java
index 0f6d255b3c..54ccfd92d9 100644
--- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/TemporaryBucket.java
+++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/TemporaryBucket.java
@@ -25,7 +25,7 @@
import com.google.common.base.Preconditions;
import java.time.Duration;
-final class TemporaryBucket implements AutoCloseable {
+public final class TemporaryBucket implements AutoCloseable {
private final BucketInfo bucket;
private final Storage storage;
@@ -44,7 +44,7 @@ private TemporaryBucket(
}
/** Return the BucketInfo from the created temporary bucket. */
- BucketInfo getBucket() {
+ public BucketInfo getBucket() {
return bucket;
}
@@ -55,11 +55,11 @@ public void close() throws Exception {
}
}
- static Builder newBuilder() {
+ public static Builder newBuilder() {
return new Builder();
}
- static final class Builder {
+ public static final class Builder {
private CleanupStrategy cleanupStrategy;
private Duration cleanupTimeoutDuration;
@@ -71,27 +71,27 @@ private Builder() {
this.cleanupTimeoutDuration = Duration.ofMinutes(1);
}
- Builder setCleanupStrategy(CleanupStrategy cleanupStrategy) {
+ public Builder setCleanupStrategy(CleanupStrategy cleanupStrategy) {
this.cleanupStrategy = cleanupStrategy;
return this;
}
- Builder setCleanupTimeoutDuration(Duration cleanupTimeoutDuration) {
+ public Builder setCleanupTimeoutDuration(Duration cleanupTimeoutDuration) {
this.cleanupTimeoutDuration = cleanupTimeoutDuration;
return this;
}
- Builder setBucketInfo(BucketInfo bucketInfo) {
+ public Builder setBucketInfo(BucketInfo bucketInfo) {
this.bucketInfo = bucketInfo;
return this;
}
- Builder setStorage(Storage storage) {
+ public Builder setStorage(Storage storage) {
this.storage = storage;
return this;
}
- TemporaryBucket build() {
+ public TemporaryBucket build() {
Preconditions.checkArgument(
cleanupStrategy != CleanupStrategy.ONLY_ON_SUCCESS, "Unable to detect success.");
Storage s = requireNonNull(storage, "storage must be non null");
diff --git a/google-cloud-storage/src/test/resources/com/google/cloud/storage/blobReadChannel.ser.properties b/google-cloud-storage/src/test/resources/com/google/cloud/storage/blobReadChannel.ser.properties
new file mode 100644
index 0000000000..c9d3dc5ff8
--- /dev/null
+++ b/google-cloud-storage/src/test/resources/com/google/cloud/storage/blobReadChannel.ser.properties
@@ -0,0 +1,70 @@
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Base 64 Encoded bytes of a BlobReadChannel circa v2.16.0
+# Generated using the following snippet:
+#
+# Storage s = StorageOptions.http()
+# .setProjectId("proj")
+# .setCredentials(NoCredentials.getInstance())
+# .build()
+# .getService();
+#
+# ReadChannel reader = s.reader(BlobId.of("buck", "obj", 1L));
+# RestorableState capture = reader.capture();
+#
+# ByteArrayOutputStream baos = new ByteArrayOutputStream();
+# try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+# oos.writeObject(capture);
+# }
+#
+# byte[] bytes = baos.toByteArray();
+# String b64Ser = Base64.getEncoder().encodeToString(bytes);
+#
+# System.out.println("b64Ser = " + b64Ser);
+#
+b64bytes=\
+ rO0ABXNyADJjb20uZ29vZ2xlLmNsb3VkLnN0b3JhZ2UuQmxvYlJlYWRDaGFubmVsJFN0YXRlSW1wbGwJWjOFWbi1AgAJSQAJY2h1bmtTaXplWgALZW5kT2ZTdHJlYW1a\
+ AAZpc09wZW5KAAVsaW1pdEoACHBvc2l0aW9uTAAEYmxvYnQAIUxjb20vZ29vZ2xlL2Nsb3VkL3N0b3JhZ2UvQmxvYklkO0wACGxhc3RFdGFndAASTGphdmEvbGFuZy9T\
+ dHJpbmc7TAAOcmVxdWVzdE9wdGlvbnN0AA9MamF2YS91dGlsL01hcDtMAA5zZXJ2aWNlT3B0aW9uc3QALUxjb20vZ29vZ2xlL2Nsb3VkL3N0b3JhZ2UvSHR0cFN0b3Jh\
+ Z2VPcHRpb25zO3hwACAAAAABf/////////8AAAAAAAAAAHNyAB9jb20uZ29vZ2xlLmNsb3VkLnN0b3JhZ2UuQmxvYklkcdHeVjWP2d0CAANMAAZidWNrZXRxAH4AAkwA\
+ CmdlbmVyYXRpb250ABBMamF2YS9sYW5nL0xvbmc7TAAEbmFtZXEAfgACeHB0AARidWNrc3IADmphdmEubGFuZy5Mb25nO4vkkMyPI98CAAFKAAV2YWx1ZXhyABBqYXZh\
+ LmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAAAAAABdAADb2JqcHNyADVjb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9y\
+ bQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAPeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAA\
+ dXEAfgARAAAAAHNyACtjb20uZ29vZ2xlLmNsb3VkLnN0b3JhZ2UuSHR0cFN0b3JhZ2VPcHRpb25ztmk+4Fw7cvMCAAFMABVyZXRyeUFsZ29yaXRobU1hbmFnZXJ0ADRM\
+ Y29tL2dvb2dsZS9jbG91ZC9zdG9yYWdlL0h0dHBSZXRyeUFsZ29yaXRobU1hbmFnZXI7eHIAJ2NvbS5nb29nbGUuY2xvdWQuc3RvcmFnZS5TdG9yYWdlT3B0aW9uc5q/\
+ 8jOW5d5PAgAAeHIAH2NvbS5nb29nbGUuY2xvdWQuU2VydmljZU9wdGlvbnN/qQsz9VFyfgIAC0wADmNsaWVudExpYlRva2VucQB+AAJMAAVjbG9ja3QAHkxjb20vZ29v\
+ Z2xlL2FwaS9jb3JlL0FwaUNsb2NrO0wAC2NyZWRlbnRpYWxzdAAdTGNvbS9nb29nbGUvYXV0aC9DcmVkZW50aWFscztMAA5oZWFkZXJQcm92aWRlcnQAJ0xjb20vZ29v\
+ Z2xlL2FwaS9nYXgvcnBjL0hlYWRlclByb3ZpZGVyO0wABGhvc3RxAH4AAkwACXByb2plY3RJZHEAfgACTAAOcXVvdGFQcm9qZWN0SWRxAH4AAkwADXJldHJ5U2V0dGlu\
+ Z3N0ACtMY29tL2dvb2dsZS9hcGkvZ2F4L3JldHJ5aW5nL1JldHJ5U2V0dGluZ3M7TAAXc2VydmljZUZhY3RvcnlDbGFzc05hbWVxAH4AAkwAGnNlcnZpY2VScGNGYWN0\
+ b3J5Q2xhc3NOYW1lcQB+AAJMABB0cmFuc3BvcnRPcHRpb25zdAAjTGNvbS9nb29nbGUvY2xvdWQvVHJhbnNwb3J0T3B0aW9uczt4cHQABGdjY2xzcgAmY29tLmdvb2ds\
+ ZS5hcGkuY29yZS5DdXJyZW50TWlsbGlzQ2xvY2usd0sHJ9YTCwIAAHhwc3IAHmNvbS5nb29nbGUuY2xvdWQuTm9DcmVkZW50aWFsc6kR5wOeLAxAAgAAeHIAKGNvbS5n\
+ b29nbGUuYXV0aC5vYXV0aDIuT0F1dGgyQ3JlZGVudGlhbHM/PX166aVRVwIABEwAEGV4cGlyYXRpb25NYXJnaW50ABRMamF2YS90aW1lL0R1cmF0aW9uO0wABGxvY2tx\
+ AH4AD0wADXJlZnJlc2hNYXJnaW5xAH4AI0wABXZhbHVldAA1TGNvbS9nb29nbGUvYXV0aC9vYXV0aDIvT0F1dGgyQ3JlZGVudGlhbHMkT0F1dGhWYWx1ZTt4cgAbY29t\
+ Lmdvb2dsZS5hdXRoLkNyZWRlbnRpYWxzCzii14w9kIECAAB4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcNAQAAAAAAAAEsAAAAAHh1cgACW0Ks8xf4BghU\
+ 4AIAAHhwAAAAAHNxAH4AJ3cNAQAAAAAAAAFoAAAAAHhwc3IAJ2NvbS5nb29nbGUuYXBpLmdheC5ycGMuTm9IZWFkZXJQcm92aWRlcmWjEqhqxXthAgAAeHB0AB5odHRw\
+ czovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb210AARwcm9qcHNyADNjb20uZ29vZ2xlLmFwaS5nYXgucmV0cnlpbmcuQXV0b1ZhbHVlX1JldHJ5U2V0dGluZ3Nym/9/a0d0\
+ swIACVoACGppdHRlcmVkSQALbWF4QXR0ZW1wdHNEABRyZXRyeURlbGF5TXVsdGlwbGllckQAFHJwY1RpbWVvdXRNdWx0aXBsaWVyTAARaW5pdGlhbFJldHJ5RGVsYXl0\
+ ABpMb3JnL3RocmVldGVuL2JwL0R1cmF0aW9uO0wAEWluaXRpYWxScGNUaW1lb3V0cQB+ADFMAA1tYXhSZXRyeURlbGF5cQB+ADFMAA1tYXhScGNUaW1lb3V0cQB+ADFM\
+ AAx0b3RhbFRpbWVvdXRxAH4AMXhyACljb20uZ29vZ2xlLmFwaS5nYXgucmV0cnlpbmcuUmV0cnlTZXR0aW5nc3Kb/39rR3SzAgAAeHABAAAABkAAAAAAAAAAP/AAAAAA\
+ AABzcgATb3JnLnRocmVldGVuLmJwLlNlcpVdhLobIkiyDAAAeHB3DQEAAAAAAAAAAQAAAAB4c3EAfgA0dw0BAAAAAAAAADIAAAAAeHNxAH4ANHcNAQAAAAAAAAAgAAAA\
+ AHhzcQB+ADR3DQEAAAAAAAAAMgAAAAB4c3EAfgA0dw0BAAAAAAAAADIAAAAAeHQAPmNvbS5nb29nbGUuY2xvdWQuc3RvcmFnZS5IdHRwU3RvcmFnZU9wdGlvbnMkSHR0\
+ cFN0b3JhZ2VGYWN0b3J5dABBY29tLmdvb2dsZS5jbG91ZC5zdG9yYWdlLkh0dHBTdG9yYWdlT3B0aW9ucyRIdHRwU3RvcmFnZVJwY0ZhY3RvcnlzcgAqY29tLmdvb2ds\
+ ZS5jbG91ZC5odHRwLkh0dHBUcmFuc3BvcnRPcHRpb25zbX9UTb2H/yICAANJAA5jb25uZWN0VGltZW91dEkAC3JlYWRUaW1lb3V0TAAdaHR0cFRyYW5zcG9ydEZhY3Rv\
+ cnlDbGFzc05hbWVxAH4AAnhw//////////90AEZjb20uZ29vZ2xlLmNsb3VkLmh0dHAuSHR0cFRyYW5zcG9ydE9wdGlvbnMkRGVmYXVsdEh0dHBUcmFuc3BvcnRGYWN0\
+ b3J5c3IAMmNvbS5nb29nbGUuY2xvdWQuc3RvcmFnZS5IdHRwUmV0cnlBbGdvcml0aG1NYW5hZ2Vy0i1ymVA0mEUCAAFMAA1yZXRyeVN0cmF0ZWd5dAAvTGNvbS9nb29n\
+ bGUvY2xvdWQvc3RvcmFnZS9TdG9yYWdlUmV0cnlTdHJhdGVneTt4cHNyADRjb20uZ29vZ2xlLmNsb3VkLnN0b3JhZ2UuRGVmYXVsdFN0b3JhZ2VSZXRyeVN0cmF0ZWd5\
+ bgaLnarjlYkCAAB4cA==
diff --git a/google-cloud-storage/src/test/resources/com/google/cloud/storage/blobWriteChannel.ser.properties b/google-cloud-storage/src/test/resources/com/google/cloud/storage/blobWriteChannel.ser.properties
index c9d3dc5ff8..d60cb17499 100644
--- a/google-cloud-storage/src/test/resources/com/google/cloud/storage/blobWriteChannel.ser.properties
+++ b/google-cloud-storage/src/test/resources/com/google/cloud/storage/blobWriteChannel.ser.properties
@@ -1,5 +1,5 @@
#
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -23,8 +23,8 @@
# .build()
# .getService();
#
-# ReadChannel reader = s.reader(BlobId.of("buck", "obj", 1L));
-# RestorableState capture = reader.capture();
+# WriteChannel reader = s.writer(BlobInfo.newBuilder("buck", "obj").build(), BlobWriteOption.doesNotExist());
+# RestorableState capture = reader.capture();
#
# ByteArrayOutputStream baos = new ByteArrayOutputStream();
# try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
@@ -37,34 +37,44 @@
# System.out.println("b64Ser = " + b64Ser);
#
b64bytes=\
- rO0ABXNyADJjb20uZ29vZ2xlLmNsb3VkLnN0b3JhZ2UuQmxvYlJlYWRDaGFubmVsJFN0YXRlSW1wbGwJWjOFWbi1AgAJSQAJY2h1bmtTaXplWgALZW5kT2ZTdHJlYW1a\
- AAZpc09wZW5KAAVsaW1pdEoACHBvc2l0aW9uTAAEYmxvYnQAIUxjb20vZ29vZ2xlL2Nsb3VkL3N0b3JhZ2UvQmxvYklkO0wACGxhc3RFdGFndAASTGphdmEvbGFuZy9T\
- dHJpbmc7TAAOcmVxdWVzdE9wdGlvbnN0AA9MamF2YS91dGlsL01hcDtMAA5zZXJ2aWNlT3B0aW9uc3QALUxjb20vZ29vZ2xlL2Nsb3VkL3N0b3JhZ2UvSHR0cFN0b3Jh\
- Z2VPcHRpb25zO3hwACAAAAABf/////////8AAAAAAAAAAHNyAB9jb20uZ29vZ2xlLmNsb3VkLnN0b3JhZ2UuQmxvYklkcdHeVjWP2d0CAANMAAZidWNrZXRxAH4AAkwA\
- CmdlbmVyYXRpb250ABBMamF2YS9sYW5nL0xvbmc7TAAEbmFtZXEAfgACeHB0AARidWNrc3IADmphdmEubGFuZy5Mb25nO4vkkMyPI98CAAFKAAV2YWx1ZXhyABBqYXZh\
- LmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAAAAAAABdAADb2JqcHNyADVjb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9y\
- bQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAPeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAA\
- dXEAfgARAAAAAHNyACtjb20uZ29vZ2xlLmNsb3VkLnN0b3JhZ2UuSHR0cFN0b3JhZ2VPcHRpb25ztmk+4Fw7cvMCAAFMABVyZXRyeUFsZ29yaXRobU1hbmFnZXJ0ADRM\
- Y29tL2dvb2dsZS9jbG91ZC9zdG9yYWdlL0h0dHBSZXRyeUFsZ29yaXRobU1hbmFnZXI7eHIAJ2NvbS5nb29nbGUuY2xvdWQuc3RvcmFnZS5TdG9yYWdlT3B0aW9uc5q/\
- 8jOW5d5PAgAAeHIAH2NvbS5nb29nbGUuY2xvdWQuU2VydmljZU9wdGlvbnN/qQsz9VFyfgIAC0wADmNsaWVudExpYlRva2VucQB+AAJMAAVjbG9ja3QAHkxjb20vZ29v\
- Z2xlL2FwaS9jb3JlL0FwaUNsb2NrO0wAC2NyZWRlbnRpYWxzdAAdTGNvbS9nb29nbGUvYXV0aC9DcmVkZW50aWFscztMAA5oZWFkZXJQcm92aWRlcnQAJ0xjb20vZ29v\
- Z2xlL2FwaS9nYXgvcnBjL0hlYWRlclByb3ZpZGVyO0wABGhvc3RxAH4AAkwACXByb2plY3RJZHEAfgACTAAOcXVvdGFQcm9qZWN0SWRxAH4AAkwADXJldHJ5U2V0dGlu\
- Z3N0ACtMY29tL2dvb2dsZS9hcGkvZ2F4L3JldHJ5aW5nL1JldHJ5U2V0dGluZ3M7TAAXc2VydmljZUZhY3RvcnlDbGFzc05hbWVxAH4AAkwAGnNlcnZpY2VScGNGYWN0\
- b3J5Q2xhc3NOYW1lcQB+AAJMABB0cmFuc3BvcnRPcHRpb25zdAAjTGNvbS9nb29nbGUvY2xvdWQvVHJhbnNwb3J0T3B0aW9uczt4cHQABGdjY2xzcgAmY29tLmdvb2ds\
- ZS5hcGkuY29yZS5DdXJyZW50TWlsbGlzQ2xvY2usd0sHJ9YTCwIAAHhwc3IAHmNvbS5nb29nbGUuY2xvdWQuTm9DcmVkZW50aWFsc6kR5wOeLAxAAgAAeHIAKGNvbS5n\
- b29nbGUuYXV0aC5vYXV0aDIuT0F1dGgyQ3JlZGVudGlhbHM/PX166aVRVwIABEwAEGV4cGlyYXRpb25NYXJnaW50ABRMamF2YS90aW1lL0R1cmF0aW9uO0wABGxvY2tx\
- AH4AD0wADXJlZnJlc2hNYXJnaW5xAH4AI0wABXZhbHVldAA1TGNvbS9nb29nbGUvYXV0aC9vYXV0aDIvT0F1dGgyQ3JlZGVudGlhbHMkT0F1dGhWYWx1ZTt4cgAbY29t\
- Lmdvb2dsZS5hdXRoLkNyZWRlbnRpYWxzCzii14w9kIECAAB4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcNAQAAAAAAAAEsAAAAAHh1cgACW0Ks8xf4BghU\
- 4AIAAHhwAAAAAHNxAH4AJ3cNAQAAAAAAAAFoAAAAAHhwc3IAJ2NvbS5nb29nbGUuYXBpLmdheC5ycGMuTm9IZWFkZXJQcm92aWRlcmWjEqhqxXthAgAAeHB0AB5odHRw\
- czovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb210AARwcm9qcHNyADNjb20uZ29vZ2xlLmFwaS5nYXgucmV0cnlpbmcuQXV0b1ZhbHVlX1JldHJ5U2V0dGluZ3Nym/9/a0d0\
- swIACVoACGppdHRlcmVkSQALbWF4QXR0ZW1wdHNEABRyZXRyeURlbGF5TXVsdGlwbGllckQAFHJwY1RpbWVvdXRNdWx0aXBsaWVyTAARaW5pdGlhbFJldHJ5RGVsYXl0\
- ABpMb3JnL3RocmVldGVuL2JwL0R1cmF0aW9uO0wAEWluaXRpYWxScGNUaW1lb3V0cQB+ADFMAA1tYXhSZXRyeURlbGF5cQB+ADFMAA1tYXhScGNUaW1lb3V0cQB+ADFM\
- AAx0b3RhbFRpbWVvdXRxAH4AMXhyACljb20uZ29vZ2xlLmFwaS5nYXgucmV0cnlpbmcuUmV0cnlTZXR0aW5nc3Kb/39rR3SzAgAAeHABAAAABkAAAAAAAAAAP/AAAAAA\
- AABzcgATb3JnLnRocmVldGVuLmJwLlNlcpVdhLobIkiyDAAAeHB3DQEAAAAAAAAAAQAAAAB4c3EAfgA0dw0BAAAAAAAAADIAAAAAeHNxAH4ANHcNAQAAAAAAAAAgAAAA\
- AHhzcQB+ADR3DQEAAAAAAAAAMgAAAAB4c3EAfgA0dw0BAAAAAAAAADIAAAAAeHQAPmNvbS5nb29nbGUuY2xvdWQuc3RvcmFnZS5IdHRwU3RvcmFnZU9wdGlvbnMkSHR0\
- cFN0b3JhZ2VGYWN0b3J5dABBY29tLmdvb2dsZS5jbG91ZC5zdG9yYWdlLkh0dHBTdG9yYWdlT3B0aW9ucyRIdHRwU3RvcmFnZVJwY0ZhY3RvcnlzcgAqY29tLmdvb2ds\
- ZS5jbG91ZC5odHRwLkh0dHBUcmFuc3BvcnRPcHRpb25zbX9UTb2H/yICAANJAA5jb25uZWN0VGltZW91dEkAC3JlYWRUaW1lb3V0TAAdaHR0cFRyYW5zcG9ydEZhY3Rv\
- cnlDbGFzc05hbWVxAH4AAnhw//////////90AEZjb20uZ29vZ2xlLmNsb3VkLmh0dHAuSHR0cFRyYW5zcG9ydE9wdGlvbnMkRGVmYXVsdEh0dHBUcmFuc3BvcnRGYWN0\
- b3J5c3IAMmNvbS5nb29nbGUuY2xvdWQuc3RvcmFnZS5IdHRwUmV0cnlBbGdvcml0aG1NYW5hZ2Vy0i1ymVA0mEUCAAFMAA1yZXRyeVN0cmF0ZWd5dAAvTGNvbS9nb29n\
- bGUvY2xvdWQvc3RvcmFnZS9TdG9yYWdlUmV0cnlTdHJhdGVneTt4cHNyADRjb20uZ29vZ2xlLmNsb3VkLnN0b3JhZ2UuRGVmYXVsdFN0b3JhZ2VSZXRyeVN0cmF0ZWd5\
- bgaLnarjlYkCAAB4cA==
+ rO0ABXNyADNjb20uZ29vZ2xlLmNsb3VkLnN0b3JhZ2UuQmxvYldyaXRlQ2hhbm5lbCRTdGF0ZUltcGyjA3jVYuVZZQIAAUwAEWFsZ29yaXRobUZvcldyaXRldAAyTGNv\
+ bS9nb29nbGUvYXBpL2dheC9yZXRyeWluZy9SZXN1bHRSZXRyeUFsZ29yaXRobTt4cgArY29tLmdvb2dsZS5jbG91ZC5CYXNlV3JpdGVDaGFubmVsJEJhc2VTdGF0ZXaH\
+ 8w86CHBzAgAHSQAJY2h1bmtTaXplWgAGaXNPcGVuSgAIcG9zaXRpb25bAAZidWZmZXJ0AAJbQkwABmVudGl0eXQAFkxqYXZhL2lvL1NlcmlhbGl6YWJsZTtMAA5zZXJ2\
+ aWNlT3B0aW9uc3QAIUxjb20vZ29vZ2xlL2Nsb3VkL1NlcnZpY2VPcHRpb25zO0wACHVwbG9hZElkdAASTGphdmEvbGFuZy9TdHJpbmc7eHAA8AAAAQAAAAAAAAAAdXIA\
+ AltCrPMX+AYIVOACAAB4cAAAAABwc3IAK2NvbS5nb29nbGUuY2xvdWQuc3RvcmFnZS5IdHRwU3RvcmFnZU9wdGlvbnO2aT7gXDty8wIAAUwAFXJldHJ5QWxnb3JpdGht\
+ TWFuYWdlcnQANExjb20vZ29vZ2xlL2Nsb3VkL3N0b3JhZ2UvSHR0cFJldHJ5QWxnb3JpdGhtTWFuYWdlcjt4cgAnY29tLmdvb2dsZS5jbG91ZC5zdG9yYWdlLlN0b3Jh\
+ Z2VPcHRpb25zmr/yM5bl3k8CAAB4cgAfY29tLmdvb2dsZS5jbG91ZC5TZXJ2aWNlT3B0aW9uc3+pCzP1UXJ+AgALTAAOY2xpZW50TGliVG9rZW5xAH4ABkwABWNsb2Nr\
+ dAAeTGNvbS9nb29nbGUvYXBpL2NvcmUvQXBpQ2xvY2s7TAALY3JlZGVudGlhbHN0AB1MY29tL2dvb2dsZS9hdXRoL0NyZWRlbnRpYWxzO0wADmhlYWRlclByb3ZpZGVy\
+ dAAnTGNvbS9nb29nbGUvYXBpL2dheC9ycGMvSGVhZGVyUHJvdmlkZXI7TAAEaG9zdHEAfgAGTAAJcHJvamVjdElkcQB+AAZMAA5xdW90YVByb2plY3RJZHEAfgAGTAAN\
+ cmV0cnlTZXR0aW5nc3QAK0xjb20vZ29vZ2xlL2FwaS9nYXgvcmV0cnlpbmcvUmV0cnlTZXR0aW5ncztMABdzZXJ2aWNlRmFjdG9yeUNsYXNzTmFtZXEAfgAGTAAac2Vy\
+ dmljZVJwY0ZhY3RvcnlDbGFzc05hbWVxAH4ABkwAEHRyYW5zcG9ydE9wdGlvbnN0ACNMY29tL2dvb2dsZS9jbG91ZC9UcmFuc3BvcnRPcHRpb25zO3hwdAAEZ2NjbHNy\
+ ACZjb20uZ29vZ2xlLmFwaS5jb3JlLkN1cnJlbnRNaWxsaXNDbG9ja6x3Swcn1hMLAgAAeHBzcgAeY29tLmdvb2dsZS5jbG91ZC5Ob0NyZWRlbnRpYWxzqRHnA54sDEAC\
+ AAB4cgAoY29tLmdvb2dsZS5hdXRoLm9hdXRoMi5PQXV0aDJDcmVkZW50aWFscz89fXrppVFXAgAETAAQZXhwaXJhdGlvbk1hcmdpbnQAFExqYXZhL3RpbWUvRHVyYXRp\
+ b247TAAEbG9ja3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wADXJlZnJlc2hNYXJnaW5xAH4AGUwABXZhbHVldAA1TGNvbS9nb29nbGUvYXV0aC9vYXV0aDIvT0F1dGgyQ3Jl\
+ ZGVudGlhbHMkT0F1dGhWYWx1ZTt4cgAbY29tLmdvb2dsZS5hdXRoLkNyZWRlbnRpYWxzCzii14w9kIECAAB4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcN\
+ AQAAAAAAAAEsAAAAAHh1cQB+AAgAAAAAc3EAfgAedw0BAAAAAAAAAWgAAAAAeHBzcgAnY29tLmdvb2dsZS5hcGkuZ2F4LnJwYy5Ob0hlYWRlclByb3ZpZGVyZaMSqGrF\
+ e2ECAAB4cHQAFWh0dHA6Ly9sb2NhbGhvc3Q6OTAwMHQABHByb2pwc3IAM2NvbS5nb29nbGUuYXBpLmdheC5yZXRyeWluZy5BdXRvVmFsdWVfUmV0cnlTZXR0aW5nc3Kb\
+ /39rR3SzAgAJWgAIaml0dGVyZWRJAAttYXhBdHRlbXB0c0QAFHJldHJ5RGVsYXlNdWx0aXBsaWVyRAAUcnBjVGltZW91dE11bHRpcGxpZXJMABFpbml0aWFsUmV0cnlE\
+ ZWxheXQAGkxvcmcvdGhyZWV0ZW4vYnAvRHVyYXRpb247TAARaW5pdGlhbFJwY1RpbWVvdXRxAH4AJ0wADW1heFJldHJ5RGVsYXlxAH4AJ0wADW1heFJwY1RpbWVvdXRx\
+ AH4AJ0wADHRvdGFsVGltZW91dHEAfgAneHIAKWNvbS5nb29nbGUuYXBpLmdheC5yZXRyeWluZy5SZXRyeVNldHRpbmdzcpv/f2tHdLMCAAB4cAEAAAAGQAAAAAAAAAA/\
+ 8AAAAAAAAHNyABNvcmcudGhyZWV0ZW4uYnAuU2VylV2EuhsiSLIMAAB4cHcNAQAAAAAAAAABAAAAAHhzcQB+ACp3DQEAAAAAAAAAMgAAAAB4c3EAfgAqdw0BAAAAAAAA\
+ ACAAAAAAeHNxAH4AKncNAQAAAAAAAAAyAAAAAHhzcQB+ACp3DQEAAAAAAAAAMgAAAAB4dAA+Y29tLmdvb2dsZS5jbG91ZC5zdG9yYWdlLkh0dHBTdG9yYWdlT3B0aW9u\
+ cyRIdHRwU3RvcmFnZUZhY3Rvcnl0AEFjb20uZ29vZ2xlLmNsb3VkLnN0b3JhZ2UuSHR0cFN0b3JhZ2VPcHRpb25zJEh0dHBTdG9yYWdlUnBjRmFjdG9yeXNyACpjb20u\
+ Z29vZ2xlLmNsb3VkLmh0dHAuSHR0cFRyYW5zcG9ydE9wdGlvbnNtf1RNvYf/IgIAA0kADmNvbm5lY3RUaW1lb3V0SQALcmVhZFRpbWVvdXRMAB1odHRwVHJhbnNwb3J0\
+ RmFjdG9yeUNsYXNzTmFtZXEAfgAGeHD//////////3QARmNvbS5nb29nbGUuY2xvdWQuaHR0cC5IdHRwVHJhbnNwb3J0T3B0aW9ucyREZWZhdWx0SHR0cFRyYW5zcG9y\
+ dEZhY3RvcnlzcgAyY29tLmdvb2dsZS5jbG91ZC5zdG9yYWdlLkh0dHBSZXRyeUFsZ29yaXRobU1hbmFnZXLSLXKZUDSYRQIAAUwADXJldHJ5U3RyYXRlZ3l0AC9MY29t\
+ L2dvb2dsZS9jbG91ZC9zdG9yYWdlL1N0b3JhZ2VSZXRyeVN0cmF0ZWd5O3hwc3IANGNvbS5nb29nbGUuY2xvdWQuc3RvcmFnZS5EZWZhdWx0U3RvcmFnZVJldHJ5U3Ry\
+ YXRlZ3luBoudquOViQIAAHhwdACQaHR0cDovL2xvY2FsaG9zdDo5MDAwL3VwbG9hZC9zdG9yYWdlL3YxL2IvYnVjay9vP3VwbG9hZFR5cGU9cmVzdW1hYmxlJnVwbG9h\
+ ZF9pZD0xNzcyNzI1NDM5ZDEyZWUzNjNmZmRlNmNiZmNlYjEzMGYzZTIxMWJiM2NjMzBlNjFhNGQ2N2I2MTU0OTUxMjIxc3IAIWNvbS5nb29nbGUuY2xvdWQuRXhjZXB0\
+ aW9uSGFuZGxlct3Z0AGsJj+JAgAETAAMaW50ZXJjZXB0b3JzdAApTGNvbS9nb29nbGUvY29tbW9uL2NvbGxlY3QvSW1tdXRhYmxlTGlzdDtMABZub25SZXRyaWFibGVF\
+ eGNlcHRpb25zdAAoTGNvbS9nb29nbGUvY29tbW9uL2NvbGxlY3QvSW1tdXRhYmxlU2V0O0wAE3JldHJpYWJsZUV4Y2VwdGlvbnNxAH4APUwACXJldHJ5SW5mb3QAD0xq\
+ YXZhL3V0aWwvU2V0O3hwc3IANmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTGlzdCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgABWwAIZWxlbWVudHN0\
+ ABNbTGphdmEvbGFuZy9PYmplY3Q7eHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAACc3IAWWNvbS5nb29nbGUuY2xvdWQuc3RvcmFnZS5EZWZh\
+ dWx0U3RvcmFnZVJldHJ5U3RyYXRlZ3kkRW1wdHlKc29uUGFyc2luZ0V4Y2VwdGlvbkludGVyY2VwdG9yz+LSc1EB+RsCAAB4cHNyAERjb20uZ29vZ2xlLmNsb3VkLnN0\
+ b3JhZ2UuRGVmYXVsdFN0b3JhZ2VSZXRyeVN0cmF0ZWd5JEludGVyY2VwdG9ySW1wbElTPf0EVOdoAgACWgAKaWRlbXBvdGVudEwAD3JldHJ5YWJsZUVycm9yc3EAfgA9\
+ eHABc3IANWNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlU2V0JFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAFbAAhlbGVtZW50c3EAfgBBeHB1cQB+AEMA\
+ AAAIc3IAK2NvbS5nb29nbGUuY2xvdWQuQmFzZVNlcnZpY2VFeGNlcHRpb24kRXJyb3LIN4LqhDNMpwIAA1oACHJlamVjdGVkTAAEY29kZXQAE0xqYXZhL2xhbmcvSW50\
+ ZWdlcjtMAAZyZWFzb25xAH4ABnhwAHNyABFqYXZhLmxhbmcuSW50ZWdlchLioKT3gYc4AgABSQAFdmFsdWV4cgAQamF2YS5sYW5nLk51bWJlcoaslR0LlOCLAgAAeHAA\
+ AAH4cHNxAH4ATABzcQB+AE8AAAH3cHNxAH4ATABzcQB+AE8AAAH2cHNxAH4ATABzcQB+AE8AAAH0cHNxAH4ATABzcQB+AE8AAAGtcHNxAH4ATABzcQB+AE8AAAGYcHNx\
+ AH4ATABwdAANaW50ZXJuYWxFcnJvcnNxAH4ATABwdAAbY29ubmVjdGlvbkNsb3NlZFByZW1hdHVyZWx5c3EAfgBJdXEAfgBDAAAAAHEAfgBgc3IAEWphdmEudXRpbC5I\
+ YXNoU2V0ukSFlZa4tzQDAAB4cHcMAAAAED9AAAAAAAAAeA==
diff --git a/google-cloud-storage/src/test/resources/com/google/cloud/storage/it/runner/registry/Dockerfile b/google-cloud-storage/src/test/resources/com/google/cloud/storage/it/runner/registry/Dockerfile
index 35df4e1df3..0b793bc300 100644
--- a/google-cloud-storage/src/test/resources/com/google/cloud/storage/it/runner/registry/Dockerfile
+++ b/google-cloud-storage/src/test/resources/com/google/cloud/storage/it/runner/registry/Dockerfile
@@ -1 +1 @@
-FROM gcr.io/cloud-devrel-public-resources/storage-testbench:v0.36.0
+FROM gcr.io/cloud-devrel-public-resources/storage-testbench:v0.37.0
diff --git a/grpc-google-cloud-storage-v2/pom.xml b/grpc-google-cloud-storage-v2/pom.xml
index c96df28ccf..825c606a69 100644
--- a/grpc-google-cloud-storage-v2/pom.xml
+++ b/grpc-google-cloud-storage-v2/pom.xml
@@ -4,13 +4,13 @@
4.0.0
com.google.api.grpc
grpc-google-cloud-storage-v2
- 2.24.0-alpha
+ 2.25.0-alpha
grpc-google-cloud-storage-v2
GRPC library for grpc-google-cloud-storage-v2
com.google.cloud
google-cloud-storage-parent
- 2.24.0
+ 2.25.0
diff --git a/grpc-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StorageGrpc.java b/grpc-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StorageGrpc.java
index b8f55baf9c..979f20d958 100644
--- a/grpc-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StorageGrpc.java
+++ b/grpc-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StorageGrpc.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/pom.xml b/pom.xml
index 8945ba764c..837179197b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
com.google.cloud
google-cloud-storage-parent
pom
- 2.24.0
+ 2.25.0
Storage Parent
https://github.com/googleapis/java-storage
@@ -14,7 +14,7 @@
com.google.cloud
google-cloud-shared-config
- 1.5.6
+ 1.5.7
@@ -54,7 +54,7 @@
UTF-8
github
google-cloud-storage-parent
- 3.13.0
+ 3.13.1
@@ -83,7 +83,7 @@
com.google.cloud
google-cloud-storage
- 2.24.0
+ 2.25.0
com.google.apis
@@ -93,7 +93,7 @@
com.google.cloud
google-cloud-pubsub
- 1.123.17
+ 1.123.18
test
@@ -124,17 +124,17 @@
com.google.api.grpc
proto-google-cloud-storage-v2
- 2.24.0-alpha
+ 2.25.0-alpha
com.google.api.grpc
grpc-google-cloud-storage-v2
- 2.24.0-alpha
+ 2.25.0-alpha
com.google.api.grpc
gapic-google-cloud-storage-v2
- 2.24.0-alpha
+ 2.25.0-alpha
com.google.cloud
diff --git a/proto-google-cloud-storage-v2/pom.xml b/proto-google-cloud-storage-v2/pom.xml
index 8d988a125a..19bbbf8f7c 100644
--- a/proto-google-cloud-storage-v2/pom.xml
+++ b/proto-google-cloud-storage-v2/pom.xml
@@ -4,13 +4,13 @@
4.0.0
com.google.api.grpc
proto-google-cloud-storage-v2
- 2.24.0-alpha
+ 2.25.0-alpha
proto-google-cloud-storage-v2
PROTO library for proto-google-cloud-storage-v2
com.google.cloud
google-cloud-storage-parent
- 2.24.0
+ 2.25.0
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/Bucket.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/Bucket.java
index 00d1991f86..ab05c01fb6 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/Bucket.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/Bucket.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/BucketAccessControl.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/BucketAccessControl.java
index 7c7d29c46b..ebb4a5a1f2 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/BucketAccessControl.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/BucketAccessControl.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/BucketAccessControlOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/BucketAccessControlOrBuilder.java
index 6b7573f3dc..95779435cb 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/BucketAccessControlOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/BucketAccessControlOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/BucketOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/BucketOrBuilder.java
index f5f20e13d2..626f06ffe4 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/BucketOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/BucketOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteRequest.java
index dc7c0173dc..c2aaa53c65 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteRequestOrBuilder.java
index 11d2a69182..891dd597af 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteResponse.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteResponse.java
index 4d7301e19a..3bf5e08167 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteResponse.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteResponse.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteResponseOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteResponseOrBuilder.java
index f192ac86a8..59a085ee31 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteResponseOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CancelResumableWriteResponseOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ChecksummedData.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ChecksummedData.java
index 8c85c618b4..8284640922 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ChecksummedData.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ChecksummedData.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ChecksummedDataOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ChecksummedDataOrBuilder.java
index 9b8aa6001b..cac4e13c5d 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ChecksummedDataOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ChecksummedDataOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CommonObjectRequestParams.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CommonObjectRequestParams.java
index 54a47e276b..659a27f6c5 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CommonObjectRequestParams.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CommonObjectRequestParams.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CommonObjectRequestParamsOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CommonObjectRequestParamsOrBuilder.java
index 327ae41411..9182e76351 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CommonObjectRequestParamsOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CommonObjectRequestParamsOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ComposeObjectRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ComposeObjectRequest.java
index 6aede839b8..51000d502d 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ComposeObjectRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ComposeObjectRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ComposeObjectRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ComposeObjectRequestOrBuilder.java
index 9e95be1591..07a5239e04 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ComposeObjectRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ComposeObjectRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ContentRange.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ContentRange.java
index 1560e20aab..25119c1320 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ContentRange.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ContentRange.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ContentRangeOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ContentRangeOrBuilder.java
index 969abe6d3b..da81df1ddf 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ContentRangeOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ContentRangeOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateBucketRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateBucketRequest.java
index 82c72702bd..959131014f 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateBucketRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateBucketRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateBucketRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateBucketRequestOrBuilder.java
index aafdbc708a..967621444a 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateBucketRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateBucketRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyRequest.java
index 24e0a7b9e1..619a750496 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyRequestOrBuilder.java
index 67b9cd539f..d35e121799 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyResponse.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyResponse.java
index c1fcf5088f..d893e943dd 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyResponse.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyResponse.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyResponseOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyResponseOrBuilder.java
index 0a77982ef8..1dbe0d5467 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyResponseOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateHmacKeyResponseOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateNotificationConfigRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateNotificationConfigRequest.java
index 1a3f99cec7..e6258c8719 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateNotificationConfigRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateNotificationConfigRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateNotificationConfigRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateNotificationConfigRequestOrBuilder.java
index 73344ab699..1ef0465349 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateNotificationConfigRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CreateNotificationConfigRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CustomerEncryption.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CustomerEncryption.java
index 27f9c05f5b..de15dd6a16 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CustomerEncryption.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CustomerEncryption.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CustomerEncryptionOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CustomerEncryptionOrBuilder.java
index 42a2b1effc..4effd943b4 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CustomerEncryptionOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/CustomerEncryptionOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteBucketRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteBucketRequest.java
index 3287d8c3df..420d964461 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteBucketRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteBucketRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteBucketRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteBucketRequestOrBuilder.java
index 7c2e583d25..fa645400a8 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteBucketRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteBucketRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteHmacKeyRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteHmacKeyRequest.java
index 408e9ff934..7361ab0de8 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteHmacKeyRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteHmacKeyRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteHmacKeyRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteHmacKeyRequestOrBuilder.java
index 7d11ae9286..184aff6b63 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteHmacKeyRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteHmacKeyRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteNotificationConfigRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteNotificationConfigRequest.java
index 95989710d6..1d2062f8b9 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteNotificationConfigRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteNotificationConfigRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteNotificationConfigRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteNotificationConfigRequestOrBuilder.java
index 6098d96ff3..1f118d6166 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteNotificationConfigRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteNotificationConfigRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteObjectRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteObjectRequest.java
index b3a9c7ccbe..d704ca0794 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteObjectRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteObjectRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteObjectRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteObjectRequestOrBuilder.java
index c93ab4af16..f746d4410c 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteObjectRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/DeleteObjectRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetBucketRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetBucketRequest.java
index 2238f8e0ae..6de2a5ae7d 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetBucketRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetBucketRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetBucketRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetBucketRequestOrBuilder.java
index 3b0735f116..85cdbc5738 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetBucketRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetBucketRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetHmacKeyRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetHmacKeyRequest.java
index c09b83335c..2ebd135484 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetHmacKeyRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetHmacKeyRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetHmacKeyRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetHmacKeyRequestOrBuilder.java
index e7ba7a09bc..88d37d0e7f 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetHmacKeyRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetHmacKeyRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetNotificationConfigRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetNotificationConfigRequest.java
index 4ad192896e..b29323e6d7 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetNotificationConfigRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetNotificationConfigRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetNotificationConfigRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetNotificationConfigRequestOrBuilder.java
index 9a2f81896b..6005568821 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetNotificationConfigRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetNotificationConfigRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetObjectRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetObjectRequest.java
index f89f58e8e8..2b87c35ae1 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetObjectRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetObjectRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetObjectRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetObjectRequestOrBuilder.java
index fd35680f3a..d1a2614b4f 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetObjectRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetObjectRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetServiceAccountRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetServiceAccountRequest.java
index aeb0778c74..14506aa3e4 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetServiceAccountRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetServiceAccountRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetServiceAccountRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetServiceAccountRequestOrBuilder.java
index 74f04937a8..be3deb7152 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetServiceAccountRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/GetServiceAccountRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/HmacKeyMetadata.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/HmacKeyMetadata.java
index fa7d9d7cfa..49e782f61a 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/HmacKeyMetadata.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/HmacKeyMetadata.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/HmacKeyMetadataOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/HmacKeyMetadataOrBuilder.java
index 8dfc5ee628..9544912199 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/HmacKeyMetadataOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/HmacKeyMetadataOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsRequest.java
index 3cf97c6082..fe4c7ff4e9 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsRequestOrBuilder.java
index 0afd2226ce..c51a104f90 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsResponse.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsResponse.java
index bf6dc24f68..ac23896f3c 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsResponse.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsResponse.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsResponseOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsResponseOrBuilder.java
index cdc9f1f78e..926e975043 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsResponseOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListBucketsResponseOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysRequest.java
index e7815eefba..0349cec433 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysRequestOrBuilder.java
index 3dbfdaea55..820453db94 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysResponse.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysResponse.java
index 076cab3182..1dec8bbaa7 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysResponse.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysResponse.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysResponseOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysResponseOrBuilder.java
index fc777c3430..39bc3f176f 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysResponseOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListHmacKeysResponseOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsRequest.java
index eab8c7183a..591c8ac417 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsRequestOrBuilder.java
index f654e44dc8..67c71c39aa 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsResponse.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsResponse.java
index 9ccb82351b..8f3d92f77d 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsResponse.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsResponse.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsResponseOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsResponseOrBuilder.java
index f1fe02eeff..9c8e9954dd 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsResponseOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListNotificationConfigsResponseOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsRequest.java
index 0096b212a3..300a8dc1fd 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsRequestOrBuilder.java
index 187cfa59cf..888aab23bd 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsResponse.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsResponse.java
index 16c00c8954..3a75847a8f 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsResponse.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsResponse.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsResponseOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsResponseOrBuilder.java
index 6fa2582940..53bdacfa40 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsResponseOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ListObjectsResponseOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/LockBucketRetentionPolicyRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/LockBucketRetentionPolicyRequest.java
index 3b6d80020c..8d04f29691 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/LockBucketRetentionPolicyRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/LockBucketRetentionPolicyRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/LockBucketRetentionPolicyRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/LockBucketRetentionPolicyRequestOrBuilder.java
index 7a5eae42f8..0d0afef1a8 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/LockBucketRetentionPolicyRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/LockBucketRetentionPolicyRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/NotificationConfig.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/NotificationConfig.java
index e94f8e124a..2199bda4c4 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/NotificationConfig.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/NotificationConfig.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/NotificationConfigOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/NotificationConfigOrBuilder.java
index 8a65871053..7684b36963 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/NotificationConfigOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/NotificationConfigOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/Object.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/Object.java
index 336e85ad65..6416b817d7 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/Object.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/Object.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectAccessControl.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectAccessControl.java
index cd90c6035b..e5d8d2dcc8 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectAccessControl.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectAccessControl.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectAccessControlOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectAccessControlOrBuilder.java
index 20ee2c6a38..ff444643f1 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectAccessControlOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectAccessControlOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectChecksums.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectChecksums.java
index 0ed622e295..50e15ab395 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectChecksums.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectChecksums.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectChecksumsOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectChecksumsOrBuilder.java
index f65c591723..81db876211 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectChecksumsOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectChecksumsOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectOrBuilder.java
index 852b2160bb..e921910569 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ObjectOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/Owner.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/Owner.java
index f46be86bb0..96b5a50330 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/Owner.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/Owner.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/OwnerOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/OwnerOrBuilder.java
index 4caffe13d6..0cfbeb1148 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/OwnerOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/OwnerOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ProjectTeam.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ProjectTeam.java
index 5097014e8b..327cdef4fd 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ProjectTeam.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ProjectTeam.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ProjectTeamOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ProjectTeamOrBuilder.java
index 3af551ee1e..bf4ae60213 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ProjectTeamOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ProjectTeamOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusRequest.java
index db5525627d..0a16b8cb86 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusRequestOrBuilder.java
index f56814f69d..9807a260e6 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusResponse.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusResponse.java
index cc6b965655..6fcbefe3d7 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusResponse.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusResponse.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusResponseOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusResponseOrBuilder.java
index d793262ac1..31e1b79742 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusResponseOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/QueryWriteStatusResponseOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectRequest.java
index ea126599a9..fae2e1c5cc 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectRequestOrBuilder.java
index 365dbdbce6..2ef4fe8a8e 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectResponse.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectResponse.java
index f360288f1a..0150ff5e09 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectResponse.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectResponse.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectResponseOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectResponseOrBuilder.java
index f8c717e1d4..420883d870 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectResponseOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ReadObjectResponseOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteObjectRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteObjectRequest.java
index d2127f0c50..9ad0ba000d 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteObjectRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteObjectRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteObjectRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteObjectRequestOrBuilder.java
index 808c42ad55..dc5beddfce 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteObjectRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteObjectRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteResponse.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteResponse.java
index fe54b1b6f0..ac31614a86 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteResponse.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteResponse.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteResponseOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteResponseOrBuilder.java
index 3e38750e35..b556f3b6c9 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteResponseOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/RewriteResponseOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceAccount.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceAccount.java
index 0632bcab4b..20742ebc07 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceAccount.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceAccount.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceAccountOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceAccountOrBuilder.java
index d6637ee207..880c17100e 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceAccountOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceAccountOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceConstants.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceConstants.java
index 7b0cc49b69..b80de230f6 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceConstants.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceConstants.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceConstantsOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceConstantsOrBuilder.java
index 9722940ed4..8468f8550d 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceConstantsOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/ServiceConstantsOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteRequest.java
index 37082d1535..30b9435ea7 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteRequestOrBuilder.java
index fb2c88680d..edd9a323ab 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteResponse.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteResponse.java
index ed5db155e2..1d8e46aa1c 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteResponse.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteResponse.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteResponseOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteResponseOrBuilder.java
index 4936df9ca2..decbf93374 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteResponseOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StartResumableWriteResponseOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StorageProto.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StorageProto.java
index 26f3553645..561de97d9b 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StorageProto.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/StorageProto.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateBucketRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateBucketRequest.java
index 10568499da..4152fc7326 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateBucketRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateBucketRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateBucketRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateBucketRequestOrBuilder.java
index 663a58a992..5917c1c8ac 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateBucketRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateBucketRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateHmacKeyRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateHmacKeyRequest.java
index 2af93d09a2..37fcd90c0c 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateHmacKeyRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateHmacKeyRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateHmacKeyRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateHmacKeyRequestOrBuilder.java
index 6dddfc06ed..f276a58154 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateHmacKeyRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateHmacKeyRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateObjectRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateObjectRequest.java
index e3b1ba4de7..f1fb42789e 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateObjectRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateObjectRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateObjectRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateObjectRequestOrBuilder.java
index 85a5d228db..e87e0c1dd9 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateObjectRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/UpdateObjectRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectRequest.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectRequest.java
index aa01ce4c5a..51a90b30e6 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectRequest.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectRequest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectRequestOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectRequestOrBuilder.java
index 04b60183cd..3d78815c6d 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectRequestOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectRequestOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectResponse.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectResponse.java
index c99f4fafc0..d5f611a4b2 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectResponse.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectResponse.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectResponseOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectResponseOrBuilder.java
index fcc3a606b1..1a27b9ac88 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectResponseOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectResponseOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectSpec.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectSpec.java
index bc3715f726..e01485764e 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectSpec.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectSpec.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectSpecOrBuilder.java b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectSpecOrBuilder.java
index 1797218b77..554af4dfbd 100644
--- a/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectSpecOrBuilder.java
+++ b/proto-google-cloud-storage-v2/src/main/java/com/google/storage/v2/WriteObjectSpecOrBuilder.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2020 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml
index 770c68667f..7b16d416c3 100644
--- a/samples/install-without-bom/pom.xml
+++ b/samples/install-without-bom/pom.xml
@@ -30,7 +30,7 @@
com.google.cloud
google-cloud-storage
- 2.23.0
+ 2.24.0
@@ -61,7 +61,7 @@
com.google.cloud
google-cloud-pubsub
- 1.123.17
+ 1.123.18
test
diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml
index 841e4a1846..d2ab7237c8 100644
--- a/samples/native-image-sample/pom.xml
+++ b/samples/native-image-sample/pom.xml
@@ -29,7 +29,7 @@
com.google.cloud
libraries-bom
- 26.18.0
+ 26.19.0
pom
import
@@ -61,7 +61,7 @@
com.google.cloud
google-cloud-pubsub
- 1.123.17
+ 1.123.18
test
@@ -114,7 +114,7 @@
org.junit.vintage
junit-vintage-engine
- 5.9.3
+ 5.10.0
test