From 12377cc716b5b83cbac9c238cf5ba788daf02970 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Thu, 18 Jun 2020 12:40:32 -0700 Subject: [PATCH 1/8] update libraries-bom version (#382) --- README.md | 16 +++++++++++++--- synth.metadata | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7afa65fb65..4dbc00089d 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file com.google.cloud libraries-bom - 6.0.0 + 7.0.0 pom import @@ -47,11 +47,11 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.cloud:google-cloud-storage:1.110.0' +compile 'com.google.cloud:google-cloud-storage:1.109.1' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "1.110.0" +libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "1.109.1" ``` [//]: # ({x-version-update-end}) @@ -207,6 +207,16 @@ display on your webpage. +## Samples + +Samples are in the [`samples/`](https://github.com/googleapis/java-storage/tree/master/samples) directory. The samples' `README.md` +has instructions for running the samples. + +| Sample | Source Code | Try it | +| --------------------------- | --------------------------------- | ------ | +| Quickstart Sample | [source code](https://github.com/googleapis/java-storage/blob/master/samples/snippets/src/main/java/com/example/storage/QuickstartSample.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-storage&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/storage/QuickstartSample.java) | + + ## Troubleshooting diff --git a/synth.metadata b/synth.metadata index 24a5838925..54bb02f27f 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-storage.git", - "sha": "5d33d641d02634491f5b3a33b94729b69790359b" + "sha": "34d1633a8d9bdff8dade812e25587d2e220f14f5" } }, { From 37f00ebc84ff4069f14e1cfa0f0a5f9552f08282 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 18 Jun 2020 19:54:07 +0000 Subject: [PATCH 2/8] chore: release 1.110.1-SNAPSHOT (#384) :robot: I have created a release \*beep\* \*boop\* --- ### Updating meta-information for bleeding-edge SNAPSHOT release. --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). --- google-cloud-storage/pom.xml | 4 ++-- pom.xml | 2 +- versions.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index d651252c10..3e2dbb0ba2 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-storage - 1.110.0 + 1.110.1-SNAPSHOT jar Google Cloud Storage https://github.com/googleapis/java-storage @@ -12,7 +12,7 @@ com.google.cloud google-cloud-storage-parent - 1.110.0 + 1.110.1-SNAPSHOT google-cloud-storage diff --git a/pom.xml b/pom.xml index a5cb6e9a84..02c4dd3949 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-storage-parent pom - 1.110.0 + 1.110.1-SNAPSHOT Storage Parent https://github.com/googleapis/java-storage diff --git a/versions.txt b/versions.txt index 9081168b0e..9d49b870bc 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-storage:1.110.0:1.110.0 \ No newline at end of file +google-cloud-storage:1.110.0:1.110.1-SNAPSHOT \ No newline at end of file From 81472a434ad6c9f9474b0c2360be0e42b10ed227 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Fri, 19 Jun 2020 16:02:07 -0700 Subject: [PATCH 3/8] [CHANGE ME] Re-generated to pick up changes from self. (#385) This PR was generated using Autosynth. :rainbow: Synth log will be available here: https://source.cloud.google.com/results/invocations/8090a8db-78af-43ce-903e-cbc4378c9d99/targets - [ ] To automatically regenerate this PR, check this box. --- README.md | 4 ++-- synth.metadata | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4dbc00089d..a9bac3a944 100644 --- a/README.md +++ b/README.md @@ -47,11 +47,11 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.cloud:google-cloud-storage:1.109.1' +compile 'com.google.cloud:google-cloud-storage:1.110.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "1.109.1" +libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "1.110.0" ``` [//]: # ({x-version-update-end}) diff --git a/synth.metadata b/synth.metadata index 54bb02f27f..79e7a4feda 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-storage.git", - "sha": "34d1633a8d9bdff8dade812e25587d2e220f14f5" + "sha": "37f00ebc84ff4069f14e1cfa0f0a5f9552f08282" } }, { From 8dfc0cbf8294a7fc426948e22e5c2182da97b630 Mon Sep 17 00:00:00 2001 From: Ajit Thakor <49403056+athakor@users.noreply.github.com> Date: Thu, 25 Jun 2020 20:38:57 +0530 Subject: [PATCH 4/8] feat: expose all the methods of notification (#141) * feat: expose all the methods of notification * feat: remove unsed code * feat: fix review changes * build: fix require upper bound dependencies errors * build: fix clirr ignore differences * feat: implement getNotification inside StorageRpcTestBase class and fix the builds checks --- .../clirr-ignored-differences.xml | 25 ++ google-cloud-storage/pom.xml | 5 + .../cloud/storage/NotificationInfo.java | 309 ++++++++++++++++++ .../com/google/cloud/storage/Storage.java | 33 ++ .../com/google/cloud/storage/StorageImpl.java | 74 +++++ .../cloud/storage/spi/v1/HttpStorageRpc.java | 15 + .../storage/spi/v1/HttpStorageRpcSpans.java | 2 + .../cloud/storage/spi/v1/StorageRpc.java | 8 + .../storage/testing/StorageRpcTestBase.java | 5 + .../cloud/storage/NotificationInfoTest.java | 102 ++++++ .../google/cloud/storage/StorageImplTest.java | 90 ++++- .../cloud/storage/it/ITStorageTest.java | 59 ++++ .../testing/StorageRpcTestBaseTest.java | 11 + pom.xml | 6 + 14 files changed, 743 insertions(+), 1 deletion(-) create mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/NotificationInfo.java create mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/NotificationInfoTest.java diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index ace7a6ef0e..2aababb9e7 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -26,4 +26,29 @@ com.google.cloud.storage.BucketInfo$Builder deleteLifecycleRules() 7013 + + com/google/cloud/storage/Storage + com.google.api.services.storage.model.Notification createNotification(java.lang.String, com.google.cloud.storage.NotificationInfo) + 7012 + + + com/google/cloud/storage/Storage + com.google.api.services.storage.model.Notification getNotification(java.lang.String, java.lang.String) + 7012 + + + com/google/cloud/storage/Storage + java.util.List listNotifications(java.lang.String) + 7012 + + + com/google/cloud/storage/Storage + boolean deleteNotification(java.lang.String, java.lang.String) + 7012 + + + com/google/cloud/storage/spi/v1/StorageRpc + com.google.api.services.storage.model.Notification getNotification(java.lang.String, java.lang.String) + 7012 + diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 3e2dbb0ba2..1be67d6b39 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -155,6 +155,11 @@ truth test + + com.google.cloud + google-cloud-pubsub + test + org.easymock easymock diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/NotificationInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/NotificationInfo.java new file mode 100644 index 0000000000..34c2d326d6 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/NotificationInfo.java @@ -0,0 +1,309 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.pathtemplate.PathTemplate; +import com.google.api.services.storage.model.Notification; +import com.google.common.base.Function; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Google Storage Notification metadata; + * + * @see Concepts and + * Terminology + */ +public class NotificationInfo implements Serializable { + + private static final long serialVersionUID = 5725883368559753810L; + private static final PathTemplate PATH_TEMPLATE = + PathTemplate.createWithoutUrlEncoding("projects/{project}/topics/{topic}"); + + public enum PayloadFormat { + JSON_API_V1, + NONE + } + + static final Function FROM_PB_FUNCTION = + new Function() { + @Override + public NotificationInfo apply(Notification pb) { + return NotificationInfo.fromPb(pb); + } + }; + static final Function TO_PB_FUNCTION = + new Function() { + @Override + public Notification apply(NotificationInfo NotificationInfo) { + return NotificationInfo.toPb(); + } + }; + private final String generatedId; + private final String topic; + private final List eventTypes; + private final Map customAttributes; + private final PayloadFormat payloadFormat; + private final String objectNamePrefix; + private final String etag; + private final String selfLink; + + public static final class Builder { + + private String generatedId; + private String topic; + private List eventTypes; + private Map customAttributes; + private PayloadFormat payloadFormat; + private String objectNamePrefix; + private String etag; + private String selfLink; + + Builder(String topic) { + this.topic = topic; + } + + Builder(NotificationInfo NotificationInfo) { + generatedId = NotificationInfo.generatedId; + etag = NotificationInfo.etag; + selfLink = NotificationInfo.selfLink; + topic = NotificationInfo.topic; + eventTypes = NotificationInfo.eventTypes; + customAttributes = NotificationInfo.customAttributes; + payloadFormat = NotificationInfo.payloadFormat; + objectNamePrefix = NotificationInfo.objectNamePrefix; + } + + Builder setGeneratedId(String generatedId) { + this.generatedId = generatedId; + return this; + } + + Builder setSelfLink(String selfLink) { + this.selfLink = selfLink; + return this; + } + + /** The name of the topic. It must have the format "projects/{project}/topics/{topic}". */ + public Builder setTopic(String topic) { + this.topic = topic; + return this; + } + + public Builder setPayloadFormat(PayloadFormat payloadFormat) { + this.payloadFormat = payloadFormat; + return this; + } + + public Builder setObjectNamePrefix(String objectNamePrefix) { + this.objectNamePrefix = objectNamePrefix; + return this; + } + + public Builder setEventTypes(Iterable eventTypes) { + this.eventTypes = eventTypes != null ? ImmutableList.copyOf(eventTypes) : null; + return this; + } + + Builder setEtag(String etag) { + this.etag = etag; + return this; + } + + public Builder setCustomAttributes(Map customAttributes) { + this.customAttributes = + customAttributes != null ? ImmutableMap.copyOf(customAttributes) : null; + return this; + } + + public NotificationInfo build() { + checkNotNull(topic); + return new NotificationInfo(this); + } + } + + NotificationInfo(Builder builder) { + generatedId = builder.generatedId; + etag = builder.etag; + selfLink = builder.selfLink; + topic = builder.topic; + eventTypes = builder.eventTypes; + customAttributes = builder.customAttributes; + payloadFormat = builder.payloadFormat; + objectNamePrefix = builder.objectNamePrefix; + } + + /** Returns the service-generated id for the notification. */ + public String getGeneratedId() { + return generatedId; + } + + /** Returns the topic to which this subscription publishes. */ + public String getTopic() { + return topic; + } + + /** Returns the canonical URI of this topic as a string. */ + public String getSelfLink() { + return selfLink; + } + + /** Returns the desired content of the Payload. */ + public PayloadFormat getPayloadFormat() { + return payloadFormat; + } + + /** Returns the object name prefix for which this notification configuration applies. */ + public String getObjectNamePrefix() { + return objectNamePrefix; + } + + /** + * Returns HTTP 1.1 Entity tag for the notification. + * + * @see Entity Tags + */ + public String getEtag() { + return etag; + } + + /** + * Returns the list of event types that this notification will apply to. If empty, notifications + * will be sent on all event types. + * + * @see Cross-Origin Resource Sharing + * (CORS) + */ + public List getEventTypes() { + return eventTypes; + } + + /** + * Returns the list of additional attributes to attach to each Cloud PubSub message published for + * this notification subscription. + * + * @see + * About Access Control Lists + */ + public Map getCustomAttributes() { + return customAttributes; + } + + /** Returns a builder for the current notification. */ + public Builder toBuilder() { + return new Builder(this); + } + + @Override + public int hashCode() { + return Objects.hash(getTopic()); + } + + @Override + public boolean equals(Object obj) { + return obj == this + || obj != null + && obj.getClass().equals(NotificationInfo.class) + && Objects.equals(toPb(), ((NotificationInfo) obj).toPb()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("topic", getTopic()).toString(); + } + + Notification toPb() { + Notification notificationPb = new Notification(); + notificationPb.setId(generatedId); + notificationPb.setEtag(etag); + if (customAttributes != null) { + notificationPb.setCustomAttributes(customAttributes); + } + if (eventTypes != null) { + notificationPb.setEventTypes(eventTypes); + } + if (objectNamePrefix != null) { + notificationPb.setObjectNamePrefix(objectNamePrefix); + } + if (payloadFormat != null) { + notificationPb.setPayloadFormat(payloadFormat.toString()); + } else { + notificationPb.setPayloadFormat(PayloadFormat.NONE.toString()); + } + notificationPb.setSelfLink(selfLink); + notificationPb.setTopic(topic); + + return notificationPb; + } + + /** + * Creates a {@code NotificationInfo} object for the provided topic name. + * + * @param topic The name of the topic. It must have the format + * "projects/{project}/topics/{topic}". + */ + public static NotificationInfo of(String topic) { + PATH_TEMPLATE.validatedMatch(topic, "topic name must be in valid format"); + return newBuilder(topic).build(); + } + + /** + * Returns a {@code NotificationInfo} builder where the topic's name is set to the provided name. + * + * @param topic The name of the topic. It must have the format + * "projects/{project}/topics/{topic}". + */ + public static Builder newBuilder(String topic) { + PATH_TEMPLATE.validatedMatch(topic, "topic name must be in valid format"); + return new Builder(topic); + } + + static NotificationInfo fromPb(Notification notificationPb) { + Builder builder = newBuilder(notificationPb.getTopic()); + if (notificationPb.getId() != null) { + builder.setGeneratedId(notificationPb.getId()); + } + if (notificationPb.getEtag() != null) { + builder.setEtag(notificationPb.getEtag()); + } + if (notificationPb.getCustomAttributes() != null) { + builder.setCustomAttributes(notificationPb.getCustomAttributes()); + } + if (notificationPb.getSelfLink() != null) { + builder.setSelfLink(notificationPb.getSelfLink()); + } + if (notificationPb.getObjectNamePrefix() != null) { + builder.setObjectNamePrefix(notificationPb.getObjectNamePrefix()); + } + if (notificationPb.getTopic() != null) { + builder.setTopic(notificationPb.getTopic()); + } + if (notificationPb.getEventTypes() != null) { + builder.setEventTypes(notificationPb.getEventTypes()); + } + if (notificationPb.getPayloadFormat() != null) { + builder.setPayloadFormat(PayloadFormat.valueOf(notificationPb.getPayloadFormat())); + } + return builder.build(); + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 988f87d39c..fa196c90da 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -21,6 +21,7 @@ import com.google.api.core.InternalExtensionOnly; import com.google.api.gax.paging.Page; +import com.google.api.services.storage.model.Notification; import com.google.auth.ServiceAccountSigner; import com.google.auth.ServiceAccountSigner.SigningException; import com.google.cloud.FieldSelector; @@ -3433,4 +3434,36 @@ List testIamPermissions( * @throws StorageException upon failure */ ServiceAccount getServiceAccount(String projectId); + + /** + * Creates a notification with the specified entity on the specified bucket. + * + * @return the notification that was created. + * @throws StorageException upon failure + */ + Notification createNotification(String bucket, NotificationInfo notification); + + /** + * Get the notification with the specified name on the specified object. + * + * @return the notification object that exist on the bucket. + * @throws StorageException upon failure + */ + Notification getNotification(String bucket, String notification); + + /** + * List the notifications for the provided bucket. + * + * @return a list of {@link Notification} objects that exist on the bucket. + * @throws StorageException upon failure + */ + List listNotifications(String bucket); + + /** + * Deletes the notification with the specified name on the specified object. + * + * @return {@code true} if the notification was deleted, {@code false} if it was not found + * @throws StorageException upon failure + */ + boolean deleteNotification(String bucket, String notification); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 0e24521eb6..ef46249d05 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -36,6 +36,7 @@ import com.google.api.gax.paging.Page; import com.google.api.services.storage.model.BucketAccessControl; +import com.google.api.services.storage.model.Notification; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.api.services.storage.model.StorageObject; import com.google.api.services.storage.model.TestIamPermissionsResponse; @@ -1698,6 +1699,79 @@ public com.google.api.services.storage.model.ServiceAccount call() { } } + @Override + public Notification createNotification(final String bucket, final NotificationInfo notification) { + final com.google.api.services.storage.model.Notification notificationPb = notification.toPb(); + try { + return runWithRetries( + new Callable() { + @Override + public com.google.api.services.storage.model.Notification call() { + return storageRpc.createNotification(bucket, notificationPb); + } + }, + getOptions().getRetrySettings(), + EXCEPTION_HANDLER, + getOptions().getClock()); + } catch (RetryHelperException e) { + throw StorageException.translateAndThrow(e); + } + } + + @Override + public Notification getNotification(final String bucket, final String notification) { + try { + return runWithRetries( + new Callable() { + @Override + public Notification call() { + return storageRpc.getNotification(bucket, notification); + } + }, + getOptions().getRetrySettings(), + EXCEPTION_HANDLER, + getOptions().getClock()); + } catch (RetryHelperException e) { + throw StorageException.translateAndThrow(e); + } + } + + @Override + public List listNotifications(final String bucket) { + try { + return runWithRetries( + new Callable>() { + @Override + public List call() { + return storageRpc.listNotifications(bucket); + } + }, + getOptions().getRetrySettings(), + EXCEPTION_HANDLER, + getOptions().getClock()); + } catch (RetryHelperException e) { + throw StorageException.translateAndThrow(e); + } + } + + @Override + public boolean deleteNotification(final String bucket, final String notification) { + try { + return runWithRetries( + new Callable() { + @Override + public Boolean call() { + return storageRpc.deleteNotification(bucket, notification); + } + }, + getOptions().getRetrySettings(), + EXCEPTION_HANDLER, + getOptions().getClock()); + } catch (RetryHelperException e) { + throw StorageException.translateAndThrow(e); + } + } + private static void addToOptionMap( StorageRpc.Option option, T defaultValue, Map map) { addToOptionMap(option, option, defaultValue, map); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index e211c0277e..97ceafa328 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -1489,6 +1489,21 @@ public Notification createNotification(String bucket, Notification notification) } } + @Override + public Notification getNotification(String bucket, String notification) { + Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_GET_NOTIFICATION); + Scope scope = tracer.withSpan(span); + try { + return storage.notifications().get(bucket, notification).execute(); + } catch (IOException ex) { + span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); + throw translate(ex); + } finally { + scope.close(); + span.end(); + } + } + @Override public Bucket lockRetentionPolicy(Bucket bucket, Map options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_LOCK_RETENTION_POLICY); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java index a5e4daf7da..11a33d8d02 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java @@ -89,6 +89,8 @@ class HttpStorageRpcSpans { static final String SPAN_NAME_LIST_NOTIFICATIONS = getTraceSpanName("listNotifications(String)"); static final String SPAN_NAME_CREATE_NOTIFICATION = getTraceSpanName("createNotification(String,Notification)"); + static final String SPAN_NAME_GET_NOTIFICATION = + getTraceSpanName("getNotification(String,String)"); static final String SPAN_LOCK_RETENTION_POLICY = getTraceSpanName("lockRetentionPolicy(String,Long)"); static final String SPAN_NAME_GET_SERVICE_ACCOUNT = getTraceSpanName("getServiceAccount(String)"); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index 36c7a5ff1b..0291d920bf 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -532,6 +532,14 @@ TestIamPermissionsResponse testIamPermissions( */ Notification createNotification(String bucket, Notification notification); + /** + * Get the notification with the specified name on the specified object. + * + * @return the notification object that exist on the bucket. + * @throws StorageException upon failure + */ + Notification getNotification(String bucket, String notification); + /** * Lock retention policy for the provided bucket. * diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java index 6bd2d487ea..0f8e651a6a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java @@ -280,6 +280,11 @@ public Notification createNotification(String bucket, Notification notification) throw new UnsupportedOperationException("Not implemented yet"); } + @Override + public Notification getNotification(String bucket, String notification) { + throw new UnsupportedOperationException("Not implemented yet"); + } + @Override public Bucket lockRetentionPolicy(Bucket bucket, Map options) { throw new UnsupportedOperationException("Not implemented yet"); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/NotificationInfoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/NotificationInfoTest.java new file mode 100644 index 0000000000..40b8a5ccf1 --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/NotificationInfoTest.java @@ -0,0 +1,102 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static org.junit.Assert.assertEquals; + +import com.google.cloud.storage.NotificationInfo.PayloadFormat; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.junit.Test; + +public class NotificationInfoTest { + + private static final String ETAG = "0xFF00"; + private static final String GENERATED_ID = "B/N:1"; + private static final String SELF_LINK = "http://storage/b/n"; + private static final List EVENT_TYPES = + ImmutableList.of("OBJECT_FINALIZE", "OBJECT_METADATA_UPDATE"); + private static final String OBJECT_NAME_PREFIX = "index.html"; + private static final PayloadFormat PAYLOAD_FORMAT = PayloadFormat.JSON_API_V1.JSON_API_V1; + private static final String TOPIC = "projects/myProject/topics/topic1"; + private static final Map CUSTOM_ATTRIBUTES = ImmutableMap.of("label1", "value1"); + private static final NotificationInfo NOTIFICATION_INFO = + NotificationInfo.newBuilder(TOPIC) + .setEtag(ETAG) + .setCustomAttributes(CUSTOM_ATTRIBUTES) + .setSelfLink(SELF_LINK) + .setEventTypes(EVENT_TYPES) + .setObjectNamePrefix(OBJECT_NAME_PREFIX) + .setPayloadFormat(PAYLOAD_FORMAT) + .setGeneratedId(GENERATED_ID) + .build(); + + @Test + public void testToBuilder() { + compareBuckets(NOTIFICATION_INFO, NOTIFICATION_INFO.toBuilder().build()); + NotificationInfo bucketInfo = NOTIFICATION_INFO.toBuilder().setGeneratedId("id").build(); + assertEquals("id", bucketInfo.getGeneratedId()); + bucketInfo = bucketInfo.toBuilder().setGeneratedId(GENERATED_ID).build(); + compareBuckets(NOTIFICATION_INFO, bucketInfo); + } + + @Test + public void testToBuilderIncomplete() { + NotificationInfo incompleteBucketInfo = NotificationInfo.newBuilder(TOPIC).build(); + compareBuckets(incompleteBucketInfo, incompleteBucketInfo.toBuilder().build()); + } + + @Test + public void testOf() { + NotificationInfo bucketInfo = NotificationInfo.of(TOPIC); + assertEquals(TOPIC, bucketInfo.getTopic()); + } + + @Test + public void testBuilder() { + assertEquals(ETAG, NOTIFICATION_INFO.getEtag()); + assertEquals(GENERATED_ID, NOTIFICATION_INFO.getGeneratedId()); + assertEquals(SELF_LINK, NOTIFICATION_INFO.getSelfLink()); + assertEquals(EVENT_TYPES, NOTIFICATION_INFO.getEventTypes()); + assertEquals(OBJECT_NAME_PREFIX, NOTIFICATION_INFO.getObjectNamePrefix()); + assertEquals(PAYLOAD_FORMAT, NOTIFICATION_INFO.getPayloadFormat()); + assertEquals(TOPIC, NOTIFICATION_INFO.getTopic()); + assertEquals(CUSTOM_ATTRIBUTES, NOTIFICATION_INFO.getCustomAttributes()); + } + + @Test + public void testToPbAndFromPb() { + compareBuckets(NOTIFICATION_INFO, NotificationInfo.fromPb(NOTIFICATION_INFO.toPb())); + NotificationInfo bucketInfo = + NotificationInfo.of(TOPIC).toBuilder().setPayloadFormat(PayloadFormat.NONE).build(); + compareBuckets(bucketInfo, NotificationInfo.fromPb(bucketInfo.toPb())); + } + + private void compareBuckets(NotificationInfo expected, NotificationInfo value) { + assertEquals(expected, value); + assertEquals(expected.getGeneratedId(), value.getGeneratedId()); + assertEquals(expected.getCustomAttributes(), value.getCustomAttributes()); + assertEquals(expected.getEtag(), value.getEtag()); + assertEquals(expected.getSelfLink(), value.getSelfLink()); + assertEquals(expected.getEventTypes(), value.getEventTypes()); + assertEquals(expected.getObjectNamePrefix(), value.getObjectNamePrefix()); + assertEquals(expected.getPayloadFormat(), value.getPayloadFormat()); + assertEquals(expected.getTopic(), value.getTopic()); + } +} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 86bbdef79f..ec06f26489 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -31,6 +31,7 @@ import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.core.ApiClock; import com.google.api.gax.paging.Page; +import com.google.api.services.storage.model.Notification; import com.google.api.services.storage.model.Policy.Bindings; import com.google.api.services.storage.model.StorageObject; import com.google.api.services.storage.model.TestIamPermissionsResponse; @@ -56,6 +57,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.io.BaseEncoding; +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; @@ -72,6 +74,7 @@ import java.security.spec.InvalidKeySpecException; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.X509EncodedKeySpec; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -369,6 +372,38 @@ public long millisTime() { .put('~', "~") .build(); + // Notification + private static final String ETAG = "0xFF00"; + private static final String GENERATED_ID = "B/N:1"; + private static final String SELF_LINK = "http://storage/b/n"; + private static final List EVENT_TYPES = + ImmutableList.of("OBJECT_FINALIZE", "OBJECT_METADATA_UPDATE"); + private static final String OBJECT_NAME_PREFIX = "index.html"; + private static final NotificationInfo.PayloadFormat PAYLOAD_FORMAT = + NotificationInfo.PayloadFormat.JSON_API_V1.JSON_API_V1; + private static final String TOPIC = "projects/myProject/topics/topic1"; + private static final Map CUSTOM_ATTRIBUTES = ImmutableMap.of("label1", "value1"); + private static final NotificationInfo NOTIFICATION_INFO_01 = + NotificationInfo.newBuilder(TOPIC) + .setEtag(ETAG) + .setCustomAttributes(CUSTOM_ATTRIBUTES) + .setSelfLink(SELF_LINK) + .setEventTypes(EVENT_TYPES) + .setObjectNamePrefix(OBJECT_NAME_PREFIX) + .setPayloadFormat(PAYLOAD_FORMAT) + .setGeneratedId(GENERATED_ID) + .build(); + private static final NotificationInfo NOTIFICATION_INFO_02 = + NotificationInfo.newBuilder(TOPIC) + .setEtag(ETAG) + .setCustomAttributes(CUSTOM_ATTRIBUTES) + .setSelfLink(SELF_LINK) + .setEventTypes(EVENT_TYPES) + .setObjectNamePrefix(OBJECT_NAME_PREFIX) + .setPayloadFormat(PAYLOAD_FORMAT) + .setGeneratedId(GENERATED_ID) + .build(); + private static final String ACCOUNT = "account"; private static PrivateKey privateKey; private static PublicKey publicKey; @@ -382,7 +417,8 @@ public long millisTime() { private Bucket expectedBucket1, expectedBucket2, expectedBucket3; @BeforeClass - public static void beforeClass() throws NoSuchAlgorithmException, InvalidKeySpecException { + public static void beforeClass() + throws NoSuchAlgorithmException, InvalidKeySpecException, IOException { KeyFactory keyFactory = KeyFactory.getInstance("RSA"); EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(BaseEncoding.base64().decode(PRIVATE_KEY_STRING)); @@ -2471,4 +2507,56 @@ public void testV4PostPolicy() { assertEquals(outputFields.get("key"), "my-object"); assertEquals("https://storage.googleapis.com/my-bucket/", policy.getUrl()); } + + @Test + public void testCreateNotification() { + Notification notification = NOTIFICATION_INFO_01.toPb(); + EasyMock.expect(storageRpcMock.createNotification(BUCKET_NAME1, notification)) + .andReturn(notification); + EasyMock.replay(storageRpcMock); + initializeService(); + Notification remoteNotification = + storage.createNotification(BUCKET_NAME1, NOTIFICATION_INFO_01); + compareBucketsNotification(remoteNotification); + } + + @Test + public void testGetNotification() { + EasyMock.expect(storageRpcMock.getNotification(BUCKET_NAME1, GENERATED_ID)) + .andReturn(NOTIFICATION_INFO_01.toPb()); + EasyMock.replay(storageRpcMock); + initializeService(); + Notification notification = storage.getNotification(BUCKET_NAME1, GENERATED_ID); + compareBucketsNotification(notification); + } + + @Test + public void testListNotification() { + EasyMock.expect(storageRpcMock.listNotifications(BUCKET_NAME1)) + .andReturn(Arrays.asList(NOTIFICATION_INFO_01.toPb(), NOTIFICATION_INFO_02.toPb())); + EasyMock.replay(storageRpcMock); + initializeService(); + List notifications = storage.listNotifications(BUCKET_NAME1); + assertEquals(2, notifications.size()); + } + + @Test + public void testDeleteNotification() { + EasyMock.expect(storageRpcMock.deleteNotification(BUCKET_NAME1, GENERATED_ID)).andReturn(true); + EasyMock.replay(storageRpcMock); + initializeService(); + Boolean isDeleted = storage.deleteNotification(BUCKET_NAME1, GENERATED_ID); + assertEquals(isDeleted, Boolean.TRUE); + } + + private void compareBucketsNotification(Notification value) { + assertEquals(GENERATED_ID, value.getId()); + assertEquals(CUSTOM_ATTRIBUTES, value.getCustomAttributes()); + assertEquals(ETAG, value.getEtag()); + assertEquals(SELF_LINK, value.getSelfLink()); + assertEquals(EVENT_TYPES, value.getEventTypes()); + assertEquals(OBJECT_NAME_PREFIX, value.getObjectNamePrefix()); + assertEquals(PAYLOAD_FORMAT.name(), value.getPayloadFormat()); + assertEquals(TOPIC, value.getTopic()); + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 7966c9281c..38fff667cd 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -33,6 +33,7 @@ import com.google.api.client.http.apache.ApacheHttpTransport; import com.google.api.client.util.DateTime; import com.google.api.gax.paging.Page; +import com.google.api.services.storage.model.Notification; import com.google.auth.ServiceAccountSigner; import com.google.auth.http.HttpTransportFactory; import com.google.auth.oauth2.GoogleCredentials; @@ -41,6 +42,7 @@ import com.google.cloud.Policy; import com.google.cloud.ReadChannel; import com.google.cloud.RestorableState; +import com.google.cloud.ServiceOptions; import com.google.cloud.TransportOptions; import com.google.cloud.WriteChannel; import com.google.cloud.http.HttpTransportOptions; @@ -54,6 +56,7 @@ import com.google.cloud.kms.v1.KeyManagementServiceGrpc.KeyManagementServiceBlockingStub; import com.google.cloud.kms.v1.KeyRingName; import com.google.cloud.kms.v1.LocationName; +import com.google.cloud.pubsub.v1.TopicAdminClient; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Acl.Role; import com.google.cloud.storage.Acl.User; @@ -68,6 +71,7 @@ import com.google.cloud.storage.CopyWriter; import com.google.cloud.storage.HmacKey; import com.google.cloud.storage.HttpMethod; +import com.google.cloud.storage.NotificationInfo; import com.google.cloud.storage.PostPolicyV4; import com.google.cloud.storage.PostPolicyV4.PostFieldsV4; import com.google.cloud.storage.ServiceAccount; @@ -120,6 +124,7 @@ import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -142,6 +147,8 @@ public class ITStorageTest { private static RemoteStorageHelper remoteStorageHelper; private static Storage storage; + private static TopicAdminClient topicAdminClient; + private static Notification notification; private static String kmsKeyOneResourcePath; private static String kmsKeyTwoResourcePath; private static Metadata requestParamsHeader = new Metadata(); @@ -196,6 +203,15 @@ public class ITStorageTest { private static final ImmutableList LIFECYCLE_RULES = ImmutableList.of(LIFECYCLE_RULE_1, LIFECYCLE_RULE_2); + private static final String PROJECT = ServiceOptions.getDefaultProjectId(); + private static final String ID = UUID.randomUUID().toString().substring(0, 8); + private static final NotificationInfo.PayloadFormat PAYLOAD_FORMAT = + NotificationInfo.PayloadFormat.JSON_API_V1.JSON_API_V1; + + private static final String TOPIC = + String.format("projects/%s/topics/test_topic_foo_%s", PROJECT, ID); + private static final Map CUSTOM_ATTRIBUTES = ImmutableMap.of("label1", "value1"); + @BeforeClass public static void beforeClass() throws IOException { remoteStorageHelper = RemoteStorageHelper.create(); @@ -213,6 +229,22 @@ public static void beforeClass() throws IOException { // Prepare KMS KeyRing for CMEK tests prepareKmsKeys(); + + // Create pubsub topic for notification. + topicAdminClient = TopicAdminClient.create(); + topicAdminClient.createTopic(TOPIC); + com.google.iam.v1.Policy policy = topicAdminClient.getIamPolicy(TOPIC); + Binding binding = + Binding.newBuilder().setRole("roles/owner").addMembers("allAuthenticatedUsers").build(); + topicAdminClient.setIamPolicy(TOPIC, policy.toBuilder().addBindings(binding).build()); + + // Create a notification on a bucket. + NotificationInfo notificationInfo = + NotificationInfo.newBuilder(TOPIC) + .setCustomAttributes(CUSTOM_ATTRIBUTES) + .setPayloadFormat(PAYLOAD_FORMAT) + .build(); + notification = storage.createNotification(BUCKET, notificationInfo); } @Before @@ -235,6 +267,13 @@ public void beforeEach() { @AfterClass public static void afterClass() throws ExecutionException, InterruptedException { if (storage != null) { + + /* Delete the specified notification on the specified bucket as well as delete the pubsub topic */ + if (topicAdminClient != null) { + storage.deleteNotification(BUCKET, notification.getId()); + topicAdminClient.deleteTopic(TOPIC); + topicAdminClient.close(); + } // In beforeClass, we make buckets auto-delete blobs older than a day old. // Here, delete all buckets older than 2 days. They should already be empty and easy. long cleanTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(2); @@ -3322,4 +3361,24 @@ public void testDeleteLifecycleRules() throws ExecutionException, InterruptedExc RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } } + + @Test + public void testGetNotification() { + Notification actualNotification = storage.getNotification(BUCKET, notification.getId()); + assertEquals(CUSTOM_ATTRIBUTES, actualNotification.getCustomAttributes()); + assertEquals(PAYLOAD_FORMAT.name(), actualNotification.getPayloadFormat()); + assertTrue(actualNotification.getTopic().contains(TOPIC)); + } + + @Test + public void testListNotification() { + List notifications = storage.listNotifications(BUCKET); + for (Notification actualNotification : notifications) { + if (actualNotification.getId().equals(notification.getId())) { + assertEquals(CUSTOM_ATTRIBUTES, actualNotification.getCustomAttributes()); + assertEquals(PAYLOAD_FORMAT.name(), actualNotification.getPayloadFormat()); + assertTrue(actualNotification.getTopic().contains(TOPIC)); + } + } + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/testing/StorageRpcTestBaseTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/testing/StorageRpcTestBaseTest.java index b798ad572b..7feef9f183 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/testing/StorageRpcTestBaseTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/testing/StorageRpcTestBaseTest.java @@ -577,6 +577,17 @@ public Notification call() { }; } + @Test + public void testGetNotification() { + rpc = + new Callable() { + @Override + public Notification call() { + return STORAGE_RPC.getNotification("bucket", "notification"); + } + }; + } + @Test public void testLockRetentionPolicy() { rpc = diff --git a/pom.xml b/pom.xml index 02c4dd3949..aa0146a4c7 100644 --- a/pom.xml +++ b/pom.xml @@ -80,6 +80,12 @@ google-api-services-storage v1-rev20200430-1.30.9 + + com.google.cloud + google-cloud-pubsub + 1.106.0 + test + org.easymock easymock From 3e02b9c4ee1ce0fb785d15b04bd36754e31831a0 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Thu, 25 Jun 2020 10:23:14 -0600 Subject: [PATCH 5/8] Revert "feat: expose all the methods of notification (#141)" (#393) This reverts commit 8dfc0cbf8294a7fc426948e22e5c2182da97b630. --- .../clirr-ignored-differences.xml | 25 -- google-cloud-storage/pom.xml | 5 - .../cloud/storage/NotificationInfo.java | 309 ------------------ .../com/google/cloud/storage/Storage.java | 33 -- .../com/google/cloud/storage/StorageImpl.java | 74 ----- .../cloud/storage/spi/v1/HttpStorageRpc.java | 15 - .../storage/spi/v1/HttpStorageRpcSpans.java | 2 - .../cloud/storage/spi/v1/StorageRpc.java | 8 - .../storage/testing/StorageRpcTestBase.java | 5 - .../cloud/storage/NotificationInfoTest.java | 102 ------ .../google/cloud/storage/StorageImplTest.java | 90 +---- .../cloud/storage/it/ITStorageTest.java | 59 ---- .../testing/StorageRpcTestBaseTest.java | 11 - pom.xml | 6 - 14 files changed, 1 insertion(+), 743 deletions(-) delete mode 100644 google-cloud-storage/src/main/java/com/google/cloud/storage/NotificationInfo.java delete mode 100644 google-cloud-storage/src/test/java/com/google/cloud/storage/NotificationInfoTest.java diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index 2aababb9e7..ace7a6ef0e 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -26,29 +26,4 @@ com.google.cloud.storage.BucketInfo$Builder deleteLifecycleRules() 7013 - - com/google/cloud/storage/Storage - com.google.api.services.storage.model.Notification createNotification(java.lang.String, com.google.cloud.storage.NotificationInfo) - 7012 - - - com/google/cloud/storage/Storage - com.google.api.services.storage.model.Notification getNotification(java.lang.String, java.lang.String) - 7012 - - - com/google/cloud/storage/Storage - java.util.List listNotifications(java.lang.String) - 7012 - - - com/google/cloud/storage/Storage - boolean deleteNotification(java.lang.String, java.lang.String) - 7012 - - - com/google/cloud/storage/spi/v1/StorageRpc - com.google.api.services.storage.model.Notification getNotification(java.lang.String, java.lang.String) - 7012 - diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 1be67d6b39..3e2dbb0ba2 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -155,11 +155,6 @@ truth test - - com.google.cloud - google-cloud-pubsub - test - org.easymock easymock diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/NotificationInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/NotificationInfo.java deleted file mode 100644 index 34c2d326d6..0000000000 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/NotificationInfo.java +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.storage; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.api.pathtemplate.PathTemplate; -import com.google.api.services.storage.model.Notification; -import com.google.common.base.Function; -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import java.io.Serializable; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * Google Storage Notification metadata; - * - * @see Concepts and - * Terminology - */ -public class NotificationInfo implements Serializable { - - private static final long serialVersionUID = 5725883368559753810L; - private static final PathTemplate PATH_TEMPLATE = - PathTemplate.createWithoutUrlEncoding("projects/{project}/topics/{topic}"); - - public enum PayloadFormat { - JSON_API_V1, - NONE - } - - static final Function FROM_PB_FUNCTION = - new Function() { - @Override - public NotificationInfo apply(Notification pb) { - return NotificationInfo.fromPb(pb); - } - }; - static final Function TO_PB_FUNCTION = - new Function() { - @Override - public Notification apply(NotificationInfo NotificationInfo) { - return NotificationInfo.toPb(); - } - }; - private final String generatedId; - private final String topic; - private final List eventTypes; - private final Map customAttributes; - private final PayloadFormat payloadFormat; - private final String objectNamePrefix; - private final String etag; - private final String selfLink; - - public static final class Builder { - - private String generatedId; - private String topic; - private List eventTypes; - private Map customAttributes; - private PayloadFormat payloadFormat; - private String objectNamePrefix; - private String etag; - private String selfLink; - - Builder(String topic) { - this.topic = topic; - } - - Builder(NotificationInfo NotificationInfo) { - generatedId = NotificationInfo.generatedId; - etag = NotificationInfo.etag; - selfLink = NotificationInfo.selfLink; - topic = NotificationInfo.topic; - eventTypes = NotificationInfo.eventTypes; - customAttributes = NotificationInfo.customAttributes; - payloadFormat = NotificationInfo.payloadFormat; - objectNamePrefix = NotificationInfo.objectNamePrefix; - } - - Builder setGeneratedId(String generatedId) { - this.generatedId = generatedId; - return this; - } - - Builder setSelfLink(String selfLink) { - this.selfLink = selfLink; - return this; - } - - /** The name of the topic. It must have the format "projects/{project}/topics/{topic}". */ - public Builder setTopic(String topic) { - this.topic = topic; - return this; - } - - public Builder setPayloadFormat(PayloadFormat payloadFormat) { - this.payloadFormat = payloadFormat; - return this; - } - - public Builder setObjectNamePrefix(String objectNamePrefix) { - this.objectNamePrefix = objectNamePrefix; - return this; - } - - public Builder setEventTypes(Iterable eventTypes) { - this.eventTypes = eventTypes != null ? ImmutableList.copyOf(eventTypes) : null; - return this; - } - - Builder setEtag(String etag) { - this.etag = etag; - return this; - } - - public Builder setCustomAttributes(Map customAttributes) { - this.customAttributes = - customAttributes != null ? ImmutableMap.copyOf(customAttributes) : null; - return this; - } - - public NotificationInfo build() { - checkNotNull(topic); - return new NotificationInfo(this); - } - } - - NotificationInfo(Builder builder) { - generatedId = builder.generatedId; - etag = builder.etag; - selfLink = builder.selfLink; - topic = builder.topic; - eventTypes = builder.eventTypes; - customAttributes = builder.customAttributes; - payloadFormat = builder.payloadFormat; - objectNamePrefix = builder.objectNamePrefix; - } - - /** Returns the service-generated id for the notification. */ - public String getGeneratedId() { - return generatedId; - } - - /** Returns the topic to which this subscription publishes. */ - public String getTopic() { - return topic; - } - - /** Returns the canonical URI of this topic as a string. */ - public String getSelfLink() { - return selfLink; - } - - /** Returns the desired content of the Payload. */ - public PayloadFormat getPayloadFormat() { - return payloadFormat; - } - - /** Returns the object name prefix for which this notification configuration applies. */ - public String getObjectNamePrefix() { - return objectNamePrefix; - } - - /** - * Returns HTTP 1.1 Entity tag for the notification. - * - * @see Entity Tags - */ - public String getEtag() { - return etag; - } - - /** - * Returns the list of event types that this notification will apply to. If empty, notifications - * will be sent on all event types. - * - * @see Cross-Origin Resource Sharing - * (CORS) - */ - public List getEventTypes() { - return eventTypes; - } - - /** - * Returns the list of additional attributes to attach to each Cloud PubSub message published for - * this notification subscription. - * - * @see - * About Access Control Lists - */ - public Map getCustomAttributes() { - return customAttributes; - } - - /** Returns a builder for the current notification. */ - public Builder toBuilder() { - return new Builder(this); - } - - @Override - public int hashCode() { - return Objects.hash(getTopic()); - } - - @Override - public boolean equals(Object obj) { - return obj == this - || obj != null - && obj.getClass().equals(NotificationInfo.class) - && Objects.equals(toPb(), ((NotificationInfo) obj).toPb()); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("topic", getTopic()).toString(); - } - - Notification toPb() { - Notification notificationPb = new Notification(); - notificationPb.setId(generatedId); - notificationPb.setEtag(etag); - if (customAttributes != null) { - notificationPb.setCustomAttributes(customAttributes); - } - if (eventTypes != null) { - notificationPb.setEventTypes(eventTypes); - } - if (objectNamePrefix != null) { - notificationPb.setObjectNamePrefix(objectNamePrefix); - } - if (payloadFormat != null) { - notificationPb.setPayloadFormat(payloadFormat.toString()); - } else { - notificationPb.setPayloadFormat(PayloadFormat.NONE.toString()); - } - notificationPb.setSelfLink(selfLink); - notificationPb.setTopic(topic); - - return notificationPb; - } - - /** - * Creates a {@code NotificationInfo} object for the provided topic name. - * - * @param topic The name of the topic. It must have the format - * "projects/{project}/topics/{topic}". - */ - public static NotificationInfo of(String topic) { - PATH_TEMPLATE.validatedMatch(topic, "topic name must be in valid format"); - return newBuilder(topic).build(); - } - - /** - * Returns a {@code NotificationInfo} builder where the topic's name is set to the provided name. - * - * @param topic The name of the topic. It must have the format - * "projects/{project}/topics/{topic}". - */ - public static Builder newBuilder(String topic) { - PATH_TEMPLATE.validatedMatch(topic, "topic name must be in valid format"); - return new Builder(topic); - } - - static NotificationInfo fromPb(Notification notificationPb) { - Builder builder = newBuilder(notificationPb.getTopic()); - if (notificationPb.getId() != null) { - builder.setGeneratedId(notificationPb.getId()); - } - if (notificationPb.getEtag() != null) { - builder.setEtag(notificationPb.getEtag()); - } - if (notificationPb.getCustomAttributes() != null) { - builder.setCustomAttributes(notificationPb.getCustomAttributes()); - } - if (notificationPb.getSelfLink() != null) { - builder.setSelfLink(notificationPb.getSelfLink()); - } - if (notificationPb.getObjectNamePrefix() != null) { - builder.setObjectNamePrefix(notificationPb.getObjectNamePrefix()); - } - if (notificationPb.getTopic() != null) { - builder.setTopic(notificationPb.getTopic()); - } - if (notificationPb.getEventTypes() != null) { - builder.setEventTypes(notificationPb.getEventTypes()); - } - if (notificationPb.getPayloadFormat() != null) { - builder.setPayloadFormat(PayloadFormat.valueOf(notificationPb.getPayloadFormat())); - } - return builder.build(); - } -} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index fa196c90da..988f87d39c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -21,7 +21,6 @@ import com.google.api.core.InternalExtensionOnly; import com.google.api.gax.paging.Page; -import com.google.api.services.storage.model.Notification; import com.google.auth.ServiceAccountSigner; import com.google.auth.ServiceAccountSigner.SigningException; import com.google.cloud.FieldSelector; @@ -3434,36 +3433,4 @@ List testIamPermissions( * @throws StorageException upon failure */ ServiceAccount getServiceAccount(String projectId); - - /** - * Creates a notification with the specified entity on the specified bucket. - * - * @return the notification that was created. - * @throws StorageException upon failure - */ - Notification createNotification(String bucket, NotificationInfo notification); - - /** - * Get the notification with the specified name on the specified object. - * - * @return the notification object that exist on the bucket. - * @throws StorageException upon failure - */ - Notification getNotification(String bucket, String notification); - - /** - * List the notifications for the provided bucket. - * - * @return a list of {@link Notification} objects that exist on the bucket. - * @throws StorageException upon failure - */ - List listNotifications(String bucket); - - /** - * Deletes the notification with the specified name on the specified object. - * - * @return {@code true} if the notification was deleted, {@code false} if it was not found - * @throws StorageException upon failure - */ - boolean deleteNotification(String bucket, String notification); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index ef46249d05..0e24521eb6 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -36,7 +36,6 @@ import com.google.api.gax.paging.Page; import com.google.api.services.storage.model.BucketAccessControl; -import com.google.api.services.storage.model.Notification; import com.google.api.services.storage.model.ObjectAccessControl; import com.google.api.services.storage.model.StorageObject; import com.google.api.services.storage.model.TestIamPermissionsResponse; @@ -1699,79 +1698,6 @@ public com.google.api.services.storage.model.ServiceAccount call() { } } - @Override - public Notification createNotification(final String bucket, final NotificationInfo notification) { - final com.google.api.services.storage.model.Notification notificationPb = notification.toPb(); - try { - return runWithRetries( - new Callable() { - @Override - public com.google.api.services.storage.model.Notification call() { - return storageRpc.createNotification(bucket, notificationPb); - } - }, - getOptions().getRetrySettings(), - EXCEPTION_HANDLER, - getOptions().getClock()); - } catch (RetryHelperException e) { - throw StorageException.translateAndThrow(e); - } - } - - @Override - public Notification getNotification(final String bucket, final String notification) { - try { - return runWithRetries( - new Callable() { - @Override - public Notification call() { - return storageRpc.getNotification(bucket, notification); - } - }, - getOptions().getRetrySettings(), - EXCEPTION_HANDLER, - getOptions().getClock()); - } catch (RetryHelperException e) { - throw StorageException.translateAndThrow(e); - } - } - - @Override - public List listNotifications(final String bucket) { - try { - return runWithRetries( - new Callable>() { - @Override - public List call() { - return storageRpc.listNotifications(bucket); - } - }, - getOptions().getRetrySettings(), - EXCEPTION_HANDLER, - getOptions().getClock()); - } catch (RetryHelperException e) { - throw StorageException.translateAndThrow(e); - } - } - - @Override - public boolean deleteNotification(final String bucket, final String notification) { - try { - return runWithRetries( - new Callable() { - @Override - public Boolean call() { - return storageRpc.deleteNotification(bucket, notification); - } - }, - getOptions().getRetrySettings(), - EXCEPTION_HANDLER, - getOptions().getClock()); - } catch (RetryHelperException e) { - throw StorageException.translateAndThrow(e); - } - } - private static void addToOptionMap( StorageRpc.Option option, T defaultValue, Map map) { addToOptionMap(option, option, defaultValue, map); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 97ceafa328..e211c0277e 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -1489,21 +1489,6 @@ public Notification createNotification(String bucket, Notification notification) } } - @Override - public Notification getNotification(String bucket, String notification) { - Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_GET_NOTIFICATION); - Scope scope = tracer.withSpan(span); - try { - return storage.notifications().get(bucket, notification).execute(); - } catch (IOException ex) { - span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); - throw translate(ex); - } finally { - scope.close(); - span.end(); - } - } - @Override public Bucket lockRetentionPolicy(Bucket bucket, Map options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_LOCK_RETENTION_POLICY); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java index 11a33d8d02..a5e4daf7da 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpcSpans.java @@ -89,8 +89,6 @@ class HttpStorageRpcSpans { static final String SPAN_NAME_LIST_NOTIFICATIONS = getTraceSpanName("listNotifications(String)"); static final String SPAN_NAME_CREATE_NOTIFICATION = getTraceSpanName("createNotification(String,Notification)"); - static final String SPAN_NAME_GET_NOTIFICATION = - getTraceSpanName("getNotification(String,String)"); static final String SPAN_LOCK_RETENTION_POLICY = getTraceSpanName("lockRetentionPolicy(String,Long)"); static final String SPAN_NAME_GET_SERVICE_ACCOUNT = getTraceSpanName("getServiceAccount(String)"); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index 0291d920bf..36c7a5ff1b 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -532,14 +532,6 @@ TestIamPermissionsResponse testIamPermissions( */ Notification createNotification(String bucket, Notification notification); - /** - * Get the notification with the specified name on the specified object. - * - * @return the notification object that exist on the bucket. - * @throws StorageException upon failure - */ - Notification getNotification(String bucket, String notification); - /** * Lock retention policy for the provided bucket. * diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java index 0f8e651a6a..6bd2d487ea 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java @@ -280,11 +280,6 @@ public Notification createNotification(String bucket, Notification notification) throw new UnsupportedOperationException("Not implemented yet"); } - @Override - public Notification getNotification(String bucket, String notification) { - throw new UnsupportedOperationException("Not implemented yet"); - } - @Override public Bucket lockRetentionPolicy(Bucket bucket, Map options) { throw new UnsupportedOperationException("Not implemented yet"); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/NotificationInfoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/NotificationInfoTest.java deleted file mode 100644 index 40b8a5ccf1..0000000000 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/NotificationInfoTest.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.storage; - -import static org.junit.Assert.assertEquals; - -import com.google.cloud.storage.NotificationInfo.PayloadFormat; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import java.util.List; -import java.util.Map; -import org.junit.Test; - -public class NotificationInfoTest { - - private static final String ETAG = "0xFF00"; - private static final String GENERATED_ID = "B/N:1"; - private static final String SELF_LINK = "http://storage/b/n"; - private static final List EVENT_TYPES = - ImmutableList.of("OBJECT_FINALIZE", "OBJECT_METADATA_UPDATE"); - private static final String OBJECT_NAME_PREFIX = "index.html"; - private static final PayloadFormat PAYLOAD_FORMAT = PayloadFormat.JSON_API_V1.JSON_API_V1; - private static final String TOPIC = "projects/myProject/topics/topic1"; - private static final Map CUSTOM_ATTRIBUTES = ImmutableMap.of("label1", "value1"); - private static final NotificationInfo NOTIFICATION_INFO = - NotificationInfo.newBuilder(TOPIC) - .setEtag(ETAG) - .setCustomAttributes(CUSTOM_ATTRIBUTES) - .setSelfLink(SELF_LINK) - .setEventTypes(EVENT_TYPES) - .setObjectNamePrefix(OBJECT_NAME_PREFIX) - .setPayloadFormat(PAYLOAD_FORMAT) - .setGeneratedId(GENERATED_ID) - .build(); - - @Test - public void testToBuilder() { - compareBuckets(NOTIFICATION_INFO, NOTIFICATION_INFO.toBuilder().build()); - NotificationInfo bucketInfo = NOTIFICATION_INFO.toBuilder().setGeneratedId("id").build(); - assertEquals("id", bucketInfo.getGeneratedId()); - bucketInfo = bucketInfo.toBuilder().setGeneratedId(GENERATED_ID).build(); - compareBuckets(NOTIFICATION_INFO, bucketInfo); - } - - @Test - public void testToBuilderIncomplete() { - NotificationInfo incompleteBucketInfo = NotificationInfo.newBuilder(TOPIC).build(); - compareBuckets(incompleteBucketInfo, incompleteBucketInfo.toBuilder().build()); - } - - @Test - public void testOf() { - NotificationInfo bucketInfo = NotificationInfo.of(TOPIC); - assertEquals(TOPIC, bucketInfo.getTopic()); - } - - @Test - public void testBuilder() { - assertEquals(ETAG, NOTIFICATION_INFO.getEtag()); - assertEquals(GENERATED_ID, NOTIFICATION_INFO.getGeneratedId()); - assertEquals(SELF_LINK, NOTIFICATION_INFO.getSelfLink()); - assertEquals(EVENT_TYPES, NOTIFICATION_INFO.getEventTypes()); - assertEquals(OBJECT_NAME_PREFIX, NOTIFICATION_INFO.getObjectNamePrefix()); - assertEquals(PAYLOAD_FORMAT, NOTIFICATION_INFO.getPayloadFormat()); - assertEquals(TOPIC, NOTIFICATION_INFO.getTopic()); - assertEquals(CUSTOM_ATTRIBUTES, NOTIFICATION_INFO.getCustomAttributes()); - } - - @Test - public void testToPbAndFromPb() { - compareBuckets(NOTIFICATION_INFO, NotificationInfo.fromPb(NOTIFICATION_INFO.toPb())); - NotificationInfo bucketInfo = - NotificationInfo.of(TOPIC).toBuilder().setPayloadFormat(PayloadFormat.NONE).build(); - compareBuckets(bucketInfo, NotificationInfo.fromPb(bucketInfo.toPb())); - } - - private void compareBuckets(NotificationInfo expected, NotificationInfo value) { - assertEquals(expected, value); - assertEquals(expected.getGeneratedId(), value.getGeneratedId()); - assertEquals(expected.getCustomAttributes(), value.getCustomAttributes()); - assertEquals(expected.getEtag(), value.getEtag()); - assertEquals(expected.getSelfLink(), value.getSelfLink()); - assertEquals(expected.getEventTypes(), value.getEventTypes()); - assertEquals(expected.getObjectNamePrefix(), value.getObjectNamePrefix()); - assertEquals(expected.getPayloadFormat(), value.getPayloadFormat()); - assertEquals(expected.getTopic(), value.getTopic()); - } -} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index ec06f26489..86bbdef79f 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -31,7 +31,6 @@ import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.core.ApiClock; import com.google.api.gax.paging.Page; -import com.google.api.services.storage.model.Notification; import com.google.api.services.storage.model.Policy.Bindings; import com.google.api.services.storage.model.StorageObject; import com.google.api.services.storage.model.TestIamPermissionsResponse; @@ -57,7 +56,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.io.BaseEncoding; -import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; @@ -74,7 +72,6 @@ import java.security.spec.InvalidKeySpecException; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.X509EncodedKeySpec; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -372,38 +369,6 @@ public long millisTime() { .put('~', "~") .build(); - // Notification - private static final String ETAG = "0xFF00"; - private static final String GENERATED_ID = "B/N:1"; - private static final String SELF_LINK = "http://storage/b/n"; - private static final List EVENT_TYPES = - ImmutableList.of("OBJECT_FINALIZE", "OBJECT_METADATA_UPDATE"); - private static final String OBJECT_NAME_PREFIX = "index.html"; - private static final NotificationInfo.PayloadFormat PAYLOAD_FORMAT = - NotificationInfo.PayloadFormat.JSON_API_V1.JSON_API_V1; - private static final String TOPIC = "projects/myProject/topics/topic1"; - private static final Map CUSTOM_ATTRIBUTES = ImmutableMap.of("label1", "value1"); - private static final NotificationInfo NOTIFICATION_INFO_01 = - NotificationInfo.newBuilder(TOPIC) - .setEtag(ETAG) - .setCustomAttributes(CUSTOM_ATTRIBUTES) - .setSelfLink(SELF_LINK) - .setEventTypes(EVENT_TYPES) - .setObjectNamePrefix(OBJECT_NAME_PREFIX) - .setPayloadFormat(PAYLOAD_FORMAT) - .setGeneratedId(GENERATED_ID) - .build(); - private static final NotificationInfo NOTIFICATION_INFO_02 = - NotificationInfo.newBuilder(TOPIC) - .setEtag(ETAG) - .setCustomAttributes(CUSTOM_ATTRIBUTES) - .setSelfLink(SELF_LINK) - .setEventTypes(EVENT_TYPES) - .setObjectNamePrefix(OBJECT_NAME_PREFIX) - .setPayloadFormat(PAYLOAD_FORMAT) - .setGeneratedId(GENERATED_ID) - .build(); - private static final String ACCOUNT = "account"; private static PrivateKey privateKey; private static PublicKey publicKey; @@ -417,8 +382,7 @@ public long millisTime() { private Bucket expectedBucket1, expectedBucket2, expectedBucket3; @BeforeClass - public static void beforeClass() - throws NoSuchAlgorithmException, InvalidKeySpecException, IOException { + public static void beforeClass() throws NoSuchAlgorithmException, InvalidKeySpecException { KeyFactory keyFactory = KeyFactory.getInstance("RSA"); EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(BaseEncoding.base64().decode(PRIVATE_KEY_STRING)); @@ -2507,56 +2471,4 @@ public void testV4PostPolicy() { assertEquals(outputFields.get("key"), "my-object"); assertEquals("https://storage.googleapis.com/my-bucket/", policy.getUrl()); } - - @Test - public void testCreateNotification() { - Notification notification = NOTIFICATION_INFO_01.toPb(); - EasyMock.expect(storageRpcMock.createNotification(BUCKET_NAME1, notification)) - .andReturn(notification); - EasyMock.replay(storageRpcMock); - initializeService(); - Notification remoteNotification = - storage.createNotification(BUCKET_NAME1, NOTIFICATION_INFO_01); - compareBucketsNotification(remoteNotification); - } - - @Test - public void testGetNotification() { - EasyMock.expect(storageRpcMock.getNotification(BUCKET_NAME1, GENERATED_ID)) - .andReturn(NOTIFICATION_INFO_01.toPb()); - EasyMock.replay(storageRpcMock); - initializeService(); - Notification notification = storage.getNotification(BUCKET_NAME1, GENERATED_ID); - compareBucketsNotification(notification); - } - - @Test - public void testListNotification() { - EasyMock.expect(storageRpcMock.listNotifications(BUCKET_NAME1)) - .andReturn(Arrays.asList(NOTIFICATION_INFO_01.toPb(), NOTIFICATION_INFO_02.toPb())); - EasyMock.replay(storageRpcMock); - initializeService(); - List notifications = storage.listNotifications(BUCKET_NAME1); - assertEquals(2, notifications.size()); - } - - @Test - public void testDeleteNotification() { - EasyMock.expect(storageRpcMock.deleteNotification(BUCKET_NAME1, GENERATED_ID)).andReturn(true); - EasyMock.replay(storageRpcMock); - initializeService(); - Boolean isDeleted = storage.deleteNotification(BUCKET_NAME1, GENERATED_ID); - assertEquals(isDeleted, Boolean.TRUE); - } - - private void compareBucketsNotification(Notification value) { - assertEquals(GENERATED_ID, value.getId()); - assertEquals(CUSTOM_ATTRIBUTES, value.getCustomAttributes()); - assertEquals(ETAG, value.getEtag()); - assertEquals(SELF_LINK, value.getSelfLink()); - assertEquals(EVENT_TYPES, value.getEventTypes()); - assertEquals(OBJECT_NAME_PREFIX, value.getObjectNamePrefix()); - assertEquals(PAYLOAD_FORMAT.name(), value.getPayloadFormat()); - assertEquals(TOPIC, value.getTopic()); - } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 38fff667cd..7966c9281c 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -33,7 +33,6 @@ import com.google.api.client.http.apache.ApacheHttpTransport; import com.google.api.client.util.DateTime; import com.google.api.gax.paging.Page; -import com.google.api.services.storage.model.Notification; import com.google.auth.ServiceAccountSigner; import com.google.auth.http.HttpTransportFactory; import com.google.auth.oauth2.GoogleCredentials; @@ -42,7 +41,6 @@ import com.google.cloud.Policy; import com.google.cloud.ReadChannel; import com.google.cloud.RestorableState; -import com.google.cloud.ServiceOptions; import com.google.cloud.TransportOptions; import com.google.cloud.WriteChannel; import com.google.cloud.http.HttpTransportOptions; @@ -56,7 +54,6 @@ import com.google.cloud.kms.v1.KeyManagementServiceGrpc.KeyManagementServiceBlockingStub; import com.google.cloud.kms.v1.KeyRingName; import com.google.cloud.kms.v1.LocationName; -import com.google.cloud.pubsub.v1.TopicAdminClient; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Acl.Role; import com.google.cloud.storage.Acl.User; @@ -71,7 +68,6 @@ import com.google.cloud.storage.CopyWriter; import com.google.cloud.storage.HmacKey; import com.google.cloud.storage.HttpMethod; -import com.google.cloud.storage.NotificationInfo; import com.google.cloud.storage.PostPolicyV4; import com.google.cloud.storage.PostPolicyV4.PostFieldsV4; import com.google.cloud.storage.ServiceAccount; @@ -124,7 +120,6 @@ import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -147,8 +142,6 @@ public class ITStorageTest { private static RemoteStorageHelper remoteStorageHelper; private static Storage storage; - private static TopicAdminClient topicAdminClient; - private static Notification notification; private static String kmsKeyOneResourcePath; private static String kmsKeyTwoResourcePath; private static Metadata requestParamsHeader = new Metadata(); @@ -203,15 +196,6 @@ public class ITStorageTest { private static final ImmutableList LIFECYCLE_RULES = ImmutableList.of(LIFECYCLE_RULE_1, LIFECYCLE_RULE_2); - private static final String PROJECT = ServiceOptions.getDefaultProjectId(); - private static final String ID = UUID.randomUUID().toString().substring(0, 8); - private static final NotificationInfo.PayloadFormat PAYLOAD_FORMAT = - NotificationInfo.PayloadFormat.JSON_API_V1.JSON_API_V1; - - private static final String TOPIC = - String.format("projects/%s/topics/test_topic_foo_%s", PROJECT, ID); - private static final Map CUSTOM_ATTRIBUTES = ImmutableMap.of("label1", "value1"); - @BeforeClass public static void beforeClass() throws IOException { remoteStorageHelper = RemoteStorageHelper.create(); @@ -229,22 +213,6 @@ public static void beforeClass() throws IOException { // Prepare KMS KeyRing for CMEK tests prepareKmsKeys(); - - // Create pubsub topic for notification. - topicAdminClient = TopicAdminClient.create(); - topicAdminClient.createTopic(TOPIC); - com.google.iam.v1.Policy policy = topicAdminClient.getIamPolicy(TOPIC); - Binding binding = - Binding.newBuilder().setRole("roles/owner").addMembers("allAuthenticatedUsers").build(); - topicAdminClient.setIamPolicy(TOPIC, policy.toBuilder().addBindings(binding).build()); - - // Create a notification on a bucket. - NotificationInfo notificationInfo = - NotificationInfo.newBuilder(TOPIC) - .setCustomAttributes(CUSTOM_ATTRIBUTES) - .setPayloadFormat(PAYLOAD_FORMAT) - .build(); - notification = storage.createNotification(BUCKET, notificationInfo); } @Before @@ -267,13 +235,6 @@ public void beforeEach() { @AfterClass public static void afterClass() throws ExecutionException, InterruptedException { if (storage != null) { - - /* Delete the specified notification on the specified bucket as well as delete the pubsub topic */ - if (topicAdminClient != null) { - storage.deleteNotification(BUCKET, notification.getId()); - topicAdminClient.deleteTopic(TOPIC); - topicAdminClient.close(); - } // In beforeClass, we make buckets auto-delete blobs older than a day old. // Here, delete all buckets older than 2 days. They should already be empty and easy. long cleanTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(2); @@ -3361,24 +3322,4 @@ public void testDeleteLifecycleRules() throws ExecutionException, InterruptedExc RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } } - - @Test - public void testGetNotification() { - Notification actualNotification = storage.getNotification(BUCKET, notification.getId()); - assertEquals(CUSTOM_ATTRIBUTES, actualNotification.getCustomAttributes()); - assertEquals(PAYLOAD_FORMAT.name(), actualNotification.getPayloadFormat()); - assertTrue(actualNotification.getTopic().contains(TOPIC)); - } - - @Test - public void testListNotification() { - List notifications = storage.listNotifications(BUCKET); - for (Notification actualNotification : notifications) { - if (actualNotification.getId().equals(notification.getId())) { - assertEquals(CUSTOM_ATTRIBUTES, actualNotification.getCustomAttributes()); - assertEquals(PAYLOAD_FORMAT.name(), actualNotification.getPayloadFormat()); - assertTrue(actualNotification.getTopic().contains(TOPIC)); - } - } - } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/testing/StorageRpcTestBaseTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/testing/StorageRpcTestBaseTest.java index 7feef9f183..b798ad572b 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/testing/StorageRpcTestBaseTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/testing/StorageRpcTestBaseTest.java @@ -577,17 +577,6 @@ public Notification call() { }; } - @Test - public void testGetNotification() { - rpc = - new Callable() { - @Override - public Notification call() { - return STORAGE_RPC.getNotification("bucket", "notification"); - } - }; - } - @Test public void testLockRetentionPolicy() { rpc = diff --git a/pom.xml b/pom.xml index aa0146a4c7..02c4dd3949 100644 --- a/pom.xml +++ b/pom.xml @@ -80,12 +80,6 @@ google-api-services-storage v1-rev20200430-1.30.9 - - com.google.cloud - google-cloud-pubsub - 1.106.0 - test - org.easymock easymock From 9457f3a76ff18552adc5f9c82f62ab8f3c207d31 Mon Sep 17 00:00:00 2001 From: Dmitry <58846611+dmitry-fa@users.noreply.github.com> Date: Thu, 25 Jun 2020 19:24:46 +0300 Subject: [PATCH 6/8] feat: add storage.upload(path) (#269) * feat: add storage.upload(path) * feat: use mockito framework * feat: update upload to return blob * feat: do not parse response from writers for signed urls * fix reviewer's comments * Update Storage.upload() functionality after merge * feat: deprecate StorageRpc.write(), introduce StorageRpc.upload() * feat: deprecate StorageRpc.write(), introduce StorageRpc.upload() * rename upload to createFrom --- .../clirr-ignored-differences.xml | 15 +- .../cloud/storage/BlobWriteChannel.java | 16 +- .../com/google/cloud/storage/Storage.java | 122 +++++++++++- .../com/google/cloud/storage/StorageImpl.java | 64 ++++++ .../cloud/storage/spi/v1/HttpStorageRpc.java | 25 ++- .../cloud/storage/spi/v1/StorageRpc.java | 21 ++ .../storage/testing/StorageRpcTestBase.java | 11 ++ .../cloud/storage/BlobWriteChannelTest.java | 139 +++++++++---- .../cloud/storage/StorageImplMockitoTest.java | 183 ++++++++++++++++++ .../google/cloud/storage/StorageImplTest.java | 3 +- .../cloud/storage/it/ITStorageTest.java | 42 ++++ 11 files changed, 589 insertions(+), 52 deletions(-) diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index ace7a6ef0e..bca2faaff1 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -2,24 +2,19 @@ - com/google/cloud/storage/Storage - com.google.cloud.storage.PostPolicyV4 generateSignedPostPolicyV4(com.google.cloud.storage.BlobInfo, long, java.util.concurrent.TimeUnit, com.google.cloud.storage.PostPolicyV4$PostFieldsV4, com.google.cloud.storage.PostPolicyV4$PostConditionsV4, com.google.cloud.storage.Storage$PostPolicyV4Option[]) 7012 - - com/google/cloud/storage/Storage - com.google.cloud.storage.PostPolicyV4 generateSignedPostPolicyV4(com.google.cloud.storage.BlobInfo, long, java.util.concurrent.TimeUnit, com.google.cloud.storage.PostPolicyV4$PostFieldsV4, com.google.cloud.storage.Storage$PostPolicyV4Option[]) - 7012 + *.Blob createFrom(*.BlobInfo, java.nio.file.Path, *.Storage$BlobWriteOption[]) - com/google/cloud/storage/Storage - com.google.cloud.storage.PostPolicyV4 generateSignedPostPolicyV4(com.google.cloud.storage.BlobInfo, long, java.util.concurrent.TimeUnit, com.google.cloud.storage.PostPolicyV4$PostConditionsV4, com.google.cloud.storage.Storage$PostPolicyV4Option[]) 7012 + com/google/cloud/storage/Storage + *.Blob createFrom(*.BlobInfo, java.io.InputStream, *.Storage$BlobWriteOption[]) - com/google/cloud/storage/Storage - com.google.cloud.storage.PostPolicyV4 generateSignedPostPolicyV4(com.google.cloud.storage.BlobInfo, long, java.util.concurrent.TimeUnit, com.google.cloud.storage.Storage$PostPolicyV4Option[]) 7012 + com/google/cloud/storage/spi/v1/StorageRpc + *.StorageObject writeWithResponse(*.String, byte[], int, long, int, boolean) com/google/cloud/storage/BucketInfo$Builder diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java index ec5376697c..0c9520849b 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java @@ -19,6 +19,7 @@ import static com.google.cloud.RetryHelper.runWithRetries; import static java.util.concurrent.Executors.callable; +import com.google.api.services.storage.model.StorageObject; import com.google.cloud.BaseWriteChannel; import com.google.cloud.RestorableState; import com.google.cloud.RetryHelper; @@ -47,6 +48,13 @@ class BlobWriteChannel extends BaseWriteChannel { super(options, null, uploadId); } + // Contains metadata of the updated object or null if upload is not completed. + private StorageObject storageObject; + + StorageObject getStorageObject() { + return storageObject; + } + @Override protected void flushBuffer(final int length, final boolean last) { try { @@ -55,9 +63,11 @@ protected void flushBuffer(final int length, final boolean last) { new Runnable() { @Override public void run() { - getOptions() - .getStorageRpcV1() - .write(getUploadId(), getBuffer(), 0, getPosition(), length, last); + storageObject = + getOptions() + .getStorageRpcV1() + .writeWithResponse( + getUploadId(), getBuffer(), 0, getPosition(), length, last); } }), getOptions().getRetrySettings(), diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 988f87d39c..eb15dd37a2 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -39,9 +39,11 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.io.BaseEncoding; +import java.io.IOException; import java.io.InputStream; import java.io.Serializable; import java.net.URL; +import java.nio.file.Path; import java.security.Key; import java.util.Arrays; import java.util.Collections; @@ -1821,7 +1823,7 @@ public static Builder newBuilder() { * Blob blob = storage.create(blobInfo); * } * - * @return a [@code Blob} with complete information + * @return a {@code Blob} with complete information * @throws StorageException upon failure */ Blob create(BlobInfo blobInfo, BlobTargetOption... options); @@ -1842,7 +1844,7 @@ public static Builder newBuilder() { * Blob blob = storage.create(blobInfo, "Hello, World!".getBytes(UTF_8)); * } * - * @return a [@code Blob} with complete information + * @return a {@code Blob} with complete information * @throws StorageException upon failure * @see Hashes and ETags */ @@ -1865,7 +1867,7 @@ public static Builder newBuilder() { * Blob blob = storage.create(blobInfo, "Hello, World!".getBytes(UTF_8), 7, 5); * } * - * @return a [@code Blob} with complete information + * @return a {@code Blob} with complete information * @throws StorageException upon failure * @see Hashes and ETags */ @@ -1908,12 +1910,124 @@ Blob create( * Blob blob = storage.create(blobInfo, content, BlobWriteOption.encryptionKey(encryptionKey)); * } * - * @return a [@code Blob} with complete information + * @return a {@code Blob} with complete information * @throws StorageException upon failure */ @Deprecated Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... options); + /** + * Uploads {@code path} to the blob using {@link #writer}. By default any MD5 and CRC32C values in + * the given {@code blobInfo} are ignored unless requested via the {@link + * BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options. Folder upload is + * not supported. + * + *

Example of uploading a file: + * + * 

{@code
+   * String bucketName = "my-unique-bucket";
+   * String fileName = "readme.txt";
+   * BlobId blobId = BlobId.of(bucketName, fileName);
+   * BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
+   * storage.createFrom(blobInfo, Paths.get(fileName));
+   * }
+ * + * @param blobInfo blob to create + * @param path file to upload + * @param options blob write options + * @return a {@code Blob} with complete information + * @throws IOException on I/O error + * @throws StorageException on server side error + * @see #createFrom(BlobInfo, Path, int, BlobWriteOption...) + */ + Blob createFrom(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException; + + /** + * Uploads {@code path} to the blob using {@link #writer} and {@code bufferSize}. By default any + * MD5 and CRC32C values in the given {@code blobInfo} are ignored unless requested via the {@link + * BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options. Folder upload is + * not supported. + * + *

{@link #createFrom(BlobInfo, Path, BlobWriteOption...)} invokes this method with a buffer + * size of 15 MiB. Users can pass alternative values. Larger buffer sizes might improve the upload + * performance but require more memory. This can cause an OutOfMemoryError or add significant + * garbage collection overhead. Smaller buffer sizes reduce memory consumption, that is noticeable + * when uploading many objects in parallel. Buffer sizes less than 256 KiB are treated as 256 KiB. + * + *

Example of uploading a humongous file: + * + * 

{@code
+   * BlobId blobId = BlobId.of(bucketName, blobName);
+   * BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("video/webm").build();
+   *
+   * int largeBufferSize = 150 * 1024 * 1024;
+   * Path file = Paths.get("humongous.file");
+   * storage.createFrom(blobInfo, file, largeBufferSize);
+   * }
+ * + * @param blobInfo blob to create + * @param path file to upload + * @param bufferSize size of the buffer I/O operations + * @param options blob write options + * @return a {@code Blob} with complete information + * @throws IOException on I/O error + * @throws StorageException on server side error + */ + Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) + throws IOException; + + /** + * Reads bytes from an input stream and uploads those bytes to the blob using {@link #writer}. By + * default any MD5 and CRC32C values in the given {@code blobInfo} are ignored unless requested + * via the {@link BlobWriteOption#md5Match()} and {@link BlobWriteOption#crc32cMatch()} options. + * + *

Example of uploading data with CRC32C checksum: + * + * 

{@code
+   * BlobId blobId = BlobId.of(bucketName, blobName);
+   * byte[] content = "Hello, world".getBytes(StandardCharsets.UTF_8);
+   * Hasher hasher = Hashing.crc32c().newHasher().putBytes(content);
+   * String crc32c = BaseEncoding.base64().encode(Ints.toByteArray(hasher.hash().asInt()));
+   * BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setCrc32c(crc32c).build();
+   * storage.createFrom(blobInfo, new ByteArrayInputStream(content), Storage.BlobWriteOption.crc32cMatch());
+   * }
+ * + * @param blobInfo blob to create + * @param content input stream to read from + * @param options blob write options + * @return a {@code Blob} with complete information + * @throws IOException on I/O error + * @throws StorageException on server side error + * @see #createFrom(BlobInfo, InputStream, int, BlobWriteOption...) + */ + Blob createFrom(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) + throws IOException; + + /** + * Reads bytes from an input stream and uploads those bytes to the blob using {@link #writer} and + * {@code bufferSize}. By default any MD5 and CRC32C values in the given {@code blobInfo} are + * ignored unless requested via the {@link BlobWriteOption#md5Match()} and {@link + * BlobWriteOption#crc32cMatch()} options. + * + *

{@link #createFrom(BlobInfo, InputStream, BlobWriteOption...)} )} invokes this method with a + * buffer size of 15 MiB. Users can pass alternative values. Larger buffer sizes might improve the + * upload performance but require more memory. This can cause an OutOfMemoryError or add + * significant garbage collection overhead. Smaller buffer sizes reduce memory consumption, that + * is noticeable when uploading many objects in parallel. Buffer sizes less than 256 KiB are + * treated as 256 KiB. + * + * @param blobInfo blob to create + * @param content input stream to read from + * @param bufferSize size of the buffer I/O operations + * @param options blob write options + * @return a {@code Blob} with complete information + * @throws IOException on I/O error + * @throws StorageException on server side error + */ + Blob createFrom( + BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) + throws IOException; + /** * Returns the requested bucket or {@code null} if not found. * diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 0e24521eb6..b1f2bfe3c3 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -48,6 +48,7 @@ import com.google.cloud.ReadChannel; import com.google.cloud.RetryHelper.RetryHelperException; import com.google.cloud.Tuple; +import com.google.cloud.WriteChannel; import com.google.cloud.storage.Acl.Entity; import com.google.cloud.storage.HmacKey.HmacKeyMetadata; import com.google.cloud.storage.PostPolicyV4.ConditionV4Type; @@ -70,12 +71,18 @@ import com.google.common.io.BaseEncoding; import com.google.common.primitives.Ints; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; import java.net.URLEncoder; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Collections; @@ -99,6 +106,9 @@ final class StorageImpl extends BaseService implements Storage { private static final String STORAGE_XML_URI_HOST_NAME = "storage.googleapis.com"; + private static final int DEFAULT_BUFFER_SIZE = 15 * 1024 * 1024; + private static final int MIN_BUFFER_SIZE = 256 * 1024; + private static final Function, Boolean> DELETE_FUNCTION = new Function, Boolean>() { @Override @@ -211,6 +221,60 @@ public StorageObject call() { } } + @Override + public Blob createFrom(BlobInfo blobInfo, Path path, BlobWriteOption... options) + throws IOException { + return createFrom(blobInfo, path, DEFAULT_BUFFER_SIZE, options); + } + + @Override + public Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) + throws IOException { + if (Files.isDirectory(path)) { + throw new StorageException(0, path + " is a directory"); + } + try (InputStream input = Files.newInputStream(path)) { + return createFrom(blobInfo, input, bufferSize, options); + } + } + + @Override + public Blob createFrom(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) + throws IOException { + return createFrom(blobInfo, content, DEFAULT_BUFFER_SIZE, options); + } + + @Override + public Blob createFrom( + BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) + throws IOException { + + BlobWriteChannel blobWriteChannel; + try (WriteChannel writer = writer(blobInfo, options)) { + blobWriteChannel = (BlobWriteChannel) writer; + uploadHelper(Channels.newChannel(content), writer, bufferSize); + } + StorageObject objectProto = blobWriteChannel.getStorageObject(); + return Blob.fromPb(this, objectProto); + } + + /* + * Uploads the given content to the storage using specified write channel and the given buffer + * size. This method does not close any channels. + */ + private static void uploadHelper(ReadableByteChannel reader, WriteChannel writer, int bufferSize) + throws IOException { + bufferSize = Math.max(bufferSize, MIN_BUFFER_SIZE); + ByteBuffer buffer = ByteBuffer.allocate(bufferSize); + writer.setChunkSize(bufferSize); + + while (reader.read(buffer) >= 0) { + buffer.flip(); + writer.write(buffer); + buffer.clear(); + } + } + @Override public Bucket get(String bucket, BucketGetOption... options) { final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb(); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index e211c0277e..519a9e9fa2 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -725,11 +725,23 @@ public void write( long destOffset, int length, boolean last) { + writeWithResponse(uploadId, toWrite, toWriteOffset, destOffset, length, last); + } + + @Override + public StorageObject writeWithResponse( + String uploadId, + byte[] toWrite, + int toWriteOffset, + long destOffset, + int length, + boolean last) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_WRITE); Scope scope = tracer.withSpan(span); + StorageObject updatedBlob = null; try { if (length == 0 && !last) { - return; + return updatedBlob; } GenericUrl url = new GenericUrl(uploadId); HttpRequest httpRequest = @@ -750,6 +762,9 @@ public void write( range.append('*'); } httpRequest.getHeaders().setContentRange(range.toString()); + if (last) { + httpRequest.setParser(storage.getObjectParser()); + } int code; String message; IOException exception = null; @@ -758,6 +773,13 @@ public void write( response = httpRequest.execute(); code = response.getStatusCode(); message = response.getStatusMessage(); + String contentType = response.getContentType(); + if (last + && (code == 200 || code == 201) + && contentType != null + && contentType.startsWith("application/json")) { + updatedBlob = response.parseAs(StorageObject.class); + } } catch (HttpResponseException ex) { exception = ex; code = ex.getStatusCode(); @@ -783,6 +805,7 @@ public void write( scope.close(); span.end(HttpStorageRpcSpans.END_SPAN_OPTIONS); } + return updatedBlob; } @Override diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index 36c7a5ff1b..7ae9c8ec1c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -325,6 +325,27 @@ void write( int length, boolean last); + /** + * Writes the provided bytes to a storage object at the provided location. If {@code last=true} + * returns metadata of the updated object, otherwise returns null. + * + * @param uploadId resumable upload ID + * @param toWrite a portion of the content + * @param toWriteOffset starting position in the {@code toWrite} array + * @param destOffset starting position in the destination data + * @param length the number of bytes to be uploaded + * @param last true, if {@code toWrite} is the final content portion + * @throws StorageException upon failure + * @return + */ + StorageObject writeWithResponse( + String uploadId, + byte[] toWrite, + int toWriteOffset, + long destOffset, + int length, + boolean last); + /** * Sends a rewrite request to open a rewrite channel. * diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java index 6bd2d487ea..7733f13eb6 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java @@ -139,6 +139,17 @@ public void write( throw new UnsupportedOperationException("Not implemented yet"); } + @Override + public StorageObject writeWithResponse( + String uploadId, + byte[] toWrite, + int toWriteOffset, + long destOffset, + int length, + boolean last) { + throw new UnsupportedOperationException("Not implemented yet"); + } + @Override public RewriteResponse openRewrite(RewriteRequest rewriteRequest) { throw new UnsupportedOperationException("Not implemented yet"); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java index fdbb932b0c..a18345be89 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java @@ -27,10 +27,14 @@ import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.api.services.storage.model.StorageObject; import com.google.cloud.RestorableState; import com.google.cloud.WriteChannel; import com.google.cloud.storage.spi.StorageRpcFactory; @@ -57,6 +61,7 @@ public class BlobWriteChannelTest { private static final String BLOB_NAME = "n"; private static final String UPLOAD_ID = "uploadid"; private static final BlobInfo BLOB_INFO = BlobInfo.newBuilder(BUCKET_NAME, BLOB_NAME).build(); + private static final StorageObject UPDATED_BLOB = new StorageObject(); private static final Map EMPTY_RPC_OPTIONS = ImmutableMap.of(); private static final int MIN_CHUNK_SIZE = 256 * 1024; private static final int DEFAULT_CHUNK_SIZE = 60 * MIN_CHUNK_SIZE; // 15MiB @@ -94,6 +99,7 @@ public void testCreate() { replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); + assertNull(writer.getStorageObject()); } @Test @@ -104,6 +110,7 @@ public void testCreateRetryableError() { replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); + assertNull(writer.getStorageObject()); } @Test @@ -131,28 +138,44 @@ public void testWriteWithoutFlush() throws IOException { public void testWriteWithFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write( - eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(CUSTOM_CHUNK_SIZE), eq(false)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(CUSTOM_CHUNK_SIZE), + eq(false))) + .andReturn(null); replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); writer.setChunkSize(CUSTOM_CHUNK_SIZE); ByteBuffer buffer = randomBuffer(CUSTOM_CHUNK_SIZE); assertEquals(CUSTOM_CHUNK_SIZE, writer.write(buffer)); assertArrayEquals(buffer.array(), capturedBuffer.getValue()); + assertNull(writer.getStorageObject()); } @Test public void testWritesAndFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write( - eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(DEFAULT_CHUNK_SIZE), eq(false)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(DEFAULT_CHUNK_SIZE), + eq(false))) + .andReturn(null); replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); ByteBuffer[] buffers = new ByteBuffer[DEFAULT_CHUNK_SIZE / MIN_CHUNK_SIZE]; for (int i = 0; i < buffers.length; i++) { buffers[i] = randomBuffer(MIN_CHUNK_SIZE); assertEquals(MIN_CHUNK_SIZE, writer.write(buffers[i])); + assertNull(writer.getStorageObject()); } for (int i = 0; i < buffers.length; i++) { assertArrayEquals( @@ -166,13 +189,17 @@ public void testWritesAndFlush() throws IOException { public void testCloseWithoutFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write(eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); writer.close(); assertArrayEquals(new byte[0], capturedBuffer.getValue()); - assertTrue(!writer.isOpen()); + assertFalse(writer.isOpen()); + assertSame(UPDATED_BLOB, writer.getStorageObject()); } @Test @@ -180,8 +207,15 @@ public void testCloseWithFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); - storageRpcMock.write( - eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(MIN_CHUNK_SIZE), eq(true)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(MIN_CHUNK_SIZE), + eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); assertTrue(writer.isOpen()); @@ -189,14 +223,18 @@ public void testCloseWithFlush() throws IOException { writer.close(); assertEquals(DEFAULT_CHUNK_SIZE, capturedBuffer.getValue().length); assertArrayEquals(buffer.array(), Arrays.copyOf(capturedBuffer.getValue(), MIN_CHUNK_SIZE)); - assertTrue(!writer.isOpen()); + assertFalse(writer.isOpen()); + assertSame(UPDATED_BLOB, writer.getStorageObject()); } @Test public void testWriteClosed() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write(eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); writer.close(); @@ -206,6 +244,7 @@ public void testWriteClosed() throws IOException { } catch (IOException ex) { // expected } + assertSame(UPDATED_BLOB, writer.getStorageObject()); } @Test @@ -213,13 +252,15 @@ public void testSaveAndRestore() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(CaptureType.ALL); Capture capturedPosition = Capture.newInstance(CaptureType.ALL); - storageRpcMock.write( - eq(UPLOAD_ID), - capture(capturedBuffer), - eq(0), - captureLong(capturedPosition), - eq(DEFAULT_CHUNK_SIZE), - eq(false)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + captureLong(capturedPosition), + eq(DEFAULT_CHUNK_SIZE), + eq(false))) + .andReturn(null); expectLastCall().times(2); replay(storageRpcMock); ByteBuffer buffer1 = randomBuffer(DEFAULT_CHUNK_SIZE); @@ -239,7 +280,10 @@ public void testSaveAndRestore() throws IOException { public void testSaveAndRestoreClosed() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write(eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); writer.close(); @@ -283,8 +327,15 @@ public void testWriteWithSignedURLAndWithoutFlush() throws IOException { public void testWriteWithSignedURLAndWithFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write( - eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(CUSTOM_CHUNK_SIZE), eq(false)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(CUSTOM_CHUNK_SIZE), + eq(false))) + .andReturn(null); replay(storageRpcMock); writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); writer.setChunkSize(CUSTOM_CHUNK_SIZE); @@ -297,8 +348,15 @@ public void testWriteWithSignedURLAndWithFlush() throws IOException { public void testWriteWithSignedURLAndFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write( - eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(DEFAULT_CHUNK_SIZE), eq(false)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(DEFAULT_CHUNK_SIZE), + eq(false))) + .andReturn(null); replay(storageRpcMock); writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); ByteBuffer[] buffers = new ByteBuffer[DEFAULT_CHUNK_SIZE / MIN_CHUNK_SIZE]; @@ -318,7 +376,10 @@ public void testWriteWithSignedURLAndFlush() throws IOException { public void testCloseWithSignedURLWithoutFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write(eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); assertTrue(writer.isOpen()); @@ -332,8 +393,15 @@ public void testCloseWithSignedURLWithFlush() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); - storageRpcMock.write( - eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(MIN_CHUNK_SIZE), eq(true)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(MIN_CHUNK_SIZE), + eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); assertTrue(writer.isOpen()); @@ -348,7 +416,10 @@ public void testCloseWithSignedURLWithFlush() throws IOException { public void testWriteWithSignedURLClosed() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(); - storageRpcMock.write(eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), capture(capturedBuffer), eq(0), eq(0L), eq(0), eq(true))) + .andReturn(UPDATED_BLOB); replay(storageRpcMock); writer = new BlobWriteChannel(options, new URL(SIGNED_URL)); writer.close(); @@ -365,13 +436,15 @@ public void testSaveAndRestoreWithSignedURL() throws IOException { expect(storageRpcMock.open(SIGNED_URL)).andReturn(UPLOAD_ID); Capture capturedBuffer = Capture.newInstance(CaptureType.ALL); Capture capturedPosition = Capture.newInstance(CaptureType.ALL); - storageRpcMock.write( - eq(UPLOAD_ID), - capture(capturedBuffer), - eq(0), - captureLong(capturedPosition), - eq(DEFAULT_CHUNK_SIZE), - eq(false)); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + captureLong(capturedPosition), + eq(DEFAULT_CHUNK_SIZE), + eq(false))) + .andReturn(null); expectLastCall().times(2); replay(storageRpcMock); ByteBuffer buffer1 = randomBuffer(DEFAULT_CHUNK_SIZE); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java index e8525c5c6d..9eaefc66fb 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java @@ -41,7 +41,11 @@ import com.google.common.io.BaseEncoding; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.Key; import java.security.KeyFactory; import java.security.NoSuchAlgorithmException; @@ -961,6 +965,185 @@ public void testCreateBlobFromStreamRetryableException() throws IOException { } } + @Test + public void testCreateFromDirectory() throws IOException { + initializeService(); + Path dir = Files.createTempDirectory("unit_"); + try { + storage.createFrom(BLOB_INFO1, dir); + fail(); + } catch (StorageException e) { + assertEquals(dir + " is a directory", e.getMessage()); + } + } + + private BlobInfo initializeUpload(byte[] bytes) { + return initializeUpload(bytes, DEFAULT_BUFFER_SIZE, EMPTY_RPC_OPTIONS); + } + + private BlobInfo initializeUpload(byte[] bytes, int bufferSize) { + return initializeUpload(bytes, bufferSize, EMPTY_RPC_OPTIONS); + } + + private BlobInfo initializeUpload( + byte[] bytes, int bufferSize, Map rpcOptions) { + String uploadId = "upload-id"; + byte[] buffer = new byte[bufferSize]; + System.arraycopy(bytes, 0, buffer, 0, bytes.length); + BlobInfo blobInfo = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + StorageObject storageObject = new StorageObject(); + storageObject.setBucket(BLOB_INFO1.getBucket()); + storageObject.setName(BLOB_INFO1.getName()); + storageObject.setSize(BigInteger.valueOf(bytes.length)); + doReturn(uploadId) + .doThrow(UNEXPECTED_CALL_EXCEPTION) + .when(storageRpcMock) + .open(blobInfo.toPb(), rpcOptions); + + doReturn(storageObject) + .doThrow(UNEXPECTED_CALL_EXCEPTION) + .when(storageRpcMock) + .writeWithResponse(uploadId, buffer, 0, 0L, bytes.length, true); + + initializeService(); + expectedUpdated = Blob.fromPb(storage, storageObject); + return blobInfo; + } + + @Test + public void testCreateFromFile() throws Exception { + byte[] dataToSend = {1, 2, 3, 4}; + Path tempFile = Files.createTempFile("testCreateFrom", ".tmp"); + Files.write(tempFile, dataToSend); + + BlobInfo blobInfo = initializeUpload(dataToSend); + Blob blob = storage.createFrom(blobInfo, tempFile); + assertEquals(expectedUpdated, blob); + } + + @Test + public void testCreateFromStream() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + + BlobInfo blobInfo = initializeUpload(dataToSend); + Blob blob = storage.createFrom(blobInfo, stream); + assertEquals(expectedUpdated, blob); + } + + @Test + public void testCreateFromWithOptions() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5, 6}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + + BlobInfo blobInfo = initializeUpload(dataToSend, DEFAULT_BUFFER_SIZE, KMS_KEY_NAME_OPTIONS); + Blob blob = + storage.createFrom(blobInfo, stream, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + assertEquals(expectedUpdated, blob); + } + + @Test + public void testCreateFromWithBufferSize() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5, 6}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + int bufferSize = MIN_BUFFER_SIZE * 2; + + BlobInfo blobInfo = initializeUpload(dataToSend, bufferSize); + Blob blob = storage.createFrom(blobInfo, stream, bufferSize); + assertEquals(expectedUpdated, blob); + } + + @Test + public void testCreateFromWithBufferSizeAndOptions() throws Exception { + byte[] dataToSend = {1, 2, 3, 4, 5, 6}; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + int bufferSize = MIN_BUFFER_SIZE * 2; + + BlobInfo blobInfo = initializeUpload(dataToSend, bufferSize, KMS_KEY_NAME_OPTIONS); + Blob blob = + storage.createFrom( + blobInfo, stream, bufferSize, Storage.BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); + assertEquals(expectedUpdated, blob); + } + + @Test + public void testCreateFromWithSmallBufferSize() throws Exception { + byte[] dataToSend = new byte[100_000]; + ByteArrayInputStream stream = new ByteArrayInputStream(dataToSend); + int smallBufferSize = 100; + + BlobInfo blobInfo = initializeUpload(dataToSend, MIN_BUFFER_SIZE); + Blob blob = storage.createFrom(blobInfo, stream, smallBufferSize); + assertEquals(expectedUpdated, blob); + } + + @Test + public void testCreateFromWithException() throws Exception { + initializeService(); + String uploadId = "id-exception"; + byte[] bytes = new byte[10]; + byte[] buffer = new byte[MIN_BUFFER_SIZE]; + System.arraycopy(bytes, 0, buffer, 0, bytes.length); + BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + doReturn(uploadId) + .doThrow(UNEXPECTED_CALL_EXCEPTION) + .when(storageRpcMock) + .open(info.toPb(), EMPTY_RPC_OPTIONS); + + Exception runtimeException = new RuntimeException("message"); + doThrow(runtimeException) + .when(storageRpcMock) + .writeWithResponse(uploadId, buffer, 0, 0L, bytes.length, true); + + InputStream input = new ByteArrayInputStream(bytes); + try { + storage.createFrom(info, input, MIN_BUFFER_SIZE); + fail(); + } catch (StorageException e) { + assertSame(runtimeException, e.getCause()); + } + } + + @Test + public void testCreateFromMultipleParts() throws Exception { + initializeService(); + String uploadId = "id-multiple-parts"; + int extraBytes = 10; + int totalSize = MIN_BUFFER_SIZE + extraBytes; + byte[] dataToSend = new byte[totalSize]; + dataToSend[0] = 42; + dataToSend[MIN_BUFFER_SIZE + 1] = 43; + + StorageObject storageObject = new StorageObject(); + storageObject.setBucket(BLOB_INFO1.getBucket()); + storageObject.setName(BLOB_INFO1.getName()); + storageObject.setSize(BigInteger.valueOf(totalSize)); + + BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); + doReturn(uploadId) + .doThrow(UNEXPECTED_CALL_EXCEPTION) + .when(storageRpcMock) + .open(info.toPb(), EMPTY_RPC_OPTIONS); + + byte[] buffer1 = new byte[MIN_BUFFER_SIZE]; + System.arraycopy(dataToSend, 0, buffer1, 0, MIN_BUFFER_SIZE); + doReturn(null) + .doThrow(UNEXPECTED_CALL_EXCEPTION) + .when(storageRpcMock) + .writeWithResponse(uploadId, buffer1, 0, 0L, MIN_BUFFER_SIZE, false); + + byte[] buffer2 = new byte[MIN_BUFFER_SIZE]; + System.arraycopy(dataToSend, MIN_BUFFER_SIZE, buffer2, 0, extraBytes); + doReturn(storageObject) + .doThrow(UNEXPECTED_CALL_EXCEPTION) + .when(storageRpcMock) + .writeWithResponse(uploadId, buffer2, 0, (long) MIN_BUFFER_SIZE, extraBytes, true); + + InputStream input = new ByteArrayInputStream(dataToSend); + Blob blob = storage.createFrom(info, input, MIN_BUFFER_SIZE); + assertEquals(Blob.fromPb(storage, storageObject), blob); + } + private void verifyChannelRead(ReadChannel channel, byte[] bytes) throws IOException { assertNotNull(channel); assertTrue(channel.isOpen()); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 86bbdef79f..6aa76a03de 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -109,7 +109,8 @@ public class StorageImplTest { "projects/gcloud-devel/locations/us/keyRings/gcs_kms_key_ring_us/cryptoKeys/key"; private static final Long RETENTION_PERIOD = 10L; private static final String USER_PROJECT = "test-project"; - + private static final int DEFAULT_BUFFER_SIZE = 15 * 1024 * 1024; + private static final int MIN_BUFFER_SIZE = 256 * 1024; // BucketInfo objects private static final BucketInfo BUCKET_INFO1 = BucketInfo.newBuilder(BUCKET_NAME1).setMetageneration(42L).build(); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 7966c9281c..3d3bda7a9d 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -109,6 +109,7 @@ import java.net.URLConnection; import java.nio.ByteBuffer; import java.nio.file.Files; +import java.nio.file.Path; import java.security.Key; import java.util.ArrayList; import java.util.Arrays; @@ -3322,4 +3323,45 @@ public void testDeleteLifecycleRules() throws ExecutionException, InterruptedExc RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } } + + @Test + public void testUploadFromDownloadTo() throws Exception { + String blobName = "test-uploadFrom-downloadTo-blob"; + BlobId blobId = BlobId.of(BUCKET, blobName); + BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build(); + + Path tempFileFrom = Files.createTempFile("ITStorageTest_", ".tmp"); + Files.write(tempFileFrom, BLOB_BYTE_CONTENT); + Blob blob = storage.createFrom(blobInfo, tempFileFrom); + assertEquals(BUCKET, blob.getBucket()); + assertEquals(blobName, blob.getName()); + assertEquals(BLOB_BYTE_CONTENT.length, (long) blob.getSize()); + + Path tempFileTo = Files.createTempFile("ITStorageTest_", ".tmp"); + storage.get(blobId).downloadTo(tempFileTo); + byte[] readBytes = Files.readAllBytes(tempFileTo); + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + } + + @Test + public void testUploadWithEncryption() throws Exception { + String blobName = "test-upload-withEncryption"; + BlobId blobId = BlobId.of(BUCKET, blobName); + BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build(); + + ByteArrayInputStream content = new ByteArrayInputStream(BLOB_BYTE_CONTENT); + Blob blob = storage.createFrom(blobInfo, content, Storage.BlobWriteOption.encryptionKey(KEY)); + + try { + blob.getContent(); + fail("StorageException was expected"); + } catch (StorageException e) { + String expectedMessage = + "The target object is encrypted by a customer-supplied encryption key."; + assertTrue(e.getMessage().contains(expectedMessage)); + assertEquals(400, e.getCode()); + } + byte[] readBytes = blob.getContent(Blob.BlobSourceOption.decryptionKey(KEY)); + assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + } } From be72027b1587b9b0a3e9e65e7a2231bdb2ae521f Mon Sep 17 00:00:00 2001 From: Ajit Thakor <49403056+athakor@users.noreply.github.com> Date: Thu, 25 Jun 2020 22:58:04 +0530 Subject: [PATCH 7/8] feat: Add support to disable logging from bucket (#390) * feat: implement disabling logging api * feat: add support to disable bucket logging * feat: modified tests * feat: addressed review changes * feat: use alternative solution * feat: use or instead of and --- .../com/google/cloud/storage/BucketInfo.java | 13 +++++++---- .../google/cloud/storage/BucketInfoTest.java | 1 + .../com/google/cloud/storage/BucketTest.java | 23 +++++++++++++++++++ .../cloud/storage/it/ITStorageTest.java | 13 +++++------ 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index f2e82e1736..9a716d7027 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -286,9 +286,14 @@ public String getLogObjectPrefix() { } Bucket.Logging toPb() { - Bucket.Logging logging = new Bucket.Logging(); - logging.setLogBucket(logBucket); - logging.setLogObjectPrefix(logObjectPrefix); + Bucket.Logging logging; + if (logBucket != null || logObjectPrefix != null) { + logging = new Bucket.Logging(); + logging.setLogBucket(logBucket); + logging.setLogObjectPrefix(logObjectPrefix); + } else { + logging = Data.nullOf(Bucket.Logging.class); + } return logging; } @@ -1310,7 +1315,7 @@ public Builder setIamConfiguration(IamConfiguration iamConfiguration) { @Override public Builder setLogging(Logging logging) { - this.logging = logging; + this.logging = logging != null ? logging : BucketInfo.Logging.newBuilder().build(); return this; } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java index 196e6b493f..b108394265 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java @@ -229,6 +229,7 @@ public void testToPbAndFromPb() { BucketInfo.newBuilder("b") .setDeleteRules(DELETE_RULES) .setLifecycleRules(LIFECYCLE_RULES) + .setLogging(LOGGING) .build(); compareBuckets(bucketInfo, BucketInfo.fromPb(bucketInfo.toPb())); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java index b9c145afa4..94050ffef1 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketTest.java @@ -863,4 +863,27 @@ public void testDeleteLifecycleRules() { Bucket actualUpdatedBucket = updatedBucket.update(); assertThat(actualUpdatedBucket.getLifecycleRules()).hasSize(0); } + + @Test + public void testUpdateBucketLogging() { + initializeExpectedBucket(6); + BucketInfo.Logging logging = + BucketInfo.Logging.newBuilder() + .setLogBucket("logs-bucket") + .setLogObjectPrefix("test-logs") + .build(); + BucketInfo bucketInfo = BucketInfo.newBuilder("b").setLogging(logging).build(); + Bucket bucket = new Bucket(serviceMockReturnsOptions, new BucketInfo.BuilderImpl(bucketInfo)); + assertThat(bucket.getLogging().getLogBucket()).isEqualTo("logs-bucket"); + assertThat(bucket.getLogging().getLogObjectPrefix()).isEqualTo("test-logs"); + Bucket expectedUpdatedBucket = bucket.toBuilder().setLogging(null).build(); + expect(storage.getOptions()).andReturn(mockOptions).times(2); + expect(storage.update(expectedUpdatedBucket)).andReturn(expectedUpdatedBucket); + replay(storage); + initializeBucket(); + Bucket updatedBucket = new Bucket(storage, new BucketInfo.BuilderImpl(expectedUpdatedBucket)); + Bucket actualUpdatedBucket = updatedBucket.update(); + assertThat(actualUpdatedBucket.getLogging().getLogBucket()).isNull(); + assertThat(actualUpdatedBucket.getLogging().getLogObjectPrefix()).isNull(); + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 3d3bda7a9d..ba4b3623f6 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -3228,13 +3228,7 @@ public void testBucketLogging() throws ExecutionException, InterruptedException try { assertNotNull(storage.create(BucketInfo.newBuilder(logsBucket).setLocation("us").build())); Policy policy = storage.getIamPolicy(logsBucket); - assertNotNull( - storage.setIamPolicy( - logsBucket, - policy - .toBuilder() - .addIdentity(StorageRoles.legacyBucketWriter(), Identity.allAuthenticatedUsers()) - .build())); + assertNotNull(policy); BucketInfo.Logging logging = BucketInfo.Logging.newBuilder() .setLogBucket(logsBucket) @@ -3245,6 +3239,11 @@ public void testBucketLogging() throws ExecutionException, InterruptedException BucketInfo.newBuilder(loggingBucket).setLocation("us").setLogging(logging).build()); assertEquals(logsBucket, bucket.getLogging().getLogBucket()); assertEquals("test-logs", bucket.getLogging().getLogObjectPrefix()); + + // Disable bucket logging. + Bucket updatedBucket = bucket.toBuilder().setLogging(null).build().update(); + assertNull(updatedBucket.getLogging()); + } finally { RemoteStorageHelper.forceDelete(storage, logsBucket, 5, TimeUnit.SECONDS); RemoteStorageHelper.forceDelete(storage, loggingBucket, 5, TimeUnit.SECONDS); From 16036acb25d418704d001b7a4eaeafcaacaf25e7 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2020 10:53:03 -0700 Subject: [PATCH 8/8] chore: release 1.111.0 (#392) * updated CHANGELOG.md [ci skip] * updated README.md [ci skip] * updated versions.txt [ci skip] * updated samples/pom.xml [ci skip] * updated samples/install-without-bom/pom.xml [ci skip] * updated samples/snapshot/pom.xml [ci skip] * updated samples/snippets/pom.xml [ci skip] * updated google-cloud-storage/pom.xml [ci skip] * updated pom.xml Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 14 ++++++++++++++ README.md | 4 ++-- google-cloud-storage/pom.xml | 4 ++-- pom.xml | 2 +- versions.txt | 2 +- 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b5776edf5..caa0aed59c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [1.111.0](https://www.github.com/googleapis/java-storage/compare/v1.110.0...v1.111.0) (2020-06-25) + + +### Features + +* add storage.upload(path) ([#269](https://www.github.com/googleapis/java-storage/issues/269)) ([9457f3a](https://www.github.com/googleapis/java-storage/commit/9457f3a76ff18552adc5f9c82f62ab8f3c207d31)) +* Add support to disable logging from bucket ([#390](https://www.github.com/googleapis/java-storage/issues/390)) ([be72027](https://www.github.com/googleapis/java-storage/commit/be72027b1587b9b0a3e9e65e7a2231bdb2ae521f)) +* expose all the methods of notification ([#141](https://www.github.com/googleapis/java-storage/issues/141)) ([8dfc0cb](https://www.github.com/googleapis/java-storage/commit/8dfc0cbf8294a7fc426948e22e5c2182da97b630)) + + +### Reverts + +* Revert "feat: expose all the methods of notification (#141)" (#393) ([3e02b9c](https://www.github.com/googleapis/java-storage/commit/3e02b9c4ee1ce0fb785d15b04bd36754e31831a0)), closes [#141](https://www.github.com/googleapis/java-storage/issues/141) [#393](https://www.github.com/googleapis/java-storage/issues/393) + ## [1.110.0](https://www.github.com/googleapis/java-storage/compare/v1.109.1...v1.110.0) (2020-06-18) diff --git a/README.md b/README.md index a9bac3a944..740c0e1638 100644 --- a/README.md +++ b/README.md @@ -47,11 +47,11 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.cloud:google-cloud-storage:1.110.0' +compile 'com.google.cloud:google-cloud-storage:1.111.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "1.110.0" +libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "1.111.0" ``` [//]: # ({x-version-update-end}) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 3e2dbb0ba2..efdf55a8d4 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-storage - 1.110.1-SNAPSHOT + 1.111.0 jar Google Cloud Storage https://github.com/googleapis/java-storage @@ -12,7 +12,7 @@ com.google.cloud google-cloud-storage-parent - 1.110.1-SNAPSHOT + 1.111.0 google-cloud-storage diff --git a/pom.xml b/pom.xml index 02c4dd3949..86f026e5e3 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-storage-parent pom - 1.110.1-SNAPSHOT + 1.111.0 Storage Parent https://github.com/googleapis/java-storage diff --git a/versions.txt b/versions.txt index 9d49b870bc..e98012626f 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-storage:1.110.0:1.110.1-SNAPSHOT \ No newline at end of file +google-cloud-storage:1.111.0:1.111.0 \ No newline at end of file