diff --git a/.github/generated-files-bot.yml b/.github/generated-files-bot.yml index 47c2ba132e..c644a24e11 100644 --- a/.github/generated-files-bot.yml +++ b/.github/generated-files-bot.yml @@ -9,3 +9,4 @@ ignoreAuthors: - 'renovate-bot' - 'yoshi-automation' - 'release-please[bot]' +- 'gcf-owl-bot[bot]' diff --git a/.github/readme/synth.metadata/synth.metadata b/.github/readme/synth.metadata/synth.metadata index 983f925002..26c407063f 100644 --- a/.github/readme/synth.metadata/synth.metadata +++ b/.github/readme/synth.metadata/synth.metadata @@ -4,14 +4,14 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-storage.git", - "sha": "df0eff16d8149ca3c65505c52f158c17a24330e0" + "sha": "513f3bb556150094d2d26aca1a4e5279cf414a53" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "bd8281a06cc7f84906e04d4843c1d3d386a980cd" + "sha": "c3e41da0fa256ad7f6b4bc76b9d069dedecdfef4" } } ] diff --git a/.github/release-please.yml b/.github/release-please.yml index 8b1f9130ff..da39bbd246 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -6,3 +6,8 @@ branches: handleGHRelease: true releaseType: java-lts branch: 1.113.14-sp + - bumpMinorPreMajor: true + handleGHRelease: true + releaseType: java-yoshi + branch: 1.106.1 + \ No newline at end of file diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 3a7beef33b..fe75998041 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -34,6 +34,22 @@ branchProtectionRules: - units (11) - 'Kokoro - Test: Integration' - cla/google + - pattern: 1.106.1 + isAdminEnforced: true + requiredApprovingReviewCount: 1 + requiresCodeOwnerReviews: true + requiresStrictStatusChecks: false + requiredStatusCheckContexts: + - dependencies (8) + - dependencies (11) + - linkage-monitor + - lint + - clirr + - units (7) + - units (8) + - units (11) + - 'Kokoro - Test: Integration' + - cla/google permissionRules: - team: yoshi-admins permission: admin diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index def8b3a2c8..0195b32f00 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,10 +19,6 @@ jobs: - run: .kokoro/build.sh env: JOB_TYPE: test - - name: coverage - uses: codecov/codecov-action@v1 - with: - name: actions ${{matrix.java}} windows: runs-on: windows-latest steps: @@ -80,4 +76,4 @@ jobs: - run: java -version - run: .kokoro/build.sh env: - JOB_TYPE: clirr \ No newline at end of file + JOB_TYPE: clirr diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 9e0dd51f73..b43a9f968f 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -23,8 +23,8 @@ cd ${scriptDir}/.. # include common functions source ${scriptDir}/common.sh -# Print out Java version -java -version +# Print out Maven & Java version +mvn -version echo ${JOB_TYPE} # attempt to install 3 times with exponential backoff (starting with 10 seconds) diff --git a/.kokoro/release/publish_javadoc11.sh b/.kokoro/release/publish_javadoc11.sh index a1f2e37743..af8238dc6c 100755 --- a/.kokoro/release/publish_javadoc11.sh +++ b/.kokoro/release/publish_javadoc11.sh @@ -42,6 +42,8 @@ mvn clean site -B -q -P docFX # copy README to docfx-yml dir and rename index.md cp README.md target/docfx-yml/index.md +# copy CHANGELOG to docfx-yml dir and rename history.md +cp CHANGELOG.md target/docfx-yml/history.md pushd target/docfx-yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c3e34b524..084115a7b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [1.115.0](https://www.github.com/googleapis/java-storage/compare/v1.114.0...v1.115.0) (2021-06-01) + + +### Features + +* add `gcf-owl-bot[bot]` to `ignoreAuthors` ([#837](https://www.github.com/googleapis/java-storage/issues/837)) ([fe8e98a](https://www.github.com/googleapis/java-storage/commit/fe8e98a229f472c1f29d206d937690660bfa1444)) + + +### Bug Fixes + +* improve error detection and reporting for BlobWriteChannel retry state ([#846](https://www.github.com/googleapis/java-storage/issues/846)) ([d0f2184](https://www.github.com/googleapis/java-storage/commit/d0f2184f4dd2d99a4315f260f35421358d14a2df)), closes [#839](https://www.github.com/googleapis/java-storage/issues/839) +* update BucketInfo translation code to properly handle lifecycle rules ([#852](https://www.github.com/googleapis/java-storage/issues/852)) ([3b1df1d](https://www.github.com/googleapis/java-storage/commit/3b1df1d00a459b134103bc8738f0294188502a37)), closes [#850](https://www.github.com/googleapis/java-storage/issues/850) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v1.2.0 ([#836](https://www.github.com/googleapis/java-storage/issues/836)) ([c1752ce](https://www.github.com/googleapis/java-storage/commit/c1752ce17d5d723d0ea36c41d98ae2bc9201fec2)) +* update kms.version to v0.88.4 ([#830](https://www.github.com/googleapis/java-storage/issues/830)) ([7e3dc28](https://www.github.com/googleapis/java-storage/commit/7e3dc287e4285a9312393179671a78c569e7e869)) +* update kms.version to v0.89.0 ([#855](https://www.github.com/googleapis/java-storage/issues/855)) ([29236e9](https://www.github.com/googleapis/java-storage/commit/29236e9d2eefb0e64b1b9bbfc532f4c3ae3e9ea4)) + ## [1.114.0](https://www.github.com/googleapis/java-storage/compare/v1.113.16...v1.114.0) (2021-05-13) diff --git a/README.md b/README.md index eae8c832f6..086565d795 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file com.google.cloud libraries-bom - 20.3.0 + 20.5.0 pom import @@ -38,25 +38,25 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-storage - 1.113.16 + 1.114.0 ``` If you are using Gradle 5.x or later, add this to your dependencies ```Groovy -implementation platform('com.google.cloud:libraries-bom:20.3.0') +implementation platform('com.google.cloud:libraries-bom:20.5.0') compile 'com.google.cloud:google-cloud-storage' ``` If you are using Gradle without BOM, add this to your dependencies ```Groovy -compile 'com.google.cloud:google-cloud-storage:1.113.16' +compile 'com.google.cloud:google-cloud-storage:1.114.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "1.113.16" +libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "1.114.0" ``` ## Authentication diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 78b40569b1..05dba83445 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-storage - 1.114.0 + 1.115.0 jar Google Cloud Storage https://github.com/googleapis/java-storage @@ -12,11 +12,11 @@ com.google.cloud google-cloud-storage-parent - 1.114.0 + 1.115.0 google-cloud-storage - 0.88.3 + 0.89.0 @@ -34,7 +34,6 @@ com.google.api-client google-api-client - 1.31.5 com.google.apis diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java index 09bc17081a..2d7a8520f4 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java @@ -26,6 +26,7 @@ import com.google.cloud.WriteChannel; import com.google.cloud.storage.spi.v1.StorageRpc; import com.google.common.collect.Maps; +import java.math.BigInteger; import java.net.URL; import java.util.Map; import java.util.concurrent.Callable; @@ -83,14 +84,52 @@ private StorageObject getRemoteStorageObject() { .get(getEntity().toPb(), Maps.newEnumMap(StorageRpc.Option.class)); } - private StorageException unrecoverableState( - int chunkOffset, int chunkLength, long localPosition, long remotePosition, boolean last) { + private static StorageException unrecoverableState( + String uploadId, + int chunkOffset, + int chunkLength, + long localPosition, + long remotePosition, + boolean last) { + return unrecoverableState( + uploadId, + chunkOffset, + chunkLength, + localPosition, + remotePosition, + last, + "Unable to recover in upload.\nThis may be a symptom of multiple clients uploading to the same upload session."); + } + + private static StorageException errorResolvingMetadataLastChunk( + String uploadId, + int chunkOffset, + int chunkLength, + long localPosition, + long remotePosition, + boolean last) { + return unrecoverableState( + uploadId, + chunkOffset, + chunkLength, + localPosition, + remotePosition, + last, + "Unable to load object metadata to determine if last chunk was successfully written"); + } + + private static StorageException unrecoverableState( + String uploadId, + int chunkOffset, + int chunkLength, + long localPosition, + long remotePosition, + boolean last, + String message) { StringBuilder sb = new StringBuilder(); - sb.append("Unable to recover in upload.\n"); - sb.append( - "This may be a symptom of multiple clients uploading to the same upload session.\n\n"); + sb.append(message).append("\n\n"); sb.append("For debugging purposes:\n"); - sb.append("uploadId: ").append(getUploadId()).append('\n'); + sb.append("uploadId: ").append(uploadId).append('\n'); sb.append("chunkOffset: ").append(chunkOffset).append('\n'); sb.append("chunkLength: ").append(chunkLength).append('\n'); sb.append("localOffset: ").append(localPosition).append('\n'); @@ -162,7 +201,7 @@ public void run() { // Get remote offset from API final long localPosition = getPosition(); // For each request it should be possible to retry from its location in this code - final long remotePosition = isRetrying() ? getRemotePosition() : getPosition(); + final long remotePosition = isRetrying() ? getRemotePosition() : localPosition; final int chunkOffset = (int) (remotePosition - localPosition); final int chunkLength = length - chunkOffset; final boolean uploadAlreadyComplete = remotePosition == -1; @@ -176,10 +215,38 @@ public void run() { if (storageObject == null) { storageObject = getRemoteStorageObject(); } + // the following checks are defined here explicitly to provide a more + // informative if either storageObject is unable to be resolved or it's size is + // unable to be determined. This scenario is a very rare case of failure that + // can arise when packets are lost. + if (storageObject == null) { + throw errorResolvingMetadataLastChunk( + getUploadId(), + chunkOffset, + chunkLength, + localPosition, + remotePosition, + lastChunk); + } // Verify that with the final chunk we match the blob length - if (storageObject.getSize().longValue() != getPosition() + length) { + BigInteger size = storageObject.getSize(); + if (size == null) { + throw errorResolvingMetadataLastChunk( + getUploadId(), + chunkOffset, + chunkLength, + localPosition, + remotePosition, + lastChunk); + } + if (size.longValue() != getPosition() + length) { throw unrecoverableState( - chunkOffset, chunkLength, localPosition, remotePosition, lastChunk); + getUploadId(), + chunkOffset, + chunkLength, + localPosition, + remotePosition, + lastChunk); } retrying = false; } else if (uploadAlreadyComplete && !lastChunk && !checkingForLastChunk) { @@ -201,7 +268,12 @@ public void run() { } else { // Case 4 && Case 8 && Case 9 throw unrecoverableState( - chunkOffset, chunkLength, localPosition, remotePosition, lastChunk); + getUploadId(), + chunkOffset, + chunkLength, + localPosition, + remotePosition, + lastChunk); } } }), diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index 5b69814ec7..c13395238b 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -43,6 +43,7 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -1838,33 +1839,52 @@ public ObjectAccessControl apply(Acl acl) { website.setNotFoundPage(notFoundPage); bucketPb.setWebsite(website); } - Set rules = new HashSet<>(); - if (deleteRules != null) { - rules.addAll( - transform( - deleteRules, - new Function() { - @Override - public Rule apply(DeleteRule deleteRule) { - return deleteRule.toPb(); - } - })); - } - if (lifecycleRules != null) { - rules.addAll( - transform( - lifecycleRules, - new Function() { - @Override - public Rule apply(LifecycleRule lifecycleRule) { - return lifecycleRule.toPb(); - } - })); - } - if (rules != null) { + if (deleteRules != null || lifecycleRules != null) { Lifecycle lifecycle = new Lifecycle(); - lifecycle.setRule(ImmutableList.copyOf(rules)); + + // Here we determine if we need to "clear" any defined Lifecycle rules by explicitly setting + // the Rule list of lifecycle to the empty list. + // In order for us to clear the rules, one of the three following must be true: + // 1. deleteRules is null while lifecycleRules is non-null and empty + // 2. lifecycleRules is null while deleteRules is non-null and empty + // 3. lifecycleRules is non-null and empty while deleteRules is non-null and empty + // If none of the above three is true, we will interpret as the Lifecycle rules being + // updated to the defined set of DeleteRule and LifecycleRule. + if ((deleteRules == null && lifecycleRules.isEmpty()) + || (lifecycleRules == null && deleteRules.isEmpty()) + || (deleteRules != null && deleteRules.isEmpty() && lifecycleRules.isEmpty())) { + lifecycle.setRule(Collections.emptyList()); + } else { + Set rules = new HashSet<>(); + if (deleteRules != null) { + rules.addAll( + transform( + deleteRules, + new Function() { + @Override + public Rule apply(DeleteRule deleteRule) { + return deleteRule.toPb(); + } + })); + } + if (lifecycleRules != null) { + rules.addAll( + transform( + lifecycleRules, + new Function() { + @Override + public Rule apply(LifecycleRule lifecycleRule) { + return lifecycleRule.toPb(); + } + })); + } + + if (!rules.isEmpty()) { + lifecycle.setRule(ImmutableList.copyOf(rules)); + } + } + bucketPb.setLifecycle(lifecycle); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java index 8def7c306d..d72901c17e 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BucketInfoTest.java @@ -19,10 +19,12 @@ import static com.google.cloud.storage.Acl.Project.ProjectRole.VIEWERS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import com.google.api.client.util.DateTime; import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.Bucket.Lifecycle; import com.google.api.services.storage.model.Bucket.Lifecycle.Rule; import com.google.cloud.storage.Acl.Project; import com.google.cloud.storage.Acl.Role; @@ -39,6 +41,7 @@ import com.google.cloud.storage.BucketInfo.RawDeleteRule; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -172,6 +175,8 @@ public class BucketInfoTest { .setLogging(LOGGING) .build(); + private static final Lifecycle EMPTY_LIFECYCLE = lifecycle(Collections.emptyList()); + @Test public void testToBuilder() { compareBuckets(BUCKET_INFO, BUCKET_INFO.toBuilder().build()); @@ -376,4 +381,70 @@ public void testLogging() { assertEquals("test-bucket", logging.getLogBucket()); assertEquals("test-", logging.getLogObjectPrefix()); } + + @Test + public void testRuleMappingIsCorrect_noMutations() { + Bucket bucket = bi().build().toPb(); + assertNull(bucket.getLifecycle()); + } + + @Test + public void testRuleMappingIsCorrect_deleteLifecycleRules() { + Bucket bucket = bi().deleteLifecycleRules().build().toPb(); + assertEquals(EMPTY_LIFECYCLE, bucket.getLifecycle()); + } + + @Test + @SuppressWarnings({"deprecation"}) + public void testRuleMappingIsCorrect_setDeleteRules_null() { + Bucket bucket = bi().setDeleteRules(null).build().toPb(); + assertNull(bucket.getLifecycle()); + } + + @Test + @SuppressWarnings({"deprecation"}) + public void testRuleMappingIsCorrect_setDeleteRules_empty() { + Bucket bucket = bi().setDeleteRules(Collections.emptyList()).build().toPb(); + assertEquals(EMPTY_LIFECYCLE, bucket.getLifecycle()); + } + + @Test + public void testRuleMappingIsCorrect_setLifecycleRules_empty() { + Bucket bucket = bi().setLifecycleRules(Collections.emptyList()).build().toPb(); + assertEquals(EMPTY_LIFECYCLE, bucket.getLifecycle()); + } + + @Test + public void testRuleMappingIsCorrect_setLifeCycleRules_nonEmpty() { + LifecycleRule lifecycleRule = + new LifecycleRule( + LifecycleAction.newDeleteAction(), LifecycleCondition.newBuilder().setAge(10).build()); + Rule lifecycleDeleteAfter10 = lifecycleRule.toPb(); + Bucket bucket = bi().setLifecycleRules(ImmutableList.of(lifecycleRule)).build().toPb(); + assertEquals(lifecycle(lifecycleDeleteAfter10), bucket.getLifecycle()); + } + + @Test + @SuppressWarnings({"deprecation"}) + public void testRuleMappingIsCorrect_setDeleteRules_nonEmpty() { + DeleteRule deleteRule = DELETE_RULES.get(0); + Rule deleteRuleAge5 = deleteRule.toPb(); + Bucket bucket = bi().setDeleteRules(ImmutableList.of(deleteRule)).build().toPb(); + assertEquals(lifecycle(deleteRuleAge5), bucket.getLifecycle()); + } + + private static Lifecycle lifecycle(Rule... rules) { + return lifecycle(Arrays.asList(rules)); + } + + private static Lifecycle lifecycle(List rules) { + Lifecycle emptyLifecycle = new Lifecycle(); + emptyLifecycle.setRule(rules); + return emptyLifecycle; + } + + private static BucketInfo.Builder bi() { + String bucketId = "bucketId"; + return BucketInfo.newBuilder(bucketId); + } } diff --git a/pom.xml b/pom.xml index 2b5a9eaed9..eec8425346 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-storage-parent pom - 1.114.0 + 1.115.0 Storage Parent https://github.com/googleapis/java-storage @@ -14,7 +14,7 @@ com.google.cloud google-cloud-shared-config - 0.11.2 + 0.12.0 @@ -63,7 +63,7 @@ UTF-8 github google-cloud-storage-parent - 1.1.0 + 1.2.0 @@ -134,7 +134,7 @@ com.google.truth truth - 1.1.2 + 1.1.3 test @@ -187,7 +187,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.2.0 + 3.3.0 html diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 005cc7f1dd..707b40cbe6 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -29,7 +29,7 @@ com.google.cloud google-cloud-storage - 1.113.16 + 1.114.0 @@ -42,7 +42,7 @@ com.google.truth truth - 1.0.1 + 1.1.3 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 9f5a496b76..2533bc8f4f 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-storage - 1.113.16 + 1.114.0 @@ -40,7 +40,7 @@ com.google.truth truth - 1.0.1 + 1.1.3 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 4e93ce42f7..7f13eaacd6 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 20.3.0 + 20.5.0 pom import @@ -53,7 +53,7 @@ com.google.truth truth - 1.0.1 + 1.1.3 test diff --git a/synth.metadata b/synth.metadata index a3a01b8b2e..69d7a143c4 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,14 +4,14 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-storage.git", - "sha": "36f4a50875dcda20f73764790122b5fd0d7bb9b4" + "sha": "340f393f81f5b17b8835166ac7a3da9e8ffea2f4" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "046994f491c02806aea60118e214a9edd67f5ab7" + "sha": "8eae0234a16b26c2ff616d305dbd9786c8b10a47" } } ], diff --git a/versions.txt b/versions.txt index 571c2c6aa3..53c0d17046 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-storage:1.114.0:1.114.0 \ No newline at end of file +google-cloud-storage:1.115.0:1.115.0 \ No newline at end of file