Skip to content

Commit 60bf976

Browse files
nidhiii-27nidhiii-27
committed
feat: implement skipIfExists option for downloadBlobs in TransferManager
1 parent 9f549d2 commit 60bf976

4 files changed

Lines changed: 194 additions & 5 deletions

File tree

‎google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/ParallelDownloadConfig.java‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,22 +34,35 @@
3434
*/
3535
public final class ParallelDownloadConfig {
3636

37+
private final boolean skipIfExists;
3738
@NonNull private final String stripPrefix;
3839
@NonNull private final Path downloadDirectory;
3940
@NonNull private final String bucketName;
4041
@NonNull private final List<BlobSourceOption> optionsPerRequest;
4142

4243
private ParallelDownloadConfig(
44+
boolean skipIfExists,
4345
@NonNull String stripPrefix,
4446
@NonNull Path downloadDirectory,
4547
@NonNull String bucketName,
4648
@NonNull List<BlobSourceOption> optionsPerRequest) {
49+
this.skipIfExists = skipIfExists;
4750
this.stripPrefix = stripPrefix;
4851
this.downloadDirectory = downloadDirectory;
4952
this.bucketName = bucketName;
5053
this.optionsPerRequest = optionsPerRequest;
5154
}
5255

56+
/**
57+
* If set, Transfer Manager will skip downloading an object if it already exists on the local
58+
* filesystem.
59+
*
60+
* @see Builder#setSkipIfExists(boolean)
61+
*/
62+
public boolean isSkipIfExists() {
63+
return skipIfExists;
64+
}
65+
5366
/**
5467
* A common prefix removed from an object's name before being written to the filesystem.
5568
*
@@ -96,20 +109,22 @@ public boolean equals(Object o) {
96109
return false;
97110
}
98111
ParallelDownloadConfig that = (ParallelDownloadConfig) o;
99-
return stripPrefix.equals(that.stripPrefix)
112+
return skipIfExists == that.skipIfExists
113+
&& stripPrefix.equals(that.stripPrefix)
100114
&& downloadDirectory.equals(that.downloadDirectory)
101115
&& bucketName.equals(that.bucketName)
102116
&& optionsPerRequest.equals(that.optionsPerRequest);
103117
}
104118

105119
@Override
106120
public int hashCode() {
107-
return Objects.hash(stripPrefix, downloadDirectory, bucketName, optionsPerRequest);
121+
return Objects.hash(skipIfExists, stripPrefix, downloadDirectory, bucketName, optionsPerRequest);
108122
}
109123

110124
@Override
111125
public String toString() {
112126
return MoreObjects.toStringHelper(this)
127+
.add("skipIfExists", skipIfExists)
113128
.add("stripPrefix", stripPrefix)
114129
.add("downloadDirectory", downloadDirectory)
115130
.add("bucketName", bucketName)
@@ -128,18 +143,32 @@ public static Builder newBuilder() {
128143

129144
public static final class Builder {
130145

146+
private boolean skipIfExists;
131147
@NonNull private String stripPrefix;
132148
@NonNull private Path downloadDirectory;
133149
@NonNull private String bucketName;
134150
@NonNull private List<BlobSourceOption> optionsPerRequest;
135151

136152
private Builder() {
153+
this.skipIfExists = false;
137154
this.stripPrefix = "";
138155
this.downloadDirectory = Paths.get("");
139156
this.bucketName = "";
140157
this.optionsPerRequest = ImmutableList.of();
141158
}
142159

160+
/**
161+
* Sets the value for skipIfExists. When set to true, Transfer Manager will skip downloading an
162+
* object if it already exists on the local filesystem.
163+
*
164+
* @return the builder instance with the value for skipIfExists modified.
165+
* @see ParallelDownloadConfig#isSkipIfExists()
166+
*/
167+
public Builder setSkipIfExists(boolean skipIfExists) {
168+
this.skipIfExists = skipIfExists;
169+
return this;
170+
}
171+
143172
/**
144173
* Sets the value for stripPrefix. This string will be removed from the beginning of all object
145174
* names before they are written to the filesystem.
@@ -197,7 +226,7 @@ public ParallelDownloadConfig build() {
197226
checkNotNull(downloadDirectory);
198227
checkNotNull(optionsPerRequest);
199228
return new ParallelDownloadConfig(
200-
stripPrefix, downloadDirectory, bucketName, optionsPerRequest);
229+
skipIfExists, stripPrefix, downloadDirectory, bucketName, optionsPerRequest);
201230
}
202231
}
203232
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/TransferManagerImpl.java‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,13 +147,30 @@ public void close() throws Exception {
147147
List<ApiFuture<DownloadResult>> downloadTasks = new ArrayList<>();
148148
if (!transferManagerConfig.isAllowDivideAndConquerDownload()) {
149149
for (BlobInfo blob : blobs) {
150-
DirectDownloadCallable callable = new DirectDownloadCallable(storage, blob, config, opts);
151-
downloadTasks.add(convert(executor.submit(callable)));
150+
Path destPath = TransferManagerUtils.createDestPath(config, blob);
151+
if (config.isSkipIfExists() && Files.exists(destPath)) {
152+
downloadTasks.add(
153+
ApiFutures.immediateFuture(
154+
DownloadResult.newBuilder(blob, TransferStatus.SKIPPED)
155+
.setOutputDestination(destPath)
156+
.build()));
157+
} else {
158+
DirectDownloadCallable callable = new DirectDownloadCallable(storage, blob, config, opts);
159+
downloadTasks.add(convert(executor.submit(callable)));
160+
}
152161
}
153162
} else {
154163
for (BlobInfo blob : blobs) {
155164
BlobInfo validatedBlob = retrieveSizeAndGeneration(storage, blob, config.getBucketName());
156165
Path destPath = TransferManagerUtils.createDestPath(config, blob);
166+
if (config.isSkipIfExists() && Files.exists(destPath)) {
167+
downloadTasks.add(
168+
ApiFutures.immediateFuture(
169+
DownloadResult.newBuilder(blob, TransferStatus.SKIPPED)
170+
.setOutputDestination(destPath)
171+
.build()));
172+
continue;
173+
}
157174
if (validatedBlob != null && qos.divideAndConquer(validatedBlob.getSize())) {
158175
DownloadResult optimisticResult =
159176
DownloadResult.newBuilder(validatedBlob, TransferStatus.SUCCESS)

‎google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITTransferManagerTest.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,69 @@ public void downloadBlobsAllowChunked() throws Exception {
377377
}
378378
}
379379

380+
@Test
381+
public void downloadBlobsSkipIfExists() throws Exception {
382+
TransferManagerConfig config =
383+
TransferManagerConfigTestingInstances.defaults(storage.getOptions());
384+
try (TransferManager transferManager = config.getService()) {
385+
String bucketName = bucket.getName();
386+
ParallelDownloadConfig parallelDownloadConfig =
387+
ParallelDownloadConfig.newBuilder()
388+
.setBucketName(bucketName)
389+
.setDownloadDirectory(baseDir)
390+
.setSkipIfExists(true)
391+
.build();
392+
// First download to ensure files exist
393+
DownloadJob job1 = transferManager.downloadBlobs(blobs, parallelDownloadConfig);
394+
List<DownloadResult> results1 = job1.getDownloadResults();
395+
assertThat(results1.stream().allMatch(r -> r.getStatus() == TransferStatus.SUCCESS)).isTrue();
396+
397+
// Second download with skipIfExists=true
398+
DownloadJob job2 = transferManager.downloadBlobs(blobs, parallelDownloadConfig);
399+
List<DownloadResult> results2 = job2.getDownloadResults();
400+
try {
401+
assertThat(results2).hasSize(3);
402+
assertThat(results2.stream().allMatch(r -> r.getStatus() == TransferStatus.SKIPPED))
403+
.isTrue();
404+
} finally {
405+
cleanUpFiles(results1);
406+
}
407+
}
408+
}
409+
410+
@Test
411+
public void downloadBlobsSkipIfExistsChunked() throws Exception {
412+
TransferManagerConfig config =
413+
TransferManagerConfigTestingInstances.defaults(storage.getOptions()).toBuilder()
414+
.setAllowDivideAndConquerDownload(true)
415+
.setPerWorkerBufferSize(128 * 1024)
416+
.build();
417+
try (TransferManager transferManager = config.getService()) {
418+
String bucketName = bucket.getName();
419+
ParallelDownloadConfig parallelDownloadConfig =
420+
ParallelDownloadConfig.newBuilder()
421+
.setBucketName(bucketName)
422+
.setDownloadDirectory(baseDir)
423+
.setSkipIfExists(true)
424+
.build();
425+
// First download to ensure files exist
426+
DownloadJob job1 = transferManager.downloadBlobs(blobs, parallelDownloadConfig);
427+
List<DownloadResult> results1 = job1.getDownloadResults();
428+
assertThat(results1.stream().allMatch(r -> r.getStatus() == TransferStatus.SUCCESS)).isTrue();
429+
430+
// Second download with skipIfExists=true
431+
DownloadJob job2 = transferManager.downloadBlobs(blobs, parallelDownloadConfig);
432+
List<DownloadResult> results2 = job2.getDownloadResults();
433+
try {
434+
assertThat(results2).hasSize(3);
435+
assertThat(results2.stream().allMatch(r -> r.getStatus() == TransferStatus.SKIPPED))
436+
.isTrue();
437+
} finally {
438+
cleanUpFiles(results1);
439+
}
440+
}
441+
}
442+
380443
@Test
381444
public void uploadFilesAllowPCU() throws Exception {
382445
TransferManagerConfig config =

‎google-cloud-storage/src/test/java/com/google/cloud/storage/transfermanager/ParallelDownloadConfigTest.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Copyright 2024 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage.transfermanager;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
21+
import com.google.cloud.storage.Storage.BlobSourceOption;
22+
import com.google.common.collect.ImmutableList;
23+
import java.nio.file.Paths;
24+
import org.junit.Test;
25+
26+
public final class ParallelDownloadConfigTest {
27+
28+
@Test
29+
public void testBuilder() {
30+
ParallelDownloadConfig config =
31+
ParallelDownloadConfig.newBuilder()
32+
.setBucketName("bucket")
33+
.setDownloadDirectory(Paths.get("dir"))
34+
.setStripPrefix("prefix")
35+
.setSkipIfExists(true)
36+
.setOptionsPerRequest(ImmutableList.of(BlobSourceOption.generationMatch(1L)))
37+
.build();
38+
39+
assertThat(config.getBucketName()).isEqualTo("bucket");
40+
assertThat(config.getDownloadDirectory()).isEqualTo(Paths.get("dir"));
41+
assertThat(config.getStripPrefix()).isEqualTo("prefix");
42+
assertThat(config.isSkipIfExists()).isTrue();
43+
assertThat(config.getOptionsPerRequest())
44+
.containsExactly(BlobSourceOption.generationMatch(1L));
45+
}
46+
47+
@Test
48+
public void testDefaultValues() {
49+
ParallelDownloadConfig config = ParallelDownloadConfig.newBuilder().setBucketName("bucket").build();
50+
51+
assertThat(config.isSkipIfExists()).isFalse();
52+
assertThat(config.getDownloadDirectory()).isEqualTo(Paths.get(""));
53+
assertThat(config.getStripPrefix()).isEqualTo("");
54+
assertThat(config.getOptionsPerRequest()).isEmpty();
55+
}
56+
57+
@Test
58+
public void testEqualsAndHashCode() {
59+
ParallelDownloadConfig config1 =
60+
ParallelDownloadConfig.newBuilder()
61+
.setBucketName("bucket")
62+
.setSkipIfExists(true)
63+
.build();
64+
ParallelDownloadConfig config2 =
65+
ParallelDownloadConfig.newBuilder()
66+
.setBucketName("bucket")
67+
.setSkipIfExists(true)
68+
.build();
69+
ParallelDownloadConfig config3 =
70+
ParallelDownloadConfig.newBuilder()
71+
.setBucketName("bucket")
72+
.setSkipIfExists(false)
73+
.build();
74+
75+
assertThat(config1).isEqualTo(config2);
76+
assertThat(config1.hashCode()).isEqualTo(config2.hashCode());
77+
assertThat(config1).isNotEqualTo(config3);
78+
assertThat(config1.hashCode()).isNotEqualTo(config3.hashCode());
79+
}
80+
}

0 commit comments

Comments
 (0)