Skip to content

Commit 5dd3c30

Browse files
mziccardmziccard
committed
Merge pull request #726 from mziccard/fix-list-prefix
Remove BlobListOption.recursive option and fix delimiter handling
2 parents 7c42116 + de9b0d6 commit 5dd3c30

12 files changed

Lines changed: 235 additions & 47 deletions

File tree

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
package com.google.gcloud.spi;
1616

17+
import static com.google.common.base.MoreObjects.firstNonNull;
1718
import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
1819
import static com.google.gcloud.spi.StorageRpc.Option.FIELDS;
1920
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
@@ -57,8 +58,9 @@
5758
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
5859
import com.google.api.services.storage.model.Objects;
5960
import com.google.api.services.storage.model.StorageObject;
60-
import com.google.common.base.MoreObjects;
61+
import com.google.common.base.Function;
6162
import com.google.common.collect.ImmutableList;
63+
import com.google.common.collect.Iterables;
6264
import com.google.common.collect.Lists;
6365
import com.google.common.collect.Maps;
6466
import com.google.gcloud.storage.StorageException;
@@ -67,6 +69,7 @@
6769
import java.io.ByteArrayOutputStream;
6870
import java.io.IOException;
6971
import java.io.InputStream;
72+
import java.math.BigInteger;
7073
import java.util.ArrayList;
7174
import java.util.Iterator;
7275
import java.util.List;
@@ -151,7 +154,7 @@ public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) {
151154
}
152155

153156
@Override
154-
public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options) {
157+
public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Option, ?> options) {
155158
try {
156159
Objects objects = storage.objects()
157160
.list(bucket)
@@ -163,13 +166,30 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
163166
.setPageToken(PAGE_TOKEN.getString(options))
164167
.setFields(FIELDS.getString(options))
165168
.execute();
166-
return Tuple.<String, Iterable<StorageObject>>of(
167-
objects.getNextPageToken(), objects.getItems());
169+
Iterable<StorageObject> storageObjects = Iterables.concat(
170+
firstNonNull(objects.getItems(), ImmutableList.<StorageObject>of()),
171+
objects.getPrefixes() != null
172+
? Lists.transform(objects.getPrefixes(), objectFromPrefix(bucket))
173+
: ImmutableList.<StorageObject>of());
174+
return Tuple.of(objects.getNextPageToken(), storageObjects);
168175
} catch (IOException ex) {
169176
throw translate(ex);
170177
}
171178
}
172179

180+
private static Function<String, StorageObject> objectFromPrefix(final String bucket) {
181+
return new Function<String, StorageObject>() {
182+
@Override
183+
public StorageObject apply(String prefix) {
184+
return new StorageObject()
185+
.set("isDirectory", true)
186+
.setBucket(bucket)
187+
.setName(prefix)
188+
.setSize(BigInteger.ZERO);
189+
}
190+
};
191+
}
192+
173193
@Override
174194
public Bucket get(Bucket bucket, Map<Option, ?> options) {
175195
try {
@@ -534,7 +554,7 @@ public String open(StorageObject object, Map<Option, ?> options) {
534554
HttpRequest httpRequest =
535555
requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object));
536556
httpRequest.getHeaders().set("X-Upload-Content-Type",
537-
MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
557+
firstNonNull(object.getContentType(), "application/octet-stream"));
538558
HttpResponse response = httpRequest.execute();
539559
if (response.getStatusCode() != 200) {
540560
GoogleJsonError error = new GoogleJsonError();

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,7 @@ void write(String uploadId, byte[] toWrite, int toWriteOffset, long destOffset,
344344
/**
345345
* Continues rewriting on an already open rewrite channel.
346346
*
347-
* @throws StorageException
347+
* @throws StorageException upon failure
348348
*/
349349
RewriteResponse continueRewrite(RewriteResponse previousResponse);
350350
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,12 @@ Builder updateTime(Long updateTime) {
290290
return this;
291291
}
292292

293+
@Override
294+
Builder isDirectory(boolean isDirectory) {
295+
infoBuilder.isDirectory(isDirectory);
296+
return this;
297+
}
298+
293299
@Override
294300
public Blob build() {
295301
return new Blob(storage, infoBuilder);

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ public StorageObject apply(BlobInfo blobInfo) {
7878
private final String contentDisposition;
7979
private final String contentLanguage;
8080
private final Integer componentCount;
81+
private final boolean isDirectory;
8182

8283
/**
8384
* This class is meant for internal use only. Users are discouraged from using this class.
@@ -187,6 +188,8 @@ public abstract static class Builder {
187188

188189
abstract Builder updateTime(Long updateTime);
189190

191+
abstract Builder isDirectory(boolean isDirectory);
192+
190193
/**
191194
* Creates a {@code BlobInfo} object.
192195
*/
@@ -215,6 +218,7 @@ static final class BuilderImpl extends Builder {
215218
private Long metageneration;
216219
private Long deleteTime;
217220
private Long updateTime;
221+
private Boolean isDirectory;
218222

219223
BuilderImpl(BlobId blobId) {
220224
this.blobId = blobId;
@@ -241,6 +245,7 @@ static final class BuilderImpl extends Builder {
241245
metageneration = blobInfo.metageneration;
242246
deleteTime = blobInfo.deleteTime;
243247
updateTime = blobInfo.updateTime;
248+
isDirectory = blobInfo.isDirectory;
244249
}
245250

246251
@Override
@@ -364,6 +369,12 @@ Builder updateTime(Long updateTime) {
364369
return this;
365370
}
366371

372+
@Override
373+
Builder isDirectory(boolean isDirectory) {
374+
this.isDirectory = isDirectory;
375+
return this;
376+
}
377+
367378
@Override
368379
public BlobInfo build() {
369380
checkNotNull(blobId);
@@ -392,6 +403,7 @@ public BlobInfo build() {
392403
metageneration = builder.metageneration;
393404
deleteTime = builder.deleteTime;
394405
updateTime = builder.updateTime;
406+
isDirectory = firstNonNull(builder.isDirectory, Boolean.FALSE);
395407
}
396408

397409
/**
@@ -588,6 +600,18 @@ public Long updateTime() {
588600
return updateTime;
589601
}
590602

603+
/**
604+
* Returns {@code true} if the current blob represents a directory. This can only happen if the
605+
* blob is returned by {@link Storage#list(String, Storage.BlobListOption...)} when the
606+
* {@link Storage.BlobListOption#currentDirectory()} option is used. When this is the case only
607+
* {@link #blobId()} and {@link #size()} are set for the current blob: {@link BlobId#name()} ends
608+
* with the '/' character, {@link BlobId#generation()} returns {@code null} and {@link #size()} is
609+
* {@code 0}.
610+
*/
611+
public boolean isDirectory() {
612+
return isDirectory;
613+
}
614+
591615
/**
592616
* Returns a builder for the current blob.
593617
*/
@@ -761,6 +785,9 @@ public Acl apply(ObjectAccessControl objectAccessControl) {
761785
}
762786
}));
763787
}
788+
if (storageObject.containsKey("isDirectory")) {
789+
builder.isDirectory(Boolean.TRUE);
790+
}
764791
return builder.build();
765792
}
766793
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -694,10 +694,17 @@ public static BlobListOption prefix(String prefix) {
694694
}
695695

696696
/**
697-
* Returns an option to specify whether blob listing should include subdirectories or not.
697+
* If specified, results are returned in a directory-like mode. Blobs whose names, after a
698+
* possible {@link #prefix(String)}, do not contain the '/' delimiter are returned as is. Blobs
699+
* whose names, after a possible {@link #prefix(String)}, contain the '/' delimiter, will have
700+
* their name truncated after the delimiter and will be returned as {@link Blob} objects where
701+
* only {@link Blob#blobId()}, {@link Blob#size()} and {@link Blob#isDirectory()} are set. For
702+
* such directory blobs, ({@link BlobId#generation()} returns {@code null}), {@link Blob#size()}
703+
* returns {@code 0} while {@link Blob#isDirectory()} returns {@code true}. Duplicate directory
704+
* blobs are omitted.
698705
*/
699-
public static BlobListOption recursive(boolean recursive) {
700-
return new BlobListOption(StorageRpc.Option.DELIMITER, recursive);
706+
public static BlobListOption currentDirectory() {
707+
return new BlobListOption(StorageRpc.Option.DELIMITER, true);
701708
}
702709

703710
/**
@@ -1289,7 +1296,8 @@ private static void checkContentType(BlobInfo blobInfo) throws IllegalArgumentEx
12891296
Page<Bucket> list(BucketListOption... options);
12901297

12911298
/**
1292-
* Lists the bucket's blobs.
1299+
* Lists the bucket's blobs. If the {@link BlobListOption#currentDirectory()} option is provided,
1300+
* results are returned in a directory-like mode.
12931301
*
12941302
* @throws StorageException upon failure
12951303
*/

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ final class StorageImpl extends BaseService<StorageOptions> implements Storage {
7676
private static final byte[] EMPTY_BYTE_ARRAY = {};
7777
private static final String EMPTY_BYTE_ARRAY_MD5 = "1B2M2Y8AsgTpgAmY7PhCfg==";
7878
private static final String EMPTY_BYTE_ARRAY_CRC32C = "AAAAAA==";
79+
private static final String PATH_DELIMITER = "/";
7980

8081
private static final Function<Tuple<Storage, Boolean>, Boolean> DELETE_FUNCTION =
8182
new Function<Tuple<Storage, Boolean>, Boolean>() {
@@ -669,7 +670,7 @@ private static <T> void addToOptionMap(StorageRpc.Option getOption, StorageRpc.O
669670
}
670671
Boolean value = (Boolean) temp.remove(DELIMITER);
671672
if (Boolean.TRUE.equals(value)) {
672-
temp.put(DELIMITER, options().pathDelimiter());
673+
temp.put(DELIMITER, PATH_DELIMITER);
673674
}
674675
if (useAsSource) {
675676
addToOptionMap(IF_GENERATION_MATCH, IF_SOURCE_GENERATION_MATCH, generation, temp);

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java‎

Lines changed: 2 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,19 @@
1616

1717
package com.google.gcloud.storage;
1818

19-
import com.google.common.base.MoreObjects;
2019
import com.google.common.collect.ImmutableSet;
2120
import com.google.gcloud.ServiceOptions;
2221
import com.google.gcloud.spi.DefaultStorageRpc;
2322
import com.google.gcloud.spi.StorageRpc;
2423
import com.google.gcloud.spi.StorageRpcFactory;
2524

26-
import java.util.Objects;
2725
import java.util.Set;
2826

2927
public class StorageOptions extends ServiceOptions<Storage, StorageRpc, StorageOptions> {
3028

3129
private static final long serialVersionUID = -7804860602287801084L;
3230
private static final String GCS_SCOPE = "https://www.googleapis.com/auth/devstorage.full_control";
3331
private static final Set<String> SCOPES = ImmutableSet.of(GCS_SCOPE);
34-
private static final String DEFAULT_PATH_DELIMITER = "/";
35-
36-
private final String pathDelimiter;
3732

3833
public static class DefaultStorageFactory implements StorageFactory {
3934

@@ -58,24 +53,10 @@ public StorageRpc create(StorageOptions options) {
5853
public static class Builder extends
5954
ServiceOptions.Builder<Storage, StorageRpc, StorageOptions, Builder> {
6055

61-
private String pathDelimiter;
62-
6356
private Builder() {}
6457

6558
private Builder(StorageOptions options) {
6659
super(options);
67-
pathDelimiter = options.pathDelimiter;
68-
}
69-
70-
/**
71-
* Sets the path delimiter for the storage service.
72-
*
73-
* @param pathDelimiter the path delimiter to set
74-
* @return the builder
75-
*/
76-
public Builder pathDelimiter(String pathDelimiter) {
77-
this.pathDelimiter = pathDelimiter;
78-
return this;
7960
}
8061

8162
@Override
@@ -86,7 +67,6 @@ public StorageOptions build() {
8667

8768
private StorageOptions(Builder builder) {
8869
super(StorageFactory.class, StorageRpcFactory.class, builder);
89-
pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER);
9070
}
9171

9272
@SuppressWarnings("unchecked")
@@ -106,13 +86,6 @@ protected Set<String> scopes() {
10686
return SCOPES;
10787
}
10888

109-
/**
110-
* Returns the storage service's path delimiter.
111-
*/
112-
public String pathDelimiter() {
113-
return pathDelimiter;
114-
}
115-
11689
/**
11790
* Returns a default {@code StorageOptions} instance.
11891
*/
@@ -128,16 +101,12 @@ public Builder toBuilder() {
128101

129102
@Override
130103
public int hashCode() {
131-
return baseHashCode() ^ Objects.hash(pathDelimiter);
104+
return baseHashCode();
132105
}
133106

134107
@Override
135108
public boolean equals(Object obj) {
136-
if (!(obj instanceof StorageOptions)) {
137-
return false;
138-
}
139-
StorageOptions other = (StorageOptions) obj;
140-
return baseEquals(other) && Objects.equals(pathDelimiter, other.pathDelimiter);
109+
return obj instanceof StorageOptions && baseEquals((StorageOptions) obj);
141110
}
142111

143112
public static Builder builder() {

0 commit comments

Comments
 (0)