Skip to content

Commit 50750d4

Browse files
mziccardmziccard
committed
Merge pull request #704 from mziccard/fix-storage-its
Handle eventally consistent blob lists in storage ITs
2 parents be913a6 + 622dcc9 commit 50750d4

1 file changed

Lines changed: 46 additions & 16 deletions

File tree

‎gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java‎

Lines changed: 46 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import com.google.common.collect.ImmutableList;
2929
import com.google.common.collect.ImmutableMap;
3030
import com.google.common.collect.ImmutableSet;
31+
import com.google.common.collect.Iterators;
3132
import com.google.common.collect.Lists;
3233
import com.google.gcloud.Page;
3334
import com.google.gcloud.ReadChannel;
@@ -100,13 +101,12 @@ public static void afterClass() throws ExecutionException, InterruptedException
100101

101102
@Test(timeout = 5000)
102103
public void testListBuckets() throws InterruptedException {
103-
Iterator<Bucket> bucketIterator =
104-
storage.list(Storage.BucketListOption.prefix(BUCKET),
105-
Storage.BucketListOption.fields()).values().iterator();
104+
Iterator<Bucket> bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET),
105+
Storage.BucketListOption.fields()).iterateAll();
106106
while (!bucketIterator.hasNext()) {
107107
Thread.sleep(500);
108108
bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET),
109-
Storage.BucketListOption.fields()).values().iterator();
109+
Storage.BucketListOption.fields()).iterateAll();
110110
}
111111
while (bucketIterator.hasNext()) {
112112
Bucket remoteBucket = bucketIterator.next();
@@ -287,8 +287,8 @@ public void testGetBlobFailNonExistingGeneration() {
287287
assertTrue(remoteBlob.delete());
288288
}
289289

290-
@Test
291-
public void testListBlobsSelectedFields() {
290+
@Test(timeout = 5000)
291+
public void testListBlobsSelectedFields() throws InterruptedException {
292292
String[] blobNames = {"test-list-blobs-selected-fields-blob1",
293293
"test-list-blobs-selected-fields-blob2"};
294294
ImmutableMap<String, String> metadata = ImmutableMap.of("k", "v");
@@ -307,19 +307,29 @@ public void testListBlobsSelectedFields() {
307307
Page<Blob> page = storage.list(BUCKET,
308308
Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"),
309309
Storage.BlobListOption.fields(BlobField.METADATA));
310-
int index = 0;
311-
for (Blob remoteBlob : page.values()) {
310+
// Listing blobs is eventually consistent, we loop until the list is of the expected size. The
311+
// test fails if timeout is reached.
312+
while (Iterators.size(page.iterateAll()) != 2) {
313+
Thread.sleep(500);
314+
page = storage.list(BUCKET,
315+
Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"),
316+
Storage.BlobListOption.fields(BlobField.METADATA));
317+
}
318+
Set<String> blobSet = ImmutableSet.of(blobNames[0], blobNames[1]);
319+
Iterator<Blob> iterator = page.iterateAll();
320+
while (iterator.hasNext()) {
321+
Blob remoteBlob = iterator.next();
312322
assertEquals(BUCKET, remoteBlob.bucket());
313-
assertEquals(blobNames[index++], remoteBlob.name());
323+
assertTrue(blobSet.contains(remoteBlob.name()));
314324
assertEquals(metadata, remoteBlob.metadata());
315325
assertNull(remoteBlob.contentType());
316326
}
317327
assertTrue(remoteBlob1.delete());
318328
assertTrue(remoteBlob2.delete());
319329
}
320330

321-
@Test
322-
public void testListBlobsEmptySelectedFields() {
331+
@Test(timeout = 5000)
332+
public void testListBlobsEmptySelectedFields() throws InterruptedException {
323333
String[] blobNames = {"test-list-blobs-empty-selected-fields-blob1",
324334
"test-list-blobs-empty-selected-fields-blob2"};
325335
BlobInfo blob1 = BlobInfo.builder(BUCKET, blobNames[0])
@@ -335,17 +345,27 @@ public void testListBlobsEmptySelectedFields() {
335345
Page<Blob> page = storage.list(BUCKET,
336346
Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"),
337347
Storage.BlobListOption.fields());
338-
int index = 0;
339-
for (Blob remoteBlob : page.values()) {
348+
// Listing blobs is eventually consistent, we loop until the list is of the expected size. The
349+
// test fails if timeout is reached.
350+
while (Iterators.size(page.iterateAll()) != 2) {
351+
Thread.sleep(500);
352+
page = storage.list(BUCKET,
353+
Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"),
354+
Storage.BlobListOption.fields());
355+
}
356+
Set<String> blobSet = ImmutableSet.of(blobNames[0], blobNames[1]);
357+
Iterator<Blob> iterator = page.iterateAll();
358+
while (iterator.hasNext()) {
359+
Blob remoteBlob = iterator.next();
340360
assertEquals(BUCKET, remoteBlob.bucket());
341-
assertEquals(blobNames[index++], remoteBlob.name());
361+
assertTrue(blobSet.contains(remoteBlob.name()));
342362
assertNull(remoteBlob.contentType());
343363
}
344364
assertTrue(remoteBlob1.delete());
345365
assertTrue(remoteBlob2.delete());
346366
}
347367

348-
@Test
368+
@Test(timeout = 15000)
349369
public void testListBlobsVersioned() throws ExecutionException, InterruptedException {
350370
String bucketName = RemoteGcsHelper.generateBucketName();
351371
Bucket bucket = storage.create(BucketInfo.builder(bucketName).versioningEnabled(true).build());
@@ -366,8 +386,18 @@ public void testListBlobsVersioned() throws ExecutionException, InterruptedExcep
366386
Page<Blob> page = storage.list(bucketName,
367387
Storage.BlobListOption.prefix("test-list-blobs-versioned-blob"),
368388
Storage.BlobListOption.versions(true));
389+
// Listing blobs is eventually consistent, we loop until the list is of the expected size. The
390+
// test fails if timeout is reached.
391+
while (Iterators.size(page.iterateAll()) != 3) {
392+
Thread.sleep(500);
393+
page = storage.list(bucketName,
394+
Storage.BlobListOption.prefix("test-list-blobs-versioned-blob"),
395+
Storage.BlobListOption.versions(true));
396+
}
369397
Set<String> blobSet = ImmutableSet.of(blobNames[0], blobNames[1]);
370-
for (Blob remoteBlob : page.values()) {
398+
Iterator<Blob> iterator = page.iterateAll();
399+
while (iterator.hasNext()) {
400+
Blob remoteBlob = iterator.next();
371401
assertEquals(bucketName, remoteBlob.bucket());
372402
assertTrue(blobSet.contains(remoteBlob.name()));
373403
assertNotNull(remoteBlob.generation());

0 commit comments

Comments
 (0)