Merge pull request #704 from mziccard/fix-storage-its

Handle eventally consistent blob lists in storage ITs

gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java

@@ -28,6 +28,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import com.google.gcloud.Page;
 import com.google.gcloud.ReadChannel;
@@ -100,13 +101,12 @@ public static void afterClass() throws ExecutionException, InterruptedException
 
   @Test(timeout = 5000)
   public void testListBuckets() throws InterruptedException {
-    Iterator<Bucket> bucketIterator =
-        storage.list(Storage.BucketListOption.prefix(BUCKET),
-        Storage.BucketListOption.fields()).values().iterator();
+    Iterator<Bucket> bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET),
+        Storage.BucketListOption.fields()).iterateAll();
     while (!bucketIterator.hasNext()) {
       Thread.sleep(500);
       bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET),
-          Storage.BucketListOption.fields()).values().iterator();
+          Storage.BucketListOption.fields()).iterateAll();
     }
     while (bucketIterator.hasNext()) {
       Bucket remoteBucket = bucketIterator.next();
@@ -287,8 +287,8 @@ public void testGetBlobFailNonExistingGeneration() {
     assertTrue(remoteBlob.delete());
   }
 
-  @Test
-  public void testListBlobsSelectedFields() {
+  @Test(timeout = 5000)
+  public void testListBlobsSelectedFields() throws InterruptedException {
     String[] blobNames = {"test-list-blobs-selected-fields-blob1",
         "test-list-blobs-selected-fields-blob2"};
     ImmutableMap<String, String> metadata = ImmutableMap.of("k", "v");
@@ -307,19 +307,29 @@ public void testListBlobsSelectedFields() {
     Page<Blob> page = storage.list(BUCKET,
         Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"),
         Storage.BlobListOption.fields(BlobField.METADATA));
-    int index = 0;
-    for (Blob remoteBlob : page.values()) {
+    // Listing blobs is eventually consistent, we loop until the list is of the expected size. The
+    // test fails if timeout is reached.
+    while (Iterators.size(page.iterateAll()) != 2) {
+      Thread.sleep(500);
+      page = storage.list(BUCKET,
+          Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"),
+          Storage.BlobListOption.fields(BlobField.METADATA));
+    }
+    Set<String> blobSet = ImmutableSet.of(blobNames[0], blobNames[1]);
+    Iterator<Blob> iterator = page.iterateAll();
+    while (iterator.hasNext()) {
+      Blob remoteBlob = iterator.next();
       assertEquals(BUCKET, remoteBlob.bucket());
-      assertEquals(blobNames[index++], remoteBlob.name());
+      assertTrue(blobSet.contains(remoteBlob.name()));
       assertEquals(metadata, remoteBlob.metadata());
       assertNull(remoteBlob.contentType());
     }
     assertTrue(remoteBlob1.delete());
     assertTrue(remoteBlob2.delete());
   }
 
-  @Test
-  public void testListBlobsEmptySelectedFields() {
+  @Test(timeout = 5000)
+  public void testListBlobsEmptySelectedFields() throws InterruptedException {
     String[] blobNames = {"test-list-blobs-empty-selected-fields-blob1",
         "test-list-blobs-empty-selected-fields-blob2"};
     BlobInfo blob1 = BlobInfo.builder(BUCKET, blobNames[0])
@@ -335,17 +345,27 @@ public void testListBlobsEmptySelectedFields() {
     Page<Blob> page = storage.list(BUCKET,
         Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"),
         Storage.BlobListOption.fields());
-    int index = 0;
-    for (Blob remoteBlob : page.values()) {
+    // Listing blobs is eventually consistent, we loop until the list is of the expected size. The
+    // test fails if timeout is reached.
+    while (Iterators.size(page.iterateAll()) != 2) {
+      Thread.sleep(500);
+      page = storage.list(BUCKET,
+          Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"),
+          Storage.BlobListOption.fields());
+    }
+    Set<String> blobSet = ImmutableSet.of(blobNames[0], blobNames[1]);
+    Iterator<Blob> iterator = page.iterateAll();
+    while (iterator.hasNext()) {
+      Blob remoteBlob = iterator.next();
       assertEquals(BUCKET, remoteBlob.bucket());
-      assertEquals(blobNames[index++], remoteBlob.name());
+      assertTrue(blobSet.contains(remoteBlob.name()));
       assertNull(remoteBlob.contentType());
     }
     assertTrue(remoteBlob1.delete());
     assertTrue(remoteBlob2.delete());
   }
 
-  @Test
+  @Test(timeout = 15000)
   public void testListBlobsVersioned() throws ExecutionException, InterruptedException {
     String bucketName = RemoteGcsHelper.generateBucketName();
     Bucket bucket = storage.create(BucketInfo.builder(bucketName).versioningEnabled(true).build());
@@ -366,8 +386,18 @@ public void testListBlobsVersioned() throws ExecutionException, InterruptedExcep
       Page<Blob> page = storage.list(bucketName,
           Storage.BlobListOption.prefix("test-list-blobs-versioned-blob"),
           Storage.BlobListOption.versions(true));
+      // Listing blobs is eventually consistent, we loop until the list is of the expected size. The
+      // test fails if timeout is reached.
+      while (Iterators.size(page.iterateAll()) != 3) {
+        Thread.sleep(500);
+        page = storage.list(bucketName,
+            Storage.BlobListOption.prefix("test-list-blobs-versioned-blob"),
+            Storage.BlobListOption.versions(true));
+      }
       Set<String> blobSet = ImmutableSet.of(blobNames[0], blobNames[1]);
-      for (Blob remoteBlob : page.values()) {
+      Iterator<Blob> iterator = page.iterateAll();
+      while (iterator.hasNext()) {
+        Blob remoteBlob = iterator.next();
         assertEquals(bucketName, remoteBlob.bucket());
         assertTrue(blobSet.contains(remoteBlob.name()));
         assertNotNull(remoteBlob.generation());