Skip to content

Commit bfd4fc4

Browse files
mziccardmziccard
committed
Refactor storage integration tests
- Add option to read keyfile with getResourceAsStream - Use -1 instead of 42 for (meta)generation - Add delay-timeout to listing - Make generateBucketName public and static in RemoteGcsHelper - Add static deleteBucketRecursively to RemoteGcsHelper - RemoteGcdHelper.create throws exception if env variables not set - Add javadoc to RemoteGcdHelper
1 parent f02bd97 commit bfd4fc4

File tree

2 files changed

+202
-55
lines changed

2 files changed

+202
-55
lines changed

‎gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java‎

Lines changed: 35 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -34,46 +34,44 @@
3434
import java.nio.ByteBuffer;
3535
import java.util.Arrays;
3636
import java.util.Calendar;
37+
import java.util.Iterator;
38+
import java.util.concurrent.ExecutionException;
39+
import java.util.concurrent.TimeUnit;
40+
import java.util.concurrent.TimeoutException;
3741

3842
import org.junit.AfterClass;
3943
import org.junit.Before;
4044
import org.junit.BeforeClass;
41-
import org.junit.Rule;
4245
import org.junit.Test;
43-
import org.junit.rules.ExpectedException;
4446

4547
public class ITStorageTest {
4648

47-
private static StorageOptions options;
4849
private static Storage storage;
4950
private static RemoteGcsHelper gcsHelper;
50-
private static String bucket;
5151

52+
private static final String bucket = RemoteGcsHelper.generateBucketName();
5253
private static final String CONTENT_TYPE = "text/plain";
5354
private static final byte[] BLOB_BYTE_CONTENT = {0xD, 0xE, 0xA, 0xD};
5455
private static final String BLOB_STRING_CONTENT = "Hello Google Cloud Storage!";
5556

56-
@Rule
57-
public ExpectedException thrown = ExpectedException.none();
58-
5957
@BeforeClass
6058
public static void beforeClass() {
61-
gcsHelper = RemoteGcsHelper.create();
62-
if (gcsHelper != null) {
63-
options = gcsHelper.options();
64-
storage = StorageFactory.instance().get(options);
65-
bucket = gcsHelper.bucket();
59+
try {
60+
gcsHelper = RemoteGcsHelper.create();
61+
storage = StorageFactory.instance().get(gcsHelper.options());
6662
storage.create(BucketInfo.of(bucket));
63+
} catch (RemoteGcsHelper.GcsHelperException e) {
64+
// ignore
6765
}
6866
}
6967

7068
@AfterClass
71-
public static void afterClass() {
69+
public static void afterClass()
70+
throws ExecutionException, TimeoutException, InterruptedException {
7271
if (storage != null) {
73-
for (BlobInfo info : storage.list(bucket)) {
74-
storage.delete(bucket, info.name());
72+
if (!RemoteGcsHelper.deleteBucketRecursively(storage, bucket, 5, TimeUnit.SECONDS)) {
73+
throw new RuntimeException("Bucket deletion timed out. Could not delete non-empty bucket");
7574
}
76-
storage.delete(bucket);
7775
}
7876
}
7977

@@ -82,11 +80,16 @@ public void beforeMethod() {
8280
org.junit.Assume.assumeNotNull(storage);
8381
}
8482

85-
@Test
86-
public void testListBuckets() {
87-
ListResult<BucketInfo> bucketList = storage.list(Storage.BucketListOption.prefix(bucket));
88-
for (BucketInfo bucketInfo : bucketList) {
89-
assertTrue(bucketInfo.name().startsWith(bucket));
83+
@Test(timeout = 5000)
84+
public void testListBuckets() throws InterruptedException {
85+
Iterator<BucketInfo> bucketIterator =
86+
storage.list(Storage.BucketListOption.prefix(bucket)).iterator();
87+
while (!bucketIterator.hasNext()) {
88+
Thread.sleep(500);
89+
bucketIterator = storage.list(Storage.BucketListOption.prefix(bucket)).iterator();
90+
}
91+
while (bucketIterator.hasNext()) {
92+
assertTrue(bucketIterator.next().name().startsWith(bucket));
9093
}
9194
}
9295

@@ -137,7 +140,7 @@ public void testCreateBlobFail() {
137140
BlobInfo blob = BlobInfo.of(bucket, blobName);
138141
assertNotNull(storage.create(blob));
139142
try {
140-
storage.create(blob.toBuilder().generation(42L).build(), BLOB_BYTE_CONTENT,
143+
storage.create(blob.toBuilder().generation(-1L).build(), BLOB_BYTE_CONTENT,
141144
Storage.BlobTargetOption.generationMatch());
142145
fail("StorageException was expected");
143146
} catch (StorageException ex) {
@@ -165,7 +168,7 @@ public void testUpdateBlobFail() {
165168
BlobInfo blob = BlobInfo.of(bucket, blobName);
166169
assertNotNull(storage.create(blob));
167170
try {
168-
storage.update(blob.toBuilder().contentType(CONTENT_TYPE).generation(42L).build(),
171+
storage.update(blob.toBuilder().contentType(CONTENT_TYPE).generation(-1L).build(),
169172
Storage.BlobTargetOption.generationMatch());
170173
fail("StorageException was expected");
171174
} catch (StorageException ex) {
@@ -187,7 +190,7 @@ public void testDeleteBlobFail() {
187190
BlobInfo blob = BlobInfo.of(bucket, blobName);
188191
assertNotNull(storage.create(blob));
189192
try {
190-
storage.delete(bucket, blob.name(), Storage.BlobSourceOption.generationMatch(42L));
193+
storage.delete(bucket, blob.name(), Storage.BlobSourceOption.generationMatch(-1L));
191194
fail("StorageException was expected");
192195
} catch (StorageException ex) {
193196
// expected
@@ -232,8 +235,8 @@ public void testComposeBlobFail() {
232235
String targetBlobName = "test-compose-blob-fail-target";
233236
BlobInfo targetBlob = BlobInfo.of(bucket, targetBlobName);
234237
Storage.ComposeRequest req = Storage.ComposeRequest.builder()
235-
.addSource(sourceBlobName1, 42L)
236-
.addSource(sourceBlobName2, 42L)
238+
.addSource(sourceBlobName1, -1L)
239+
.addSource(sourceBlobName2, -1L)
237240
.target(targetBlob)
238241
.build();
239242
try {
@@ -290,7 +293,7 @@ public void testCopyBlobFail() {
290293
Storage.CopyRequest req = new Storage.CopyRequest.Builder()
291294
.source(bucket, sourceBlobName)
292295
.target(BlobInfo.builder(bucket, targetBlobName).build())
293-
.sourceOptions(Storage.BlobSourceOption.metagenerationMatch(42L))
296+
.sourceOptions(Storage.BlobSourceOption.metagenerationMatch(-1L))
294297
.build();
295298
try {
296299
storage.copy(req);
@@ -362,11 +365,11 @@ public void testBatchRequestFail() {
362365
String blobName = "test-batch-request-blob-fail";
363366
BlobInfo blob = BlobInfo.of(bucket, blobName);
364367
assertNotNull(storage.create(blob));
365-
BlobInfo updatedBlob = blob.toBuilder().generation(42L).build();
368+
BlobInfo updatedBlob = blob.toBuilder().generation(-1L).build();
366369
BatchRequest batchRequest = BatchRequest.builder()
367370
.update(updatedBlob, Storage.BlobTargetOption.generationMatch())
368-
.delete(bucket, blobName, Storage.BlobSourceOption.generationMatch(42L))
369-
.get(bucket, blobName, Storage.BlobSourceOption.generationMatch(42L))
371+
.delete(bucket, blobName, Storage.BlobSourceOption.generationMatch(-1L))
372+
.get(bucket, blobName, Storage.BlobSourceOption.generationMatch(-1L))
370373
.build();
371374
BatchResponse updateResponse = storage.apply(batchRequest);
372375
assertEquals(1, updateResponse.updates().size());
@@ -407,7 +410,7 @@ public void testReadChannelFail() throws UnsupportedEncodingException, IOExcepti
407410
BlobInfo blob = BlobInfo.of(bucket, blobName);
408411
assertNotNull(storage.create(blob));
409412
try (BlobReadChannel reader =
410-
storage.reader(bucket, blobName, Storage.BlobSourceOption.metagenerationMatch(42L))) {
413+
storage.reader(bucket, blobName, Storage.BlobSourceOption.metagenerationMatch(-1L))) {
411414
reader.read(ByteBuffer.allocate(42));
412415
fail("StorageException was expected");
413416
} catch (StorageException ex) {
@@ -419,7 +422,7 @@ public void testReadChannelFail() throws UnsupportedEncodingException, IOExcepti
419422
@Test
420423
public void testWriteChannelFail() throws UnsupportedEncodingException, IOException {
421424
String blobName = "test-write-channel-blob-fail";
422-
BlobInfo blob = BlobInfo.builder(bucket, blobName).generation(42L).build();
425+
BlobInfo blob = BlobInfo.builder(bucket, blobName).generation(-1L).build();
423426
try {
424427
try (BlobWriteChannel writer =
425428
storage.writer(blob, Storage.BlobTargetOption.generationMatch())) {

‎gcloud-java-storage/src/test/java/com/google/gcloud/storage/RemoteGcsHelper.java‎

Lines changed: 167 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,23 @@
1616

1717
package com.google.gcloud.storage;
1818

19+
import com.google.common.collect.ImmutableMap;
1920
import com.google.gcloud.AuthCredentials;
21+
import com.google.gcloud.storage.RemoteGcsHelper.Option.KeyFromClasspath;
2022

21-
import java.io.File;
2223
import java.io.FileInputStream;
2324
import java.io.FileNotFoundException;
24-
import java.io.IOException;
2525
import java.io.InputStream;
26+
import java.io.IOException;
27+
import java.util.Map;
2628
import java.util.UUID;
29+
import java.util.concurrent.Callable;
30+
import java.util.concurrent.ExecutionException;
31+
import java.util.concurrent.ExecutorService;
32+
import java.util.concurrent.Executors;
33+
import java.util.concurrent.Future;
34+
import java.util.concurrent.TimeUnit;
35+
import java.util.concurrent.TimeoutException;
2736
import java.util.logging.Level;
2837
import java.util.logging.Logger;
2938

@@ -38,53 +47,188 @@ public class RemoteGcsHelper {
3847
private static final String PRIVATE_KEY_ENV_VAR = "GCLOUD_TESTS_KEY";
3948

4049
private final StorageOptions options;
41-
private final String bucket;
4250

43-
private RemoteGcsHelper(StorageOptions options, String bucket) {
51+
private RemoteGcsHelper(StorageOptions options) {
4452
this.options = options;
45-
this.bucket = bucket;
4653
}
4754

55+
/**
56+
* Returns a {@StorageOptions} object to be used for testing.
57+
*/
4858
public StorageOptions options() {
4959
return options;
5060
}
5161

52-
public String bucket() {
53-
return bucket;
62+
/**
63+
* Delete a bucket recursively. Objects in the bucket are listed and deleted until bucket deletion
64+
* succeeds or {@code timeout} expires.
65+
*
66+
* @param storage the storage service to be used to issue requests
67+
* @param bucket the bucket to be deleted
68+
* @param timeout the maximum time to wait
69+
* @param unit the time unit of the timeout argument
70+
* @return true if deletion succeeded, false if timeout expired.
71+
* @throws InterruptedException if the thread deleting the bucket is interrupted while waiting
72+
* @throws ExecutionException if an exception was thrown while deleting bucket or bucket objects
73+
*/
74+
public static Boolean deleteBucketRecursively(Storage storage, String bucket, long timeout,
75+
TimeUnit unit) throws InterruptedException, ExecutionException {
76+
ExecutorService executor = Executors.newSingleThreadExecutor();
77+
Future<Boolean> future = executor.submit(new DeleteBucketTask(storage, bucket));
78+
try {
79+
return future.get(timeout, unit);
80+
} catch (TimeoutException ex) {
81+
return false;
82+
}
5483
}
55-
56-
private static String generateBucketName() {
84+
85+
/**
86+
* Returns a bucket name generated using a random UUID.
87+
*/
88+
public static String generateBucketName() {
5789
return BUCKET_NAME_PREFIX + UUID.randomUUID().toString();
5890
}
5991

60-
public static RemoteGcsHelper create() {
61-
if (System.getenv(PROJECT_ID_ENV_VAR) == null || System.getenv(PRIVATE_KEY_ENV_VAR) == null) {
62-
if (log.isLoggable(Level.WARNING)) {
63-
log.log(Level.INFO, "Environment variables {0} and {1} not set", new String[] {
64-
PROJECT_ID_ENV_VAR, PRIVATE_KEY_ENV_VAR});
65-
}
66-
return null;
67-
}
92+
/**
93+
* Creates a {@code RemoteGcsHelper} object.
94+
*
95+
* @param options creation options
96+
* @return A {@code RemoteGcsHelper} object for the provided options.
97+
* @throws com.google.gcloud.storage.RemoteGcsHelper.GcsHelperException if environment variables
98+
* {@code GCLOUD_TESTS_PROJECT_ID} and {@code GCLOUD_TESTS_KEY_PATH} are not set or if the file
99+
* pointed by {@code GCLOUD_TESTS_KEY_PATH} does not exist
100+
*/
101+
public static RemoteGcsHelper create(Option... options) throws GcsHelperException {
102+
boolean keyFromClassPath = false;
103+
Map<Class<? extends Option>, Option> optionsMap = Option.asImmutableMap(options);
104+
if (optionsMap.containsKey(KeyFromClasspath.class)) {
105+
keyFromClassPath =
106+
((KeyFromClasspath) optionsMap.get(KeyFromClasspath.class)).keyFromClasspath();
107+
}
68108
String projectId = System.getenv(PROJECT_ID_ENV_VAR);
69109
String stringKeyPath = System.getenv(PRIVATE_KEY_ENV_VAR);
70-
File keyFile = new File(stringKeyPath);
110+
if (projectId == null) {
111+
String message = "Environment variable " + PROJECT_ID_ENV_VAR + " not set";
112+
if (log.isLoggable(Level.WARNING)) {
113+
log.log(Level.WARNING, message);
114+
}
115+
throw new GcsHelperException(message);
116+
}
117+
if (stringKeyPath == null) {
118+
String message = "Environment variable " + PRIVATE_KEY_ENV_VAR + " not set";
119+
if (log.isLoggable(Level.WARNING)) {
120+
log.log(Level.WARNING, message);
121+
}
122+
throw new GcsHelperException(message);
123+
}
71124
try {
72-
InputStream keyFileStream = new FileInputStream(keyFile);
73-
StorageOptions options = StorageOptions.builder()
125+
InputStream keyFileStream;
126+
if (keyFromClassPath) {
127+
keyFileStream = RemoteGcsHelper.class.getResourceAsStream(stringKeyPath);
128+
if (keyFileStream == null) {
129+
throw new FileNotFoundException(stringKeyPath + " not found in classpath");
130+
}
131+
} else {
132+
keyFileStream = new FileInputStream(stringKeyPath);
133+
}
134+
StorageOptions storageOptions = StorageOptions.builder()
74135
.authCredentials(AuthCredentials.createForJson(keyFileStream))
75136
.projectId(projectId)
76137
.build();
77-
return new RemoteGcsHelper(options, generateBucketName());
138+
return new RemoteGcsHelper(storageOptions);
78139
} catch (FileNotFoundException ex) {
79140
if (log.isLoggable(Level.WARNING)) {
80141
log.log(Level.WARNING, ex.getMessage());
81142
}
82-
return null;
143+
throw GcsHelperException.translate(ex);
83144
} catch (IOException ex) {
84145
if (log.isLoggable(Level.WARNING)) {
85146
log.log(Level.WARNING, ex.getMessage());
86147
}
87-
return null;
148+
throw GcsHelperException.translate(ex);
149+
}
150+
}
151+
152+
private static class DeleteBucketTask implements Callable<Boolean> {
153+
154+
private Storage storage;
155+
private String bucket;
156+
157+
public DeleteBucketTask(Storage storage, String bucket) {
158+
this.storage = storage;
159+
this.bucket = bucket;
160+
}
161+
162+
@Override
163+
public Boolean call() throws Exception {
164+
while (true) {
165+
for (BlobInfo info : storage.list(bucket)) {
166+
storage.delete(bucket, info.name());
167+
}
168+
try {
169+
storage.delete(bucket);
170+
return true;
171+
} catch (StorageException e) {
172+
if (e.code() == 409) {
173+
Thread.sleep(500);
174+
} else {
175+
throw e;
176+
}
177+
}
178+
}
179+
}
180+
}
181+
182+
public static abstract class Option implements java.io.Serializable {
183+
184+
private static final long serialVersionUID = 8849118657896662369L;
185+
186+
public static final class KeyFromClasspath extends Option {
187+
188+
private static final long serialVersionUID = -5506049413185246821L;
189+
190+
private final boolean keyFromClasspath;
191+
192+
public KeyFromClasspath(boolean keyFromClasspath) {
193+
this.keyFromClasspath = keyFromClasspath;
194+
}
195+
196+
public boolean keyFromClasspath() {
197+
return keyFromClasspath;
198+
}
199+
}
200+
201+
Option() {
202+
// package protected
203+
}
204+
205+
public static KeyFromClasspath keyFromClassPath() {
206+
return new KeyFromClasspath(true);
207+
}
208+
209+
static Map<Class<? extends Option>, Option> asImmutableMap(Option... options) {
210+
ImmutableMap.Builder<Class<? extends Option>, Option> builder = ImmutableMap.builder();
211+
for (Option option : options) {
212+
builder.put(option.getClass(), option);
213+
}
214+
return builder.build();
215+
}
216+
}
217+
218+
public static class GcsHelperException extends RuntimeException {
219+
220+
private static final long serialVersionUID = -7756074894502258736L;
221+
222+
public GcsHelperException(String message) {
223+
super(message);
224+
}
225+
226+
public GcsHelperException(String message, Throwable cause) {
227+
super(message, cause);
228+
}
229+
230+
public static GcsHelperException translate(Exception ex) {
231+
return new GcsHelperException(ex.getMessage(), ex);
88232
}
89233
}
90234
}

0 commit comments

Comments
 (0)