Skip to content

Commit d9c6b85

Browse files
aozarovaozarov
committed
Merge pull request #294 from mziccard/add-support-for-rewrite
Add support for blob rewrite
2 parents ff9f60c + 23215a1 commit d9c6b85

11 files changed

Lines changed: 763 additions & 110 deletions

File tree

‎gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import com.google.gcloud.storage.BlobId;
2525
import com.google.gcloud.storage.BlobInfo;
2626
import com.google.gcloud.storage.BlobReadChannel;
27+
import com.google.gcloud.storage.CopyWriter;
2728
import com.google.gcloud.storage.BlobWriteChannel;
2829
import com.google.gcloud.storage.Bucket;
2930
import com.google.gcloud.storage.BucketInfo;
@@ -366,8 +367,8 @@ public String params() {
366367
private static class CopyAction extends StorageAction<CopyRequest> {
367368
@Override
368369
public void run(Storage storage, CopyRequest request) {
369-
BlobInfo copiedBlobInfo = storage.copy(request);
370-
System.out.println("Copied " + copiedBlobInfo);
370+
CopyWriter copyWriter = storage.copy(request);
371+
System.out.println("Copied " + copyWriter.result());
371372
}
372373

373374
@Override

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java‎

Lines changed: 43 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ public class DefaultStorageRpc implements StorageRpc {
7878

7979
// see: https://cloud.google.com/storage/docs/concepts-techniques#practices
8080
private static final Set<Integer> RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 429, 408);
81+
private static final long MEGABYTE = 1024L * 1024L;
8182

8283
public DefaultStorageRpc(StorageOptions options) {
8384
HttpTransport transport = options.httpTransportFactory().create();
@@ -320,30 +321,6 @@ public StorageObject compose(Iterable<StorageObject> sources, StorageObject targ
320321
}
321322
}
322323

323-
@Override
324-
public StorageObject copy(StorageObject source, Map<Option, ?> sourceOptions,
325-
StorageObject target, Map<Option, ?> targetOptions) throws StorageException {
326-
try {
327-
return storage
328-
.objects()
329-
.copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
330-
target.getContentType() != null ? target : null)
331-
.setProjection(DEFAULT_PROJECTION)
332-
.setIfSourceMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
333-
.setIfSourceMetagenerationNotMatch(
334-
IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
335-
.setIfSourceGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
336-
.setIfSourceGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
337-
.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
338-
.setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
339-
.setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
340-
.setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
341-
.execute();
342-
} catch (IOException ex) {
343-
throw translate(ex);
344-
}
345-
}
346-
347324
@Override
348325
public byte[] load(StorageObject from, Map<Option, ?> options)
349326
throws StorageException {
@@ -521,4 +498,46 @@ public String open(StorageObject object, Map<Option, ?> options)
521498
throw translate(ex);
522499
}
523500
}
501+
502+
@Override
503+
public RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException {
504+
return rewrite(rewriteRequest, null);
505+
}
506+
507+
@Override
508+
public RewriteResponse continueRewrite(RewriteResponse previousResponse) throws StorageException {
509+
return rewrite(previousResponse.rewriteRequest, previousResponse.rewriteToken);
510+
}
511+
512+
private RewriteResponse rewrite(RewriteRequest req, String token) throws StorageException {
513+
try {
514+
Long maxBytesRewrittenPerCall = req.megabytesRewrittenPerCall != null
515+
? req.megabytesRewrittenPerCall * MEGABYTE : null;
516+
com.google.api.services.storage.model.RewriteResponse rewriteReponse = storage.objects()
517+
.rewrite(req.source.getBucket(), req.source.getName(), req.target.getBucket(),
518+
req.target.getName(), req.target.getContentType() != null ? req.target : null)
519+
.setRewriteToken(token)
520+
.setMaxBytesRewrittenPerCall(maxBytesRewrittenPerCall)
521+
.setProjection(DEFAULT_PROJECTION)
522+
.setIfSourceMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(req.sourceOptions))
523+
.setIfSourceMetagenerationNotMatch(
524+
IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(req.sourceOptions))
525+
.setIfSourceGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(req.sourceOptions))
526+
.setIfSourceGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(req.sourceOptions))
527+
.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(req.targetOptions))
528+
.setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(req.targetOptions))
529+
.setIfGenerationMatch(IF_GENERATION_MATCH.getLong(req.targetOptions))
530+
.setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(req.targetOptions))
531+
.execute();
532+
return new RewriteResponse(
533+
req,
534+
rewriteReponse.getResource(),
535+
rewriteReponse.getObjectSize().longValue(),
536+
rewriteReponse.getDone(),
537+
rewriteReponse.getRewriteToken(),
538+
rewriteReponse.getTotalBytesRewritten().longValue());
539+
} catch (IOException ex) {
540+
throw translate(ex);
541+
}
542+
}
524543
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java‎

Lines changed: 88 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import java.io.InputStream;
2828
import java.util.List;
2929
import java.util.Map;
30+
import java.util.Objects;
3031

3132
public interface StorageRpc {
3233

@@ -132,6 +133,89 @@ public BatchResponse(Map<StorageObject, Tuple<Boolean, StorageException>> delete
132133
}
133134
}
134135

136+
class RewriteRequest {
137+
138+
public final StorageObject source;
139+
public final Map<StorageRpc.Option, ?> sourceOptions;
140+
public final StorageObject target;
141+
public final Map<StorageRpc.Option, ?> targetOptions;
142+
public final Long megabytesRewrittenPerCall;
143+
144+
public RewriteRequest(StorageObject source, Map<StorageRpc.Option, ?> sourceOptions,
145+
StorageObject target, Map<StorageRpc.Option, ?> targetOptions,
146+
Long megabytesRewrittenPerCall) {
147+
this.source = source;
148+
this.sourceOptions = sourceOptions;
149+
this.target = target;
150+
this.targetOptions = targetOptions;
151+
this.megabytesRewrittenPerCall = megabytesRewrittenPerCall;
152+
}
153+
154+
@Override
155+
public boolean equals(Object obj) {
156+
if (obj == null) {
157+
return false;
158+
}
159+
if (!(obj instanceof RewriteRequest)) {
160+
return false;
161+
}
162+
final RewriteRequest other = (RewriteRequest) obj;
163+
return Objects.equals(this.source, other.source)
164+
&& Objects.equals(this.sourceOptions, other.sourceOptions)
165+
&& Objects.equals(this.target, other.target)
166+
&& Objects.equals(this.targetOptions, other.targetOptions)
167+
&& Objects.equals(this.megabytesRewrittenPerCall, other.megabytesRewrittenPerCall);
168+
}
169+
170+
@Override
171+
public int hashCode() {
172+
return Objects.hash(source, sourceOptions, target, targetOptions, megabytesRewrittenPerCall);
173+
}
174+
}
175+
176+
class RewriteResponse {
177+
178+
public final RewriteRequest rewriteRequest;
179+
public final StorageObject result;
180+
public final long blobSize;
181+
public final boolean isDone;
182+
public final String rewriteToken;
183+
public final long totalBytesRewritten;
184+
185+
public RewriteResponse(RewriteRequest rewriteRequest, StorageObject result, long blobSize,
186+
boolean isDone, String rewriteToken, long totalBytesRewritten) {
187+
this.rewriteRequest = rewriteRequest;
188+
this.result = result;
189+
this.blobSize = blobSize;
190+
this.isDone = isDone;
191+
this.rewriteToken = rewriteToken;
192+
this.totalBytesRewritten = totalBytesRewritten;
193+
}
194+
195+
@Override
196+
public boolean equals(Object obj) {
197+
if (obj == null) {
198+
return false;
199+
}
200+
if (!(obj instanceof RewriteResponse)) {
201+
return false;
202+
}
203+
final RewriteResponse other = (RewriteResponse) obj;
204+
return Objects.equals(this.rewriteRequest, other.rewriteRequest)
205+
&& Objects.equals(this.result, other.result)
206+
&& Objects.equals(this.rewriteToken, other.rewriteToken)
207+
&& this.blobSize == other.blobSize
208+
&& Objects.equals(this.isDone, other.isDone)
209+
&& this.totalBytesRewritten == other.totalBytesRewritten;
210+
}
211+
212+
@Override
213+
public int hashCode() {
214+
return Objects.hash(rewriteRequest, result, blobSize, isDone, rewriteToken,
215+
totalBytesRewritten);
216+
}
217+
}
218+
135219
Bucket create(Bucket bucket, Map<Option, ?> options) throws StorageException;
136220

137221
StorageObject create(StorageObject object, InputStream content, Map<Option, ?> options)
@@ -161,9 +245,6 @@ StorageObject patch(StorageObject storageObject, Map<Option, ?> options)
161245
StorageObject compose(Iterable<StorageObject> sources, StorageObject target,
162246
Map<Option, ?> targetOptions) throws StorageException;
163247

164-
StorageObject copy(StorageObject source, Map<Option, ?> sourceOptions,
165-
StorageObject target, Map<Option, ?> targetOptions) throws StorageException;
166-
167248
byte[] load(StorageObject storageObject, Map<Option, ?> options)
168249
throws StorageException;
169250

@@ -174,4 +255,8 @@ byte[] read(StorageObject from, Map<Option, ?> options, long position, int bytes
174255

175256
void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
176257
long destOffset, int length, boolean last) throws StorageException;
258+
259+
RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException;
260+
261+
RewriteResponse continueRewrite(RewriteResponse previousResponse) throws StorageException;
177262
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -213,19 +213,20 @@ public Blob update(BlobInfo blobInfo, BlobTargetOption... options) {
213213
}
214214

215215
/**
216-
* Copies this blob to the specified target. Possibly copying also some of the metadata
217-
* (e.g. content-type).
216+
* Sends a copy request for the current blob to the target blob. Possibly also some of the
217+
* metadata are copied (e.g. content-type).
218218
*
219219
* @param targetBlob target blob's id
220220
* @param options source blob options
221-
* @return the copied blob
221+
* @return a {@link CopyWriter} object that can be used to get information on the newly created
222+
* blob or to complete the copy if more than one RPC request is needed
222223
* @throws StorageException upon failure
223224
*/
224-
public Blob copyTo(BlobId targetBlob, BlobSourceOption... options) {
225-
BlobInfo updatedInfo = info.toBuilder().blobId(targetBlob).build();
225+
public CopyWriter copyTo(BlobId targetBlob, BlobSourceOption... options) {
226+
BlobInfo updatedInfo = BlobInfo.builder(targetBlob).build();
226227
CopyRequest copyRequest = CopyRequest.builder().source(info.bucket(), info.name())
227228
.sourceOptions(convert(info, options)).target(updatedInfo).build();
228-
return new Blob(storage, storage.copy(copyRequest));
229+
return storage.copy(copyRequest);
229230
}
230231

231232
/**
@@ -240,33 +241,35 @@ public boolean delete(BlobSourceOption... options) {
240241
}
241242

242243
/**
243-
* Copies this blob to the target bucket, preserving its name. Possibly copying also some of the
244-
* metadata (e.g. content-type).
244+
* Sends a copy request for the current blob to the target bucket, preserving its name. Possibly
245+
* copying also some of the metadata (e.g. content-type).
245246
*
246247
* @param targetBucket target bucket's name
247248
* @param options source blob options
248-
* @return the copied blob
249+
* @return a {@link CopyWriter} object that can be used to get information on the newly created
250+
* blob or to complete the copy if more than one RPC request is needed
249251
* @throws StorageException upon failure
250252
*/
251-
public Blob copyTo(String targetBucket, BlobSourceOption... options) {
253+
public CopyWriter copyTo(String targetBucket, BlobSourceOption... options) {
252254
return copyTo(targetBucket, info.name(), options);
253255
}
254256

255257
/**
256-
* Copies this blob to the target bucket with a new name. Possibly copying also some of the
257-
* metadata (e.g. content-type).
258+
* Sends a copy request for the current blob to the target blob. Possibly also some of the
259+
* metadata are copied (e.g. content-type).
258260
*
259261
* @param targetBucket target bucket's name
260262
* @param targetBlob target blob's name
261263
* @param options source blob options
262-
* @return the copied blob
264+
* @return a {@link CopyWriter} object that can be used to get information on the newly created
265+
* blob or to complete the copy if more than one RPC request is needed
263266
* @throws StorageException upon failure
264267
*/
265-
public Blob copyTo(String targetBucket, String targetBlob, BlobSourceOption... options) {
266-
BlobInfo updatedInfo = info.toBuilder().blobId(BlobId.of(targetBucket, targetBlob)).build();
268+
public CopyWriter copyTo(String targetBucket, String targetBlob, BlobSourceOption... options) {
269+
BlobInfo updatedInfo = BlobInfo.builder(targetBucket, targetBlob).build();
267270
CopyRequest copyRequest = CopyRequest.builder().source(info.bucket(), info.name())
268271
.sourceOptions(convert(info, options)).target(updatedInfo).build();
269-
return new Blob(storage, storage.copy(copyRequest));
272+
return storage.copy(copyRequest);
270273
}
271274

272275
/**

0 commit comments

Comments
 (0)