Skip to content

Commit 4c2f44e

Browse files
BenWhiteheadBenWhitehead
authored
feat: update Storage.createFrom(BlobInfo, Path) to have 150% higher throughput (#2059)
When uploading a file where we are able to rewind to an arbitrary offset, we can be more optimistic in the way we send requests to GCS. Add new code middleware to allow PUTing an entire file to GCS in a single request, and using query resumable session to recover from the specific offset in the case of retryable error. ### Benchmark Results #### Methodology Generate a random file on disk of size `128KiB..2GiB` from `/dev/urandom`, then upload the generated file using `Storage.createFrom(BlobInfo, Path)`. Perform each 4096 times. Run on a c2-standard-60 instance is us-central1 against a regional bucket located in us-central1. #### Results The following summary of throughput in MiB/s as observed between the existing implementation, and the new implementation proposed in this PR. ``` count mean std min 50% 75% 90% 99% max runId ApiName createFrom - existing JSON 4096.0 66.754 10.988 3.249 67.317 73.476 78.961 91.197 107.247 createFrom - new JSON 4096.0 158.769 67.105 4.600 170.680 218.618 240.992 266.297 305.205 ``` #### Comparison When comparing the new implementation to the existing implementation we get the following improvement to throughput (higher is better): ``` stat pct mean 137.841 50% 153.547 90% 205.204 99% 192.003 ```
1 parent d54b9cd commit 4c2f44e

25 files changed

Lines changed: 3184 additions & 20 deletions

‎google-cloud-storage/src/main/java/com/google/cloud/storage/ByteRangeSpec.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616

1717
package com.google.cloud.storage;
1818

19+
import static com.google.api.client.util.Preconditions.checkNotNull;
20+
import static com.google.common.base.Preconditions.checkArgument;
21+
1922
import com.google.api.core.InternalApi;
2023
import com.google.common.base.MoreObjects;
2124
import com.google.common.base.MoreObjects.ToStringHelper;
@@ -124,6 +127,20 @@ static ByteRangeSpec explicitClosed(
124127
return create(beginOffset, endOffsetInclusive, LeftClosedRightClosedByteRangeSpec::new);
125128
}
126129

130+
static ByteRangeSpec parse(String string) {
131+
checkNotNull(string, "Range header is null");
132+
checkArgument(string.startsWith("bytes="), "malformed Range header value: %s", string);
133+
134+
int i = string.indexOf('-');
135+
String minS = string.substring(6, i);
136+
String maxS = string.substring(i + 1);
137+
138+
long min = Long.parseLong(minS);
139+
long max = Long.parseLong(maxS);
140+
141+
return explicitClosed(min, max);
142+
}
143+
127144
private static ByteRangeSpec create(
128145
@Nullable Long beginOffset,
129146
@Nullable Long length,

‎google-cloud-storage/src/main/java/com/google/cloud/storage/ByteSizeConstants.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,16 @@ final class ByteSizeConstants {
2323
static final int _256KiB = 256 * _1KiB;
2424
static final int _384KiB = 384 * _1KiB;
2525
static final int _512KiB = 512 * _1KiB;
26+
static final int _768KiB = 768 * _1KiB;
2627
static final int _1MiB = 1024 * _1KiB;
2728
static final int _2MiB = 2 * _1MiB;
2829
static final int _16MiB = 16 * _1MiB;
2930
static final int _32MiB = 32 * _1MiB;
3031

32+
static final long _128KiBL = 131072L;
33+
static final long _256KiBL = 262144L;
34+
static final long _512KiBL = 524288L;
35+
static final long _768KiBL = 786432L;
36+
3137
private ByteSizeConstants() {}
3238
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/HttpClientContext.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.api.client.http.HttpHeaders;
20+
import com.google.api.client.http.HttpRequestFactory;
21+
import com.google.api.client.json.JsonObjectParser;
22+
import com.google.api.client.util.ObjectParser;
23+
import com.google.cloud.storage.spi.v1.StorageRpc;
24+
import io.opencensus.trace.Span;
25+
import io.opencensus.trace.Tracer;
26+
import io.opencensus.trace.Tracing;
27+
import java.util.List;
28+
import org.checkerframework.checker.nullness.qual.NonNull;
29+
import org.checkerframework.checker.nullness.qual.Nullable;
30+
31+
final class HttpClientContext {
32+
33+
private final HttpRequestFactory requestFactory;
34+
private final ObjectParser objectParser;
35+
private final Tracer tracer;
36+
37+
private HttpClientContext(
38+
HttpRequestFactory requestFactory, ObjectParser objectParser, Tracer tracer) {
39+
this.requestFactory = requestFactory;
40+
this.objectParser = objectParser;
41+
this.tracer = tracer;
42+
}
43+
44+
@SuppressWarnings({"unchecked", "SameParameterValue"})
45+
static @Nullable String firstHeaderValue(
46+
@NonNull HttpHeaders headers, @NonNull String headerName) {
47+
Object v = headers.get(headerName);
48+
// HttpHeaders doesn't type its get method, so we have to jump through hoops here
49+
if (v instanceof List) {
50+
List<String> list = (List<String>) v;
51+
return list.get(0);
52+
} else {
53+
return null;
54+
}
55+
}
56+
57+
public HttpRequestFactory getRequestFactory() {
58+
return requestFactory;
59+
}
60+
61+
public ObjectParser getObjectParser() {
62+
return objectParser;
63+
}
64+
65+
public Tracer getTracer() {
66+
return tracer;
67+
}
68+
69+
public Span startSpan(String name) {
70+
// record events is hardcoded to true in HttpStorageRpc, preserve it here
71+
return tracer.spanBuilder(name).setRecordEvents(true).startSpan();
72+
}
73+
74+
static HttpClientContext from(StorageRpc storageRpc) {
75+
return new HttpClientContext(
76+
storageRpc.getStorage().getRequestFactory(),
77+
storageRpc.getStorage().getObjectParser(),
78+
Tracing.getTracer());
79+
}
80+
81+
public static HttpClientContext of(
82+
HttpRequestFactory requestFactory, JsonObjectParser jsonObjectParser) {
83+
return new HttpClientContext(requestFactory, jsonObjectParser, Tracing.getTracer());
84+
}
85+
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/HttpContentRange.java‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import static com.google.common.base.Preconditions.checkArgument;
20+
21+
import com.google.common.base.MoreObjects;
22+
import java.util.Objects;
23+
import java.util.function.UnaryOperator;
24+
25+
abstract class HttpContentRange {
26+
27+
private final boolean finalizing;
28+
29+
private HttpContentRange(boolean finalizing) {
30+
this.finalizing = finalizing;
31+
}
32+
33+
public abstract String getHeaderValue();
34+
35+
public boolean isFinalizing() {
36+
return finalizing;
37+
}
38+
39+
static Total of(ByteRangeSpec spec, long size) {
40+
checkArgument(size >= 0, "size must be >= 0");
41+
checkArgument(size >= spec.endOffsetInclusive(), "size must be >= end");
42+
return new Total(spec, size);
43+
}
44+
45+
static Incomplete of(ByteRangeSpec spec) {
46+
return new Incomplete(spec);
47+
}
48+
49+
static Size of(long size) {
50+
checkArgument(size >= 0, "size must be >= 0");
51+
return new Size(size);
52+
}
53+
54+
static Query query() {
55+
return Query.INSTANCE;
56+
}
57+
58+
static HttpContentRange parse(String string) {
59+
if ("bytes */*".equals(string)) {
60+
return HttpContentRange.query();
61+
} else if (string.startsWith("bytes */")) {
62+
return HttpContentRange.of(Long.parseLong(string.substring(8)));
63+
} else {
64+
int idxDash = string.indexOf('-');
65+
int idxSlash = string.indexOf('/');
66+
67+
String beginS = string.substring(6, idxDash);
68+
String endS = string.substring(idxDash + 1, idxSlash);
69+
long begin = Long.parseLong(beginS);
70+
long end = Long.parseLong(endS);
71+
if (string.endsWith("/*")) {
72+
return HttpContentRange.of(ByteRangeSpec.explicitClosed(begin, end));
73+
} else {
74+
String sizeS = string.substring(idxSlash + 1);
75+
long size = Long.parseLong(sizeS);
76+
return HttpContentRange.of(ByteRangeSpec.explicitClosed(begin, end), size);
77+
}
78+
}
79+
}
80+
81+
static final class Incomplete extends HttpContentRange implements HasRange<Incomplete> {
82+
83+
private final ByteRangeSpec spec;
84+
85+
private Incomplete(ByteRangeSpec spec) {
86+
super(false);
87+
this.spec = spec;
88+
}
89+
90+
@Override
91+
public String getHeaderValue() {
92+
return String.format("bytes %d-%d/*", spec.beginOffset(), spec.endOffsetInclusive());
93+
}
94+
95+
@Override
96+
public ByteRangeSpec range() {
97+
return spec;
98+
}
99+
100+
@Override
101+
public Incomplete map(UnaryOperator<ByteRangeSpec> f) {
102+
return new Incomplete(f.apply(spec));
103+
}
104+
105+
@Override
106+
public boolean equals(Object o) {
107+
if (this == o) {
108+
return true;
109+
}
110+
if (!(o instanceof Incomplete)) {
111+
return false;
112+
}
113+
Incomplete that = (Incomplete) o;
114+
return Objects.equals(spec, that.spec);
115+
}
116+
117+
@Override
118+
public int hashCode() {
119+
return Objects.hash(spec);
120+
}
121+
122+
@Override
123+
public String toString() {
124+
return MoreObjects.toStringHelper(this).add("spec", spec).toString();
125+
}
126+
}
127+
128+
static final class Total extends HttpContentRange implements HasRange<Total>, HasSize {
129+
130+
private final ByteRangeSpec spec;
131+
private final long size;
132+
133+
private Total(ByteRangeSpec spec, long size) {
134+
super(true);
135+
this.spec = spec;
136+
this.size = size;
137+
}
138+
139+
@Override
140+
public String getHeaderValue() {
141+
return String.format("bytes %d-%d/%d", spec.beginOffset(), spec.endOffsetInclusive(), size);
142+
}
143+
144+
@Override
145+
public long getSize() {
146+
return size;
147+
}
148+
149+
@Override
150+
public ByteRangeSpec range() {
151+
return spec;
152+
}
153+
154+
@Override
155+
public Total map(UnaryOperator<ByteRangeSpec> f) {
156+
return new Total(f.apply(spec), size);
157+
}
158+
159+
@Override
160+
public boolean equals(Object o) {
161+
if (this == o) {
162+
return true;
163+
}
164+
if (!(o instanceof Total)) {
165+
return false;
166+
}
167+
Total total = (Total) o;
168+
return size == total.size && Objects.equals(spec, total.spec);
169+
}
170+
171+
@Override
172+
public int hashCode() {
173+
return Objects.hash(spec, size);
174+
}
175+
176+
@Override
177+
public String toString() {
178+
return MoreObjects.toStringHelper(this).add("spec", spec).add("size", size).toString();
179+
}
180+
}
181+
182+
static final class Size extends HttpContentRange implements HasSize {
183+
184+
private final long size;
185+
186+
private Size(long size) {
187+
super(true);
188+
this.size = size;
189+
}
190+
191+
@Override
192+
public String getHeaderValue() {
193+
return String.format("bytes */%d", size);
194+
}
195+
196+
@Override
197+
public long getSize() {
198+
return size;
199+
}
200+
201+
@Override
202+
public boolean equals(Object o) {
203+
if (this == o) {
204+
return true;
205+
}
206+
if (!(o instanceof Size)) {
207+
return false;
208+
}
209+
Size size1 = (Size) o;
210+
return size == size1.size;
211+
}
212+
213+
@Override
214+
public int hashCode() {
215+
return Objects.hash(size);
216+
}
217+
218+
@Override
219+
public String toString() {
220+
return MoreObjects.toStringHelper(this).add("size", size).toString();
221+
}
222+
}
223+
224+
static final class Query extends HttpContentRange {
225+
226+
private static final Query INSTANCE = new Query();
227+
228+
private Query() {
229+
super(false);
230+
}
231+
232+
@Override
233+
public String getHeaderValue() {
234+
return "bytes */*";
235+
}
236+
}
237+
238+
interface HasRange<T extends HttpContentRange> {
239+
240+
ByteRangeSpec range();
241+
242+
T map(UnaryOperator<ByteRangeSpec> f);
243+
}
244+
245+
interface HasSize {
246+
247+
long getSize();
248+
}
249+
}

0 commit comments

Comments
 (0)