Skip to content

Commit 277f985

Browse files
thiyaguk09thiyaguk09
authored
feat(Storage): Enable full object checksum validation on JSON path (#8825)
1 parent 650b5e3 commit 277f985

10 files changed

Lines changed: 460 additions & 10 deletions

File tree

‎Core/src/Upload/MultipartUploader.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,9 @@ private function prepareRequest()
9898
$headers['Content-Length'] = $size;
9999
}
100100

101+
$customHeaders = $this->requestOptions['restOptions']['headers'] ?? [];
102+
$headers = array_merge($headers, $customHeaders);
103+
101104
return new Request(
102105
'POST',
103106
$this->uri,

‎Core/src/Upload/ResumableUploader.php‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,16 @@ public function upload()
170170
'Content-Range' => "bytes $rangeStart-$rangeEnd/$size",
171171
];
172172

173+
$customHeaders = $this->requestOptions['restOptions']['headers'] ?? [];
174+
175+
// Check if this chunk is the final one
176+
$isFinalChunk = ($size !== '*' && (int) ($rangeEnd + 1) === (int) $size);
177+
if (!$isFinalChunk) {
178+
unset($customHeaders['X-Goog-Hash']);
179+
}
180+
181+
$headers = array_merge($headers, $customHeaders);
182+
173183
$request = new Request(
174184
'PUT',
175185
$resumeUri,

‎Core/src/Upload/StreamableUploader.php‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,12 @@ public function upload($writeSize = null)
4343
return [];
4444
}
4545

46+
$isFinalRequest = ($writeSize === null);
47+
4648
// find or create the resumeUri
4749
$resumeUri = $this->getResumeUri();
4850

49-
if ($writeSize) {
51+
if ($writeSize !== null) {
5052
$rangeEnd = $this->rangeStart + $writeSize - 1;
5153
$data = $this->data->read($writeSize);
5254
} else {
@@ -62,6 +64,14 @@ public function upload($writeSize = null)
6264
'Content-Range' => "bytes {$this->rangeStart}-$rangeEnd/*"
6365
];
6466

67+
$customHeaders = $this->requestOptions['restOptions']['headers'] ?? [];
68+
69+
// Only include X-Goog-Hash if this is the final request
70+
if (!$isFinalRequest) {
71+
unset($customHeaders['X-Goog-Hash']);
72+
}
73+
$headers = array_merge($headers, $customHeaders);
74+
6575
$request = new Request(
6676
'PUT',
6777
$resumeUri,

‎Core/tests/Unit/Upload/MultipartUploaderTest.php‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,48 @@ public function testUploadsAsyncData()
8585
$actualPromise->wait()
8686
);
8787
}
88+
89+
public function testUploadsWithCustomHeaders()
90+
{
91+
$customHeaders = [
92+
'X-Goog-Custom-Header' => 'custom-value',
93+
'User-Agent' => 'custom-ua'
94+
];
95+
96+
$requestWrapper = $this->prophesize(RequestWrapper::class);
97+
$stream = Utils::streamFor('abcd');
98+
$successBody = '{"canI":"kickIt"}';
99+
$response = new Response(200, [], $successBody);
100+
101+
$requestWrapper->send(
102+
Argument::that(function (RequestInterface $request) use ($customHeaders) {
103+
foreach ($customHeaders as $key => $value) {
104+
if ($request->getHeaderLine($key) !== $value) {
105+
return false;
106+
}
107+
}
108+
109+
$contentType = $request->getHeaderLine('Content-Type');
110+
return str_contains($contentType, 'multipart/related')
111+
&& str_contains($contentType, 'boundary=boundary');
112+
}),
113+
Argument::type('array')
114+
)->willReturn($response);
115+
116+
$uploader = new MultipartUploader(
117+
$requestWrapper->reveal(),
118+
$stream,
119+
'http://www.example.com',
120+
[
121+
'restOptions' => [
122+
'headers' => $customHeaders
123+
]
124+
]
125+
);
126+
127+
$this->assertEquals(json_decode($successBody, true), $uploader->upload());
128+
}
129+
88130
/**
89131
* @dataProvider streamSizes
90132
*/

‎Core/tests/Unit/Upload/ResumableUploaderTest.php‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,119 @@ public function testRetryOptionsPassing()
238238
$this->assertTrue($retryListenerCalled);
239239
}
240240

241+
public function testUploadSendsGoogHashOnFinalChunk()
242+
{
243+
$hashValue = 'crc32c=abc123';
244+
$resumeUri = 'http://some-resume-uri.example.com';
245+
246+
$this->requestWrapper->send(
247+
Argument::which('getMethod', 'POST'),
248+
Argument::type('array')
249+
)->willReturn(new Response(200, ['Location' => $resumeUri]));
250+
251+
$this->requestWrapper->send(
252+
Argument::that(function (RequestInterface $request) use ($hashValue) {
253+
return $request->getHeaderLine('X-Goog-Hash') === $hashValue;
254+
}),
255+
Argument::type('array')
256+
)->willReturn(new Response(200, [], $this->successBody));
257+
258+
$uploader = new ResumableUploader(
259+
$this->requestWrapper->reveal(),
260+
$this->stream,
261+
'http://www.example.com',
262+
[
263+
'restOptions' => [
264+
'headers' => ['X-Goog-Hash' => $hashValue]
265+
]
266+
]
267+
);
268+
269+
$this->assertEquals(json_decode($this->successBody, true), $uploader->upload());
270+
}
271+
272+
public function testUploadDoesNotSendGoogHashOnIntermediateChunk()
273+
{
274+
$hashValue = 'crc32c=abc123';
275+
$resumeUri = 'http://some-resume-uri.example.com';
276+
277+
$this->requestWrapper->send(
278+
Argument::which('getMethod', 'POST'),
279+
Argument::type('array')
280+
)->willReturn(new Response(200, ['Location' => $resumeUri]));
281+
282+
$this->requestWrapper->send(
283+
Argument::that(function (RequestInterface $request) {
284+
return $request->getHeaderLine('Content-Range') === 'bytes 0-1/4'
285+
&& !$request->hasHeader('X-Goog-Hash');
286+
}),
287+
Argument::type('array')
288+
)->willReturn(new Response(308, ['Range' => 'bytes 0-1']));
289+
290+
$this->requestWrapper->send(
291+
Argument::that(function (RequestInterface $request) {
292+
return $request->getHeaderLine('Content-Range') === 'bytes 2-3/4';
293+
}),
294+
Argument::type('array')
295+
)->willReturn(new Response(200, [], $this->successBody));
296+
297+
$uploader = new ResumableUploader(
298+
$this->requestWrapper->reveal(),
299+
$this->stream, // size 4
300+
'http://www.example.com',
301+
[
302+
'chunkSize' => 2, // Force multi-chunk upload
303+
'restOptions' => [
304+
'headers' => ['X-Goog-Hash' => $hashValue]
305+
]
306+
]
307+
);
308+
309+
$this->assertEquals(json_decode($this->successBody, true), $uploader->upload());
310+
}
311+
312+
public function testGoogHashOnlyOnFinalChunkOfMultiChunkUpload()
313+
{
314+
$hashValue = 'crc32c=abc123';
315+
$resumeUri = 'http://some-resume-uri.example.com';
316+
$this->stream = Utils::streamFor('01234567'); // 8 bytes total
317+
318+
$this->requestWrapper->send(
319+
Argument::which('getMethod', 'POST'),
320+
Argument::type('array')
321+
)->willReturn(new Response(200, ['Location' => $resumeUri]));
322+
323+
$this->requestWrapper->send(
324+
Argument::that(function (RequestInterface $request) {
325+
return $request->getHeaderLine('Content-Range') === 'bytes 0-3/8'
326+
&& !$request->hasHeader('X-Goog-Hash');
327+
}),
328+
Argument::type('array')
329+
)->willReturn(new Response(308, ['Range' => 'bytes 0-3']));
330+
331+
$this->requestWrapper->send(
332+
Argument::that(function (RequestInterface $request) use ($hashValue) {
333+
return $request->getHeaderLine('Content-Range') === 'bytes 4-7/8'
334+
&& $request->getHeaderLine('X-Goog-Hash') === $hashValue;
335+
}),
336+
Argument::type('array')
337+
)->willReturn(new Response(200, [], $this->successBody));
338+
339+
$uploader = new ResumableUploader(
340+
$this->requestWrapper->reveal(),
341+
$this->stream,
342+
'http://www.example.com',
343+
[
344+
'chunkSize' => 4,
345+
'restOptions' => [
346+
'headers' => ['X-Goog-Hash' => $hashValue]
347+
]
348+
]
349+
);
350+
351+
$this->assertEquals(json_decode($this->successBody, true), $uploader->upload());
352+
}
353+
241354
public function testThrowsExceptionWhenAttemptsAsyncUpload()
242355
{
243356
$this->expectException(GoogleException::class);

‎Core/tests/Unit/Upload/StreamableUploaderTest.php‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,115 @@ public function testLastChunkSendsCorrectHeaders()
174174
$uploader->upload();
175175
}
176176

177+
public function testUploadWithCustomGoogHashHeader()
178+
{
179+
$hashValue = 'md5=abc123';
180+
$resumeUri = 'http://some-resume-uri.example.com';
181+
182+
$resumeResponse = new Response(200, ['Location' => $resumeUri]);
183+
$this->requestWrapper->send(
184+
Argument::type(RequestInterface::class),
185+
Argument::any()
186+
)->willReturn($resumeResponse)->shouldBeCalled();
187+
188+
$uploadResponse = new Response(200, ['Location' => $resumeUri], $this->successBody);
189+
$this->requestWrapper->send(
190+
Argument::that(function ($request) use ($resumeUri, $hashValue) {
191+
return (string) $request->getUri() === $resumeUri
192+
&& $request->getHeaderLine('X-Goog-Hash') === $hashValue;
193+
}),
194+
Argument::any()
195+
)->willReturn($uploadResponse)->shouldBeCalled();
196+
197+
$uploader = new StreamableUploader(
198+
$this->requestWrapper->reveal(),
199+
$this->stream,
200+
'http://www.example.com',
201+
[
202+
'restOptions' => [
203+
'headers' => [
204+
'X-Goog-Hash' => $hashValue,
205+
'Other-Header' => 'should-be-ignored'
206+
]
207+
]
208+
]
209+
);
210+
211+
$this->stream->write("some data");
212+
213+
$this->assertEquals(json_decode($this->successBody, true), $uploader->upload());
214+
}
215+
216+
public function testUploadDoesNotSendGoogHashWhenConditionNotMet()
217+
{
218+
$hashValue = 'md5=abc123';
219+
$resumeUri = 'http://some-resume-uri.example.com';
220+
221+
$resumeResponse = new Response(200, ['Location' => $resumeUri]);
222+
$this->requestWrapper->send(
223+
Argument::which('getMethod', 'POST'),
224+
Argument::type('array')
225+
)->willReturn($resumeResponse);
226+
227+
$uploadResponse = new Response(200, [], $this->successBody);
228+
$this->requestWrapper->send(
229+
Argument::that(function (RequestInterface $request) {
230+
return !$request->hasHeader('X-Goog-Hash');
231+
}),
232+
Argument::type('array')
233+
)->willReturn($uploadResponse);
234+
235+
$uploader = new StreamableUploader(
236+
$this->requestWrapper->reveal(),
237+
$this->stream,
238+
'http://www.example.com',
239+
[
240+
'restOptions' => [
241+
'headers' => ['X-Goog-Hash' => $hashValue]
242+
]
243+
]
244+
);
245+
246+
$this->stream->write("0123456789ABCDEF");
247+
248+
$this->assertEquals(json_decode($this->successBody, true), $uploader->upload(16));
249+
}
250+
251+
public function testUploadSendsGoogHashOnFinalStep()
252+
{
253+
$hashValue = 'md5=finalHash';
254+
$resumeUri = 'http://some-resume-uri.example.com';
255+
256+
$resumeResponse = new Response(200, ['Location' => $resumeUri]);
257+
$this->requestWrapper->send(
258+
Argument::which('getMethod', 'POST'),
259+
Argument::type('array')
260+
)->willReturn($resumeResponse)->shouldBeCalled();
261+
262+
$uploadResponse = new Response(200, [], $this->successBody);
263+
$this->requestWrapper->send(
264+
Argument::that(function (RequestInterface $request) use ($hashValue) {
265+
return $request->getHeaderLine('X-Goog-Hash') === $hashValue;
266+
}),
267+
Argument::type('array')
268+
)->willReturn($uploadResponse)->shouldBeCalled();
269+
270+
$uploader = new StreamableUploader(
271+
$this->requestWrapper->reveal(),
272+
$this->stream,
273+
'http://www.example.com',
274+
[
275+
'restOptions' => [
276+
'headers' => ['X-Goog-Hash' => $hashValue]
277+
]
278+
]
279+
);
280+
281+
$this->stream->write("final data");
282+
283+
$this->assertEquals(json_decode($this->successBody, true), $uploader->upload());
284+
}
285+
177286
public function testThrowsExceptionWhenAttemptsAsyncUpload()
178287
{
179288
$this->expectException(GoogleException::class);

‎Storage/composer.json‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,19 @@
55
"minimum-stability": "stable",
66
"require": {
77
"php": "^8.1",
8-
"google/cloud-core": "^1.57",
8+
"google/cloud-core": "^1.72.0",
99
"ramsey/uuid": "^4.2.3"
1010
},
1111
"require-dev": {
1212
"phpunit/phpunit": "^9.0",
1313
"phpspec/prophecy-phpunit": "^2.0",
1414
"squizlabs/php_codesniffer": "2.*",
15-
"phpdocumentor/reflection": "^5.3.3||^6.0",
16-
"phpdocumentor/reflection-docblock": "^5.3.3||^6.0",
15+
"phpdocumentor/reflection": "^6.0",
16+
"phpdocumentor/reflection-docblock": "^5.3.3",
1717
"erusev/parsedown": "^1.6",
1818
"phpseclib/phpseclib": "^2.0||^3.0",
19-
"google/cloud-pubsub": "^2.0"
19+
"google/cloud-pubsub": "^2.0",
20+
"nikic/php-parser": "^5"
2021
},
2122
"suggest": {
2223
"phpseclib/phpseclib": "May be used in place of OpenSSL for creating signed Cloud Storage URLs. Please require version ^2.",

0 commit comments

Comments
 (0)