diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java new file mode 100644 index 000000000000..5bcadcf9e3af --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializer.java @@ -0,0 +1,82 @@ +/* + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Canonical extension header serializer. + * + * @see + * Canonical Extension Headers + */ +public class CanonicalExtensionHeadersSerializer { + + private static final char HEADER_SEPARATOR = ':'; + + public StringBuilder serialize(Map canonicalizedExtensionHeaders) { + + StringBuilder serializedHeaders = new StringBuilder(); + + if (canonicalizedExtensionHeaders == null || + canonicalizedExtensionHeaders.isEmpty()) { + + return serializedHeaders; + } + + // Make all custom header names lowercase. + Map lowercaseHeaders = new HashMap<>(); + for (String headerName : new ArrayList<>(canonicalizedExtensionHeaders.keySet())) { + + String lowercaseHeaderName = headerName.toLowerCase(); + + // If present, remove the x-goog-encryption-key and x-goog-encryption-key-sha256 headers. + if ("x-goog-encryption-key".equals(lowercaseHeaderName) || + "x-goog-encryption-key-sha256".equals(lowercaseHeaderName)) { + + continue; + } + + lowercaseHeaders.put(lowercaseHeaderName, canonicalizedExtensionHeaders.get(headerName)); + } + + // Sort all custom headers by header name using a lexicographical sort by code point value. + List sortedHeaderNames = new ArrayList<>(lowercaseHeaders.keySet()); + Collections.sort(sortedHeaderNames); + + for (String headerName : sortedHeaderNames) { + serializedHeaders + .append(headerName).append(HEADER_SEPARATOR) + .append(lowercaseHeaders.get(headerName) + // Remove any whitespace around the colon that appears after the header name. + .trim() + // Replace any folding whitespace or newlines (CRLF or LF) with a single space. + .replaceAll("[\\s]{2,}"," ") + .replaceAll("(\\t|\\r?\\n)+", " ")) + // Append a newline (U+000A) to each custom header. + .append(SignatureInfo.COMPONENT_SEPARATOR); + } + + // Concatenate all custom headers + return serializedHeaders; + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java new file mode 100644 index 000000000000..bd9ae3ee34c8 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/SignatureInfo.java @@ -0,0 +1,168 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static com.google.common.base.Preconditions.checkArgument; +import java.net.URI; +import java.util.Map; + +/** + * Signature Info holds payload components of the string that requires signing. + * + * @see + * Components + */ +public class SignatureInfo { + + public static final char COMPONENT_SEPARATOR = '\n'; + + private final HttpMethod httpVerb; + private final String contentMd5; + private final String contentType; + private final long expiration; + private final Map canonicalizedExtensionHeaders; + private final URI canonicalizedResource; + + private SignatureInfo(Builder builder) { + this.httpVerb = builder.httpVerb; + this.contentMd5 = builder.contentMd5; + this.contentType = builder.contentType; + this.expiration = builder.expiration; + this.canonicalizedExtensionHeaders = builder.canonicalizedExtensionHeaders; + this.canonicalizedResource = builder.canonicalizedResource; + } + + /** + * Constructs payload to be signed. + * + * @return paylod to sign + * @see Signed URLs + */ + public String constructUnsignedPayload() { + StringBuilder payload = new StringBuilder(); + + payload.append(httpVerb.name()).append(COMPONENT_SEPARATOR); + if (contentMd5 != null) { + payload.append(contentMd5); + } + payload.append(COMPONENT_SEPARATOR); + + if (contentType != null) { + payload.append(contentType); + } + payload.append(COMPONENT_SEPARATOR); + + payload.append(expiration).append(COMPONENT_SEPARATOR); + + if (canonicalizedExtensionHeaders != null) { + payload.append(new CanonicalExtensionHeadersSerializer() + .serialize(canonicalizedExtensionHeaders)); + } + + payload.append(canonicalizedResource); + + return payload.toString(); + } + + public HttpMethod getHttpVerb() { + return httpVerb; + } + + public String getContentMd5() { + return contentMd5; + } + + public String getContentType() { + return contentType; + } + + public long getExpiration() { + return expiration; + } + + public Map getCanonicalizedExtensionHeaders() { + return canonicalizedExtensionHeaders; + } + + public URI getCanonicalizedResource() { + return canonicalizedResource; + } + + public static final class Builder { + + private final HttpMethod httpVerb; + private String contentMd5; + private String contentType; + private final long expiration; + private Map canonicalizedExtensionHeaders; + private final URI canonicalizedResource; + + /** + * Constructs builder. + * + * @param httpVerb the HTTP method + * @param expiration the EPOX expiration date + * @param canonicalizedResource the resource URI + * @throws IllegalArgumentException if required field is not provided. + */ + public Builder(HttpMethod httpVerb, long expiration, URI canonicalizedResource) { + this.httpVerb = httpVerb; + this.expiration = expiration; + this.canonicalizedResource = canonicalizedResource; + } + + public Builder(SignatureInfo signatureInfo) { + this.httpVerb = signatureInfo.httpVerb; + this.contentMd5 = signatureInfo.contentMd5; + this.contentType = signatureInfo.contentType; + this.expiration = signatureInfo.expiration; + this.canonicalizedExtensionHeaders = signatureInfo.canonicalizedExtensionHeaders; + this.canonicalizedResource = signatureInfo.canonicalizedResource; + } + + public Builder setContentMd5(String contentMd5) { + this.contentMd5 = contentMd5; + + return this; + } + + public Builder setContentType(String contentType) { + this.contentType = contentType; + + return this; + } + + public Builder setCanonicalizedExtensionHeaders( + Map canonicalizedExtensionHeaders) { + this.canonicalizedExtensionHeaders = canonicalizedExtensionHeaders; + + return this; + } + + /** + * Creates an {@code SignatureInfo} object from this builder. + */ + public SignatureInfo build() { + checkArgument(httpVerb != null, "Required HTTP method"); + checkArgument(canonicalizedResource != null, "Required canonicalized resource"); + checkArgument(expiration >= 0, "Expiration must be greater than or equal to zero"); + + return new SignatureInfo(this); + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 5fd9fe38ae0e..5fb4e725db3a 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -46,6 +46,7 @@ import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -854,7 +855,7 @@ class SignUrlOption implements Serializable { private final Object value; enum Option { - HTTP_METHOD, CONTENT_TYPE, MD5, SERVICE_ACCOUNT_CRED + HTTP_METHOD, CONTENT_TYPE, MD5, EXT_HEADERS, SERVICE_ACCOUNT_CRED } private SignUrlOption(Option option, Object value) { @@ -874,7 +875,7 @@ Object getValue() { * The HTTP method to be used with the signed URL. */ public static SignUrlOption httpMethod(HttpMethod httpMethod) { - return new SignUrlOption(Option.HTTP_METHOD, httpMethod.name()); + return new SignUrlOption(Option.HTTP_METHOD, httpMethod); } /** @@ -892,6 +893,16 @@ public static SignUrlOption withContentType() { public static SignUrlOption withMd5() { return new SignUrlOption(Option.MD5, true); } + + /** + * Use it if signature should include the blob's canonicalized extended headers. + * When used, users of the signed URL should include the canonicalized extended headers with + * their request. + * @see + */ + public static SignUrlOption withExtHeaders(Map extHeaders) { + return new SignUrlOption(Option.EXT_HEADERS, extHeaders); + } /** * Provides a service account signer to sign the URL. If not provided an attempt will be made to diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 1a7dc91e18b9..49a412c6ab13 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -66,6 +66,7 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.net.URLEncoder; import java.util.Arrays; @@ -509,54 +510,81 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio "Signing key was not provided and could not be derived"); credentials = (ServiceAccountSigner) this.getOptions().getCredentials(); } - // construct signature - see https://cloud.google.com/storage/docs/access-control#Signed-URLs - StringBuilder stBuilder = new StringBuilder(); - if (optionMap.containsKey(SignUrlOption.Option.HTTP_METHOD)) { - stBuilder.append(optionMap.get(SignUrlOption.Option.HTTP_METHOD)); - } else { - stBuilder.append(HttpMethod.GET); - } - stBuilder.append('\n'); - if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.MD5), false)) { - checkArgument(blobInfo.getMd5() != null, "Blob is missing a value for md5"); - stBuilder.append(blobInfo.getMd5()); - } - stBuilder.append('\n'); - if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.CONTENT_TYPE), false)) { - checkArgument(blobInfo.getContentType() != null, "Blob is missing a value for content-type"); - stBuilder.append(blobInfo.getContentType()); - } - stBuilder.append('\n'); + long expiration = TimeUnit.SECONDS.convert( - getOptions().getClock().millisTime() + unit.toMillis(duration), TimeUnit.MILLISECONDS); - stBuilder.append(expiration).append('\n'); - StringBuilder path = new StringBuilder(); - if (!blobInfo.getBucket().startsWith("/")) { - path.append('/'); + getOptions().getClock().millisTime() + unit.toMillis(duration), TimeUnit.MILLISECONDS); + + StringBuilder stPath = new StringBuilder(); + if (!blobInfo.getBucket().startsWith(PATH_DELIMITER)) { + stPath.append(PATH_DELIMITER); } - path.append(blobInfo.getBucket()); - if (!blobInfo.getBucket().endsWith("/")) { - path.append('/'); + stPath.append(blobInfo.getBucket()); + if (!blobInfo.getBucket().endsWith(PATH_DELIMITER)) { + stPath.append(PATH_DELIMITER); } - if (blobInfo.getName().startsWith("/")) { - path.setLength(path.length() - 1); + if (blobInfo.getName().startsWith(PATH_DELIMITER)) { + stPath.setLength(stPath.length() - 1); } + String escapedName = UrlEscapers.urlFragmentEscaper().escape(blobInfo.getName()); - path.append(escapedName.replace("?", "%3F")); - stBuilder.append(path); - try { - byte[] signatureBytes = credentials.sign(stBuilder.toString().getBytes(UTF_8)); - stBuilder = new StringBuilder("https://storage.googleapis.com").append(path); + stPath.append(escapedName.replace("?", "%3F")); + + URI path = URI.create(stPath.toString()); + + try { + SignatureInfo signatureInfo = buildSignatureInfo(optionMap, blobInfo, expiration, path); + byte[] signatureBytes = + credentials.sign(signatureInfo.constructUnsignedPayload().getBytes(UTF_8)); + StringBuilder stBuilder = new StringBuilder("https://storage.googleapis.com").append(path); String signature = URLEncoder.encode(BaseEncoding.base64().encode(signatureBytes), UTF_8.name()); stBuilder.append("?GoogleAccessId=").append(credentials.getAccount()); stBuilder.append("&Expires=").append(expiration); stBuilder.append("&Signature=").append(signature); + return new URL(stBuilder.toString()); + } catch (MalformedURLException | UnsupportedEncodingException ex) { throw new IllegalStateException(ex); } } + + /** + * Builds signature info. + * @param optionMap the option map + * @param blobInfo the blob info + * @param expiration the expiration in seconds + * @param path the resource URI + * @return signature info + */ + private SignatureInfo buildSignatureInfo(Map optionMap, + BlobInfo blobInfo, long expiration, URI path) { + + HttpMethod httpVerb = optionMap.containsKey(SignUrlOption.Option.HTTP_METHOD) + ? (HttpMethod) optionMap.get(SignUrlOption.Option.HTTP_METHOD) + : HttpMethod.GET; + + SignatureInfo.Builder signatureInfoBuilder = + new SignatureInfo.Builder(httpVerb, expiration, path); + + if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.MD5), false)) { + checkArgument(blobInfo.getMd5() != null, "Blob is missing a value for md5"); + signatureInfoBuilder.setContentMd5(blobInfo.getMd5()); + } + + if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.CONTENT_TYPE), false)) { + checkArgument(blobInfo.getContentType() != null, "Blob is missing a value for content-type"); + signatureInfoBuilder.setContentType(blobInfo.getContentType()); + } + + @SuppressWarnings("unchecked") + Map extHeaders = + (Map) (optionMap.containsKey(SignUrlOption.Option.EXT_HEADERS) + ? (Map) optionMap.get(SignUrlOption.Option.EXT_HEADERS) + : Collections.emptyMap()); + + return signatureInfoBuilder.setCanonicalizedExtensionHeaders(extHeaders).build(); + } @Override public List get(BlobId... blobIds) { diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializerTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializerTest.java new file mode 100644 index 000000000000..599c9a5ba16d --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/CanonicalExtensionHeadersSerializerTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static org.junit.Assert.assertEquals; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.junit.Before; +import org.junit.Test; + +public class CanonicalExtensionHeadersSerializerTest { + + private CanonicalExtensionHeadersSerializer serializer; + + @Before + public void setUp() { + serializer = new CanonicalExtensionHeadersSerializer(); + } + + @Test + public void givenNoHeadersWhenSerializeThenProduceNothing() { + + StringBuilder sb = serializer.serialize(Collections.emptyMap()); + + assertEquals(sb.toString(), ""); + } + + @Test + public void givenNullHeadersWhenSerializeThenProduceNothing() { + + StringBuilder sb = serializer.serialize(null); + + assertEquals(sb.toString(), ""); + } + + @Test + public void givenEncryptionHeadersWhenSerializeThenAreRemvoed() { + + Map encryptionHeaders = new HashMap<>(); + encryptionHeaders.put("x-goog-encryption-key", ""); + encryptionHeaders.put("x-goog-encryption-key-sha256", ""); + + StringBuilder sb = serializer.serialize(encryptionHeaders); + + assertEquals(sb.toString(), ""); + } + + @Test + public void givenHeadersWhenSerializeThenSuccess() { + + Map encryptionHeaders = new HashMap<>(); + encryptionHeaders.put("x-goog-encryption-key", ""); + encryptionHeaders.put("x-GOOg-acl", " \n public-read "); + encryptionHeaders.put("x-goog-encryption-key-sha256", ""); + encryptionHeaders.put("X-goog-meta-OWNER", " myself and others \n"); + + StringBuilder sb = serializer.serialize(encryptionHeaders); + + assertEquals(sb.toString(), "x-goog-acl:public-read\nx-goog-meta-owner:myself and others\n"); + } +} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/SignatureInfoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/SignatureInfoTest.java new file mode 100644 index 000000000000..cd27b6aa805e --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/SignatureInfoTest.java @@ -0,0 +1,71 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static org.junit.Assert.assertEquals; + +import com.google.cloud.storage.SignatureInfo.Builder; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import org.junit.Test; + +public class SignatureInfoTest { + + private static final String RESOURCE = "/bucketName/blobName"; + + @Test(expected = IllegalArgumentException.class) + public void requireHttpVerb() { + + new SignatureInfo.Builder(null, 0L, URI.create(RESOURCE)).build(); + } + + @Test(expected = IllegalArgumentException.class) + public void requireResource() { + + new SignatureInfo.Builder(HttpMethod.GET, 0L, null).build(); + } + + @Test + public void constructUnsignedPayload() { + + Builder builder = new SignatureInfo.Builder(HttpMethod.PUT, 0L, URI.create(RESOURCE)); + + String unsignedPayload = builder.build().constructUnsignedPayload(); + + assertEquals("PUT\n\n\n0\n" + RESOURCE, unsignedPayload); + } + + @Test + public void constructUnsignedPayloadWithExtensionHeaders() { + + Builder builder = new SignatureInfo.Builder(HttpMethod.PUT, 0L, URI.create(RESOURCE)); + + Map extensionHeaders = new HashMap<>(); + extensionHeaders.put("x-goog-acl", "public-read"); + extensionHeaders.put("x-goog-meta-owner", "myself"); + + builder.setCanonicalizedExtensionHeaders(extensionHeaders); + + String unsignedPayload = builder.build().constructUnsignedPayload(); + + String rawPayload = + "PUT\n\n\n0\nx-goog-acl:public-read\nx-goog-meta-owner:myself\n" + RESOURCE; + + assertEquals(rawPayload, unsignedPayload); + } +} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 3c8a6ba38f0f..ab40409a67fa 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -74,6 +74,7 @@ import java.security.spec.InvalidKeySpecException; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.X509EncodedKeySpec; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -1672,6 +1673,63 @@ public void testSignUrlForBlobWithSpecialChars() signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name())))); } } + + @Test + public void testSignUrlWithExtHeaders() + throws NoSuchAlgorithmException, InvalidKeyException, SignatureException, + UnsupportedEncodingException { + EasyMock.replay(storageRpcMock); + ServiceAccountCredentials credentials = + new ServiceAccountCredentials(null, ACCOUNT, privateKey, null, null); + storage = options.toBuilder().setCredentials(credentials).build().getService(); + Map extHeaders = new HashMap(); + extHeaders.put("x-goog-acl", "public-read"); + extHeaders.put("x-goog-meta-owner", "myself"); + URL url = + storage.signUrl( + BLOB_INFO1, + 14, + TimeUnit.DAYS, + Storage.SignUrlOption.httpMethod(HttpMethod.PUT), + Storage.SignUrlOption.withContentType(), + Storage.SignUrlOption.withExtHeaders(extHeaders)); + String stringUrl = url.toString(); + String expectedUrl = + new StringBuilder("https://storage.googleapis.com/") + .append(BUCKET_NAME1) + .append('/') + .append(BLOB_NAME1) + .append("?GoogleAccessId=") + .append(ACCOUNT) + .append("&Expires=") + .append(42L + 1209600) + .append("&Signature=") + .toString(); + assertTrue(stringUrl.startsWith(expectedUrl)); + String signature = stringUrl.substring(expectedUrl.length()); + + StringBuilder signedMessageBuilder = new StringBuilder(); + signedMessageBuilder + .append(HttpMethod.PUT) + .append('\n') + .append('\n') + .append(BLOB_INFO1.getContentType()) + .append('\n') + .append(42L + 1209600) + .append('\n') + .append("x-goog-acl:public-read\n") + .append("x-goog-meta-owner:myself\n") + .append('/') + .append(BUCKET_NAME1) + .append('/') + .append(BLOB_NAME1); + + Signature signer = Signature.getInstance("SHA256withRSA"); + signer.initVerify(publicKey); + signer.update(signedMessageBuilder.toString().getBytes(UTF_8)); + assertTrue( + signer.verify(BaseEncoding.base64().decode(URLDecoder.decode(signature, UTF_8.name())))); + } @Test public void testSignUrlForBlobWithSlashes()