From 20dc759683e798aa207878378a658bedfef78c39 Mon Sep 17 00:00:00 2001 From: Google Cloud Datastore <> Date: Tue, 14 May 2013 12:08:02 -0700 Subject: [PATCH 01/54] Initial commit --- .../services/datastore/client/Datastore.java | 124 ++++++ .../datastore/client/DatastoreException.java | 31 ++ .../datastore/client/DatastoreFactory.java | 128 ++++++ .../datastore/client/DatastoreHelper.java | 413 ++++++++++++++++++ .../datastore/client/DatastoreOptions.java | 111 +++++ .../client/LocalDevelopmentDatastore.java | 253 +++++++++++ .../LocalDevelopmentDatastoreException.java | 30 ++ .../LocalDevelopmentDatastoreFactory.java | 41 ++ .../services/datastore/client/RemoteRpc.java | 109 +++++ 9 files changed, 1240 insertions(+) create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/Datastore.java create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreException.java create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreException.java create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreFactory.java create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/Datastore.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/Datastore.java new file mode 100644 index 000000000..f3b0a3107 --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/Datastore.java @@ -0,0 +1,124 @@ +/* + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + +import com.google.api.client.http.HttpStatusCodes; +import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest; +import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse; +import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest; +import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse; +import com.google.api.services.datastore.DatastoreV1.BlindWriteRequest; +import com.google.api.services.datastore.DatastoreV1.BlindWriteResponse; +import com.google.api.services.datastore.DatastoreV1.CommitRequest; +import com.google.api.services.datastore.DatastoreV1.CommitResponse; +import com.google.api.services.datastore.DatastoreV1.LookupRequest; +import com.google.api.services.datastore.DatastoreV1.LookupResponse; +import com.google.api.services.datastore.DatastoreV1.RollbackRequest; +import com.google.api.services.datastore.DatastoreV1.RollbackResponse; +import com.google.api.services.datastore.DatastoreV1.RunQueryRequest; +import com.google.api.services.datastore.DatastoreV1.RunQueryResponse; + +import java.io.IOException; + +/** + * Provides access to the Datastore. + * + */ +public class Datastore { + + final RemoteRpc remoteRpc; + + Datastore(RemoteRpc remoteRpc) { + this.remoteRpc = remoteRpc; + } + + /** + * Reset the RPC count. + */ + public void resetRpcCount() { + remoteRpc.resetRpcCount(); + } + + /** + * Returns the number of RPC calls made since the client was created + * or {@link #resetRpcCount} was called. + */ + public int getRpcCount() { + return remoteRpc.getRpcCount(); + } + + private DatastoreException invalidResponseException(String method, IOException exception) { + return RemoteRpc.makeException(remoteRpc.getUrl(), method, + HttpStatusCodes.STATUS_CODE_SERVICE_UNAVAILABLE, "Invalid response", exception); + } + + public AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws DatastoreException { + try { + return AllocateIdsResponse.parseFrom(remoteRpc.call("allocateIds", request)); + } catch (IOException exception) { + throw invalidResponseException("allocateIds", exception); + } + } + + public BeginTransactionResponse beginTransaction(BeginTransactionRequest request) + throws DatastoreException { + try { + return BeginTransactionResponse.parseFrom(remoteRpc.call("beginTransaction", request)); + } catch (IOException exception) { + throw invalidResponseException("beginTransaction", exception); + } + } + + public CommitResponse commit(CommitRequest request) throws DatastoreException { + try { + return CommitResponse.parseFrom(remoteRpc.call("commit", request)); + } catch (IOException exception) { + throw invalidResponseException("commit", exception); + } + } + + public LookupResponse lookup(LookupRequest request) throws DatastoreException { + try { + return LookupResponse.parseFrom(remoteRpc.call("lookup", request)); + } catch (IOException exception) { + throw invalidResponseException("lookup", exception); + } + } + + public RollbackResponse rollback(RollbackRequest request) throws DatastoreException { + try { + return RollbackResponse.parseFrom(remoteRpc.call("rollback", request)); + } catch (IOException exception) { + throw invalidResponseException("rollback", exception); + } + } + + public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreException { + try { + return RunQueryResponse.parseFrom(remoteRpc.call("runQuery", request)); + } catch (IOException exception) { + throw invalidResponseException("runQuery", exception); + } + } + + public BlindWriteResponse blindWrite(BlindWriteRequest request) throws DatastoreException { + try { + return BlindWriteResponse.parseFrom(remoteRpc.call("blindWrite", request)); + } catch (IOException exception) { + throw invalidResponseException("blindWrite", exception); + } + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreException.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreException.java new file mode 100644 index 000000000..54f144474 --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreException.java @@ -0,0 +1,31 @@ +/* + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + +/** + * Indicates an error in a {@link Datastore} call. + * + */ +public class DatastoreException extends Exception { + public final String methodName; + public final int code; + + public DatastoreException(String methodName, int code, String message, Throwable cause) { + super(message, cause); + this.methodName = methodName; + this.code = code; + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java new file mode 100644 index 000000000..acd72c80d --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java @@ -0,0 +1,128 @@ +/* + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.logging.ConsoleHandler; +import java.util.logging.Formatter; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.logging.StreamHandler; + +/** + * Client factory for {@link Datastore}. + * + */ +public class DatastoreFactory { + private static final Logger logger = Logger.getLogger(DatastoreFactory.class.getName()); + + /** Singleton factory instance. */ + private static final DatastoreFactory INSTANCE = new DatastoreFactory(); + private static final String VERSION = "v1beta1"; + + // Lazy load this because we might be running inside App Engine and this + // class isn't on the whitelist. + private static ConsoleHandler methodHandler; + + public static DatastoreFactory get() { + return INSTANCE; + } + + // TODO(user): Support something other than console handler for when we're + // running in App Engine + private static synchronized StreamHandler getStreamHandler() { + if (methodHandler == null) { + methodHandler = new ConsoleHandler(); + methodHandler.setFormatter(new Formatter() { + @Override + public String format(LogRecord record) { + return record.getMessage() + "\n"; + } + }); + methodHandler.setLevel(Level.FINE); + } + return methodHandler; + } + + DatastoreFactory() { } + + RemoteRpc newRemoteRpc(DatastoreOptions options) { + if (options == null) { + throw new IllegalArgumentException("options not set"); + } + HttpRequestFactory client = makeClient(options); + return new RemoteRpc(client, buildUrl(options)); + } + + /** + * Provides access to a datastore using the provided options. Logs + * into the application using the credentials available via these + * options. + * + * @throws IllegalArgumentException if the server or credentials weren't provided. + */ + public Datastore create(DatastoreOptions options) throws IllegalArgumentException { + return new Datastore(newRemoteRpc(options)); + } + + /** + * Constructs a Google APIs HTTP client with the associated credentials. + */ + public HttpRequestFactory makeClient(DatastoreOptions options) { + Credential credential = options.getCredential(); + if (credential == null) { + logger.warning("Not using any credentials"); + return new NetHttpTransport().createRequestFactory(); + } + HttpTransport transport = credential.getTransport(); + return transport.createRequestFactory(credential); + } + + /** + * Build a valid datastore URL. + */ + private String buildUrl(DatastoreOptions options) { + try { + if (options.getDataset() == null) { + throw new IllegalArgumentException("datastore dataset not set in options"); + } + URI validUri = new URI(String.format("%s/datastore/%s/datasets/%s", + options.getHost(), VERSION, options.getDataset())); + return validUri.toString(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + } + + /** + * Starts logging datastore method calls to the console. (Useful within tests.) + */ + public static void logMethodCalls() { + Logger logger = Logger.getLogger(Datastore.class.getName()); + logger.setLevel(Level.FINE); + if (!Arrays.asList(logger.getHandlers()).contains(getStreamHandler())) { + logger.addHandler(getStreamHandler()); + } + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java new file mode 100644 index 000000000..ab576bdbf --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java @@ -0,0 +1,413 @@ +/* + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.compute.ComputeCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.jackson.JacksonFactory; +import com.google.api.services.datastore.DatastoreV1.CompositeFilter; +import com.google.api.services.datastore.DatastoreV1.Entity; +import com.google.api.services.datastore.DatastoreV1.EntityOrBuilder; +import com.google.api.services.datastore.DatastoreV1.Filter; +import com.google.api.services.datastore.DatastoreV1.Key; +import com.google.api.services.datastore.DatastoreV1.Key.PathElement; +import com.google.api.services.datastore.DatastoreV1.Property; +import com.google.api.services.datastore.DatastoreV1.PropertyFilter; +import com.google.api.services.datastore.DatastoreV1.PropertyOrBuilder; +import com.google.api.services.datastore.DatastoreV1.PropertyOrder; +import com.google.api.services.datastore.DatastoreV1.PropertyReference; +import com.google.api.services.datastore.DatastoreV1.Value; +import com.google.api.services.datastore.DatastoreV1.ValueOrBuilder; +import com.google.protobuf.ByteString; + +import java.io.File; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +/** + * Helper methods for {@link Datastore}. + * + */ +public class DatastoreHelper { + private static final Logger logger = Logger.getLogger(DatastoreHelper.class.getName()); + + private DatastoreHelper() {} + + /** + * Attempts to get credentials from Google Compute Engine. + * + * @return valid credentials or {@code null} + */ + public static Credential getComputeEngineCredential() + throws GeneralSecurityException, IOException { + NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + try { + // Try to connect using Google Compute Engine service account credentials. + ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory()); + // Force token refresh to detect if we are running on Google Compute Engine. + credential.refreshToken(); + return credential; + } catch (IOException e) { + return null; + } + } + + /** + * Constructs credentials for the given account and key. + * + * @param account the account to use. + * @param privateKeyFile the file name from which to get the private key. + * @return valid credentials or {@code null} + */ + public static Credential getServiceAccountCredential(String account, String privateKeyFile) + throws GeneralSecurityException, IOException { + NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + JacksonFactory jsonFactory = new JacksonFactory(); + return new GoogleCredential.Builder() + .setTransport(transport) + .setJsonFactory(jsonFactory) + .setServiceAccountId(account) + .setServiceAccountScopes(DatastoreOptions.SCOPE) + .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile)) + .build(); + } + + /** + * Uses the following enviorment variables to construct a {@link Datastore}: + * DATASTORE_DATASET - the datastore dataset id + * DATASTORE_HOST - the host to use to access the datastore + * e.g: https://www.googleapis.com/datastore/v1/datasets/{dataset} + * DATASTORE_SERVICE_ACCOUNT - (optional) service account name + * DATASTORE_PRIVATE_KEY_FILE - (optional) service account private key file + * + * Preference of credentials is: + * - ComputeEngine + * - Service Account (specified by DATASTORE_SERVICE_ACCOUNT and DATASTORE_PRIVATE_KEY_FILE) + * - no-credentials (for local development environment) + */ + public static DatastoreOptions.Builder getOptionsfromEnv() + throws GeneralSecurityException, IOException { + DatastoreOptions.Builder options = new DatastoreOptions.Builder(); + options.dataset(System.getenv("DATASTORE_DATASET")); + options.host(System.getenv("DATASTORE_HOST")); + Credential credential = getComputeEngineCredential(); + if (credential != null) { + logger.info("Using Compute Engine credential."); + } else if (System.getenv("DATASTORE_SERVICE_ACCOUNT") != null && + System.getenv("DATASTORE_PRIVATE_KEY_FILE") != null) { + credential = getServiceAccountCredential(System.getenv("DATASTORE_SERVICE_ACCOUNT"), + System.getenv("DATASTORE_PRIVATE_KEY_FILE")); + logger.info("Using JWT Service Account credential."); + } + options.credential(credential); + return options; + } + + /** + * @see #getOptionsfromEnv() + */ + public static Datastore getDatastoreFromEnv() throws GeneralSecurityException, IOException { + return DatastoreFactory.get().create(getOptionsfromEnv().build()); + } + + /** + * Make a sort order for use in a query. + */ + public static PropertyOrder.Builder makeOrder(String property, + PropertyOrder.Direction direction) { + return PropertyOrder.newBuilder() + .setProperty(makePropertyReference(property)) + .setDirection(direction); + } + + /** + * Make a filter on a property for use in a query. + */ + public static Filter.Builder makeFilter(String property, PropertyFilter.Operator operator, + Value value) { + return Filter.newBuilder() + .setPropertyFilter(PropertyFilter.newBuilder() + .setProperty(makePropertyReference(property)) + .setOperator(operator) + .setValue(value)); + } + + /** + * Make a filter on a property for use in a query. + */ + public static Filter.Builder makeFilter(String property, PropertyFilter.Operator operator, + Value.Builder value) { + return makeFilter(property, operator, value.build()); + } + + /** + * Make a composite filter from the given sub-filters. + * + * Uses AND to combine filters. + */ + public static Filter.Builder makeFilter(Filter ...subfilters) { + return makeCompositeFilter(Arrays.asList(subfilters)); + } + + /** + * Make a composite filter from the given sub-filters. + * + * Uses AND to combine filters. + */ + public static Filter.Builder makeCompositeFilter(Iterable subfilters) { + return Filter.newBuilder() + .setCompositeFilter(CompositeFilter.newBuilder() + .addAllFilter(subfilters) + .setOperator(CompositeFilter.Operator.AND)); + } + + /** + * Make an entity property with the specified value. + */ + public static Property.Builder makeProperty(String name, Value value) { + return Property.newBuilder().setName(name).addValue(value); + } + + /** + * Make an entity property with the specified value. + */ + public static Property.Builder makeProperty(String name, Value.Builder value) { + return makeProperty(name, value.build()); + } + + /** + * Make an entity property with the specified values. + */ + public static Property.Builder makeProperty(String name, Iterable values) { + return Property.newBuilder().setName(name).setMulti(true).addAllValue(values); + } + + /** + * Make an entity property with the specified values. + */ + public static Property.Builder makeProperty(String name, Value ...values) { + return makeProperty(name, Arrays.asList(values)); + } + + /** + * Make an entity property with the specified values. + */ + public static Property.Builder makeProperty(String name, Value.Builder ...builders) { + Property.Builder prop = makeProperty(name, Collections.emptyList()); + for (Value.Builder builder : builders) { + prop.addValue(builder); + } + return prop; + } + + /** + * Make a property reference for use in a query. + */ + public static PropertyReference.Builder makePropertyReference(String propertyName) { + return PropertyReference.newBuilder().setName(propertyName); + } + + /** + * Make a key value. + */ + public static Value.Builder makeValue(Key key) { + return Value.newBuilder().setKeyValue(key); + } + + /** + * Make a key value. + */ + public static Value.Builder makeValue(Key.Builder key) { + return makeValue(key.build()); + } + + /** + * Make an integer value. + */ + public static Value.Builder makeValue(long key) { + return Value.newBuilder().setIntegerValue(key); + } + + /** + * Make a floating point value. + */ + public static Value.Builder makeValue(double value) { + return Value.newBuilder().setDoubleValue(value); + } + + /** + * Make a floating point value. + */ + public static Value.Builder makeValue(boolean value) { + return Value.newBuilder().setBooleanValue(value); + } + + /** + * Make a string value. + */ + public static Value.Builder makeValue(String value) { + return Value.newBuilder().setStringValue(value); + } + + /** + * Make a key value. + */ + public static Value.Builder makeValue(Entity entity) { + return Value.newBuilder().setEntityValue(entity); + } + + /** + * Make a entity value. + */ + public static Value.Builder makeValue(Entity.Builder entity) { + return makeValue(entity.build()); + } + + /** + * Make a entity value. + */ + public static Value.Builder makeValue(ByteString blob) { + return Value.newBuilder().setBlobValue(blob); + } + + /** + * Make a date value given a time in milliseconds. + */ + public static Value.Builder makeValue(Date date) { + return Value.newBuilder().setTimestampMicrosecondsValue(date.getTime() * 1000L); + } + + /** + * Make a key from the specified path of kind/id-or-name pairs. + * + * The id-or-name values must be either Key, String, Long, Integer or Short. + * + * The last id-or-name value may be omitted, in which case an entity without + * an id is created (for use with automatic id allocation). + */ + public static Key.Builder makeKey(Object... elements) { + Key.Builder key = Key.newBuilder(); + for (int pathIndex = 0; pathIndex < elements.length; pathIndex += 2) { + PathElement.Builder pathElement = PathElement.newBuilder(); + Object element = elements[pathIndex]; + if (element instanceof Key) { + key.addAllPathElement(((Key) element).getPathElementList()); + // We increment by 2, but since we got a Key argument we're only consuming 1 element in this + // iteration of the loop. Decrement the index so that when we jump by 2 we end up in the + // right spot. + pathIndex--; + } else { + String kind; + try { + kind = (String) element; + } catch (ClassCastException e) { + throw new IllegalArgumentException("Expected string or Key, got: " + element.getClass()); + } + pathElement.setKind(kind); + if (pathIndex + 1 < elements.length) { + Object value = elements[pathIndex + 1]; + if (value instanceof String) { + pathElement.setName((String) value); + } else if (value instanceof Long) { + pathElement.setId((Long) value); + } else if (value instanceof Integer) { + pathElement.setId((Integer) value); + } else if (value instanceof Short) { + pathElement.setId((Short) value); + } else { + throw new IllegalArgumentException( + "Expected string or integer, got: " + value.getClass()); + } + } + key.addPathElement(pathElement); + } + } + return key; + } + + /** + * Return a map of property name to java object. + * + * Looses microseconds on timestamp values, meaning and indexing information. + * + * @param entity the entity to search for the property. + * @return The property value, or null if it's not found. + */ + public static Map getPropertyMap(EntityOrBuilder entity) { + Map result = new HashMap(); + for (PropertyOrBuilder property : entity.getPropertyList()) { + Object value; + if (property.getMulti()) { + List values = new ArrayList(property.getValueCount()); + for (ValueOrBuilder subValue : property.getValueList()) { + values.add(getValue(subValue)); + } + value = values; + } else { + value = getValue(property.getValue(0)); + } + result.put(property.getName(), value); + } + return Collections.unmodifiableMap(result); + } + + /** + * Convert a Value to a Java Object. Ignores meaning. + * + * Looses microseconds on timestamp values, meaning and indexing information. + */ + public static Object getValue(ValueOrBuilder value) { + if (value.hasBooleanValue()) { + return value.getBooleanValue(); + } + if (value.hasIntegerValue()) { + return value.getIntegerValue(); + } + if (value.hasDoubleValue()) { + return value.getDoubleValue(); + } + if (value.hasTimestampMicrosecondsValue()) { + return new Date(value.getTimestampMicrosecondsValue() / 1000L); + } + if (value.hasKeyValue()) { + return value.getKeyValue(); + } + if (value.hasBlobKeyValue()) { + return value; // Returning value directly to avoid loosing type info. + } + if (value.hasStringValue()) { + return value.getStringValue(); + } + if (value.hasBlobValue()) { + return value.getBlobValue(); + } + if (value.hasEntityValue()) { + return value.getEntityValue(); + } + return null; + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java new file mode 100644 index 000000000..44278959e --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java @@ -0,0 +1,111 @@ +/* + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + +import com.google.api.client.auth.oauth2.Credential; + +/** + * A mutable object containing settings for the datastore. + * + *

Example for connecting to a datastore:

+ * + * 
+ * DatastoreOptions options = new DatastoreOptions()
+ *     .dataset("my-dataset-id"),
+ *     .credential(DatastoreHelper.getGceCredential());
+ * DatastoreFactory.get().create(options);
+ *
+ * + *

+ * The options should be passed to {@link DatastoreFactory#create}. + *

+ * + */ +public class DatastoreOptions { + private final String dataset; + private final String host; + private static final String DEFAULT_HOST = "https://www.googleapis.com"; + + private final Credential credential; + public static final String SCOPE = "https://www.googleapis.com/auth/datastore" + + " https://www.googleapis.com/auth/userinfo.email"; + + DatastoreOptions(String dataset, String host, Credential credential) { + this.dataset = dataset; + this.host = host != null ? host : DEFAULT_HOST; + this.credential = credential; + } + + /** + * Builder for {@link DatastoreOptions}. + */ + public static class Builder { + private String dataset; + private String host; + private Credential credential; + + public Builder() { } + + public Builder(DatastoreOptions options) { + this.dataset = options.dataset; + this.host = options.host; + this.credential = options.credential; + } + + public DatastoreOptions build() { + return new DatastoreOptions(dataset, host, credential); + } + + /** + * Sets the dataset used to access the datastore. + */ + public Builder dataset(String newDataset) { + dataset = newDataset; + return this; + } + + /** + * Sets the host used to access the datastore. + */ + public Builder host(String newHost) { + host = newHost; + return this; + } + + /** + * Sets the Google APIs credentials used to access the API. + */ + public Builder credential(Credential newCredential) { + credential = newCredential; + return this; + } + } + + // === getters === + + public String getDataset() { + return dataset; + } + + public String getHost() { + return host; + } + + public Credential getCredential() { + return credential; + } + +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java new file mode 100644 index 000000000..5b92cfc17 --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java @@ -0,0 +1,253 @@ +/* + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.UrlEncodedContent; +import com.google.api.client.util.Preconditions; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * An extension to {@link Datastore} that provides lifecycle management for a development datastore + * server. + * + * In order to use a development datastore for a JUnit 4 test you might do something like this: + * + * 
+ * public class MyTest {
+ *
+ *   static LocalDevelopmentDatastore datastore;
+ *
+ *   {@literal @}BeforeClass
+ *   public static void startLocalDatastore() throws LocalDevelopmentDatastoreException {
+ *     DatastoreOptions opts = new DatastoreOptions.Builder()
+ *         .host("http://localhost:8080")
+ *         .dataset("myapp")
+ *         .build();
+ *     datastore = LocalDevelopmentDatastoreFactory.get().create(opts);
+ *     datastore.start("/usr/local/gcdsdk", "myapp");
+ *   }
+ *
+ *   {@literal @}Before
+ *   public void setUp() throws LocalDevelopmentDatastoreException {
+ *     datastore.clearDatastore();
+ *   }
+ *
+ *   {@literal @}AfterClass
+ *   public static void stopLocalDatastore() throws LocalDevelopmentDatastoreException {
+ *     datastore.stopDatastore();
+ *   }
+ *
+ *   {@literal @}Test
+ *   public void testFoo1() { }
+
+ *   {@literal @}Test
+ *   public void testFoo2() { }
+ *
+ * }
+ *
+ * + */ +public class LocalDevelopmentDatastore extends Datastore { + private static final int STARTUP_TIMEOUT_SECS = 5; + + private final String host; + + /** Internal state lifecycle management. */ + enum State {NEW, STARTED, STOPPED} + + private State state = State.NEW; + + LocalDevelopmentDatastore(RemoteRpc rpc, String host) { + super(rpc); + this.host = host; + } + + /** + * Clears all data in the Datastore. + * + * @throws LocalDevelopmentDatastoreException + */ + public void clear() throws LocalDevelopmentDatastoreException { + HttpRequestFactory client = remoteRpc.getHttpRequestFactory(); + try { + Map params = new HashMap(); + params.put("action", "Clear Datastore"); + UrlEncodedContent content = new UrlEncodedContent(params); + GenericUrl url = new GenericUrl(host + "/_ah/admin/datastore"); + HttpResponse httpResponse = client.buildPostRequest(url, content).execute(); + if (!httpResponse.isSuccessStatusCode()) { + throw new LocalDevelopmentDatastoreException( + "Clear Datastore returned http status " + httpResponse.getStatusCode()); + } + } catch (IOException e) { + throw new LocalDevelopmentDatastoreException( + "Exception trying to clear the dev datastore", e); + } + } + + /** + * Starts the local datastore. It is the caller's responsibility to call {@link #stop}. Note that + * receiving an exception does not indicate that the server did not start. We recommend calling + * {@link #stop} to ensure the server is not running regardless of the result of this method. + * + * @param sdkPath The path to the GCD SDK, eg /usr/local/dev/gcd + * @param dataset The name of the GCD dataset + * @param cmdLineOptions Command line options to pass to the script that launches the dev server + * @throws LocalDevelopmentDatastoreException If {@link #start} has already been called or the + * server does not start successfully. + */ + public synchronized void start(String sdkPath, String dataset, String... cmdLineOptions) + throws LocalDevelopmentDatastoreException { + Preconditions.checkNotNull(sdkPath, "sdkPath cannot be null"); + Preconditions.checkNotNull(dataset, "dataset cannot be null"); + Preconditions.checkState(state == State.NEW, "Cannot call start() more than once."); + try { + startDatastoreInternal(sdkPath, dataset, cmdLineOptions); + state = State.STARTED; + } finally { + if (state != State.STARTED) { + // If we're not able to start the server we don't want people trying again. Just move it + // straight to the STOPPED state. + state = State.STOPPED; + } + } + } + + void startDatastoreInternal(String sdkPath, String dataset, String... cmdLineOptions) + throws LocalDevelopmentDatastoreException { + List cmd = Arrays.asList("./gcd.sh", "start", dataset, "--allow_remote_shutdown", + "--property=datastore.no_storage"); + cmd.addAll(Arrays.asList(cmdLineOptions)); + ProcessBuilder builder = new ProcessBuilder(cmd); + builder.directory(new File(sdkPath)); + builder.redirectErrorStream(true); + Process devDatastoreServerProcess; + try { + devDatastoreServerProcess = builder.start(); + } catch (IOException e) { + throw new LocalDevelopmentDatastoreException("Could not start dev server", e); + } + StartupMonitor monitor = new StartupMonitor(devDatastoreServerProcess.getInputStream()); + try { + monitor.start(); + if (!monitor.startupCompleteLatch.await(STARTUP_TIMEOUT_SECS, TimeUnit.SECONDS)) { + throw new LocalDevelopmentDatastoreException("Dev server did not start within 5 seconds"); + } + if (!monitor.success) { + throw new LocalDevelopmentDatastoreException("Server did not start normally"); + } + } catch (InterruptedException e) { + // not sure why this would happen + throw new LocalDevelopmentDatastoreException("Received an interrupt", e); + } + } + + /** + * Stops the local datastore. Multiple calls are allowed. + * + * @throws LocalDevelopmentDatastoreException + */ + public synchronized void stop() throws LocalDevelopmentDatastoreException { + // We intentionally don't check the internal state. If people want to try and stop the server + // multiple times that's fine. + stopDatastoreInternal(); + state = State.STOPPED; + } + + void stopDatastoreInternal() throws LocalDevelopmentDatastoreException { + // No need to kill the process we started, this function will take care of it. + HttpRequestFactory client = remoteRpc.getHttpRequestFactory(); + Map params = new HashMap(); + UrlEncodedContent content = new UrlEncodedContent(params); + GenericUrl url = new GenericUrl(host + "/_ah/admin/quit"); + try { + HttpResponse httpResponse = client.buildPostRequest(url, content).execute(); + if (!httpResponse.isSuccessStatusCode()) { + throw new LocalDevelopmentDatastoreException( + "Request to shutdown local datastore returned http error code " + + httpResponse.getStatusCode()); + } + } catch (IOException e) { + throw new LocalDevelopmentDatastoreException( + "Exception trying to stop the dev datastore", e); + } + } + + /** + * Monitors the provided input stream for evidence that the dev server has started successfully + * and redirects the output of the dev server process to sysout for this process. + */ + static class StartupMonitor extends Thread { + + private final InputStream inputStream; + private volatile boolean success = false; + /** This latch will reach 0 once server startup has completed. */ + private final CountDownLatch startupCompleteLatch = new CountDownLatch(1); + + StartupMonitor(InputStream inputStream) { + this.inputStream = inputStream; + setDaemon(true); + } + + @Override + public void run() { + try { + BufferedReader br = new BufferedReader(new InputStreamReader(inputStream)); + String line; + while ((line = br.readLine()) != null) { + // redirect to sysout for our process + System.out.println(line); + if (!success && line.contains("Dev App Server is now running")) { + success = true; + startupCompleteLatch.countDown(); + } + } + } catch (IOException ioe) { + if (!success) { + System.err.println("Received an IOException before the dev server startup completed. " + + "Dev server is in an unknown state."); + } else { + // We got an exception after the server started successfully. We'll lose the ability + // to log the output of the dev server but there's no need to shut anything down. + System.err.println("Received an exception handling output from the dev server. " + + "Logging will stop but the dev server is probably ok."); + } + ioe.printStackTrace(); + } finally { + if (!success) { + // Either the stream is closed (indicates server shut down) or we received an Exception + // while processing the stream contents. Either way we can tell the calling thread to stop + // waiting. + startupCompleteLatch.countDown(); + } + } + } + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreException.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreException.java new file mode 100644 index 000000000..deaec1957 --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreException.java @@ -0,0 +1,30 @@ +/* + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + +/** + * An exception related to the local development {@link Datastore}. + * + */ +public class LocalDevelopmentDatastoreException extends Exception { + public LocalDevelopmentDatastoreException(String message) { + super(message); + } + + public LocalDevelopmentDatastoreException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreFactory.java new file mode 100644 index 000000000..c0c90c7f0 --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreFactory.java @@ -0,0 +1,41 @@ +/* + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + + +/** + * Factory for {@link LocalDevelopmentDatastore}. + * + */ +public class LocalDevelopmentDatastoreFactory extends DatastoreFactory { + + /** Singleton factory instance. */ + private static final LocalDevelopmentDatastoreFactory INSTANCE = + new LocalDevelopmentDatastoreFactory(); + + public static LocalDevelopmentDatastoreFactory get() { + return INSTANCE; + } + + LocalDevelopmentDatastoreFactory() { } + + @Override + public LocalDevelopmentDatastore create(DatastoreOptions options) + throws IllegalArgumentException { + RemoteRpc rpc = newRemoteRpc(options); + return new LocalDevelopmentDatastore(rpc, options.getHost()); + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java new file mode 100644 index 000000000..60f30395f --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java @@ -0,0 +1,109 @@ +/* + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpResponseException; +import com.google.api.client.http.HttpStatusCodes; +import com.google.api.client.http.protobuf.ProtoHttpContent; + +import com.google.protobuf.MessageLite; + +import java.io.IOException; +import java.io.InputStream; +import java.util.logging.Logger; + +/** + * An RPC transport that sends protocol buffers over HTTP. + * + */ +class RemoteRpc { + private static final Logger logger = Logger.getLogger(RemoteRpc.class.getName()); + + private HttpRequestFactory client; + private String url; + private int rpcCount = 0; + + RemoteRpc(HttpRequestFactory client, String url) { + this.client = client; + this.url = url; + try { + resolveURL("dummyRpc"); + } catch (Exception e) { + throw new IllegalArgumentException( + "Unable to construct RemoteRpc due to unsupported url: <" + url + ">", e); + } + } + + /** + * Makes an RPC call using the client. Logs how long it took and any exceptions. + * + * NOTE: The request could be an InputStream too, but the http client will need to + * find its length, which will require buffering it anyways. + * + * @throws DatastoreException if the RPC fails. + */ + InputStream call(String methodName, MessageLite request) throws DatastoreException { + logger.fine("remote datastore call " + methodName); + + long startTime = System.currentTimeMillis(); + try { + HttpResponse httpResponse; + try { + rpcCount++; + ProtoHttpContent payload = new ProtoHttpContent(request); + httpResponse = client.buildPostRequest(resolveURL(methodName), payload).execute(); + return httpResponse.getContent(); + } catch (HttpResponseException e) { + throw makeException(url, methodName, e.getStatusCode(), e.getContent(), e); + } catch (IOException e) { + throw makeException(url, methodName, HttpStatusCodes.STATUS_CODE_SERVICE_UNAVAILABLE, + "I/O error", e); + } + } finally { + long elapsedTime = System.currentTimeMillis() - startTime; + logger.fine("remote datastore call " + methodName + " took " + elapsedTime + " ms"); + } + } + + void resetRpcCount() { + rpcCount = 0; + } + + int getRpcCount() { + return rpcCount; + } + + String getUrl() { + return url; + } + + GenericUrl resolveURL(String path) { + return new GenericUrl(url + "/" + path); + } + + HttpRequestFactory getHttpRequestFactory() { + return client; + } + + static DatastoreException makeException( + String url, String methodName, int code, String message, Throwable cause) { + logger.fine("remote datastore call " + methodName + " against " + url + " failed: " + message); + return new DatastoreException(methodName, code, message, cause); + } +} From ce63aab012e9d3331ddb9bbedb36162db313a540 Mon Sep 17 00:00:00 2001 From: Ed Davisson Date: Tue, 6 Aug 2013 19:25:34 -0700 Subject: [PATCH 02/54] v1beta1-rev2-1.0.1 ------------------ - GQL support. - Metadata query support. - Command line tool improvements. - Microsoft Windows support (`gcd.cmd`). - Testing mode. - More intuitive `update_indexes` command (renamed to `updateindexes`). - New `create` command and simplified `start` command. - Improved integration with existing App Engine applications. - Ruby samples. - Java helper for query splitting. --- .../datastore/client/DatastoreFactory.java | 16 +- .../datastore/client/DatastoreHelper.java | 71 +++++- .../datastore/client/DatastoreOptions.java | 8 +- .../datastore/client/QuerySplitter.java | 42 ++++ .../datastore/client/QuerySplitterImpl.java | 232 ++++++++++++++++++ 5 files changed, 361 insertions(+), 8 deletions(-) create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java index acd72c80d..a8d0492be 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java @@ -72,7 +72,8 @@ RemoteRpc newRemoteRpc(DatastoreOptions options) { throw new IllegalArgumentException("options not set"); } HttpRequestFactory client = makeClient(options); - return new RemoteRpc(client, buildUrl(options)); + return new RemoteRpc(client, + buildUrl(options, System.getenv("DATASTORE_URL_INTERNAL_OVERRIDE"))); } /** @@ -102,14 +103,19 @@ public HttpRequestFactory makeClient(DatastoreOptions options) { /** * Build a valid datastore URL. */ - private String buildUrl(DatastoreOptions options) { + String buildUrl(DatastoreOptions options, String overrideUrl) { try { if (options.getDataset() == null) { throw new IllegalArgumentException("datastore dataset not set in options"); } - URI validUri = new URI(String.format("%s/datastore/%s/datasets/%s", - options.getHost(), VERSION, options.getDataset())); - return validUri.toString(); + String url; + if (overrideUrl != null) { + url = String.format("%s/datasets/%s", overrideUrl, options.getDataset()); + } else { + url = String.format("%s/datastore/%s/datasets/%s", + options.getHost(), VERSION, options.getDataset()); + } + return new URI(url).toString(); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java index ab576bdbf..65ddb3c45 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java @@ -42,8 +42,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.logging.Logger; @@ -55,6 +57,59 @@ public class DatastoreHelper { private static final Logger logger = Logger.getLogger(DatastoreHelper.class.getName()); + /** The property used in the Datastore to give us a random distribution. **/ + public static final String SCATTER_PROPERTY_NAME = "__scatter__"; + + /** The property used in the Datastore to get the key of the entity. **/ + public static final String KEY_PROPERTY_NAME = "__key__"; + + /** + * Comparator for Keys + */ + private static final class KeyComparator implements Comparator { + + static final KeyComparator INSTANCE = new KeyComparator(); + + private int comparePathElement(PathElement thisElement, PathElement otherElement) { + int result = thisElement.getKind().compareTo(otherElement.getKind()); + if (result != 0) { + return result; + } + if (thisElement.hasId()) { + if (!otherElement.hasId()) { + return -1; + } + return Long.valueOf(thisElement.getId()).compareTo(otherElement.getId()); + } + if (otherElement.hasId()) { + return 1; + } + + return thisElement.getName().compareTo(otherElement.getName()); + } + + @Override + public int compare(Key thisKey, Key otherKey) { + if (!thisKey.getPartitionId().equals(otherKey.getPartitionId())) { + throw new IllegalArgumentException("Cannot compare keys with different partition ids."); + } + + Iterator thisPath = thisKey.getPathElementList().iterator(); + Iterator otherPath = otherKey.getPathElementList().iterator(); + while (thisPath.hasNext()) { + if (!otherPath.hasNext()) { + return 1; + } + int result = comparePathElement(thisPath.next(), otherPath.next()); + if (result != 0) { + return result; + } + } + + return otherPath.hasNext() ? -1 : 0; + } + } + private DatastoreHelper() {} /** @@ -91,7 +146,7 @@ public static Credential getServiceAccountCredential(String account, String priv .setTransport(transport) .setJsonFactory(jsonFactory) .setServiceAccountId(account) - .setServiceAccountScopes(DatastoreOptions.SCOPE) + .setServiceAccountScopes(DatastoreOptions.SCOPES) .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile)) .build(); } @@ -134,6 +189,20 @@ public static Datastore getDatastoreFromEnv() throws GeneralSecurityException, I return DatastoreFactory.get().create(getOptionsfromEnv().build()); } + /** + * Gets a {@link QuerySplitter}. + * + * The returned {@link QuerySplitter#getSplits} cannot accept a query that contains inequality + * filters, a sort filter, or a missing kind. + */ + public static QuerySplitter getQuerySplitter() { + return QuerySplitterImpl.INSTANCE; + } + + public static Comparator getKeyComparator() { + return KeyComparator.INSTANCE; + } + /** * Make a sort order for use in a query. */ diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java index 44278959e..dc6bf1db3 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java @@ -17,6 +17,9 @@ import com.google.api.client.auth.oauth2.Credential; +import java.util.Arrays; +import java.util.List; + /** * A mutable object containing settings for the datastore. * @@ -40,8 +43,9 @@ public class DatastoreOptions { private static final String DEFAULT_HOST = "https://www.googleapis.com"; private final Credential credential; - public static final String SCOPE = "https://www.googleapis.com/auth/datastore" + - " https://www.googleapis.com/auth/userinfo.email"; + public static final List SCOPES = Arrays.asList( + "https://www.googleapis.com/auth/datastore", + "https://www.googleapis.com/auth/userinfo.email"); DatastoreOptions(String dataset, String host, Credential credential) { this.dataset = dataset; diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java new file mode 100644 index 000000000..f9b750b95 --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java @@ -0,0 +1,42 @@ +/* + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + +import com.google.api.services.datastore.DatastoreV1.Query; + +import java.util.List; + +/** + * Provides the ability to split a query into multiple shards. + * + */ +public interface QuerySplitter { + + /** + * Returns a list of sharded {@link Query}s for the given query. + * + * This will create up to the desired number of splits, however it may return less splits if + * the desired number of splits is unavailable. + * + * @param query the query to split. + * @param numSplits the desired number of splits. + * @param datastore the datastore to run on. + * @throws DatastoreException if there was a datastore error while generating query splits. + * @throws IllegalArgumentException if the given query or numSplits was invalid. + */ + List getSplits(Query query, int numSplits, Datastore datastore) + throws DatastoreException; +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java new file mode 100644 index 000000000..f36b5e67e --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java @@ -0,0 +1,232 @@ +/* + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + +import com.google.api.services.datastore.DatastoreV1.EntityResult; +import com.google.api.services.datastore.DatastoreV1.Filter; +import com.google.api.services.datastore.DatastoreV1.Key; +import com.google.api.services.datastore.DatastoreV1.PropertyExpression; +import com.google.api.services.datastore.DatastoreV1.PropertyFilter; +import com.google.api.services.datastore.DatastoreV1.PropertyFilter.Operator; +import com.google.api.services.datastore.DatastoreV1.PropertyOrder.Direction; +import com.google.api.services.datastore.DatastoreV1.PropertyReference; +import com.google.api.services.datastore.DatastoreV1.Query; +import com.google.api.services.datastore.DatastoreV1.QueryResultBatch; +import com.google.api.services.datastore.DatastoreV1.QueryResultBatch.MoreResultsType; +import com.google.api.services.datastore.DatastoreV1.RunQueryRequest; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * An implementation of {@link QuerySplitter} for the Datastore. + * + * Runs a query on the scatter property to get a random sampling of entities. + * + **/ +final class QuerySplitterImpl implements QuerySplitter { + + /** The number of keys to sample for each split. **/ + private static final int KEYS_PER_SPLIT = 32; + + static final QuerySplitter INSTANCE = new QuerySplitterImpl(); + + QuerySplitterImpl() { + // No initialization required. + } + + /** + * @see QuerySplitter#getSplits + */ + @Override + public List getSplits(Query query, int numSplits, Datastore datastore) + throws DatastoreException, IllegalArgumentException { + + validateQuery(query); + validateSplitSize(numSplits); + + List splits = new ArrayList(); + List scatterKeys = getScatterKeys(numSplits, query, datastore); + Key lastKey = null; + for (Key nextKey : getSplitKey(scatterKeys, numSplits)) { + splits.add(createSplit(lastKey, nextKey, query)); + lastKey = nextKey; + } + splits.add(createSplit(lastKey, null, query)); + return splits; + } + + /** + * Helper to determine if a filter operator is an inequality. + */ + private boolean isInequality(Operator operator) { + return operator == Operator.LESS_THAN || + operator == Operator.LESS_THAN_OR_EQUAL || + operator == Operator.GREATER_THAN || + operator == Operator.GREATER_THAN_OR_EQUAL; + } + + /** + * Verify that the given number of splits is not out of bounds. + * @param numSplits the number of splits. + * @throws IllegalArgumentException if the split size is invalid. + */ + private void validateSplitSize(int numSplits) throws IllegalArgumentException { + if (numSplits < 1) { + throw new IllegalArgumentException("The number of splits must be greater than 0."); + } + } + + /** + * Validate that we only have allowable filters. + * + * Note that equality and ancestor filters are allowed, however they may result in + * inefficient sharding. + */ + private void validateFilter(Filter filter) throws IllegalArgumentException { + if (filter.hasCompositeFilter()) { + for (Filter subFilter : filter.getCompositeFilter().getFilterList()) { + validateFilter(subFilter); + } + } else if (filter.hasPropertyFilter()) { + if (isInequality(filter.getPropertyFilter().getOperator())) { + throw new IllegalArgumentException("Query cannot have an inequality filter.", + new IllegalArgumentException()); + } + } + } + + /** + * Verify that the given query can be properly scattered. + * + * @param query the query to verify + * @throws IllegalArgumentException if the query is invalid. + */ + private void validateQuery(Query query) throws IllegalArgumentException { + if (query.getKindCount() != 1) { + throw new IllegalArgumentException("Query must have exactly one kind."); + } + if (query.getOrderCount() != 0) { + throw new IllegalArgumentException("Query cannot have a sort order."); + } + if (query.hasFilter()) { + validateFilter(query.getFilter()); + } + } + + /** + * Create a new {@link Query} given the query and range. + * + * @param lastKey the previous key. If null then assumed to be the beginning. + * @param nextKey the next key. If null then assumed to be the end. + * @param query the desired query. + */ + private Query createSplit(Key lastKey, Key nextKey, Query query) { + List keyFilters = new ArrayList(); + if (query.hasFilter()) { + keyFilters.add(query.getFilter()); + } + if (lastKey != null) { + keyFilters.add(DatastoreHelper.makeFilter( + DatastoreHelper.KEY_PROPERTY_NAME, PropertyFilter.Operator.GREATER_THAN_OR_EQUAL, + DatastoreHelper.makeValue(lastKey)).build()); + } + if (nextKey != null) { + keyFilters.add(DatastoreHelper.makeFilter( + DatastoreHelper.KEY_PROPERTY_NAME, PropertyFilter.Operator.LESS_THAN, + DatastoreHelper.makeValue(nextKey)).build()); + } + return Query.newBuilder(query).setFilter( + DatastoreHelper.makeCompositeFilter(keyFilters)).build(); + } + + /** + * Given a number of desired splits gets a list of scatter keys with multiples at each split. + * + * @param numSplits the number of desired splits. + * @param query the user query. + * @param datastore the datastore containing the data. + * @throws DatastoreException if there was an error when executing the datastore query. + */ + private List getScatterKeys(int numSplits, Query query, Datastore datastore) + throws DatastoreException { + Query.Builder scatterPointQuery = createScatterQuery(query, numSplits); + + List keySplits = new ArrayList(); + + QueryResultBatch batch = null; + do { + batch = datastore.runQuery(RunQueryRequest.newBuilder().setQuery(query).build()).getBatch(); + for (EntityResult result : batch.getEntityResultList()) { + keySplits.add(result.getEntity().getKey()); + } + scatterPointQuery.setStartCursor(batch.getEndCursor()); + scatterPointQuery.setLimit(scatterPointQuery.getLimit() - batch.getEntityResultCount()); + } while (batch.getMoreResults() == MoreResultsType.NOT_FINISHED); + + Collections.sort(keySplits, DatastoreHelper.getKeyComparator()); + return keySplits; + } + + /** + * Creates a scatter query from the given user query + * + * @param query the user's query. + * @param numSplits the number of splits to create. + */ + private Query.Builder createScatterQuery(Query query, int numSplits) { + // TODO(user): We can potentially support better splits with equality filters in our query + // if there exists a composite index on property, __scatter__, __key__. Until an API for + // metadata exists, this isn't possible. Note that ancestor and inequality queries fall into + // the same category. + Query.Builder scatterPointQuery = Query.newBuilder(); + scatterPointQuery.addAllKind(query.getKindList()); + scatterPointQuery.addOrder(DatastoreHelper.makeOrder( + DatastoreHelper.SCATTER_PROPERTY_NAME, Direction.ASCENDING)); + scatterPointQuery.setLimit(numSplits * KEYS_PER_SPLIT - 1); + scatterPointQuery.addProjection(PropertyExpression.newBuilder().setProperty( + PropertyReference.newBuilder().setName("__key__"))); + return scatterPointQuery; + } + + /** + * Given a list of keys and a number of splits find the keys to split on. + * @param keys the list of keys. + * @param numSplits the number of splits. + */ + private Iterable getSplitKey(List keys, int numSplits) { + // If the number of keys is less than the number of splits, we are limited in the number of + // splits we can make. + if (keys.size() < numSplits - 1) { + return keys; + } + + // Calculate the number of keys per split. This should be {@link KEYS_PER_SPLIT}, but may + // be less if there are not KEYS_PER_SPLIT * numSplits scatter properties. + double numKeysPerSplit = Math.max(1.0, (keys.size() + 1) / numSplits); + + List keysList = new ArrayList(numSplits - 1); + // Grab the last sample for each split, otherwise the first split will be too small. + for (int i = 1; i < numSplits; i++) { + int splitIndex = ((int) (i * numKeysPerSplit)) - 1; + keysList.add(keys.get(splitIndex)); + } + + return keysList; + } + +} From 45831c286a3609e18868e1d23f9503f723a03d41 Mon Sep 17 00:00:00 2001 From: Ed Davisson Date: Tue, 24 Sep 2013 15:17:26 -0700 Subject: [PATCH 03/54] v1beta2-rev1-2.0.0 ------------------ - API changes: - `BlindWrite` method merged into `Commit`. - Added `list_value` to `Value` and changed `value` to a non-repeated field in `Property`. - In JSON API, string constants are now uppercase and underscore-separated instead of camel-cased (e.g. `LESS_THAN_OR_EQUAL` instead of `lessThanOrEqual`). - GQL changes: - New synthetic literals: `BLOB`, `BLOBKEY`, `DATETIME`, `KEY`. - Support for `IS NULL`. - Fixed partition ID handling for binding arguments. - Documentation changes: - All documentation has been updated to the v1beta2 API. - Getting started guide for Node.js now uses v0.4.5 of google-api-nodejs-client. - Fixed partition ID handling for query requests that include an explicit partition ID. - Fixed scopes in discovery document. - https://github.com/GoogleCloudPlatform/google-cloud-datastore/issues/9 - Fixed an issue where command line tool didn't work for some locales. - https://github.com/GoogleCloudPlatform/google-cloud-datastore/issues/12 --- .../services/datastore/client/Datastore.java | 10 - .../datastore/client/DatastoreException.java | 18 +- .../datastore/client/DatastoreFactory.java | 2 +- .../datastore/client/DatastoreHelper.java | 201 ++++++++++++------ .../datastore/client/QuerySplitter.java | 6 +- .../datastore/client/QuerySplitterImpl.java | 101 +++++---- 6 files changed, 219 insertions(+), 119 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/Datastore.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/Datastore.java index f3b0a3107..6765609f3 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/Datastore.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/Datastore.java @@ -20,8 +20,6 @@ import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse; import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest; import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse; -import com.google.api.services.datastore.DatastoreV1.BlindWriteRequest; -import com.google.api.services.datastore.DatastoreV1.BlindWriteResponse; import com.google.api.services.datastore.DatastoreV1.CommitRequest; import com.google.api.services.datastore.DatastoreV1.CommitResponse; import com.google.api.services.datastore.DatastoreV1.LookupRequest; @@ -113,12 +111,4 @@ public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreExcept throw invalidResponseException("runQuery", exception); } } - - public BlindWriteResponse blindWrite(BlindWriteRequest request) throws DatastoreException { - try { - return BlindWriteResponse.parseFrom(remoteRpc.call("blindWrite", request)); - } catch (IOException exception) { - throw invalidResponseException("blindWrite", exception); - } - } } diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreException.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreException.java index 54f144474..09c4cac4b 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreException.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreException.java @@ -20,8 +20,22 @@ * */ public class DatastoreException extends Exception { - public final String methodName; - public final int code; + private final String methodName; + private final int code; + + /** + * @return the HTTP response code + */ + public int getCode() { + return code; + } + + /** + * @return the datastore method name + */ + public String getMethodName() { + return methodName; + } public DatastoreException(String methodName, int code, String message, Throwable cause) { super(message, cause); diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java index a8d0492be..232554690 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java @@ -39,7 +39,7 @@ public class DatastoreFactory { /** Singleton factory instance. */ private static final DatastoreFactory INSTANCE = new DatastoreFactory(); - private static final String VERSION = "v1beta1"; + private static final String VERSION = "v1beta2"; // Lazy load this because we might be running inside App Engine and this // class isn't on the whitelist. diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java index 65ddb3c45..b099b11f6 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java @@ -39,7 +39,6 @@ import java.io.File; import java.io.IOException; import java.security.GeneralSecurityException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; @@ -238,8 +237,8 @@ public static Filter.Builder makeFilter(String property, PropertyFilter.Operator * * Uses AND to combine filters. */ - public static Filter.Builder makeFilter(Filter ...subfilters) { - return makeCompositeFilter(Arrays.asList(subfilters)); + public static Filter.Builder makeFilter(Filter... subfilters) { + return makeFilter(Arrays.asList(subfilters)); } /** @@ -247,7 +246,7 @@ public static Filter.Builder makeFilter(Filter ...subfilters) { * * Uses AND to combine filters. */ - public static Filter.Builder makeCompositeFilter(Iterable subfilters) { + public static Filter.Builder makeFilter(Iterable subfilters) { return Filter.newBuilder() .setCompositeFilter(CompositeFilter.newBuilder() .addAllFilter(subfilters) @@ -258,7 +257,7 @@ public static Filter.Builder makeCompositeFilter(Iterable subfilters) { * Make an entity property with the specified value. */ public static Property.Builder makeProperty(String name, Value value) { - return Property.newBuilder().setName(name).addValue(value); + return Property.newBuilder().setName(name).setValue(value); } /** @@ -269,35 +268,35 @@ public static Property.Builder makeProperty(String name, Value.Builder value) { } /** - * Make an entity property with the specified values. + * Make a property reference for use in a query. */ - public static Property.Builder makeProperty(String name, Iterable values) { - return Property.newBuilder().setName(name).setMulti(true).addAllValue(values); + public static PropertyReference.Builder makePropertyReference(String propertyName) { + return PropertyReference.newBuilder().setName(propertyName); } /** - * Make an entity property with the specified values. + * Make an entity value with the specified list values. */ - public static Property.Builder makeProperty(String name, Value ...values) { - return makeProperty(name, Arrays.asList(values)); + public static Value.Builder makeValue(Iterable listValues) { + return Value.newBuilder().addAllListValue(listValues); } /** - * Make an entity property with the specified values. + * Make an entity value with the specified list values. */ - public static Property.Builder makeProperty(String name, Value.Builder ...builders) { - Property.Builder prop = makeProperty(name, Collections.emptyList()); - for (Value.Builder builder : builders) { - prop.addValue(builder); - } - return prop; + public static Value.Builder makeValue(Value... values) { + return makeValue(Arrays.asList(values)); } /** - * Make a property reference for use in a query. + * Make an entity value with the specified list values. */ - public static PropertyReference.Builder makePropertyReference(String propertyName) { - return PropertyReference.newBuilder().setName(propertyName); + public static Value.Builder makeValue(Value.Builder... builders) { + Value.Builder value = Value.newBuilder(); + for (Value.Builder builder : builders) { + value.addListValue(builder); + } + return value; } /** @@ -419,64 +418,140 @@ public static Key.Builder makeKey(Object... elements) { } /** - * Return a map of property name to java object. - * - * Looses microseconds on timestamp values, meaning and indexing information. - * - * @param entity the entity to search for the property. - * @return The property value, or null if it's not found. + * @return a map of property name to value */ - public static Map getPropertyMap(EntityOrBuilder entity) { - Map result = new HashMap(); + public static Map getPropertyMap(EntityOrBuilder entity) { + Map result = new HashMap(); for (PropertyOrBuilder property : entity.getPropertyList()) { - Object value; - if (property.getMulti()) { - List values = new ArrayList(property.getValueCount()); - for (ValueOrBuilder subValue : property.getValueList()) { - values.add(getValue(subValue)); - } - value = values; - } else { - value = getValue(property.getValue(0)); - } - result.put(property.getName(), value); + result.put(property.getName(), property.getValue()); } return Collections.unmodifiableMap(result); } /** - * Convert a Value to a Java Object. Ignores meaning. - * - * Looses microseconds on timestamp values, meaning and indexing information. + * @return the double contained in value + * @throws IllegalArgumentException if the value does not contain a double. */ - public static Object getValue(ValueOrBuilder value) { - if (value.hasBooleanValue()) { - return value.getBooleanValue(); + public static double getDouble(ValueOrBuilder value) { + if (!value.hasDoubleValue()) { + throw new IllegalArgumentException("Value does not contain a double."); } - if (value.hasIntegerValue()) { - return value.getIntegerValue(); + return value.getDoubleValue(); + } + + /** + * @return the key contained in value + * @throws IllegalArgumentException if the value does not contain a key. + */ + public static Key getKey(ValueOrBuilder value) { + if (!value.hasKeyValue()) { + throw new IllegalArgumentException("Value does not contain a key."); + } + return value.getKeyValue(); + } + + /** + * @return the blob contained in value + * @throws IllegalArgumentException if the value does not contain a blob. + */ + public static ByteString getByteString(ValueOrBuilder value) { + if (value.getMeaning() == 18 && value.hasStringValue()) { + return value.getStringValueBytes(); + } else if (value.hasBlobValue()) { + return value.getBlobValue(); } - if (value.hasDoubleValue()) { - return value.getDoubleValue(); + throw new IllegalArgumentException("Value does not contain a bob."); + } + + /** + * @return the blob key contained in value + * @throws IllegalArgumentException if the value does not contain a blob key. + */ + public static String getBlobKey(ValueOrBuilder value) { + if (value.getMeaning() == 18 && value.hasStringValue()) { + return value.getStringValue(); + } else if (value.hasBlobKeyValue()) { + return value.getBlobKeyValue(); } - if (value.hasTimestampMicrosecondsValue()) { - return new Date(value.getTimestampMicrosecondsValue() / 1000L); + throw new IllegalArgumentException("Value does not contain a bob key."); + } + + /** + * @return the entity contained in value + * @throws IllegalArgumentException if the value does not contain an entity. + */ + public static Entity getEntity(ValueOrBuilder value) { + if (!value.hasEntityValue()) { + throw new IllegalArgumentException("Value does not contain an Entity."); } - if (value.hasKeyValue()) { - return value.getKeyValue(); + return value.getEntityValue(); + } + + /** + * @return the string contained in value + * @throws IllegalArgumentException if the value does not contain a string. + */ + public static String getString(ValueOrBuilder value) { + if (!value.hasStringValue()) { + throw new IllegalArgumentException("Value does not contain a string."); } - if (value.hasBlobKeyValue()) { - return value; // Returning value directly to avoid loosing type info. + return value.getStringValue(); + } + + /** + * @return the boolean contained in value + * @throws IllegalArgumentException if the value does not contain a boolean. + */ + public static boolean getBoolean(ValueOrBuilder value) { + if (!value.hasBooleanValue()) { + throw new IllegalArgumentException("Value does not contain a boolean."); } - if (value.hasStringValue()) { - return value.getStringValue(); + return value.getBooleanValue(); + } + + /** + * @return the long contained in value + * @throws IllegalArgumentException if the value does not contain a long. + */ + public static long getLong(ValueOrBuilder value) { + if (!value.hasIntegerValue()) { + throw new IllegalArgumentException("Value does not contain an integer."); } - if (value.hasBlobValue()) { - return value.getBlobValue(); + return value.getIntegerValue(); + } + + /** + * @return the timestamp in microseconds contained in value + * @throws IllegalArgumentException if the value does not contain a timestamp. + */ + public static long getTimestamp(ValueOrBuilder value) { + if (value.getMeaning() == 18 && value.hasIntegerValue()) { + return value.getIntegerValue(); + } else if (value.hasTimestampMicrosecondsValue()) { + return value.getTimestampMicrosecondsValue(); } - if (value.hasEntityValue()) { - return value.getEntityValue(); + throw new IllegalArgumentException("Value does not contain a timestamp."); + } + + /** + * @return the list contained in value + * @throws IllegalArgumentException if the value does not contain a list. + */ + public static List getList(ValueOrBuilder value) { + if (value.getListValueCount() == 0) { + throw new IllegalArgumentException("Value does not contain a list."); } - return null; + return value.getListValueList(); + } + + /** + * Convert a timestamp value into a {@link Date} clipping off the microseconds. + * + * @param value a timestamp value to convert + * @return the resulting {@link Date} + * @throws IllegalArgumentException if the value does not contain a timestamp. + */ + public static Date toDate(ValueOrBuilder value) { + return new Date(getTimestamp(value) / 1000); } } diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java index f9b750b95..a2228b9c4 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java @@ -28,8 +28,10 @@ public interface QuerySplitter { /** * Returns a list of sharded {@link Query}s for the given query. * - * This will create up to the desired number of splits, however it may return less splits if - * the desired number of splits is unavailable. + *

This will create up to the desired number of splits, however it may return less splits if + * the desired number of splits is unavailable. This will happen if the number of split points + * provided by the underlying Datastore is less than the desired number, which will occur if the + * number of results for the query is too small. * * @param query the query to split. * @param numSplits the desired number of splits. diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java index f36b5e67e..290965504 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java @@ -15,6 +15,8 @@ */ package com.google.api.services.datastore.client; +import static com.google.api.services.datastore.client.DatastoreHelper.makeFilter; + import com.google.api.services.datastore.DatastoreV1.EntityResult; import com.google.api.services.datastore.DatastoreV1.Filter; import com.google.api.services.datastore.DatastoreV1.Key; @@ -30,12 +32,14 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.EnumSet; import java.util.List; /** - * An implementation of {@link QuerySplitter} for the Datastore. + * Provides the ability to split a query into multiple shards using Cloud Datastore. * - * Runs a query on the scatter property to get a random sampling of entities. + *

This implementation of the QuerySplitter uses the __scatter__ property to gather + * random split points for a query. * **/ final class QuerySplitterImpl implements QuerySplitter { @@ -43,15 +47,15 @@ final class QuerySplitterImpl implements QuerySplitter { /** The number of keys to sample for each split. **/ private static final int KEYS_PER_SPLIT = 32; + private static final EnumSet UNSUPPORTED_OPERATORS = EnumSet.of(Operator.LESS_THAN, + Operator.LESS_THAN_OR_EQUAL, Operator.GREATER_THAN, Operator.GREATER_THAN_OR_EQUAL); + static final QuerySplitter INSTANCE = new QuerySplitterImpl(); - QuerySplitterImpl() { + private QuerySplitterImpl() { // No initialization required. } - /** - * @see QuerySplitter#getSplits - */ @Override public List getSplits(Query query, int numSplits, Datastore datastore) throws DatastoreException, IllegalArgumentException { @@ -59,7 +63,7 @@ public List getSplits(Query query, int numSplits, Datastore datastore) validateQuery(query); validateSplitSize(numSplits); - List splits = new ArrayList(); + List splits = new ArrayList(numSplits); List scatterKeys = getScatterKeys(numSplits, query, datastore); Key lastKey = null; for (Key nextKey : getSplitKey(scatterKeys, numSplits)) { @@ -70,16 +74,6 @@ public List getSplits(Query query, int numSplits, Datastore datastore) return splits; } - /** - * Helper to determine if a filter operator is an inequality. - */ - private boolean isInequality(Operator operator) { - return operator == Operator.LESS_THAN || - operator == Operator.LESS_THAN_OR_EQUAL || - operator == Operator.GREATER_THAN || - operator == Operator.GREATER_THAN_OR_EQUAL; - } - /** * Verify that the given number of splits is not out of bounds. * @param numSplits the number of splits. @@ -92,9 +86,9 @@ private void validateSplitSize(int numSplits) throws IllegalArgumentException { } /** - * Validate that we only have allowable filters. + * Validates that we only have allowable filters. * - * Note that equality and ancestor filters are allowed, however they may result in + *

Note that equality and ancestor filters are allowed, however they may result in * inefficient sharding. */ private void validateFilter(Filter filter) throws IllegalArgumentException { @@ -103,15 +97,14 @@ private void validateFilter(Filter filter) throws IllegalArgumentException { validateFilter(subFilter); } } else if (filter.hasPropertyFilter()) { - if (isInequality(filter.getPropertyFilter().getOperator())) { - throw new IllegalArgumentException("Query cannot have an inequality filter.", - new IllegalArgumentException()); + if (UNSUPPORTED_OPERATORS.contains(filter.getPropertyFilter().getOperator())) { + throw new IllegalArgumentException("Query cannot have any inequality filters."); } } } /** - * Verify that the given query can be properly scattered. + * Verifies that the given query can be properly scattered. * * @param query the query to verify * @throws IllegalArgumentException if the query is invalid. @@ -121,7 +114,7 @@ private void validateQuery(Query query) throws IllegalArgumentException { throw new IllegalArgumentException("Query must have exactly one kind."); } if (query.getOrderCount() != 0) { - throw new IllegalArgumentException("Query cannot have a sort order."); + throw new IllegalArgumentException("Query cannot have any sort orders."); } if (query.hasFilter()) { validateFilter(query.getFilter()); @@ -141,21 +134,26 @@ private Query createSplit(Key lastKey, Key nextKey, Query query) { keyFilters.add(query.getFilter()); } if (lastKey != null) { - keyFilters.add(DatastoreHelper.makeFilter( - DatastoreHelper.KEY_PROPERTY_NAME, PropertyFilter.Operator.GREATER_THAN_OR_EQUAL, - DatastoreHelper.makeValue(lastKey)).build()); + Filter lowerBound = DatastoreHelper.makeFilter(DatastoreHelper.KEY_PROPERTY_NAME, + PropertyFilter.Operator.GREATER_THAN_OR_EQUAL, + DatastoreHelper.makeValue(lastKey)).build(); + keyFilters.add(lowerBound); } if (nextKey != null) { - keyFilters.add(DatastoreHelper.makeFilter( - DatastoreHelper.KEY_PROPERTY_NAME, PropertyFilter.Operator.LESS_THAN, - DatastoreHelper.makeValue(nextKey)).build()); + Filter upperBound = DatastoreHelper.makeFilter(DatastoreHelper.KEY_PROPERTY_NAME, + PropertyFilter.Operator.LESS_THAN, + DatastoreHelper.makeValue(nextKey)).build(); + keyFilters.add(upperBound); } - return Query.newBuilder(query).setFilter( - DatastoreHelper.makeCompositeFilter(keyFilters)).build(); + return Query.newBuilder(query).setFilter(makeFilter(keyFilters)).build(); } /** - * Given a number of desired splits gets a list of scatter keys with multiples at each split. + * Gets a list of split keys given a desired number of splits. + * + *

This list will contain multiple split keys for each split. Only a single split key + * will be chosen as the split point, however providing multiple keys allows for more uniform + * sharding. * * @param numSplits the number of desired splits. * @param query the user query. @@ -168,16 +166,17 @@ private List getScatterKeys(int numSplits, Query query, Datastore datastore List keySplits = new ArrayList(); - QueryResultBatch batch = null; + QueryResultBatch batch; do { - batch = datastore.runQuery(RunQueryRequest.newBuilder().setQuery(query).build()).getBatch(); + RunQueryRequest scatterRequest = RunQueryRequest.newBuilder() + .setQuery(scatterPointQuery).build(); + batch = datastore.runQuery(scatterRequest).getBatch(); for (EntityResult result : batch.getEntityResultList()) { keySplits.add(result.getEntity().getKey()); } scatterPointQuery.setStartCursor(batch.getEndCursor()); scatterPointQuery.setLimit(scatterPointQuery.getLimit() - batch.getEntityResultCount()); } while (batch.getMoreResults() == MoreResultsType.NOT_FINISHED); - Collections.sort(keySplits, DatastoreHelper.getKeyComparator()); return keySplits; } @@ -197,7 +196,12 @@ private Query.Builder createScatterQuery(Query query, int numSplits) { scatterPointQuery.addAllKind(query.getKindList()); scatterPointQuery.addOrder(DatastoreHelper.makeOrder( DatastoreHelper.SCATTER_PROPERTY_NAME, Direction.ASCENDING)); - scatterPointQuery.setLimit(numSplits * KEYS_PER_SPLIT - 1); + // There is a split containing entities before and after each scatter entity: + // ||---*------*------*------*------*------*------*---|| = scatter entity + // If we represent each split as a region before a scatter entity, there is an extra region + // following the last scatter point. Thus, we do not need the scatter entities for the last + // region. + scatterPointQuery.setLimit((numSplits - 1) * KEYS_PER_SPLIT); scatterPointQuery.addProjection(PropertyExpression.newBuilder().setProperty( PropertyReference.newBuilder().setName("__key__"))); return scatterPointQuery; @@ -205,6 +209,7 @@ private Query.Builder createScatterQuery(Query query, int numSplits) { /** * Given a list of keys and a number of splits find the keys to split on. + * * @param keys the list of keys. * @param numSplits the number of splits. */ @@ -215,14 +220,28 @@ private Iterable getSplitKey(List keys, int numSplits) { return keys; } - // Calculate the number of keys per split. This should be {@link KEYS_PER_SPLIT}, but may - // be less if there are not KEYS_PER_SPLIT * numSplits scatter properties. - double numKeysPerSplit = Math.max(1.0, (keys.size() + 1) / numSplits); + // Calculate the number of keys per split. This should be KEYS_PER_SPLIT, but may + // be less if there are not KEYS_PER_SPLIT * (numSplits - 1) scatter entities. + // + // Consider the following dataset, where - represents an entity and * represents an entity + // that is returned as a scatter entity: + // ||---*-----*----*-----*-----*------*----*----|| + // If we want 4 splits in this data, the optimal split would look like: + // ||---*-----*----*-----*-----*------*----*----|| + // | | | + // The scatter keys in the last region are not useful to us, so we never request them: + // ||---*-----*----*-----*-----*------*---------|| + // | | | + // With 6 scatter keys we want to set scatter points at indexes: 1, 3, 5. + // + // We keep this as a double so that any "fractional" keys per split get distributed throughout + // the splits and don't make the last split significantly larger than the rest. + double numKeysPerSplit = Math.max(1.0, ((double) keys.size()) / (numSplits - 1)); List keysList = new ArrayList(numSplits - 1); // Grab the last sample for each split, otherwise the first split will be too small. for (int i = 1; i < numSplits; i++) { - int splitIndex = ((int) (i * numKeysPerSplit)) - 1; + int splitIndex = (int) Math.round(i * numKeysPerSplit) - 1; keysList.add(keys.get(splitIndex)); } From 4c28630668dca7b6145dfc0ac3d131c256966a49 Mon Sep 17 00:00:00 2001 From: Johan Euphrosine Date: Wed, 2 Oct 2013 16:50:25 -0700 Subject: [PATCH 04/54] python: fix #23 java: add http initializer proto: better comments --- .../datastore/client/DatastoreFactory.java | 2 +- .../datastore/client/DatastoreOptions.java | 24 ++++++++++++++++--- .../services/datastore/client/RemoteRpc.java | 17 +++++++++---- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java index 232554690..18a232e3f 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java @@ -72,7 +72,7 @@ RemoteRpc newRemoteRpc(DatastoreOptions options) { throw new IllegalArgumentException("options not set"); } HttpRequestFactory client = makeClient(options); - return new RemoteRpc(client, + return new RemoteRpc(client, options.getInitializer(), buildUrl(options, System.getenv("DATASTORE_URL_INTERNAL_OVERRIDE"))); } diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java index dc6bf1db3..f20b096cc 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java @@ -16,6 +16,7 @@ package com.google.api.services.datastore.client; import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.HttpRequestInitializer; import java.util.Arrays; import java.util.List; @@ -42,14 +43,18 @@ public class DatastoreOptions { private final String host; private static final String DEFAULT_HOST = "https://www.googleapis.com"; + private final HttpRequestInitializer initializer; + private final Credential credential; public static final List SCOPES = Arrays.asList( "https://www.googleapis.com/auth/datastore", "https://www.googleapis.com/auth/userinfo.email"); - DatastoreOptions(String dataset, String host, Credential credential) { + DatastoreOptions(String dataset, String host, HttpRequestInitializer initializer, + Credential credential) { this.dataset = dataset; this.host = host != null ? host : DEFAULT_HOST; + this.initializer = initializer; this.credential = credential; } @@ -59,6 +64,7 @@ public class DatastoreOptions { public static class Builder { private String dataset; private String host; + private HttpRequestInitializer initializer; private Credential credential; public Builder() { } @@ -66,11 +72,12 @@ public Builder() { } public Builder(DatastoreOptions options) { this.dataset = options.dataset; this.host = options.host; + this.initializer = options.initializer; this.credential = options.credential; } public DatastoreOptions build() { - return new DatastoreOptions(dataset, host, credential); + return new DatastoreOptions(dataset, host, initializer, credential); } /** @@ -89,6 +96,14 @@ public Builder host(String newHost) { return this; } + /** + * Sets the (optional) initializer to run on HTTP requests to the API. + */ + public Builder initializer(HttpRequestInitializer newInitializer) { + initializer = newInitializer; + return this; + } + /** * Sets the Google APIs credentials used to access the API. */ @@ -108,8 +123,11 @@ public String getHost() { return host; } + public HttpRequestInitializer getInitializer() { + return initializer; + } + public Credential getCredential() { return credential; } - } diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java index 60f30395f..edc5894b4 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java @@ -16,12 +16,13 @@ package com.google.api.services.datastore.client; import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpStatusCodes; import com.google.api.client.http.protobuf.ProtoHttpContent; - import com.google.protobuf.MessageLite; import java.io.IOException; @@ -35,12 +36,14 @@ class RemoteRpc { private static final Logger logger = Logger.getLogger(RemoteRpc.class.getName()); - private HttpRequestFactory client; - private String url; + private final HttpRequestFactory client; + private final HttpRequestInitializer initializer; + private final String url; private int rpcCount = 0; - RemoteRpc(HttpRequestFactory client, String url) { + RemoteRpc(HttpRequestFactory client, HttpRequestInitializer initializer, String url) { this.client = client; + this.initializer = initializer; this.url = url; try { resolveURL("dummyRpc"); @@ -67,7 +70,11 @@ InputStream call(String methodName, MessageLite request) throws DatastoreExcepti try { rpcCount++; ProtoHttpContent payload = new ProtoHttpContent(request); - httpResponse = client.buildPostRequest(resolveURL(methodName), payload).execute(); + HttpRequest httpRequest = client.buildPostRequest(resolveURL(methodName), payload); + if (initializer != null) { + initializer.initialize(httpRequest); + } + httpResponse = httpRequest.execute(); return httpResponse.getContent(); } catch (HttpResponseException e) { throw makeException(url, methodName, e.getStatusCode(), e.getContent(), e); From 118b49dce984c4993b9cc874db47a904442b1dc3 Mon Sep 17 00:00:00 2001 From: Ed Davisson Date: Mon, 16 Dec 2013 11:14:46 -0800 Subject: [PATCH 05/54] v1beta2-rev1-2.1.0 --- .../api/services/datastore/client/DatastoreFactory.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java index 18a232e3f..26d0a6a9e 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java @@ -39,7 +39,9 @@ public class DatastoreFactory { /** Singleton factory instance. */ private static final DatastoreFactory INSTANCE = new DatastoreFactory(); - private static final String VERSION = "v1beta2"; + + /** API version. */ + public static final String VERSION = "v1beta2"; // Lazy load this because we might be running inside App Engine and this // class isn't on the whitelist. From 49a43b9655712806c0c58b27be91adb4c996c9cc Mon Sep 17 00:00:00 2001 From: bmenasha Date: Tue, 5 May 2015 10:53:07 -0400 Subject: [PATCH 06/54] changed class usage in comment to match code In the class comment, the example code was invoking the wrong methods: datastore.stopDatastore() is named datastore.stop() datastore.clearDatastore() is named datastore.clear() --- .../services/datastore/client/LocalDevelopmentDatastore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java index 5b92cfc17..3a65df8eb 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java @@ -56,12 +56,12 @@ * * {@literal @}Before * public void setUp() throws LocalDevelopmentDatastoreException { - * datastore.clearDatastore(); + * datastore.clear(); * } * * {@literal @}AfterClass * public static void stopLocalDatastore() throws LocalDevelopmentDatastoreException { - * datastore.stopDatastore(); + * datastore.stop(); * } * * {@literal @}Test From f4f606f0b119cd41d3639c3414b59c28e5942262 Mon Sep 17 00:00:00 2001 From: Steven Kim Date: Tue, 26 May 2015 16:11:08 -0400 Subject: [PATCH 07/54] Corrects error message on exceptions --- .../google/api/services/datastore/client/DatastoreHelper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java index b099b11f6..0035b5c10 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java @@ -460,7 +460,7 @@ public static ByteString getByteString(ValueOrBuilder value) { } else if (value.hasBlobValue()) { return value.getBlobValue(); } - throw new IllegalArgumentException("Value does not contain a bob."); + throw new IllegalArgumentException("Value does not contain a blob."); } /** @@ -473,7 +473,7 @@ public static String getBlobKey(ValueOrBuilder value) { } else if (value.hasBlobKeyValue()) { return value.getBlobKeyValue(); } - throw new IllegalArgumentException("Value does not contain a bob key."); + throw new IllegalArgumentException("Value does not contain a blob key."); } /** From 229dc4c62cfab33fddfacff00aca3a3fea4b0071 Mon Sep 17 00:00:00 2001 From: Ed Davisson Date: Mon, 31 Aug 2015 13:50:43 -0700 Subject: [PATCH 08/54] v1beta2-rev1-3.0.0 --- .../client/BaseDatastoreFactory.java | 113 ++++++++++++++++++ .../datastore/client/DatastoreFactory.java | 79 +----------- .../datastore/client/DatastoreHelper.java | 78 +++++++++--- .../datastore/client/DatastoreOptions.java | 41 +++++-- .../client/LocalDevelopmentDatastore.java | 104 ++++++++++++---- .../LocalDevelopmentDatastoreFactory.java | 8 +- .../LocalDevelopmentDatastoreOptions.java | 53 ++++++++ .../datastore/client/QuerySplitter.java | 23 +++- .../datastore/client/QuerySplitterImpl.java | 21 +++- .../services/datastore/client/RemoteRpc.java | 15 +-- 10 files changed, 399 insertions(+), 136 deletions(-) create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/BaseDatastoreFactory.java create mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreOptions.java diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/BaseDatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/BaseDatastoreFactory.java new file mode 100644 index 000000000..e71d87794 --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/BaseDatastoreFactory.java @@ -0,0 +1,113 @@ +/* + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; + +import java.util.Arrays; +import java.util.logging.ConsoleHandler; +import java.util.logging.Formatter; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.logging.StreamHandler; + +/** + * Client factory for {@link Datastore}. + * + */ +abstract class BaseDatastoreFactory { + + // Non-javadoc. + // This class allows RemoteRpc to be used by factory implementations defined + // outside of this package. + public static class RemoteRpc + extends com.google.api.services.datastore.client.RemoteRpc { + public RemoteRpc(HttpRequestFactory client, HttpRequestInitializer initializer, String url) { + super(client, initializer, url); + } + } + + private static final Logger logger = Logger.getLogger(BaseDatastoreFactory.class.getName()); + + // Lazy load this because we might be running inside App Engine and this + // class isn't on the whitelist. + private static ConsoleHandler methodHandler; + + protected abstract String buildUrl(DatastoreOptions options, String overrideUrl); + public abstract T create(DatastoreOptions options) throws IllegalArgumentException; + + // TODO(user): Support something other than console handler for when we're + // running in App Engine + private static synchronized StreamHandler getStreamHandler() { + if (methodHandler == null) { + methodHandler = new ConsoleHandler(); + methodHandler.setFormatter(new Formatter() { + @Override + public String format(LogRecord record) { + return record.getMessage() + "\n"; + } + }); + methodHandler.setLevel(Level.FINE); + } + return methodHandler; + } + + protected BaseDatastoreFactory() { } + + protected RemoteRpc + newRemoteRpc(DatastoreOptions options) { + return newRemoteRpc(options, System.getenv("DATASTORE_URL_INTERNAL_OVERRIDE")); + } + + protected RemoteRpc newRemoteRpc(DatastoreOptions options, String urlOverride) { + if (options == null) { + throw new IllegalArgumentException("options not set"); + } + HttpRequestFactory client = makeClient(options); + return new RemoteRpc(client, options.getInitializer(), buildUrl(options, urlOverride)); + } + + /** + * Constructs a Google APIs HTTP client with the associated credentials. + */ + public HttpRequestFactory makeClient(DatastoreOptions options) { + Credential credential = options.getCredential(); + if (credential == null) { + logger.warning("Not using any credentials"); + } + HttpTransport transport = options.getTransport(); + if (transport == null) { + transport = credential == null ? new NetHttpTransport() : credential.getTransport(); + } + return transport.createRequestFactory(credential); + } + + /** + * Starts logging datastore method calls to the console. (Useful within tests.) + */ + public static void logMethodCalls() { + Logger logger = Logger.getLogger(Datastore.class.getName()); + logger.setLevel(Level.FINE); + if (!Arrays.asList(logger.getHandlers()).contains(getStreamHandler())) { + logger.addHandler(getStreamHandler()); + } + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java index 26d0a6a9e..cfd9d84c8 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java @@ -15,69 +15,24 @@ */ package com.google.api.services.datastore.client; -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.http.HttpRequestFactory; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; - import java.net.URI; import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.logging.ConsoleHandler; -import java.util.logging.Formatter; -import java.util.logging.Level; -import java.util.logging.LogRecord; -import java.util.logging.Logger; -import java.util.logging.StreamHandler; /** * Client factory for {@link Datastore}. * */ -public class DatastoreFactory { - private static final Logger logger = Logger.getLogger(DatastoreFactory.class.getName()); - +public class DatastoreFactory extends BaseDatastoreFactory{ /** Singleton factory instance. */ private static final DatastoreFactory INSTANCE = new DatastoreFactory(); - + /** API version. */ public static final String VERSION = "v1beta2"; - // Lazy load this because we might be running inside App Engine and this - // class isn't on the whitelist. - private static ConsoleHandler methodHandler; - public static DatastoreFactory get() { return INSTANCE; } - // TODO(user): Support something other than console handler for when we're - // running in App Engine - private static synchronized StreamHandler getStreamHandler() { - if (methodHandler == null) { - methodHandler = new ConsoleHandler(); - methodHandler.setFormatter(new Formatter() { - @Override - public String format(LogRecord record) { - return record.getMessage() + "\n"; - } - }); - methodHandler.setLevel(Level.FINE); - } - return methodHandler; - } - - DatastoreFactory() { } - - RemoteRpc newRemoteRpc(DatastoreOptions options) { - if (options == null) { - throw new IllegalArgumentException("options not set"); - } - HttpRequestFactory client = makeClient(options); - return new RemoteRpc(client, options.getInitializer(), - buildUrl(options, System.getenv("DATASTORE_URL_INTERNAL_OVERRIDE"))); - } - /** * Provides access to a datastore using the provided options. Logs * into the application using the credentials available via these @@ -85,27 +40,16 @@ RemoteRpc newRemoteRpc(DatastoreOptions options) { * * @throws IllegalArgumentException if the server or credentials weren't provided. */ + @Override public Datastore create(DatastoreOptions options) throws IllegalArgumentException { return new Datastore(newRemoteRpc(options)); } - /** - * Constructs a Google APIs HTTP client with the associated credentials. - */ - public HttpRequestFactory makeClient(DatastoreOptions options) { - Credential credential = options.getCredential(); - if (credential == null) { - logger.warning("Not using any credentials"); - return new NetHttpTransport().createRequestFactory(); - } - HttpTransport transport = credential.getTransport(); - return transport.createRequestFactory(credential); - } - /** * Build a valid datastore URL. */ - String buildUrl(DatastoreOptions options, String overrideUrl) { + @Override + protected String buildUrl(DatastoreOptions options, String overrideUrl) { try { if (options.getDataset() == null) { throw new IllegalArgumentException("datastore dataset not set in options"); @@ -122,15 +66,4 @@ String buildUrl(DatastoreOptions options, String overrideUrl) { throw new IllegalArgumentException(e); } } - - /** - * Starts logging datastore method calls to the console. (Useful within tests.) - */ - public static void logMethodCalls() { - Logger logger = Logger.getLogger(Datastore.class.getName()); - logger.setLevel(Level.FINE); - if (!Arrays.asList(logger.getHandlers()).contains(getStreamHandler())) { - logger.addHandler(getStreamHandler()); - } - } -} +} \ No newline at end of file diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java index 0035b5c10..9724ea3fb 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java @@ -39,7 +39,9 @@ import java.io.File; import java.io.IOException; import java.security.GeneralSecurityException; +import java.security.PrivateKey; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; @@ -53,7 +55,7 @@ * Helper methods for {@link Datastore}. * */ -public class DatastoreHelper { +public final class DatastoreHelper { private static final Logger logger = Logger.getLogger(DatastoreHelper.class.getName()); /** The property used in the Datastore to give us a random distribution. **/ @@ -139,53 +141,93 @@ public static Credential getComputeEngineCredential() */ public static Credential getServiceAccountCredential(String account, String privateKeyFile) throws GeneralSecurityException, IOException { + return getServiceAccountCredential(account, privateKeyFile, DatastoreOptions.SCOPES); + } + + /** + * Constructs credentials for the given account and key file. + * + * @param account the account to use. + * @param privateKeyFile the file name from which to get the private key. + * @param serviceAccountScopes Collection of OAuth scopes to use with the the service + * account flow or {@code null} if not. + * @return valid credentials or {@code null} + */ + public static Credential getServiceAccountCredential(String account, String privateKeyFile, + Collection serviceAccountScopes) throws GeneralSecurityException, IOException { + return getCredentialBuilderWithoutPrivateKey(account, serviceAccountScopes) + .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile)) + .build(); + } + + /** + * Constructs credentials for the given account and key. + * + * @param account the account to use. + * @param privateKey the private key for the given account. + * @param serviceAccountScopes Collection of OAuth scopes to use with the the service + * account flow or {@code null} if not. + * @return valid credentials or {@code null} + */ + public static Credential getServiceAccountCredential(String account, PrivateKey privateKey, + Collection serviceAccountScopes) throws GeneralSecurityException, IOException { + return getCredentialBuilderWithoutPrivateKey(account, serviceAccountScopes) + .setServiceAccountPrivateKey(privateKey) + .build(); + } + + private static GoogleCredential.Builder getCredentialBuilderWithoutPrivateKey( + String account, Collection scopes) throws GeneralSecurityException, IOException { NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); JacksonFactory jsonFactory = new JacksonFactory(); return new GoogleCredential.Builder() .setTransport(transport) .setJsonFactory(jsonFactory) .setServiceAccountId(account) - .setServiceAccountScopes(DatastoreOptions.SCOPES) - .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile)) - .build(); + .setServiceAccountScopes(scopes); } /** - * Uses the following enviorment variables to construct a {@link Datastore}: + * Uses the following environment variables to construct a {@link Datastore}: * DATASTORE_DATASET - the datastore dataset id * DATASTORE_HOST - the host to use to access the datastore - * e.g: https://www.googleapis.com/datastore/v1/datasets/{dataset} + * e.g: https://www.googleapis.com * DATASTORE_SERVICE_ACCOUNT - (optional) service account name * DATASTORE_PRIVATE_KEY_FILE - (optional) service account private key file * * Preference of credentials is: - * - ComputeEngine * - Service Account (specified by DATASTORE_SERVICE_ACCOUNT and DATASTORE_PRIVATE_KEY_FILE) + * - ComputeEngine * - no-credentials (for local development environment) */ - public static DatastoreOptions.Builder getOptionsfromEnv() + public static DatastoreOptions.Builder getOptionsFromEnv() throws GeneralSecurityException, IOException { DatastoreOptions.Builder options = new DatastoreOptions.Builder(); options.dataset(System.getenv("DATASTORE_DATASET")); options.host(System.getenv("DATASTORE_HOST")); - Credential credential = getComputeEngineCredential(); - if (credential != null) { - logger.info("Using Compute Engine credential."); - } else if (System.getenv("DATASTORE_SERVICE_ACCOUNT") != null && - System.getenv("DATASTORE_PRIVATE_KEY_FILE") != null) { + Credential credential; + if (System.getenv("DATASTORE_SERVICE_ACCOUNT") != null + && System.getenv("DATASTORE_PRIVATE_KEY_FILE") != null) { credential = getServiceAccountCredential(System.getenv("DATASTORE_SERVICE_ACCOUNT"), System.getenv("DATASTORE_PRIVATE_KEY_FILE")); logger.info("Using JWT Service Account credential."); + } else { + credential = getComputeEngineCredential(); + if (credential != null) { + logger.info("Using Compute Engine credential."); + } else { + logger.info("Using no credential."); + } } options.credential(credential); return options; } /** - * @see #getOptionsfromEnv() + * @see #getOptionsFromEnv() */ public static Datastore getDatastoreFromEnv() throws GeneralSecurityException, IOException { - return DatastoreFactory.get().create(getOptionsfromEnv().build()); + return DatastoreFactory.get().create(getOptionsFromEnv().build()); } /** @@ -328,7 +370,7 @@ public static Value.Builder makeValue(double value) { } /** - * Make a floating point value. + * Make a boolean value. */ public static Value.Builder makeValue(boolean value) { return Value.newBuilder().setBooleanValue(value); @@ -342,7 +384,7 @@ public static Value.Builder makeValue(String value) { } /** - * Make a key value. + * Make an entity value. */ public static Value.Builder makeValue(Entity entity) { return Value.newBuilder().setEntityValue(entity); @@ -356,7 +398,7 @@ public static Value.Builder makeValue(Entity.Builder entity) { } /** - * Make a entity value. + * Make a ByteString value. */ public static Value.Builder makeValue(ByteString blob) { return Value.newBuilder().setBlobValue(blob); diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java index f20b096cc..a66f22eb8 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java @@ -17,19 +17,21 @@ import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; import java.util.Arrays; import java.util.List; /** - * A mutable object containing settings for the datastore. + * An immutable object containing settings for the datastore. * *

Example for connecting to a datastore:

* * 
- * DatastoreOptions options = new DatastoreOptions()
- *     .dataset("my-dataset-id"),
- *     .credential(DatastoreHelper.getGceCredential());
+ * DatastoreOptions options = new DatastoreOptions.Builder()
+ *     .dataset("my-dataset-id")
+ *     .credential(DatastoreHelper.getComputeEngineCredential())
+ *     .build();
  * DatastoreFactory.get().create(options);
  *
* @@ -46,16 +48,17 @@ public class DatastoreOptions { private final HttpRequestInitializer initializer; private final Credential credential; + private final HttpTransport transport; public static final List SCOPES = Arrays.asList( "https://www.googleapis.com/auth/datastore", "https://www.googleapis.com/auth/userinfo.email"); - DatastoreOptions(String dataset, String host, HttpRequestInitializer initializer, - Credential credential) { - this.dataset = dataset; - this.host = host != null ? host : DEFAULT_HOST; - this.initializer = initializer; - this.credential = credential; + DatastoreOptions(Builder b) { + this.dataset = b.dataset; + this.host = b.host != null ? b.host : DEFAULT_HOST; + this.initializer = b.initializer; + this.credential = b.credential; + this.transport = b.transport; } /** @@ -66,6 +69,7 @@ public static class Builder { private String host; private HttpRequestInitializer initializer; private Credential credential; + private HttpTransport transport; public Builder() { } @@ -74,10 +78,11 @@ public Builder(DatastoreOptions options) { this.host = options.host; this.initializer = options.initializer; this.credential = options.credential; + this.transport = options.transport; } public DatastoreOptions build() { - return new DatastoreOptions(dataset, host, initializer, credential); + return new DatastoreOptions(this); } /** @@ -111,6 +116,14 @@ public Builder credential(Credential newCredential) { credential = newCredential; return this; } + + /** + * Sets the transport used to access the API. + */ + public Builder transport(HttpTransport transport) { + this.transport = transport; + return this; + } } // === getters === @@ -130,4 +143,8 @@ public HttpRequestInitializer getInitializer() { public Credential getCredential() { return credential; } -} + + public HttpTransport getTransport() { + return transport; + } +} \ No newline at end of file diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java index 3a65df8eb..39f81665d 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java @@ -15,17 +15,21 @@ */ package com.google.api.services.datastore.client; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; + import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.UrlEncodedContent; -import com.google.api.client.util.Preconditions; import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.file.Files; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -51,7 +55,7 @@ * .dataset("myapp") * .build(); * datastore = LocalDevelopmentDatastoreFactory.get().create(opts); - * datastore.start("/usr/local/gcdsdk", "myapp"); + * datastore.start("/usr/local/gcd-sdk", "myapp"); * } * * {@literal @}Before @@ -75,18 +79,23 @@ * */ public class LocalDevelopmentDatastore extends Datastore { - private static final int STARTUP_TIMEOUT_SECS = 5; + private static final int STARTUP_TIMEOUT_SECS = 30; private final String host; + private LocalDevelopmentDatastoreOptions localDevelopmentOptions; /** Internal state lifecycle management. */ enum State {NEW, STARTED, STOPPED} - private State state = State.NEW; + private volatile State state = State.NEW; + + private File datasetDirectory; - LocalDevelopmentDatastore(RemoteRpc rpc, String host) { + LocalDevelopmentDatastore(RemoteRpc rpc, String host, + LocalDevelopmentDatastoreOptions localDevelopmentOptions) { super(rpc); this.host = host; + this.localDevelopmentOptions = localDevelopmentOptions; } /** @@ -125,9 +134,9 @@ public void clear() throws LocalDevelopmentDatastoreException { */ public synchronized void start(String sdkPath, String dataset, String... cmdLineOptions) throws LocalDevelopmentDatastoreException { - Preconditions.checkNotNull(sdkPath, "sdkPath cannot be null"); - Preconditions.checkNotNull(dataset, "dataset cannot be null"); - Preconditions.checkState(state == State.NEW, "Cannot call start() more than once."); + checkNotNull(sdkPath, "sdkPath cannot be null"); + checkNotNull(dataset, "dataset cannot be null"); + checkState(state == State.NEW, "Cannot call start() more than once."); try { startDatastoreInternal(sdkPath, dataset, cmdLineOptions); state = State.STARTED; @@ -142,23 +151,29 @@ public synchronized void start(String sdkPath, String dataset, String... cmdLine void startDatastoreInternal(String sdkPath, String dataset, String... cmdLineOptions) throws LocalDevelopmentDatastoreException { - List cmd = Arrays.asList("./gcd.sh", "start", dataset, "--allow_remote_shutdown", - "--property=datastore.no_storage"); + File datasetDirectory = createDatastore(sdkPath, dataset); + List cmd = new ArrayList( + Arrays.asList("./gcd.sh", "start", "--allow_remote_shutdown", "--store_on_disk=false")); cmd.addAll(Arrays.asList(cmdLineOptions)); - ProcessBuilder builder = new ProcessBuilder(cmd); - builder.directory(new File(sdkPath)); - builder.redirectErrorStream(true); - Process devDatastoreServerProcess; + cmd.add(datasetDirectory.getPath()); + final Process gcdStartProcess; try { - devDatastoreServerProcess = builder.start(); + gcdStartProcess = newGcdProcess(sdkPath, cmd).start(); } catch (IOException e) { throw new LocalDevelopmentDatastoreException("Could not start dev server", e); } - StartupMonitor monitor = new StartupMonitor(devDatastoreServerProcess.getInputStream()); + // Ensure we don't leak the stub instance if tests end prematurely. + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + gcdStartProcess.destroy(); + } + }); + StartupMonitor monitor = new StartupMonitor(gcdStartProcess.getInputStream()); try { monitor.start(); if (!monitor.startupCompleteLatch.await(STARTUP_TIMEOUT_SECS, TimeUnit.SECONDS)) { - throw new LocalDevelopmentDatastoreException("Dev server did not start within 5 seconds"); + throw new LocalDevelopmentDatastoreException("Dev server did not start within 30 seconds"); } if (!monitor.success) { throw new LocalDevelopmentDatastoreException("Server did not start normally"); @@ -169,6 +184,42 @@ void startDatastoreInternal(String sdkPath, String dataset, String... cmdLineOpt } } + private File createDatastore(String sdkPath, String dataset) + throws LocalDevelopmentDatastoreException { + try { + datasetDirectory = Files.createTempDirectory("local-development-datastore").toFile(); + } catch (IOException e) { + throw new LocalDevelopmentDatastoreException("Could not create dataset tmp directory", e); + } + List cmd = Arrays.asList("./gcd.sh", "create", "--project_id=" + dataset, + datasetDirectory.getPath()); + try { + int retCode = newGcdProcess(sdkPath, cmd).start().waitFor(); + if (retCode != 0) { + throw new LocalDevelopmentDatastoreException( + String.format("Could not create dataset (retcode=%d)", retCode)); + } + } catch (IOException e) { + throw new LocalDevelopmentDatastoreException("Could not create dataset", e); + } catch (InterruptedException e) { + throw new LocalDevelopmentDatastoreException("Received an interrupt", e); + } + return datasetDirectory; + } + + private ProcessBuilder newGcdProcess(String sdkPath, List cmd) { + ProcessBuilder builder = new ProcessBuilder(cmd); + builder.directory(new File(sdkPath)); + builder.redirectErrorStream(true); + builder.environment().putAll(localDevelopmentOptions.getEnvVars()); + return builder; + } + + public File getDatasetDirectory() { + checkState(state == State.STARTED); + return datasetDirectory; + } + /** * Stops the local datastore. Multiple calls are allowed. * @@ -178,7 +229,20 @@ public synchronized void stop() throws LocalDevelopmentDatastoreException { // We intentionally don't check the internal state. If people want to try and stop the server // multiple times that's fine. stopDatastoreInternal(); - state = State.STOPPED; + if (state != State.STOPPED) { + state = State.STOPPED; + if (datasetDirectory != null) { + try { + Process process = + new ProcessBuilder("rm", "-r", datasetDirectory.getAbsolutePath()).start(); + if (process.waitFor() != 0) { + throw new IOException("Temp directory delete exited with " + process.exitValue()); + } + } catch (IOException | InterruptedException e) { + throw new IllegalStateException("Dataset directory wipe failed.", e); + } + } + } } void stopDatastoreInternal() throws LocalDevelopmentDatastoreException { @@ -191,8 +255,8 @@ void stopDatastoreInternal() throws LocalDevelopmentDatastoreException { HttpResponse httpResponse = client.buildPostRequest(url, content).execute(); if (!httpResponse.isSuccessStatusCode()) { throw new LocalDevelopmentDatastoreException( - "Request to shutdown local datastore returned http error code " + - httpResponse.getStatusCode()); + "Request to shutdown local datastore returned http error code " + + httpResponse.getStatusCode()); } } catch (IOException e) { throw new LocalDevelopmentDatastoreException( diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreFactory.java index c0c90c7f0..824f6b604 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreFactory.java @@ -35,7 +35,13 @@ public static LocalDevelopmentDatastoreFactory get() { @Override public LocalDevelopmentDatastore create(DatastoreOptions options) throws IllegalArgumentException { + return create(options, new LocalDevelopmentDatastoreOptions.Builder().build()); + } + + public LocalDevelopmentDatastore create(DatastoreOptions options, + LocalDevelopmentDatastoreOptions localDevelopmentOptions) { RemoteRpc rpc = newRemoteRpc(options); - return new LocalDevelopmentDatastore(rpc, options.getHost()); + return new LocalDevelopmentDatastore(rpc, options.getHost(), + localDevelopmentOptions); } } diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreOptions.java new file mode 100644 index 000000000..b59f8734e --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreOptions.java @@ -0,0 +1,53 @@ +/* + * Copyright 2014 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.api.services.datastore.client; + +import java.util.HashMap; +import java.util.Map; + +/** + * An immutable object containing settings for a {@link LocalDevelopmentDatastore}. + */ +public class LocalDevelopmentDatastoreOptions { + private final Map envVars; + + LocalDevelopmentDatastoreOptions(Map envVars) { + this.envVars = envVars; + } + + /** + * Builder for {@link LocalDevelopmentDatastoreOptions}. + */ + public static class Builder { + private final Map envVars = new HashMap(); + + public LocalDevelopmentDatastoreOptions build() { + return new LocalDevelopmentDatastoreOptions(envVars); + } + + /** + * Adds an environment variable to pass to the GCD tool. + */ + public Builder addEnvVar(String var, String value) { + envVars.put(var, value); + return this; + } + } + + public Map getEnvVars() { + return envVars; + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java index a2228b9c4..528349cc9 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java @@ -15,6 +15,7 @@ */ package com.google.api.services.datastore.client; +import com.google.api.services.datastore.DatastoreV1.PartitionId; import com.google.api.services.datastore.DatastoreV1.Query; import java.util.List; @@ -38,7 +39,27 @@ public interface QuerySplitter { * @param datastore the datastore to run on. * @throws DatastoreException if there was a datastore error while generating query splits. * @throws IllegalArgumentException if the given query or numSplits was invalid. + * @deprecated Use {@link getSplits(Query, PartitionId, int, Datastore)} instead, which provides + * the ability to supply a namespace. */ - List getSplits(Query query, int numSplits, Datastore datastore) + @Deprecated + List getSplits(Query query, int numSplits, Datastore datastore) throws DatastoreException; + + /** + * Returns a list of sharded {@link Query}s for the given query. + * + *

This will create up to the desired number of splits, however it may return less splits if + * the desired number of splits is unavailable. This will happen if the number of split points + * provided by the underlying Datastore is less than the desired number, which will occur if the + * number of results for the query is too small. + * + * @param query the query to split. + * @param partition the partition to run in. + * @param numSplits the desired number of splits. + * @param datastore the datastore to run on. + * @throws DatastoreException if there was a datastore error while generating query splits. + * @throws IllegalArgumentException if the given query or numSplits was invalid. + */ + List getSplits(Query query, PartitionId partition, int numSplits, Datastore datastore) throws DatastoreException; } diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java index 290965504..215633b25 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java @@ -20,6 +20,7 @@ import com.google.api.services.datastore.DatastoreV1.EntityResult; import com.google.api.services.datastore.DatastoreV1.Filter; import com.google.api.services.datastore.DatastoreV1.Key; +import com.google.api.services.datastore.DatastoreV1.PartitionId; import com.google.api.services.datastore.DatastoreV1.PropertyExpression; import com.google.api.services.datastore.DatastoreV1.PropertyFilter; import com.google.api.services.datastore.DatastoreV1.PropertyFilter.Operator; @@ -57,14 +58,21 @@ private QuerySplitterImpl() { } @Override + @Deprecated public List getSplits(Query query, int numSplits, Datastore datastore) throws DatastoreException, IllegalArgumentException { + return getSplits(query, PartitionId.newBuilder().build(), numSplits, datastore); + } + @Override + public List getSplits( + Query query, PartitionId partition, int numSplits, Datastore datastore) + throws DatastoreException, IllegalArgumentException { validateQuery(query); validateSplitSize(numSplits); List splits = new ArrayList(numSplits); - List scatterKeys = getScatterKeys(numSplits, query, datastore); + List scatterKeys = getScatterKeys(numSplits, query, partition, datastore); Key lastKey = null; for (Key nextKey : getSplitKey(scatterKeys, numSplits)) { splits.add(createSplit(lastKey, nextKey, query)); @@ -157,10 +165,12 @@ private Query createSplit(Key lastKey, Key nextKey, Query query) { * * @param numSplits the number of desired splits. * @param query the user query. + * @param partition the partition to run the query in. * @param datastore the datastore containing the data. * @throws DatastoreException if there was an error when executing the datastore query. */ - private List getScatterKeys(int numSplits, Query query, Datastore datastore) + private List getScatterKeys( + int numSplits, Query query, PartitionId partition, Datastore datastore) throws DatastoreException { Query.Builder scatterPointQuery = createScatterQuery(query, numSplits); @@ -168,8 +178,11 @@ private List getScatterKeys(int numSplits, Query query, Datastore datastore QueryResultBatch batch; do { - RunQueryRequest scatterRequest = RunQueryRequest.newBuilder() - .setQuery(scatterPointQuery).build(); + RunQueryRequest scatterRequest = + RunQueryRequest.newBuilder() + .setPartitionId(partition) + .setQuery(scatterPointQuery) + .build(); batch = datastore.runQuery(scatterRequest).getBatch(); for (EntityResult result : batch.getEntityResultList()) { keySplits.add(result.getEntity().getKey()); diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java index edc5894b4..80c6d5c81 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.InputStream; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; /** @@ -39,7 +40,7 @@ class RemoteRpc { private final HttpRequestFactory client; private final HttpRequestInitializer initializer; private final String url; - private int rpcCount = 0; + private final AtomicInteger rpcCount = new AtomicInteger(0); RemoteRpc(HttpRequestFactory client, HttpRequestInitializer initializer, String url) { this.client = client; @@ -61,14 +62,14 @@ class RemoteRpc { * * @throws DatastoreException if the RPC fails. */ - InputStream call(String methodName, MessageLite request) throws DatastoreException { + public InputStream call(String methodName, MessageLite request) throws DatastoreException { logger.fine("remote datastore call " + methodName); long startTime = System.currentTimeMillis(); try { HttpResponse httpResponse; try { - rpcCount++; + rpcCount.incrementAndGet(); ProtoHttpContent payload = new ProtoHttpContent(request); HttpRequest httpRequest = client.buildPostRequest(resolveURL(methodName), payload); if (initializer != null) { @@ -89,14 +90,14 @@ InputStream call(String methodName, MessageLite request) throws DatastoreExcepti } void resetRpcCount() { - rpcCount = 0; + rpcCount.set(0); } int getRpcCount() { - return rpcCount; + return rpcCount.get(); } - String getUrl() { + public String getUrl() { return url; } @@ -108,7 +109,7 @@ HttpRequestFactory getHttpRequestFactory() { return client; } - static DatastoreException makeException( + public static DatastoreException makeException( String url, String methodName, int code, String message, Throwable cause) { logger.fine("remote datastore call " + methodName + " against " + url + " failed: " + message); return new DatastoreException(methodName, code, message, cause); From 3a74434b6dbafb159ff34b2e203f5b7dcf71563b Mon Sep 17 00:00:00 2001 From: Ed Davisson Date: Thu, 10 Sep 2015 18:47:29 -0700 Subject: [PATCH 09/54] v1beta2-rev1-3.0.1 --- .../api/services/datastore/client/QuerySplitterImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java index 215633b25..c351981a0 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java @@ -137,6 +137,9 @@ private void validateQuery(Query query) throws IllegalArgumentException { * @param query the desired query. */ private Query createSplit(Key lastKey, Key nextKey, Query query) { + if (lastKey == null && nextKey == null) { + return query; + } List keyFilters = new ArrayList(); if (query.hasFilter()) { keyFilters.add(query.getFilter()); From 35cdd302923693cb4f809d432d5d3d134c7666d5 Mon Sep 17 00:00:00 2001 From: Pieter De Maeyer Date: Wed, 23 Sep 2015 09:50:29 +0200 Subject: [PATCH 10/54] Fix bug in LocalDevelopmentDatastore --- .../services/datastore/client/LocalDevelopmentDatastore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java index 39f81665d..018238ccb 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java +++ b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java @@ -191,7 +191,7 @@ private File createDatastore(String sdkPath, String dataset) } catch (IOException e) { throw new LocalDevelopmentDatastoreException("Could not create dataset tmp directory", e); } - List cmd = Arrays.asList("./gcd.sh", "create", "--project_id=" + dataset, + List cmd = Arrays.asList("./gcd.sh", "create", "--dataset_id=" + dataset, datasetDirectory.getPath()); try { int retCode = newGcdProcess(sdkPath, cmd).start().waitFor(); From 34bfe3c13a23f8d76d45e2167cd6e71071ec8fb7 Mon Sep 17 00:00:00 2001 From: Patrick Costello Date: Tue, 29 Mar 2016 14:40:00 -0700 Subject: [PATCH 11/54] Update to v1beta3. Change-Id: I045ae46d2ac2c4a8685ed8538137a8dbdf94028d --- .../datastore/client/DatastoreFactory.java | 69 -- .../datastore/client/DatastoreHelper.java | 599 -------------- .../datastore/client/DatastoreOptions.java | 150 ---- .../v1beta3}/client/Datastore.java | 37 +- .../v1beta3}/client/DatastoreException.java | 28 +- .../v1beta3/client/DatastoreFactory.java} | 119 +-- .../v1beta3/client/DatastoreHelper.java | 749 ++++++++++++++++++ .../v1beta3/client/DatastoreOptions.java | 188 +++++ .../client/LocalDevelopmentDatastore.java | 66 +- .../LocalDevelopmentDatastoreException.java | 5 +- .../LocalDevelopmentDatastoreFactory.java | 8 +- .../LocalDevelopmentDatastoreOptions.java | 4 +- .../v1beta3}/client/QuerySplitter.java | 30 +- .../v1beta3}/client/QuerySplitterImpl.java | 78 +- .../v1beta3}/client/RemoteRpc.java | 80 +- .../v1beta3/client/DatastoreFactoryTest.java | 107 +++ .../v1beta3/client/DatastoreHelperTest.java | 355 +++++++++ .../v1beta3/client/DatastoreTest.java | 362 +++++++++ .../client/LocalDevelopmentDatastoreTest.java | 96 +++ .../v1beta3/client/RemoteRpcTest.java | 87 ++ 20 files changed, 2199 insertions(+), 1018 deletions(-) delete mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java delete mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java delete mode 100644 datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java rename datastore-v1-proto-client/src/main/java/com/google/{api/services/datastore => datastore/v1beta3}/client/Datastore.java (70%) rename datastore-v1-proto-client/src/main/java/com/google/{api/services/datastore => datastore/v1beta3}/client/DatastoreException.java (69%) rename datastore-v1-proto-client/src/main/java/com/google/{api/services/datastore/client/BaseDatastoreFactory.java => datastore/v1beta3/client/DatastoreFactory.java} (55%) create mode 100644 datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java create mode 100644 datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreOptions.java rename datastore-v1-proto-client/src/main/java/com/google/{api/services/datastore => datastore/v1beta3}/client/LocalDevelopmentDatastore.java (86%) rename datastore-v1-proto-client/src/main/java/com/google/{api/services/datastore => datastore/v1beta3}/client/LocalDevelopmentDatastoreException.java (89%) rename datastore-v1-proto-client/src/main/java/com/google/{api/services/datastore => datastore/v1beta3}/client/LocalDevelopmentDatastoreFactory.java (89%) rename datastore-v1-proto-client/src/main/java/com/google/{api/services/datastore => datastore/v1beta3}/client/LocalDevelopmentDatastoreOptions.java (93%) rename datastore-v1-proto-client/src/main/java/com/google/{api/services/datastore => datastore/v1beta3}/client/QuerySplitter.java (55%) rename datastore-v1-proto-client/src/main/java/com/google/{api/services/datastore => datastore/v1beta3}/client/QuerySplitterImpl.java (79%) rename datastore-v1-proto-client/src/main/java/com/google/{api/services/datastore => datastore/v1beta3}/client/RemoteRpc.java (53%) create mode 100644 datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreFactoryTest.java create mode 100644 datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreHelperTest.java create mode 100644 datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java create mode 100644 datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreTest.java create mode 100644 datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/RemoteRpcTest.java diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java deleted file mode 100644 index cfd9d84c8..000000000 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreFactory.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2013 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.api.services.datastore.client; - -import java.net.URI; -import java.net.URISyntaxException; - -/** - * Client factory for {@link Datastore}. - * - */ -public class DatastoreFactory extends BaseDatastoreFactory{ - /** Singleton factory instance. */ - private static final DatastoreFactory INSTANCE = new DatastoreFactory(); - - /** API version. */ - public static final String VERSION = "v1beta2"; - - public static DatastoreFactory get() { - return INSTANCE; - } - - /** - * Provides access to a datastore using the provided options. Logs - * into the application using the credentials available via these - * options. - * - * @throws IllegalArgumentException if the server or credentials weren't provided. - */ - @Override - public Datastore create(DatastoreOptions options) throws IllegalArgumentException { - return new Datastore(newRemoteRpc(options)); - } - - /** - * Build a valid datastore URL. - */ - @Override - protected String buildUrl(DatastoreOptions options, String overrideUrl) { - try { - if (options.getDataset() == null) { - throw new IllegalArgumentException("datastore dataset not set in options"); - } - String url; - if (overrideUrl != null) { - url = String.format("%s/datasets/%s", overrideUrl, options.getDataset()); - } else { - url = String.format("%s/datastore/%s/datasets/%s", - options.getHost(), VERSION, options.getDataset()); - } - return new URI(url).toString(); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } - } -} \ No newline at end of file diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java deleted file mode 100644 index 9724ea3fb..000000000 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreHelper.java +++ /dev/null @@ -1,599 +0,0 @@ -/* - * Copyright 2013 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.api.services.datastore.client; - -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.compute.ComputeCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.jackson.JacksonFactory; -import com.google.api.services.datastore.DatastoreV1.CompositeFilter; -import com.google.api.services.datastore.DatastoreV1.Entity; -import com.google.api.services.datastore.DatastoreV1.EntityOrBuilder; -import com.google.api.services.datastore.DatastoreV1.Filter; -import com.google.api.services.datastore.DatastoreV1.Key; -import com.google.api.services.datastore.DatastoreV1.Key.PathElement; -import com.google.api.services.datastore.DatastoreV1.Property; -import com.google.api.services.datastore.DatastoreV1.PropertyFilter; -import com.google.api.services.datastore.DatastoreV1.PropertyOrBuilder; -import com.google.api.services.datastore.DatastoreV1.PropertyOrder; -import com.google.api.services.datastore.DatastoreV1.PropertyReference; -import com.google.api.services.datastore.DatastoreV1.Value; -import com.google.api.services.datastore.DatastoreV1.ValueOrBuilder; -import com.google.protobuf.ByteString; - -import java.io.File; -import java.io.IOException; -import java.security.GeneralSecurityException; -import java.security.PrivateKey; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; - -/** - * Helper methods for {@link Datastore}. - * - */ -public final class DatastoreHelper { - private static final Logger logger = Logger.getLogger(DatastoreHelper.class.getName()); - - /** The property used in the Datastore to give us a random distribution. **/ - public static final String SCATTER_PROPERTY_NAME = "__scatter__"; - - /** The property used in the Datastore to get the key of the entity. **/ - public static final String KEY_PROPERTY_NAME = "__key__"; - - /** - * Comparator for Keys - */ - private static final class KeyComparator implements Comparator { - - static final KeyComparator INSTANCE = new KeyComparator(); - - private int comparePathElement(PathElement thisElement, PathElement otherElement) { - int result = thisElement.getKind().compareTo(otherElement.getKind()); - if (result != 0) { - return result; - } - if (thisElement.hasId()) { - if (!otherElement.hasId()) { - return -1; - } - return Long.valueOf(thisElement.getId()).compareTo(otherElement.getId()); - } - if (otherElement.hasId()) { - return 1; - } - - return thisElement.getName().compareTo(otherElement.getName()); - } - - @Override - public int compare(Key thisKey, Key otherKey) { - if (!thisKey.getPartitionId().equals(otherKey.getPartitionId())) { - throw new IllegalArgumentException("Cannot compare keys with different partition ids."); - } - - Iterator thisPath = thisKey.getPathElementList().iterator(); - Iterator otherPath = otherKey.getPathElementList().iterator(); - while (thisPath.hasNext()) { - if (!otherPath.hasNext()) { - return 1; - } - int result = comparePathElement(thisPath.next(), otherPath.next()); - if (result != 0) { - return result; - } - } - - return otherPath.hasNext() ? -1 : 0; - } - } - - private DatastoreHelper() {} - - /** - * Attempts to get credentials from Google Compute Engine. - * - * @return valid credentials or {@code null} - */ - public static Credential getComputeEngineCredential() - throws GeneralSecurityException, IOException { - NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); - try { - // Try to connect using Google Compute Engine service account credentials. - ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory()); - // Force token refresh to detect if we are running on Google Compute Engine. - credential.refreshToken(); - return credential; - } catch (IOException e) { - return null; - } - } - - /** - * Constructs credentials for the given account and key. - * - * @param account the account to use. - * @param privateKeyFile the file name from which to get the private key. - * @return valid credentials or {@code null} - */ - public static Credential getServiceAccountCredential(String account, String privateKeyFile) - throws GeneralSecurityException, IOException { - return getServiceAccountCredential(account, privateKeyFile, DatastoreOptions.SCOPES); - } - - /** - * Constructs credentials for the given account and key file. - * - * @param account the account to use. - * @param privateKeyFile the file name from which to get the private key. - * @param serviceAccountScopes Collection of OAuth scopes to use with the the service - * account flow or {@code null} if not. - * @return valid credentials or {@code null} - */ - public static Credential getServiceAccountCredential(String account, String privateKeyFile, - Collection serviceAccountScopes) throws GeneralSecurityException, IOException { - return getCredentialBuilderWithoutPrivateKey(account, serviceAccountScopes) - .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile)) - .build(); - } - - /** - * Constructs credentials for the given account and key. - * - * @param account the account to use. - * @param privateKey the private key for the given account. - * @param serviceAccountScopes Collection of OAuth scopes to use with the the service - * account flow or {@code null} if not. - * @return valid credentials or {@code null} - */ - public static Credential getServiceAccountCredential(String account, PrivateKey privateKey, - Collection serviceAccountScopes) throws GeneralSecurityException, IOException { - return getCredentialBuilderWithoutPrivateKey(account, serviceAccountScopes) - .setServiceAccountPrivateKey(privateKey) - .build(); - } - - private static GoogleCredential.Builder getCredentialBuilderWithoutPrivateKey( - String account, Collection scopes) throws GeneralSecurityException, IOException { - NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); - JacksonFactory jsonFactory = new JacksonFactory(); - return new GoogleCredential.Builder() - .setTransport(transport) - .setJsonFactory(jsonFactory) - .setServiceAccountId(account) - .setServiceAccountScopes(scopes); - } - - /** - * Uses the following environment variables to construct a {@link Datastore}: - * DATASTORE_DATASET - the datastore dataset id - * DATASTORE_HOST - the host to use to access the datastore - * e.g: https://www.googleapis.com - * DATASTORE_SERVICE_ACCOUNT - (optional) service account name - * DATASTORE_PRIVATE_KEY_FILE - (optional) service account private key file - * - * Preference of credentials is: - * - Service Account (specified by DATASTORE_SERVICE_ACCOUNT and DATASTORE_PRIVATE_KEY_FILE) - * - ComputeEngine - * - no-credentials (for local development environment) - */ - public static DatastoreOptions.Builder getOptionsFromEnv() - throws GeneralSecurityException, IOException { - DatastoreOptions.Builder options = new DatastoreOptions.Builder(); - options.dataset(System.getenv("DATASTORE_DATASET")); - options.host(System.getenv("DATASTORE_HOST")); - Credential credential; - if (System.getenv("DATASTORE_SERVICE_ACCOUNT") != null - && System.getenv("DATASTORE_PRIVATE_KEY_FILE") != null) { - credential = getServiceAccountCredential(System.getenv("DATASTORE_SERVICE_ACCOUNT"), - System.getenv("DATASTORE_PRIVATE_KEY_FILE")); - logger.info("Using JWT Service Account credential."); - } else { - credential = getComputeEngineCredential(); - if (credential != null) { - logger.info("Using Compute Engine credential."); - } else { - logger.info("Using no credential."); - } - } - options.credential(credential); - return options; - } - - /** - * @see #getOptionsFromEnv() - */ - public static Datastore getDatastoreFromEnv() throws GeneralSecurityException, IOException { - return DatastoreFactory.get().create(getOptionsFromEnv().build()); - } - - /** - * Gets a {@link QuerySplitter}. - * - * The returned {@link QuerySplitter#getSplits} cannot accept a query that contains inequality - * filters, a sort filter, or a missing kind. - */ - public static QuerySplitter getQuerySplitter() { - return QuerySplitterImpl.INSTANCE; - } - - public static Comparator getKeyComparator() { - return KeyComparator.INSTANCE; - } - - /** - * Make a sort order for use in a query. - */ - public static PropertyOrder.Builder makeOrder(String property, - PropertyOrder.Direction direction) { - return PropertyOrder.newBuilder() - .setProperty(makePropertyReference(property)) - .setDirection(direction); - } - - /** - * Make a filter on a property for use in a query. - */ - public static Filter.Builder makeFilter(String property, PropertyFilter.Operator operator, - Value value) { - return Filter.newBuilder() - .setPropertyFilter(PropertyFilter.newBuilder() - .setProperty(makePropertyReference(property)) - .setOperator(operator) - .setValue(value)); - } - - /** - * Make a filter on a property for use in a query. - */ - public static Filter.Builder makeFilter(String property, PropertyFilter.Operator operator, - Value.Builder value) { - return makeFilter(property, operator, value.build()); - } - - /** - * Make a composite filter from the given sub-filters. - * - * Uses AND to combine filters. - */ - public static Filter.Builder makeFilter(Filter... subfilters) { - return makeFilter(Arrays.asList(subfilters)); - } - - /** - * Make a composite filter from the given sub-filters. - * - * Uses AND to combine filters. - */ - public static Filter.Builder makeFilter(Iterable subfilters) { - return Filter.newBuilder() - .setCompositeFilter(CompositeFilter.newBuilder() - .addAllFilter(subfilters) - .setOperator(CompositeFilter.Operator.AND)); - } - - /** - * Make an entity property with the specified value. - */ - public static Property.Builder makeProperty(String name, Value value) { - return Property.newBuilder().setName(name).setValue(value); - } - - /** - * Make an entity property with the specified value. - */ - public static Property.Builder makeProperty(String name, Value.Builder value) { - return makeProperty(name, value.build()); - } - - /** - * Make a property reference for use in a query. - */ - public static PropertyReference.Builder makePropertyReference(String propertyName) { - return PropertyReference.newBuilder().setName(propertyName); - } - - /** - * Make an entity value with the specified list values. - */ - public static Value.Builder makeValue(Iterable listValues) { - return Value.newBuilder().addAllListValue(listValues); - } - - /** - * Make an entity value with the specified list values. - */ - public static Value.Builder makeValue(Value... values) { - return makeValue(Arrays.asList(values)); - } - - /** - * Make an entity value with the specified list values. - */ - public static Value.Builder makeValue(Value.Builder... builders) { - Value.Builder value = Value.newBuilder(); - for (Value.Builder builder : builders) { - value.addListValue(builder); - } - return value; - } - - /** - * Make a key value. - */ - public static Value.Builder makeValue(Key key) { - return Value.newBuilder().setKeyValue(key); - } - - /** - * Make a key value. - */ - public static Value.Builder makeValue(Key.Builder key) { - return makeValue(key.build()); - } - - /** - * Make an integer value. - */ - public static Value.Builder makeValue(long key) { - return Value.newBuilder().setIntegerValue(key); - } - - /** - * Make a floating point value. - */ - public static Value.Builder makeValue(double value) { - return Value.newBuilder().setDoubleValue(value); - } - - /** - * Make a boolean value. - */ - public static Value.Builder makeValue(boolean value) { - return Value.newBuilder().setBooleanValue(value); - } - - /** - * Make a string value. - */ - public static Value.Builder makeValue(String value) { - return Value.newBuilder().setStringValue(value); - } - - /** - * Make an entity value. - */ - public static Value.Builder makeValue(Entity entity) { - return Value.newBuilder().setEntityValue(entity); - } - - /** - * Make a entity value. - */ - public static Value.Builder makeValue(Entity.Builder entity) { - return makeValue(entity.build()); - } - - /** - * Make a ByteString value. - */ - public static Value.Builder makeValue(ByteString blob) { - return Value.newBuilder().setBlobValue(blob); - } - - /** - * Make a date value given a time in milliseconds. - */ - public static Value.Builder makeValue(Date date) { - return Value.newBuilder().setTimestampMicrosecondsValue(date.getTime() * 1000L); - } - - /** - * Make a key from the specified path of kind/id-or-name pairs. - * - * The id-or-name values must be either Key, String, Long, Integer or Short. - * - * The last id-or-name value may be omitted, in which case an entity without - * an id is created (for use with automatic id allocation). - */ - public static Key.Builder makeKey(Object... elements) { - Key.Builder key = Key.newBuilder(); - for (int pathIndex = 0; pathIndex < elements.length; pathIndex += 2) { - PathElement.Builder pathElement = PathElement.newBuilder(); - Object element = elements[pathIndex]; - if (element instanceof Key) { - key.addAllPathElement(((Key) element).getPathElementList()); - // We increment by 2, but since we got a Key argument we're only consuming 1 element in this - // iteration of the loop. Decrement the index so that when we jump by 2 we end up in the - // right spot. - pathIndex--; - } else { - String kind; - try { - kind = (String) element; - } catch (ClassCastException e) { - throw new IllegalArgumentException("Expected string or Key, got: " + element.getClass()); - } - pathElement.setKind(kind); - if (pathIndex + 1 < elements.length) { - Object value = elements[pathIndex + 1]; - if (value instanceof String) { - pathElement.setName((String) value); - } else if (value instanceof Long) { - pathElement.setId((Long) value); - } else if (value instanceof Integer) { - pathElement.setId((Integer) value); - } else if (value instanceof Short) { - pathElement.setId((Short) value); - } else { - throw new IllegalArgumentException( - "Expected string or integer, got: " + value.getClass()); - } - } - key.addPathElement(pathElement); - } - } - return key; - } - - /** - * @return a map of property name to value - */ - public static Map getPropertyMap(EntityOrBuilder entity) { - Map result = new HashMap(); - for (PropertyOrBuilder property : entity.getPropertyList()) { - result.put(property.getName(), property.getValue()); - } - return Collections.unmodifiableMap(result); - } - - /** - * @return the double contained in value - * @throws IllegalArgumentException if the value does not contain a double. - */ - public static double getDouble(ValueOrBuilder value) { - if (!value.hasDoubleValue()) { - throw new IllegalArgumentException("Value does not contain a double."); - } - return value.getDoubleValue(); - } - - /** - * @return the key contained in value - * @throws IllegalArgumentException if the value does not contain a key. - */ - public static Key getKey(ValueOrBuilder value) { - if (!value.hasKeyValue()) { - throw new IllegalArgumentException("Value does not contain a key."); - } - return value.getKeyValue(); - } - - /** - * @return the blob contained in value - * @throws IllegalArgumentException if the value does not contain a blob. - */ - public static ByteString getByteString(ValueOrBuilder value) { - if (value.getMeaning() == 18 && value.hasStringValue()) { - return value.getStringValueBytes(); - } else if (value.hasBlobValue()) { - return value.getBlobValue(); - } - throw new IllegalArgumentException("Value does not contain a blob."); - } - - /** - * @return the blob key contained in value - * @throws IllegalArgumentException if the value does not contain a blob key. - */ - public static String getBlobKey(ValueOrBuilder value) { - if (value.getMeaning() == 18 && value.hasStringValue()) { - return value.getStringValue(); - } else if (value.hasBlobKeyValue()) { - return value.getBlobKeyValue(); - } - throw new IllegalArgumentException("Value does not contain a blob key."); - } - - /** - * @return the entity contained in value - * @throws IllegalArgumentException if the value does not contain an entity. - */ - public static Entity getEntity(ValueOrBuilder value) { - if (!value.hasEntityValue()) { - throw new IllegalArgumentException("Value does not contain an Entity."); - } - return value.getEntityValue(); - } - - /** - * @return the string contained in value - * @throws IllegalArgumentException if the value does not contain a string. - */ - public static String getString(ValueOrBuilder value) { - if (!value.hasStringValue()) { - throw new IllegalArgumentException("Value does not contain a string."); - } - return value.getStringValue(); - } - - /** - * @return the boolean contained in value - * @throws IllegalArgumentException if the value does not contain a boolean. - */ - public static boolean getBoolean(ValueOrBuilder value) { - if (!value.hasBooleanValue()) { - throw new IllegalArgumentException("Value does not contain a boolean."); - } - return value.getBooleanValue(); - } - - /** - * @return the long contained in value - * @throws IllegalArgumentException if the value does not contain a long. - */ - public static long getLong(ValueOrBuilder value) { - if (!value.hasIntegerValue()) { - throw new IllegalArgumentException("Value does not contain an integer."); - } - return value.getIntegerValue(); - } - - /** - * @return the timestamp in microseconds contained in value - * @throws IllegalArgumentException if the value does not contain a timestamp. - */ - public static long getTimestamp(ValueOrBuilder value) { - if (value.getMeaning() == 18 && value.hasIntegerValue()) { - return value.getIntegerValue(); - } else if (value.hasTimestampMicrosecondsValue()) { - return value.getTimestampMicrosecondsValue(); - } - throw new IllegalArgumentException("Value does not contain a timestamp."); - } - - /** - * @return the list contained in value - * @throws IllegalArgumentException if the value does not contain a list. - */ - public static List getList(ValueOrBuilder value) { - if (value.getListValueCount() == 0) { - throw new IllegalArgumentException("Value does not contain a list."); - } - return value.getListValueList(); - } - - /** - * Convert a timestamp value into a {@link Date} clipping off the microseconds. - * - * @param value a timestamp value to convert - * @return the resulting {@link Date} - * @throws IllegalArgumentException if the value does not contain a timestamp. - */ - public static Date toDate(ValueOrBuilder value) { - return new Date(getTimestamp(value) / 1000); - } -} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java deleted file mode 100644 index a66f22eb8..000000000 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreOptions.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2013 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.api.services.datastore.client; - -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.http.HttpRequestInitializer; -import com.google.api.client.http.HttpTransport; - -import java.util.Arrays; -import java.util.List; - -/** - * An immutable object containing settings for the datastore. - * - *

Example for connecting to a datastore:

- * - * 
- * DatastoreOptions options = new DatastoreOptions.Builder()
- *     .dataset("my-dataset-id")
- *     .credential(DatastoreHelper.getComputeEngineCredential())
- *     .build();
- * DatastoreFactory.get().create(options);
- *
- * - *

- * The options should be passed to {@link DatastoreFactory#create}. - *

- * - */ -public class DatastoreOptions { - private final String dataset; - private final String host; - private static final String DEFAULT_HOST = "https://www.googleapis.com"; - - private final HttpRequestInitializer initializer; - - private final Credential credential; - private final HttpTransport transport; - public static final List SCOPES = Arrays.asList( - "https://www.googleapis.com/auth/datastore", - "https://www.googleapis.com/auth/userinfo.email"); - - DatastoreOptions(Builder b) { - this.dataset = b.dataset; - this.host = b.host != null ? b.host : DEFAULT_HOST; - this.initializer = b.initializer; - this.credential = b.credential; - this.transport = b.transport; - } - - /** - * Builder for {@link DatastoreOptions}. - */ - public static class Builder { - private String dataset; - private String host; - private HttpRequestInitializer initializer; - private Credential credential; - private HttpTransport transport; - - public Builder() { } - - public Builder(DatastoreOptions options) { - this.dataset = options.dataset; - this.host = options.host; - this.initializer = options.initializer; - this.credential = options.credential; - this.transport = options.transport; - } - - public DatastoreOptions build() { - return new DatastoreOptions(this); - } - - /** - * Sets the dataset used to access the datastore. - */ - public Builder dataset(String newDataset) { - dataset = newDataset; - return this; - } - - /** - * Sets the host used to access the datastore. - */ - public Builder host(String newHost) { - host = newHost; - return this; - } - - /** - * Sets the (optional) initializer to run on HTTP requests to the API. - */ - public Builder initializer(HttpRequestInitializer newInitializer) { - initializer = newInitializer; - return this; - } - - /** - * Sets the Google APIs credentials used to access the API. - */ - public Builder credential(Credential newCredential) { - credential = newCredential; - return this; - } - - /** - * Sets the transport used to access the API. - */ - public Builder transport(HttpTransport transport) { - this.transport = transport; - return this; - } - } - - // === getters === - - public String getDataset() { - return dataset; - } - - public String getHost() { - return host; - } - - public HttpRequestInitializer getInitializer() { - return initializer; - } - - public Credential getCredential() { - return credential; - } - - public HttpTransport getTransport() { - return transport; - } -} \ No newline at end of file diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/Datastore.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/Datastore.java similarity index 70% rename from datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/Datastore.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/Datastore.java index 6765609f3..bd5a02809 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/Datastore.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/Datastore.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Google Inc. All Rights Reserved. + * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,27 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.api.services.datastore.client; +package com.google.datastore.v1beta3.client; -import com.google.api.client.http.HttpStatusCodes; -import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest; -import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse; -import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest; -import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse; -import com.google.api.services.datastore.DatastoreV1.CommitRequest; -import com.google.api.services.datastore.DatastoreV1.CommitResponse; -import com.google.api.services.datastore.DatastoreV1.LookupRequest; -import com.google.api.services.datastore.DatastoreV1.LookupResponse; -import com.google.api.services.datastore.DatastoreV1.RollbackRequest; -import com.google.api.services.datastore.DatastoreV1.RollbackResponse; -import com.google.api.services.datastore.DatastoreV1.RunQueryRequest; -import com.google.api.services.datastore.DatastoreV1.RunQueryResponse; +import com.google.datastore.v1beta3.AllocateIdsRequest; +import com.google.datastore.v1beta3.AllocateIdsResponse; +import com.google.datastore.v1beta3.BeginTransactionRequest; +import com.google.datastore.v1beta3.BeginTransactionResponse; +import com.google.datastore.v1beta3.CommitRequest; +import com.google.datastore.v1beta3.CommitResponse; +import com.google.datastore.v1beta3.LookupRequest; +import com.google.datastore.v1beta3.LookupResponse; +import com.google.datastore.v1beta3.RollbackRequest; +import com.google.datastore.v1beta3.RollbackResponse; +import com.google.datastore.v1beta3.RunQueryRequest; +import com.google.datastore.v1beta3.RunQueryResponse; +import com.google.rpc.Code; import java.io.IOException; /** - * Provides access to the Datastore. - * + * Provides access to Cloud Datastore. */ public class Datastore { @@ -59,8 +58,8 @@ public int getRpcCount() { } private DatastoreException invalidResponseException(String method, IOException exception) { - return RemoteRpc.makeException(remoteRpc.getUrl(), method, - HttpStatusCodes.STATUS_CODE_SERVICE_UNAVAILABLE, "Invalid response", exception); + return RemoteRpc.makeException(remoteRpc.getUrl(), method, Code.UNAVAILABLE, + "Invalid response", exception); } public AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws DatastoreException { diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreException.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreException.java similarity index 69% rename from datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreException.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreException.java index 09c4cac4b..70bfc2f4a 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/DatastoreException.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreException.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Google Inc. All Rights Reserved. + * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,20 +13,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.api.services.datastore.client; +package com.google.datastore.v1beta3.client; + +import com.google.rpc.Code; /** * Indicates an error in a {@link Datastore} call. - * */ public class DatastoreException extends Exception { private final String methodName; - private final int code; + private final Code code; + + public DatastoreException(String methodName, Code code, String message, Throwable cause) { + super(message, cause); + this.methodName = methodName; + this.code = code; + } /** - * @return the HTTP response code + * @return the canonical error code */ - public int getCode() { + public Code getCode() { return code; } @@ -36,10 +43,9 @@ public int getCode() { public String getMethodName() { return methodName; } - - public DatastoreException(String methodName, int code, String message, Throwable cause) { - super(message, cause); - this.methodName = methodName; - this.code = code; + + @Override + public String toString() { + return String.format("%s, code=%s", super.toString(), code); } } diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/BaseDatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java similarity index 55% rename from datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/BaseDatastoreFactory.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java index e71d87794..60cc87c18 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/BaseDatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Google Inc. All Rights Reserved. + * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,14 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.api.services.datastore.client; +package com.google.datastore.v1beta3.client; + +import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.http.HttpRequestFactory; -import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; +import java.net.URI; +import java.net.URISyntaxException; import java.util.Arrays; import java.util.logging.ConsoleHandler; import java.util.logging.Formatter; @@ -31,67 +34,44 @@ /** * Client factory for {@link Datastore}. - * */ -abstract class BaseDatastoreFactory { - - // Non-javadoc. - // This class allows RemoteRpc to be used by factory implementations defined - // outside of this package. - public static class RemoteRpc - extends com.google.api.services.datastore.client.RemoteRpc { - public RemoteRpc(HttpRequestFactory client, HttpRequestInitializer initializer, String url) { - super(client, initializer, url); - } - } - - private static final Logger logger = Logger.getLogger(BaseDatastoreFactory.class.getName()); +public class DatastoreFactory { + private static final Logger logger = Logger.getLogger(DatastoreFactory.class.getName()); // Lazy load this because we might be running inside App Engine and this // class isn't on the whitelist. private static ConsoleHandler methodHandler; - protected abstract String buildUrl(DatastoreOptions options, String overrideUrl); - public abstract T create(DatastoreOptions options) throws IllegalArgumentException; + /** API version. */ + public static final String VERSION = "v1beta3"; - // TODO(user): Support something other than console handler for when we're - // running in App Engine - private static synchronized StreamHandler getStreamHandler() { - if (methodHandler == null) { - methodHandler = new ConsoleHandler(); - methodHandler.setFormatter(new Formatter() { - @Override - public String format(LogRecord record) { - return record.getMessage() + "\n"; - } - }); - methodHandler.setLevel(Level.FINE); - } - return methodHandler; - } + public static final String DEFAULT_HOST = "https://datastore.googleapis.com"; - protected BaseDatastoreFactory() { } + /** Singleton factory instance. */ + private static final DatastoreFactory INSTANCE = new DatastoreFactory(); - protected RemoteRpc - newRemoteRpc(DatastoreOptions options) { - return newRemoteRpc(options, System.getenv("DATASTORE_URL_INTERNAL_OVERRIDE")); + public static DatastoreFactory get() { + return INSTANCE; } - protected RemoteRpc newRemoteRpc(DatastoreOptions options, String urlOverride) { - if (options == null) { - throw new IllegalArgumentException("options not set"); - } - HttpRequestFactory client = makeClient(options); - return new RemoteRpc(client, options.getInitializer(), buildUrl(options, urlOverride)); + /** + * Provides access to a datastore using the provided options. Logs + * into the application using the credentials available via these + * options. + * + * @throws IllegalArgumentException if the server or credentials weren't provided. + */ + public Datastore create(DatastoreOptions options) throws IllegalArgumentException { + return new Datastore(newRemoteRpc(options)); } - + /** * Constructs a Google APIs HTTP client with the associated credentials. */ public HttpRequestFactory makeClient(DatastoreOptions options) { Credential credential = options.getCredential(); if (credential == null) { - logger.warning("Not using any credentials"); + logger.info("Not using any credentials"); } HttpTransport transport = options.getTransport(); if (transport == null) { @@ -110,4 +90,51 @@ public static void logMethodCalls() { logger.addHandler(getStreamHandler()); } } + + /** + * Build a valid datastore URL. + */ + String buildProjectEndpoint(DatastoreOptions options) { + if (options.getProjectEndpoint() != null) { + return options.getProjectEndpoint(); + } + // DatastoreOptions ensures either project endpoint or project ID is set. + String projectId = checkNotNull(options.getProjectId()); + if (options.getLocalHost() != null) { + return validateUrl(String.format("http://%s/datastore/%s/projects/%s", + options.getLocalHost(), VERSION, projectId)); + } + return validateUrl(String.format("%s/%s/projects/%s", + DEFAULT_HOST, VERSION, projectId)); + } + + protected RemoteRpc newRemoteRpc(DatastoreOptions options) { + checkNotNull(options); + HttpRequestFactory client = makeClient(options); + return new RemoteRpc(client, options.getInitializer(), buildProjectEndpoint(options)); + } + + static String validateUrl(String url) { + try { + return new URI(url).toString(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + } + + // TODO: Support something other than console handler for when we're + // running in App Engine + private static synchronized StreamHandler getStreamHandler() { + if (methodHandler == null) { + methodHandler = new ConsoleHandler(); + methodHandler.setFormatter(new Formatter() { + @Override + public String format(LogRecord record) { + return record.getMessage() + "\n"; + } + }); + methodHandler.setLevel(Level.FINE); + } + return methodHandler; + } } diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java new file mode 100644 index 000000000..a170cc0fa --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java @@ -0,0 +1,749 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1beta3.client; + +import static com.google.datastore.v1beta3.client.DatastoreFactory.validateUrl; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson.JacksonFactory; +import com.google.datastore.v1beta3.ArrayValue; +import com.google.datastore.v1beta3.CompositeFilter; +import com.google.datastore.v1beta3.Entity; +import com.google.datastore.v1beta3.Filter; +import com.google.datastore.v1beta3.Key; +import com.google.datastore.v1beta3.Key.PathElement; +import com.google.datastore.v1beta3.Key.PathElement.IdTypeCase; +import com.google.datastore.v1beta3.Mutation; +import com.google.datastore.v1beta3.PartitionId; +import com.google.datastore.v1beta3.PropertyFilter; +import com.google.datastore.v1beta3.PropertyOrder; +import com.google.datastore.v1beta3.PropertyReference; +import com.google.datastore.v1beta3.Value; +import com.google.datastore.v1beta3.Value.ValueTypeCase; +import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; +import com.google.protobuf.TimestampOrBuilder; +import com.google.type.LatLng; + +import java.io.File; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.security.PrivateKey; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Helper methods for {@link Datastore}. + */ +// TODO: Accept OrBuilders when possible. +public final class DatastoreHelper { + private static final Logger logger = Logger.getLogger(DatastoreHelper.class.getName()); + + private static final int MICROSECONDS_PER_SECOND = 1000 * 1000; + private static final int NANOSECONDS_PER_MICROSECOND = 1000; + + /** The property used in the Datastore to give us a random distribution. **/ + public static final String SCATTER_PROPERTY_NAME = "__scatter__"; + + /** The property used in the Datastore to get the key of the entity. **/ + public static final String KEY_PROPERTY_NAME = "__key__"; + + /** + * Name of the environment variable used to set the project ID. + */ + public static final String PROJECT_ID_ENV_VAR = "DATASTORE_PROJECT_ID"; + + /** + * Name of the environment variable used to set the local host. + */ + public static final String LOCAL_HOST_ENV_VAR = "DATASTORE_EMULATOR_HOST"; + + /** + * Name of the environment variable used to set the service account. + */ + public static final String SERVICE_ACCOUNT_ENV_VAR = "DATASTORE_SERVICE_ACCOUNT"; + + /** + * Name of the environment variable used to set the private key file. + */ + public static final String PRIVATE_KEY_FILE_ENV_VAR = "DATASTORE_PRIVATE_KEY_FILE"; + + private static final String URL_OVERRIDE_ENV_VAR = "__DATASTORE_URL_OVERRIDE"; + + /** + * Comparator for Keys + */ + private static final class KeyComparator implements Comparator { + + static final KeyComparator INSTANCE = new KeyComparator(); + + private int comparePathElement(PathElement thisElement, PathElement otherElement) { + int result = thisElement.getKind().compareTo(otherElement.getKind()); + if (result != 0) { + return result; + } + if (thisElement.getIdTypeCase() == IdTypeCase.ID) { + if (otherElement.getIdTypeCase() != IdTypeCase.ID) { + return -1; + } + return Long.valueOf(thisElement.getId()).compareTo(otherElement.getId()); + } + if (otherElement.getIdTypeCase() == IdTypeCase.ID) { + return 1; + } + + return thisElement.getName().compareTo(otherElement.getName()); + } + + @Override + public int compare(Key thisKey, Key otherKey) { + if (!thisKey.getPartitionId().equals(otherKey.getPartitionId())) { + throw new IllegalArgumentException("Cannot compare keys with different partition ids."); + } + + Iterator thisPath = thisKey.getPathList().iterator(); + Iterator otherPath = otherKey.getPathList().iterator(); + while (thisPath.hasNext()) { + if (!otherPath.hasNext()) { + return 1; + } + int result = comparePathElement(thisPath.next(), otherPath.next()); + if (result != 0) { + return result; + } + } + + return otherPath.hasNext() ? -1 : 0; + } + } + + private DatastoreHelper() {} + + private static HttpTransport newTransport() throws GeneralSecurityException, IOException { + return GoogleNetHttpTransport.newTrustedTransport(); + } + + static JsonFactory newJsonFactory() { + return new JacksonFactory(); + } + + /** + * Constructs credentials for the given account and key. + * + * @param serviceAccountId service account ID (typically an e-mail address). + * @param privateKeyFile the file name from which to get the private key. + * @return valid credentials or {@code null} + */ + public static Credential getServiceAccountCredential(String serviceAccountId, + String privateKeyFile) throws GeneralSecurityException, IOException { + return getServiceAccountCredential(serviceAccountId, privateKeyFile, DatastoreOptions.SCOPES); + } + + /** + * Constructs credentials for the given account and key file. + * + * @param serviceAccountId service account ID (typically an e-mail address). + * @param privateKeyFile the file name from which to get the private key. + * @param serviceAccountScopes Collection of OAuth scopes to use with the the service + * account flow or {@code null} if not. + * @return valid credentials or {@code null} + */ + public static Credential getServiceAccountCredential(String serviceAccountId, + String privateKeyFile, Collection serviceAccountScopes) + throws GeneralSecurityException, IOException { + return getCredentialBuilderWithoutPrivateKey(serviceAccountId, serviceAccountScopes) + .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile)) + .build(); + } + + /** + * Constructs credentials for the given account and key. + * + * @param serviceAccountId service account ID (typically an e-mail address). + * @param privateKey the private key for the given account. + * @param serviceAccountScopes Collection of OAuth scopes to use with the the service + * account flow or {@code null} if not. + * @return valid credentials or {@code null} + */ + public static Credential getServiceAccountCredential(String serviceAccountId, + PrivateKey privateKey, Collection serviceAccountScopes) + throws GeneralSecurityException, IOException { + return getCredentialBuilderWithoutPrivateKey(serviceAccountId, serviceAccountScopes) + .setServiceAccountPrivateKey(privateKey) + .build(); + } + + private static GoogleCredential.Builder getCredentialBuilderWithoutPrivateKey( + String serviceAccountId, Collection serviceAccountScopes) + throws GeneralSecurityException, IOException { + HttpTransport transport = newTransport(); + JsonFactory jsonFactory = newJsonFactory(); + return new GoogleCredential.Builder() + .setTransport(transport) + .setJsonFactory(jsonFactory) + .setServiceAccountId(serviceAccountId) + .setServiceAccountScopes(serviceAccountScopes); + } + + /** + * Constructs a {@link Datastore} from environment variables and/or the Compute Engine metadata + * server. + * + *

The project ID is determined from, in order of preference: + *

    + *
  • DATASTORE_PROJECT_ID environment variable + *
  • Compute Engine + *
+ * + *

Credentials are taken from, in order of preference: + *

    + *
  1. No credentials (if the DATASTORE_EMULATOR_HOST environment variable is set) + *
  2. Service Account specified by the DATASTORE_SERVICE_ACCOUNT and DATASTORE_PRIVATE_KEY_FILE + * environment variables + *
  3. Google Application Default as described at + * {@link "https://developers.google.com/identity/protocols/application-default-credentials"} + *
+ */ + public static DatastoreOptions.Builder getOptionsFromEnv() + throws GeneralSecurityException, IOException { + DatastoreOptions.Builder options = new DatastoreOptions.Builder(); + setProjectEndpointFromEnv(options); + options.credential(getCredentialFromEnv()); + return options; + } + + private static Credential getCredentialFromEnv() throws GeneralSecurityException, IOException { + if (System.getenv(LOCAL_HOST_ENV_VAR) != null) { + logger.log(Level.INFO, "{0} environment variable was set. Not using credentials.", + new Object[] {LOCAL_HOST_ENV_VAR}); + return null; + } + String serviceAccount = System.getenv(SERVICE_ACCOUNT_ENV_VAR); + String privateKeyFile = System.getenv(PRIVATE_KEY_FILE_ENV_VAR); + if (serviceAccount != null && privateKeyFile != null) { + logger.log(Level.INFO, "{0} and {1} environment variables were set. " + + "Using service account credential.", + new Object[] {SERVICE_ACCOUNT_ENV_VAR, PRIVATE_KEY_FILE_ENV_VAR}); + return getServiceAccountCredential(serviceAccount, privateKeyFile); + } + return GoogleCredential.getApplicationDefault() + .createScoped(DatastoreOptions.SCOPES); + } + + /** + * Determines the project id from the environment. Uses the following sources in order of + * preference: + *
    + *
  1. Value of the DATASTORE_PROJECT_ID environment variable + *
  2. Compute Engine + *
+ * + * @throws IllegalStateException if the project ID cannot be determined + */ + private static String getProjectIdFromEnv() { + if (System.getenv(PROJECT_ID_ENV_VAR) != null) { + return System.getenv(PROJECT_ID_ENV_VAR); + } + String projectIdFromComputeEngine = getProjectIdFromComputeEngine(); + if (projectIdFromComputeEngine != null) { + return projectIdFromComputeEngine; + } + throw new IllegalStateException(String.format("Could not determine project ID." + + " If you are not running on Compute Engine, set the" + + " %s environment variable.", PROJECT_ID_ENV_VAR)); + } + + /** + * Gets the project ID from the Compute Engine metadata server. Returns {@code null} if the + * project ID cannot be determined (because, for instance, the code is not running on Compute + * Engine). + */ + public static String getProjectIdFromComputeEngine() { + HttpTransport transport; + try { + transport = newTransport(); + } catch (GeneralSecurityException | IOException e) { + logger.log(Level.WARNING, "Failed to create HttpTransport.", e); + return null; + } + try { + GenericUrl projectIdUrl = + new GenericUrl("http://metadata/computeMetadata/v1/project/project-id"); + HttpRequest request = transport.createRequestFactory().buildGetRequest(projectIdUrl); + request.getHeaders().set("Metadata-Flavor", "Google"); + return request.execute().parseAsString(); + } catch (IOException e) { + logger.log(Level.INFO, "Could not determine project ID from Compute Engine.", e); + return null; + } + } + + private static void setProjectEndpointFromEnv(DatastoreOptions.Builder options) { + // DATASTORE_HOST is deprecated. + if (System.getenv("DATASTORE_HOST") != null) { + logger.warning(String.format( + "Ignoring value of environment variable DATASTORE_HOST. " + + "To point datastore to a host running locally, use " + + "the environment variable %s.", + LOCAL_HOST_ENV_VAR)); + } + String projectId = getProjectIdFromEnv(); + if (System.getenv(URL_OVERRIDE_ENV_VAR) != null) { + options.projectEndpoint(validateUrl(String.format("%s/projects/%s", + System.getenv(URL_OVERRIDE_ENV_VAR), projectId))); + return; + } + if (System.getenv(LOCAL_HOST_ENV_VAR) != null) { + options.projectId(projectId); + options.localHost(System.getenv(LOCAL_HOST_ENV_VAR)); + return; + } + options.projectId(projectId); + return; + } + + /** + * @see #getOptionsFromEnv() + */ + public static Datastore getDatastoreFromEnv() throws GeneralSecurityException, IOException { + return DatastoreFactory.get().create(getOptionsFromEnv().build()); + } + + /** + * Gets a {@link QuerySplitter}. + * + * The returned {@link QuerySplitter#getSplits} cannot accept a query that contains inequality + * filters, a sort filter, or a missing kind. + */ + public static QuerySplitter getQuerySplitter() { + return QuerySplitterImpl.INSTANCE; + } + + public static Comparator getKeyComparator() { + return KeyComparator.INSTANCE; + } + + /** + * Make a sort order for use in a query. + */ + public static PropertyOrder.Builder makeOrder(String property, + PropertyOrder.Direction direction) { + return PropertyOrder.newBuilder() + .setProperty(makePropertyReference(property)) + .setDirection(direction); + } + + /** + * Make a filter on a property for use in a query. + */ + public static Filter.Builder makeFilter(String property, PropertyFilter.Operator operator, + Value value) { + return Filter.newBuilder() + .setPropertyFilter(PropertyFilter.newBuilder() + .setProperty(makePropertyReference(property)) + .setOp(operator) + .setValue(value)); + } + + /** + * Make a filter on a property for use in a query. + */ + public static Filter.Builder makeFilter(String property, PropertyFilter.Operator operator, + Value.Builder value) { + return makeFilter(property, operator, value.build()); + } + + /** + * Make a composite filter from the given sub-filters using AND to combine filters. + */ + public static Filter.Builder makeAndFilter(Filter... subfilters) { + return makeAndFilter(Arrays.asList(subfilters)); + } + + /** + * Make a composite filter from the given sub-filters using AND to combine filters. + */ + public static Filter.Builder makeAndFilter(Iterable subfilters) { + return Filter.newBuilder() + .setCompositeFilter(CompositeFilter.newBuilder() + .addAllFilters(subfilters) + .setOp(CompositeFilter.Operator.AND)); + } + + /** + * Make a property reference for use in a query. + */ + public static PropertyReference.Builder makePropertyReference(String propertyName) { + return PropertyReference.newBuilder().setName(propertyName); + } + + /** + * Make an array value containing the specified values. + */ + public static Value.Builder makeValue(Iterable values) { + return Value.newBuilder().setArrayValue(ArrayValue.newBuilder().addAllValues(values)); + } + + /** + * Make a list value containing the specified values. + */ + public static Value.Builder makeValue(Value value1, Value value2, Value... rest) { + ArrayValue.Builder arrayValue = ArrayValue.newBuilder(); + arrayValue.addValues(value1); + arrayValue.addValues(value2); + arrayValue.addAllValues(Arrays.asList(rest)); + return Value.newBuilder().setArrayValue(arrayValue); + } + + /** + * Make an array value containing the specified values. + */ + public static Value.Builder makeValue(Value.Builder value1, Value.Builder value2, + Value.Builder... rest) { + ArrayValue.Builder arrayValue = ArrayValue.newBuilder(); + arrayValue.addValues(value1); + arrayValue.addValues(value2); + for (Value.Builder builder : rest) { + arrayValue.addValues(builder); + } + return Value.newBuilder().setArrayValue(arrayValue); + } + + /** + * Make a key value. + */ + public static Value.Builder makeValue(Key key) { + return Value.newBuilder().setKeyValue(key); + } + + /** + * Make a key value. + */ + public static Value.Builder makeValue(Key.Builder key) { + return makeValue(key.build()); + } + + /** + * Make an integer value. + */ + public static Value.Builder makeValue(long key) { + return Value.newBuilder().setIntegerValue(key); + } + + /** + * Make a floating point value. + */ + public static Value.Builder makeValue(double value) { + return Value.newBuilder().setDoubleValue(value); + } + + /** + * Make a boolean value. + */ + public static Value.Builder makeValue(boolean value) { + return Value.newBuilder().setBooleanValue(value); + } + + /** + * Make a string value. + */ + public static Value.Builder makeValue(String value) { + return Value.newBuilder().setStringValue(value); + } + + /** + * Make an entity value. + */ + public static Value.Builder makeValue(Entity entity) { + return Value.newBuilder().setEntityValue(entity); + } + + /** + * Make a entity value. + */ + public static Value.Builder makeValue(Entity.Builder entity) { + return makeValue(entity.build()); + } + + /** + * Make a ByteString value. + */ + public static Value.Builder makeValue(ByteString blob) { + return Value.newBuilder().setBlobValue(blob); + } + + /** + * Make a timestamp value given a date. + */ + public static Value.Builder makeValue(Date date) { + return Value.newBuilder().setTimestampValue(toTimestamp(date.getTime() * 1000L)); + } + + private static Timestamp.Builder toTimestamp(long microseconds) { + long seconds = microseconds / MICROSECONDS_PER_SECOND; + long microsecondsRemainder = microseconds % MICROSECONDS_PER_SECOND; + if (microsecondsRemainder < 0) { + // Nanos must be positive even if microseconds is negative. + // Java modulus doesn't take care of this for us. + microsecondsRemainder += MICROSECONDS_PER_SECOND; + seconds -= 1; + } + return Timestamp.newBuilder() + .setSeconds(seconds) + .setNanos((int) microsecondsRemainder * NANOSECONDS_PER_MICROSECOND); + } + + /** + * Makes a GeoPoint value. + */ + public static Value.Builder makeValue(LatLng value) { + return Value.newBuilder().setGeoPointValue(value); + } + + /** + * Makes a GeoPoint value. + */ + public static Value.Builder makeValue(LatLng.Builder value) { + return makeValue(value.build()); + } + + /** + * Make a key from the specified path of kind/id-or-name pairs + * and/or Keys. + * + *

The id-or-name values must be either String, Long, Integer or Short. + * + *

The last id-or-name value may be omitted, in which case an entity without + * an id is created (for use with automatic id allocation). + * + *

The PartitionIds of all Keys in the path must be equal. The returned + * Key.Builder will use this PartitionId. + */ + public static Key.Builder makeKey(Object... elements) { + Key.Builder key = Key.newBuilder(); + PartitionId partitionId = null; + for (int pathIndex = 0; pathIndex < elements.length; pathIndex += 2) { + PathElement.Builder pathElement = PathElement.newBuilder(); + Object element = elements[pathIndex]; + if (element instanceof Key) { + Key subKey = (Key) element; + if (partitionId == null) { + partitionId = subKey.getPartitionId(); + } else if (!partitionId.equals(subKey.getPartitionId())) { + throw new IllegalArgumentException("Partition IDs did not match, found: " + + partitionId + " and " + subKey.getPartitionId()); + } + key.addAllPath(((Key) element).getPathList()); + // We increment by 2, but since we got a Key argument we're only consuming 1 element in this + // iteration of the loop. Decrement the index so that when we jump by 2 we end up in the + // right spot. + pathIndex--; + } else { + String kind; + try { + kind = (String) element; + } catch (ClassCastException e) { + throw new IllegalArgumentException("Expected string or Key, got: " + element.getClass()); + } + pathElement.setKind(kind); + if (pathIndex + 1 < elements.length) { + Object value = elements[pathIndex + 1]; + if (value instanceof String) { + pathElement.setName((String) value); + } else if (value instanceof Long) { + pathElement.setId((Long) value); + } else if (value instanceof Integer) { + pathElement.setId((Integer) value); + } else if (value instanceof Short) { + pathElement.setId((Short) value); + } else { + throw new IllegalArgumentException( + "Expected string or integer, got: " + value.getClass()); + } + } + key.addPath(pathElement); + } + } + if (partitionId != null && !partitionId.equals(PartitionId.getDefaultInstance())) { + key.setPartitionId(partitionId); + } + return key; + } + + /** + * @return the double contained in value + * @throws IllegalArgumentException if the value does not contain a double. + */ + public static double getDouble(Value value) { + if (value.getValueTypeCase() != ValueTypeCase.DOUBLE_VALUE) { + throw new IllegalArgumentException("Value does not contain a double."); + } + return value.getDoubleValue(); + } + + /** + * @return the key contained in value + * @throws IllegalArgumentException if the value does not contain a key. + */ + public static Key getKey(Value value) { + if (value.getValueTypeCase() != ValueTypeCase.KEY_VALUE) { + throw new IllegalArgumentException("Value does not contain a key."); + } + return value.getKeyValue(); + } + + /** + * @return the blob contained in value + * @throws IllegalArgumentException if the value does not contain a blob. + */ + public static ByteString getByteString(Value value) { + if (value.getMeaning() == 18 && value.getValueTypeCase() == ValueTypeCase.STRING_VALUE) { + return value.getStringValueBytes(); + } else if (value.getValueTypeCase() == ValueTypeCase.BLOB_VALUE) { + return value.getBlobValue(); + } + throw new IllegalArgumentException("Value does not contain a blob."); + } + + /** + * @return the entity contained in value + * @throws IllegalArgumentException if the value does not contain an entity. + */ + public static Entity getEntity(Value value) { + if (value.getValueTypeCase() != ValueTypeCase.ENTITY_VALUE) { + throw new IllegalArgumentException("Value does not contain an Entity."); + } + return value.getEntityValue(); + } + + /** + * @return the string contained in value + * @throws IllegalArgumentException if the value does not contain a string. + */ + public static String getString(Value value) { + if (value.getValueTypeCase() != ValueTypeCase.STRING_VALUE) { + throw new IllegalArgumentException("Value does not contain a string."); + } + return value.getStringValue(); + } + + /** + * @return the boolean contained in value + * @throws IllegalArgumentException if the value does not contain a boolean. + */ + public static boolean getBoolean(Value value) { + if (value.getValueTypeCase() != ValueTypeCase.BOOLEAN_VALUE) { + throw new IllegalArgumentException("Value does not contain a boolean."); + } + return value.getBooleanValue(); + } + + /** + * @return the long contained in value + * @throws IllegalArgumentException if the value does not contain a long. + */ + public static long getLong(Value value) { + if (value.getValueTypeCase() != ValueTypeCase.INTEGER_VALUE) { + throw new IllegalArgumentException("Value does not contain an integer."); + } + return value.getIntegerValue(); + } + + /** + * @return the timestamp in microseconds contained in value + * @throws IllegalArgumentException if the value does not contain a timestamp. + */ + public static long getTimestamp(Value value) { + if (value.getMeaning() == 18 && value.getValueTypeCase() == ValueTypeCase.INTEGER_VALUE) { + return value.getIntegerValue(); + } else if (value.getValueTypeCase() == ValueTypeCase.TIMESTAMP_VALUE) { + return toMicroseconds(value.getTimestampValue()); + } + throw new IllegalArgumentException("Value does not contain a timestamp."); + } + + private static long toMicroseconds(TimestampOrBuilder timestamp) { + // Nanosecond precision is lost. + return timestamp.getSeconds() * MICROSECONDS_PER_SECOND + + timestamp.getNanos() / NANOSECONDS_PER_MICROSECOND; + } + + /** + * @return the array contained in value as a list. + * @throws IllegalArgumentException if the value does not contain an array. + */ + public static List getList(Value value) { + if (value.getValueTypeCase() != ValueTypeCase.ARRAY_VALUE) { + throw new IllegalArgumentException("Value does not contain an array."); + } + return value.getArrayValue().getValuesList(); + } + + /** + * Convert a timestamp value into a {@link Date} clipping off the microseconds. + * + * @param value a timestamp value to convert + * @return the resulting {@link Date} + * @throws IllegalArgumentException if the value does not contain a timestamp. + */ + public static Date toDate(Value value) { + return new Date(getTimestamp(value) / 1000); + } + + /** + * @param entity the entity to insert + * @return a mutation that will insert an entity + */ + public static Mutation.Builder makeInsert(Entity entity) { + return Mutation.newBuilder().setInsert(entity); + } + + /** + * @param entity the entity to update + * @return a mutation that will update an entity + */ + public static Mutation.Builder makeUpdate(Entity entity) { + return Mutation.newBuilder().setUpdate(entity); + } + + /** + * @param entity the entity to upsert + * @return a mutation that will upsert an entity + */ + public static Mutation.Builder makeUpsert(Entity entity) { + return Mutation.newBuilder().setUpsert(entity); + } + + /** + * @param key the key of the entity to delete + * @return a mutation that will delete an entity + */ + public static Mutation.Builder makeDelete(Key key) { + return Mutation.newBuilder().setDelete(key); + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreOptions.java new file mode 100644 index 000000000..fbcdf77a3 --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreOptions.java @@ -0,0 +1,188 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1beta3.client; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; + +import java.util.Arrays; +import java.util.List; + +/** + * An immutable object containing settings for the datastore. + * + *

Example for connecting to a datastore: + * + * 

+ * DatastoreOptions options = new DatastoreOptions.Builder()
+ *     .projectId("my-project-id")
+ *     .credential(DatastoreHelper.getComputeEngineCredential())
+ *     .build();
+ * DatastoreFactory.get().create(options);
+ *
+ * + *

The options should be passed to {@link DatastoreFactory#create}. + */ +public class DatastoreOptions { + private final String projectId; + private final String projectEndpoint; + private final String localHost; + + private final HttpRequestInitializer initializer; + + private final Credential credential; + private final HttpTransport transport; + public static final List SCOPES = Arrays.asList( + "https://www.googleapis.com/auth/datastore"); + + DatastoreOptions(Builder b) { + checkArgument(b.projectId != null || b.projectEndpoint != null, + "Either project ID or project endpoint must be provided."); + this.projectId = b.projectId; + this.projectEndpoint = b.projectEndpoint; + this.localHost = b.localHost; + this.initializer = b.initializer; + this.credential = b.credential; + this.transport = b.transport; + } + + /** + * Builder for {@link DatastoreOptions}. + */ + public static class Builder { + private static final String PROJECT_ENDPOINT_AND_PROJECT_ID_ERROR = + "Cannot set both project endpoint and project ID."; + private static final String PROJECT_ENDPOINT_AND_LOCAL_HOST_ERROR = + "Cannot set both project endpoint and local host."; + + private String projectId; + private String projectEndpoint; + private String localHost; + private HttpRequestInitializer initializer; + private Credential credential; + private HttpTransport transport; + + public Builder() { } + + public Builder(DatastoreOptions options) { + this.projectId = options.projectId; + this.projectEndpoint = options.projectEndpoint; + this.localHost = options.localHost; + this.initializer = options.initializer; + this.credential = options.credential; + this.transport = options.transport; + } + + public DatastoreOptions build() { + return new DatastoreOptions(this); + } + + /** + * Sets the project ID used to access Datastore. + */ + public Builder projectId(String projectId) { + checkArgument(projectEndpoint == null, PROJECT_ENDPOINT_AND_PROJECT_ID_ERROR); + this.projectId = projectId; + return this; + } + + /** + * Sets the host used to access Datastore. + */ + public Builder localHost(String localHost) { + checkArgument(projectEndpoint == null, PROJECT_ENDPOINT_AND_LOCAL_HOST_ERROR); + DatastoreFactory.validateUrl(localHost); + if (includesScheme(localHost)) { + throw new IllegalArgumentException( + String.format("Local host \"%s\" must not include scheme.", localHost)); + } + this.localHost = localHost; + return this; + } + + /** + * Sets the project endpoint used to access Datastore. Prefer using {@link #projectId} + * and/or {@link #localHost} when possible. + */ + public Builder projectEndpoint(String projectEndpoint) { + checkArgument(projectId == null, PROJECT_ENDPOINT_AND_PROJECT_ID_ERROR); + checkArgument(localHost == null, PROJECT_ENDPOINT_AND_LOCAL_HOST_ERROR); + DatastoreFactory.validateUrl(projectEndpoint); + if (!includesScheme(projectEndpoint)) { + throw new IllegalArgumentException(String.format( + "Project endpoint \"%s\" must include scheme.", projectEndpoint)); + } + this.projectEndpoint = projectEndpoint; + return this; + } + + /** + * Sets the (optional) initializer to run on HTTP requests to Datastore. + */ + public Builder initializer(HttpRequestInitializer initializer) { + this.initializer = initializer; + return this; + } + + /** + * Sets the Google APIs {@link Credential} used to access Datastore. + */ + public Builder credential(Credential credential) { + this.credential = credential; + return this; + } + + /** + * Sets the transport used to access Datastore. + */ + public Builder transport(HttpTransport transport) { + this.transport = transport; + return this; + } + + private static boolean includesScheme(String url) { + return url.startsWith("http://") || url.startsWith("https://"); + } + } + + public String getProjectId() { + return projectId; + } + + public String getProjectEndpoint() { + return projectEndpoint; + } + + public String getLocalHost() { + return localHost; + } + + public HttpRequestInitializer getInitializer() { + return initializer; + } + + public Credential getCredential() { + return credential; + } + + public HttpTransport getTransport() { + return transport; + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastore.java similarity index 86% rename from datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastore.java index 39f81665d..0a0d7a045 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastore.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastore.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Google Inc. All Rights Reserved. + * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,15 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.api.services.datastore.client; - -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; +package com.google.datastore.v1beta3.client; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.UrlEncodedContent; +import com.google.api.client.util.Preconditions; import java.io.BufferedReader; import java.io.File; @@ -51,11 +49,11 @@ * {@literal @}BeforeClass * public static void startLocalDatastore() throws LocalDevelopmentDatastoreException { * DatastoreOptions opts = new DatastoreOptions.Builder() - * .host("http://localhost:8080") - * .dataset("myapp") + * .localHost("localhost:8080") + * .projectId("my-project-id") * .build(); * datastore = LocalDevelopmentDatastoreFactory.get().create(opts); - * datastore.start("/usr/local/gcd-sdk", "myapp"); + * datastore.start("/usr/local/gcd-sdk", "my-project-id"); * } * * {@literal @}Before @@ -76,7 +74,6 @@ * * } * - * */ public class LocalDevelopmentDatastore extends Datastore { private static final int STARTUP_TIMEOUT_SECS = 30; @@ -89,12 +86,12 @@ enum State {NEW, STARTED, STOPPED} private volatile State state = State.NEW; - private File datasetDirectory; + private File projectDirectory; - LocalDevelopmentDatastore(RemoteRpc rpc, String host, + LocalDevelopmentDatastore(RemoteRpc rpc, String localHost, LocalDevelopmentDatastoreOptions localDevelopmentOptions) { super(rpc); - this.host = host; + this.host = "http://" + localHost; this.localDevelopmentOptions = localDevelopmentOptions; } @@ -127,18 +124,18 @@ public void clear() throws LocalDevelopmentDatastoreException { * {@link #stop} to ensure the server is not running regardless of the result of this method. * * @param sdkPath The path to the GCD SDK, eg /usr/local/dev/gcd - * @param dataset The name of the GCD dataset + * @param projectId The GCD project ID * @param cmdLineOptions Command line options to pass to the script that launches the dev server * @throws LocalDevelopmentDatastoreException If {@link #start} has already been called or the * server does not start successfully. */ - public synchronized void start(String sdkPath, String dataset, String... cmdLineOptions) + public synchronized void start(String sdkPath, String projectId, String... cmdLineOptions) throws LocalDevelopmentDatastoreException { - checkNotNull(sdkPath, "sdkPath cannot be null"); - checkNotNull(dataset, "dataset cannot be null"); - checkState(state == State.NEW, "Cannot call start() more than once."); + Preconditions.checkNotNull(sdkPath, "sdkPath cannot be null"); + Preconditions.checkNotNull(projectId, "projectId cannot be null"); + Preconditions.checkState(state == State.NEW, "Cannot call start() more than once."); try { - startDatastoreInternal(sdkPath, dataset, cmdLineOptions); + startDatastoreInternal(sdkPath, projectId, cmdLineOptions); state = State.STARTED; } finally { if (state != State.STARTED) { @@ -149,13 +146,13 @@ public synchronized void start(String sdkPath, String dataset, String... cmdLine } } - void startDatastoreInternal(String sdkPath, String dataset, String... cmdLineOptions) + void startDatastoreInternal(String sdkPath, String projectId, String... cmdLineOptions) throws LocalDevelopmentDatastoreException { - File datasetDirectory = createDatastore(sdkPath, dataset); + File projectDirectory = createProjectDirectory(sdkPath, projectId); List cmd = new ArrayList( Arrays.asList("./gcd.sh", "start", "--allow_remote_shutdown", "--store_on_disk=false")); cmd.addAll(Arrays.asList(cmdLineOptions)); - cmd.add(datasetDirectory.getPath()); + cmd.add(projectDirectory.getPath()); final Process gcdStartProcess; try { gcdStartProcess = newGcdProcess(sdkPath, cmd).start(); @@ -184,27 +181,27 @@ public void run() { } } - private File createDatastore(String sdkPath, String dataset) + private File createProjectDirectory(String sdkPath, String projectId) throws LocalDevelopmentDatastoreException { try { - datasetDirectory = Files.createTempDirectory("local-development-datastore").toFile(); + projectDirectory = Files.createTempDirectory("local-development-datastore").toFile(); } catch (IOException e) { - throw new LocalDevelopmentDatastoreException("Could not create dataset tmp directory", e); + throw new LocalDevelopmentDatastoreException("Could not create project tmp directory", e); } - List cmd = Arrays.asList("./gcd.sh", "create", "--project_id=" + dataset, - datasetDirectory.getPath()); + List cmd = Arrays.asList("./gcd.sh", "create", "--project_id=" + projectId, + projectDirectory.getPath()); try { int retCode = newGcdProcess(sdkPath, cmd).start().waitFor(); if (retCode != 0) { throw new LocalDevelopmentDatastoreException( - String.format("Could not create dataset (retcode=%d)", retCode)); + String.format("Could not create project (retcode=%d)", retCode)); } } catch (IOException e) { - throw new LocalDevelopmentDatastoreException("Could not create dataset", e); + throw new LocalDevelopmentDatastoreException("Could not create project", e); } catch (InterruptedException e) { throw new LocalDevelopmentDatastoreException("Received an interrupt", e); } - return datasetDirectory; + return projectDirectory; } private ProcessBuilder newGcdProcess(String sdkPath, List cmd) { @@ -215,11 +212,6 @@ private ProcessBuilder newGcdProcess(String sdkPath, List cmd) { return builder; } - public File getDatasetDirectory() { - checkState(state == State.STARTED); - return datasetDirectory; - } - /** * Stops the local datastore. Multiple calls are allowed. * @@ -231,15 +223,15 @@ public synchronized void stop() throws LocalDevelopmentDatastoreException { stopDatastoreInternal(); if (state != State.STOPPED) { state = State.STOPPED; - if (datasetDirectory != null) { + if (projectDirectory != null) { try { Process process = - new ProcessBuilder("rm", "-r", datasetDirectory.getAbsolutePath()).start(); + new ProcessBuilder("rm", "-r", projectDirectory.getAbsolutePath()).start(); if (process.waitFor() != 0) { throw new IOException("Temp directory delete exited with " + process.exitValue()); } } catch (IOException | InterruptedException e) { - throw new IllegalStateException("Dataset directory wipe failed.", e); + throw new IllegalStateException("Project directory wipe failed.", e); } } } diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreException.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreException.java similarity index 89% rename from datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreException.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreException.java index deaec1957..258dcd853 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreException.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreException.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Google Inc. All Rights Reserved. + * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.api.services.datastore.client; +package com.google.datastore.v1beta3.client; /** * An exception related to the local development {@link Datastore}. - * */ public class LocalDevelopmentDatastoreException extends Exception { public LocalDevelopmentDatastoreException(String message) { diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreFactory.java similarity index 89% rename from datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreFactory.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreFactory.java index 824f6b604..37302ab5f 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Google Inc. All Rights Reserved. + * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,12 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.api.services.datastore.client; - +package com.google.datastore.v1beta3.client; /** * Factory for {@link LocalDevelopmentDatastore}. - * */ public class LocalDevelopmentDatastoreFactory extends DatastoreFactory { @@ -41,7 +39,7 @@ public LocalDevelopmentDatastore create(DatastoreOptions options) public LocalDevelopmentDatastore create(DatastoreOptions options, LocalDevelopmentDatastoreOptions localDevelopmentOptions) { RemoteRpc rpc = newRemoteRpc(options); - return new LocalDevelopmentDatastore(rpc, options.getHost(), + return new LocalDevelopmentDatastore(rpc, options.getLocalHost(), localDevelopmentOptions); } } diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreOptions.java similarity index 93% rename from datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreOptions.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreOptions.java index b59f8734e..8fd519ed2 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/LocalDevelopmentDatastoreOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreOptions.java @@ -1,5 +1,5 @@ /* - * Copyright 2014 Google Inc. All Rights Reserved. + * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.api.services.datastore.client; +package com.google.datastore.v1beta3.client; import java.util.HashMap; import java.util.Map; diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/QuerySplitter.java similarity index 55% rename from datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/QuerySplitter.java index 528349cc9..190cf783a 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitter.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/QuerySplitter.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Google Inc. All Rights Reserved. + * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,16 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.api.services.datastore.client; +package com.google.datastore.v1beta3.client; -import com.google.api.services.datastore.DatastoreV1.PartitionId; -import com.google.api.services.datastore.DatastoreV1.Query; +import com.google.datastore.v1beta3.PartitionId; +import com.google.datastore.v1beta3.Query; import java.util.List; /** * Provides the ability to split a query into multiple shards. - * */ public interface QuerySplitter { @@ -35,26 +34,7 @@ public interface QuerySplitter { * number of results for the query is too small. * * @param query the query to split. - * @param numSplits the desired number of splits. - * @param datastore the datastore to run on. - * @throws DatastoreException if there was a datastore error while generating query splits. - * @throws IllegalArgumentException if the given query or numSplits was invalid. - * @deprecated Use {@link getSplits(Query, PartitionId, int, Datastore)} instead, which provides - * the ability to supply a namespace. - */ - @Deprecated - List getSplits(Query query, int numSplits, Datastore datastore) throws DatastoreException; - - /** - * Returns a list of sharded {@link Query}s for the given query. - * - *

This will create up to the desired number of splits, however it may return less splits if - * the desired number of splits is unavailable. This will happen if the number of split points - * provided by the underlying Datastore is less than the desired number, which will occur if the - * number of results for the query is too small. - * - * @param query the query to split. - * @param partition the partition to run in. + * @param partition the partition the query is running in. * @param numSplits the desired number of splits. * @param datastore the datastore to run on. * @throws DatastoreException if there was a datastore error while generating query splits. diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/QuerySplitterImpl.java similarity index 79% rename from datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/QuerySplitterImpl.java index c351981a0..dab377148 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/QuerySplitterImpl.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/QuerySplitterImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Google Inc. All Rights Reserved. + * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,23 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.api.services.datastore.client; +package com.google.datastore.v1beta3.client; -import static com.google.api.services.datastore.client.DatastoreHelper.makeFilter; +import static com.google.datastore.v1beta3.client.DatastoreHelper.makeAndFilter; -import com.google.api.services.datastore.DatastoreV1.EntityResult; -import com.google.api.services.datastore.DatastoreV1.Filter; -import com.google.api.services.datastore.DatastoreV1.Key; -import com.google.api.services.datastore.DatastoreV1.PartitionId; -import com.google.api.services.datastore.DatastoreV1.PropertyExpression; -import com.google.api.services.datastore.DatastoreV1.PropertyFilter; -import com.google.api.services.datastore.DatastoreV1.PropertyFilter.Operator; -import com.google.api.services.datastore.DatastoreV1.PropertyOrder.Direction; -import com.google.api.services.datastore.DatastoreV1.PropertyReference; -import com.google.api.services.datastore.DatastoreV1.Query; -import com.google.api.services.datastore.DatastoreV1.QueryResultBatch; -import com.google.api.services.datastore.DatastoreV1.QueryResultBatch.MoreResultsType; -import com.google.api.services.datastore.DatastoreV1.RunQueryRequest; +import com.google.datastore.v1beta3.EntityResult; +import com.google.datastore.v1beta3.Filter; +import com.google.datastore.v1beta3.Key; +import com.google.datastore.v1beta3.PartitionId; +import com.google.datastore.v1beta3.Projection; +import com.google.datastore.v1beta3.PropertyFilter; +import com.google.datastore.v1beta3.PropertyFilter.Operator; +import com.google.datastore.v1beta3.PropertyOrder.Direction; +import com.google.datastore.v1beta3.PropertyReference; +import com.google.datastore.v1beta3.Query; +import com.google.datastore.v1beta3.QueryResultBatch; +import com.google.datastore.v1beta3.QueryResultBatch.MoreResultsType; +import com.google.datastore.v1beta3.RunQueryRequest; import java.util.ArrayList; import java.util.Collections; @@ -41,8 +41,7 @@ * *

This implementation of the QuerySplitter uses the __scatter__ property to gather * random split points for a query. - * - **/ + */ final class QuerySplitterImpl implements QuerySplitter { /** The number of keys to sample for each split. **/ @@ -57,17 +56,11 @@ private QuerySplitterImpl() { // No initialization required. } - @Override - @Deprecated - public List getSplits(Query query, int numSplits, Datastore datastore) - throws DatastoreException, IllegalArgumentException { - return getSplits(query, PartitionId.newBuilder().build(), numSplits, datastore); - } - @Override public List getSplits( Query query, PartitionId partition, int numSplits, Datastore datastore) throws DatastoreException, IllegalArgumentException { + validateQuery(query); validateSplitSize(numSplits); @@ -100,14 +93,20 @@ private void validateSplitSize(int numSplits) throws IllegalArgumentException { * inefficient sharding. */ private void validateFilter(Filter filter) throws IllegalArgumentException { - if (filter.hasCompositeFilter()) { - for (Filter subFilter : filter.getCompositeFilter().getFilterList()) { - validateFilter(subFilter); - } - } else if (filter.hasPropertyFilter()) { - if (UNSUPPORTED_OPERATORS.contains(filter.getPropertyFilter().getOperator())) { - throw new IllegalArgumentException("Query cannot have any inequality filters."); - } + switch (filter.getFilterTypeCase()) { + case COMPOSITE_FILTER: + for (Filter subFilter : filter.getCompositeFilter().getFiltersList()) { + validateFilter(subFilter); + } + break; + case PROPERTY_FILTER: + if (UNSUPPORTED_OPERATORS.contains(filter.getPropertyFilter().getOp())) { + throw new IllegalArgumentException("Query cannot have any inequality filters."); + } + break; + default: + throw new IllegalArgumentException( + "Unsupported filter type: " + filter.getFilterTypeCase()); } } @@ -156,7 +155,7 @@ private Query createSplit(Key lastKey, Key nextKey, Query query) { DatastoreHelper.makeValue(nextKey)).build(); keyFilters.add(upperBound); } - return Query.newBuilder(query).setFilter(makeFilter(keyFilters)).build(); + return Query.newBuilder(query).setFilter(makeAndFilter(keyFilters)).build(); } /** @@ -187,11 +186,12 @@ private List getScatterKeys( .setQuery(scatterPointQuery) .build(); batch = datastore.runQuery(scatterRequest).getBatch(); - for (EntityResult result : batch.getEntityResultList()) { + for (EntityResult result : batch.getEntityResultsList()) { keySplits.add(result.getEntity().getKey()); } scatterPointQuery.setStartCursor(batch.getEndCursor()); - scatterPointQuery.setLimit(scatterPointQuery.getLimit() - batch.getEntityResultCount()); + scatterPointQuery.getLimitBuilder().setValue( + scatterPointQuery.getLimit().getValue() - batch.getEntityResultsCount()); } while (batch.getMoreResults() == MoreResultsType.NOT_FINISHED); Collections.sort(keySplits, DatastoreHelper.getKeyComparator()); return keySplits; @@ -204,7 +204,7 @@ private List getScatterKeys( * @param numSplits the number of splits to create. */ private Query.Builder createScatterQuery(Query query, int numSplits) { - // TODO(user): We can potentially support better splits with equality filters in our query + // TODO(pcostello): We can potentially support better splits with equality filters in our query // if there exists a composite index on property, __scatter__, __key__. Until an API for // metadata exists, this isn't possible. Note that ancestor and inequality queries fall into // the same category. @@ -217,8 +217,8 @@ private Query.Builder createScatterQuery(Query query, int numSplits) { // If we represent each split as a region before a scatter entity, there is an extra region // following the last scatter point. Thus, we do not need the scatter entities for the last // region. - scatterPointQuery.setLimit((numSplits - 1) * KEYS_PER_SPLIT); - scatterPointQuery.addProjection(PropertyExpression.newBuilder().setProperty( + scatterPointQuery.getLimitBuilder().setValue((numSplits - 1) * KEYS_PER_SPLIT); + scatterPointQuery.addProjection(Projection.newBuilder().setProperty( PropertyReference.newBuilder().setName("__key__"))); return scatterPointQuery; } diff --git a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/RemoteRpc.java similarity index 53% rename from datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/RemoteRpc.java index 80c6d5c81..bf7918706 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/api/services/datastore/client/RemoteRpc.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/RemoteRpc.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Google Inc. All Rights Reserved. + * Copyright 2015 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,30 +13,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.api.services.datastore.client; +package com.google.datastore.v1beta3.client; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; -import com.google.api.client.http.HttpResponseException; -import com.google.api.client.http.HttpStatusCodes; import com.google.api.client.http.protobuf.ProtoHttpContent; +import com.google.api.client.util.IOUtils; import com.google.protobuf.MessageLite; +import com.google.rpc.Code; +import com.google.rpc.Status; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.Charset; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; /** * An RPC transport that sends protocol buffers over HTTP. - * */ class RemoteRpc { private static final Logger logger = Logger.getLogger(RemoteRpc.class.getName()); + private static final String API_FORMAT_VERSION_HEADER = "X-Goog-Api-Format-Version"; + private static final String API_FORMAT_VERSION = "2"; + private final HttpRequestFactory client; private final HttpRequestInitializer initializer; private final String url; @@ -72,16 +77,22 @@ public InputStream call(String methodName, MessageLite request) throws Datastore rpcCount.incrementAndGet(); ProtoHttpContent payload = new ProtoHttpContent(request); HttpRequest httpRequest = client.buildPostRequest(resolveURL(methodName), payload); + httpRequest.getHeaders().put(API_FORMAT_VERSION_HEADER, API_FORMAT_VERSION); + // Don't throw an HTTPResponseException on error. It converts the response to a String and + // throws away the original, whereas we need the raw bytes to parse it as a proto. + httpRequest.setThrowExceptionOnExecuteError(false); if (initializer != null) { initializer.initialize(httpRequest); } httpResponse = httpRequest.execute(); + if (!httpResponse.isSuccessStatusCode()) { + throw makeException(url, methodName, httpResponse.getContent(), + httpResponse.getContentType(), httpResponse.getContentCharset(), null, + httpResponse.getStatusCode()); + } return httpResponse.getContent(); - } catch (HttpResponseException e) { - throw makeException(url, methodName, e.getStatusCode(), e.getContent(), e); } catch (IOException e) { - throw makeException(url, methodName, HttpStatusCodes.STATUS_CODE_SERVICE_UNAVAILABLE, - "I/O error", e); + throw makeException(url, methodName, Code.UNAVAILABLE, "I/O error", e); } } finally { long elapsedTime = System.currentTimeMillis() - startTime; @@ -102,16 +113,59 @@ public String getUrl() { } GenericUrl resolveURL(String path) { - return new GenericUrl(url + "/" + path); + return new GenericUrl(url + ":" + path); } HttpRequestFactory getHttpRequestFactory() { return client; } - - public static DatastoreException makeException( - String url, String methodName, int code, String message, Throwable cause) { + + public static DatastoreException makeException(String url, String methodName, Code code, + String message, Throwable cause) { logger.fine("remote datastore call " + methodName + " against " + url + " failed: " + message); return new DatastoreException(methodName, code, message, cause); } + + static DatastoreException makeException(String url, String methodName, InputStream content, + String contentType, Charset contentCharset, Throwable cause, int httpStatusCode) { + if (!contentType.equals("application/x-protobuf")) { + String responseContent; + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copy(content, out); + responseContent = out.toString(contentCharset.name()); + } catch (IOException e) { + responseContent = ""; + } + return makeException(url, methodName, Code.INTERNAL, + String.format( + "Non-protobuf error: %s. HTTP status code was %d.", responseContent, httpStatusCode), + cause); + } + + Status rpcStatus; + try { + rpcStatus = Status.parseFrom(content); + } catch (IOException e) { + return makeException(url, methodName, Code.INTERNAL, + String.format( + "Unable to parse Status protocol buffer: HTTP status code was %s.", httpStatusCode), + e); + } + + Code code = Code.forNumber(rpcStatus.getCode()); + if (code == null) { + return makeException(url, methodName, Code.INTERNAL, + String.format( + "Invalid error code: %d. Message: %s.", rpcStatus.getCode(), rpcStatus.getMessage()), + cause); + } else if (code == Code.OK) { + return makeException(url, methodName, Code.INTERNAL, + String.format("Unexpected OK error code with HTTP status code of %d. Message: %s.", + httpStatusCode, rpcStatus.getMessage()), + cause); + } + + return makeException(url, methodName, code, rpcStatus.getMessage(), cause); + } } diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreFactoryTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreFactoryTest.java new file mode 100644 index 000000000..3a76d6110 --- /dev/null +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreFactoryTest.java @@ -0,0 +1,107 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1beta3.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.javanet.NetHttpTransport; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Test for {@link DatastoreFactory}. + */ +@RunWith(JUnit4.class) +public class DatastoreFactoryTest { + private static final String PROJECT_ID = "project-id"; + + private DatastoreFactory factory = DatastoreFactory.get(); + + /** + * Without specifying a credential or transport, the factory will create + * a default transport on its own. + */ + @Test + public void makeClient_Default() { + DatastoreOptions options = new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .build(); + HttpRequestFactory f = factory.makeClient(options); + assertNotNull(f.getTransport()); + assertTrue(f.getTransport() instanceof NetHttpTransport); + } + + /** + * Specifying a credential, but not a transport, the factory will use the + * transport from the credential. + */ + @Test + public void makeClient_WithCredential() { + NetHttpTransport transport = new NetHttpTransport(); + GoogleCredential credential = new GoogleCredential.Builder() + .setTransport(transport) + .build(); + DatastoreOptions options = new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .credential(credential) + .build(); + HttpRequestFactory f = factory.makeClient(options); + assertEquals(transport, f.getTransport()); + } + + /** + * Specifying a transport, but not a credential, the factory will use the + * transport specified. + */ + @Test + public void makeClient_WithTransport() { + NetHttpTransport transport = new NetHttpTransport(); + DatastoreOptions options = new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .transport(transport) + .build(); + HttpRequestFactory f = factory.makeClient(options); + assertEquals(transport, f.getTransport()); + } + + /** + * Specifying both credential and transport, the factory will use the + * transport specified and not the one in the credential. + */ + @Test + public void makeClient_WithCredentialTransport() { + NetHttpTransport credTransport = new NetHttpTransport(); + NetHttpTransport transport = new NetHttpTransport(); + GoogleCredential credential = new GoogleCredential.Builder() + .setTransport(credTransport) + .build(); + DatastoreOptions options = new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .credential(credential) + .transport(transport) + .build(); + HttpRequestFactory f = factory.makeClient(options); + assertNotSame(credTransport, f.getTransport()); + assertEquals(transport, f.getTransport()); + } +} diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreHelperTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreHelperTest.java new file mode 100644 index 000000000..11ec8aeed --- /dev/null +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreHelperTest.java @@ -0,0 +1,355 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1beta3.client; + +import static com.google.datastore.v1beta3.client.DatastoreHelper.getByteString; +import static com.google.datastore.v1beta3.client.DatastoreHelper.makeKey; +import static com.google.datastore.v1beta3.client.DatastoreHelper.makeValue; +import static com.google.datastore.v1beta3.client.DatastoreHelper.toDate; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import com.google.datastore.v1beta3.Key; +import com.google.datastore.v1beta3.PartitionId; +import com.google.datastore.v1beta3.Value; +import com.google.datastore.v1beta3.Value.ValueTypeCase; +import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Date; + +/** + * Tests for {@link DatastoreHelper}. + */ +@RunWith(JUnit4.class) +public class DatastoreHelperTest { + + private static final Key PARENT = Key.newBuilder() + .addPath(Key.PathElement.newBuilder() + .setKind("Parent") + .setId(23L)) + .build(); + private static final Key GRANDPARENT = Key.newBuilder() + .addPath(Key.PathElement.newBuilder() + .setKind("Grandparent") + .setId(24L)) + .build(); + private static final Key CHILD = Key.newBuilder() + .addPath(Key.PathElement.newBuilder() + .setKind("Child") + .setId(26L)) + .build(); + + @Test + public void testMakeKey_BadTypeForKind() { + try { + DatastoreHelper.makeKey(new Object()); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testMakeKey_BadTypeForNameId() { + try { + DatastoreHelper.makeKey("kind", new Object()); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testMakeKey_Empty() { + assertEquals(Key.newBuilder().build(), DatastoreHelper.makeKey().build()); + } + + @Test + public void testMakeKey_Incomplete() { + assertEquals( + Key.newBuilder() + .addPath(Key.PathElement.newBuilder().setKind("Foo")) + .build(), + makeKey("Foo").build()); + } + + @Test + public void testMakeKey_IdInt() { + assertEquals( + Key.newBuilder() + .addPath(Key.PathElement.newBuilder().setKind("Foo").setId(1)) + .build(), + makeKey("Foo", 1).build()); + } + + @Test + public void testMakeKey_IdLong() { + assertEquals( + Key.newBuilder() + .addPath(Key.PathElement.newBuilder().setKind("Foo").setId(1)) + .build(), + makeKey("Foo", 1L).build()); + } + + @Test + public void testMakeKey_IdShort() { + assertEquals( + Key.newBuilder() + .addPath(Key.PathElement.newBuilder().setKind("Foo").setId(1)) + .build(), + makeKey("Foo", (short) 1).build()); + } + + @Test + public void testMakeKey_Name() { + assertEquals( + Key.newBuilder() + .addPath(Key.PathElement.newBuilder().setKind("Foo").setName("hi")) + .build(), + makeKey("Foo", "hi").build()); + } + + @Test + public void testMakeKey_KindNameKind() { + assertEquals( + Key.newBuilder() + .addPath(Key.PathElement.newBuilder().setKind("Foo").setName("hi")) + .addPath(Key.PathElement.newBuilder().setKind("Bar")) + .build(), + makeKey("Foo", "hi", "Bar").build()); + } + + @Test + public void testMakeKey_KeyKind() { + // 1 key at the beginning of the series + assertEquals( + Key.newBuilder() + .addPath(PARENT.getPath(0)) + .addPath(Key.PathElement.newBuilder().setKind("Child")) + .build(), + makeKey(PARENT, "Child").build()); + } + + @Test + public void testMakeKey_KindIdKeyKind() { + // 1 key in the middle of the series + assertEquals( + Key.newBuilder() + .addPath(Key.PathElement.newBuilder().setKind("Grandparent").setId(24L)) + .addPath(PARENT.getPath(0)) + .addPath(Key.PathElement.newBuilder().setKind("Child")) + .build(), + makeKey("Grandparent", 24L, PARENT, "Child").build()); + } + + @Test + public void testMakeKey_KindIdKey() { + // 1 key at the end of the series + assertEquals( + Key.newBuilder() + .addPath(Key.PathElement.newBuilder().setKind("Grandparent").setId(24L)) + .addPath(PARENT.getPath(0)) + .build(), + makeKey("Grandparent", 24L, PARENT).build()); + } + + @Test + public void testMakeKey_KeyKindIdKey() { + // 1 key at the beginning and 1 key at the end of the series + assertEquals( + Key.newBuilder() + .addPath(GRANDPARENT.getPath(0)) + .addPath(Key.PathElement.newBuilder().setKind("Parent").setId(23L)) + .addPath(CHILD.getPath(0)) + .build(), + makeKey(GRANDPARENT, "Parent", 23, CHILD).build()); + } + + @Test + public void testMakeKey_Key() { + // Just 1 key + assertEquals( + Key.newBuilder() + .addPath(CHILD.getPath(0)) + .build(), + makeKey(CHILD).build()); + } + + @Test + public void testMakeKey_KeyKey() { + // Just 2 keys + assertEquals( + Key.newBuilder() + .addPath(PARENT.getPath(0)) + .addPath(CHILD.getPath(0)) + .build(), + makeKey(PARENT, CHILD).build()); + } + + @Test + public void testMakeKey_KeyKeyKey() { + // Just 3 keys + assertEquals( + Key.newBuilder() + .addPath(GRANDPARENT.getPath(0)) + .addPath(PARENT.getPath(0)) + .addPath(CHILD.getPath(0)) + .build(), + makeKey(GRANDPARENT, PARENT, CHILD).build()); + } + + @Test + public void testMakeKey_KeyMultiLevelKey() { + // 1 key with 2 elements + assertEquals( + Key.newBuilder() + .addPath(GRANDPARENT.getPath(0)) + .addPath(PARENT.getPath(0)) + .addPath(CHILD.getPath(0)) + .build(), + makeKey(GRANDPARENT, makeKey(PARENT, CHILD).build()).build()); + } + + @Test + public void testMakeKey_MultiLevelKeyKey() { + // 1 key with 2 elements + assertEquals( + Key.newBuilder() + .addPath(GRANDPARENT.getPath(0)) + .addPath(PARENT.getPath(0)) + .addPath(CHILD.getPath(0)) + .build(), + makeKey(makeKey(GRANDPARENT, PARENT).build(), CHILD).build()); + } + + @Test + public void testMakeKey_MultiLevelKey() { + // 1 key with 3 elements + assertEquals( + Key.newBuilder() + .addPath(GRANDPARENT.getPath(0)) + .addPath(PARENT.getPath(0)) + .addPath(CHILD.getPath(0)) + .build(), + makeKey(makeKey(GRANDPARENT, PARENT, CHILD).build()).build()); + } + + @Test + public void testMakeKey_PartitionId() { + PartitionId partitionId = PartitionId.newBuilder() + .setNamespaceId("namespace-id") + .build(); + Key parent = PARENT.toBuilder() + .setPartitionId(partitionId) + .build(); + assertEquals( + Key.newBuilder() + .setPartitionId(partitionId) + .addPath(PARENT.getPath(0)) + .addPath(Key.PathElement.newBuilder().setKind("Child")) + .build(), + makeKey(parent, "Child").build()); + } + + @Test + public void testMakeKey_NonMatchingPartitionId2() { + PartitionId partitionId1 = PartitionId.newBuilder() + .setNamespaceId("namespace-id") + .build(); + PartitionId partitionId2 = PartitionId.newBuilder() + .setNamespaceId("another-namespace-id") + .build(); + try { + makeKey( + PARENT.toBuilder().setPartitionId(partitionId1).build(), + CHILD.toBuilder().setPartitionId(partitionId2).build()); + fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testMakeTimestampValue() throws Exception { + // Test cases with nanos == 0. + assertConversion(-50_000, -50, 0); + assertConversion(-1_000, -1, 0); + assertConversion(0, 0, 0); + assertConversion(1_000, 1, 0); + assertConversion(2_000, 2, 0); + assertConversion(100_000, 100, 0); + + // Test cases with nanos % 1_000_000 == 0 (no loss of precision). + assertConversion(2, 0, 2_000_000); + assertConversion(1_003, 1, 3_000_000); + assertConversion(2_005, 2, 5_000_000); + + // Timestamp specification requires that nanos >= 0 even if the timestamp + // is before the epoch. + assertConversion(0, 0, 0); + assertConversion(-250, -1, 750_000_000); // 1/4 second before epoch + assertConversion(-500, -1, 500_000_000); // 1/2 second before epoch + assertConversion(-750, -1, 250_000_000); // 3/4 second before epoch + + // If nanos % 1_000_000 != 0, precision is lost (via truncation) when + // converting to milliseconds. + assertTimestampToMilliseconds(3_100, 3, 100_000_999); + assertMillisecondsToTimestamp(3_100, 3, 100_000_000); + assertTimestampToMilliseconds(5_999, 5, 999_999_999); + assertMillisecondsToTimestamp(5_999, 5, 999_000_000); + assertTimestampToMilliseconds(7_100, 7, 100_000_001); + assertMillisecondsToTimestamp(7_100, 7, 100_000_000); + } + + private void assertConversion(long millis, long seconds, int nanos) { + assertMillisecondsToTimestamp(millis, seconds, nanos); + assertTimestampToMilliseconds(millis, seconds, nanos); + } + + private void assertMillisecondsToTimestamp(long millis, long seconds, long nanos) { + Value timestampValue = makeValue(new Date(millis)).build(); + assertEquals(ValueTypeCase.TIMESTAMP_VALUE, timestampValue.getValueTypeCase()); + assertEquals(seconds, timestampValue.getTimestampValue().getSeconds()); + assertEquals(nanos, timestampValue.getTimestampValue().getNanos()); + } + + private void assertTimestampToMilliseconds(long millis, long seconds, int nanos) { + Value.Builder value = Value.newBuilder().setTimestampValue(Timestamp.newBuilder() + .setSeconds(seconds) + .setNanos(nanos)); + assertEquals(millis, DatastoreHelper.toDate(value.build()).getTime()); + } + + @Test + public void testProjectionHandling() { + assertEquals(ByteString.copyFromUtf8("hi"), + getByteString(makeValue("hi").setMeaning(18).build())); + try { + getByteString(makeValue("hi").build()); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + } + + assertEquals(new Date(1), toDate(makeValue(1000).setMeaning(18).build())); + try { + toDate(makeValue(1000).build()); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + } + } +} diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java new file mode 100644 index 000000000..00ce00256 --- /dev/null +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java @@ -0,0 +1,362 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1beta3.client; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpStatusCodes; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.api.client.testing.util.TestableByteArrayInputStream; +import com.google.common.collect.Iterables; +import com.google.datastore.v1beta3.AllocateIdsRequest; +import com.google.datastore.v1beta3.AllocateIdsResponse; +import com.google.datastore.v1beta3.BeginTransactionRequest; +import com.google.datastore.v1beta3.BeginTransactionResponse; +import com.google.datastore.v1beta3.CommitRequest; +import com.google.datastore.v1beta3.CommitResponse; +import com.google.datastore.v1beta3.EntityResult; +import com.google.datastore.v1beta3.LookupRequest; +import com.google.datastore.v1beta3.LookupResponse; +import com.google.datastore.v1beta3.QueryResultBatch; +import com.google.datastore.v1beta3.RollbackRequest; +import com.google.datastore.v1beta3.RollbackResponse; +import com.google.datastore.v1beta3.RunQueryRequest; +import com.google.datastore.v1beta3.RunQueryResponse; +import com.google.protobuf.ByteString; +import com.google.protobuf.Message; +import com.google.rpc.Code; +import com.google.rpc.Status; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.List; + +/** + * Tests for {@link DatastoreFactory} and {@link Datastore}. + */ +@RunWith(JUnit4.class) +public class DatastoreTest { + private static final String PROJECT_ID = "project-id"; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private DatastoreFactory factory = new MockDatastoreFactory(); + private DatastoreOptions.Builder options = new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .credential(new MockCredential()); + + @Test + public void options_NoProjectIdOrProjectEndpoint() throws Exception { + options = new DatastoreOptions.Builder(); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Either project ID or project endpoint must be provided"); + options.build(); + } + + @Test + public void options_ProjectIdAndProjectEndpoint() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Cannot set both project endpoint and project ID"); + options = new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .projectEndpoint("http://prom-qa/datastore/v1beta42/projects/project-id"); + } + + @Test + public void options_LocalHostAndProjectEndpoint() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Cannot set both project endpoint and local host"); + options = new DatastoreOptions.Builder() + .localHost("localhost:8080") + .projectEndpoint("http://prom-qa/datastore/v1beta42/projects/project-id"); + } + + @Test + public void options_InvalidLocalHost() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Illegal character"); + new DatastoreOptions.Builder() + .localHost("!not a valid url!"); + } + + @Test + public void options_SchemeInLocalHost() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Local host \"http://localhost:8080\" must not include scheme"); + new DatastoreOptions.Builder() + .localHost("http://localhost:8080"); + } + + @Test + public void create_NullOptions() throws Exception { + thrown.expect(NullPointerException.class); + factory.create(null); + } + + @Test + public void create_LocalHost() { + Datastore datastore = factory.create(new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .localHost("localhost:8080") + .build()); + assertThat(datastore.remoteRpc.getUrl()) + .isEqualTo("http://localhost:8080/datastore/v1beta3/projects/project-id"); + } + + @Test + public void create_DefaultHost() { + Datastore datastore = factory.create(new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .build()); + assertThat(datastore.remoteRpc.getUrl()) + .isEqualTo("https://datastore.googleapis.com/v1beta3/projects/project-id"); + } + + @Test + public void create_ProjectEndpoint() { + Datastore datastore = factory.create(new DatastoreOptions.Builder() + .projectEndpoint("http://prom-qa/datastore/v1beta42/projects/project-id") + .build()); + assertThat(datastore.remoteRpc.getUrl()) + .isEqualTo("http://prom-qa/datastore/v1beta42/projects/project-id"); + } + + @Test + public void initializer() throws Exception { + options.initializer(new HttpRequestInitializer() { + @Override + public void initialize(HttpRequest request) { + request.getHeaders().setCookie("magic"); + } + }); + Datastore datastore = factory.create(options.build()); + MockDatastoreFactory mockClient = (MockDatastoreFactory) factory; + AllocateIdsRequest request = AllocateIdsRequest.newBuilder().build(); + AllocateIdsResponse response = AllocateIdsResponse.newBuilder().build(); + mockClient.setNextResponse(response); + assertEquals(response, datastore.allocateIds(request)); + assertEquals("magic", mockClient.lastCookies.get(0)); + } + + @Test + public void allocateIds() throws Exception { + AllocateIdsRequest.Builder request = AllocateIdsRequest.newBuilder(); + AllocateIdsResponse.Builder response = AllocateIdsResponse.newBuilder(); + expectRpc("allocateIds", request.build(), response.build()); + } + + @Test + public void lookup() throws Exception { + LookupRequest.Builder request = LookupRequest.newBuilder(); + LookupResponse.Builder response = LookupResponse.newBuilder(); + expectRpc("lookup", request.build(), response.build()); + } + + @Test + public void beginTransaction() throws Exception { + BeginTransactionRequest.Builder request = BeginTransactionRequest.newBuilder(); + BeginTransactionResponse.Builder response = BeginTransactionResponse.newBuilder(); + response.setTransaction(ByteString.copyFromUtf8("project-id")); + expectRpc("beginTransaction", request.build(), response.build()); + } + + @Test + public void commit() throws Exception { + CommitRequest.Builder request = CommitRequest.newBuilder(); + request.setTransaction(ByteString.copyFromUtf8("project-id")); + CommitResponse.Builder response = CommitResponse.newBuilder(); + expectRpc("commit", request.build(), response.build()); + } + + @Test + public void rollback() throws Exception { + RollbackRequest.Builder request = RollbackRequest.newBuilder(); + request.setTransaction(ByteString.copyFromUtf8("project-id")); + RollbackResponse.Builder response = RollbackResponse.newBuilder(); + expectRpc("rollback", request.build(), response.build()); + } + + @Test + public void runQuery() throws Exception { + RunQueryRequest.Builder request = RunQueryRequest.newBuilder(); + request.getQueryBuilder(); + RunQueryResponse.Builder response = RunQueryResponse.newBuilder(); + response.getBatchBuilder() + .setEntityResultType(EntityResult.ResultType.FULL) + .setMoreResults(QueryResultBatch.MoreResultsType.NOT_FINISHED); + expectRpc("runQuery", request.build(), response.build()); + } + + private void expectRpc(String methodName, Message request, Message response) throws Exception { + Datastore datastore = factory.create(options.build()); + MockDatastoreFactory mockClient = (MockDatastoreFactory) factory; + + mockClient.setNextResponse(response); + @SuppressWarnings("rawtypes") + Class[] methodArgs = { request.getClass() }; + Method call = Datastore.class.getMethod(methodName, methodArgs); + Object[] callArgs = { request }; + assertEquals(response, call.invoke(datastore, callArgs)); + + assertEquals("/v1beta3/projects/project-id:" + methodName, mockClient.lastPath); + assertEquals("application/x-protobuf", mockClient.lastMimeType); + assertEquals("2", mockClient.lastApiFormatHeaderValue); + assertArrayEquals(request.toByteArray(), mockClient.lastBody); + assertEquals(1, datastore.getRpcCount()); + + datastore.resetRpcCount(); + assertEquals(0, datastore.getRpcCount()); + + mockClient.setNextError(400, Code.INVALID_ARGUMENT, "oops"); + try { + call.invoke(datastore, callArgs); + fail(); + } catch (InvocationTargetException targetException) { + DatastoreException exception = (DatastoreException) targetException.getCause(); + assertEquals(Code.INVALID_ARGUMENT, exception.getCode()); + assertEquals(methodName, exception.getMethodName()); + assertEquals("oops", exception.getMessage()); + } + + IOException ioException = new IOException("non"); + mockClient.setNextException(ioException); + try { + call.invoke(datastore, callArgs); + fail(); + } catch (InvocationTargetException targetException) { + DatastoreException exception = (DatastoreException) targetException.getCause(); + assertEquals(Code.UNAVAILABLE, exception.getCode()); + assertEquals(methodName, exception.getMethodName()); + assertEquals("I/O error", exception.getMessage()); + assertSame(ioException, exception.getCause()); + } + + assertEquals(2, datastore.getRpcCount()); + } + + private static class MockCredential extends Credential { + MockCredential() { + super(new AccessMethod() { + @Override + public void intercept(HttpRequest request, String accessToken) throws IOException { + } + @Override + public String getAccessTokenFromRequest(HttpRequest request) { + return "MockAccessToken"; + } + }); + } + } + + private static class MockDatastoreFactory extends DatastoreFactory { + int nextStatus; + Message nextResponse; + Status nextError; + IOException nextException; + + String lastPath; + String lastMimeType; + byte[] lastBody; + List lastCookies; + String lastApiFormatHeaderValue; + + void setNextResponse(Message response) { + nextStatus = HttpStatusCodes.STATUS_CODE_OK; + nextResponse = response; + nextError = null; + nextException = null; + } + + void setNextError(int status, Code code, String message) { + nextStatus = status; + nextResponse = null; + nextError = makeErrorContent(message, code); + nextException = null; + } + + void setNextException(IOException exception) { + nextStatus = 0; + nextResponse = null; + nextError = null; + nextException = exception; + } + + @Override + public HttpRequestFactory makeClient(DatastoreOptions options) { + HttpTransport transport = new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return new MockLowLevelHttpRequest(url) { + @Override + public LowLevelHttpResponse execute() throws IOException { + lastPath = new GenericUrl(getUrl()).getRawPath(); + lastMimeType = getContentType(); + lastCookies = getHeaderValues("Cookie"); + lastApiFormatHeaderValue = + Iterables.getOnlyElement(getHeaderValues("X-Goog-Api-Format-Version")); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + getStreamingContent().writeTo(out); + lastBody = out.toByteArray(); + if (nextException != null) { + throw nextException; + } + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse() + .setStatusCode(nextStatus) + .setContentType("application/x-protobuf"); + if (nextError != null) { + assertNull(nextResponse); + response.setContent(new TestableByteArrayInputStream(nextError.toByteArray())); + } else { + response.setContent(new TestableByteArrayInputStream(nextResponse.toByteArray())); + } + return response; + } + }; + } + }; + Credential credential = options.getCredential(); + return transport.createRequestFactory(credential); + } + } + + private static Status makeErrorContent(String message, Code code) { + return Status.newBuilder().setCode(code.getNumber()).setMessage(message).build(); + } +} diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreTest.java new file mode 100644 index 000000000..d0c789f3e --- /dev/null +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreTest.java @@ -0,0 +1,96 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1beta3.client; + +import static org.junit.Assert.fail; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link LocalDevelopmentDatastore}. + */ +@RunWith(JUnit4.class) +public class LocalDevelopmentDatastoreTest { + + private static final LocalDevelopmentDatastoreOptions options = + new LocalDevelopmentDatastoreOptions.Builder().build(); + + @Test + public void testArgs() throws LocalDevelopmentDatastoreException { + LocalDevelopmentDatastore datastore = new LocalDevelopmentDatastore(null, "blar", options) { + @Override + void startDatastoreInternal(String sdkPath, String dataset, String... cmdLineOpts) { + // no-op for testing + } + }; + + try { + datastore.start(null, "dataset"); + fail("expected exception"); + } catch (NullPointerException npe) { + // good + } + + try { + datastore.start("path/to/sdk", null); + fail("expected exception"); + } catch (NullPointerException npe) { + // good + } + + datastore.start("path/to/sdk", "dataset"); + } + + @Test + public void testLifecycle() throws LocalDevelopmentDatastoreException { + LocalDevelopmentDatastore datastore = new LocalDevelopmentDatastore(null, "blar", options) { + @Override + void startDatastoreInternal(String sdkPath, String dataset, String... cmdLineOpts) { + // no-op for testing + } + + @Override + protected void stopDatastoreInternal() { + // no-op for testing + } + }; + + String sdkPath = "/yar"; + String myApp = "myapp"; + + datastore.start(sdkPath, myApp); + try { + datastore.start(sdkPath, myApp); + fail("expected exception"); + } catch (IllegalStateException e) { + // good + } + + datastore.stop(); + // it's ok to stop if we've already stopped + datastore.stop(); + + // once we've stopped we can't start again + try { + datastore.start(sdkPath, myApp); + fail("expected exception"); + } catch (IllegalStateException e) { + // good + } + } +} diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/RemoteRpcTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/RemoteRpcTest.java new file mode 100644 index 000000000..bde089ad3 --- /dev/null +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/RemoteRpcTest.java @@ -0,0 +1,87 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1beta3.client; + +import static org.junit.Assert.assertEquals; + +import com.google.api.client.util.Charsets; +import com.google.rpc.Code; +import com.google.rpc.Status; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayInputStream; + +/** + * Test for {@link RemoteRpc}. + */ +@RunWith(JUnit4.class) +public class RemoteRpcTest { + + private static final String METHOD_NAME = "methodName"; + + @Test + public void testException() { + Status statusProto = + Status.newBuilder() + .setCode(Code.UNAUTHENTICATED_VALUE) + .setMessage("The request does not have valid authentication credentials.") + .build(); + DatastoreException exception = RemoteRpc.makeException("url", METHOD_NAME, + new ByteArrayInputStream(statusProto.toByteArray()), "application/x-protobuf", + Charsets.UTF_8, new RuntimeException(), 401); + assertEquals(Code.UNAUTHENTICATED, exception.getCode()); + assertEquals("The request does not have valid authentication credentials.", + exception.getMessage()); + assertEquals(METHOD_NAME, exception.getMethodName()); + } + + @Test + public void testInvalidProtoException() { + DatastoreException exception = RemoteRpc.makeException("url", METHOD_NAME, + new ByteArrayInputStream("".getBytes()), "application/x-protobuf", + Charsets.UTF_8, new RuntimeException(), 401); + assertEquals(Code.INTERNAL, exception.getCode()); + assertEquals("Unable to parse Status protocol buffer: HTTP status code was 401.", + exception.getMessage()); + assertEquals(METHOD_NAME, exception.getMethodName()); + } + + @Test + public void testEmptyProtoException() { + Status statusProto = Status.newBuilder().build(); + DatastoreException exception = RemoteRpc.makeException("url", METHOD_NAME, + new ByteArrayInputStream(statusProto.toByteArray()), "application/x-protobuf", + Charsets.UTF_8, new RuntimeException(), 401); + assertEquals(Code.INTERNAL, exception.getCode()); + assertEquals("Unexpected OK error code with HTTP status code of 401. Message: .", + exception.getMessage()); + assertEquals(METHOD_NAME, exception.getMethodName()); + } + + @Test + public void testPlainTextException() { + DatastoreException exception = RemoteRpc.makeException("url", METHOD_NAME, + new ByteArrayInputStream("Text Error".getBytes()), "text/plain", Charsets.UTF_8, + new RuntimeException(), 401); + assertEquals(Code.INTERNAL, exception.getCode()); + assertEquals( + "Non-protobuf error: Text Error. HTTP status code was 401.", exception.getMessage()); + assertEquals(METHOD_NAME, exception.getMethodName()); + } +} From 6b53466752e414de59ecf6876448a937e4d0ddce Mon Sep 17 00:00:00 2001 From: Patrick Costello Date: Wed, 30 Mar 2016 10:41:35 -0700 Subject: [PATCH 12/54] Update RemoteRpc to use correct version of proto library. --- .../java/com/google/datastore/v1beta3/client/RemoteRpc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/RemoteRpc.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/RemoteRpc.java index bf7918706..5cd091947 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/RemoteRpc.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/RemoteRpc.java @@ -153,7 +153,7 @@ static DatastoreException makeException(String url, String methodName, InputStre e); } - Code code = Code.forNumber(rpcStatus.getCode()); + Code code = Code.valueOf(rpcStatus.getCode()); if (code == null) { return makeException(url, methodName, Code.INTERNAL, String.format( From 17d14c7a9c79f8f80b13f27e6f518d483a41b8e5 Mon Sep 17 00:00:00 2001 From: Patrick Costello Date: Wed, 6 Apr 2016 11:19:17 -0700 Subject: [PATCH 13/54] Add a helper for ancestor filters. --- .../google/datastore/v1beta3/client/DatastoreHelper.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java index a170cc0fa..9d54d323e 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java @@ -358,6 +358,15 @@ public static PropertyOrder.Builder makeOrder(String property, .setDirection(direction); } + /** + * Makes an ancestor filter. + */ + public static Filter.Builder makeAncestorFilter(Key ancestor) { + return makeFilter( + DatastoreHelper.KEY_PROPERTY_NAME, PropertyFilter.Operator.HAS_ANCESTOR, + makeValue(ancestor)); + } + /** * Make a filter on a property for use in a query. */ From a4a4247a5c92221a51c2278f38b1fe2ab95572d2 Mon Sep 17 00:00:00 2001 From: Ajay Kannan Date: Thu, 7 Apr 2016 14:08:57 -0700 Subject: [PATCH 14/54] Remove log that credentials aren't provided. --- .../com/google/datastore/v1beta3/client/DatastoreFactory.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java index 60cc87c18..1b16c17bc 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java @@ -70,9 +70,6 @@ public Datastore create(DatastoreOptions options) throws IllegalArgumentExceptio */ public HttpRequestFactory makeClient(DatastoreOptions options) { Credential credential = options.getCredential(); - if (credential == null) { - logger.info("Not using any credentials"); - } HttpTransport transport = options.getTransport(); if (transport == null) { transport = credential == null ? new NetHttpTransport() : credential.getTransport(); From 1bfd8bbed986ef721fe0031fd974f13328a5fc4b Mon Sep 17 00:00:00 2001 From: Ed Davisson Date: Thu, 7 Apr 2016 16:05:08 -0700 Subject: [PATCH 15/54] Datastore.java closes InputStreams --- .../datastore/v1beta3/client/Datastore.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/Datastore.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/Datastore.java index bd5a02809..dee8f2df2 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/Datastore.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/Datastore.java @@ -30,6 +30,7 @@ import com.google.rpc.Code; import java.io.IOException; +import java.io.InputStream; /** * Provides access to Cloud Datastore. @@ -63,8 +64,8 @@ private DatastoreException invalidResponseException(String method, IOException e } public AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws DatastoreException { - try { - return AllocateIdsResponse.parseFrom(remoteRpc.call("allocateIds", request)); + try (InputStream is = remoteRpc.call("allocateIds", request)) { + return AllocateIdsResponse.parseFrom(is); } catch (IOException exception) { throw invalidResponseException("allocateIds", exception); } @@ -72,40 +73,40 @@ public AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws Datast public BeginTransactionResponse beginTransaction(BeginTransactionRequest request) throws DatastoreException { - try { - return BeginTransactionResponse.parseFrom(remoteRpc.call("beginTransaction", request)); + try (InputStream is = remoteRpc.call("beginTransaction", request)) { + return BeginTransactionResponse.parseFrom(is); } catch (IOException exception) { throw invalidResponseException("beginTransaction", exception); } } public CommitResponse commit(CommitRequest request) throws DatastoreException { - try { - return CommitResponse.parseFrom(remoteRpc.call("commit", request)); + try (InputStream is = remoteRpc.call("commit", request)) { + return CommitResponse.parseFrom(is); } catch (IOException exception) { throw invalidResponseException("commit", exception); } } public LookupResponse lookup(LookupRequest request) throws DatastoreException { - try { - return LookupResponse.parseFrom(remoteRpc.call("lookup", request)); + try (InputStream is = remoteRpc.call("lookup", request)) { + return LookupResponse.parseFrom(is); } catch (IOException exception) { throw invalidResponseException("lookup", exception); } } public RollbackResponse rollback(RollbackRequest request) throws DatastoreException { - try { - return RollbackResponse.parseFrom(remoteRpc.call("rollback", request)); + try (InputStream is = remoteRpc.call("rollback", request)) { + return RollbackResponse.parseFrom(is); } catch (IOException exception) { throw invalidResponseException("rollback", exception); } } public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreException { - try { - return RunQueryResponse.parseFrom(remoteRpc.call("runQuery", request)); + try (InputStream is = remoteRpc.call("runQuery", request)) { + return RunQueryResponse.parseFrom(is); } catch (IOException exception) { throw invalidResponseException("runQuery", exception); } From 54cc83131d0a2645726af6a4b03347c1f3f7c849 Mon Sep 17 00:00:00 2001 From: Ed Davisson Date: Mon, 9 May 2016 10:36:36 -0700 Subject: [PATCH 16/54] Do all URL validation in DatastoreFactory --- .../v1beta3/client/DatastoreFactory.java | 2 +- .../v1beta3/client/DatastoreHelper.java | 6 ++-- .../v1beta3/client/DatastoreOptions.java | 8 ++--- .../v1beta3/client/DatastoreTest.java | 29 +++++++++++++++++-- 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java index 1b16c17bc..fd72d4069 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java @@ -111,7 +111,7 @@ protected RemoteRpc newRemoteRpc(DatastoreOptions options) { return new RemoteRpc(client, options.getInitializer(), buildProjectEndpoint(options)); } - static String validateUrl(String url) { + private static String validateUrl(String url) { try { return new URI(url).toString(); } catch (URISyntaxException e) { diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java index 9d54d323e..143c47e1d 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java @@ -15,8 +15,6 @@ */ package com.google.datastore.v1beta3.client; -import static com.google.datastore.v1beta3.client.DatastoreFactory.validateUrl; - import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; @@ -314,8 +312,8 @@ private static void setProjectEndpointFromEnv(DatastoreOptions.Builder options) } String projectId = getProjectIdFromEnv(); if (System.getenv(URL_OVERRIDE_ENV_VAR) != null) { - options.projectEndpoint(validateUrl(String.format("%s/projects/%s", - System.getenv(URL_OVERRIDE_ENV_VAR), projectId))); + options.projectEndpoint(String.format("%s/projects/%s", + System.getenv(URL_OVERRIDE_ENV_VAR), projectId)); return; } if (System.getenv(LOCAL_HOST_ENV_VAR) != null) { diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreOptions.java index fbcdf77a3..a5a2e16c4 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreOptions.java @@ -108,7 +108,6 @@ public Builder projectId(String projectId) { */ public Builder localHost(String localHost) { checkArgument(projectEndpoint == null, PROJECT_ENDPOINT_AND_LOCAL_HOST_ERROR); - DatastoreFactory.validateUrl(localHost); if (includesScheme(localHost)) { throw new IllegalArgumentException( String.format("Local host \"%s\" must not include scheme.", localHost)); @@ -124,7 +123,6 @@ public Builder localHost(String localHost) { public Builder projectEndpoint(String projectEndpoint) { checkArgument(projectId == null, PROJECT_ENDPOINT_AND_PROJECT_ID_ERROR); checkArgument(localHost == null, PROJECT_ENDPOINT_AND_LOCAL_HOST_ERROR); - DatastoreFactory.validateUrl(projectEndpoint); if (!includesScheme(projectEndpoint)) { throw new IllegalArgumentException(String.format( "Project endpoint \"%s\" must include scheme.", projectEndpoint)); @@ -156,7 +154,7 @@ public Builder transport(HttpTransport transport) { this.transport = transport; return this; } - + private static boolean includesScheme(String url) { return url.startsWith("http://") || url.startsWith("https://"); } @@ -165,7 +163,7 @@ private static boolean includesScheme(String url) { public String getProjectId() { return projectId; } - + public String getProjectEndpoint() { return projectEndpoint; } @@ -181,7 +179,7 @@ public HttpRequestInitializer getInitializer() { public Credential getCredential() { return credential; } - + public HttpTransport getTransport() { return transport; } diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java index 00ce00256..b4cd7086c 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java @@ -87,7 +87,7 @@ public void options_NoProjectIdOrProjectEndpoint() throws Exception { options = new DatastoreOptions.Builder(); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Either project ID or project endpoint must be provided"); - options.build(); + factory.create(options.build()); } @Test @@ -112,8 +112,10 @@ public void options_LocalHostAndProjectEndpoint() throws Exception { public void options_InvalidLocalHost() throws Exception { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Illegal character"); - new DatastoreOptions.Builder() - .localHost("!not a valid url!"); + factory.create(new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .localHost("!not a valid url!") + .build()); } @Test @@ -140,6 +142,16 @@ public void create_LocalHost() { .isEqualTo("http://localhost:8080/datastore/v1beta3/projects/project-id"); } + @Test + public void create_LocalHostIp() { + Datastore datastore = factory.create(new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .localHost("127.0.0.1:8080") + .build()); + assertThat(datastore.remoteRpc.getUrl()) + .isEqualTo("http://127.0.0.1:8080/datastore/v1beta3/projects/project-id"); + } + @Test public void create_DefaultHost() { Datastore datastore = factory.create(new DatastoreOptions.Builder() @@ -158,6 +170,17 @@ public void create_ProjectEndpoint() { .isEqualTo("http://prom-qa/datastore/v1beta42/projects/project-id"); } + @Test + public void create_ProjectEndpointNoScheme() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage( + "Project endpoint \"prom-qa/datastore/v1beta42/projects/project-id\" must" + + " include scheme."); + factory.create(new DatastoreOptions.Builder() + .projectEndpoint("prom-qa/datastore/v1beta42/projects/project-id") + .build()); + } + @Test public void initializer() throws Exception { options.initializer(new HttpRequestInitializer() { From a3152476a2367f4679888bb534a8941ce226376a Mon Sep 17 00:00:00 2001 From: Ed Davisson Date: Mon, 9 May 2016 10:53:59 -0700 Subject: [PATCH 17/54] s/prom-qa/localhost:1234/ --- .../google/datastore/v1beta3/client/DatastoreTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java index b4cd7086c..44e520605 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java @@ -96,7 +96,7 @@ public void options_ProjectIdAndProjectEndpoint() throws Exception { thrown.expectMessage("Cannot set both project endpoint and project ID"); options = new DatastoreOptions.Builder() .projectId(PROJECT_ID) - .projectEndpoint("http://prom-qa/datastore/v1beta42/projects/project-id"); + .projectEndpoint("http://localhost:1234/datastore/v1beta42/projects/project-id"); } @Test @@ -105,7 +105,7 @@ public void options_LocalHostAndProjectEndpoint() throws Exception { thrown.expectMessage("Cannot set both project endpoint and local host"); options = new DatastoreOptions.Builder() .localHost("localhost:8080") - .projectEndpoint("http://prom-qa/datastore/v1beta42/projects/project-id"); + .projectEndpoint("http://localhost:1234/datastore/v1beta42/projects/project-id"); } @Test @@ -174,10 +174,10 @@ public void create_ProjectEndpoint() { public void create_ProjectEndpointNoScheme() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage( - "Project endpoint \"prom-qa/datastore/v1beta42/projects/project-id\" must" + "Project endpoint \"localhost:1234/datastore/v1beta42/projects/project-id\" must" + " include scheme."); factory.create(new DatastoreOptions.Builder() - .projectEndpoint("prom-qa/datastore/v1beta42/projects/project-id") + .projectEndpoint("localhost:1234/datastore/v1beta42/projects/project-id") .build()); } From 29249d4ee511efdd4e1d72a605405085ed2dda35 Mon Sep 17 00:00:00 2001 From: Ed Davisson Date: Mon, 15 Aug 2016 11:03:28 -0700 Subject: [PATCH 18/54] Update to use the v1 API --- .../{v1beta3 => v1}/client/Datastore.java | 26 ++++++------- .../client/DatastoreException.java | 2 +- .../client/DatastoreFactory.java | 4 +- .../client/DatastoreHelper.java | 30 +++++++-------- .../client/DatastoreOptions.java | 2 +- .../client/LocalDevelopmentDatastore.java | 2 +- .../LocalDevelopmentDatastoreException.java | 2 +- .../LocalDevelopmentDatastoreFactory.java | 2 +- .../LocalDevelopmentDatastoreOptions.java | 2 +- .../{v1beta3 => v1}/client/QuerySplitter.java | 6 +-- .../client/QuerySplitterImpl.java | 30 +++++++-------- .../{v1beta3 => v1}/client/RemoteRpc.java | 2 +- .../client/DatastoreFactoryTest.java | 2 +- .../client/DatastoreHelperTest.java | 18 ++++----- .../{v1beta3 => v1}/client/DatastoreTest.java | 38 +++++++++---------- .../client/LocalDevelopmentDatastoreTest.java | 2 +- .../{v1beta3 => v1}/client/RemoteRpcTest.java | 2 +- 17 files changed, 86 insertions(+), 86 deletions(-) rename datastore-v1-proto-client/src/main/java/com/google/datastore/{v1beta3 => v1}/client/Datastore.java (82%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/{v1beta3 => v1}/client/DatastoreException.java (96%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/{v1beta3 => v1}/client/DatastoreFactory.java (97%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/{v1beta3 => v1}/client/DatastoreHelper.java (97%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/{v1beta3 => v1}/client/DatastoreOptions.java (99%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/{v1beta3 => v1}/client/LocalDevelopmentDatastore.java (99%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/{v1beta3 => v1}/client/LocalDevelopmentDatastoreException.java (95%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/{v1beta3 => v1}/client/LocalDevelopmentDatastoreFactory.java (97%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/{v1beta3 => v1}/client/LocalDevelopmentDatastoreOptions.java (97%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/{v1beta3 => v1}/client/QuerySplitter.java (92%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/{v1beta3 => v1}/client/QuerySplitterImpl.java (92%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/{v1beta3 => v1}/client/RemoteRpc.java (99%) rename datastore-v1-proto-client/src/test/java/com/google/datastore/{v1beta3 => v1}/client/DatastoreFactoryTest.java (98%) rename datastore-v1-proto-client/src/test/java/com/google/datastore/{v1beta3 => v1}/client/DatastoreHelperTest.java (95%) rename datastore-v1-proto-client/src/test/java/com/google/datastore/{v1beta3 => v1}/client/DatastoreTest.java (92%) rename datastore-v1-proto-client/src/test/java/com/google/datastore/{v1beta3 => v1}/client/LocalDevelopmentDatastoreTest.java (98%) rename datastore-v1-proto-client/src/test/java/com/google/datastore/{v1beta3 => v1}/client/RemoteRpcTest.java (98%) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/Datastore.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java similarity index 82% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/Datastore.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java index dee8f2df2..e753dc331 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/Datastore.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; -import com.google.datastore.v1beta3.AllocateIdsRequest; -import com.google.datastore.v1beta3.AllocateIdsResponse; -import com.google.datastore.v1beta3.BeginTransactionRequest; -import com.google.datastore.v1beta3.BeginTransactionResponse; -import com.google.datastore.v1beta3.CommitRequest; -import com.google.datastore.v1beta3.CommitResponse; -import com.google.datastore.v1beta3.LookupRequest; -import com.google.datastore.v1beta3.LookupResponse; -import com.google.datastore.v1beta3.RollbackRequest; -import com.google.datastore.v1beta3.RollbackResponse; -import com.google.datastore.v1beta3.RunQueryRequest; -import com.google.datastore.v1beta3.RunQueryResponse; +import com.google.datastore.v1.AllocateIdsRequest; +import com.google.datastore.v1.AllocateIdsResponse; +import com.google.datastore.v1.BeginTransactionRequest; +import com.google.datastore.v1.BeginTransactionResponse; +import com.google.datastore.v1.CommitRequest; +import com.google.datastore.v1.CommitResponse; +import com.google.datastore.v1.LookupRequest; +import com.google.datastore.v1.LookupResponse; +import com.google.datastore.v1.RollbackRequest; +import com.google.datastore.v1.RollbackResponse; +import com.google.datastore.v1.RunQueryRequest; +import com.google.datastore.v1.RunQueryResponse; import com.google.rpc.Code; import java.io.IOException; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreException.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreException.java similarity index 96% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreException.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreException.java index 70bfc2f4a..e259aa9e6 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreException.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreException.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; import com.google.rpc.Code; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java similarity index 97% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java index fd72d4069..e5de9f784 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; import static com.google.common.base.Preconditions.checkNotNull; @@ -43,7 +43,7 @@ public class DatastoreFactory { private static ConsoleHandler methodHandler; /** API version. */ - public static final String VERSION = "v1beta3"; + public static final String VERSION = "v1"; public static final String DEFAULT_HOST = "https://datastore.googleapis.com"; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java similarity index 97% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java index 143c47e1d..6b2c042be 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreHelper.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; @@ -23,20 +23,20 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson.JacksonFactory; -import com.google.datastore.v1beta3.ArrayValue; -import com.google.datastore.v1beta3.CompositeFilter; -import com.google.datastore.v1beta3.Entity; -import com.google.datastore.v1beta3.Filter; -import com.google.datastore.v1beta3.Key; -import com.google.datastore.v1beta3.Key.PathElement; -import com.google.datastore.v1beta3.Key.PathElement.IdTypeCase; -import com.google.datastore.v1beta3.Mutation; -import com.google.datastore.v1beta3.PartitionId; -import com.google.datastore.v1beta3.PropertyFilter; -import com.google.datastore.v1beta3.PropertyOrder; -import com.google.datastore.v1beta3.PropertyReference; -import com.google.datastore.v1beta3.Value; -import com.google.datastore.v1beta3.Value.ValueTypeCase; +import com.google.datastore.v1.ArrayValue; +import com.google.datastore.v1.CompositeFilter; +import com.google.datastore.v1.Entity; +import com.google.datastore.v1.Filter; +import com.google.datastore.v1.Key; +import com.google.datastore.v1.Key.PathElement; +import com.google.datastore.v1.Key.PathElement.IdTypeCase; +import com.google.datastore.v1.Mutation; +import com.google.datastore.v1.PartitionId; +import com.google.datastore.v1.PropertyFilter; +import com.google.datastore.v1.PropertyOrder; +import com.google.datastore.v1.PropertyReference; +import com.google.datastore.v1.Value; +import com.google.datastore.v1.Value.ValueTypeCase; import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; import com.google.protobuf.TimestampOrBuilder; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java similarity index 99% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreOptions.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java index a5a2e16c4..ac76133c9 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/DatastoreOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastore.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastore.java similarity index 99% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastore.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastore.java index 0a0d7a045..7e6e6b158 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastore.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastore.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequestFactory; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreException.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreException.java similarity index 95% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreException.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreException.java index 258dcd853..f335674a8 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreException.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreException.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; /** * An exception related to the local development {@link Datastore}. diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreFactory.java similarity index 97% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreFactory.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreFactory.java index 37302ab5f..12ddbe859 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreFactory.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; /** * Factory for {@link LocalDevelopmentDatastore}. diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreOptions.java similarity index 97% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreOptions.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreOptions.java index 8fd519ed2..0d4b00124 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreOptions.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; import java.util.HashMap; import java.util.Map; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/QuerySplitter.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitter.java similarity index 92% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/QuerySplitter.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitter.java index 190cf783a..e220ad35c 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/QuerySplitter.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitter.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; -import com.google.datastore.v1beta3.PartitionId; -import com.google.datastore.v1beta3.Query; +import com.google.datastore.v1.PartitionId; +import com.google.datastore.v1.Query; import java.util.List; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/QuerySplitterImpl.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java similarity index 92% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/QuerySplitterImpl.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java index dab377148..c10ef0563 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/QuerySplitterImpl.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java @@ -13,23 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; -import static com.google.datastore.v1beta3.client.DatastoreHelper.makeAndFilter; +import static com.google.datastore.v1.client.DatastoreHelper.makeAndFilter; -import com.google.datastore.v1beta3.EntityResult; -import com.google.datastore.v1beta3.Filter; -import com.google.datastore.v1beta3.Key; -import com.google.datastore.v1beta3.PartitionId; -import com.google.datastore.v1beta3.Projection; -import com.google.datastore.v1beta3.PropertyFilter; -import com.google.datastore.v1beta3.PropertyFilter.Operator; -import com.google.datastore.v1beta3.PropertyOrder.Direction; -import com.google.datastore.v1beta3.PropertyReference; -import com.google.datastore.v1beta3.Query; -import com.google.datastore.v1beta3.QueryResultBatch; -import com.google.datastore.v1beta3.QueryResultBatch.MoreResultsType; -import com.google.datastore.v1beta3.RunQueryRequest; +import com.google.datastore.v1.EntityResult; +import com.google.datastore.v1.Filter; +import com.google.datastore.v1.Key; +import com.google.datastore.v1.PartitionId; +import com.google.datastore.v1.Projection; +import com.google.datastore.v1.PropertyFilter; +import com.google.datastore.v1.PropertyFilter.Operator; +import com.google.datastore.v1.PropertyOrder.Direction; +import com.google.datastore.v1.PropertyReference; +import com.google.datastore.v1.Query; +import com.google.datastore.v1.QueryResultBatch; +import com.google.datastore.v1.QueryResultBatch.MoreResultsType; +import com.google.datastore.v1.RunQueryRequest; import java.util.ArrayList; import java.util.Collections; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/RemoteRpc.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java similarity index 99% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/RemoteRpc.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java index 5cd091947..386775c54 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1beta3/client/RemoteRpc.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequest; diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreFactoryTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreFactoryTest.java similarity index 98% rename from datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreFactoryTest.java rename to datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreFactoryTest.java index 3a76d6110..22e145996 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreFactoryTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreFactoryTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreHelperTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreHelperTest.java similarity index 95% rename from datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreHelperTest.java rename to datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreHelperTest.java index 11ec8aeed..1bd166bcc 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreHelperTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreHelperTest.java @@ -13,19 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; -import static com.google.datastore.v1beta3.client.DatastoreHelper.getByteString; -import static com.google.datastore.v1beta3.client.DatastoreHelper.makeKey; -import static com.google.datastore.v1beta3.client.DatastoreHelper.makeValue; -import static com.google.datastore.v1beta3.client.DatastoreHelper.toDate; +import static com.google.datastore.v1.client.DatastoreHelper.getByteString; +import static com.google.datastore.v1.client.DatastoreHelper.makeKey; +import static com.google.datastore.v1.client.DatastoreHelper.makeValue; +import static com.google.datastore.v1.client.DatastoreHelper.toDate; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import com.google.datastore.v1beta3.Key; -import com.google.datastore.v1beta3.PartitionId; -import com.google.datastore.v1beta3.Value; -import com.google.datastore.v1beta3.Value.ValueTypeCase; +import com.google.datastore.v1.Key; +import com.google.datastore.v1.PartitionId; +import com.google.datastore.v1.Value; +import com.google.datastore.v1.Value.ValueTypeCase; import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java similarity index 92% rename from datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java rename to datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java index 44e520605..323568e01 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/DatastoreTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertArrayEquals; @@ -36,20 +36,20 @@ import com.google.api.client.testing.http.MockLowLevelHttpResponse; import com.google.api.client.testing.util.TestableByteArrayInputStream; import com.google.common.collect.Iterables; -import com.google.datastore.v1beta3.AllocateIdsRequest; -import com.google.datastore.v1beta3.AllocateIdsResponse; -import com.google.datastore.v1beta3.BeginTransactionRequest; -import com.google.datastore.v1beta3.BeginTransactionResponse; -import com.google.datastore.v1beta3.CommitRequest; -import com.google.datastore.v1beta3.CommitResponse; -import com.google.datastore.v1beta3.EntityResult; -import com.google.datastore.v1beta3.LookupRequest; -import com.google.datastore.v1beta3.LookupResponse; -import com.google.datastore.v1beta3.QueryResultBatch; -import com.google.datastore.v1beta3.RollbackRequest; -import com.google.datastore.v1beta3.RollbackResponse; -import com.google.datastore.v1beta3.RunQueryRequest; -import com.google.datastore.v1beta3.RunQueryResponse; +import com.google.datastore.v1.AllocateIdsRequest; +import com.google.datastore.v1.AllocateIdsResponse; +import com.google.datastore.v1.BeginTransactionRequest; +import com.google.datastore.v1.BeginTransactionResponse; +import com.google.datastore.v1.CommitRequest; +import com.google.datastore.v1.CommitResponse; +import com.google.datastore.v1.EntityResult; +import com.google.datastore.v1.LookupRequest; +import com.google.datastore.v1.LookupResponse; +import com.google.datastore.v1.QueryResultBatch; +import com.google.datastore.v1.RollbackRequest; +import com.google.datastore.v1.RollbackResponse; +import com.google.datastore.v1.RunQueryRequest; +import com.google.datastore.v1.RunQueryResponse; import com.google.protobuf.ByteString; import com.google.protobuf.Message; import com.google.rpc.Code; @@ -139,7 +139,7 @@ public void create_LocalHost() { .localHost("localhost:8080") .build()); assertThat(datastore.remoteRpc.getUrl()) - .isEqualTo("http://localhost:8080/datastore/v1beta3/projects/project-id"); + .isEqualTo("http://localhost:8080/datastore/v1/projects/project-id"); } @Test @@ -149,7 +149,7 @@ public void create_LocalHostIp() { .localHost("127.0.0.1:8080") .build()); assertThat(datastore.remoteRpc.getUrl()) - .isEqualTo("http://127.0.0.1:8080/datastore/v1beta3/projects/project-id"); + .isEqualTo("http://127.0.0.1:8080/datastore/v1/projects/project-id"); } @Test @@ -158,7 +158,7 @@ public void create_DefaultHost() { .projectId(PROJECT_ID) .build()); assertThat(datastore.remoteRpc.getUrl()) - .isEqualTo("https://datastore.googleapis.com/v1beta3/projects/project-id"); + .isEqualTo("https://datastore.googleapis.com/v1/projects/project-id"); } @Test @@ -258,7 +258,7 @@ private void expectRpc(String methodName, Message request, Message response) thr Object[] callArgs = { request }; assertEquals(response, call.invoke(datastore, callArgs)); - assertEquals("/v1beta3/projects/project-id:" + methodName, mockClient.lastPath); + assertEquals("/v1/projects/project-id:" + methodName, mockClient.lastPath); assertEquals("application/x-protobuf", mockClient.lastMimeType); assertEquals("2", mockClient.lastApiFormatHeaderValue); assertArrayEquals(request.toByteArray(), mockClient.lastBody); diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreTest.java similarity index 98% rename from datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreTest.java rename to datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreTest.java index d0c789f3e..e7cd32bd5 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/LocalDevelopmentDatastoreTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; import static org.junit.Assert.fail; diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/RemoteRpcTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java similarity index 98% rename from datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/RemoteRpcTest.java rename to datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java index bde089ad3..bceee04b3 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1beta3/client/RemoteRpcTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.datastore.v1beta3.client; +package com.google.datastore.v1.client; import static org.junit.Assert.assertEquals; From aef3e35d6141935a53944f6545dd7965a83959fa Mon Sep 17 00:00:00 2001 From: eddavisson Date: Wed, 17 Aug 2016 10:18:57 -0700 Subject: [PATCH 19/54] Remove 'datatore' from path and prepare for release (#135) --- .../java/com/google/datastore/v1/client/DatastoreFactory.java | 4 ++-- .../java/com/google/datastore/v1/client/DatastoreTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java index e5de9f784..809f8beab 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java @@ -64,7 +64,7 @@ public static DatastoreFactory get() { public Datastore create(DatastoreOptions options) throws IllegalArgumentException { return new Datastore(newRemoteRpc(options)); } - + /** * Constructs a Google APIs HTTP client with the associated credentials. */ @@ -98,7 +98,7 @@ String buildProjectEndpoint(DatastoreOptions options) { // DatastoreOptions ensures either project endpoint or project ID is set. String projectId = checkNotNull(options.getProjectId()); if (options.getLocalHost() != null) { - return validateUrl(String.format("http://%s/datastore/%s/projects/%s", + return validateUrl(String.format("http://%s/%s/projects/%s", options.getLocalHost(), VERSION, projectId)); } return validateUrl(String.format("%s/%s/projects/%s", diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java index 323568e01..bcdbe18de 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java @@ -139,7 +139,7 @@ public void create_LocalHost() { .localHost("localhost:8080") .build()); assertThat(datastore.remoteRpc.getUrl()) - .isEqualTo("http://localhost:8080/datastore/v1/projects/project-id"); + .isEqualTo("http://localhost:8080/v1/projects/project-id"); } @Test @@ -149,7 +149,7 @@ public void create_LocalHostIp() { .localHost("127.0.0.1:8080") .build()); assertThat(datastore.remoteRpc.getUrl()) - .isEqualTo("http://127.0.0.1:8080/datastore/v1/projects/project-id"); + .isEqualTo("http://127.0.0.1:8080/v1/projects/project-id"); } @Test From e1c8dc8c5060f48410c9fa284e78a7019f84e190 Mon Sep 17 00:00:00 2001 From: Ed Davisson Date: Mon, 12 Sep 2016 17:31:47 -0700 Subject: [PATCH 20/54] Rename Java testing harness --- .../{LocalDevelopmentDatastore.java => DatastoreEmulator.java} | 0 ...entDatastoreException.java => DatastoreEmulatorException.java} | 0 ...lopmentDatastoreFactory.java => DatastoreEmulatorFactory.java} | 0 ...lopmentDatastoreOptions.java => DatastoreEmulatorOptions.java} | 0 ...alDevelopmentDatastoreTest.java => DatastoreEmulatorTest.java} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/{LocalDevelopmentDatastore.java => DatastoreEmulator.java} (100%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/{LocalDevelopmentDatastoreException.java => DatastoreEmulatorException.java} (100%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/{LocalDevelopmentDatastoreFactory.java => DatastoreEmulatorFactory.java} (100%) rename datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/{LocalDevelopmentDatastoreOptions.java => DatastoreEmulatorOptions.java} (100%) rename datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/{LocalDevelopmentDatastoreTest.java => DatastoreEmulatorTest.java} (100%) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastore.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java similarity index 100% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastore.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreException.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorException.java similarity index 100% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreException.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorException.java diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorFactory.java similarity index 100% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreFactory.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorFactory.java diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorOptions.java similarity index 100% rename from datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreOptions.java rename to datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorOptions.java diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreEmulatorTest.java similarity index 100% rename from datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/LocalDevelopmentDatastoreTest.java rename to datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreEmulatorTest.java From 8d62ff4142d70e9439950090472dacebbaead11d Mon Sep 17 00:00:00 2001 From: Ed Davisson Date: Mon, 12 Sep 2016 14:47:36 -0700 Subject: [PATCH 21/54] Update test harnesses to use the emulator. --- .../v1/client/DatastoreEmulator.java | 218 +++++++++--------- .../v1/client/DatastoreEmulatorException.java | 10 +- .../v1/client/DatastoreEmulatorFactory.java | 25 +- .../v1/client/DatastoreEmulatorOptions.java | 28 +-- .../v1/client/DatastoreEmulatorTest.java | 68 +++--- 5 files changed, 172 insertions(+), 177 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java index 7e6e6b158..2dc80fe06 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java @@ -15,12 +15,12 @@ */ package com.google.datastore.v1.client; +import static com.google.api.client.util.Preconditions.checkNotNull; +import static com.google.api.client.util.Preconditions.checkState; + import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpResponse; -import com.google.api.client.http.UrlEncodedContent; -import com.google.api.client.util.Preconditions; - import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -29,57 +29,55 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; +import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; /** - * An extension to {@link Datastore} that provides lifecycle management for a development datastore - * server. + * An extension to {@link Datastore} that provides lifecycle management for a datastore emulator. * - * In order to use a development datastore for a JUnit 4 test you might do something like this: + *

In order to use the emulator for a JUnit 4 test you might do something like this: * * 

  * public class MyTest {
  *
- *   static LocalDevelopmentDatastore datastore;
+ *   static DatastoreEmulator datastore;
  *
  *   {@literal @}BeforeClass
- *   public static void startLocalDatastore() throws LocalDevelopmentDatastoreException {
- *     DatastoreOptions opts = new DatastoreOptions.Builder()
+ *   public static void startEmulator() throws DatastoreEmulatorException {
+ *     DatastoreOptions options = new DatastoreOptions.Builder()
  *         .localHost("localhost:8080")
  *         .projectId("my-project-id")
  *         .build();
- *     datastore = LocalDevelopmentDatastoreFactory.get().create(opts);
- *     datastore.start("/usr/local/gcd-sdk", "my-project-id");
+ *     datastore = DatastoreEmulatorFactory.get().create(options);
+ *     datastore.start("/usr/local/cloud-datastore-emulator", "my-project-id");
  *   }
  *
  *   {@literal @}Before
- *   public void setUp() throws LocalDevelopmentDatastoreException {
+ *   public void setUp() throws DatastoreEmulatorException {
  *     datastore.clear();
  *   }
  *
  *   {@literal @}AfterClass
- *   public static void stopLocalDatastore() throws LocalDevelopmentDatastoreException {
+ *   public static void stopEmulator() throws DatastoreEmulatorException {
  *     datastore.stop();
  *   }
  *
  *   {@literal @}Test
  *   public void testFoo1() { }
-
+ *
  *   {@literal @}Test
  *   public void testFoo2() { }
  *
  * }
  *
*/ -public class LocalDevelopmentDatastore extends Datastore { +public class DatastoreEmulator extends Datastore { private static final int STARTUP_TIMEOUT_SECS = 30; private final String host; - private LocalDevelopmentDatastoreOptions localDevelopmentOptions; + private final DatastoreEmulatorOptions options; /** Internal state lifecycle management. */ enum State {NEW, STARTED, STOPPED} @@ -88,54 +86,49 @@ enum State {NEW, STARTED, STOPPED} private File projectDirectory; - LocalDevelopmentDatastore(RemoteRpc rpc, String localHost, - LocalDevelopmentDatastoreOptions localDevelopmentOptions) { + DatastoreEmulator(RemoteRpc rpc, String localHost, DatastoreEmulatorOptions options) { super(rpc); this.host = "http://" + localHost; - this.localDevelopmentOptions = localDevelopmentOptions; + this.options = options; } /** - * Clears all data in the Datastore. + * Clears all data in the emulator. * - * @throws LocalDevelopmentDatastoreException + * @throws DatastoreEmulatorException if the clear fails */ - public void clear() throws LocalDevelopmentDatastoreException { + public void clear() throws DatastoreEmulatorException { HttpRequestFactory client = remoteRpc.getHttpRequestFactory(); try { - Map params = new HashMap(); - params.put("action", "Clear Datastore"); - UrlEncodedContent content = new UrlEncodedContent(params); - GenericUrl url = new GenericUrl(host + "/_ah/admin/datastore"); - HttpResponse httpResponse = client.buildPostRequest(url, content).execute(); + GenericUrl url = new GenericUrl(host + "/reset"); + HttpResponse httpResponse = client.buildPostRequest(url, null).execute(); if (!httpResponse.isSuccessStatusCode()) { - throw new LocalDevelopmentDatastoreException( - "Clear Datastore returned http status " + httpResponse.getStatusCode()); + throw new DatastoreEmulatorException( + "POST request to /reset returned HTTP status " + httpResponse.getStatusCode()); } } catch (IOException e) { - throw new LocalDevelopmentDatastoreException( - "Exception trying to clear the dev datastore", e); + throw new DatastoreEmulatorException("Exception trying to clear the emulator", e); } } /** - * Starts the local datastore. It is the caller's responsibility to call {@link #stop}. Note that + * Starts the emulator. It is the caller's responsibility to call {@link #stop}. Note that * receiving an exception does not indicate that the server did not start. We recommend calling * {@link #stop} to ensure the server is not running regardless of the result of this method. * - * @param sdkPath The path to the GCD SDK, eg /usr/local/dev/gcd - * @param projectId The GCD project ID - * @param cmdLineOptions Command line options to pass to the script that launches the dev server - * @throws LocalDevelopmentDatastoreException If {@link #start} has already been called or the - * server does not start successfully. + * @param emulatorDir The path to the emulator directory, e.g. /usr/local/cloud-datastore-emulator + * @param projectId The project ID + * @param commandLineOptions Command line options to pass to the emulator on startup + * @throws DatastoreEmulatorException If {@link #start} has already been called or the server does + * not start successfully. */ - public synchronized void start(String sdkPath, String projectId, String... cmdLineOptions) - throws LocalDevelopmentDatastoreException { - Preconditions.checkNotNull(sdkPath, "sdkPath cannot be null"); - Preconditions.checkNotNull(projectId, "projectId cannot be null"); - Preconditions.checkState(state == State.NEW, "Cannot call start() more than once."); + public synchronized void start(String emulatorDir, String projectId, String... commandLineOptions) + throws DatastoreEmulatorException { + checkNotNull(emulatorDir, "emulatorDir cannot be null"); + checkNotNull(projectId, "projectId cannot be null"); + checkState(state == State.NEW, "Cannot call start() more than once."); try { - startDatastoreInternal(sdkPath, projectId, cmdLineOptions); + startEmulatorInternal(emulatorDir, projectId, commandLineOptions); state = State.STARTED; } finally { if (state != State.STARTED) { @@ -146,81 +139,89 @@ public synchronized void start(String sdkPath, String projectId, String... cmdLi } } - void startDatastoreInternal(String sdkPath, String projectId, String... cmdLineOptions) - throws LocalDevelopmentDatastoreException { - File projectDirectory = createProjectDirectory(sdkPath, projectId); - List cmd = new ArrayList( - Arrays.asList("./gcd.sh", "start", "--allow_remote_shutdown", "--store_on_disk=false")); - cmd.addAll(Arrays.asList(cmdLineOptions)); + void startEmulatorInternal(String emulatorDir, String projectId, String... commandLineOptions) + throws DatastoreEmulatorException { + projectDirectory = createProjectDirectory(emulatorDir, projectId); + List cmd = + new ArrayList<>(Arrays.asList("./cloud_datastore_emulator", "start", "--testing")); + Collections.addAll(cmd, commandLineOptions); cmd.add(projectDirectory.getPath()); - final Process gcdStartProcess; + final Process emulatorStartProcess; try { - gcdStartProcess = newGcdProcess(sdkPath, cmd).start(); + emulatorStartProcess = newEmulatorProcess(emulatorDir, cmd).start(); } catch (IOException e) { - throw new LocalDevelopmentDatastoreException("Could not start dev server", e); - } - // Ensure we don't leak the stub instance if tests end prematurely. - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - gcdStartProcess.destroy(); + throw new DatastoreEmulatorException("Could not start emulator", e); } - }); - StartupMonitor monitor = new StartupMonitor(gcdStartProcess.getInputStream()); + // Ensure we don't leak the emulator instance if tests end prematurely. + Runtime.getRuntime() + .addShutdownHook( + new Thread() { + @Override + public void run() { + emulatorStartProcess.destroy(); + } + }); + StartupMonitor monitor = new StartupMonitor(emulatorStartProcess.getInputStream()); try { monitor.start(); if (!monitor.startupCompleteLatch.await(STARTUP_TIMEOUT_SECS, TimeUnit.SECONDS)) { - throw new LocalDevelopmentDatastoreException("Dev server did not start within 30 seconds"); + throw new DatastoreEmulatorException("Emulator did not start within 30 seconds"); } if (!monitor.success) { - throw new LocalDevelopmentDatastoreException("Server did not start normally"); + throw new DatastoreEmulatorException("Emulator did not start normally"); } } catch (InterruptedException e) { - // not sure why this would happen - throw new LocalDevelopmentDatastoreException("Received an interrupt", e); + Thread.currentThread().interrupt(); + throw new DatastoreEmulatorException("Received an interrupt", e); } } - private File createProjectDirectory(String sdkPath, String projectId) - throws LocalDevelopmentDatastoreException { + private File createProjectDirectory(String emulatorDir, String projectId) + throws DatastoreEmulatorException { + File projectDirectory; try { - projectDirectory = Files.createTempDirectory("local-development-datastore").toFile(); + projectDirectory = Files.createTempDirectory("datastore-emulator").toFile(); } catch (IOException e) { - throw new LocalDevelopmentDatastoreException("Could not create project tmp directory", e); + throw new DatastoreEmulatorException("Could not create temporary project directory", e); } - List cmd = Arrays.asList("./gcd.sh", "create", "--project_id=" + projectId, - projectDirectory.getPath()); + List cmd = + Arrays.asList( + "./cloud_datastore_emulator", + "create", + "--project_id=" + projectId, + projectDirectory.getPath()); try { - int retCode = newGcdProcess(sdkPath, cmd).start().waitFor(); + int retCode = newEmulatorProcess(emulatorDir, cmd).start().waitFor(); if (retCode != 0) { - throw new LocalDevelopmentDatastoreException( + throw new DatastoreEmulatorException( String.format("Could not create project (retcode=%d)", retCode)); } } catch (IOException e) { - throw new LocalDevelopmentDatastoreException("Could not create project", e); + throw new DatastoreEmulatorException("Could not create project", e); } catch (InterruptedException e) { - throw new LocalDevelopmentDatastoreException("Received an interrupt", e); + Thread.currentThread().interrupt(); + throw new DatastoreEmulatorException("Received an interrupt", e); } return projectDirectory; } - private ProcessBuilder newGcdProcess(String sdkPath, List cmd) { + private ProcessBuilder newEmulatorProcess(String emulatorDir, List cmd) { ProcessBuilder builder = new ProcessBuilder(cmd); - builder.directory(new File(sdkPath)); + builder.directory(new File(emulatorDir)); builder.redirectErrorStream(true); - builder.environment().putAll(localDevelopmentOptions.getEnvVars()); + builder.environment().putAll(options.getEnvVars()); return builder; } /** - * Stops the local datastore. Multiple calls are allowed. + * Stops the emulator. Multiple calls are allowed. * - * @throws LocalDevelopmentDatastoreException + * @throws DatastoreEmulatorException if the emulator cannot be stopped */ - public synchronized void stop() throws LocalDevelopmentDatastoreException { + public synchronized void stop() throws DatastoreEmulatorException { // We intentionally don't check the internal state. If people want to try and stop the server // multiple times that's fine. - stopDatastoreInternal(); + stopEmulatorInternal(); if (state != State.STOPPED) { state = State.STOPPED; if (projectDirectory != null) { @@ -228,40 +229,44 @@ public synchronized void stop() throws LocalDevelopmentDatastoreException { Process process = new ProcessBuilder("rm", "-r", projectDirectory.getAbsolutePath()).start(); if (process.waitFor() != 0) { - throw new IOException("Temp directory delete exited with " + process.exitValue()); + throw new IOException( + "Temporary project directory deletion exited with " + process.exitValue()); } - } catch (IOException | InterruptedException e) { - throw new IllegalStateException("Project directory wipe failed.", e); + } catch (IOException e) { + throw new IllegalStateException("Could not delete temporary project directory", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Could not delete temporary project directory", e); } } } } - void stopDatastoreInternal() throws LocalDevelopmentDatastoreException { + public synchronized File getProjectDirectory() { + checkState(state == State.STARTED); + return projectDirectory; + } + + void stopEmulatorInternal() throws DatastoreEmulatorException { // No need to kill the process we started, this function will take care of it. HttpRequestFactory client = remoteRpc.getHttpRequestFactory(); - Map params = new HashMap(); - UrlEncodedContent content = new UrlEncodedContent(params); - GenericUrl url = new GenericUrl(host + "/_ah/admin/quit"); + GenericUrl url = new GenericUrl(host + "/shutdown"); try { - HttpResponse httpResponse = client.buildPostRequest(url, content).execute(); + HttpResponse httpResponse = client.buildPostRequest(url, null).execute(); if (!httpResponse.isSuccessStatusCode()) { - throw new LocalDevelopmentDatastoreException( - "Request to shutdown local datastore returned http error code " - + httpResponse.getStatusCode()); + throw new DatastoreEmulatorException( + "POST request to /shutdown returned HTTP status " + httpResponse.getStatusCode()); } } catch (IOException e) { - throw new LocalDevelopmentDatastoreException( - "Exception trying to stop the dev datastore", e); + throw new DatastoreEmulatorException("Exception trying to stop the emulator", e); } } /** - * Monitors the provided input stream for evidence that the dev server has started successfully - * and redirects the output of the dev server process to sysout for this process. + * Monitors the provided input stream for evidence that the emulator has started successfully and + * redirects the output of the emulator process to sysout for this process. */ static class StartupMonitor extends Thread { - private final InputStream inputStream; private volatile boolean success = false; /** This latch will reach 0 once server startup has completed. */ @@ -278,8 +283,9 @@ public void run() { BufferedReader br = new BufferedReader(new InputStreamReader(inputStream)); String line; while ((line = br.readLine()) != null) { - // redirect to sysout for our process + // Redirect to sysout for our process. System.out.println(line); + // TODO(pcostello): Just make a request to "/" and look for an HTTP 200. if (!success && line.contains("Dev App Server is now running")) { success = true; startupCompleteLatch.countDown(); @@ -287,13 +293,15 @@ public void run() { } } catch (IOException ioe) { if (!success) { - System.err.println("Received an IOException before the dev server startup completed. " - + "Dev server is in an unknown state."); + System.err.println( + "Received an IOException before emulator startup completed. " + + "Emulator is in an unknown state."); } else { // We got an exception after the server started successfully. We'll lose the ability - // to log the output of the dev server but there's no need to shut anything down. - System.err.println("Received an exception handling output from the dev server. " - + "Logging will stop but the dev server is probably ok."); + // to log the output of the emulator but there's no need to shut anything down. + System.err.println( + "Received an exception handling output from the emulator. " + + "Logging will stop but the emulator is probably ok."); } ioe.printStackTrace(); } finally { diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorException.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorException.java index f335674a8..f0f3a177f 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorException.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorException.java @@ -15,15 +15,13 @@ */ package com.google.datastore.v1.client; -/** - * An exception related to the local development {@link Datastore}. - */ -public class LocalDevelopmentDatastoreException extends Exception { - public LocalDevelopmentDatastoreException(String message) { +/** An exception related to the {@link DatastoreEmulator}. */ +public class DatastoreEmulatorException extends Exception { + public DatastoreEmulatorException(String message) { super(message); } - public LocalDevelopmentDatastoreException(String message, Throwable cause) { + public DatastoreEmulatorException(String message, Throwable cause) { super(message, cause); } } diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorFactory.java index 12ddbe859..c8cc6ab29 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorFactory.java @@ -15,31 +15,26 @@ */ package com.google.datastore.v1.client; -/** - * Factory for {@link LocalDevelopmentDatastore}. - */ -public class LocalDevelopmentDatastoreFactory extends DatastoreFactory { +/** Factory for {@link DatastoreEmulator}. */ +public class DatastoreEmulatorFactory extends DatastoreFactory { /** Singleton factory instance. */ - private static final LocalDevelopmentDatastoreFactory INSTANCE = - new LocalDevelopmentDatastoreFactory(); + private static final DatastoreEmulatorFactory INSTANCE = new DatastoreEmulatorFactory(); - public static LocalDevelopmentDatastoreFactory get() { + public static DatastoreEmulatorFactory get() { return INSTANCE; } - LocalDevelopmentDatastoreFactory() { } + DatastoreEmulatorFactory() {} @Override - public LocalDevelopmentDatastore create(DatastoreOptions options) - throws IllegalArgumentException { - return create(options, new LocalDevelopmentDatastoreOptions.Builder().build()); + public DatastoreEmulator create(DatastoreOptions options) throws IllegalArgumentException { + return create(options, new DatastoreEmulatorOptions.Builder().build()); } - public LocalDevelopmentDatastore create(DatastoreOptions options, - LocalDevelopmentDatastoreOptions localDevelopmentOptions) { + public DatastoreEmulator create( + DatastoreOptions options, DatastoreEmulatorOptions localDevelopmentOptions) { RemoteRpc rpc = newRemoteRpc(options); - return new LocalDevelopmentDatastore(rpc, options.getLocalHost(), - localDevelopmentOptions); + return new DatastoreEmulator(rpc, options.getLocalHost(), localDevelopmentOptions); } } diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorOptions.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorOptions.java index 0d4b00124..446b60cec 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorOptions.java @@ -18,35 +18,29 @@ import java.util.HashMap; import java.util.Map; -/** - * An immutable object containing settings for a {@link LocalDevelopmentDatastore}. - */ -public class LocalDevelopmentDatastoreOptions { +/** An immutable object containing settings for a {@link DatastoreEmulator}. */ +public class DatastoreEmulatorOptions { private final Map envVars; - - LocalDevelopmentDatastoreOptions(Map envVars) { + + DatastoreEmulatorOptions(Map envVars) { this.envVars = envVars; } - - /** - * Builder for {@link LocalDevelopmentDatastoreOptions}. - */ + + /** Builder for {@link DatastoreEmulatorOptions}. */ public static class Builder { private final Map envVars = new HashMap(); - public LocalDevelopmentDatastoreOptions build() { - return new LocalDevelopmentDatastoreOptions(envVars); + public DatastoreEmulatorOptions build() { + return new DatastoreEmulatorOptions(envVars); } - - /** - * Adds an environment variable to pass to the GCD tool. - */ + + /** Adds an environment variable to pass to the emulator. */ public Builder addEnvVar(String var, String value) { envVars.put(var, value); return this; } } - + public Map getEnvVars() { return envVars; } diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreEmulatorTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreEmulatorTest.java index e7cd32bd5..fbe573b76 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreEmulatorTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreEmulatorTest.java @@ -21,73 +21,73 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** - * Tests for {@link LocalDevelopmentDatastore}. - */ +/** Tests for {@link DatastoreEmulator}. */ @RunWith(JUnit4.class) -public class LocalDevelopmentDatastoreTest { +public class DatastoreEmulatorTest { - private static final LocalDevelopmentDatastoreOptions options = - new LocalDevelopmentDatastoreOptions.Builder().build(); + private static final DatastoreEmulatorOptions options = + new DatastoreEmulatorOptions.Builder().build(); @Test - public void testArgs() throws LocalDevelopmentDatastoreException { - LocalDevelopmentDatastore datastore = new LocalDevelopmentDatastore(null, "blar", options) { - @Override - void startDatastoreInternal(String sdkPath, String dataset, String... cmdLineOpts) { - // no-op for testing - } - }; + public void testArgs() throws DatastoreEmulatorException { + DatastoreEmulator datastore = + new DatastoreEmulator(null, "blar", options) { + @Override + void startEmulatorInternal(String emulatorDir, String projectId, String... cmdLineOpts) { + // no-op for testing + } + }; try { - datastore.start(null, "dataset"); + datastore.start(null, "projectId"); fail("expected exception"); } catch (NullPointerException npe) { // good } try { - datastore.start("path/to/sdk", null); + datastore.start("path/to/emulator", null); fail("expected exception"); } catch (NullPointerException npe) { // good } - datastore.start("path/to/sdk", "dataset"); + datastore.start("path/to/emulator", "projectId"); } @Test - public void testLifecycle() throws LocalDevelopmentDatastoreException { - LocalDevelopmentDatastore datastore = new LocalDevelopmentDatastore(null, "blar", options) { - @Override - void startDatastoreInternal(String sdkPath, String dataset, String... cmdLineOpts) { - // no-op for testing - } + public void testLifecycle() throws DatastoreEmulatorException { + DatastoreEmulator datastore = + new DatastoreEmulator(null, "blar", options) { + @Override + void startEmulatorInternal(String emulatorDir, String projectId, String... cmdLineOpts) { + // no-op for testing + } - @Override - protected void stopDatastoreInternal() { - // no-op for testing - } - }; + @Override + protected void stopEmulatorInternal() { + // no-op for testing + } + }; - String sdkPath = "/yar"; - String myApp = "myapp"; + String emulatorDir = "/yar"; + String myProject = "myproject"; - datastore.start(sdkPath, myApp); + datastore.start(emulatorDir, myProject); try { - datastore.start(sdkPath, myApp); + datastore.start(emulatorDir, myProject); fail("expected exception"); } catch (IllegalStateException e) { // good } datastore.stop(); - // it's ok to stop if we've already stopped + // It's ok to stop if we've already stopped. datastore.stop(); - // once we've stopped we can't start again + // Once we've stopped we can't start again. try { - datastore.start(sdkPath, myApp); + datastore.start(emulatorDir, myProject); fail("expected exception"); } catch (IllegalStateException e) { // good From c77c3cd27c7c65b424ee3dc8fa1c390230cc37ca Mon Sep 17 00:00:00 2001 From: Ed Davisson Date: Tue, 10 Jan 2017 12:41:45 -0800 Subject: [PATCH 22/54] Add a host() option to DatastoreOptions --- .../datastore/v1/client/DatastoreFactory.java | 5 +- .../datastore/v1/client/DatastoreOptions.java | 51 ++++++++++++++----- .../datastore/v1/client/DatastoreTest.java | 49 +++++++++++++++++- 3 files changed, 91 insertions(+), 14 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java index 809f8beab..8fe5ad7f2 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java @@ -97,7 +97,10 @@ String buildProjectEndpoint(DatastoreOptions options) { } // DatastoreOptions ensures either project endpoint or project ID is set. String projectId = checkNotNull(options.getProjectId()); - if (options.getLocalHost() != null) { + if (options.getHost() != null) { + return validateUrl(String.format("https://%s/%s/projects/%s", + options.getHost(), VERSION, projectId)); + } else if (options.getLocalHost() != null) { return validateUrl(String.format("http://%s/%s/projects/%s", options.getLocalHost(), VERSION, projectId)); } diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java index ac76133c9..8f3523a95 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java @@ -43,6 +43,7 @@ public class DatastoreOptions { private final String projectId; private final String projectEndpoint; + private final String host; private final String localHost; private final HttpRequestInitializer initializer; @@ -57,6 +58,7 @@ public class DatastoreOptions { "Either project ID or project endpoint must be provided."); this.projectId = b.projectId; this.projectEndpoint = b.projectEndpoint; + this.host = b.host; this.localHost = b.localHost; this.initializer = b.initializer; this.credential = b.credential; @@ -67,13 +69,14 @@ public class DatastoreOptions { * Builder for {@link DatastoreOptions}. */ public static class Builder { - private static final String PROJECT_ENDPOINT_AND_PROJECT_ID_ERROR = + private static final String PROJECT_ENDPOINT_AND_PROJECT_ID_ERROR = "Cannot set both project endpoint and project ID."; - private static final String PROJECT_ENDPOINT_AND_LOCAL_HOST_ERROR = - "Cannot set both project endpoint and local host."; + private static final String PROJECT_ENDPOINT_AND_HOST_ERROR = + "Can set at most one of project endpoint, host, and local host."; private String projectId; private String projectEndpoint; + private String host; private String localHost; private HttpRequestInitializer initializer; private Credential credential; @@ -84,6 +87,7 @@ public Builder() { } public Builder(DatastoreOptions options) { this.projectId = options.projectId; this.projectEndpoint = options.projectEndpoint; + this.host = options.host; this.localHost = options.localHost; this.initializer = options.initializer; this.credential = options.credential; @@ -95,7 +99,7 @@ public DatastoreOptions build() { } /** - * Sets the project ID used to access Datastore. + * Sets the project ID used to access Cloud Datastore. */ public Builder projectId(String projectId) { checkArgument(projectEndpoint == null, PROJECT_ENDPOINT_AND_PROJECT_ID_ERROR); @@ -104,10 +108,26 @@ public Builder projectId(String projectId) { } /** - * Sets the host used to access Datastore. + * Sets the host used to access Cloud Datastore. To connect to the Cloud Datastore Emulator, + * use {@link #localHost} instead. + */ + public Builder host(String host) { + checkArgument(projectEndpoint == null && localHost == null, PROJECT_ENDPOINT_AND_HOST_ERROR); + if (includesScheme(host)) { + throw new IllegalArgumentException( + String.format("Host \"%s\" must not include scheme.", host)); + } + this.host = host; + return this; + } + + /** + * Configures the client to access Cloud Datastore on a local host (typically a Cloud Datastore + * Emulator instance). Call this method also configures the client not to attach credentials + * to requests. */ public Builder localHost(String localHost) { - checkArgument(projectEndpoint == null, PROJECT_ENDPOINT_AND_LOCAL_HOST_ERROR); + checkArgument(projectEndpoint == null && host == null, PROJECT_ENDPOINT_AND_HOST_ERROR); if (includesScheme(localHost)) { throw new IllegalArgumentException( String.format("Local host \"%s\" must not include scheme.", localHost)); @@ -117,12 +137,15 @@ public Builder localHost(String localHost) { } /** - * Sets the project endpoint used to access Datastore. Prefer using {@link #projectId} - * and/or {@link #localHost} when possible. + * @deprecated Use {@link #projectId} and/or {@link #host}/{@link #localHost} instead. + * + * Sets the project endpoint used to access Cloud Datastore. Prefer using {@link #projectId} + * and/or {@link #host}/{@link #localHost} when possible. */ + @Deprecated public Builder projectEndpoint(String projectEndpoint) { checkArgument(projectId == null, PROJECT_ENDPOINT_AND_PROJECT_ID_ERROR); - checkArgument(localHost == null, PROJECT_ENDPOINT_AND_LOCAL_HOST_ERROR); + checkArgument(localHost == null && host == null, PROJECT_ENDPOINT_AND_HOST_ERROR); if (!includesScheme(projectEndpoint)) { throw new IllegalArgumentException(String.format( "Project endpoint \"%s\" must include scheme.", projectEndpoint)); @@ -132,7 +155,7 @@ public Builder projectEndpoint(String projectEndpoint) { } /** - * Sets the (optional) initializer to run on HTTP requests to Datastore. + * Sets the (optional) initializer to run on HTTP requests to Cloud Datastore. */ public Builder initializer(HttpRequestInitializer initializer) { this.initializer = initializer; @@ -140,7 +163,7 @@ public Builder initializer(HttpRequestInitializer initializer) { } /** - * Sets the Google APIs {@link Credential} used to access Datastore. + * Sets the Google APIs {@link Credential} used to access Cloud Datastore. */ public Builder credential(Credential credential) { this.credential = credential; @@ -148,7 +171,7 @@ public Builder credential(Credential credential) { } /** - * Sets the transport used to access Datastore. + * Sets the transport used to access Cloud Datastore. */ public Builder transport(HttpTransport transport) { this.transport = transport; @@ -168,6 +191,10 @@ public String getProjectEndpoint() { return projectEndpoint; } + public String getHost() { + return host; + } + public String getLocalHost() { return localHost; } diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java index bcdbe18de..4454be66c 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java @@ -102,12 +102,30 @@ public void options_ProjectIdAndProjectEndpoint() throws Exception { @Test public void options_LocalHostAndProjectEndpoint() throws Exception { thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("Cannot set both project endpoint and local host"); + thrown.expectMessage("Can set at most one of project endpoint, host, and local host"); options = new DatastoreOptions.Builder() .localHost("localhost:8080") .projectEndpoint("http://localhost:1234/datastore/v1beta42/projects/project-id"); } + @Test + public void options_HostAndProjectEndpoint() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Can set at most one of project endpoint, host, and local host"); + options = new DatastoreOptions.Builder() + .host("foo-datastore.googleapis.com") + .projectEndpoint("http://localhost:1234/datastore/v1beta42/projects/project-id"); + } + + @Test + public void options_HostAndLocalHost() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Can set at most one of project endpoint, host, and local host"); + options = new DatastoreOptions.Builder() + .host("foo-datastore.googleapis.com") + .localHost("localhost:8080"); + } + @Test public void options_InvalidLocalHost() throws Exception { thrown.expect(IllegalArgumentException.class); @@ -126,12 +144,41 @@ public void options_SchemeInLocalHost() { .localHost("http://localhost:8080"); } + @Test + public void options_InvalidHost() throws Exception { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("Illegal character"); + factory.create(new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .host("!not a valid url!") + .build()); + } + + @Test + public void options_SchemeInHost() { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage( + "Host \"http://foo-datastore.googleapis.com\" must not include scheme"); + new DatastoreOptions.Builder() + .host("http://foo-datastore.googleapis.com"); + } + @Test public void create_NullOptions() throws Exception { thrown.expect(NullPointerException.class); factory.create(null); } + @Test + public void create_Host() { + Datastore datastore = factory.create(new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .host("foo-datastore.googleapis.com") + .build()); + assertThat(datastore.remoteRpc.getUrl()) + .isEqualTo("https://foo-datastore.googleapis.com/v1/projects/project-id"); + } + @Test public void create_LocalHost() { Datastore datastore = factory.create(new DatastoreOptions.Builder() From 6f1ffea6a36bb5b4699440f5182ca7fc333965b3 Mon Sep 17 00:00:00 2001 From: Will Hayworth Date: Mon, 9 Oct 2017 17:23:47 -0700 Subject: [PATCH 23/54] Sync from CL 170707635. --- .../google/datastore/v1/client/Datastore.java | 3 +- .../v1/client/DatastoreEmulator.java | 104 ++++---- .../v1/client/DatastoreEmulatorOptions.java | 53 +++- .../datastore/v1/client/DatastoreFactory.java | 1 + .../datastore/v1/client/DatastoreOptions.java | 2 - .../v1/client/QuerySplitterImpl.java | 6 +- .../google/datastore/v1/client/RemoteRpc.java | 110 +++++++- .../v1/client/DatastoreEmulatorTest.java | 7 +- .../datastore/v1/client/DatastoreTest.java | 30 ++- .../datastore/v1/client/RemoteRpcTest.java | 243 +++++++++++++++++- 10 files changed, 485 insertions(+), 74 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java index e753dc331..e10598770 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java @@ -28,12 +28,13 @@ import com.google.datastore.v1.RunQueryRequest; import com.google.datastore.v1.RunQueryResponse; import com.google.rpc.Code; - import java.io.IOException; import java.io.InputStream; /** * Provides access to Cloud Datastore. + * + *

This class is thread-safe. */ public class Datastore { diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java index 2dc80fe06..a6c656df5 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java @@ -18,18 +18,16 @@ import static com.google.api.client.util.Preconditions.checkNotNull; import static com.google.api.client.util.Preconditions.checkState; -import com.google.api.client.http.GenericUrl; -import com.google.api.client.http.HttpRequestFactory; -import com.google.api.client.http.HttpResponse; import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -98,17 +96,7 @@ enum State {NEW, STARTED, STOPPED} * @throws DatastoreEmulatorException if the clear fails */ public void clear() throws DatastoreEmulatorException { - HttpRequestFactory client = remoteRpc.getHttpRequestFactory(); - try { - GenericUrl url = new GenericUrl(host + "/reset"); - HttpResponse httpResponse = client.buildPostRequest(url, null).execute(); - if (!httpResponse.isSuccessStatusCode()) { - throw new DatastoreEmulatorException( - "POST request to /reset returned HTTP status " + httpResponse.getStatusCode()); - } - } catch (IOException e) { - throw new DatastoreEmulatorException("Exception trying to clear the emulator", e); - } + sendEmptyRequest("/reset", "POST"); } /** @@ -121,14 +109,31 @@ public void clear() throws DatastoreEmulatorException { * @param commandLineOptions Command line options to pass to the emulator on startup * @throws DatastoreEmulatorException If {@link #start} has already been called or the server does * not start successfully. + * @deprecated prefer setting options in the emulator options and calling {#start()}. */ + @Deprecated public synchronized void start(String emulatorDir, String projectId, String... commandLineOptions) throws DatastoreEmulatorException { checkNotNull(emulatorDir, "emulatorDir cannot be null"); checkNotNull(projectId, "projectId cannot be null"); checkState(state == State.NEW, "Cannot call start() more than once."); try { - startEmulatorInternal(emulatorDir, projectId, commandLineOptions); + startEmulatorInternal( + emulatorDir + "/cloud_datastore_emulator", projectId, Arrays.asList(commandLineOptions)); + state = State.STARTED; + } finally { + if (state != State.STARTED) { + // If we're not able to start the server we don't want people trying again. Just move it + // straight to the STOPPED state. + state = State.STOPPED; + } + } + } + + public synchronized void start() throws DatastoreEmulatorException { + checkState(state == State.NEW, "Cannot call start() more than once."); + try { + startEmulatorInternal(options.getCmd(), options.getProjectId(), options.getCmdLineOptions()); state = State.STARTED; } finally { if (state != State.STARTED) { @@ -139,16 +144,15 @@ public synchronized void start(String emulatorDir, String projectId, String... c } } - void startEmulatorInternal(String emulatorDir, String projectId, String... commandLineOptions) + void startEmulatorInternal(String emulatorCmd, String projectId, List commandLineOptions) throws DatastoreEmulatorException { - projectDirectory = createProjectDirectory(emulatorDir, projectId); - List cmd = - new ArrayList<>(Arrays.asList("./cloud_datastore_emulator", "start", "--testing")); - Collections.addAll(cmd, commandLineOptions); + projectDirectory = createProjectDirectory(emulatorCmd, projectId); + List cmd = new ArrayList<>(Arrays.asList(emulatorCmd, "start", "--testing")); + cmd.addAll(commandLineOptions); cmd.add(projectDirectory.getPath()); final Process emulatorStartProcess; try { - emulatorStartProcess = newEmulatorProcess(emulatorDir, cmd).start(); + emulatorStartProcess = newEmulatorProcess(cmd).start(); } catch (IOException e) { throw new DatastoreEmulatorException("Could not start emulator", e); } @@ -176,7 +180,7 @@ public void run() { } } - private File createProjectDirectory(String emulatorDir, String projectId) + private File createProjectDirectory(String emulatorCmd, String projectId) throws DatastoreEmulatorException { File projectDirectory; try { @@ -186,12 +190,9 @@ private File createProjectDirectory(String emulatorDir, String projectId) } List cmd = Arrays.asList( - "./cloud_datastore_emulator", - "create", - "--project_id=" + projectId, - projectDirectory.getPath()); + emulatorCmd, "create", "--project_id=" + projectId, projectDirectory.getPath()); try { - int retCode = newEmulatorProcess(emulatorDir, cmd).start().waitFor(); + int retCode = newEmulatorProcess(cmd).start().waitFor(); if (retCode != 0) { throw new DatastoreEmulatorException( String.format("Could not create project (retcode=%d)", retCode)); @@ -205,9 +206,8 @@ private File createProjectDirectory(String emulatorDir, String projectId) return projectDirectory; } - private ProcessBuilder newEmulatorProcess(String emulatorDir, List cmd) { + private ProcessBuilder newEmulatorProcess(List cmd) { ProcessBuilder builder = new ProcessBuilder(cmd); - builder.directory(new File(emulatorDir)); builder.redirectErrorStream(true); builder.environment().putAll(options.getEnvVars()); return builder; @@ -242,26 +242,15 @@ public synchronized void stop() throws DatastoreEmulatorException { } } + protected void stopEmulatorInternal() throws DatastoreEmulatorException { + sendEmptyRequest("/shutdown", "POST"); + } + public synchronized File getProjectDirectory() { checkState(state == State.STARTED); return projectDirectory; } - void stopEmulatorInternal() throws DatastoreEmulatorException { - // No need to kill the process we started, this function will take care of it. - HttpRequestFactory client = remoteRpc.getHttpRequestFactory(); - GenericUrl url = new GenericUrl(host + "/shutdown"); - try { - HttpResponse httpResponse = client.buildPostRequest(url, null).execute(); - if (!httpResponse.isSuccessStatusCode()) { - throw new DatastoreEmulatorException( - "POST request to /shutdown returned HTTP status " + httpResponse.getStatusCode()); - } - } catch (IOException e) { - throw new DatastoreEmulatorException("Exception trying to stop the emulator", e); - } - } - /** * Monitors the provided input stream for evidence that the emulator has started successfully and * redirects the output of the emulator process to sysout for this process. @@ -314,4 +303,29 @@ public void run() { } } } + + /** Send an empty request using a standard HTTP connection. */ + private void sendEmptyRequest(String path, String method) throws DatastoreEmulatorException { + HttpURLConnection connection = null; + try { + URL url = new URL(this.host + path); + connection = (HttpURLConnection) url.openConnection(); + connection.setDoOutput(true); + connection.setRequestMethod(method); + connection.getOutputStream().close(); + if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { + throw new DatastoreEmulatorException( + String.format( + "%s request to %s returned HTTP status %s", + method, path, connection.getResponseCode())); + } + } catch (IOException e) { + throw new DatastoreEmulatorException( + String.format("Exception connecting to emulator on %s request to %s", method, path), e); + } finally { + if (connection != null) { + connection.disconnect(); + } + } + } } diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorOptions.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorOptions.java index 446b60cec..83348319b 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulatorOptions.java @@ -15,23 +15,37 @@ */ package com.google.datastore.v1.client; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; +import javax.annotation.Nullable; /** An immutable object containing settings for a {@link DatastoreEmulator}. */ public class DatastoreEmulatorOptions { private final Map envVars; + private final String cmd; + private final List cmdLineOptions; + private final String projectId; - DatastoreEmulatorOptions(Map envVars) { + DatastoreEmulatorOptions( + Map envVars, String cmd, List cmdLineOptions, String projectId) { this.envVars = envVars; + this.cmd = cmd; + this.cmdLineOptions = cmdLineOptions; + this.projectId = projectId; } /** Builder for {@link DatastoreEmulatorOptions}. */ public static class Builder { - private final Map envVars = new HashMap(); + private final Map envVars = new HashMap<>(); + private final List cmdLineOptions = new ArrayList<>(); + private String cmd = "./cloud_datastore_emulator"; + private String projectId; public DatastoreEmulatorOptions build() { - return new DatastoreEmulatorOptions(envVars); + return new DatastoreEmulatorOptions(envVars, cmd, cmdLineOptions, projectId); } /** Adds an environment variable to pass to the emulator. */ @@ -39,9 +53,42 @@ public Builder addEnvVar(String var, String value) { envVars.put(var, value); return this; } + + public Builder addCmdLineOption(String option) { + cmdLineOptions.add(option); + return this; + } + + public Builder addCmdLineOptions(Collection options) { + cmdLineOptions.addAll(options); + return this; + } + + public Builder setCommand(String cmd) { + this.cmd = cmd; + return this; + } + + public Builder setProjectId(String projectId) { + this.projectId = projectId; + return this; + } } public Map getEnvVars() { return envVars; } + + public List getCmdLineOptions() { + return cmdLineOptions; + } + + public String getCmd() { + return cmd; + } + + @Nullable + public String getProjectId() { + return projectId; + } } diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java index 8fe5ad7f2..acbea4f16 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java @@ -73,6 +73,7 @@ public HttpRequestFactory makeClient(DatastoreOptions options) { HttpTransport transport = options.getTransport(); if (transport == null) { transport = credential == null ? new NetHttpTransport() : credential.getTransport(); + transport = transport == null ? new NetHttpTransport() : transport; } return transport.createRequestFactory(credential); } diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java index 8f3523a95..4ccde164f 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java @@ -16,12 +16,10 @@ package com.google.datastore.v1.client; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; - import java.util.Arrays; import java.util.List; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java index c10ef0563..91c7fb4b2 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java @@ -61,10 +61,14 @@ public List getSplits( Query query, PartitionId partition, int numSplits, Datastore datastore) throws DatastoreException, IllegalArgumentException { + List splits = new ArrayList(numSplits); + if (numSplits == 1) { + splits.add(query); + return splits; + } validateQuery(query); validateSplitSize(numSplits); - List splits = new ArrayList(numSplits); List scatterKeys = getScatterKeys(numSplits, query, partition, datastore); Key lastKey = null; for (Key nextKey : getSplitKey(scatterKeys, numSplits)) { diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java index 386775c54..e01383b66 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java @@ -20,25 +20,118 @@ import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpStatusCodes; import com.google.api.client.http.protobuf.ProtoHttpContent; import com.google.api.client.util.IOUtils; +import com.google.common.base.Preconditions; import com.google.protobuf.MessageLite; import com.google.rpc.Code; import com.google.rpc.Status; - import java.io.ByteArrayOutputStream; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Field; +import java.net.SocketTimeoutException; import java.nio.charset.Charset; import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; import java.util.logging.Logger; +import java.util.zip.GZIPInputStream; /** * An RPC transport that sends protocol buffers over HTTP. + * + *

This class is thread-safe. */ class RemoteRpc { private static final Logger logger = Logger.getLogger(RemoteRpc.class.getName()); + /** + * An input stream that wraps a {@link GZIPInputStream} and forces it to fully consume its + * underlying {@link InputStream} before calling close() on it. + * + * @see href="https://github.com/google/google-http-java-client/issues/367" + */ + static class GzipFixingInputStream extends InputStream { + + // How many extra read() calls we have made on the underlying stream. + int callsToRead = 0; + + // Experimentally, it is only necessary to consume 1 extra byte. Consuming a bit more than that + // should not affect performance, but we set an upper bound for safety. + private static final int MAX_BYTES_TO_CONSUME = 100; + + private static final Field gzipUnderlyingInputStreamField = getGzipUnderlyingInputStreamField(); + + private static Field getGzipUnderlyingInputStreamField() { + try { + // FilterInputStream is a superclass of GZIPInputStream and stores the underlying + // InputStream. + Field gzipInputStreamIsField = FilterInputStream.class.getDeclaredField("in"); + gzipInputStreamIsField.setAccessible(true); + return gzipInputStreamIsField; + } catch (Exception e) { + logger.log( + Level.INFO, + "Failed to find field \"in\" in FilterInputStream. This" + + " may prevent keep-alive from working correctly.", + e); + return null; + } + } + + public static InputStream maybeWrap(InputStream inputStream) { + if (gzipUnderlyingInputStreamField != null && inputStream instanceof GZIPInputStream) { + return new GzipFixingInputStream((GZIPInputStream) inputStream); + } + return inputStream; + } + + private final GZIPInputStream gzipInputStream; + + private GzipFixingInputStream(GZIPInputStream gzipInputStream) { + Preconditions.checkNotNull(gzipUnderlyingInputStreamField); + this.gzipInputStream = gzipInputStream; + } + + @Override + public int read() throws IOException { + return gzipInputStream.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return gzipInputStream.read(b, off, len); + } + + @Override + public void close() throws IOException { + // If possible, finish consuming the underlying InputStream before closing it. + if (gzipUnderlyingInputStreamField != null) { + try { + InputStream underlyingInputStream = + (InputStream) gzipUnderlyingInputStreamField.get(gzipInputStream); + boolean reachedEndOfStream = false; + while (!reachedEndOfStream && callsToRead < MAX_BYTES_TO_CONSUME) { + callsToRead++; + if (underlyingInputStream.read() == -1) { + reachedEndOfStream = true; + } + } + if (!reachedEndOfStream) { + logger.log(Level.FINER, "Gave up consuming underlying InputStream"); + } + } catch (Exception e) { + // If this fails for any reason, log and move on. + logger.log( + Level.FINER, "Failed to consume underlying InputStream from GZIPInputStream", e); + } + } + gzipInputStream.close(); + } + } + private static final String API_FORMAT_VERSION_HEADER = "X-Goog-Api-Format-Version"; private static final String API_FORMAT_VERSION = "2"; @@ -81,6 +174,9 @@ public InputStream call(String methodName, MessageLite request) throws Datastore // Don't throw an HTTPResponseException on error. It converts the response to a String and // throws away the original, whereas we need the raw bytes to parse it as a proto. httpRequest.setThrowExceptionOnExecuteError(false); + // Datastore requests typically time out after 60s; set the read timeout to slightly longer + // than that by default (can be overridden via the HttpRequestInitializer). + httpRequest.setReadTimeout(65 * 1000); if (initializer != null) { initializer.initialize(httpRequest); } @@ -90,7 +186,9 @@ public InputStream call(String methodName, MessageLite request) throws Datastore httpResponse.getContentType(), httpResponse.getContentCharset(), null, httpResponse.getStatusCode()); } - return httpResponse.getContent(); + return GzipFixingInputStream.maybeWrap(httpResponse.getContent()); + } catch (SocketTimeoutException e) { + throw makeException(url, methodName, Code.DEADLINE_EXCEEDED, "Deadline exceeded", e); } catch (IOException e) { throw makeException(url, methodName, Code.UNAVAILABLE, "I/O error", e); } @@ -119,7 +217,7 @@ GenericUrl resolveURL(String path) { HttpRequestFactory getHttpRequestFactory() { return client; } - + public static DatastoreException makeException(String url, String methodName, Code code, String message, Throwable cause) { logger.fine("remote datastore call " + methodName + " against " + url + " failed: " + message); @@ -160,6 +258,12 @@ static DatastoreException makeException(String url, String methodName, InputStre "Invalid error code: %d. Message: %s.", rpcStatus.getCode(), rpcStatus.getMessage()), cause); } else if (code == Code.OK) { + // We can end up here because there was no response body (and we successfully parsed an + // empty Status message). This may happen for 401s in particular due to special handling + // in low-level HTTP libraries. + if (httpStatusCode == HttpStatusCodes.STATUS_CODE_UNAUTHORIZED) { + return makeException(url, methodName, Code.UNAUTHENTICATED, "Unauthenticated.", cause); + } return makeException(url, methodName, Code.INTERNAL, String.format("Unexpected OK error code with HTTP status code of %d. Message: %s.", httpStatusCode, rpcStatus.getMessage()), diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreEmulatorTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreEmulatorTest.java index fbe573b76..477bd6a50 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreEmulatorTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreEmulatorTest.java @@ -17,6 +17,7 @@ import static org.junit.Assert.fail; +import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -33,7 +34,8 @@ public void testArgs() throws DatastoreEmulatorException { DatastoreEmulator datastore = new DatastoreEmulator(null, "blar", options) { @Override - void startEmulatorInternal(String emulatorDir, String projectId, String... cmdLineOpts) { + void startEmulatorInternal( + String emulatorDir, String projectId, List cmdLineOpts) { // no-op for testing } }; @@ -60,7 +62,8 @@ public void testLifecycle() throws DatastoreEmulatorException { DatastoreEmulator datastore = new DatastoreEmulator(null, "blar", options) { @Override - void startEmulatorInternal(String emulatorDir, String projectId, String... cmdLineOpts) { + void startEmulatorInternal( + String emulatorDir, String projectId, List cmdLineOpts) { // no-op for testing } diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java index 4454be66c..6a48d2580 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java @@ -54,18 +54,17 @@ import com.google.protobuf.Message; import com.google.rpc.Code; import com.google.rpc.Status; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.net.SocketTimeoutException; import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; /** * Tests for {@link DatastoreFactory} and {@link Datastore}. @@ -325,7 +324,20 @@ private void expectRpc(String methodName, Message request, Message response) thr assertEquals("oops", exception.getMessage()); } - IOException ioException = new IOException("non"); + SocketTimeoutException socketTimeoutException = new SocketTimeoutException("ste"); + mockClient.setNextException(socketTimeoutException); + try { + call.invoke(datastore, callArgs); + fail(); + } catch (InvocationTargetException targetException) { + DatastoreException exception = (DatastoreException) targetException.getCause(); + assertEquals(Code.DEADLINE_EXCEEDED, exception.getCode()); + assertEquals(methodName, exception.getMethodName()); + assertEquals("Deadline exceeded", exception.getMessage()); + assertSame(socketTimeoutException, exception.getCause()); + } + + IOException ioException = new IOException("ioe"); mockClient.setNextException(ioException); try { call.invoke(datastore, callArgs); @@ -338,7 +350,7 @@ private void expectRpc(String methodName, Message request, Message response) thr assertSame(ioException, exception.getCause()); } - assertEquals(2, datastore.getRpcCount()); + assertEquals(3, datastore.getRpcCount()); } private static class MockCredential extends Credential { diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java index bceee04b3..66b26c6e5 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java @@ -16,17 +16,28 @@ package com.google.datastore.v1.client; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; import com.google.api.client.util.Charsets; +import com.google.datastore.v1.BeginTransactionResponse; +import com.google.datastore.v1.client.RemoteRpc.GzipFixingInputStream; +import com.google.protobuf.ByteString; import com.google.rpc.Code; import com.google.rpc.Status; - +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.GZIPOutputStream; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import java.io.ByteArrayInputStream; - /** * Test for {@link RemoteRpc}. */ @@ -61,19 +72,43 @@ public void testInvalidProtoException() { exception.getMessage()); assertEquals(METHOD_NAME, exception.getMethodName()); } - + @Test public void testEmptyProtoException() { Status statusProto = Status.newBuilder().build(); - DatastoreException exception = RemoteRpc.makeException("url", METHOD_NAME, - new ByteArrayInputStream(statusProto.toByteArray()), "application/x-protobuf", - Charsets.UTF_8, new RuntimeException(), 401); + DatastoreException exception = + RemoteRpc.makeException( + "url", + METHOD_NAME, + new ByteArrayInputStream(statusProto.toByteArray()), + "application/x-protobuf", + Charsets.UTF_8, + new RuntimeException(), + 404); assertEquals(Code.INTERNAL, exception.getCode()); - assertEquals("Unexpected OK error code with HTTP status code of 401. Message: .", + assertEquals( + "Unexpected OK error code with HTTP status code of 404. Message: .", exception.getMessage()); assertEquals(METHOD_NAME, exception.getMethodName()); } + @Test + public void testEmptyProtoExceptionUnauthenticated() { + Status statusProto = Status.newBuilder().build(); + DatastoreException exception = + RemoteRpc.makeException( + "url", + METHOD_NAME, + new ByteArrayInputStream(statusProto.toByteArray()), + "application/x-protobuf", + Charsets.UTF_8, + new RuntimeException(), + 401); + assertEquals(Code.UNAUTHENTICATED, exception.getCode()); + assertEquals("Unauthenticated.", exception.getMessage()); + assertEquals(METHOD_NAME, exception.getMethodName()); + } + @Test public void testPlainTextException() { DatastoreException exception = RemoteRpc.makeException("url", METHOD_NAME, @@ -84,4 +119,196 @@ public void testPlainTextException() { "Non-protobuf error: Text Error. HTTP status code was 401.", exception.getMessage()); assertEquals(METHOD_NAME, exception.getMethodName()); } + + @Test + public void testGzipHack_NonGzip() throws Exception { + BeginTransactionResponse resp = newBeginTransactionResp(); + InjectedTestValues injectedTestValues = + new InjectedTestValues(resp.toByteArray(), new byte[0], false); + RemoteRpc rpc = newRemoteRpc(injectedTestValues); + + InputStream is = rpc.call("beginTransaction", BeginTransactionResponse.getDefaultInstance()); + BeginTransactionResponse parsedResp = BeginTransactionResponse.parseFrom(is); + is.close(); + + assertEquals(resp, parsedResp); + assertFalse(is instanceof GzipFixingInputStream); + } + + @Test + public void testGzipHack_Gzip() throws Exception { + BeginTransactionResponse resp = newBeginTransactionResp(); + InjectedTestValues injectedTestValues = new InjectedTestValues(gzip(resp), new byte[1], true); + RemoteRpc rpc = newRemoteRpc(injectedTestValues); + + InputStream is = rpc.call("beginTransaction", BeginTransactionResponse.getDefaultInstance()); + BeginTransactionResponse parsedResp = BeginTransactionResponse.parseFrom(is); + is.close(); + + assertEquals(resp, parsedResp); + assertTrue(is instanceof GzipFixingInputStream); + assertEquals(1, ((GzipFixingInputStream) is).callsToRead); + // Check that the underlying stream is exhausted. + assertEquals(-1, injectedTestValues.inputStream.read()); + } + + @Test + public void testGzipHack_GzipTooManyExtraBytes() throws Exception { + BeginTransactionResponse resp = newBeginTransactionResp(); + // NOTE(eddavisson): We might expect 101 extra bytes to be enough that the underlying input + // stream is not exhausted, but this is not the case (likely due to a buffer somewhere). 1000 + // extra bytes seems to be enough. We check the value of callsToRead directly to make sure + // we eventually stopped trying to consume the underlying stream. + InjectedTestValues injectedTestValues = + new InjectedTestValues(gzip(resp), new byte[1000], true); + RemoteRpc rpc = newRemoteRpc(injectedTestValues); + + InputStream is = rpc.call("beginTransaction", BeginTransactionResponse.getDefaultInstance()); + BeginTransactionResponse parsedResp = BeginTransactionResponse.parseFrom(is); + is.close(); + + assertEquals(resp, parsedResp); + assertTrue(is instanceof GzipFixingInputStream); + assertEquals(100, ((GzipFixingInputStream) is).callsToRead); + // Check that the underlying stream is _not_ exhausted. + assertNotEquals(-1, injectedTestValues.inputStream.read()); + } + + private static BeginTransactionResponse newBeginTransactionResp() { + return BeginTransactionResponse.newBuilder() + .setTransaction(ByteString.copyFromUtf8("blah-blah-blah")) + .build(); + } + + private static RemoteRpc newRemoteRpc(InjectedTestValues injectedTestValues) { + return new RemoteRpc( + new MyHttpTransport(injectedTestValues).createRequestFactory(), + null, + "https://www.example.com/v1/projects/p"); + } + + private byte[] gzip(BeginTransactionResponse resp) throws IOException { + ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); + try (GZIPOutputStream gzipOut = new GZIPOutputStream(bytesOut)) { + resp.writeTo(gzipOut); + } + return bytesOut.toByteArray(); + } + + private static class InjectedTestValues { + private final InputStream inputStream; + private final int contentLength; + private final boolean isGzip; + + public InjectedTestValues(byte[] messageBytes, byte[] additionalBytes, boolean isGzip) { + byte[] allBytes = concat(messageBytes, additionalBytes); + this.inputStream = new ByteArrayInputStream(allBytes); + this.contentLength = allBytes.length; + this.isGzip = isGzip; + } + + private static byte[] concat(byte[] a, byte[] b) { + byte[] c = new byte[a.length + b.length]; + System.arraycopy(a, 0, c, 0, a.length); + System.arraycopy(b, 0, c, a.length, b.length); + return c; + } + } + + /** {@link HttpTransport} that allows injection of the returned {@link LowLevelHttpRequest}. */ + private static class MyHttpTransport extends HttpTransport { + + private final InjectedTestValues injectedTestValues; + + public MyHttpTransport(InjectedTestValues injectedTestValues) { + this.injectedTestValues = injectedTestValues; + } + + @Override + protected LowLevelHttpRequest buildRequest(String method, String url) throws IOException { + return new MyLowLevelHttpRequest(injectedTestValues); + } + } + + /** + * {@link LowLevelHttpRequest} that allows injection of the returned {@link LowLevelHttpResponse}. + */ + private static class MyLowLevelHttpRequest extends LowLevelHttpRequest { + + private final InjectedTestValues injectedTestValues; + + public MyLowLevelHttpRequest(InjectedTestValues injectedTestValues) { + this.injectedTestValues = injectedTestValues; + } + + @Override + public void addHeader(String name, String value) throws IOException { + // Do nothing. + } + + @Override + public LowLevelHttpResponse execute() throws IOException { + return new MyLowLevelHttpResponse(injectedTestValues); + } + } + + /** {@link LowLevelHttpResponse} that allows injected properties. */ + private static class MyLowLevelHttpResponse extends LowLevelHttpResponse { + + private final InjectedTestValues injectedTestValues; + + public MyLowLevelHttpResponse(InjectedTestValues injectedTestValues) { + this.injectedTestValues = injectedTestValues; + } + + @Override + public InputStream getContent() throws IOException { + return injectedTestValues.inputStream; + } + + @Override + public String getContentEncoding() throws IOException { + return injectedTestValues.isGzip ? "gzip" : ""; + } + + @Override + public long getContentLength() throws IOException { + return injectedTestValues.contentLength; + } + + @Override + public String getContentType() throws IOException { + return "application/x-protobuf"; + } + + @Override + public String getStatusLine() throws IOException { + return null; + } + + @Override + public int getStatusCode() throws IOException { + return 200; + } + + @Override + public String getReasonPhrase() throws IOException { + return null; + } + + @Override + public int getHeaderCount() throws IOException { + return 0; + } + + @Override + public String getHeaderName(int index) throws IOException { + return null; + } + + @Override + public String getHeaderValue(int index) throws IOException { + return null; + } + } } From 3caba37fc546f293f1b63c6a5bf3ccbd894a30df Mon Sep 17 00:00:00 2001 From: wsh Date: Wed, 11 Oct 2017 18:53:08 -0700 Subject: [PATCH 24/54] Remove V1 ID reservation flag guarding in low-level Java proto client now that the feature has been released. Bump proto library version to 1.5. Bump proto client version to 1.5. Bump google-http-client version to 1.23. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=171906809 --- .../java/com/google/datastore/v1/client/Datastore.java | 10 ++++++++++ .../com/google/datastore/v1/client/DatastoreTest.java | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java index e10598770..eb33dc627 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java @@ -23,6 +23,8 @@ import com.google.datastore.v1.CommitResponse; import com.google.datastore.v1.LookupRequest; import com.google.datastore.v1.LookupResponse; +import com.google.datastore.v1.ReserveIdsRequest; +import com.google.datastore.v1.ReserveIdsResponse; import com.google.datastore.v1.RollbackRequest; import com.google.datastore.v1.RollbackResponse; import com.google.datastore.v1.RunQueryRequest; @@ -97,6 +99,14 @@ public LookupResponse lookup(LookupRequest request) throws DatastoreException { } } + public ReserveIdsResponse reserveIds(ReserveIdsRequest request) throws DatastoreException { + try (InputStream is = remoteRpc.call("reserveIds", request)) { + return ReserveIdsResponse.parseFrom(is); + } catch (IOException exception) { + throw invalidResponseException("reserveIds", exception); + } + } + public RollbackResponse rollback(RollbackRequest request) throws DatastoreException { try (InputStream is = remoteRpc.call("rollback", request)) { return RollbackResponse.parseFrom(is); diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java index 6a48d2580..617960bd3 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java @@ -46,6 +46,8 @@ import com.google.datastore.v1.LookupRequest; import com.google.datastore.v1.LookupResponse; import com.google.datastore.v1.QueryResultBatch; +import com.google.datastore.v1.ReserveIdsRequest; +import com.google.datastore.v1.ReserveIdsResponse; import com.google.datastore.v1.RollbackRequest; import com.google.datastore.v1.RollbackResponse; import com.google.datastore.v1.RunQueryRequest; @@ -274,6 +276,13 @@ public void commit() throws Exception { expectRpc("commit", request.build(), response.build()); } + @Test + public void reserveIds() throws Exception { + ReserveIdsRequest.Builder request = ReserveIdsRequest.newBuilder(); + ReserveIdsResponse.Builder response = ReserveIdsResponse.newBuilder(); + expectRpc("reserveIds", request.build(), response.build()); + } + @Test public void rollback() throws Exception { RollbackRequest.Builder request = RollbackRequest.newBuilder(); From b48b0915bb75cc8cc497f763686e368782ed87f2 Mon Sep 17 00:00:00 2001 From: Christian Maan Date: Wed, 25 Oct 2017 10:37:37 +0200 Subject: [PATCH 25/54] =?UTF-8?q?=F0=9F=9A=91=20fix=20http=20connection=20?= =?UTF-8?q?leak=20on=20Datastore=20errors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/google/datastore/v1/client/RemoteRpc.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java index e01383b66..f63440d31 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java @@ -182,9 +182,11 @@ public InputStream call(String methodName, MessageLite request) throws Datastore } httpResponse = httpRequest.execute(); if (!httpResponse.isSuccessStatusCode()) { - throw makeException(url, methodName, httpResponse.getContent(), - httpResponse.getContentType(), httpResponse.getContentCharset(), null, - httpResponse.getStatusCode()); + try (InputStream content = httpResponse.getContent()) { + throw makeException(url, methodName, content, + httpResponse.getContentType(), httpResponse.getContentCharset(), null, + httpResponse.getStatusCode()); + } } return GzipFixingInputStream.maybeWrap(httpResponse.getContent()); } catch (SocketTimeoutException e) { @@ -230,7 +232,7 @@ static DatastoreException makeException(String url, String methodName, InputStre String responseContent; try { ByteArrayOutputStream out = new ByteArrayOutputStream(); - IOUtils.copy(content, out); + IOUtils.copy(content, out, false); responseContent = out.toString(contentCharset.name()); } catch (IOException e) { responseContent = ""; From cff0e2e311e2217c86e1daab5f5921c69fe0da4a Mon Sep 17 00:00:00 2001 From: eddavisson Date: Thu, 14 Dec 2017 14:14:15 -0800 Subject: [PATCH 26/54] Also use GzipFixingInputStream for error response content. This will allow connection reuse on errors where there was an actual response but the HTTP response code was not 200. This applies to cases where ESF returned an error or forwarded one from the backend (e.g. validation errors, contention, etc.). ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=179095258 --- .../src/main/java/com/google/datastore/v1/client/RemoteRpc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java index f63440d31..3576c02e0 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java @@ -182,7 +182,7 @@ public InputStream call(String methodName, MessageLite request) throws Datastore } httpResponse = httpRequest.execute(); if (!httpResponse.isSuccessStatusCode()) { - try (InputStream content = httpResponse.getContent()) { + try (InputStream content = GzipFixingInputStream.maybeWrap(httpResponse.getContent())) { throw makeException(url, methodName, content, httpResponse.getContentType(), httpResponse.getContentCharset(), null, httpResponse.getStatusCode()); From 09ce8dfce2da4417c807d1667201309565f9b67c Mon Sep 17 00:00:00 2001 From: wsh Date: Tue, 2 Jan 2018 13:20:39 -0800 Subject: [PATCH 27/54] Cache project ID information retrieved from Compute Engine. Note: this uses an AtomicReference to avoid synchronized static methods (and the class-level lock they entail). ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=180584318 --- .../datastore/v1/client/DatastoreHelper.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java index 6b2c042be..79132cce7 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java @@ -41,7 +41,6 @@ import com.google.protobuf.Timestamp; import com.google.protobuf.TimestampOrBuilder; import com.google.type.LatLng; - import java.io.File; import java.io.IOException; import java.security.GeneralSecurityException; @@ -52,8 +51,10 @@ import java.util.Date; import java.util.Iterator; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; +import javax.annotation.Nullable; /** * Helper methods for {@link Datastore}. @@ -93,6 +94,8 @@ public final class DatastoreHelper { private static final String URL_OVERRIDE_ENV_VAR = "__DATASTORE_URL_OVERRIDE"; + private static final AtomicReference projectIdFromComputeEngine = new AtomicReference<>(); + /** * Comparator for Keys */ @@ -281,20 +284,31 @@ private static String getProjectIdFromEnv() { * project ID cannot be determined (because, for instance, the code is not running on Compute * Engine). */ + @Nullable public static String getProjectIdFromComputeEngine() { + String cachedProjectId = projectIdFromComputeEngine.get(); + return cachedProjectId != null ? cachedProjectId : queryProjectIdFromComputeEngine(); + } + + @Nullable + private static String queryProjectIdFromComputeEngine() { HttpTransport transport; + try { transport = newTransport(); } catch (GeneralSecurityException | IOException e) { logger.log(Level.WARNING, "Failed to create HttpTransport.", e); return null; } + try { GenericUrl projectIdUrl = new GenericUrl("http://metadata/computeMetadata/v1/project/project-id"); HttpRequest request = transport.createRequestFactory().buildGetRequest(projectIdUrl); request.getHeaders().set("Metadata-Flavor", "Google"); - return request.execute().parseAsString(); + String result = request.execute().parseAsString(); + projectIdFromComputeEngine.set(result); + return result; } catch (IOException e) { logger.log(Level.INFO, "Could not determine project ID from Compute Engine.", e); return null; From 3f6b85f5588bf4ded57ac59fa41684b2b5bcde35 Mon Sep 17 00:00:00 2001 From: Elliotte Rusty Harold Date: Tue, 29 Oct 2019 06:40:25 -0400 Subject: [PATCH 28/54] update to jackson2 --- .../java/com/google/datastore/v1/client/DatastoreHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java index 79132cce7..1b187f1c8 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java @@ -22,7 +22,7 @@ import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson.JacksonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; import com.google.datastore.v1.ArrayValue; import com.google.datastore.v1.CompositeFilter; import com.google.datastore.v1.Entity; From 058ca50bb82d654a7cd508b947322e53b6d8ef37 Mon Sep 17 00:00:00 2001 From: Elliotte Rusty Harold Date: Tue, 29 Oct 2019 06:52:46 -0400 Subject: [PATCH 29/54] fix assorted warnings --- .../java/com/google/datastore/v1/client/DatastoreFactory.java | 3 +-- .../java/com/google/datastore/v1/client/DatastoreOptions.java | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java index acbea4f16..bece878d5 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java @@ -36,7 +36,6 @@ * Client factory for {@link Datastore}. */ public class DatastoreFactory { - private static final Logger logger = Logger.getLogger(DatastoreFactory.class.getName()); // Lazy load this because we might be running inside App Engine and this // class isn't on the whitelist. @@ -61,7 +60,7 @@ public static DatastoreFactory get() { * * @throws IllegalArgumentException if the server or credentials weren't provided. */ - public Datastore create(DatastoreOptions options) throws IllegalArgumentException { + public Datastore create(DatastoreOptions options) { return new Datastore(newRemoteRpc(options)); } diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java index 4ccde164f..d70d3c00c 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java @@ -135,10 +135,10 @@ public Builder localHost(String localHost) { } /** - * @deprecated Use {@link #projectId} and/or {@link #host}/{@link #localHost} instead. - * * Sets the project endpoint used to access Cloud Datastore. Prefer using {@link #projectId} * and/or {@link #host}/{@link #localHost} when possible. + * + * @deprecated Use {@link #projectId} and/or {@link #host}/{@link #localHost} instead. */ @Deprecated public Builder projectEndpoint(String projectEndpoint) { From f2aa3ddc349a8cd9688d0f770a0f08ae1b17e5c3 Mon Sep 17 00:00:00 2001 From: Elliotte Rusty Harold Date: Fri, 1 Nov 2019 12:14:33 -0400 Subject: [PATCH 30/54] update http client (#239) * update http client * update .gitignore * remove no longer needed gzip hack * remove redundant tests --- .gitignore | 1 + .../google/datastore/v1/client/RemoteRpc.java | 96 +------------------ .../datastore/v1/client/RemoteRpcTest.java | 58 ++--------- 3 files changed, 12 insertions(+), 143 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..b83d22266 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java index 3576c02e0..1c422844d 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java @@ -23,21 +23,16 @@ import com.google.api.client.http.HttpStatusCodes; import com.google.api.client.http.protobuf.ProtoHttpContent; import com.google.api.client.util.IOUtils; -import com.google.common.base.Preconditions; import com.google.protobuf.MessageLite; import com.google.rpc.Code; import com.google.rpc.Status; import java.io.ByteArrayOutputStream; -import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.Field; import java.net.SocketTimeoutException; import java.nio.charset.Charset; import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; import java.util.logging.Logger; -import java.util.zip.GZIPInputStream; /** * An RPC transport that sends protocol buffers over HTTP. @@ -47,91 +42,6 @@ class RemoteRpc { private static final Logger logger = Logger.getLogger(RemoteRpc.class.getName()); - /** - * An input stream that wraps a {@link GZIPInputStream} and forces it to fully consume its - * underlying {@link InputStream} before calling close() on it. - * - * @see href="https://github.com/google/google-http-java-client/issues/367" - */ - static class GzipFixingInputStream extends InputStream { - - // How many extra read() calls we have made on the underlying stream. - int callsToRead = 0; - - // Experimentally, it is only necessary to consume 1 extra byte. Consuming a bit more than that - // should not affect performance, but we set an upper bound for safety. - private static final int MAX_BYTES_TO_CONSUME = 100; - - private static final Field gzipUnderlyingInputStreamField = getGzipUnderlyingInputStreamField(); - - private static Field getGzipUnderlyingInputStreamField() { - try { - // FilterInputStream is a superclass of GZIPInputStream and stores the underlying - // InputStream. - Field gzipInputStreamIsField = FilterInputStream.class.getDeclaredField("in"); - gzipInputStreamIsField.setAccessible(true); - return gzipInputStreamIsField; - } catch (Exception e) { - logger.log( - Level.INFO, - "Failed to find field \"in\" in FilterInputStream. This" - + " may prevent keep-alive from working correctly.", - e); - return null; - } - } - - public static InputStream maybeWrap(InputStream inputStream) { - if (gzipUnderlyingInputStreamField != null && inputStream instanceof GZIPInputStream) { - return new GzipFixingInputStream((GZIPInputStream) inputStream); - } - return inputStream; - } - - private final GZIPInputStream gzipInputStream; - - private GzipFixingInputStream(GZIPInputStream gzipInputStream) { - Preconditions.checkNotNull(gzipUnderlyingInputStreamField); - this.gzipInputStream = gzipInputStream; - } - - @Override - public int read() throws IOException { - return gzipInputStream.read(); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - return gzipInputStream.read(b, off, len); - } - - @Override - public void close() throws IOException { - // If possible, finish consuming the underlying InputStream before closing it. - if (gzipUnderlyingInputStreamField != null) { - try { - InputStream underlyingInputStream = - (InputStream) gzipUnderlyingInputStreamField.get(gzipInputStream); - boolean reachedEndOfStream = false; - while (!reachedEndOfStream && callsToRead < MAX_BYTES_TO_CONSUME) { - callsToRead++; - if (underlyingInputStream.read() == -1) { - reachedEndOfStream = true; - } - } - if (!reachedEndOfStream) { - logger.log(Level.FINER, "Gave up consuming underlying InputStream"); - } - } catch (Exception e) { - // If this fails for any reason, log and move on. - logger.log( - Level.FINER, "Failed to consume underlying InputStream from GZIPInputStream", e); - } - } - gzipInputStream.close(); - } - } - private static final String API_FORMAT_VERSION_HEADER = "X-Goog-Api-Format-Version"; private static final String API_FORMAT_VERSION = "2"; @@ -182,13 +92,13 @@ public InputStream call(String methodName, MessageLite request) throws Datastore } httpResponse = httpRequest.execute(); if (!httpResponse.isSuccessStatusCode()) { - try (InputStream content = GzipFixingInputStream.maybeWrap(httpResponse.getContent())) { + try (InputStream content = httpResponse.getContent()) { throw makeException(url, methodName, content, httpResponse.getContentType(), httpResponse.getContentCharset(), null, httpResponse.getStatusCode()); } } - return GzipFixingInputStream.maybeWrap(httpResponse.getContent()); + return httpResponse.getContent(); } catch (SocketTimeoutException e) { throw makeException(url, methodName, Code.DEADLINE_EXCEEDED, "Deadline exceeded", e); } catch (IOException e) { @@ -253,7 +163,7 @@ static DatastoreException makeException(String url, String methodName, InputStre e); } - Code code = Code.valueOf(rpcStatus.getCode()); + Code code = Code.forNumber(rpcStatus.getCode()); if (code == null) { return makeException(url, methodName, Code.INTERNAL, String.format( diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java index 66b26c6e5..86f2559f4 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java @@ -16,16 +16,12 @@ package com.google.datastore.v1.client; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.LowLevelHttpRequest; import com.google.api.client.http.LowLevelHttpResponse; import com.google.api.client.util.Charsets; import com.google.datastore.v1.BeginTransactionResponse; -import com.google.datastore.v1.client.RemoteRpc.GzipFixingInputStream; import com.google.protobuf.ByteString; import com.google.rpc.Code; import com.google.rpc.Status; @@ -121,60 +117,22 @@ public void testPlainTextException() { } @Test - public void testGzipHack_NonGzip() throws Exception { - BeginTransactionResponse resp = newBeginTransactionResp(); + public void testGzip() throws IOException, DatastoreException { + BeginTransactionResponse response = newBeginTransactionResponse(); InjectedTestValues injectedTestValues = - new InjectedTestValues(resp.toByteArray(), new byte[0], false); + new InjectedTestValues(gzip(response), new byte[1], true); RemoteRpc rpc = newRemoteRpc(injectedTestValues); InputStream is = rpc.call("beginTransaction", BeginTransactionResponse.getDefaultInstance()); - BeginTransactionResponse parsedResp = BeginTransactionResponse.parseFrom(is); + BeginTransactionResponse parsedResponse = BeginTransactionResponse.parseFrom(is); is.close(); - assertEquals(resp, parsedResp); - assertFalse(is instanceof GzipFixingInputStream); - } - - @Test - public void testGzipHack_Gzip() throws Exception { - BeginTransactionResponse resp = newBeginTransactionResp(); - InjectedTestValues injectedTestValues = new InjectedTestValues(gzip(resp), new byte[1], true); - RemoteRpc rpc = newRemoteRpc(injectedTestValues); - - InputStream is = rpc.call("beginTransaction", BeginTransactionResponse.getDefaultInstance()); - BeginTransactionResponse parsedResp = BeginTransactionResponse.parseFrom(is); - is.close(); - - assertEquals(resp, parsedResp); - assertTrue(is instanceof GzipFixingInputStream); - assertEquals(1, ((GzipFixingInputStream) is).callsToRead); + assertEquals(response, parsedResponse); // Check that the underlying stream is exhausted. assertEquals(-1, injectedTestValues.inputStream.read()); } - @Test - public void testGzipHack_GzipTooManyExtraBytes() throws Exception { - BeginTransactionResponse resp = newBeginTransactionResp(); - // NOTE(eddavisson): We might expect 101 extra bytes to be enough that the underlying input - // stream is not exhausted, but this is not the case (likely due to a buffer somewhere). 1000 - // extra bytes seems to be enough. We check the value of callsToRead directly to make sure - // we eventually stopped trying to consume the underlying stream. - InjectedTestValues injectedTestValues = - new InjectedTestValues(gzip(resp), new byte[1000], true); - RemoteRpc rpc = newRemoteRpc(injectedTestValues); - - InputStream is = rpc.call("beginTransaction", BeginTransactionResponse.getDefaultInstance()); - BeginTransactionResponse parsedResp = BeginTransactionResponse.parseFrom(is); - is.close(); - - assertEquals(resp, parsedResp); - assertTrue(is instanceof GzipFixingInputStream); - assertEquals(100, ((GzipFixingInputStream) is).callsToRead); - // Check that the underlying stream is _not_ exhausted. - assertNotEquals(-1, injectedTestValues.inputStream.read()); - } - - private static BeginTransactionResponse newBeginTransactionResp() { + private static BeginTransactionResponse newBeginTransactionResponse() { return BeginTransactionResponse.newBuilder() .setTransaction(ByteString.copyFromUtf8("blah-blah-blah")) .build(); @@ -187,10 +145,10 @@ private static RemoteRpc newRemoteRpc(InjectedTestValues injectedTestValues) { "https://www.example.com/v1/projects/p"); } - private byte[] gzip(BeginTransactionResponse resp) throws IOException { + private byte[] gzip(BeginTransactionResponse response) throws IOException { ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); try (GZIPOutputStream gzipOut = new GZIPOutputStream(bytesOut)) { - resp.writeTo(gzipOut); + response.writeTo(gzipOut); } return bytesOut.toByteArray(); } From f55449c712b4717fe73c1ce48f8973932d5f21ed Mon Sep 17 00:00:00 2001 From: Vasu Nori <48367170+vnorigoog@users.noreply.github.com> Date: Tue, 30 Mar 2021 15:51:29 -0700 Subject: [PATCH 31/54] feat: add optional end-to-end checksum support for http 1.1 requests (#261) Update RemoteRpc to optionally support end-to-end checksums. If enabled, when sending a request to datastore a checksum will be calculated and included as a header of the request. When receiving a response from datastore and the checksum header is present, a checksum of the payload will be computed and validated to match the values specified in the header. If validation of the response fails, an IOException will be thrown to signal possible payload corruption. By default, checksum validation is not enabled. To enable checksum validation the environment variable `GOOGLE_CLOUD_DATASTORE_HTTP_ENABLE_E2E_CHECKSUM` should be set to `true`. --- .../client/ChecksumEnforcingInputStream.java | 109 ++++++++++++++++++ .../v1/client/EndToEndChecksumHandler.java | 77 +++++++++++++ .../google/datastore/v1/client/RemoteRpc.java | 38 +++++- .../ChecksumEnforcingInputStreamTest.java | 109 ++++++++++++++++++ .../client/EndToEndChecksumHandlerTest.java | 74 ++++++++++++ .../datastore/v1/client/RemoteRpcTest.java | 45 ++++++++ 6 files changed, 449 insertions(+), 3 deletions(-) create mode 100644 datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/ChecksumEnforcingInputStream.java create mode 100644 datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/EndToEndChecksumHandler.java create mode 100644 datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java create mode 100644 datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/EndToEndChecksumHandlerTest.java diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/ChecksumEnforcingInputStream.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/ChecksumEnforcingInputStream.java new file mode 100644 index 000000000..f74dc004c --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/ChecksumEnforcingInputStream.java @@ -0,0 +1,109 @@ +/* + * Copyright 2021 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1.client; + +import com.google.common.annotations.VisibleForTesting; +import com.google.api.client.http.HttpResponse; +import java.io.IOException; +import java.io.InputStream; +import java.security.MessageDigest; + +/** This class provides End-to-End Checksum API for http protocol. */ +class ChecksumEnforcingInputStream extends InputStream { + private final InputStream delegate; + private final MessageDigest messageDigest; + private final String expectedChecksum; + + ChecksumEnforcingInputStream(InputStream originalInputStream, + HttpResponse response, + MessageDigest digest) { + this(originalInputStream, EndToEndChecksumHandler.getChecksumHeader(response), digest); + } + + @VisibleForTesting + ChecksumEnforcingInputStream(InputStream originalInputStream, + String checksum, + MessageDigest digest) { + delegate = originalInputStream; + expectedChecksum = checksum; + messageDigest = digest; + } + + @Override + public int available() throws IOException { + return delegate.available(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + @Override + public void mark(int readlimit) { + throw new RuntimeException("mark(int) Not Supported"); + } + + @Override + public boolean markSupported() { + // This class doesn't support mark, reset methods! + return false; + } + + @Override + public int read() throws IOException { + throw new RuntimeException("read() Not Supported"); + } + + @Override + public int read(byte[] b) throws IOException { + throw new RuntimeException("read(byte[]) Not Supported"); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (len <= 0) return 0; + int i = delegate.read(b, off, len); + if (i > 0) { + messageDigest.update(b, off, i); + } else { + // no more payload to read. compute checksum and verify + if (!expectedChecksum.equalsIgnoreCase( + com.google.common.io.BaseEncoding.base16().encode(messageDigest.digest()))) { + throw new IOException("possible memory corruption on payload detected"); + } + } + return i; + } + + @Override + public void reset() throws IOException { + throw new RuntimeException("reset() Not Supported"); + } + + @Override + public long skip(long n) throws IOException { + if (n <= 0) return 0; + // TODO: handle the case of n > Integer.MAX_VALUE ( that is, n > (2GB - 1). It is highly + // unlikely that callers will want to skip that many bytes. That is the entire payload + if (n > Integer.MAX_VALUE) { + throw new IOException("can't skip more than Integer.MAX bytes"); + } + int intSkip = (int) n; + byte[] b = new byte[intSkip]; + return read(b, 0, intSkip); + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/EndToEndChecksumHandler.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/EndToEndChecksumHandler.java new file mode 100644 index 000000000..591ebe3f6 --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/EndToEndChecksumHandler.java @@ -0,0 +1,77 @@ +/* + * Copyright 2021 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1.client; + +import com.google.api.client.http.HttpResponse; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** This class provides End-to-End Checksum API for http protocol. */ +class EndToEndChecksumHandler { + /** The checksum http header on http requests */ + static final String HTTP_REQUEST_CHECKSUM_HEADER = "x-request-checksum-348659783"; + /** The checksum http header on http responses */ + static final String HTTP_RESPONSE_CHECKSUM_HEADER = "x-response-checksum-348659783"; + /** Algorithm used for checksum */ + private static final String MD5 = "MD5"; + + /** + * Create and return checksum as a string value for the input 'bytes'. + * + * @param bytes raw message for which the checksum is being computed + * @return computed checksum as a hex string + * @throws RuntimeException if MD5 Algorithm is not found in the VM + */ + static String computeChecksum(byte[] bytes) { + if (bytes == null || (bytes.length == 0)) { + return null; + } + return com.google.common.io.BaseEncoding.base16().encode( + getMessageDigestInstance().digest(bytes)); + } + + /** + * Validates the checksum for the given input 'bytes' and returns true if valid, false otherwise. + * + * @param checksum the checksum as a hex string + * @param bytes the raw message for which the checksum was sent + * @return {@code true} if input checksum is valid for the input bytes; {@code false} otherwise + */ + static boolean validateChecksum(String checksum, byte[] bytes) { + return checksum != null + && !checksum.isEmpty() + && bytes != null + && bytes.length > 0 + && checksum.equalsIgnoreCase(computeChecksum(bytes)); + } + + static MessageDigest getMessageDigestInstance() { + try { + return MessageDigest.getInstance(MD5); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("MD5 algorithm is not found when computing checksum!"); + } + } + + static boolean hasChecksumHeader(HttpResponse response) { + String checksum = getChecksumHeader(response); + return checksum != null && !checksum.isEmpty(); + } + + static String getChecksumHeader(HttpResponse response) { + return response.getHeaders().getFirstHeaderStringValue(HTTP_RESPONSE_CHECKSUM_HEADER); + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java index 1c422844d..e0b2e378b 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java @@ -23,6 +23,7 @@ import com.google.api.client.http.HttpStatusCodes; import com.google.api.client.http.protobuf.ProtoHttpContent; import com.google.api.client.util.IOUtils; +import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.MessageLite; import com.google.rpc.Code; import com.google.rpc.Status; @@ -42,13 +43,17 @@ class RemoteRpc { private static final Logger logger = Logger.getLogger(RemoteRpc.class.getName()); - private static final String API_FORMAT_VERSION_HEADER = "X-Goog-Api-Format-Version"; + @VisibleForTesting + static final String API_FORMAT_VERSION_HEADER = "X-Goog-Api-Format-Version"; private static final String API_FORMAT_VERSION = "2"; private final HttpRequestFactory client; private final HttpRequestInitializer initializer; private final String url; private final AtomicInteger rpcCount = new AtomicInteger(0); + // Not final - so it can be set/reset in Unittests + private static boolean enableE2EChecksum = Boolean.parseBoolean( + System.getenv("GOOGLE_CLOUD_DATASTORE_HTTP_ENABLE_E2E_CHECKSUM")); RemoteRpc(HttpRequestFactory client, HttpRequestInitializer initializer, String url) { this.client = client; @@ -80,7 +85,7 @@ public InputStream call(String methodName, MessageLite request) throws Datastore rpcCount.incrementAndGet(); ProtoHttpContent payload = new ProtoHttpContent(request); HttpRequest httpRequest = client.buildPostRequest(resolveURL(methodName), payload); - httpRequest.getHeaders().put(API_FORMAT_VERSION_HEADER, API_FORMAT_VERSION); + setHeaders(request, httpRequest); // Don't throw an HTTPResponseException on error. It converts the response to a String and // throws away the original, whereas we need the raw bytes to parse it as a proto. httpRequest.setThrowExceptionOnExecuteError(false); @@ -98,7 +103,12 @@ public InputStream call(String methodName, MessageLite request) throws Datastore httpResponse.getStatusCode()); } } - return httpResponse.getContent(); + InputStream inputStream = httpResponse.getContent(); + return enableE2EChecksum && EndToEndChecksumHandler.hasChecksumHeader(httpResponse) + ? new ChecksumEnforcingInputStream(inputStream, + httpResponse, + EndToEndChecksumHandler.getMessageDigestInstance()) + : inputStream; } catch (SocketTimeoutException e) { throw makeException(url, methodName, Code.DEADLINE_EXCEEDED, "Deadline exceeded", e); } catch (IOException e) { @@ -110,6 +120,28 @@ public InputStream call(String methodName, MessageLite request) throws Datastore } } + @VisibleForTesting + void setHeaders(MessageLite request, HttpRequest httpRequest) { + httpRequest.getHeaders().put(API_FORMAT_VERSION_HEADER, API_FORMAT_VERSION); + if (enableE2EChecksum && request != null) { + String checksum = EndToEndChecksumHandler.computeChecksum(request.toByteArray()); + if (checksum != null) { + httpRequest.getHeaders().put(EndToEndChecksumHandler.HTTP_REQUEST_CHECKSUM_HEADER, + checksum); + } + } + } + + @VisibleForTesting + HttpRequestFactory getClient() { + return client; + } + + @VisibleForTesting + static void setSystemEnvE2EChecksum(boolean enableE2EChecksum) { + RemoteRpc.enableE2EChecksum = enableE2EChecksum; + } + void resetRpcCount() { rpcCount.set(0); } diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java new file mode 100644 index 000000000..32d43fb44 --- /dev/null +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java @@ -0,0 +1,109 @@ +/* + * Copyright 2021 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1.client; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.security.MessageDigest; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Test for {@link ChecksumEnforcingInputStream}. */ +@RunWith(JUnit4.class) +public class ChecksumEnforcingInputStreamTest { + private final MessageDigest digest = EndToEndChecksumHandler.getMessageDigestInstance(); + + public void test(int payloadSize) throws Exception { + ChecksumEnforcingInputStream testInstance = setUpData(payloadSize); + // read 1000 bytes at a time + // Since checksum should be correct, do not expect IOException + byte[] buf = new byte[1000]; + try { + while (testInstance.read(buf, 0, 1000) != -1) { + // do nothing with the bytes read + } + } catch (IOException e) { + fail("checksum verification failed!"); + } + } + + @Test + public void read_withValidChecksum_differentPayloadSizes() throws Exception { + // test with various payload sizes (1, 2, 2**2, 2**3 etc upto 2**28 = 256MB) + for (int i = 0, payloadSize = 1; i < 29; i++) { + long start = System.currentTimeMillis(); + test(payloadSize); + payloadSize *= 2; + long duration = System.currentTimeMillis() - start; + // log test duration times for bigger payloads + if (i > 20) { + System.out.println("Test duration for payloadsize = 2** " + i + " is: " + duration + "ms"); + } + } + } + + @Test + public void read_withInvalidChecksum() { + // build a test instance with invalidchecksum + ChecksumEnforcingInputStream instance = new ChecksumEnforcingInputStream( + new ByteArrayInputStream("hello there".getBytes(UTF_8)), + "this checksum is invalid", + digest); + // read 1000 bytes at a time + // Since checksum should be correct, do not expect IOException + byte[] buf = new byte[1000]; + try { + while (instance.read(buf, 0, 1000) != -1) { + // do nothing with the bytes read + } + } catch (IOException e) { + // this is expected + return; + } + fail("should have failed"); + } + + @Test + public void markNotSupported() throws Exception { + ChecksumEnforcingInputStream testInstance = setUpData(1); + assertFalse(testInstance.markSupported()); + } + + private ChecksumEnforcingInputStream setUpData(int payloadSize) throws Exception { + // setup a String of size = input param: payloadSize + String str = "This is a repeating string."; + String payload; + if (payloadSize > str.length()) { + int num = payloadSize / str.length(); + StringBuffer buf = new StringBuffer(); + for (int i = 0; i < num; i++) { + buf.append(str); + } + payload = buf.toString(); + } else { + payload = str.substring(0, payloadSize); + } + byte[] bytes = payload.getBytes(UTF_8); + String expectedChecksum = EndToEndChecksumHandler.computeChecksum(bytes); + return new ChecksumEnforcingInputStream(new ByteArrayInputStream(bytes), + expectedChecksum, digest); + } +} diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/EndToEndChecksumHandlerTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/EndToEndChecksumHandlerTest.java new file mode 100644 index 000000000..593ccd23d --- /dev/null +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/EndToEndChecksumHandlerTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2021 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1.client; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Test for {@link EndToEndChecksumHandler}. */ +@RunWith(JUnit4.class) +public class EndToEndChecksumHandlerTest { + private byte[] payloadBytes = "This is a long string with numbers 1234, 134.56 ".getBytes(UTF_8); + + @Test + public void validateChecksum_correctChecksum() { + String computed = EndToEndChecksumHandler.computeChecksum(payloadBytes); + assertTrue(EndToEndChecksumHandler.validateChecksum(computed, payloadBytes)); + } + + @Test + public void validateChecksum_incorrectChecksum() { + String computed = EndToEndChecksumHandler.computeChecksum("random string".getBytes(UTF_8)); + assertFalse(EndToEndChecksumHandler.validateChecksum(computed, payloadBytes)); + } + + @Test + public void validateChecksum_nullChecksum() { + assertFalse(EndToEndChecksumHandler.validateChecksum(null, payloadBytes)); + } + + @Test + public void validateChecksum_emptyChecksum() { + assertFalse(EndToEndChecksumHandler.validateChecksum("", payloadBytes)); + } + + @Test + public void validateChecksum_nullPayload() { + assertFalse(EndToEndChecksumHandler.validateChecksum("foo", null)); + } + + @Test + public void validateChecksum_emptyPayload() { + assertFalse(EndToEndChecksumHandler.validateChecksum("foo", new byte[0])); + } + + @Test + public void computeChecksum_nullInputBytes() { + assertNull(EndToEndChecksumHandler.computeChecksum(null)); + } + + @Test + public void computeChecksum_emptyArrayForInputBytes() { + assertNull(EndToEndChecksumHandler.computeChecksum(new byte[0])); + } +} diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java index 86f2559f4..377f92d8c 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java @@ -16,13 +16,19 @@ package com.google.datastore.v1.client; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.LowLevelHttpRequest; import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.http.protobuf.ProtoHttpContent; import com.google.api.client.util.Charsets; import com.google.datastore.v1.BeginTransactionResponse; +import com.google.datastore.v1.RollbackRequest; import com.google.protobuf.ByteString; +import com.google.protobuf.MessageLite; import com.google.rpc.Code; import com.google.rpc.Status; import java.io.ByteArrayInputStream; @@ -132,6 +138,45 @@ public void testGzip() throws IOException, DatastoreException { assertEquals(-1, injectedTestValues.inputStream.read()); } + @Test + public void testHttpHeaders_expectE2eChecksumHeader() throws IOException { + // Enable E2E-Checksum system env variable + RemoteRpc.setSystemEnvE2EChecksum(true); + MessageLite request = RollbackRequest.newBuilder() + .setTransaction(ByteString.copyFromUtf8("project-id")) + .build(); + RemoteRpc rpc = newRemoteRpc(new InjectedTestValues(gzip(newBeginTransactionResponse()), + new byte[1], true)); + HttpRequest httpRequest = rpc.getClient().buildPostRequest(rpc.resolveURL("blah"), + new ProtoHttpContent(request)); + rpc.setHeaders(request, httpRequest); + assertNotNull(httpRequest.getHeaders() + .getFirstHeaderStringValue(RemoteRpc.API_FORMAT_VERSION_HEADER)); + // Expect to find e2e-checksum header + String header = httpRequest.getHeaders().getFirstHeaderStringValue( + EndToEndChecksumHandler.HTTP_REQUEST_CHECKSUM_HEADER); + assertEquals(32, header.length()); + } + + @Test + public void testHttpHeaders_doNotExpectE2eChecksumHeader() throws IOException { + // disable E2E-Checksum system env variable + RemoteRpc.setSystemEnvE2EChecksum(false); + MessageLite request = RollbackRequest.newBuilder() + .setTransaction(ByteString.copyFromUtf8("project-id")) + .build(); + RemoteRpc rpc = newRemoteRpc(new InjectedTestValues(gzip(newBeginTransactionResponse()), + new byte[1], true)); + HttpRequest httpRequest = rpc.getClient().buildPostRequest(rpc.resolveURL("blah"), + new ProtoHttpContent(request)); + rpc.setHeaders(request, httpRequest); + assertNotNull(httpRequest.getHeaders() + .getFirstHeaderStringValue(RemoteRpc.API_FORMAT_VERSION_HEADER)); + // Do not expect to find e2e-checksum header + assertNull(httpRequest.getHeaders().getFirstHeaderStringValue( + EndToEndChecksumHandler.HTTP_REQUEST_CHECKSUM_HEADER)); + } + private static BeginTransactionResponse newBeginTransactionResponse() { return BeginTransactionResponse.newBuilder() .setTransaction(ByteString.copyFromUtf8("blah-blah-blah")) From 76faa6b88e9ed204ed62800ad5ad6195445ccaed Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 11 May 2021 19:02:03 +0000 Subject: [PATCH 32/54] chore: release 1.106.5-SNAPSHOT (#419) :robot: I have created a release \*beep\* \*boop\* --- ### Updating meta-information for bleeding-edge SNAPSHOT release. --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- google-cloud-datastore-bom/pom.xml | 6 +++--- google-cloud-datastore/pom.xml | 4 ++-- pom.xml | 4 ++-- proto-google-cloud-datastore-v1/pom.xml | 4 ++-- versions.txt | 8 ++++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index 640e3feae..d39cb3790 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-datastore-bom - 1.106.4 + 1.106.5-SNAPSHOT pom com.google.cloud @@ -63,12 +63,12 @@ com.google.api.grpc proto-google-cloud-datastore-v1 - 0.89.4 + 0.89.5-SNAPSHOT com.google.cloud google-cloud-datastore - 1.106.4 + 1.106.5-SNAPSHOT diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index 4c48c408e..2c4d600d5 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-datastore - 1.106.4 + 1.106.5-SNAPSHOT jar Google Cloud Datastore https://github.com/googleapis/java-datastore @@ -12,7 +12,7 @@ com.google.cloud google-cloud-datastore-parent - 1.106.4 + 1.106.5-SNAPSHOT google-cloud-datastore diff --git a/pom.xml b/pom.xml index f4e2ab9b0..08cac4f86 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-datastore-parent pom - 1.106.4 + 1.106.5-SNAPSHOT Google Cloud Datastore Parent https://github.com/googleapis/java-datastore @@ -167,7 +167,7 @@ com.google.api.grpc proto-google-cloud-datastore-v1 - 0.89.4 + 0.89.5-SNAPSHOT com.google.cloud.datastore diff --git a/proto-google-cloud-datastore-v1/pom.xml b/proto-google-cloud-datastore-v1/pom.xml index bdf94fed8..ef1b94849 100644 --- a/proto-google-cloud-datastore-v1/pom.xml +++ b/proto-google-cloud-datastore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.89.4 + 0.89.5-SNAPSHOT proto-google-cloud-datastore-v1 PROTO library for proto-google-cloud-datastore-v1 com.google.cloud google-cloud-datastore-parent - 1.106.4 + 1.106.5-SNAPSHOT diff --git a/versions.txt b/versions.txt index ef38c90f9..3e5c2218c 100644 --- a/versions.txt +++ b/versions.txt @@ -1,8 +1,8 @@ # Format: # module:released-version:current-version -google-cloud-datastore:1.106.4:1.106.4 -google-cloud-datastore-bom:1.106.4:1.106.4 -google-cloud-datastore-parent:1.106.4:1.106.4 -proto-google-cloud-datastore-v1:0.89.4:0.89.4 +google-cloud-datastore:1.106.4:1.106.5-SNAPSHOT +google-cloud-datastore-bom:1.106.4:1.106.5-SNAPSHOT +google-cloud-datastore-parent:1.106.4:1.106.5-SNAPSHOT +proto-google-cloud-datastore-v1:0.89.4:0.89.5-SNAPSHOT From 9e943c1cc4722d7e8ef75c5c726975ef19f11fea Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 11 May 2021 16:32:05 -0700 Subject: [PATCH 33/54] chore: regenerate README (#416) This PR was generated using Autosynth. :rainbow:

Log from Synthtool ``` 2021-05-10 17:34:02,307 synthtool [DEBUG] > Executing /root/.cache/synthtool/java-datastore/.github/readme/synth.py. On branch autosynth-readme nothing to commit, working tree clean 2021-05-10 17:34:05,034 synthtool [DEBUG] > Wrote metadata to .github/readme/synth.metadata/synth.metadata. ```
Full log will be available here: https://source.cloud.google.com/results/invocations/90f248bc-b652-4080-b66a-69c8eb61f7db/targets - [ ] To automatically regenerate this PR, check this box. (May take up to 24 hours.) --- .github/readme/synth.metadata/synth.metadata | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/readme/synth.metadata/synth.metadata b/.github/readme/synth.metadata/synth.metadata index 4e3f2a881..0a3f6b80b 100644 --- a/.github/readme/synth.metadata/synth.metadata +++ b/.github/readme/synth.metadata/synth.metadata @@ -4,14 +4,14 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-datastore.git", - "sha": "71b84e8d2af39deb5d07b0b2643dbdd0afc80ae8" + "sha": "0e4ac4be5af744fdb5fe4d8a5aeaa95d9ac8ead5" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "06a8cd0ff7e81b05e6c503eab510ec622384caa7" + "sha": "6726988c677bb78385868bfc48dbfa2fe981d44a" } } ] diff --git a/README.md b/README.md index 5a5030280..e11b958a2 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies ```Groovy -implementation platform('com.google.cloud:libraries-bom:20.2.0') +implementation platform('com.google.cloud:libraries-bom:20.3.0') compile 'com.google.cloud:google-cloud-datastore' ``` From 0607c604a6ef4d5b33505ef0ce61ef3c275694e9 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 11 May 2021 16:46:04 -0700 Subject: [PATCH 34/54] chore: regenerate README (#421) This PR was generated using Autosynth. :rainbow:
Log from Synthtool ``` 2021-05-11 23:40:16,301 synthtool [DEBUG] > Executing /root/.cache/synthtool/java-datastore/.github/readme/synth.py. On branch autosynth-readme nothing to commit, working tree clean 2021-05-11 23:40:17,684 synthtool [DEBUG] > Wrote metadata to .github/readme/synth.metadata/synth.metadata. ```
Full log will be available here: https://source.cloud.google.com/results/invocations/aa65d86a-b74e-4c2d-9df0-8a4a39904113/targets - [ ] To automatically regenerate this PR, check this box. (May take up to 24 hours.) --- .github/readme/synth.metadata/synth.metadata | 4 ++-- README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/readme/synth.metadata/synth.metadata b/.github/readme/synth.metadata/synth.metadata index 0a3f6b80b..856780743 100644 --- a/.github/readme/synth.metadata/synth.metadata +++ b/.github/readme/synth.metadata/synth.metadata @@ -4,14 +4,14 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-datastore.git", - "sha": "0e4ac4be5af744fdb5fe4d8a5aeaa95d9ac8ead5" + "sha": "9e943c1cc4722d7e8ef75c5c726975ef19f11fea" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "6726988c677bb78385868bfc48dbfa2fe981d44a" + "sha": "f7e7449a641ce47f2a04ed577dec48e109325e76" } } ] diff --git a/README.md b/README.md index e11b958a2..204cb225c 100644 --- a/README.md +++ b/README.md @@ -51,12 +51,12 @@ compile 'com.google.cloud:google-cloud-datastore' ``` If you are using Gradle without BOM, add this to your dependencies ```Groovy -compile 'com.google.cloud:google-cloud-datastore:1.106.3' +compile 'com.google.cloud:google-cloud-datastore:1.106.4' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-datastore" % "1.106.3" +libraryDependencies += "com.google.cloud" % "google-cloud-datastore" % "1.106.4" ``` ## Authentication From a1ea05f975365a7eba6607aeae1d144b8c49453c Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 13 May 2021 15:58:02 +0200 Subject: [PATCH 35/54] chore(deps): update dependency com.google.cloud:libraries-bom to v20.3.0 (#415) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:libraries-bom](https://togithub.com/GoogleCloudPlatform/cloud-opensource-java) | `20.2.0` -> `20.3.0` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:libraries-bom/20.3.0/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:libraries-bom/20.3.0/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:libraries-bom/20.3.0/compatibility-slim/20.2.0)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:libraries-bom/20.3.0/confidence-slim/20.2.0)](https://docs.renovatebot.com/merge-confidence/) | --- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻️ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-datastore). --- samples/snippets/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 283310a11..70c6099b2 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 20.2.0 + 20.3.0 pom import From af7528f49250916ac4a5ce0b51b6150f780c22d5 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Thu, 13 May 2021 07:12:08 -0700 Subject: [PATCH 36/54] chore: regenerate README (#422) This PR was generated using Autosynth. :rainbow:
Log from Synthtool ``` 2021-05-13 14:02:50,568 synthtool [DEBUG] > Executing /root/.cache/synthtool/java-datastore/.github/readme/synth.py. On branch autosynth-readme nothing to commit, working tree clean 2021-05-13 14:02:51,812 synthtool [DEBUG] > Wrote metadata to .github/readme/synth.metadata/synth.metadata. ```
Full log will be available here: https://source.cloud.google.com/results/invocations/cec070fa-bb3b-4290-bc82-9829f3cb2b09/targets - [ ] To automatically regenerate this PR, check this box. (May take up to 24 hours.) --- .github/readme/synth.metadata/synth.metadata | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/readme/synth.metadata/synth.metadata b/.github/readme/synth.metadata/synth.metadata index 856780743..2b8c6a4f9 100644 --- a/.github/readme/synth.metadata/synth.metadata +++ b/.github/readme/synth.metadata/synth.metadata @@ -4,14 +4,14 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-datastore.git", - "sha": "9e943c1cc4722d7e8ef75c5c726975ef19f11fea" + "sha": "a1ea05f975365a7eba6607aeae1d144b8c49453c" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "f7e7449a641ce47f2a04ed577dec48e109325e76" + "sha": "bd8281a06cc7f84906e04d4843c1d3d386a980cd" } } ] diff --git a/README.md b/README.md index 204cb225c..bb17743d5 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file com.google.cloud libraries-bom - 20.2.0 + 20.3.0 pom import From b842b7d140f311b00cdb108330ccd72cd25aa1e9 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Thu, 13 May 2021 16:38:33 -0700 Subject: [PATCH 37/54] build(java): remove codecov action (#423) This PR was generated using Autosynth. :rainbow: Synth log will be available here: https://source.cloud.google.com/results/invocations/e6b0cbbb-c315-4d47-a85f-ad59f2a76c78/targets - [ ] To automatically regenerate this PR, check this box. (May take up to 24 hours.) Source-Link: https://github.com/googleapis/synthtool/commit/4f4b1b9b8d8b52f1e9e4a76165896debce5ab7f1 --- .github/workflows/ci.yaml | 6 +----- synth.metadata | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index def8b3a2c..0195b32f0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,10 +19,6 @@ jobs: - run: .kokoro/build.sh env: JOB_TYPE: test - - name: coverage - uses: codecov/codecov-action@v1 - with: - name: actions ${{matrix.java}} windows: runs-on: windows-latest steps: @@ -80,4 +76,4 @@ jobs: - run: java -version - run: .kokoro/build.sh env: - JOB_TYPE: clirr \ No newline at end of file + JOB_TYPE: clirr diff --git a/synth.metadata b/synth.metadata index 35efcba29..317c8f55d 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-datastore.git", - "sha": "03b6d5ed1ff9e4741b4085ebf5e919861eed1a28" + "sha": "af7528f49250916ac4a5ce0b51b6150f780c22d5" } }, { @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "046994f491c02806aea60118e214a9edd67f5ab7" + "sha": "4f4b1b9b8d8b52f1e9e4a76165896debce5ab7f1" } } ], From 449ea8805f26108ef60a92be5add9782e63a9809 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Fri, 14 May 2021 14:11:22 -0400 Subject: [PATCH 38/54] delete .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index b83d22266..000000000 --- a/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target/ From 5aa175ea2a687e9165a30df56c5fe4109e4a7a0f Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Fri, 14 May 2021 14:12:40 -0400 Subject: [PATCH 39/54] move pom --- datastore-v1-proto-client/src/pom.xml | 108 ++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 datastore-v1-proto-client/src/pom.xml diff --git a/datastore-v1-proto-client/src/pom.xml b/datastore-v1-proto-client/src/pom.xml new file mode 100644 index 000000000..91835e84b --- /dev/null +++ b/datastore-v1-proto-client/src/pom.xml @@ -0,0 +1,108 @@ + + + + 4.0.0 + datastore-v1-proto-client + 1.6.3 + + ${project.groupId}:${project.artifactId} + + jar + + Low level client for accessing Google Cloud Datastore v1. + + + com.google.cloud.datastore + datastore-v1-proto-client-parent + 1.6.3 + + + + + com.google.api.grpc + proto-google-cloud-datastore-v1 + + + + com.google.http-client + google-http-client + + + + com.google.http-client + google-http-client-protobuf + + + + com.google.http-client + google-http-client-jackson2 + + + + com.google.oauth-client + google-oauth-client + + + + com.google.api-client + google-api-client + + + + com.google.guava + guava + + + + + junit + junit + test + + + + com.google.truth + truth + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + org.apache.maven.plugins + maven-gpg-plugin + + + + From 9f115b77e3d18e92a7f9f45f49e61c525bc6648e Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Fri, 14 May 2021 14:13:21 -0400 Subject: [PATCH 40/54] move pom --- datastore-v1-proto-client/{src => }/pom.xml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename datastore-v1-proto-client/{src => }/pom.xml (100%) diff --git a/datastore-v1-proto-client/src/pom.xml b/datastore-v1-proto-client/pom.xml similarity index 100% rename from datastore-v1-proto-client/src/pom.xml rename to datastore-v1-proto-client/pom.xml From 62411b9eb95c490df48cd711a633310ced8baf53 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Fri, 14 May 2021 14:26:25 -0400 Subject: [PATCH 41/54] add module, run formatter, fix any failing builds --- datastore-v1-proto-client/pom.xml | 53 ++-- .../client/ChecksumEnforcingInputStream.java | 14 +- .../google/datastore/v1/client/Datastore.java | 12 +- .../v1/client/DatastoreEmulator.java | 6 +- .../v1/client/DatastoreException.java | 14 +- .../datastore/v1/client/DatastoreFactory.java | 46 ++-- .../datastore/v1/client/DatastoreHelper.java | 252 ++++++++---------- .../datastore/v1/client/DatastoreOptions.java | 43 ++- .../v1/client/EndToEndChecksumHandler.java | 6 +- .../datastore/v1/client/QuerySplitter.java | 5 +- .../v1/client/QuerySplitterImpl.java | 57 ++-- .../google/datastore/v1/client/RemoteRpc.java | 69 +++-- .../ChecksumEnforcingInputStreamTest.java | 17 +- .../v1/client/DatastoreFactoryTest.java | 64 ++--- .../v1/client/DatastoreHelperTest.java | 95 +++---- .../datastore/v1/client/DatastoreTest.java | 218 +++++++-------- .../client/EndToEndChecksumHandlerTest.java | 3 +- .../datastore/v1/client/RemoteRpcTest.java | 95 ++++--- pom.xml | 1 + versions.txt | 1 + 20 files changed, 503 insertions(+), 568 deletions(-) diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index 91835e84b..718f03bd7 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -18,19 +18,18 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 datastore-v1-proto-client - 1.6.3 + 1.6.4-SNAPSHOT - ${project.groupId}:${project.artifactId} + + com.google.cloud + google-cloud-datastore-parent + 1.106.5-SNAPSHOT + jar Low level client for accessing Google Cloud Datastore v1. - - com.google.cloud.datastore - datastore-v1-proto-client-parent - 1.6.3 - @@ -68,6 +67,21 @@ guava + + com.google.code.findbugs + jsr305 + + + + com.google.api.grpc + proto-google-common-protos + + + + com.google.protobuf + protobuf-java + + junit @@ -78,31 +92,8 @@ com.google.truth truth + 1.1.2 test - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.apache.maven.plugins - maven-source-plugin - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - - org.apache.maven.plugins - maven-gpg-plugin - - - diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/ChecksumEnforcingInputStream.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/ChecksumEnforcingInputStream.java index f74dc004c..2b9674917 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/ChecksumEnforcingInputStream.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/ChecksumEnforcingInputStream.java @@ -1,5 +1,5 @@ /* - * Copyright 2021 Google LLC. All Rights Reserved. + * Copyright 2021 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,8 @@ */ package com.google.datastore.v1.client; -import com.google.common.annotations.VisibleForTesting; import com.google.api.client.http.HttpResponse; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.io.InputStream; import java.security.MessageDigest; @@ -27,16 +27,14 @@ class ChecksumEnforcingInputStream extends InputStream { private final MessageDigest messageDigest; private final String expectedChecksum; - ChecksumEnforcingInputStream(InputStream originalInputStream, - HttpResponse response, - MessageDigest digest) { + ChecksumEnforcingInputStream( + InputStream originalInputStream, HttpResponse response, MessageDigest digest) { this(originalInputStream, EndToEndChecksumHandler.getChecksumHeader(response), digest); } @VisibleForTesting - ChecksumEnforcingInputStream(InputStream originalInputStream, - String checksum, - MessageDigest digest) { + ChecksumEnforcingInputStream( + InputStream originalInputStream, String checksum, MessageDigest digest) { delegate = originalInputStream; expectedChecksum = checksum; messageDigest = digest; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java index eb33dc627..8426b6e28 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/Datastore.java @@ -46,24 +46,22 @@ public class Datastore { this.remoteRpc = remoteRpc; } - /** - * Reset the RPC count. - */ + /** Reset the RPC count. */ public void resetRpcCount() { remoteRpc.resetRpcCount(); } /** - * Returns the number of RPC calls made since the client was created - * or {@link #resetRpcCount} was called. + * Returns the number of RPC calls made since the client was created or {@link #resetRpcCount} was + * called. */ public int getRpcCount() { return remoteRpc.getRpcCount(); } private DatastoreException invalidResponseException(String method, IOException exception) { - return RemoteRpc.makeException(remoteRpc.getUrl(), method, Code.UNAVAILABLE, - "Invalid response", exception); + return RemoteRpc.makeException( + remoteRpc.getUrl(), method, Code.UNAVAILABLE, "Invalid response", exception); } public AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws DatastoreException { diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java index a6c656df5..721073e12 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreEmulator.java @@ -78,7 +78,11 @@ public class DatastoreEmulator extends Datastore { private final DatastoreEmulatorOptions options; /** Internal state lifecycle management. */ - enum State {NEW, STARTED, STOPPED} + enum State { + NEW, + STARTED, + STOPPED + } private volatile State state = State.NEW; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreException.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreException.java index e259aa9e6..b5ff17c4d 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreException.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreException.java @@ -17,9 +17,7 @@ import com.google.rpc.Code; -/** - * Indicates an error in a {@link Datastore} call. - */ +/** Indicates an error in a {@link Datastore} call. */ public class DatastoreException extends Exception { private final String methodName; private final Code code; @@ -30,20 +28,16 @@ public DatastoreException(String methodName, Code code, String message, Throwabl this.code = code; } - /** - * @return the canonical error code - */ + /** @return the canonical error code */ public Code getCode() { return code; } - /** - * @return the datastore method name - */ + /** @return the datastore method name */ public String getMethodName() { return methodName; } - + @Override public String toString() { return String.format("%s, code=%s", super.toString(), code); diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java index bece878d5..5a3f4183b 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreFactory.java @@ -21,7 +21,6 @@ import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; - import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; @@ -32,9 +31,7 @@ import java.util.logging.Logger; import java.util.logging.StreamHandler; -/** - * Client factory for {@link Datastore}. - */ +/** Client factory for {@link Datastore}. */ public class DatastoreFactory { // Lazy load this because we might be running inside App Engine and this @@ -54,9 +51,8 @@ public static DatastoreFactory get() { } /** - * Provides access to a datastore using the provided options. Logs - * into the application using the credentials available via these - * options. + * Provides access to a datastore using the provided options. Logs into the application using the + * credentials available via these options. * * @throws IllegalArgumentException if the server or credentials weren't provided. */ @@ -64,9 +60,7 @@ public Datastore create(DatastoreOptions options) { return new Datastore(newRemoteRpc(options)); } - /** - * Constructs a Google APIs HTTP client with the associated credentials. - */ + /** Constructs a Google APIs HTTP client with the associated credentials. */ public HttpRequestFactory makeClient(DatastoreOptions options) { Credential credential = options.getCredential(); HttpTransport transport = options.getTransport(); @@ -77,9 +71,7 @@ public HttpRequestFactory makeClient(DatastoreOptions options) { return transport.createRequestFactory(credential); } - /** - * Starts logging datastore method calls to the console. (Useful within tests.) - */ + /** Starts logging datastore method calls to the console. (Useful within tests.) */ public static void logMethodCalls() { Logger logger = Logger.getLogger(Datastore.class.getName()); logger.setLevel(Level.FINE); @@ -88,9 +80,7 @@ public static void logMethodCalls() { } } - /** - * Build a valid datastore URL. - */ + /** Build a valid datastore URL. */ String buildProjectEndpoint(DatastoreOptions options) { if (options.getProjectEndpoint() != null) { return options.getProjectEndpoint(); @@ -98,14 +88,13 @@ String buildProjectEndpoint(DatastoreOptions options) { // DatastoreOptions ensures either project endpoint or project ID is set. String projectId = checkNotNull(options.getProjectId()); if (options.getHost() != null) { - return validateUrl(String.format("https://%s/%s/projects/%s", - options.getHost(), VERSION, projectId)); + return validateUrl( + String.format("https://%s/%s/projects/%s", options.getHost(), VERSION, projectId)); } else if (options.getLocalHost() != null) { - return validateUrl(String.format("http://%s/%s/projects/%s", - options.getLocalHost(), VERSION, projectId)); + return validateUrl( + String.format("http://%s/%s/projects/%s", options.getLocalHost(), VERSION, projectId)); } - return validateUrl(String.format("%s/%s/projects/%s", - DEFAULT_HOST, VERSION, projectId)); + return validateUrl(String.format("%s/%s/projects/%s", DEFAULT_HOST, VERSION, projectId)); } protected RemoteRpc newRemoteRpc(DatastoreOptions options) { @@ -127,12 +116,13 @@ private static String validateUrl(String url) { private static synchronized StreamHandler getStreamHandler() { if (methodHandler == null) { methodHandler = new ConsoleHandler(); - methodHandler.setFormatter(new Formatter() { - @Override - public String format(LogRecord record) { - return record.getMessage() + "\n"; - } - }); + methodHandler.setFormatter( + new Formatter() { + @Override + public String format(LogRecord record) { + return record.getMessage() + "\n"; + } + }); methodHandler.setLevel(Level.FINE); } return methodHandler; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java index 1b187f1c8..5b092c987 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreHelper.java @@ -56,9 +56,7 @@ import java.util.logging.Logger; import javax.annotation.Nullable; -/** - * Helper methods for {@link Datastore}. - */ +/** Helper methods for {@link Datastore}. */ // TODO: Accept OrBuilders when possible. public final class DatastoreHelper { private static final Logger logger = Logger.getLogger(DatastoreHelper.class.getName()); @@ -66,39 +64,29 @@ public final class DatastoreHelper { private static final int MICROSECONDS_PER_SECOND = 1000 * 1000; private static final int NANOSECONDS_PER_MICROSECOND = 1000; - /** The property used in the Datastore to give us a random distribution. **/ + /** The property used in the Datastore to give us a random distribution. * */ public static final String SCATTER_PROPERTY_NAME = "__scatter__"; - /** The property used in the Datastore to get the key of the entity. **/ + /** The property used in the Datastore to get the key of the entity. * */ public static final String KEY_PROPERTY_NAME = "__key__"; - /** - * Name of the environment variable used to set the project ID. - */ + /** Name of the environment variable used to set the project ID. */ public static final String PROJECT_ID_ENV_VAR = "DATASTORE_PROJECT_ID"; - - /** - * Name of the environment variable used to set the local host. - */ + + /** Name of the environment variable used to set the local host. */ public static final String LOCAL_HOST_ENV_VAR = "DATASTORE_EMULATOR_HOST"; - - /** - * Name of the environment variable used to set the service account. - */ + + /** Name of the environment variable used to set the service account. */ public static final String SERVICE_ACCOUNT_ENV_VAR = "DATASTORE_SERVICE_ACCOUNT"; - - /** - * Name of the environment variable used to set the private key file. - */ + + /** Name of the environment variable used to set the private key file. */ public static final String PRIVATE_KEY_FILE_ENV_VAR = "DATASTORE_PRIVATE_KEY_FILE"; private static final String URL_OVERRIDE_ENV_VAR = "__DATASTORE_URL_OVERRIDE"; private static final AtomicReference projectIdFromComputeEngine = new AtomicReference<>(); - /** - * Comparator for Keys - */ + /** Comparator for Keys */ private static final class KeyComparator implements Comparator { static final KeyComparator INSTANCE = new KeyComparator(); @@ -160,8 +148,8 @@ static JsonFactory newJsonFactory() { * @param privateKeyFile the file name from which to get the private key. * @return valid credentials or {@code null} */ - public static Credential getServiceAccountCredential(String serviceAccountId, - String privateKeyFile) throws GeneralSecurityException, IOException { + public static Credential getServiceAccountCredential( + String serviceAccountId, String privateKeyFile) throws GeneralSecurityException, IOException { return getServiceAccountCredential(serviceAccountId, privateKeyFile, DatastoreOptions.SCOPES); } @@ -170,13 +158,13 @@ public static Credential getServiceAccountCredential(String serviceAccountId, * * @param serviceAccountId service account ID (typically an e-mail address). * @param privateKeyFile the file name from which to get the private key. - * @param serviceAccountScopes Collection of OAuth scopes to use with the the service - * account flow or {@code null} if not. + * @param serviceAccountScopes Collection of OAuth scopes to use with the the service account flow + * or {@code null} if not. * @return valid credentials or {@code null} */ - public static Credential getServiceAccountCredential(String serviceAccountId, - String privateKeyFile, Collection serviceAccountScopes) - throws GeneralSecurityException, IOException { + public static Credential getServiceAccountCredential( + String serviceAccountId, String privateKeyFile, Collection serviceAccountScopes) + throws GeneralSecurityException, IOException { return getCredentialBuilderWithoutPrivateKey(serviceAccountId, serviceAccountScopes) .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFile)) .build(); @@ -187,12 +175,12 @@ public static Credential getServiceAccountCredential(String serviceAccountId, * * @param serviceAccountId service account ID (typically an e-mail address). * @param privateKey the private key for the given account. - * @param serviceAccountScopes Collection of OAuth scopes to use with the the service - * account flow or {@code null} if not. + * @param serviceAccountScopes Collection of OAuth scopes to use with the the service account flow + * or {@code null} if not. * @return valid credentials or {@code null} */ - public static Credential getServiceAccountCredential(String serviceAccountId, - PrivateKey privateKey, Collection serviceAccountScopes) + public static Credential getServiceAccountCredential( + String serviceAccountId, PrivateKey privateKey, Collection serviceAccountScopes) throws GeneralSecurityException, IOException { return getCredentialBuilderWithoutPrivateKey(serviceAccountId, serviceAccountScopes) .setServiceAccountPrivateKey(privateKey) @@ -214,20 +202,22 @@ private static GoogleCredential.Builder getCredentialBuilderWithoutPrivateKey( /** * Constructs a {@link Datastore} from environment variables and/or the Compute Engine metadata * server. - * + * *

The project ID is determined from, in order of preference: + * *

    *
  • DATASTORE_PROJECT_ID environment variable *
  • Compute Engine *
* *

Credentials are taken from, in order of preference: + * *

    *
  1. No credentials (if the DATASTORE_EMULATOR_HOST environment variable is set) *
  2. Service Account specified by the DATASTORE_SERVICE_ACCOUNT and DATASTORE_PRIVATE_KEY_FILE - * environment variables - *
  3. Google Application Default as described at - * {@link "https://developers.google.com/identity/protocols/application-default-credentials"} + * environment variables + *
  4. Google Application Default as described at {@link + * "https://developers.google.com/identity/protocols/application-default-credentials"} *
*/ public static DatastoreOptions.Builder getOptionsFromEnv() @@ -240,25 +230,28 @@ public static DatastoreOptions.Builder getOptionsFromEnv() private static Credential getCredentialFromEnv() throws GeneralSecurityException, IOException { if (System.getenv(LOCAL_HOST_ENV_VAR) != null) { - logger.log(Level.INFO, "{0} environment variable was set. Not using credentials.", + logger.log( + Level.INFO, + "{0} environment variable was set. Not using credentials.", new Object[] {LOCAL_HOST_ENV_VAR}); return null; } String serviceAccount = System.getenv(SERVICE_ACCOUNT_ENV_VAR); String privateKeyFile = System.getenv(PRIVATE_KEY_FILE_ENV_VAR); if (serviceAccount != null && privateKeyFile != null) { - logger.log(Level.INFO, "{0} and {1} environment variables were set. " - + "Using service account credential.", + logger.log( + Level.INFO, + "{0} and {1} environment variables were set. " + "Using service account credential.", new Object[] {SERVICE_ACCOUNT_ENV_VAR, PRIVATE_KEY_FILE_ENV_VAR}); return getServiceAccountCredential(serviceAccount, privateKeyFile); } - return GoogleCredential.getApplicationDefault() - .createScoped(DatastoreOptions.SCOPES); + return GoogleCredential.getApplicationDefault().createScoped(DatastoreOptions.SCOPES); } /** * Determines the project id from the environment. Uses the following sources in order of * preference: + * *
    *
  1. Value of the DATASTORE_PROJECT_ID environment variable *
  2. Compute Engine @@ -274,9 +267,12 @@ private static String getProjectIdFromEnv() { if (projectIdFromComputeEngine != null) { return projectIdFromComputeEngine; } - throw new IllegalStateException(String.format("Could not determine project ID." - + " If you are not running on Compute Engine, set the" - + " %s environment variable.", PROJECT_ID_ENV_VAR)); + throw new IllegalStateException( + String.format( + "Could not determine project ID." + + " If you are not running on Compute Engine, set the" + + " %s environment variable.", + PROJECT_ID_ENV_VAR)); } /** @@ -318,16 +314,17 @@ private static String queryProjectIdFromComputeEngine() { private static void setProjectEndpointFromEnv(DatastoreOptions.Builder options) { // DATASTORE_HOST is deprecated. if (System.getenv("DATASTORE_HOST") != null) { - logger.warning(String.format( - "Ignoring value of environment variable DATASTORE_HOST. " - + "To point datastore to a host running locally, use " - + "the environment variable %s.", - LOCAL_HOST_ENV_VAR)); + logger.warning( + String.format( + "Ignoring value of environment variable DATASTORE_HOST. " + + "To point datastore to a host running locally, use " + + "the environment variable %s.", + LOCAL_HOST_ENV_VAR)); } String projectId = getProjectIdFromEnv(); if (System.getenv(URL_OVERRIDE_ENV_VAR) != null) { - options.projectEndpoint(String.format("%s/projects/%s", - System.getenv(URL_OVERRIDE_ENV_VAR), projectId)); + options.projectEndpoint( + String.format("%s/projects/%s", System.getenv(URL_OVERRIDE_ENV_VAR), projectId)); return; } if (System.getenv(LOCAL_HOST_ENV_VAR) != null) { @@ -339,9 +336,7 @@ private static void setProjectEndpointFromEnv(DatastoreOptions.Builder options) return; } - /** - * @see #getOptionsFromEnv() - */ + /** @see #getOptionsFromEnv() */ public static Datastore getDatastoreFromEnv() throws GeneralSecurityException, IOException { return DatastoreFactory.get().create(getOptionsFromEnv().build()); } @@ -349,7 +344,7 @@ public static Datastore getDatastoreFromEnv() throws GeneralSecurityException, I /** * Gets a {@link QuerySplitter}. * - * The returned {@link QuerySplitter#getSplits} cannot accept a query that contains inequality + *

    The returned {@link QuerySplitter#getSplits} cannot accept a query that contains inequality * filters, a sort filter, or a missing kind. */ public static QuerySplitter getQuerySplitter() { @@ -360,79 +355,64 @@ public static Comparator getKeyComparator() { return KeyComparator.INSTANCE; } - /** - * Make a sort order for use in a query. - */ - public static PropertyOrder.Builder makeOrder(String property, - PropertyOrder.Direction direction) { + /** Make a sort order for use in a query. */ + public static PropertyOrder.Builder makeOrder( + String property, PropertyOrder.Direction direction) { return PropertyOrder.newBuilder() .setProperty(makePropertyReference(property)) .setDirection(direction); } - /** - * Makes an ancestor filter. - */ + /** Makes an ancestor filter. */ public static Filter.Builder makeAncestorFilter(Key ancestor) { return makeFilter( - DatastoreHelper.KEY_PROPERTY_NAME, PropertyFilter.Operator.HAS_ANCESTOR, + DatastoreHelper.KEY_PROPERTY_NAME, + PropertyFilter.Operator.HAS_ANCESTOR, makeValue(ancestor)); } - /** - * Make a filter on a property for use in a query. - */ - public static Filter.Builder makeFilter(String property, PropertyFilter.Operator operator, - Value value) { + /** Make a filter on a property for use in a query. */ + public static Filter.Builder makeFilter( + String property, PropertyFilter.Operator operator, Value value) { return Filter.newBuilder() - .setPropertyFilter(PropertyFilter.newBuilder() - .setProperty(makePropertyReference(property)) - .setOp(operator) - .setValue(value)); + .setPropertyFilter( + PropertyFilter.newBuilder() + .setProperty(makePropertyReference(property)) + .setOp(operator) + .setValue(value)); } - /** - * Make a filter on a property for use in a query. - */ - public static Filter.Builder makeFilter(String property, PropertyFilter.Operator operator, - Value.Builder value) { + /** Make a filter on a property for use in a query. */ + public static Filter.Builder makeFilter( + String property, PropertyFilter.Operator operator, Value.Builder value) { return makeFilter(property, operator, value.build()); } - /** - * Make a composite filter from the given sub-filters using AND to combine filters. - */ + /** Make a composite filter from the given sub-filters using AND to combine filters. */ public static Filter.Builder makeAndFilter(Filter... subfilters) { return makeAndFilter(Arrays.asList(subfilters)); } - /** - * Make a composite filter from the given sub-filters using AND to combine filters. - */ + /** Make a composite filter from the given sub-filters using AND to combine filters. */ public static Filter.Builder makeAndFilter(Iterable subfilters) { return Filter.newBuilder() - .setCompositeFilter(CompositeFilter.newBuilder() - .addAllFilters(subfilters) - .setOp(CompositeFilter.Operator.AND)); + .setCompositeFilter( + CompositeFilter.newBuilder() + .addAllFilters(subfilters) + .setOp(CompositeFilter.Operator.AND)); } - /** - * Make a property reference for use in a query. - */ + /** Make a property reference for use in a query. */ public static PropertyReference.Builder makePropertyReference(String propertyName) { return PropertyReference.newBuilder().setName(propertyName); } - /** - * Make an array value containing the specified values. - */ + /** Make an array value containing the specified values. */ public static Value.Builder makeValue(Iterable values) { return Value.newBuilder().setArrayValue(ArrayValue.newBuilder().addAllValues(values)); } - /** - * Make a list value containing the specified values. - */ + /** Make a list value containing the specified values. */ public static Value.Builder makeValue(Value value1, Value value2, Value... rest) { ArrayValue.Builder arrayValue = ArrayValue.newBuilder(); arrayValue.addValues(value1); @@ -441,11 +421,9 @@ public static Value.Builder makeValue(Value value1, Value value2, Value... rest) return Value.newBuilder().setArrayValue(arrayValue); } - /** - * Make an array value containing the specified values. - */ - public static Value.Builder makeValue(Value.Builder value1, Value.Builder value2, - Value.Builder... rest) { + /** Make an array value containing the specified values. */ + public static Value.Builder makeValue( + Value.Builder value1, Value.Builder value2, Value.Builder... rest) { ArrayValue.Builder arrayValue = ArrayValue.newBuilder(); arrayValue.addValues(value1); arrayValue.addValues(value2); @@ -455,72 +433,52 @@ public static Value.Builder makeValue(Value.Builder value1, Value.Builder value2 return Value.newBuilder().setArrayValue(arrayValue); } - /** - * Make a key value. - */ + /** Make a key value. */ public static Value.Builder makeValue(Key key) { return Value.newBuilder().setKeyValue(key); } - /** - * Make a key value. - */ + /** Make a key value. */ public static Value.Builder makeValue(Key.Builder key) { return makeValue(key.build()); } - /** - * Make an integer value. - */ + /** Make an integer value. */ public static Value.Builder makeValue(long key) { return Value.newBuilder().setIntegerValue(key); } - /** - * Make a floating point value. - */ + /** Make a floating point value. */ public static Value.Builder makeValue(double value) { return Value.newBuilder().setDoubleValue(value); } - /** - * Make a boolean value. - */ + /** Make a boolean value. */ public static Value.Builder makeValue(boolean value) { return Value.newBuilder().setBooleanValue(value); } - /** - * Make a string value. - */ + /** Make a string value. */ public static Value.Builder makeValue(String value) { return Value.newBuilder().setStringValue(value); } - /** - * Make an entity value. - */ + /** Make an entity value. */ public static Value.Builder makeValue(Entity entity) { return Value.newBuilder().setEntityValue(entity); } - /** - * Make a entity value. - */ + /** Make a entity value. */ public static Value.Builder makeValue(Entity.Builder entity) { return makeValue(entity.build()); } - /** - * Make a ByteString value. - */ + /** Make a ByteString value. */ public static Value.Builder makeValue(ByteString blob) { return Value.newBuilder().setBlobValue(blob); } - /** - * Make a timestamp value given a date. - */ + /** Make a timestamp value given a date. */ public static Value.Builder makeValue(Date date) { return Value.newBuilder().setTimestampValue(toTimestamp(date.getTime() * 1000L)); } @@ -539,45 +497,43 @@ private static Timestamp.Builder toTimestamp(long microseconds) { .setNanos((int) microsecondsRemainder * NANOSECONDS_PER_MICROSECOND); } - /** - * Makes a GeoPoint value. - */ + /** Makes a GeoPoint value. */ public static Value.Builder makeValue(LatLng value) { return Value.newBuilder().setGeoPointValue(value); } - /** - * Makes a GeoPoint value. - */ + /** Makes a GeoPoint value. */ public static Value.Builder makeValue(LatLng.Builder value) { return makeValue(value.build()); } /** - * Make a key from the specified path of kind/id-or-name pairs - * and/or Keys. + * Make a key from the specified path of kind/id-or-name pairs and/or Keys. * *

    The id-or-name values must be either String, Long, Integer or Short. * - *

    The last id-or-name value may be omitted, in which case an entity without - * an id is created (for use with automatic id allocation). + *

    The last id-or-name value may be omitted, in which case an entity without an id is created + * (for use with automatic id allocation). * - *

    The PartitionIds of all Keys in the path must be equal. The returned - * Key.Builder will use this PartitionId. + *

    The PartitionIds of all Keys in the path must be equal. The returned Key.Builder will use + * this PartitionId. */ public static Key.Builder makeKey(Object... elements) { Key.Builder key = Key.newBuilder(); PartitionId partitionId = null; for (int pathIndex = 0; pathIndex < elements.length; pathIndex += 2) { PathElement.Builder pathElement = PathElement.newBuilder(); - Object element = elements[pathIndex]; + Object element = elements[pathIndex]; if (element instanceof Key) { Key subKey = (Key) element; if (partitionId == null) { partitionId = subKey.getPartitionId(); } else if (!partitionId.equals(subKey.getPartitionId())) { - throw new IllegalArgumentException("Partition IDs did not match, found: " - + partitionId + " and " + subKey.getPartitionId()); + throw new IllegalArgumentException( + "Partition IDs did not match, found: " + + partitionId + + " and " + + subKey.getPartitionId()); } key.addAllPath(((Key) element).getPathList()); // We increment by 2, but since we got a Key argument we're only consuming 1 element in this diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java index d70d3c00c..9ad9ca552 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/DatastoreOptions.java @@ -48,11 +48,12 @@ public class DatastoreOptions { private final Credential credential; private final HttpTransport transport; - public static final List SCOPES = Arrays.asList( - "https://www.googleapis.com/auth/datastore"); + public static final List SCOPES = + Arrays.asList("https://www.googleapis.com/auth/datastore"); DatastoreOptions(Builder b) { - checkArgument(b.projectId != null || b.projectEndpoint != null, + checkArgument( + b.projectId != null || b.projectEndpoint != null, "Either project ID or project endpoint must be provided."); this.projectId = b.projectId; this.projectEndpoint = b.projectEndpoint; @@ -63,9 +64,7 @@ public class DatastoreOptions { this.transport = b.transport; } - /** - * Builder for {@link DatastoreOptions}. - */ + /** Builder for {@link DatastoreOptions}. */ public static class Builder { private static final String PROJECT_ENDPOINT_AND_PROJECT_ID_ERROR = "Cannot set both project endpoint and project ID."; @@ -80,7 +79,7 @@ public static class Builder { private Credential credential; private HttpTransport transport; - public Builder() { } + public Builder() {} public Builder(DatastoreOptions options) { this.projectId = options.projectId; @@ -96,9 +95,7 @@ public DatastoreOptions build() { return new DatastoreOptions(this); } - /** - * Sets the project ID used to access Cloud Datastore. - */ + /** Sets the project ID used to access Cloud Datastore. */ public Builder projectId(String projectId) { checkArgument(projectEndpoint == null, PROJECT_ENDPOINT_AND_PROJECT_ID_ERROR); this.projectId = projectId; @@ -106,8 +103,8 @@ public Builder projectId(String projectId) { } /** - * Sets the host used to access Cloud Datastore. To connect to the Cloud Datastore Emulator, - * use {@link #localHost} instead. + * Sets the host used to access Cloud Datastore. To connect to the Cloud Datastore Emulator, use + * {@link #localHost} instead. */ public Builder host(String host) { checkArgument(projectEndpoint == null && localHost == null, PROJECT_ENDPOINT_AND_HOST_ERROR); @@ -121,8 +118,8 @@ public Builder host(String host) { /** * Configures the client to access Cloud Datastore on a local host (typically a Cloud Datastore - * Emulator instance). Call this method also configures the client not to attach credentials - * to requests. + * Emulator instance). Call this method also configures the client not to attach credentials to + * requests. */ public Builder localHost(String localHost) { checkArgument(projectEndpoint == null && host == null, PROJECT_ENDPOINT_AND_HOST_ERROR); @@ -137,7 +134,7 @@ public Builder localHost(String localHost) { /** * Sets the project endpoint used to access Cloud Datastore. Prefer using {@link #projectId} * and/or {@link #host}/{@link #localHost} when possible. - * + * * @deprecated Use {@link #projectId} and/or {@link #host}/{@link #localHost} instead. */ @Deprecated @@ -145,32 +142,26 @@ public Builder projectEndpoint(String projectEndpoint) { checkArgument(projectId == null, PROJECT_ENDPOINT_AND_PROJECT_ID_ERROR); checkArgument(localHost == null && host == null, PROJECT_ENDPOINT_AND_HOST_ERROR); if (!includesScheme(projectEndpoint)) { - throw new IllegalArgumentException(String.format( - "Project endpoint \"%s\" must include scheme.", projectEndpoint)); + throw new IllegalArgumentException( + String.format("Project endpoint \"%s\" must include scheme.", projectEndpoint)); } this.projectEndpoint = projectEndpoint; return this; } - /** - * Sets the (optional) initializer to run on HTTP requests to Cloud Datastore. - */ + /** Sets the (optional) initializer to run on HTTP requests to Cloud Datastore. */ public Builder initializer(HttpRequestInitializer initializer) { this.initializer = initializer; return this; } - /** - * Sets the Google APIs {@link Credential} used to access Cloud Datastore. - */ + /** Sets the Google APIs {@link Credential} used to access Cloud Datastore. */ public Builder credential(Credential credential) { this.credential = credential; return this; } - /** - * Sets the transport used to access Cloud Datastore. - */ + /** Sets the transport used to access Cloud Datastore. */ public Builder transport(HttpTransport transport) { this.transport = transport; return this; diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/EndToEndChecksumHandler.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/EndToEndChecksumHandler.java index 591ebe3f6..06d08d499 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/EndToEndChecksumHandler.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/EndToEndChecksumHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2021 Google LLC. All Rights Reserved. + * Copyright 2021 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,8 +39,8 @@ static String computeChecksum(byte[] bytes) { if (bytes == null || (bytes.length == 0)) { return null; } - return com.google.common.io.BaseEncoding.base16().encode( - getMessageDigestInstance().digest(bytes)); + return com.google.common.io.BaseEncoding.base16() + .encode(getMessageDigestInstance().digest(bytes)); } /** diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitter.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitter.java index e220ad35c..44b1c2c50 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitter.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitter.java @@ -17,12 +17,9 @@ import com.google.datastore.v1.PartitionId; import com.google.datastore.v1.Query; - import java.util.List; -/** - * Provides the ability to split a query into multiple shards. - */ +/** Provides the ability to split a query into multiple shards. */ public interface QuerySplitter { /** diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java index 91c7fb4b2..d1055cdd3 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java @@ -30,7 +30,6 @@ import com.google.datastore.v1.QueryResultBatch; import com.google.datastore.v1.QueryResultBatch.MoreResultsType; import com.google.datastore.v1.RunQueryRequest; - import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; @@ -39,16 +38,20 @@ /** * Provides the ability to split a query into multiple shards using Cloud Datastore. * - *

    This implementation of the QuerySplitter uses the __scatter__ property to gather - * random split points for a query. + *

    This implementation of the QuerySplitter uses the __scatter__ property to gather random split + * points for a query. */ final class QuerySplitterImpl implements QuerySplitter { - /** The number of keys to sample for each split. **/ + /** The number of keys to sample for each split. * */ private static final int KEYS_PER_SPLIT = 32; - private static final EnumSet UNSUPPORTED_OPERATORS = EnumSet.of(Operator.LESS_THAN, - Operator.LESS_THAN_OR_EQUAL, Operator.GREATER_THAN, Operator.GREATER_THAN_OR_EQUAL); + private static final EnumSet UNSUPPORTED_OPERATORS = + EnumSet.of( + Operator.LESS_THAN, + Operator.LESS_THAN_OR_EQUAL, + Operator.GREATER_THAN, + Operator.GREATER_THAN_OR_EQUAL); static final QuerySplitter INSTANCE = new QuerySplitterImpl(); @@ -81,6 +84,7 @@ public List getSplits( /** * Verify that the given number of splits is not out of bounds. + * * @param numSplits the number of splits. * @throws IllegalArgumentException if the split size is invalid. */ @@ -93,8 +97,8 @@ private void validateSplitSize(int numSplits) throws IllegalArgumentException { /** * Validates that we only have allowable filters. * - *

    Note that equality and ancestor filters are allowed, however they may result in - * inefficient sharding. + *

    Note that equality and ancestor filters are allowed, however they may result in inefficient + * sharding. */ private void validateFilter(Filter filter) throws IllegalArgumentException { switch (filter.getFilterTypeCase()) { @@ -148,15 +152,21 @@ private Query createSplit(Key lastKey, Key nextKey, Query query) { keyFilters.add(query.getFilter()); } if (lastKey != null) { - Filter lowerBound = DatastoreHelper.makeFilter(DatastoreHelper.KEY_PROPERTY_NAME, - PropertyFilter.Operator.GREATER_THAN_OR_EQUAL, - DatastoreHelper.makeValue(lastKey)).build(); + Filter lowerBound = + DatastoreHelper.makeFilter( + DatastoreHelper.KEY_PROPERTY_NAME, + PropertyFilter.Operator.GREATER_THAN_OR_EQUAL, + DatastoreHelper.makeValue(lastKey)) + .build(); keyFilters.add(lowerBound); } if (nextKey != null) { - Filter upperBound = DatastoreHelper.makeFilter(DatastoreHelper.KEY_PROPERTY_NAME, - PropertyFilter.Operator.LESS_THAN, - DatastoreHelper.makeValue(nextKey)).build(); + Filter upperBound = + DatastoreHelper.makeFilter( + DatastoreHelper.KEY_PROPERTY_NAME, + PropertyFilter.Operator.LESS_THAN, + DatastoreHelper.makeValue(nextKey)) + .build(); keyFilters.add(upperBound); } return Query.newBuilder(query).setFilter(makeAndFilter(keyFilters)).build(); @@ -165,9 +175,8 @@ private Query createSplit(Key lastKey, Key nextKey, Query query) { /** * Gets a list of split keys given a desired number of splits. * - *

    This list will contain multiple split keys for each split. Only a single split key - * will be chosen as the split point, however providing multiple keys allows for more uniform - * sharding. + *

    This list will contain multiple split keys for each split. Only a single split key will be + * chosen as the split point, however providing multiple keys allows for more uniform sharding. * * @param numSplits the number of desired splits. * @param query the user query. @@ -194,8 +203,9 @@ private List getScatterKeys( keySplits.add(result.getEntity().getKey()); } scatterPointQuery.setStartCursor(batch.getEndCursor()); - scatterPointQuery.getLimitBuilder().setValue( - scatterPointQuery.getLimit().getValue() - batch.getEntityResultsCount()); + scatterPointQuery + .getLimitBuilder() + .setValue(scatterPointQuery.getLimit().getValue() - batch.getEntityResultsCount()); } while (batch.getMoreResults() == MoreResultsType.NOT_FINISHED); Collections.sort(keySplits, DatastoreHelper.getKeyComparator()); return keySplits; @@ -214,16 +224,16 @@ private Query.Builder createScatterQuery(Query query, int numSplits) { // the same category. Query.Builder scatterPointQuery = Query.newBuilder(); scatterPointQuery.addAllKind(query.getKindList()); - scatterPointQuery.addOrder(DatastoreHelper.makeOrder( - DatastoreHelper.SCATTER_PROPERTY_NAME, Direction.ASCENDING)); + scatterPointQuery.addOrder( + DatastoreHelper.makeOrder(DatastoreHelper.SCATTER_PROPERTY_NAME, Direction.ASCENDING)); // There is a split containing entities before and after each scatter entity: // ||---*------*------*------*------*------*------*---|| = scatter entity // If we represent each split as a region before a scatter entity, there is an extra region // following the last scatter point. Thus, we do not need the scatter entities for the last // region. scatterPointQuery.getLimitBuilder().setValue((numSplits - 1) * KEYS_PER_SPLIT); - scatterPointQuery.addProjection(Projection.newBuilder().setProperty( - PropertyReference.newBuilder().setName("__key__"))); + scatterPointQuery.addProjection( + Projection.newBuilder().setProperty(PropertyReference.newBuilder().setName("__key__"))); return scatterPointQuery; } @@ -267,5 +277,4 @@ private Iterable getSplitKey(List keys, int numSplits) { return keysList; } - } diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java index e0b2e378b..17763c9c1 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java @@ -43,8 +43,7 @@ class RemoteRpc { private static final Logger logger = Logger.getLogger(RemoteRpc.class.getName()); - @VisibleForTesting - static final String API_FORMAT_VERSION_HEADER = "X-Goog-Api-Format-Version"; + @VisibleForTesting static final String API_FORMAT_VERSION_HEADER = "X-Goog-Api-Format-Version"; private static final String API_FORMAT_VERSION = "2"; private final HttpRequestFactory client; @@ -52,8 +51,8 @@ class RemoteRpc { private final String url; private final AtomicInteger rpcCount = new AtomicInteger(0); // Not final - so it can be set/reset in Unittests - private static boolean enableE2EChecksum = Boolean.parseBoolean( - System.getenv("GOOGLE_CLOUD_DATASTORE_HTTP_ENABLE_E2E_CHECKSUM")); + private static boolean enableE2EChecksum = + Boolean.parseBoolean(System.getenv("GOOGLE_CLOUD_DATASTORE_HTTP_ENABLE_E2E_CHECKSUM")); RemoteRpc(HttpRequestFactory client, HttpRequestInitializer initializer, String url) { this.client = client; @@ -70,8 +69,8 @@ class RemoteRpc { /** * Makes an RPC call using the client. Logs how long it took and any exceptions. * - * NOTE: The request could be an InputStream too, but the http client will need to - * find its length, which will require buffering it anyways. + *

    NOTE: The request could be an InputStream too, but the http client will need to find its + * length, which will require buffering it anyways. * * @throws DatastoreException if the RPC fails. */ @@ -98,16 +97,20 @@ public InputStream call(String methodName, MessageLite request) throws Datastore httpResponse = httpRequest.execute(); if (!httpResponse.isSuccessStatusCode()) { try (InputStream content = httpResponse.getContent()) { - throw makeException(url, methodName, content, - httpResponse.getContentType(), httpResponse.getContentCharset(), null, + throw makeException( + url, + methodName, + content, + httpResponse.getContentType(), + httpResponse.getContentCharset(), + null, httpResponse.getStatusCode()); } } InputStream inputStream = httpResponse.getContent(); return enableE2EChecksum && EndToEndChecksumHandler.hasChecksumHeader(httpResponse) - ? new ChecksumEnforcingInputStream(inputStream, - httpResponse, - EndToEndChecksumHandler.getMessageDigestInstance()) + ? new ChecksumEnforcingInputStream( + inputStream, httpResponse, EndToEndChecksumHandler.getMessageDigestInstance()) : inputStream; } catch (SocketTimeoutException e) { throw makeException(url, methodName, Code.DEADLINE_EXCEEDED, "Deadline exceeded", e); @@ -126,8 +129,9 @@ void setHeaders(MessageLite request, HttpRequest httpRequest) { if (enableE2EChecksum && request != null) { String checksum = EndToEndChecksumHandler.computeChecksum(request.toByteArray()); if (checksum != null) { - httpRequest.getHeaders().put(EndToEndChecksumHandler.HTTP_REQUEST_CHECKSUM_HEADER, - checksum); + httpRequest + .getHeaders() + .put(EndToEndChecksumHandler.HTTP_REQUEST_CHECKSUM_HEADER, checksum); } } } @@ -162,14 +166,20 @@ HttpRequestFactory getHttpRequestFactory() { return client; } - public static DatastoreException makeException(String url, String methodName, Code code, - String message, Throwable cause) { - logger.fine("remote datastore call " + methodName + " against " + url + " failed: " + message); + public static DatastoreException makeException( + String url, String methodName, Code code, String message, Throwable cause) { + logger.fine("remote datastore call " + methodName + " against " + url + " failed: " + message); return new DatastoreException(methodName, code, message, cause); } - static DatastoreException makeException(String url, String methodName, InputStream content, - String contentType, Charset contentCharset, Throwable cause, int httpStatusCode) { + static DatastoreException makeException( + String url, + String methodName, + InputStream content, + String contentType, + Charset contentCharset, + Throwable cause, + int httpStatusCode) { if (!contentType.equals("application/x-protobuf")) { String responseContent; try { @@ -179,7 +189,10 @@ static DatastoreException makeException(String url, String methodName, InputStre } catch (IOException e) { responseContent = ""; } - return makeException(url, methodName, Code.INTERNAL, + return makeException( + url, + methodName, + Code.INTERNAL, String.format( "Non-protobuf error: %s. HTTP status code was %d.", responseContent, httpStatusCode), cause); @@ -189,7 +202,10 @@ static DatastoreException makeException(String url, String methodName, InputStre try { rpcStatus = Status.parseFrom(content); } catch (IOException e) { - return makeException(url, methodName, Code.INTERNAL, + return makeException( + url, + methodName, + Code.INTERNAL, String.format( "Unable to parse Status protocol buffer: HTTP status code was %s.", httpStatusCode), e); @@ -197,7 +213,10 @@ static DatastoreException makeException(String url, String methodName, InputStre Code code = Code.forNumber(rpcStatus.getCode()); if (code == null) { - return makeException(url, methodName, Code.INTERNAL, + return makeException( + url, + methodName, + Code.INTERNAL, String.format( "Invalid error code: %d. Message: %s.", rpcStatus.getCode(), rpcStatus.getMessage()), cause); @@ -208,8 +227,12 @@ static DatastoreException makeException(String url, String methodName, InputStre if (httpStatusCode == HttpStatusCodes.STATUS_CODE_UNAUTHORIZED) { return makeException(url, methodName, Code.UNAUTHENTICATED, "Unauthenticated.", cause); } - return makeException(url, methodName, Code.INTERNAL, - String.format("Unexpected OK error code with HTTP status code of %d. Message: %s.", + return makeException( + url, + methodName, + Code.INTERNAL, + String.format( + "Unexpected OK error code with HTTP status code of %d. Message: %s.", httpStatusCode, rpcStatus.getMessage()), cause); } diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java index 32d43fb44..997205667 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2021 Google LLC. All Rights Reserved. + * Copyright 2021 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,7 +38,7 @@ public void test(int payloadSize) throws Exception { byte[] buf = new byte[1000]; try { while (testInstance.read(buf, 0, 1000) != -1) { - // do nothing with the bytes read + // do nothing with the bytes read } } catch (IOException e) { fail("checksum verification failed!"); @@ -63,10 +63,11 @@ public void read_withValidChecksum_differentPayloadSizes() throws Exception { @Test public void read_withInvalidChecksum() { // build a test instance with invalidchecksum - ChecksumEnforcingInputStream instance = new ChecksumEnforcingInputStream( - new ByteArrayInputStream("hello there".getBytes(UTF_8)), - "this checksum is invalid", - digest); + ChecksumEnforcingInputStream instance = + new ChecksumEnforcingInputStream( + new ByteArrayInputStream("hello there".getBytes(UTF_8)), + "this checksum is invalid", + digest); // read 1000 bytes at a time // Since checksum should be correct, do not expect IOException byte[] buf = new byte[1000]; @@ -103,7 +104,7 @@ private ChecksumEnforcingInputStream setUpData(int payloadSize) throws Exception } byte[] bytes = payload.getBytes(UTF_8); String expectedChecksum = EndToEndChecksumHandler.computeChecksum(bytes); - return new ChecksumEnforcingInputStream(new ByteArrayInputStream(bytes), - expectedChecksum, digest); + return new ChecksumEnforcingInputStream( + new ByteArrayInputStream(bytes), expectedChecksum, digest); } } diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreFactoryTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreFactoryTest.java index 22e145996..0414bcbe9 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreFactoryTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreFactoryTest.java @@ -23,83 +23,69 @@ import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.javanet.NetHttpTransport; - import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** - * Test for {@link DatastoreFactory}. - */ +/** Test for {@link DatastoreFactory}. */ @RunWith(JUnit4.class) public class DatastoreFactoryTest { private static final String PROJECT_ID = "project-id"; - + private DatastoreFactory factory = DatastoreFactory.get(); /** - * Without specifying a credential or transport, the factory will create - * a default transport on its own. + * Without specifying a credential or transport, the factory will create a default transport on + * its own. */ @Test public void makeClient_Default() { - DatastoreOptions options = new DatastoreOptions.Builder() - .projectId(PROJECT_ID) - .build(); + DatastoreOptions options = new DatastoreOptions.Builder().projectId(PROJECT_ID).build(); HttpRequestFactory f = factory.makeClient(options); assertNotNull(f.getTransport()); assertTrue(f.getTransport() instanceof NetHttpTransport); } /** - * Specifying a credential, but not a transport, the factory will use the - * transport from the credential. + * Specifying a credential, but not a transport, the factory will use the transport from the + * credential. */ @Test public void makeClient_WithCredential() { NetHttpTransport transport = new NetHttpTransport(); - GoogleCredential credential = new GoogleCredential.Builder() - .setTransport(transport) - .build(); - DatastoreOptions options = new DatastoreOptions.Builder() - .projectId(PROJECT_ID) - .credential(credential) - .build(); + GoogleCredential credential = new GoogleCredential.Builder().setTransport(transport).build(); + DatastoreOptions options = + new DatastoreOptions.Builder().projectId(PROJECT_ID).credential(credential).build(); HttpRequestFactory f = factory.makeClient(options); assertEquals(transport, f.getTransport()); } - - /** - * Specifying a transport, but not a credential, the factory will use the - * transport specified. - */ + + /** Specifying a transport, but not a credential, the factory will use the transport specified. */ @Test public void makeClient_WithTransport() { NetHttpTransport transport = new NetHttpTransport(); - DatastoreOptions options = new DatastoreOptions.Builder() - .projectId(PROJECT_ID) - .transport(transport) - .build(); + DatastoreOptions options = + new DatastoreOptions.Builder().projectId(PROJECT_ID).transport(transport).build(); HttpRequestFactory f = factory.makeClient(options); assertEquals(transport, f.getTransport()); } - + /** - * Specifying both credential and transport, the factory will use the - * transport specified and not the one in the credential. + * Specifying both credential and transport, the factory will use the transport specified and not + * the one in the credential. */ @Test public void makeClient_WithCredentialTransport() { NetHttpTransport credTransport = new NetHttpTransport(); NetHttpTransport transport = new NetHttpTransport(); - GoogleCredential credential = new GoogleCredential.Builder() - .setTransport(credTransport) - .build(); - DatastoreOptions options = new DatastoreOptions.Builder() - .projectId(PROJECT_ID) - .credential(credential) - .transport(transport) - .build(); + GoogleCredential credential = + new GoogleCredential.Builder().setTransport(credTransport).build(); + DatastoreOptions options = + new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .credential(credential) + .transport(transport) + .build(); HttpRequestFactory f = factory.makeClient(options); assertNotSame(credTransport, f.getTransport()); assertEquals(transport, f.getTransport()); diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreHelperTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreHelperTest.java index 1bd166bcc..5fcb214e8 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreHelperTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreHelperTest.java @@ -28,34 +28,23 @@ import com.google.datastore.v1.Value.ValueTypeCase; import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; - +import java.util.Date; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import java.util.Date; - -/** - * Tests for {@link DatastoreHelper}. - */ +/** Tests for {@link DatastoreHelper}. */ @RunWith(JUnit4.class) public class DatastoreHelperTest { - private static final Key PARENT = Key.newBuilder() - .addPath(Key.PathElement.newBuilder() - .setKind("Parent") - .setId(23L)) - .build(); - private static final Key GRANDPARENT = Key.newBuilder() - .addPath(Key.PathElement.newBuilder() - .setKind("Grandparent") - .setId(24L)) - .build(); - private static final Key CHILD = Key.newBuilder() - .addPath(Key.PathElement.newBuilder() - .setKind("Child") - .setId(26L)) - .build(); + private static final Key PARENT = + Key.newBuilder().addPath(Key.PathElement.newBuilder().setKind("Parent").setId(23L)).build(); + private static final Key GRANDPARENT = + Key.newBuilder() + .addPath(Key.PathElement.newBuilder().setKind("Grandparent").setId(24L)) + .build(); + private static final Key CHILD = + Key.newBuilder().addPath(Key.PathElement.newBuilder().setKind("Child").setId(26L)).build(); @Test public void testMakeKey_BadTypeForKind() { @@ -83,45 +72,35 @@ public void testMakeKey_Empty() { @Test public void testMakeKey_Incomplete() { assertEquals( - Key.newBuilder() - .addPath(Key.PathElement.newBuilder().setKind("Foo")) - .build(), + Key.newBuilder().addPath(Key.PathElement.newBuilder().setKind("Foo")).build(), makeKey("Foo").build()); } @Test public void testMakeKey_IdInt() { assertEquals( - Key.newBuilder() - .addPath(Key.PathElement.newBuilder().setKind("Foo").setId(1)) - .build(), + Key.newBuilder().addPath(Key.PathElement.newBuilder().setKind("Foo").setId(1)).build(), makeKey("Foo", 1).build()); } @Test public void testMakeKey_IdLong() { assertEquals( - Key.newBuilder() - .addPath(Key.PathElement.newBuilder().setKind("Foo").setId(1)) - .build(), + Key.newBuilder().addPath(Key.PathElement.newBuilder().setKind("Foo").setId(1)).build(), makeKey("Foo", 1L).build()); } @Test public void testMakeKey_IdShort() { assertEquals( - Key.newBuilder() - .addPath(Key.PathElement.newBuilder().setKind("Foo").setId(1)) - .build(), + Key.newBuilder().addPath(Key.PathElement.newBuilder().setKind("Foo").setId(1)).build(), makeKey("Foo", (short) 1).build()); } @Test public void testMakeKey_Name() { assertEquals( - Key.newBuilder() - .addPath(Key.PathElement.newBuilder().setKind("Foo").setName("hi")) - .build(), + Key.newBuilder().addPath(Key.PathElement.newBuilder().setKind("Foo").setName("hi")).build(), makeKey("Foo", "hi").build()); } @@ -184,21 +163,14 @@ public void testMakeKey_KeyKindIdKey() { @Test public void testMakeKey_Key() { // Just 1 key - assertEquals( - Key.newBuilder() - .addPath(CHILD.getPath(0)) - .build(), - makeKey(CHILD).build()); + assertEquals(Key.newBuilder().addPath(CHILD.getPath(0)).build(), makeKey(CHILD).build()); } @Test public void testMakeKey_KeyKey() { // Just 2 keys assertEquals( - Key.newBuilder() - .addPath(PARENT.getPath(0)) - .addPath(CHILD.getPath(0)) - .build(), + Key.newBuilder().addPath(PARENT.getPath(0)).addPath(CHILD.getPath(0)).build(), makeKey(PARENT, CHILD).build()); } @@ -252,12 +224,8 @@ public void testMakeKey_MultiLevelKey() { @Test public void testMakeKey_PartitionId() { - PartitionId partitionId = PartitionId.newBuilder() - .setNamespaceId("namespace-id") - .build(); - Key parent = PARENT.toBuilder() - .setPartitionId(partitionId) - .build(); + PartitionId partitionId = PartitionId.newBuilder().setNamespaceId("namespace-id").build(); + Key parent = PARENT.toBuilder().setPartitionId(partitionId).build(); assertEquals( Key.newBuilder() .setPartitionId(partitionId) @@ -269,12 +237,9 @@ public void testMakeKey_PartitionId() { @Test public void testMakeKey_NonMatchingPartitionId2() { - PartitionId partitionId1 = PartitionId.newBuilder() - .setNamespaceId("namespace-id") - .build(); - PartitionId partitionId2 = PartitionId.newBuilder() - .setNamespaceId("another-namespace-id") - .build(); + PartitionId partitionId1 = PartitionId.newBuilder().setNamespaceId("namespace-id").build(); + PartitionId partitionId2 = + PartitionId.newBuilder().setNamespaceId("another-namespace-id").build(); try { makeKey( PARENT.toBuilder().setPartitionId(partitionId1).build(), @@ -302,9 +267,9 @@ public void testMakeTimestampValue() throws Exception { // Timestamp specification requires that nanos >= 0 even if the timestamp // is before the epoch. assertConversion(0, 0, 0); - assertConversion(-250, -1, 750_000_000); // 1/4 second before epoch - assertConversion(-500, -1, 500_000_000); // 1/2 second before epoch - assertConversion(-750, -1, 250_000_000); // 3/4 second before epoch + assertConversion(-250, -1, 750_000_000); // 1/4 second before epoch + assertConversion(-500, -1, 500_000_000); // 1/2 second before epoch + assertConversion(-750, -1, 250_000_000); // 3/4 second before epoch // If nanos % 1_000_000 != 0, precision is lost (via truncation) when // converting to milliseconds. @@ -329,16 +294,16 @@ private void assertMillisecondsToTimestamp(long millis, long seconds, long nanos } private void assertTimestampToMilliseconds(long millis, long seconds, int nanos) { - Value.Builder value = Value.newBuilder().setTimestampValue(Timestamp.newBuilder() - .setSeconds(seconds) - .setNanos(nanos)); + Value.Builder value = + Value.newBuilder() + .setTimestampValue(Timestamp.newBuilder().setSeconds(seconds).setNanos(nanos)); assertEquals(millis, DatastoreHelper.toDate(value.build()).getTime()); } @Test public void testProjectionHandling() { - assertEquals(ByteString.copyFromUtf8("hi"), - getByteString(makeValue("hi").setMeaning(18).build())); + assertEquals( + ByteString.copyFromUtf8("hi"), getByteString(makeValue("hi").setMeaning(18).build())); try { getByteString(makeValue("hi").build()); fail("Expected IllegalArgumentException"); diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java index 617960bd3..a4a51cd15 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreTest.java @@ -68,20 +68,16 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** - * Tests for {@link DatastoreFactory} and {@link Datastore}. - */ +/** Tests for {@link DatastoreFactory} and {@link Datastore}. */ @RunWith(JUnit4.class) public class DatastoreTest { private static final String PROJECT_ID = "project-id"; - @Rule - public ExpectedException thrown = ExpectedException.none(); + @Rule public ExpectedException thrown = ExpectedException.none(); private DatastoreFactory factory = new MockDatastoreFactory(); - private DatastoreOptions.Builder options = new DatastoreOptions.Builder() - .projectId(PROJECT_ID) - .credential(new MockCredential()); + private DatastoreOptions.Builder options = + new DatastoreOptions.Builder().projectId(PROJECT_ID).credential(new MockCredential()); @Test public void options_NoProjectIdOrProjectEndpoint() throws Exception { @@ -95,73 +91,73 @@ public void options_NoProjectIdOrProjectEndpoint() throws Exception { public void options_ProjectIdAndProjectEndpoint() throws Exception { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Cannot set both project endpoint and project ID"); - options = new DatastoreOptions.Builder() - .projectId(PROJECT_ID) - .projectEndpoint("http://localhost:1234/datastore/v1beta42/projects/project-id"); + options = + new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .projectEndpoint("http://localhost:1234/datastore/v1beta42/projects/project-id"); } @Test public void options_LocalHostAndProjectEndpoint() throws Exception { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Can set at most one of project endpoint, host, and local host"); - options = new DatastoreOptions.Builder() - .localHost("localhost:8080") - .projectEndpoint("http://localhost:1234/datastore/v1beta42/projects/project-id"); + options = + new DatastoreOptions.Builder() + .localHost("localhost:8080") + .projectEndpoint("http://localhost:1234/datastore/v1beta42/projects/project-id"); } @Test public void options_HostAndProjectEndpoint() throws Exception { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Can set at most one of project endpoint, host, and local host"); - options = new DatastoreOptions.Builder() - .host("foo-datastore.googleapis.com") - .projectEndpoint("http://localhost:1234/datastore/v1beta42/projects/project-id"); + options = + new DatastoreOptions.Builder() + .host("foo-datastore.googleapis.com") + .projectEndpoint("http://localhost:1234/datastore/v1beta42/projects/project-id"); } @Test public void options_HostAndLocalHost() throws Exception { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Can set at most one of project endpoint, host, and local host"); - options = new DatastoreOptions.Builder() - .host("foo-datastore.googleapis.com") - .localHost("localhost:8080"); + options = + new DatastoreOptions.Builder() + .host("foo-datastore.googleapis.com") + .localHost("localhost:8080"); } @Test public void options_InvalidLocalHost() throws Exception { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Illegal character"); - factory.create(new DatastoreOptions.Builder() - .projectId(PROJECT_ID) - .localHost("!not a valid url!") - .build()); + factory.create( + new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .localHost("!not a valid url!") + .build()); } @Test public void options_SchemeInLocalHost() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Local host \"http://localhost:8080\" must not include scheme"); - new DatastoreOptions.Builder() - .localHost("http://localhost:8080"); + new DatastoreOptions.Builder().localHost("http://localhost:8080"); } @Test public void options_InvalidHost() throws Exception { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Illegal character"); - factory.create(new DatastoreOptions.Builder() - .projectId(PROJECT_ID) - .host("!not a valid url!") - .build()); + factory.create( + new DatastoreOptions.Builder().projectId(PROJECT_ID).host("!not a valid url!").build()); } @Test public void options_SchemeInHost() { thrown.expect(IllegalArgumentException.class); - thrown.expectMessage( - "Host \"http://foo-datastore.googleapis.com\" must not include scheme"); - new DatastoreOptions.Builder() - .host("http://foo-datastore.googleapis.com"); + thrown.expectMessage("Host \"http://foo-datastore.googleapis.com\" must not include scheme"); + new DatastoreOptions.Builder().host("http://foo-datastore.googleapis.com"); } @Test @@ -172,48 +168,55 @@ public void create_NullOptions() throws Exception { @Test public void create_Host() { - Datastore datastore = factory.create(new DatastoreOptions.Builder() - .projectId(PROJECT_ID) - .host("foo-datastore.googleapis.com") - .build()); + Datastore datastore = + factory.create( + new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .host("foo-datastore.googleapis.com") + .build()); assertThat(datastore.remoteRpc.getUrl()) .isEqualTo("https://foo-datastore.googleapis.com/v1/projects/project-id"); } @Test public void create_LocalHost() { - Datastore datastore = factory.create(new DatastoreOptions.Builder() - .projectId(PROJECT_ID) - .localHost("localhost:8080") - .build()); + Datastore datastore = + factory.create( + new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .localHost("localhost:8080") + .build()); assertThat(datastore.remoteRpc.getUrl()) .isEqualTo("http://localhost:8080/v1/projects/project-id"); } @Test public void create_LocalHostIp() { - Datastore datastore = factory.create(new DatastoreOptions.Builder() - .projectId(PROJECT_ID) - .localHost("127.0.0.1:8080") - .build()); + Datastore datastore = + factory.create( + new DatastoreOptions.Builder() + .projectId(PROJECT_ID) + .localHost("127.0.0.1:8080") + .build()); assertThat(datastore.remoteRpc.getUrl()) .isEqualTo("http://127.0.0.1:8080/v1/projects/project-id"); } @Test public void create_DefaultHost() { - Datastore datastore = factory.create(new DatastoreOptions.Builder() - .projectId(PROJECT_ID) - .build()); + Datastore datastore = + factory.create(new DatastoreOptions.Builder().projectId(PROJECT_ID).build()); assertThat(datastore.remoteRpc.getUrl()) .isEqualTo("https://datastore.googleapis.com/v1/projects/project-id"); } @Test public void create_ProjectEndpoint() { - Datastore datastore = factory.create(new DatastoreOptions.Builder() - .projectEndpoint("http://prom-qa/datastore/v1beta42/projects/project-id") - .build()); + Datastore datastore = + factory.create( + new DatastoreOptions.Builder() + .projectEndpoint("http://prom-qa/datastore/v1beta42/projects/project-id") + .build()); assertThat(datastore.remoteRpc.getUrl()) .isEqualTo("http://prom-qa/datastore/v1beta42/projects/project-id"); } @@ -223,20 +226,22 @@ public void create_ProjectEndpointNoScheme() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage( "Project endpoint \"localhost:1234/datastore/v1beta42/projects/project-id\" must" - + " include scheme."); - factory.create(new DatastoreOptions.Builder() - .projectEndpoint("localhost:1234/datastore/v1beta42/projects/project-id") - .build()); + + " include scheme."); + factory.create( + new DatastoreOptions.Builder() + .projectEndpoint("localhost:1234/datastore/v1beta42/projects/project-id") + .build()); } @Test public void initializer() throws Exception { - options.initializer(new HttpRequestInitializer() { - @Override - public void initialize(HttpRequest request) { - request.getHeaders().setCookie("magic"); - } - }); + options.initializer( + new HttpRequestInitializer() { + @Override + public void initialize(HttpRequest request) { + request.getHeaders().setCookie("magic"); + } + }); Datastore datastore = factory.create(options.build()); MockDatastoreFactory mockClient = (MockDatastoreFactory) factory; AllocateIdsRequest request = AllocateIdsRequest.newBuilder().build(); @@ -296,7 +301,8 @@ public void runQuery() throws Exception { RunQueryRequest.Builder request = RunQueryRequest.newBuilder(); request.getQueryBuilder(); RunQueryResponse.Builder response = RunQueryResponse.newBuilder(); - response.getBatchBuilder() + response + .getBatchBuilder() .setEntityResultType(EntityResult.ResultType.FULL) .setMoreResults(QueryResultBatch.MoreResultsType.NOT_FINISHED); expectRpc("runQuery", request.build(), response.build()); @@ -308,9 +314,9 @@ private void expectRpc(String methodName, Message request, Message response) thr mockClient.setNextResponse(response); @SuppressWarnings("rawtypes") - Class[] methodArgs = { request.getClass() }; + Class[] methodArgs = {request.getClass()}; Method call = Datastore.class.getMethod(methodName, methodArgs); - Object[] callArgs = { request }; + Object[] callArgs = {request}; assertEquals(response, call.invoke(datastore, callArgs)); assertEquals("/v1/projects/project-id:" + methodName, mockClient.lastPath); @@ -364,15 +370,16 @@ private void expectRpc(String methodName, Message request, Message response) thr private static class MockCredential extends Credential { MockCredential() { - super(new AccessMethod() { - @Override - public void intercept(HttpRequest request, String accessToken) throws IOException { - } - @Override - public String getAccessTokenFromRequest(HttpRequest request) { - return "MockAccessToken"; - } - }); + super( + new AccessMethod() { + @Override + public void intercept(HttpRequest request, String accessToken) throws IOException {} + + @Override + public String getAccessTokenFromRequest(HttpRequest request) { + return "MockAccessToken"; + } + }); } } @@ -411,37 +418,40 @@ void setNextException(IOException exception) { @Override public HttpRequestFactory makeClient(DatastoreOptions options) { - HttpTransport transport = new MockHttpTransport() { - @Override - public LowLevelHttpRequest buildRequest(String method, String url) { - return new MockLowLevelHttpRequest(url) { - @Override - public LowLevelHttpResponse execute() throws IOException { - lastPath = new GenericUrl(getUrl()).getRawPath(); - lastMimeType = getContentType(); - lastCookies = getHeaderValues("Cookie"); - lastApiFormatHeaderValue = - Iterables.getOnlyElement(getHeaderValues("X-Goog-Api-Format-Version")); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - getStreamingContent().writeTo(out); - lastBody = out.toByteArray(); - if (nextException != null) { - throw nextException; - } - MockLowLevelHttpResponse response = new MockLowLevelHttpResponse() - .setStatusCode(nextStatus) - .setContentType("application/x-protobuf"); - if (nextError != null) { - assertNull(nextResponse); - response.setContent(new TestableByteArrayInputStream(nextError.toByteArray())); - } else { - response.setContent(new TestableByteArrayInputStream(nextResponse.toByteArray())); + HttpTransport transport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return new MockLowLevelHttpRequest(url) { + @Override + public LowLevelHttpResponse execute() throws IOException { + lastPath = new GenericUrl(getUrl()).getRawPath(); + lastMimeType = getContentType(); + lastCookies = getHeaderValues("Cookie"); + lastApiFormatHeaderValue = + Iterables.getOnlyElement(getHeaderValues("X-Goog-Api-Format-Version")); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + getStreamingContent().writeTo(out); + lastBody = out.toByteArray(); + if (nextException != null) { + throw nextException; + } + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setStatusCode(nextStatus) + .setContentType("application/x-protobuf"); + if (nextError != null) { + assertNull(nextResponse); + response.setContent(new TestableByteArrayInputStream(nextError.toByteArray())); + } else { + response.setContent( + new TestableByteArrayInputStream(nextResponse.toByteArray())); + } + return response; } - return response; - } - }; - } - }; + }; + } + }; Credential credential = options.getCredential(); return transport.createRequestFactory(credential); } diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/EndToEndChecksumHandlerTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/EndToEndChecksumHandlerTest.java index 593ccd23d..b0a7e4e86 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/EndToEndChecksumHandlerTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/EndToEndChecksumHandlerTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2021 Google LLC. All Rights Reserved. + * Copyright 2021 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,6 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java index 377f92d8c..27f300865 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java @@ -40,9 +40,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** - * Test for {@link RemoteRpc}. - */ +/** Test for {@link RemoteRpc}. */ @RunWith(JUnit4.class) public class RemoteRpcTest { @@ -55,22 +53,35 @@ public void testException() { .setCode(Code.UNAUTHENTICATED_VALUE) .setMessage("The request does not have valid authentication credentials.") .build(); - DatastoreException exception = RemoteRpc.makeException("url", METHOD_NAME, - new ByteArrayInputStream(statusProto.toByteArray()), "application/x-protobuf", - Charsets.UTF_8, new RuntimeException(), 401); + DatastoreException exception = + RemoteRpc.makeException( + "url", + METHOD_NAME, + new ByteArrayInputStream(statusProto.toByteArray()), + "application/x-protobuf", + Charsets.UTF_8, + new RuntimeException(), + 401); assertEquals(Code.UNAUTHENTICATED, exception.getCode()); - assertEquals("The request does not have valid authentication credentials.", - exception.getMessage()); + assertEquals( + "The request does not have valid authentication credentials.", exception.getMessage()); assertEquals(METHOD_NAME, exception.getMethodName()); } @Test public void testInvalidProtoException() { - DatastoreException exception = RemoteRpc.makeException("url", METHOD_NAME, - new ByteArrayInputStream("".getBytes()), "application/x-protobuf", - Charsets.UTF_8, new RuntimeException(), 401); + DatastoreException exception = + RemoteRpc.makeException( + "url", + METHOD_NAME, + new ByteArrayInputStream("".getBytes()), + "application/x-protobuf", + Charsets.UTF_8, + new RuntimeException(), + 401); assertEquals(Code.INTERNAL, exception.getCode()); - assertEquals("Unable to parse Status protocol buffer: HTTP status code was 401.", + assertEquals( + "Unable to parse Status protocol buffer: HTTP status code was 401.", exception.getMessage()); assertEquals(METHOD_NAME, exception.getMethodName()); } @@ -113,9 +124,15 @@ public void testEmptyProtoExceptionUnauthenticated() { @Test public void testPlainTextException() { - DatastoreException exception = RemoteRpc.makeException("url", METHOD_NAME, - new ByteArrayInputStream("Text Error".getBytes()), "text/plain", Charsets.UTF_8, - new RuntimeException(), 401); + DatastoreException exception = + RemoteRpc.makeException( + "url", + METHOD_NAME, + new ByteArrayInputStream("Text Error".getBytes()), + "text/plain", + Charsets.UTF_8, + new RuntimeException(), + 401); assertEquals(Code.INTERNAL, exception.getCode()); assertEquals( "Non-protobuf error: Text Error. HTTP status code was 401.", exception.getMessage()); @@ -142,19 +159,21 @@ public void testGzip() throws IOException, DatastoreException { public void testHttpHeaders_expectE2eChecksumHeader() throws IOException { // Enable E2E-Checksum system env variable RemoteRpc.setSystemEnvE2EChecksum(true); - MessageLite request = RollbackRequest.newBuilder() - .setTransaction(ByteString.copyFromUtf8("project-id")) - .build(); - RemoteRpc rpc = newRemoteRpc(new InjectedTestValues(gzip(newBeginTransactionResponse()), - new byte[1], true)); - HttpRequest httpRequest = rpc.getClient().buildPostRequest(rpc.resolveURL("blah"), - new ProtoHttpContent(request)); + MessageLite request = + RollbackRequest.newBuilder().setTransaction(ByteString.copyFromUtf8("project-id")).build(); + RemoteRpc rpc = + newRemoteRpc( + new InjectedTestValues(gzip(newBeginTransactionResponse()), new byte[1], true)); + HttpRequest httpRequest = + rpc.getClient().buildPostRequest(rpc.resolveURL("blah"), new ProtoHttpContent(request)); rpc.setHeaders(request, httpRequest); - assertNotNull(httpRequest.getHeaders() - .getFirstHeaderStringValue(RemoteRpc.API_FORMAT_VERSION_HEADER)); + assertNotNull( + httpRequest.getHeaders().getFirstHeaderStringValue(RemoteRpc.API_FORMAT_VERSION_HEADER)); // Expect to find e2e-checksum header - String header = httpRequest.getHeaders().getFirstHeaderStringValue( - EndToEndChecksumHandler.HTTP_REQUEST_CHECKSUM_HEADER); + String header = + httpRequest + .getHeaders() + .getFirstHeaderStringValue(EndToEndChecksumHandler.HTTP_REQUEST_CHECKSUM_HEADER); assertEquals(32, header.length()); } @@ -162,19 +181,21 @@ public void testHttpHeaders_expectE2eChecksumHeader() throws IOException { public void testHttpHeaders_doNotExpectE2eChecksumHeader() throws IOException { // disable E2E-Checksum system env variable RemoteRpc.setSystemEnvE2EChecksum(false); - MessageLite request = RollbackRequest.newBuilder() - .setTransaction(ByteString.copyFromUtf8("project-id")) - .build(); - RemoteRpc rpc = newRemoteRpc(new InjectedTestValues(gzip(newBeginTransactionResponse()), - new byte[1], true)); - HttpRequest httpRequest = rpc.getClient().buildPostRequest(rpc.resolveURL("blah"), - new ProtoHttpContent(request)); + MessageLite request = + RollbackRequest.newBuilder().setTransaction(ByteString.copyFromUtf8("project-id")).build(); + RemoteRpc rpc = + newRemoteRpc( + new InjectedTestValues(gzip(newBeginTransactionResponse()), new byte[1], true)); + HttpRequest httpRequest = + rpc.getClient().buildPostRequest(rpc.resolveURL("blah"), new ProtoHttpContent(request)); rpc.setHeaders(request, httpRequest); - assertNotNull(httpRequest.getHeaders() - .getFirstHeaderStringValue(RemoteRpc.API_FORMAT_VERSION_HEADER)); + assertNotNull( + httpRequest.getHeaders().getFirstHeaderStringValue(RemoteRpc.API_FORMAT_VERSION_HEADER)); // Do not expect to find e2e-checksum header - assertNull(httpRequest.getHeaders().getFirstHeaderStringValue( - EndToEndChecksumHandler.HTTP_REQUEST_CHECKSUM_HEADER)); + assertNull( + httpRequest + .getHeaders() + .getFirstHeaderStringValue(EndToEndChecksumHandler.HTTP_REQUEST_CHECKSUM_HEADER)); } private static BeginTransactionResponse newBeginTransactionResponse() { diff --git a/pom.xml b/pom.xml index 08cac4f86..15ae9819a 100644 --- a/pom.xml +++ b/pom.xml @@ -220,6 +220,7 @@ + datastore-v1-proto-client proto-google-cloud-datastore-v1 google-cloud-datastore google-cloud-datastore-bom diff --git a/versions.txt b/versions.txt index 3e5c2218c..3b94ddf8d 100644 --- a/versions.txt +++ b/versions.txt @@ -5,4 +5,5 @@ google-cloud-datastore:1.106.4:1.106.5-SNAPSHOT google-cloud-datastore-bom:1.106.4:1.106.5-SNAPSHOT google-cloud-datastore-parent:1.106.4:1.106.5-SNAPSHOT proto-google-cloud-datastore-v1:0.89.4:0.89.5-SNAPSHOT +datastore-v1-proto-client:1.6.3:1.6.4-SNAPSHOT From 4efd9b359be85948faadbd8b8a15dfaed45c4da7 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 17 May 2021 03:46:22 +0200 Subject: [PATCH 42/54] chore(deps): update dependency com.google.cloud:libraries-bom to v20.4.0 (#424) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [com.google.cloud:libraries-bom](https://togithub.com/GoogleCloudPlatform/cloud-opensource-java) | `20.3.0` -> `20.4.0` | [![age](https://badges.renovateapi.com/packages/maven/com.google.cloud:libraries-bom/20.4.0/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/maven/com.google.cloud:libraries-bom/20.4.0/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/maven/com.google.cloud:libraries-bom/20.4.0/compatibility-slim/20.3.0)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/maven/com.google.cloud:libraries-bom/20.4.0/confidence-slim/20.3.0)](https://docs.renovatebot.com/merge-confidence/) | --- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻️ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-datastore). --- samples/snippets/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 70c6099b2..d28eda28f 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 20.3.0 + 20.4.0 pom import From f89df6f219c2bd702e18758fc22150c6c3485e0e Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Sun, 16 May 2021 19:02:06 -0700 Subject: [PATCH 43/54] chore: regenerate README (#426) This PR was generated using Autosynth. :rainbow:

    Log from Synthtool ``` 2021-05-17 01:52:22,297 synthtool [DEBUG] > Executing /root/.cache/synthtool/java-datastore/.github/readme/synth.py. On branch autosynth-readme nothing to commit, working tree clean 2021-05-17 01:52:23,611 synthtool [DEBUG] > Wrote metadata to .github/readme/synth.metadata/synth.metadata. ```
    Full log will be available here: https://source.cloud.google.com/results/invocations/2da7b935-431c-46ae-a34d-28078566718f/targets - [ ] To automatically regenerate this PR, check this box. (May take up to 24 hours.) --- .github/readme/synth.metadata/synth.metadata | 4 ++-- README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/readme/synth.metadata/synth.metadata b/.github/readme/synth.metadata/synth.metadata index 2b8c6a4f9..6cc78ae40 100644 --- a/.github/readme/synth.metadata/synth.metadata +++ b/.github/readme/synth.metadata/synth.metadata @@ -4,14 +4,14 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-datastore.git", - "sha": "a1ea05f975365a7eba6607aeae1d144b8c49453c" + "sha": "4efd9b359be85948faadbd8b8a15dfaed45c4da7" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "bd8281a06cc7f84906e04d4843c1d3d386a980cd" + "sha": "4f4b1b9b8d8b52f1e9e4a76165896debce5ab7f1" } } ] diff --git a/README.md b/README.md index bb17743d5..5d9e8efd5 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file com.google.cloud libraries-bom - 20.3.0 + 20.4.0 pom import @@ -45,7 +45,7 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies ```Groovy -implementation platform('com.google.cloud:libraries-bom:20.3.0') +implementation platform('com.google.cloud:libraries-bom:20.4.0') compile 'com.google.cloud:google-cloud-datastore' ``` From 5fd6621a2d92c9b03fb3d0a349fe778181e91061 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Mon, 17 May 2021 11:47:24 -0400 Subject: [PATCH 44/54] have dependencies use snapshot version --- datastore-v1-proto-client/pom.xml | 1 + pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index 718f03bd7..ba8a9b317 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -17,6 +17,7 @@ 4.0.0 + com.google.cloud.datastore datastore-v1-proto-client 1.6.4-SNAPSHOT diff --git a/pom.xml b/pom.xml index 15ae9819a..d6e4bdb40 100644 --- a/pom.xml +++ b/pom.xml @@ -172,7 +172,7 @@ com.google.cloud.datastore datastore-v1-proto-client - 1.6.3 + 1.6.4-SNAPSHOT com.google.api.grpc From 156e3dfbed326d8ebcf30bff0c27ff01a61ee8a8 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Mon, 17 May 2021 12:34:39 -0400 Subject: [PATCH 45/54] close input stream after each run --- .../datastore/v1/client/ChecksumEnforcingInputStreamTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java index 997205667..31981617d 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java @@ -42,6 +42,8 @@ public void test(int payloadSize) throws Exception { } } catch (IOException e) { fail("checksum verification failed!"); + } finally { + testInstance.close(); } } From 32ee2204f87f56207a30079afb93553f0ba17172 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Mon, 17 May 2021 12:37:01 -0400 Subject: [PATCH 46/54] use try with resources --- .../v1/client/ChecksumEnforcingInputStreamTest.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java index 31981617d..9807354e2 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java @@ -32,18 +32,15 @@ public class ChecksumEnforcingInputStreamTest { private final MessageDigest digest = EndToEndChecksumHandler.getMessageDigestInstance(); public void test(int payloadSize) throws Exception { - ChecksumEnforcingInputStream testInstance = setUpData(payloadSize); // read 1000 bytes at a time // Since checksum should be correct, do not expect IOException - byte[] buf = new byte[1000]; - try { + try (ChecksumEnforcingInputStream testInstance = setUpData(payloadSize)) { + byte[] buf = new byte[1000]; while (testInstance.read(buf, 0, 1000) != -1) { // do nothing with the bytes read } } catch (IOException e) { fail("checksum verification failed!"); - } finally { - testInstance.close(); } } From 756784b2033a1eff3068fbb2a85047018ff9683e Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Mon, 17 May 2021 12:41:12 -0400 Subject: [PATCH 47/54] use try with resources --- .../ChecksumEnforcingInputStreamTest.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java index 9807354e2..571210414 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java @@ -62,15 +62,13 @@ public void read_withValidChecksum_differentPayloadSizes() throws Exception { @Test public void read_withInvalidChecksum() { // build a test instance with invalidchecksum - ChecksumEnforcingInputStream instance = - new ChecksumEnforcingInputStream( - new ByteArrayInputStream("hello there".getBytes(UTF_8)), - "this checksum is invalid", - digest); // read 1000 bytes at a time // Since checksum should be correct, do not expect IOException - byte[] buf = new byte[1000]; - try { + try (ChecksumEnforcingInputStream instance = new ChecksumEnforcingInputStream( + new ByteArrayInputStream("hello there".getBytes(UTF_8)), + "this checksum is invalid", + digest)) { + byte[] buf = new byte[1000]; while (instance.read(buf, 0, 1000) != -1) { // do nothing with the bytes read } @@ -83,8 +81,9 @@ public void read_withInvalidChecksum() { @Test public void markNotSupported() throws Exception { - ChecksumEnforcingInputStream testInstance = setUpData(1); - assertFalse(testInstance.markSupported()); + try (ChecksumEnforcingInputStream testInstance = setUpData(1)) { + assertFalse(testInstance.markSupported()); + } } private ChecksumEnforcingInputStream setUpData(int payloadSize) throws Exception { @@ -93,7 +92,7 @@ private ChecksumEnforcingInputStream setUpData(int payloadSize) throws Exception String payload; if (payloadSize > str.length()) { int num = payloadSize / str.length(); - StringBuffer buf = new StringBuffer(); + StringBuilder buf = new StringBuilder(); for (int i = 0; i < num; i++) { buf.append(str); } From 320dc1239980c25af0938cf2aad319cc25b5a848 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Mon, 17 May 2021 15:11:47 -0400 Subject: [PATCH 48/54] add test config --- .kokoro/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 23b1f5d04..a16e4d777 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -47,6 +47,7 @@ set +e case ${JOB_TYPE} in test) + export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=128m" mvn test -B -Dclirr.skip=true -Denforcer.skip=true RETURN_CODE=$? ;; From a655613178d143dc94d2fe107e2321a33d983a16 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Mon, 17 May 2021 15:13:46 -0400 Subject: [PATCH 49/54] adjust xmx value --- .kokoro/build.sh | 1 - datastore-v1-proto-client/pom.xml | 26 +++++++++++++++++++ .../ChecksumEnforcingInputStreamTest.java | 3 ++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index a16e4d777..23b1f5d04 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -47,7 +47,6 @@ set +e case ${JOB_TYPE} in test) - export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=128m" mvn test -B -Dclirr.skip=true -Denforcer.skip=true RETURN_CODE=$? ;; diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index ba8a9b317..621995b87 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -97,4 +97,30 @@ test + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + + + **/*SmokeTest.java + **/IT*.java + + sponge_log + -Xmx2048m + + + + org.apache.maven.surefire + surefire-junit47 + 3.0.0-M5 + + + + + diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java index 571210414..47592f987 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java @@ -64,7 +64,8 @@ public void read_withInvalidChecksum() { // build a test instance with invalidchecksum // read 1000 bytes at a time // Since checksum should be correct, do not expect IOException - try (ChecksumEnforcingInputStream instance = new ChecksumEnforcingInputStream( + try (ChecksumEnforcingInputStream instance = + new ChecksumEnforcingInputStream( new ByteArrayInputStream("hello there".getBytes(UTF_8)), "this checksum is invalid", digest)) { From 3699b36878fb9d50c50e7638ed1e46fd44ebcb41 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 17 May 2021 22:56:41 +0200 Subject: [PATCH 50/54] chore(deps): update dependency com.google.cloud:google-cloud-datastore to v1.106.4 (#420) --- samples/install-without-bom/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 36255b925..632a7e95e 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -29,7 +29,7 @@ com.google.cloud google-cloud-datastore - 1.106.3 + 1.106.4 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 2fb3f6bd1..519b74ced 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-datastore - 1.106.3 + 1.106.4 From 778e75b27f31a0ef0058af4d3e867e05bc8993c7 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Mon, 17 May 2021 14:04:15 -0700 Subject: [PATCH 51/54] chore: regenerate README (#428) This PR was generated using Autosynth. :rainbow:
    Log from Synthtool ``` 2021-05-17 20:58:51,777 synthtool [DEBUG] > Executing /root/.cache/synthtool/java-datastore/.github/readme/synth.py. On branch autosynth-readme nothing to commit, working tree clean 2021-05-17 20:58:53,146 synthtool [DEBUG] > Wrote metadata to .github/readme/synth.metadata/synth.metadata. ```
    Full log will be available here: https://source.cloud.google.com/results/invocations/765f04fe-d4f4-4558-808a-89f065eb0b37/targets - [ ] To automatically regenerate this PR, check this box. (May take up to 24 hours.) --- .github/readme/synth.metadata/synth.metadata | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/readme/synth.metadata/synth.metadata b/.github/readme/synth.metadata/synth.metadata index 6cc78ae40..bbf2ef4ef 100644 --- a/.github/readme/synth.metadata/synth.metadata +++ b/.github/readme/synth.metadata/synth.metadata @@ -4,14 +4,14 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-datastore.git", - "sha": "4efd9b359be85948faadbd8b8a15dfaed45c4da7" + "sha": "3699b36878fb9d50c50e7638ed1e46fd44ebcb41" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "4f4b1b9b8d8b52f1e9e4a76165896debce5ab7f1" + "sha": "c86c7a60985644eab557949363a38301d40d78d2" } } ] diff --git a/README.md b/README.md index 5d9e8efd5..e74704c89 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-datastore - 1.106.3 + 1.106.4 ``` From 586709b7a1a2f9b8ca93c872b0a42c762d92894d Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Mon, 17 May 2021 16:18:16 -0700 Subject: [PATCH 52/54] chore: add changelog to cloud-rad (#429) This PR was generated using Autosynth. :rainbow: Synth log will be available here: https://source.cloud.google.com/results/invocations/81e4ad9c-db2f-4f59-8745-b52a48665ccc/targets - [ ] To automatically regenerate this PR, check this box. (May take up to 24 hours.) Source-Link: https://github.com/googleapis/synthtool/commit/c86c7a60985644eab557949363a38301d40d78d2 --- .kokoro/release/publish_javadoc11.sh | 2 ++ synth.metadata | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.kokoro/release/publish_javadoc11.sh b/.kokoro/release/publish_javadoc11.sh index 682e0bfcd..4a6619862 100755 --- a/.kokoro/release/publish_javadoc11.sh +++ b/.kokoro/release/publish_javadoc11.sh @@ -42,6 +42,8 @@ mvn clean site -B -q -P docFX # copy README to docfx-yml dir and rename index.md cp README.md target/docfx-yml/index.md +# copy CHANGELOG to docfx-yml dir and rename history.md +cp CHANGELOG.md target/docfx-yml/history.md pushd target/docfx-yml diff --git a/synth.metadata b/synth.metadata index 317c8f55d..fa8f84830 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-datastore.git", - "sha": "af7528f49250916ac4a5ce0b51b6150f780c22d5" + "sha": "778e75b27f31a0ef0058af4d3e867e05bc8993c7" } }, { @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "4f4b1b9b8d8b52f1e9e4a76165896debce5ab7f1" + "sha": "c86c7a60985644eab557949363a38301d40d78d2" } } ], From 86c4dfe1a9ae4fdcf1b788281be0f57d14c6d353 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 19 May 2021 03:36:29 +0200 Subject: [PATCH 53/54] deps: update dependency com.google.cloud:google-cloud-shared-dependencies to v1.2.0 (#430) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d6e4bdb40..8fd4ad444 100644 --- a/pom.xml +++ b/pom.xml @@ -159,7 +159,7 @@ com.google.cloud google-cloud-shared-dependencies - 1.1.0 + 1.2.0 pom import From ed336d67c45376726822f792dc992ae85b23a93c Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 19 May 2021 01:44:08 +0000 Subject: [PATCH 54/54] chore: release 1.106.5 (#431) :robot: I have created a release \*beep\* \*boop\* --- ### [1.106.5](https://www.github.com/googleapis/java-datastore/compare/v1.106.4...v1.106.5) (2021-05-19) ### Dependencies * update dependency com.google.cloud:google-cloud-shared-dependencies to v1.2.0 ([#430](https://www.github.com/googleapis/java-datastore/issues/430)) ([86c4dfe](https://www.github.com/googleapis/java-datastore/commit/86c4dfe1a9ae4fdcf1b788281be0f57d14c6d353)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 7 +++++++ datastore-v1-proto-client/pom.xml | 4 ++-- google-cloud-datastore-bom/pom.xml | 6 +++--- google-cloud-datastore/pom.xml | 4 ++-- pom.xml | 6 +++--- proto-google-cloud-datastore-v1/pom.xml | 4 ++-- versions.txt | 10 +++++----- 7 files changed, 24 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06a5b92e5..11b4807a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +### [1.106.5](https://www.github.com/googleapis/java-datastore/compare/v1.106.4...v1.106.5) (2021-05-19) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v1.2.0 ([#430](https://www.github.com/googleapis/java-datastore/issues/430)) ([86c4dfe](https://www.github.com/googleapis/java-datastore/commit/86c4dfe1a9ae4fdcf1b788281be0f57d14c6d353)) + ### [1.106.4](https://www.github.com/googleapis/java-datastore/compare/v1.106.3...v1.106.4) (2021-05-11) diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index 621995b87..05c191ed0 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -19,12 +19,12 @@ 4.0.0 com.google.cloud.datastore datastore-v1-proto-client - 1.6.4-SNAPSHOT + 1.6.4 com.google.cloud google-cloud-datastore-parent - 1.106.5-SNAPSHOT + 1.106.5 jar diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index d39cb3790..feddbb6ec 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-datastore-bom - 1.106.5-SNAPSHOT + 1.106.5 pom com.google.cloud @@ -63,12 +63,12 @@ com.google.api.grpc proto-google-cloud-datastore-v1 - 0.89.5-SNAPSHOT + 0.89.5 com.google.cloud google-cloud-datastore - 1.106.5-SNAPSHOT + 1.106.5 diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index 2c4d600d5..1c773cd5e 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-datastore - 1.106.5-SNAPSHOT + 1.106.5 jar Google Cloud Datastore https://github.com/googleapis/java-datastore @@ -12,7 +12,7 @@ com.google.cloud google-cloud-datastore-parent - 1.106.5-SNAPSHOT + 1.106.5 google-cloud-datastore diff --git a/pom.xml b/pom.xml index 8fd4ad444..4309b2940 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-datastore-parent pom - 1.106.5-SNAPSHOT + 1.106.5 Google Cloud Datastore Parent https://github.com/googleapis/java-datastore @@ -167,12 +167,12 @@ com.google.api.grpc proto-google-cloud-datastore-v1 - 0.89.5-SNAPSHOT + 0.89.5 com.google.cloud.datastore datastore-v1-proto-client - 1.6.4-SNAPSHOT + 1.6.4 com.google.api.grpc diff --git a/proto-google-cloud-datastore-v1/pom.xml b/proto-google-cloud-datastore-v1/pom.xml index ef1b94849..2cc19db6a 100644 --- a/proto-google-cloud-datastore-v1/pom.xml +++ b/proto-google-cloud-datastore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.89.5-SNAPSHOT + 0.89.5 proto-google-cloud-datastore-v1 PROTO library for proto-google-cloud-datastore-v1 com.google.cloud google-cloud-datastore-parent - 1.106.5-SNAPSHOT + 1.106.5 diff --git a/versions.txt b/versions.txt index 3b94ddf8d..44e843cc3 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-datastore:1.106.4:1.106.5-SNAPSHOT -google-cloud-datastore-bom:1.106.4:1.106.5-SNAPSHOT -google-cloud-datastore-parent:1.106.4:1.106.5-SNAPSHOT -proto-google-cloud-datastore-v1:0.89.4:0.89.5-SNAPSHOT -datastore-v1-proto-client:1.6.3:1.6.4-SNAPSHOT +google-cloud-datastore:1.106.5:1.106.5 +google-cloud-datastore-bom:1.106.5:1.106.5 +google-cloud-datastore-parent:1.106.5:1.106.5 +proto-google-cloud-datastore-v1:0.89.5:0.89.5 +datastore-v1-proto-client:1.6.4:1.6.4