diff --git a/CHANGELOG.md b/CHANGELOG.md index ef05ced8a7..64aa224d37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [2.28.2](https://github.com/googleapis/java-bigquerystorage/compare/v2.28.1...v2.28.2) (2023-01-18) + + +### Bug Fixes + +* Add a timeout on retry for retryable errors ([#1930](https://github.com/googleapis/java-bigquerystorage/issues/1930)) ([2d648cf](https://github.com/googleapis/java-bigquerystorage/commit/2d648cf9706a6e7bc155e8769ba7dda2a6bc3061)) +* Add precision overwritten to 9 digit if the passed in JSON type is FLOAT or DOUBLE ([#1932](https://github.com/googleapis/java-bigquerystorage/issues/1932)) ([417bc6c](https://github.com/googleapis/java-bigquerystorage/commit/417bc6c76f7b9fa602721c3c183c487c5aab2e09)) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-bigquery to v2.20.2 ([#1929](https://github.com/googleapis/java-bigquerystorage/issues/1929)) ([a95ae9d](https://github.com/googleapis/java-bigquerystorage/commit/a95ae9d708bd3e2d8f55297a6004a0d937c8d83f)) + ## [2.28.1](https://github.com/googleapis/java-bigquerystorage/compare/v2.28.0...v2.28.1) (2023-01-12) diff --git a/README.md b/README.md index 364cf27829..fa5a31e151 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: com.google.cloud libraries-bom - 26.3.0 + 26.4.0 pom import @@ -41,7 +41,7 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-bigquerystorage - 2.26.0 + 2.28.1 ``` @@ -49,20 +49,20 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.3.0') +implementation platform('com.google.cloud:libraries-bom:26.4.0') implementation 'com.google.cloud:google-cloud-bigquerystorage' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigquerystorage:2.28.0' +implementation 'com.google.cloud:google-cloud-bigquerystorage:2.28.1' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquerystorage" % "2.28.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquerystorage" % "2.28.1" ``` ## Authentication diff --git a/google-cloud-bigquerystorage-bom/pom.xml b/google-cloud-bigquerystorage-bom/pom.xml index 83949a45e1..b11807833e 100644 --- a/google-cloud-bigquerystorage-bom/pom.xml +++ b/google-cloud-bigquerystorage-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquerystorage-bom - 2.28.1 + 2.28.2 pom com.google.cloud @@ -52,37 +52,37 @@ com.google.cloud google-cloud-bigquerystorage - 2.28.1 + 2.28.2 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta1 - 0.152.1 + 0.152.2 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta2 - 0.152.1 + 0.152.2 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1 - 2.28.1 + 2.28.2 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta1 - 0.152.1 + 0.152.2 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta2 - 0.152.1 + 0.152.2 com.google.api.grpc proto-google-cloud-bigquerystorage-v1 - 2.28.1 + 2.28.2 diff --git a/google-cloud-bigquerystorage/clirr-ignored-differences.xml b/google-cloud-bigquerystorage/clirr-ignored-differences.xml index 080a8c33f3..b33dced533 100644 --- a/google-cloud-bigquerystorage/clirr-ignored-differences.xml +++ b/google-cloud-bigquerystorage/clirr-ignored-differences.xml @@ -76,4 +76,14 @@ com/google/cloud/bigquery/storage/v1/ConnectionWorker com.google.cloud.bigquery.storage.v1.TableSchema getUpdatedSchema() + + 7004 + com/google/cloud/bigquery/storage/v1/ConnectionWorker + ConnectionWorker(java.lang.String, com.google.cloud.bigquery.storage.v1.ProtoSchema, long, long, com.google.api.gax.batching.FlowController$LimitExceededBehavior, java.lang.String, com.google.cloud.bigquery.storage.v1.BigQueryWriteClient, boolean) + + + 7004 + com/google/cloud/bigquery/storage/v1/ConnectionWorkerPool + ConnectionWorkerPool(long, long, com.google.api.gax.batching.FlowController$LimitExceededBehavior, java.lang.String, com.google.cloud.bigquery.storage.v1.BigQueryWriteClient, boolean) + diff --git a/google-cloud-bigquerystorage/pom.xml b/google-cloud-bigquerystorage/pom.xml index f8bfffa18a..342384fd8b 100644 --- a/google-cloud-bigquerystorage/pom.xml +++ b/google-cloud-bigquerystorage/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquerystorage - 2.28.1 + 2.28.2 jar BigQuery Storage https://github.com/googleapis/java-bigquerystorage @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquerystorage-parent - 2.28.1 + 2.28.2 google-cloud-bigquerystorage diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java index 3520ad0a98..50086e95e2 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java @@ -31,6 +31,7 @@ import io.grpc.Status.Code; import io.grpc.StatusRuntimeException; import java.io.IOException; +import java.time.Duration; import java.util.Comparator; import java.util.Deque; import java.util.HashMap; @@ -61,6 +62,7 @@ public class ConnectionWorker implements AutoCloseable { private Lock lock; private Condition hasMessageInWaitingQueue; private Condition inflightReduced; + private static Duration maxRetryDuration = Duration.ofMinutes(5); /* * The identifier of the current stream to write to. This stream name can change during @@ -114,6 +116,9 @@ public class ConnectionWorker implements AutoCloseable { @GuardedBy("lock") private long conectionRetryCountWithoutCallback = 0; + @GuardedBy("lock") + private long connectionRetryStartTime = 0; + /* * If false, streamConnection needs to be reset. */ @@ -201,6 +206,7 @@ public ConnectionWorker( ProtoSchema writerSchema, long maxInflightRequests, long maxInflightBytes, + Duration maxRetryDuration, FlowController.LimitExceededBehavior limitExceededBehavior, String traceId, BigQueryWriteClient client, @@ -210,6 +216,7 @@ public ConnectionWorker( this.hasMessageInWaitingQueue = lock.newCondition(); this.inflightReduced = lock.newCondition(); this.streamName = streamName; + this.maxRetryDuration = maxRetryDuration; if (writerSchema == null) { throw new StatusRuntimeException( Status.fromCode(Code.INVALID_ARGUMENT) @@ -237,6 +244,7 @@ public void run() { } private void resetConnection() { + log.info("Reconnecting for stream:" + streamName); this.streamConnection = new StreamConnection( this.client, @@ -618,6 +626,9 @@ private void requestCallback(AppendRowsResponse response) { if (conectionRetryCountWithoutCallback != 0) { conectionRetryCountWithoutCallback = 0; } + if (connectionRetryStartTime != 0) { + connectionRetryStartTime = 0; + } if (!this.inflightRequestQueue.isEmpty()) { requestWrapper = pollInflightRequestQueue(); } else if (inflightCleanuped) { @@ -686,15 +697,25 @@ private void doneCallback(Throwable finalStatus) { try { this.streamConnectionIsConnected = false; if (connectionFinalStatus == null) { + if (connectionRetryStartTime == 0) { + connectionRetryStartTime = System.currentTimeMillis(); + } // If the error can be retried, don't set it here, let it try to retry later on. - if (isRetriableError(finalStatus) && !userClosed) { + if (isRetriableError(finalStatus) + && !userClosed + && (maxRetryDuration.toMillis() == 0f + || System.currentTimeMillis() - connectionRetryStartTime + <= maxRetryDuration.toMillis())) { this.conectionRetryCountWithoutCallback++; log.info( "Retriable error " + finalStatus.toString() + " received, retry count " + conectionRetryCountWithoutCallback - + " for stream " + + ", millis left to retry " + + (maxRetryDuration.toMillis() + - (System.currentTimeMillis() - connectionRetryStartTime)) + + ", for stream " + streamName); } else { Exceptions.StorageException storageException = Exceptions.toStorageException(finalStatus); diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPool.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPool.java index e119f4c560..121b1d0e28 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPool.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPool.java @@ -57,6 +57,11 @@ public class ConnectionWorkerPool { */ private final long maxInflightBytes; + /* + * Max retry duration for retryable errors. + */ + private final java.time.Duration maxRetryDuration; + /* * Behavior when inflight queue is exceeded. Only supports Block or Throw, default is Block. */ @@ -196,12 +201,14 @@ public abstract static class Builder { public ConnectionWorkerPool( long maxInflightRequests, long maxInflightBytes, + java.time.Duration maxRetryDuration, FlowController.LimitExceededBehavior limitExceededBehavior, String traceId, BigQueryWriteClient client, boolean ownsBigQueryWriteClient) { this.maxInflightRequests = maxInflightRequests; this.maxInflightBytes = maxInflightBytes; + this.maxRetryDuration = maxRetryDuration; this.limitExceededBehavior = limitExceededBehavior; this.traceId = traceId; this.client = client; @@ -356,6 +363,7 @@ private ConnectionWorker createConnectionWorker(String streamName, ProtoSchema w writeSchema, maxInflightRequests, maxInflightBytes, + maxRetryDuration, limitExceededBehavior, traceId, client, diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java index eebe7538aa..e2cc1cc6b0 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java @@ -28,6 +28,7 @@ import com.google.protobuf.Message; import com.google.protobuf.UninitializedMessageException; import java.math.BigDecimal; +import java.math.RoundingMode; import java.time.LocalDate; import java.util.List; import java.util.logging.Logger; @@ -49,6 +50,7 @@ */ public class JsonToProtoMessage { private static final Logger LOG = Logger.getLogger(JsonToProtoMessage.class.getName()); + private static int NUMERIC_SCALE = 9; private static ImmutableMap FieldTypeToDebugMessage = new ImmutableMap.Builder() .put(FieldDescriptor.Type.BOOL, "boolean") @@ -315,10 +317,15 @@ private static void fillField( new BigDecimal(((Number) val).longValue()))); return; } else if (val instanceof Float || val instanceof Double) { + // In JSON, the precision passed in is machine dependent. We should round the number + // before passing to backend. + BigDecimal bigDecimal = new BigDecimal(String.valueOf(val)); + if (bigDecimal.scale() > 9) { + bigDecimal = bigDecimal.setScale(NUMERIC_SCALE, RoundingMode.HALF_UP); + } protoMsg.setField( fieldDescriptor, - BigDecimalByteStringEncoder.encodeToNumericByteString( - new BigDecimal(String.valueOf(val)))); + BigDecimalByteStringEncoder.encodeToNumericByteString(bigDecimal)); return; } else if (val instanceof BigDecimal) { protoMsg.setField( @@ -559,10 +566,13 @@ private static void fillRepeatedField( new BigDecimal(((Number) val).longValue()))); added = true; } else if (val instanceof Float || val instanceof Double) { + BigDecimal bigDecimal = new BigDecimal(String.valueOf(val)); + if (bigDecimal.scale() > 9) { + bigDecimal = bigDecimal.setScale(NUMERIC_SCALE, RoundingMode.HALF_UP); + } protoMsg.addRepeatedField( fieldDescriptor, - BigDecimalByteStringEncoder.encodeToNumericByteString( - new BigDecimal(String.valueOf(val)))); + BigDecimalByteStringEncoder.encodeToNumericByteString(bigDecimal)); added = true; } else if (val instanceof BigDecimal) { protoMsg.addRepeatedField( diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java index 4d07dfdd91..ff7dad474d 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java @@ -29,6 +29,7 @@ import io.grpc.Status.Code; import io.grpc.StatusRuntimeException; import java.io.IOException; +import java.time.Duration; import java.util.Map; import java.util.Objects; import java.util.UUID; @@ -193,6 +194,7 @@ private StreamWriter(Builder builder) throws IOException { builder.writerSchema, builder.maxInflightRequest, builder.maxInflightBytes, + builder.maxRetryDuration, builder.limitExceededBehavior, builder.traceId, getBigQueryWriteClient(builder), @@ -251,6 +253,7 @@ private StreamWriter(Builder builder) throws IOException { return new ConnectionWorkerPool( builder.maxInflightRequest, builder.maxInflightBytes, + builder.maxRetryDuration, builder.limitExceededBehavior, builder.traceId, client, @@ -494,6 +497,8 @@ public static final class Builder { private boolean enableConnectionPool = false; + private java.time.Duration maxRetryDuration = Duration.ofMinutes(5); + private Builder(String streamName) { this.streamName = Preconditions.checkNotNull(streamName); this.client = null; @@ -602,6 +607,15 @@ public Builder setLimitExceededBehavior( return this; } + /* + * Max duration to retry on retryable errors. Default is 5 minutes. You can allow unlimited + * retry by setting the value to be 0. + */ + public Builder setMaxRetryDuration(java.time.Duration maxRetryDuration) { + this.maxRetryDuration = maxRetryDuration; + return this; + } + /** Builds the {@code StreamWriterV2}. */ public StreamWriter build() throws IOException { return new StreamWriter(this); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPoolTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPoolTest.java index 08543f539d..961ad3fdc1 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPoolTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPoolTest.java @@ -153,7 +153,8 @@ private void testSendRequestsToMultiTable( .setMaxConnectionsPerRegion(maxConnections) .build()); ConnectionWorkerPool connectionWorkerPool = - createConnectionWorkerPool(maxRequests, /*maxBytes=*/ 100000); + createConnectionWorkerPool( + maxRequests, /*maxBytes=*/ 100000, java.time.Duration.ofSeconds(5)); // Sets the sleep time to simulate requests stuck in connection. testBigQueryWrite.setResponseSleep(Duration.ofMillis(50L)); @@ -206,7 +207,8 @@ public void testMultiStreamClosed_multiplexingEnabled() throws Exception { ConnectionWorkerPool.setOptions( Settings.builder().setMaxConnectionsPerRegion(10).setMinConnectionsPerRegion(5).build()); ConnectionWorkerPool connectionWorkerPool = - createConnectionWorkerPool(/*maxRequests=*/ 3, /*maxBytes=*/ 1000); + createConnectionWorkerPool( + /*maxRequests=*/ 3, /*maxBytes=*/ 1000, java.time.Duration.ofSeconds(5)); // Sets the sleep time to simulate requests stuck in connection. testBigQueryWrite.setResponseSleep(Duration.ofMillis(50L)); @@ -255,7 +257,8 @@ public void testMultiStreamAppend_appendWhileClosing() throws Exception { ConnectionWorkerPool.setOptions( Settings.builder().setMaxConnectionsPerRegion(10).setMinConnectionsPerRegion(5).build()); ConnectionWorkerPool connectionWorkerPool = - createConnectionWorkerPool(/*maxRequests=*/ 3, /*maxBytes=*/ 100000); + createConnectionWorkerPool( + /*maxRequests=*/ 3, /*maxBytes=*/ 100000, java.time.Duration.ofSeconds(5)); // Sets the sleep time to simulate requests stuck in connection. testBigQueryWrite.setResponseSleep(Duration.ofMillis(50L)); @@ -368,11 +371,13 @@ private ProtoRows createProtoRows(String[] messages) { return rowsBuilder.build(); } - ConnectionWorkerPool createConnectionWorkerPool(long maxRequests, long maxBytes) { + ConnectionWorkerPool createConnectionWorkerPool( + long maxRequests, long maxBytes, java.time.Duration maxRetryDuration) { ConnectionWorkerPool.enableTestingLogic(); return new ConnectionWorkerPool( maxRequests, maxBytes, + maxRetryDuration, FlowController.LimitExceededBehavior.Block, TEST_TRACE_ID, client, diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerTest.java index a2258ad430..8db4b072b1 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerTest.java @@ -290,16 +290,23 @@ private AppendRowsResponse createAppendResponse(long offset) { private ConnectionWorker createConnectionWorker() throws IOException { // By default use only the first table as table reference. - return createConnectionWorker(TEST_STREAM_1, TEST_TRACE_ID, 100, 1000); + return createConnectionWorker( + TEST_STREAM_1, TEST_TRACE_ID, 100, 1000, java.time.Duration.ofSeconds(5)); } private ConnectionWorker createConnectionWorker( - String streamName, String traceId, long maxRequests, long maxBytes) throws IOException { + String streamName, + String traceId, + long maxRequests, + long maxBytes, + java.time.Duration maxRetryDuration) + throws IOException { return new ConnectionWorker( streamName, createProtoSchema("foo"), maxRequests, maxBytes, + maxRetryDuration, FlowController.LimitExceededBehavior.Block, TEST_TRACE_ID, client, diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/FakeBigQueryWrite.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/FakeBigQueryWrite.java index 5ba2f2aa1e..d707bbf976 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/FakeBigQueryWrite.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/FakeBigQueryWrite.java @@ -91,6 +91,10 @@ public void setTimesToClose(long numberTimesToClose) { serviceImpl.setTimesToClose(numberTimesToClose); } + public void setCloseForeverAfter(long closeForeverAfter) { + serviceImpl.setCloseForeverAfter(closeForeverAfter); + } + public long getConnectionCount() { return serviceImpl.getConnectionCount(); } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/FakeBigQueryWriteImpl.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/FakeBigQueryWriteImpl.java index 02223ace82..db900100ad 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/FakeBigQueryWriteImpl.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/FakeBigQueryWriteImpl.java @@ -57,6 +57,7 @@ class FakeBigQueryWriteImpl extends BigQueryWriteGrpc.BigQueryWriteImplBase { private long closeAfter = 0; private long recordCount = 0; private long connectionCount = 0; + private long closeForeverAfter = 0; // Record whether the first record has been seen on a connection. private final Map, Boolean> connectionToFirstRequest = @@ -177,6 +178,9 @@ public void onNext(AppendRowsRequest value) { && (numberTimesToClose == 0 || connectionCount <= numberTimesToClose)) { LOG.info("Shutting down connection from test..."); responseObserver.onError(Status.ABORTED.asException()); + } else if (closeForeverAfter > 0 && recordCount > closeForeverAfter) { + LOG.info("Shutting down connection from test..."); + responseObserver.onError(Status.ABORTED.asException()); } else { final Response response = responses.get(offset); sendResponse(response, responseObserver); @@ -279,4 +283,10 @@ public void setCloseEveryNAppends(long closeAfter) { public void setTimesToClose(long numberTimesToClose) { this.numberTimesToClose = numberTimesToClose; } + + /* The connection will forever return failure after numberTimesToClose. This option shouldn't + * be used together with setCloseEveryNAppends and setTimesToClose*/ + public void setCloseForeverAfter(long closeForeverAfter) { + this.closeForeverAfter = closeForeverAfter; + } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java index 463208302d..62daf66950 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java @@ -20,6 +20,7 @@ import com.google.cloud.bigquery.storage.test.JsonTest.*; import com.google.cloud.bigquery.storage.test.SchemaTest.*; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.protobuf.ByteString; import com.google.protobuf.Descriptors.Descriptor; @@ -685,6 +686,60 @@ public void testDouble() throws Exception { assertEquals(expectedProto, protoMsg); } + @Test + public void testDoubleHighPrecision() throws Exception { + TableSchema tableSchema = + TableSchema.newBuilder() + .addFields( + TableFieldSchema.newBuilder() + .setName("numeric") + .setType(TableFieldSchema.Type.NUMERIC) + .build()) + .build(); + TestNumeric expectedProto = + TestNumeric.newBuilder() + .setNumeric( + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal("3.400500513"))) + .build(); + JSONObject json = new JSONObject(); + json.put("numeric", 3.400500512978076); + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage( + TestNumeric.getDescriptor(), tableSchema, json); + assertEquals(expectedProto, protoMsg); + } + + @Test + public void testDoubleHighPrecision_RepeatedField() throws Exception { + TableSchema tableSchema = + TableSchema.newBuilder() + .addFields( + 0, + TableFieldSchema.newBuilder() + .setName("bignumeric") + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.REPEATED) + .build()) + .build(); + TestBignumeric expectedProto = + TestBignumeric.newBuilder() + .addBignumeric( + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal("3.400500513"))) + .addBignumeric( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0.1"))) + .addBignumeric( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0.12"))) + .build(); + JSONObject json = new JSONObject(); + json.put("bignumeric", ImmutableList.of(3.400500512978076, 0.10000000000055, 0.12)); + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage( + TestBignumeric.getDescriptor(), tableSchema, json); + assertEquals(expectedProto, protoMsg); + } + @Test public void testTimestamp() throws Exception { TableSchema tableSchema = diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/StreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/StreamWriterTest.java index 50e43fe45d..f8822e231f 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/StreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/StreamWriterTest.java @@ -25,6 +25,7 @@ import com.google.api.gax.core.NoCredentialsProvider; import com.google.api.gax.grpc.testing.MockGrpcService; import com.google.api.gax.grpc.testing.MockServiceHelper; +import com.google.api.gax.rpc.AbortedException; import com.google.api.gax.rpc.ApiException; import com.google.api.gax.rpc.StatusCode.Code; import com.google.api.gax.rpc.UnknownException; @@ -129,6 +130,7 @@ private StreamWriter getMultiplexingTestStreamWriter() throws IOException { .setTraceId(TEST_TRACE_ID) .setLocation("US") .setEnableConnectionPool(true) + .setMaxRetryDuration(java.time.Duration.ofSeconds(5)) .build(); } @@ -136,6 +138,7 @@ private StreamWriter getTestStreamWriter() throws IOException { return StreamWriter.newBuilder(TEST_STREAM_1, client) .setWriterSchema(createProtoSchema()) .setTraceId(TEST_TRACE_ID) + .setMaxRetryDuration(java.time.Duration.ofSeconds(5)) .build(); } @@ -884,6 +887,48 @@ public void testAppendWithResetSuccess() throws Exception { } } + @Test + public void testAppendWithResetNeverSuccess() throws Exception { + try (StreamWriter writer = getTestStreamWriter()) { + testBigQueryWrite.setCloseForeverAfter(1); + long appendCount = 100; + for (long i = 0; i < appendCount; i++) { + testBigQueryWrite.addResponse(createAppendResponse(i)); + } + List> futures = new ArrayList<>(); + for (long i = 0; i < appendCount; i++) { + futures.add(sendTestMessage(writer, new String[] {String.valueOf(i)}, i)); + } + // first request succeeded. + assertEquals(futures.get(0).get().getAppendResult().getOffset().getValue(), 0); + // after 5 seconds, the requests will bail out. + for (int i = 1; i < appendCount; i++) { + assertFutureException(AbortedException.class, futures.get(i)); + } + } + } + + @Test + public void testAppendWithResetNeverSuccessWithMultiplexing() throws Exception { + try (StreamWriter writer = getMultiplexingTestStreamWriter()) { + testBigQueryWrite.setCloseForeverAfter(1); + long appendCount = 100; + for (long i = 0; i < appendCount; i++) { + testBigQueryWrite.addResponse(createAppendResponse(i)); + } + List> futures = new ArrayList<>(); + for (long i = 0; i < appendCount; i++) { + futures.add(sendTestMessage(writer, new String[] {String.valueOf(i)}, i)); + } + // first request succeeded. + assertEquals(futures.get(0).get().getAppendResult().getOffset().getValue(), 0); + // after 5 seconds, the requests will bail out. + for (int i = 1; i < appendCount; i++) { + assertFutureException(AbortedException.class, futures.get(i)); + } + } + } + // This test is setup for the server to force a retry after all records are sent. Ensure the // records are resent, even if no new records are appeneded. @Test diff --git a/grpc-google-cloud-bigquerystorage-v1/pom.xml b/grpc-google-cloud-bigquerystorage-v1/pom.xml index 889a0c7268..ebea7dd0b9 100644 --- a/grpc-google-cloud-bigquerystorage-v1/pom.xml +++ b/grpc-google-cloud-bigquerystorage-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1 - 2.28.1 + 2.28.2 grpc-google-cloud-bigquerystorage-v1 GRPC library for grpc-google-cloud-bigquerystorage-v1 com.google.cloud google-cloud-bigquerystorage-parent - 2.28.1 + 2.28.2 diff --git a/grpc-google-cloud-bigquerystorage-v1beta1/pom.xml b/grpc-google-cloud-bigquerystorage-v1beta1/pom.xml index 4115e45e6f..aa14da081b 100644 --- a/grpc-google-cloud-bigquerystorage-v1beta1/pom.xml +++ b/grpc-google-cloud-bigquerystorage-v1beta1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta1 - 0.152.1 + 0.152.2 grpc-google-cloud-bigquerystorage-v1beta1 GRPC library for grpc-google-cloud-bigquerystorage-v1beta1 com.google.cloud google-cloud-bigquerystorage-parent - 2.28.1 + 2.28.2 diff --git a/grpc-google-cloud-bigquerystorage-v1beta2/pom.xml b/grpc-google-cloud-bigquerystorage-v1beta2/pom.xml index 09c096e42a..3b84d2d6a4 100644 --- a/grpc-google-cloud-bigquerystorage-v1beta2/pom.xml +++ b/grpc-google-cloud-bigquerystorage-v1beta2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta2 - 0.152.1 + 0.152.2 grpc-google-cloud-bigquerystorage-v1beta2 GRPC library for grpc-google-cloud-bigquerystorage-v1beta2 com.google.cloud google-cloud-bigquerystorage-parent - 2.28.1 + 2.28.2 diff --git a/pom.xml b/pom.xml index 402b737d0d..8d08251cf3 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquerystorage-parent pom - 2.28.1 + 2.28.2 BigQuery Storage Parent https://github.com/googleapis/java-bigquerystorage @@ -83,37 +83,37 @@ com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta1 - 0.152.1 + 0.152.2 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta2 - 0.152.1 + 0.152.2 com.google.api.grpc proto-google-cloud-bigquerystorage-v1 - 2.28.1 + 2.28.2 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta1 - 0.152.1 + 0.152.2 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta2 - 0.152.1 + 0.152.2 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1 - 2.28.1 + 2.28.2 com.google.cloud google-cloud-bigquerystorage - 2.28.1 + 2.28.2 org.json @@ -132,7 +132,7 @@ com.google.cloud google-cloud-bigquery - 2.20.1 + 2.20.2 test diff --git a/proto-google-cloud-bigquerystorage-v1/pom.xml b/proto-google-cloud-bigquerystorage-v1/pom.xml index 65c82b233f..9c06521ac4 100644 --- a/proto-google-cloud-bigquerystorage-v1/pom.xml +++ b/proto-google-cloud-bigquerystorage-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1 - 2.28.1 + 2.28.2 proto-google-cloud-bigquerystorage-v1 PROTO library for proto-google-cloud-bigquerystorage-v1 com.google.cloud google-cloud-bigquerystorage-parent - 2.28.1 + 2.28.2 diff --git a/proto-google-cloud-bigquerystorage-v1beta1/pom.xml b/proto-google-cloud-bigquerystorage-v1beta1/pom.xml index f967404534..0ae0a22aa8 100644 --- a/proto-google-cloud-bigquerystorage-v1beta1/pom.xml +++ b/proto-google-cloud-bigquerystorage-v1beta1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta1 - 0.152.1 + 0.152.2 proto-google-cloud-bigquerystorage-v1beta1 PROTO library for proto-google-cloud-bigquerystorage-v1beta1 com.google.cloud google-cloud-bigquerystorage-parent - 2.28.1 + 2.28.2 diff --git a/proto-google-cloud-bigquerystorage-v1beta2/pom.xml b/proto-google-cloud-bigquerystorage-v1beta2/pom.xml index 07823be670..347f3c7f09 100644 --- a/proto-google-cloud-bigquerystorage-v1beta2/pom.xml +++ b/proto-google-cloud-bigquerystorage-v1beta2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta2 - 0.152.1 + 0.152.2 proto-google-cloud-bigquerystorage-v1beta2 PROTO library for proto-google-cloud-bigquerystorage-v1beta2 com.google.cloud google-cloud-bigquerystorage-parent - 2.28.1 + 2.28.2 diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index fc1cc15469..97bba2a7e9 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -30,14 +30,14 @@ com.google.cloud google-cloud-bigquerystorage - 2.26.0 + 2.28.1 com.google.cloud google-cloud-bigquery - 2.20.1 + 2.20.2 org.apache.avro diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 9245d86623..94ecf42095 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -29,14 +29,14 @@ com.google.cloud google-cloud-bigquerystorage - 2.28.1 + 2.28.2 com.google.cloud google-cloud-bigquery - 2.20.1 + 2.20.2 org.apache.avro diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 73e5356aa9..2e77fdb494 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -31,7 +31,7 @@ com.google.cloud libraries-bom - 26.3.0 + 26.4.0 pom import @@ -48,7 +48,7 @@ com.google.cloud google-cloud-bigquery - 2.20.1 + 2.20.2 org.apache.avro diff --git a/tutorials/JsonWriterDefaultStream/pom.xml b/tutorials/JsonWriterDefaultStream/pom.xml index 8a7beea401..cce456b673 100644 --- a/tutorials/JsonWriterDefaultStream/pom.xml +++ b/tutorials/JsonWriterDefaultStream/pom.xml @@ -19,12 +19,12 @@ com.google.cloud google-cloud-bigquerystorage - 2.26.0 + 2.28.1 com.google.cloud google-cloud-bigquery - 2.20.1 + 2.20.2 org.apache.avro diff --git a/versions.txt b/versions.txt index 0b00a0c634..dbd515dc0c 100644 --- a/versions.txt +++ b/versions.txt @@ -1,10 +1,10 @@ # Format: # module:released-version:current-version -google-cloud-bigquerystorage:2.28.1:2.28.1 -grpc-google-cloud-bigquerystorage-v1beta1:0.152.1:0.152.1 -grpc-google-cloud-bigquerystorage-v1beta2:0.152.1:0.152.1 -grpc-google-cloud-bigquerystorage-v1:2.28.1:2.28.1 -proto-google-cloud-bigquerystorage-v1beta1:0.152.1:0.152.1 -proto-google-cloud-bigquerystorage-v1beta2:0.152.1:0.152.1 -proto-google-cloud-bigquerystorage-v1:2.28.1:2.28.1 +google-cloud-bigquerystorage:2.28.2:2.28.2 +grpc-google-cloud-bigquerystorage-v1beta1:0.152.2:0.152.2 +grpc-google-cloud-bigquerystorage-v1beta2:0.152.2:0.152.2 +grpc-google-cloud-bigquerystorage-v1:2.28.2:2.28.2 +proto-google-cloud-bigquerystorage-v1beta1:0.152.2:0.152.2 +proto-google-cloud-bigquerystorage-v1beta2:0.152.2:0.152.2 +proto-google-cloud-bigquerystorage-v1:2.28.2:2.28.2