diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..2f51be7bc3 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,76 @@ +on: + push: + branches: + - master + pull_request: +name: ci +jobs: + units: + runs-on: ubuntu-latest + strategy: + matrix: + java: [7, 8, 11] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: test + - name: coverage + uses: codecov/codecov-action@v1 + with: + name: actions ${{ matrix.java }} + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/build.bat + env: + JOB_TYPE: test + dependencies: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/dependencies.sh + linkage-monitor: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/linkage-monitor.sh + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: lint + clirr: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: clirr diff --git a/.kokoro/build.bat b/.kokoro/build.bat index ed794c3d8d..05826ad93f 100644 --- a/.kokoro/build.bat +++ b/.kokoro/build.bat @@ -1,3 +1,3 @@ :: See documentation in type-shell-output.bat -"C:\Program Files\Git\bin\bash.exe" github/java-bigquerystorage/.kokoro/build.sh +"C:\Program Files\Git\bin\bash.exe" %~dp0build.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 4052d6abe0..16443c41ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +### [0.131.1](https://www.github.com/googleapis/java-bigquerystorage/compare/v0.131.0...v0.131.1) (2020-05-04) + + +### Bug Fixes + +* flaky writeapi manual client tests ([#238](https://www.github.com/googleapis/java-bigquerystorage/issues/238)) ([89c8623](https://www.github.com/googleapis/java-bigquerystorage/commit/89c8623e082cacdc8e0843bffb67da4dc8b79df3)) +* more writeapi manual client test issues ([#241](https://www.github.com/googleapis/java-bigquerystorage/issues/241)) ([65c5ec9](https://www.github.com/googleapis/java-bigquerystorage/commit/65c5ec9c27e901b1633402d5fbbbbb83f956ed97)) + ## [0.131.0](https://www.github.com/googleapis/java-bigquerystorage/compare/v0.130.0...v0.131.0) (2020-04-28) diff --git a/README.md b/README.md index eaf1f89504..7501cbaeba 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file com.google.cloud libraries-bom - 5.2.0 + 5.3.0 pom import @@ -50,11 +50,11 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.cloud:google-cloud-bigquerystorage:0.131.0-beta' +compile 'com.google.cloud:google-cloud-bigquerystorage:0.131.1-beta' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquerystorage" % "0.131.0-beta" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquerystorage" % "0.131.1-beta" ``` [//]: # ({x-version-update-end}) diff --git a/google-cloud-bigquerystorage-bom/pom.xml b/google-cloud-bigquerystorage-bom/pom.xml index 73a84dd771..30b90fe8ad 100644 --- a/google-cloud-bigquerystorage-bom/pom.xml +++ b/google-cloud-bigquerystorage-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquerystorage-bom - 0.131.0-beta + 0.131.1-beta pom com.google.cloud @@ -63,48 +63,48 @@ com.google.api.grpc proto-google-cloud-bigquerystorage-v1alpha2 - 0.96.0 + 0.96.1 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta1 - 0.96.0 + 0.96.1 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta2 - 0.96.0 + 0.96.1 com.google.api.grpc proto-google-cloud-bigquerystorage-v1 - 0.96.0 + 0.96.1 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1alpha2 - 0.96.0 + 0.96.1 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta1 - 0.96.0 + 0.96.1 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta2 - 0.96.0 + 0.96.1 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1 - 0.96.0 + 0.96.1 com.google.cloud google-cloud-bigquerystorage - 0.131.0-beta + 0.131.1-beta diff --git a/google-cloud-bigquerystorage/pom.xml b/google-cloud-bigquerystorage/pom.xml index 4f6230e87d..7246183acc 100644 --- a/google-cloud-bigquerystorage/pom.xml +++ b/google-cloud-bigquerystorage/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquerystorage - 0.131.0-beta + 0.131.1-beta jar BigQuery Storage https://github.com/googleapis/java-bigquerystorage @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquerystorage-parent - 0.131.0-beta + 0.131.1-beta google-cloud-bigquerystorage diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/DirectWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/DirectWriter.java index 295638f74f..4b3032b615 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/DirectWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/DirectWriter.java @@ -102,4 +102,9 @@ public static void testSetStub( BigQueryWriteClient stub, int maxTableEntry, SchemaCompact schemaCheck) { cache = WriterCache.getTestInstance(stub, maxTableEntry, schemaCheck); } + + /** Clears the underlying cache and all the transport connections. */ + public static void clearCache() { + cache.clear(); + } } diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/StreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/StreamWriter.java index cd27c741e5..b9dd306935 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/StreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/StreamWriter.java @@ -165,10 +165,14 @@ private StreamWriter(Builder builder) Instant.ofEpochSecond( stream.getCreateTime().getSeconds(), stream.getCreateTime().getNanos()); if (stream.getType() == Stream.WriteStream.Type.PENDING && stream.hasCommitTime()) { + backgroundResources.shutdown(); + backgroundResources.awaitTermination(1, TimeUnit.MINUTES); throw new IllegalStateException( "Cannot write to a stream that is already committed: " + streamName); } if (createTime.plus(streamTTL).compareTo(Instant.now()) < 0) { + backgroundResources.shutdown(); + backgroundResources.awaitTermination(1, TimeUnit.MINUTES); throw new IllegalStateException( "Cannot write to a stream that is already expired: " + streamName); } @@ -247,7 +251,7 @@ public ApiFuture append(AppendRowsRequest message) { */ public void refreshAppend() throws IOException, InterruptedException { synchronized (this) { - Preconditions.checkState(!shutdown.get(), "Cannot append on a shut-down writer."); + Preconditions.checkState(!shutdown.get(), "Cannot shut down on a shut-down writer."); // There could be a moment, stub is not yet initialized. if (clientStream != null) { clientStream.closeSend(); @@ -475,6 +479,7 @@ public RetrySettings getRetrySettings() { public void shutdown() { Preconditions.checkState( !shutdown.getAndSet(true), "Cannot shut down a writer already shut-down."); + LOG.info("Shutdown called on writer"); if (currentAlarmFuture != null && activeAlarm.getAndSet(false)) { currentAlarmFuture.cancel(false); } @@ -684,10 +689,6 @@ > getApiMaxInflightRequests()) { */ public Builder setRetrySettings(RetrySettings retrySettings) { Preconditions.checkNotNull(retrySettings); - Preconditions.checkArgument( - retrySettings.getTotalTimeout().compareTo(MIN_TOTAL_TIMEOUT) >= 0); - Preconditions.checkArgument( - retrySettings.getInitialRpcTimeout().compareTo(MIN_RPC_TIMEOUT) >= 0); this.retrySettings = retrySettings; return this; } diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/WriterCache.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/WriterCache.java index 9b7cb1fd18..68eb59d4af 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/WriterCache.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/WriterCache.java @@ -18,8 +18,11 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalListener; +import com.google.common.cache.RemovalNotification; import com.google.protobuf.Descriptors.Descriptor; import java.io.IOException; +import java.util.concurrent.ConcurrentMap; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -52,6 +55,15 @@ private WriterCache(BigQueryWriteClient stub, int maxTableEntry, SchemaCompact c writerCache = CacheBuilder.newBuilder() .maximumSize(maxTableEntry) + .removalListener( + new RemovalListener>() { + @Override + public void onRemoval( + RemovalNotification> + removalNotification) { + removalNotification.getValue().invalidateAll(); + } + }) .>build(); } @@ -134,6 +146,14 @@ public StreamWriter getTableWriter(String tableName, Descriptor userSchema) tableEntry = CacheBuilder.newBuilder() .maximumSize(MAX_WRITERS_PER_TABLE) + .removalListener( + new RemovalListener() { + @Override + public void onRemoval( + RemovalNotification removalNotification) { + removalNotification.getValue().close(); + } + }) .build(); writer = CreateNewWriter(streamName); tableEntry.put(userSchema, writer); @@ -144,6 +164,22 @@ public StreamWriter getTableWriter(String tableName, Descriptor userSchema) return writer; } + /** Clear the cache and close all the writers in the cache. */ + public void clear() { + synchronized (this) { + ConcurrentMap> map = writerCache.asMap(); + for (String key : map.keySet()) { + Cache entry = writerCache.getIfPresent(key); + ConcurrentMap entryMap = entry.asMap(); + for (Descriptor descriptor : entryMap.keySet()) { + StreamWriter writer = entry.getIfPresent(descriptor); + writer.close(); + } + } + writerCache.cleanUp(); + } + } + @VisibleForTesting public long cachedTableCount() { synchronized (writerCache) { diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/DirectWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/DirectWriterTest.java index f6a0948802..c3b059c777 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/DirectWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/DirectWriterTest.java @@ -24,15 +24,15 @@ import com.google.api.gax.grpc.testing.MockGrpcService; import com.google.api.gax.grpc.testing.MockServiceHelper; import com.google.cloud.bigquery.storage.test.Test.*; +import com.google.common.collect.Sets; import com.google.protobuf.AbstractMessage; import com.google.protobuf.Timestamp; import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutionException; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; import org.junit.*; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -42,6 +42,8 @@ @RunWith(JUnit4.class) public class DirectWriterTest { + private static final Logger LOG = Logger.getLogger(DirectWriterTest.class.getName()); + private static final String TEST_TABLE = "projects/p/datasets/d/tables/t"; private static final String TEST_STREAM = "projects/p/datasets/d/tables/t/streams/s"; private static final String TEST_STREAM_2 = "projects/p/datasets/d/tables/t/streams/s2"; @@ -86,7 +88,7 @@ public void tearDown() throws Exception { } /** Response mocks for create a new writer */ - void WriterCreationResponseMock(String testStreamName, List responseOffsets) { + void WriterCreationResponseMock(String testStreamName, Set responseOffsets) { // Response from CreateWriteStream Stream.WriteStream expectedResponse = Stream.WriteStream.newBuilder().setName(testStreamName).build(); @@ -117,7 +119,7 @@ public void testWriteSuccess() throws Exception { FooType m1 = FooType.newBuilder().setFoo("m1").build(); FooType m2 = FooType.newBuilder().setFoo("m2").build(); - WriterCreationResponseMock(TEST_STREAM, Arrays.asList(Long.valueOf(0L))); + WriterCreationResponseMock(TEST_STREAM, Sets.newHashSet(Long.valueOf(0L))); ApiFuture ret = DirectWriter.append(TEST_TABLE, Arrays.asList(m1, m2)); verify(schemaCheck).check(TEST_TABLE, FooType.getDescriptor()); assertEquals(Long.valueOf(0L), ret.get()); @@ -159,7 +161,7 @@ public void testWriteSuccess() throws Exception { assertEquals(expectRequest.toString(), actualRequests.get(3).toString()); // Write with a different schema. - WriterCreationResponseMock(TEST_STREAM_2, Arrays.asList(Long.valueOf(0L))); + WriterCreationResponseMock(TEST_STREAM_2, Sets.newHashSet(Long.valueOf(0L))); AllSupportedTypes m3 = AllSupportedTypes.newBuilder().setStringValue("s").build(); ret = DirectWriter.append(TEST_TABLE, Arrays.asList(m3)); verify(schemaCheck).check(TEST_TABLE, AllSupportedTypes.getDescriptor()); @@ -181,6 +183,8 @@ public void testWriteSuccess() throws Exception { ((Storage.CreateWriteStreamRequest) actualRequests.get(4)).getWriteStream().getType()); assertEquals(TEST_STREAM_2, ((Storage.GetWriteStreamRequest) actualRequests.get(5)).getName()); assertEquals(expectRequest.toString(), actualRequests.get(6).toString()); + + DirectWriter.clearCache(); } @Test @@ -195,15 +199,17 @@ public void testWriteBadTableName() throws Exception { } catch (IllegalArgumentException expected) { assertEquals("Invalid table name: abc", expected.getMessage()); } + + DirectWriter.clearCache(); } @Test public void testConcurrentAccess() throws Exception { - WriterCache cache = WriterCache.getTestInstance(client, 2, schemaCheck); + DirectWriter.testSetStub(client, 2, schemaCheck); final FooType m1 = FooType.newBuilder().setFoo("m1").build(); final FooType m2 = FooType.newBuilder().setFoo("m2").build(); - final List expectedOffset = - Arrays.asList( + final Set expectedOffset = + Sets.newHashSet( Long.valueOf(0L), Long.valueOf(2L), Long.valueOf(4L), @@ -221,12 +227,21 @@ public void run() { try { ApiFuture result = DirectWriter.append(TEST_TABLE, Arrays.asList(m1, m2)); - assertTrue(expectedOffset.remove(result.get())); - } catch (IOException | InterruptedException | ExecutionException e) { - fail(e.getMessage()); + synchronized (expectedOffset) { + assertTrue(expectedOffset.remove(result.get())); + } + } catch (Exception e) { + fail(e.toString()); } } }); } + executor.shutdown(); + try { + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + LOG.info(e.toString()); + } + DirectWriter.clearCache(); } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/StreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/StreamWriterTest.java index 950419fdc9..3dba7524d0 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/StreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/StreamWriterTest.java @@ -33,6 +33,7 @@ import com.google.api.gax.grpc.testing.LocalChannelProvider; import com.google.api.gax.grpc.testing.MockGrpcService; import com.google.api.gax.grpc.testing.MockServiceHelper; +import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.DataLossException; import com.google.cloud.bigquery.storage.test.Test.FooType; import com.google.cloud.bigquery.storage.v1alpha2.Storage.*; @@ -135,8 +136,9 @@ private ApiFuture sendTestMessage(StreamWriter writer, Strin @Test public void testTableName() throws Exception { - StreamWriter writer = getTestStreamWriterBuilder().build(); - assertEquals("projects/p/datasets/d/tables/t", writer.getTableNameString()); + try (StreamWriter writer = getTestStreamWriterBuilder().build()) { + assertEquals("projects/p/datasets/d/tables/t", writer.getTableNameString()); + } } @Test @@ -175,7 +177,7 @@ public void testAppendByDuration() throws Exception { .getSerializedRowsCount()); assertEquals( true, testBigQueryWrite.getAppendRequests().get(0).getProtoRows().hasWriterSchema()); - writer.shutdown(); + writer.close(); } @Test @@ -228,7 +230,7 @@ public void testAppendByNumBatchedMessages() throws Exception { .getSerializedRowsCount()); assertEquals( false, testBigQueryWrite.getAppendRequests().get(1).getProtoRows().hasWriterSchema()); - writer.shutdown(); + writer.close(); } @Test @@ -264,7 +266,7 @@ public void testAppendByNumBytes() throws Exception { assertEquals(3, testBigQueryWrite.getAppendRequests().size()); - writer.shutdown(); + writer.close(); } @Test @@ -429,7 +431,8 @@ public void testFlowControlBehaviorException() throws Exception { try { appendFuture2.get(); Assert.fail("This should fail"); - } catch (ExecutionException e) { + } catch (Exception e) { + LOG.info("ControlFlow test exception: " + e.toString()); assertEquals("The maximum number of batch elements: 1 have been reached.", e.getMessage()); } assertEquals(1L, appendFuture1.get().getOffset()); @@ -437,7 +440,7 @@ public void testFlowControlBehaviorException() throws Exception { } @Test - public void testStreamReconnection() throws Exception { + public void testStreamReconnectionTransient() throws Exception { StreamWriter writer = getTestStreamWriterBuilder() .setBatchingSettings( @@ -448,7 +451,6 @@ public void testStreamReconnection() throws Exception { .build()) .build(); - // Case 1: Request succeeded after retry since the error is transient. StatusRuntimeException transientError = new StatusRuntimeException(Status.UNAVAILABLE); testBigQueryWrite.addException(transientError); testBigQueryWrite.addResponse(AppendRowsResponse.newBuilder().setOffset(0).build()); @@ -456,9 +458,20 @@ public void testStreamReconnection() throws Exception { assertEquals(false, future1.isDone()); // Retry is scheduled to be 7 seconds later. assertEquals(0L, future1.get().getOffset()); + writer.close(); + } - LOG.info("======CASE II"); - // Case 2 : Request failed since the error is not transient. + @Test + public void testStreamReconnectionPermanant() throws Exception { + StreamWriter writer = + getTestStreamWriterBuilder() + .setBatchingSettings( + StreamWriter.Builder.DEFAULT_BATCHING_SETTINGS + .toBuilder() + .setDelayThreshold(Duration.ofSeconds(100000)) + .setElementCountThreshold(1L) + .build()) + .build(); StatusRuntimeException permanentError = new StatusRuntimeException(Status.INVALID_ARGUMENT); testBigQueryWrite.addException(permanentError); ApiFuture future2 = sendTestMessage(writer, new String[] {"m2"}); @@ -468,11 +481,26 @@ public void testStreamReconnection() throws Exception { } catch (ExecutionException e) { assertEquals(permanentError.toString(), e.getCause().getCause().toString()); } + writer.close(); + } - LOG.info("======CASE III"); - // Case 3: Failed after retried max retry times. - testBigQueryWrite.addException(transientError); - testBigQueryWrite.addException(transientError); + @Test + public void testStreamReconnectionExceedRetry() throws Exception { + StreamWriter writer = + getTestStreamWriterBuilder() + .setBatchingSettings( + StreamWriter.Builder.DEFAULT_BATCHING_SETTINGS + .toBuilder() + .setDelayThreshold(Duration.ofSeconds(100000)) + .setElementCountThreshold(1L) + .build()) + .setRetrySettings( + RetrySettings.newBuilder() + .setMaxRetryDelay(Duration.ofMillis(100)) + .setMaxAttempts(1) + .build()) + .build(); + StatusRuntimeException transientError = new StatusRuntimeException(Status.UNAVAILABLE); testBigQueryWrite.addException(transientError); testBigQueryWrite.addException(transientError); ApiFuture future3 = sendTestMessage(writer, new String[] {"m3"}); @@ -812,5 +840,7 @@ public void testExistingClient() throws Exception { StreamWriter writer = StreamWriter.newBuilder(TEST_STREAM, client).build(); writer.close(); assertFalse(client.isShutdown()); + client.shutdown(); + client.awaitTermination(1, TimeUnit.MINUTES); } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/WriterCacheTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/WriterCacheTest.java index 47ad647e66..eb249ddd39 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/WriterCacheTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/WriterCacheTest.java @@ -33,6 +33,7 @@ import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import org.junit.*; import org.junit.runner.RunWith; @@ -143,6 +144,7 @@ public void testCreateNewWriter() throws Exception { assertEquals(TEST_TABLE, writer.getTableNameString()); assertEquals(TEST_STREAM, writer.getStreamNameString()); assertEquals(1, cache.cachedTableCount()); + cache.clear(); } @Test @@ -173,6 +175,7 @@ public void testWriterExpired() throws Exception { "Cannot write to a stream that is already expired: projects/p/datasets/d/tables/t/streams/s", e.getMessage()); } + cache.clear(); } @Test @@ -216,6 +219,7 @@ public void testWriterWithNewSchema() throws Exception { assertEquals(TEST_STREAM_3, writer4.getStreamNameString()); assertEquals(TEST_STREAM_4, writer5.getStreamNameString()); assertEquals(1, cache.cachedTableCount()); + cache.clear(); } @Test @@ -259,6 +263,7 @@ public void testWriterWithDifferentTable() throws Exception { assertEquals(TEST_STREAM_31, writer4.getStreamNameString()); assertEquals(TEST_STREAM, writer5.getStreamNameString()); assertEquals(2, cache.cachedTableCount()); + cache.clear(); } @Test @@ -275,11 +280,17 @@ public void testConcurrentAccess() throws Exception { public void run() { try { assertTrue(cache.getTableWriter(TEST_TABLE, FooType.getDescriptor()) != null); - } catch (IOException | InterruptedException e) { + } catch (Exception e) { fail(e.getMessage()); } } }); } + executor.shutdown(); + try { + executor.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + LOG.info(e.toString()); + } } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/it/ITBigQueryWriteManualClientTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/it/ITBigQueryWriteManualClientTest.java index 04c831ccc6..da6e144323 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/it/ITBigQueryWriteManualClientTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/it/ITBigQueryWriteManualClientTest.java @@ -391,5 +391,12 @@ public Long call() throws IOException, InterruptedException, ExecutionException assertTrue(expectedOffset.remove(response.get())); } assertTrue(expectedOffset.isEmpty()); + executor.shutdown(); + try { + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + LOG.info(e.toString()); + } + DirectWriter.clearCache(); } } diff --git a/grpc-google-cloud-bigquerystorage-v1/pom.xml b/grpc-google-cloud-bigquerystorage-v1/pom.xml index f47279fc0a..bc8d0f09af 100644 --- a/grpc-google-cloud-bigquerystorage-v1/pom.xml +++ b/grpc-google-cloud-bigquerystorage-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1 - 0.96.0 + 0.96.1 grpc-google-cloud-bigquerystorage-v1 GRPC library for grpc-google-cloud-bigquerystorage-v1 com.google.cloud google-cloud-bigquerystorage-parent - 0.131.0-beta + 0.131.1-beta diff --git a/grpc-google-cloud-bigquerystorage-v1alpha2/pom.xml b/grpc-google-cloud-bigquerystorage-v1alpha2/pom.xml index 6b954591f7..1aa6035916 100644 --- a/grpc-google-cloud-bigquerystorage-v1alpha2/pom.xml +++ b/grpc-google-cloud-bigquerystorage-v1alpha2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1alpha2 - 0.96.0 + 0.96.1 grpc-google-cloud-bigquerystorage-v1alpha2 GRPC library for grpc-google-cloud-bigquerystorage-v1alpha2 com.google.cloud google-cloud-bigquerystorage-parent - 0.131.0-beta + 0.131.1-beta diff --git a/grpc-google-cloud-bigquerystorage-v1beta1/pom.xml b/grpc-google-cloud-bigquerystorage-v1beta1/pom.xml index efaa6ad929..519c01f7c5 100644 --- a/grpc-google-cloud-bigquerystorage-v1beta1/pom.xml +++ b/grpc-google-cloud-bigquerystorage-v1beta1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta1 - 0.96.0 + 0.96.1 grpc-google-cloud-bigquerystorage-v1beta1 GRPC library for grpc-google-cloud-bigquerystorage-v1beta1 com.google.cloud google-cloud-bigquerystorage-parent - 0.131.0-beta + 0.131.1-beta diff --git a/grpc-google-cloud-bigquerystorage-v1beta2/pom.xml b/grpc-google-cloud-bigquerystorage-v1beta2/pom.xml index bcf08340fc..6cbd47baff 100644 --- a/grpc-google-cloud-bigquerystorage-v1beta2/pom.xml +++ b/grpc-google-cloud-bigquerystorage-v1beta2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta2 - 0.96.0 + 0.96.1 grpc-google-cloud-bigquerystorage-v1beta2 GRPC library for grpc-google-cloud-bigquerystorage-v1beta2 com.google.cloud google-cloud-bigquerystorage-parent - 0.131.0-beta + 0.131.1-beta diff --git a/pom.xml b/pom.xml index c7c0a7c742..0208ba6ba6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquerystorage-parent pom - 0.131.0-beta + 0.131.1-beta BigQuery Storage Parent https://github.com/googleapis/java-bigquerystorage @@ -103,47 +103,47 @@ com.google.api.grpc proto-google-cloud-bigquerystorage-v1alpha2 - 0.96.0 + 0.96.1 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta1 - 0.96.0 + 0.96.1 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta2 - 0.96.0 + 0.96.1 com.google.api.grpc proto-google-cloud-bigquerystorage-v1 - 0.96.0 + 0.96.1 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1alpha2 - 0.96.0 + 0.96.1 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta1 - 0.96.0 + 0.96.1 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta2 - 0.96.0 + 0.96.1 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1 - 0.96.0 + 0.96.1 com.google.cloud google-cloud-bigquerystorage - 0.131.0-beta + 0.131.1-beta com.fasterxml.jackson.core diff --git a/proto-google-cloud-bigquerystorage-v1/pom.xml b/proto-google-cloud-bigquerystorage-v1/pom.xml index 87835f51b7..6a693a53c4 100644 --- a/proto-google-cloud-bigquerystorage-v1/pom.xml +++ b/proto-google-cloud-bigquerystorage-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1 - 0.96.0 + 0.96.1 proto-google-cloud-bigquerystorage-v1 PROTO library for proto-google-cloud-bigquerystorage-v1 com.google.cloud google-cloud-bigquerystorage-parent - 0.131.0-beta + 0.131.1-beta diff --git a/proto-google-cloud-bigquerystorage-v1alpha2/pom.xml b/proto-google-cloud-bigquerystorage-v1alpha2/pom.xml index b6c6ace0f7..cf78fe82d0 100644 --- a/proto-google-cloud-bigquerystorage-v1alpha2/pom.xml +++ b/proto-google-cloud-bigquerystorage-v1alpha2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1alpha2 - 0.96.0 + 0.96.1 proto-google-cloud-bigquerystorage-v1alpha2 PROTO library for proto-google-cloud-bigquerystorage-v1alpha2 com.google.cloud google-cloud-bigquerystorage-parent - 0.131.0-beta + 0.131.1-beta diff --git a/proto-google-cloud-bigquerystorage-v1beta1/pom.xml b/proto-google-cloud-bigquerystorage-v1beta1/pom.xml index f19ff3be6c..f65681936b 100644 --- a/proto-google-cloud-bigquerystorage-v1beta1/pom.xml +++ b/proto-google-cloud-bigquerystorage-v1beta1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta1 - 0.96.0 + 0.96.1 proto-google-cloud-bigquerystorage-v1beta1 PROTO library for proto-google-cloud-bigquerystorage-v1beta1 com.google.cloud google-cloud-bigquerystorage-parent - 0.131.0-beta + 0.131.1-beta diff --git a/proto-google-cloud-bigquerystorage-v1beta2/pom.xml b/proto-google-cloud-bigquerystorage-v1beta2/pom.xml index 5d14cac077..ebbfa155ec 100644 --- a/proto-google-cloud-bigquerystorage-v1beta2/pom.xml +++ b/proto-google-cloud-bigquerystorage-v1beta2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta2 - 0.96.0 + 0.96.1 proto-google-cloud-bigquerystorage-v1beta2 PROTO library for proto-google-cloud-bigquerystorage-v1beta2 com.google.cloud google-cloud-bigquerystorage-parent - 0.131.0-beta + 0.131.1-beta diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index be539afe1e..5251248ec1 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -29,7 +29,7 @@ com.google.cloud google-cloud-bigquerystorage - 0.130.0-beta + 0.131.0-beta diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 19513bd221..aae487f47a 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-bigquerystorage - 0.131.0-beta + 0.131.1-beta diff --git a/synth.metadata b/synth.metadata index 6102832074..3bc8037095 100644 --- a/synth.metadata +++ b/synth.metadata @@ -10,47 +10,49 @@ { "git": { "name": ".", - "remote": "https://github.com/googleapis/java-bigquerystorage.git", - "sha": "066a994adc3add6222c5367a27fbd5e600dff900" + "remote": "git@github.com:stephaniewang526/java-bigquerystorage.git", + "sha": "c6d9d9ac2c2ef5a901525c9adda64082d826e67e" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "28e76243c23cc282efbb288cb558c174e3e5e9ee", - "internalRef": "308294748" + "sha": "aaff764c185e18a6c73227357c3df5fa60fec85a", + "internalRef": "309426927" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "28e76243c23cc282efbb288cb558c174e3e5e9ee", - "internalRef": "308294748" + "sha": "aaff764c185e18a6c73227357c3df5fa60fec85a", + "internalRef": "309426927" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "28e76243c23cc282efbb288cb558c174e3e5e9ee", - "internalRef": "308294748" + "sha": "aaff764c185e18a6c73227357c3df5fa60fec85a", + "internalRef": "309426927" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "28e76243c23cc282efbb288cb558c174e3e5e9ee", - "internalRef": "308294748" + "sha": "aaff764c185e18a6c73227357c3df5fa60fec85a", + "internalRef": "309426927", + "log": "aaff764c185e18a6c73227357c3df5fa60fec85a\nchore: enable gapicv2 and annotate protos for recommendationengine/v1beta1 API\n\nCommitter: @miraleung\nPiperOrigin-RevId: 309426927\n\nf78da0d54c78aa8f66d52a448db1f7ec0e0fd591\nfix: replace all retry configs removed during GAPIC config migration\n\nPiperOrigin-RevId: 309415646\n\n7e1c7603c8cb2f05ef29842405654ff8a9d3d108\nAdd an OAuth scope annotation in build_service proto file\n\nPiperOrigin-RevId: 309058279\n\n3f5f8a2258c6a41f9fbf7b80acbca631dda0a952\nfeat: added support for project id in scope\nfeat: added support for order_by in SearchAllResources rpc\nfeat: added support for search on location, labels and networkTags fields\ndocs: updated according to above features\n\nPiperOrigin-RevId: 308922843\n\n756b174de4a122461993c1c583345533d819936d\nfix: Add missing method_signature annotations for BigTable Admin Backup RPCs\n\nThe added method_signatures reflect method flattenings in the GAPIC v1 config.\n\nPiperOrigin-RevId: 308824110\n\nc284e4f849e4b085a297d336cef8721ab69ba013\nConfigure retry for Diaglogflow v2.Environments.ListEnvironments and v2beta1.Environments.ListEnvironments.\n\nPiperOrigin-RevId: 308749074\n\nff17eefa184f6985e5d9a8ee3e725da95a06cd23\nchore: add Python target for orgpolicy BUILD.bazel\n\nPiperOrigin-RevId: 308747658\n\n371206424e6db4df2cb3734bb1013dcaac9dbe4b\nRemove raw html from proto comments as this breaks client library documentation generators.\n\nSee https://aip.dev/192\n\n> Any formatting in comments must be in CommonMark. Headings and tables must not be used, as these cause problems for several tools, and are unsuitable for client library reference documentation.\n>\n> Comments should use code font for property names and for literals (such as true).\n>\n> Raw HTML must not be used.\n\nPiperOrigin-RevId: 308716969\n\nc2bfceedfc9f01356229bd206ad25e250aa9551c\nGenerating BUILD.bazel for maps/routes/v1alpha.\n\nPiperOrigin-RevId: 308691010\n\n5dca7f1079a4c127245c0f1dbb8293b434fbd0c8\nSupport purchasing capacity commitments in the BigQuery Reservation v1 API.\n\nPiperOrigin-RevId: 308671886\n\n" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "f8a9933e5e98202b04ef427f28d1d79221190fa4" + "sha": "cdddf139b36000b3a7c65fd2a7781e253262359a", + "log": "cdddf139b36000b3a7c65fd2a7781e253262359a\nfeat: implement multiple version for nodejs lib (#504)\n\n* feat: synthool generate src/index for nodejs client library\n\n* fix the linters\n\n* merge upstream\n\n* remove '-' in jinja tempalte\n\n* rename the functions\n\n* add try, finally to avoid change direcory\n\n* optional for versions and default verison\n958a803ee0c43523fa3e6e0bb016276a69614c8e\nfix: update requirements.txt (#516)\n\n* fix: remove version update check\n\nbecause autosynth now directly calls the synthtool code in this repo.\n\n* fix: add 'deprecation' dot requirements.txt\n\n* fix: add `protobuf` to requirements.txt\n\n* lint\n6b685a261479e27fd959dd7b1ff253d6cf595172\nfix: synthtool path (#515)\n\n\n46fb62a3ebda62cc6c1c98e4f1eaf91965eab580\nfix: kokoro path (#514)\n\n* fix: path to .kokoro-autosynth\n\n* fix: path in ruby, dotnet, and nodejs cnfgs\n7c1a92d00de654db2fb440243129c61393a694e2\nfix: path to build.sh (#513)\n\n\n720b7bff0da03ed9e67cb2362f2d291241fa59ae\nfix: typo (#512)\n\n\n969a2340e74c73227e7c1638ed7650abcac22ee4\nchore: merge autosynth repo into this repo (#508)\n\n* chore: merge autosynth repo into this repo\n\n* fix: add autsynth directory to code coverage reports\n01b6f23d24b27878b48667ce597876d66b59780e\nfix(python): install testutils from pypi (#503)\n\nhttps://pypi.org/project/google-cloud-testutils/\n716f741f2d307b48cbe8a5bc3bc883571212344a\nfix(python): adjust regex for fix_pb2_headers (#500)\n\nFixes regex and updates copyright year to 2020. \n\nBefore:\n```python\n# -*- coding: utf-8 -*-\n\n# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: google/type/timeofday.proto\n```\n\nAfter:\n```python\n# -*- coding: utf-8 -*-\n\n# Copyright 2020 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: google/type/timeofday.proto\n```\n21ce80f9d4785dc75aa36d26944bddd5a1b7e25b\nfeat: better Node.js postprocessing (#498)\n\n* feat: better Node.js postprocessing\n\n* docs: docstring for is_gapic_library\n\n* PR feedback\n\n* pass parameter by name\n\nCo-authored-by: Jeffrey Rennie \nCo-authored-by: Benjamin E. Coe \nf5e4c17dc78a966dbf29961dd01f9bbd63e20a04\nfeat: Add include_protos parameter support for GAPICBazel (#489)\n\n* feat: Add include_protos parameter support for GAPICBazel. This is needed to migrate python libraries, which are heavily dependent on this feature.\n\n* add proto_output_path parameter, which is needed for non-standard cases (cloudbuild, containeranalysis, which are under devtools directory)\n\nCo-authored-by: Justin Beckwith \nCo-authored-by: Jeffrey Rennie \n19465d3ec5e5acdb01521d8f3bddd311bcbee28d\nbuild: use codecov's action, now that it's authless (#499)\n\n\n" } } ], diff --git a/synth.py b/synth.py index be86b40f42..5956987ef1 100644 --- a/synth.py +++ b/synth.py @@ -35,27 +35,6 @@ destination_name='bigquerystorage', ) -java.common_templates() - -# TODO: Remove all below s.replace() logic when upstream correction is made in gapic https://github.com/googleapis/gapic-generator/issues/3181 -# Remove line added by gapic generator -s.replace("google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/MockBigQueryWriteImpl.java", - "final Object response = responses.remove();", - "") - -# Add back lines removed by gapic generator -s.replace("google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/MockBigQueryWriteImpl.java", - """ - public void onNext(AppendRowsRequest value) { - if (response instanceof AppendRowsResponse) { - """, - """ - public void onNext(AppendRowsRequest value) { - requests.add(value); - final Object response = responses.remove(); - if (response instanceof AppendRowsResponse) { - """ - ) - -# Re-run java code formatter after making hacky code change -os.system("mvn com.coveo:fmt-maven-plugin:format") \ No newline at end of file +java.common_templates(excludes=[ + '.kokoro/build.bat' # TODO: remove this when we switch to actions +]) \ No newline at end of file diff --git a/versions.txt b/versions.txt index 9541b8146a..dda6e853ad 100644 --- a/versions.txt +++ b/versions.txt @@ -1,12 +1,12 @@ # Format: # module:released-version:current-version -proto-google-cloud-bigquerystorage-v1alpha2:0.96.0:0.96.0 -proto-google-cloud-bigquerystorage-v1beta1:0.96.0:0.96.0 -proto-google-cloud-bigquerystorage-v1beta2:0.96.0:0.96.0 -proto-google-cloud-bigquerystorage-v1:0.96.0:0.96.0 -grpc-google-cloud-bigquerystorage-v1alpha2:0.96.0:0.96.0 -grpc-google-cloud-bigquerystorage-v1beta1:0.96.0:0.96.0 -grpc-google-cloud-bigquerystorage-v1beta2:0.96.0:0.96.0 -grpc-google-cloud-bigquerystorage-v1:0.96.0:0.96.0 -google-cloud-bigquerystorage:0.131.0-beta:0.131.0-beta +proto-google-cloud-bigquerystorage-v1alpha2:0.96.1:0.96.1 +proto-google-cloud-bigquerystorage-v1beta1:0.96.1:0.96.1 +proto-google-cloud-bigquerystorage-v1beta2:0.96.1:0.96.1 +proto-google-cloud-bigquerystorage-v1:0.96.1:0.96.1 +grpc-google-cloud-bigquerystorage-v1alpha2:0.96.1:0.96.1 +grpc-google-cloud-bigquerystorage-v1beta1:0.96.1:0.96.1 +grpc-google-cloud-bigquerystorage-v1beta2:0.96.1:0.96.1 +grpc-google-cloud-bigquerystorage-v1:0.96.1:0.96.1 +google-cloud-bigquerystorage:0.131.1-beta:0.131.1-beta