diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 5d788be0b5..88d9b926c1 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest - digest: sha256:a744afda1956beb7bd6dc8d0b740a7839bbe402e34e2f4ef24589074db53ceb0 + digest: sha256:5b8c790f57cca57e6b37ba25f79291265c218cea3e6ba9714b001910ab3f1419 diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 5454c3fc5d..e490e6cd70 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -91,7 +91,6 @@ samples) pushd ${SAMPLES_DIR} mvn -B \ - -Penable-samples \ -ntp \ -DtrimStackTrace=false \ -Dclirr.skip=true \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 50caea76ae..4e23ce49a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [2.5.0](https://www.github.com/googleapis/java-bigquery/compare/v2.4.1...v2.5.0) (2021-12-01) + + +### Features + +* add support for BI Engine Statistics ([#1723](https://www.github.com/googleapis/java-bigquery/issues/1723)) ([13cc6e6](https://www.github.com/googleapis/java-bigquery/commit/13cc6e608fd501067f7c5dcd2f5b9a03c078b065)) + ### [2.4.1](https://www.github.com/googleapis/java-bigquery/compare/v2.4.0...v2.4.1) (2021-11-16) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f2dbdee06b..b65dd279c9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,12 +53,12 @@ mvn -Penable-integration-tests clean verify ## Code Samples -Code Samples must be bundled in separate Maven modules, and guarded by a -Maven profile with the name `enable-samples`. +All code samples must be in compliance with the [java sample formatting guide][3]. +Code Samples must be bundled in separate Maven modules. The samples must be separate from the primary project for a few reasons: -1. Primary projects have a minimum Java version of Java 7 whereas samples have - a minimum Java version of Java 8. Due to this we need the ability to +1. Primary projects have a minimum Java version of Java 8 whereas samples can have + Java version of Java 11. Due to this we need the ability to selectively exclude samples from a build run. 2. Many code samples depend on external GCP services and need credentials to access the service. @@ -68,39 +68,16 @@ The samples must be separate from the primary project for a few reasons: ### Building ```bash -mvn -Penable-samples clean verify +mvn clean verify ``` Some samples require access to GCP services and require a service account: ```bash export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account.json -mvn -Penable-samples clean verify +mvn clean verify ``` -### Profile Config - -1. To add samples in a profile to your Maven project, add the following to your -`pom.xml` - - ```xml - - [...] - - - enable-samples - - sample - - - - [...] - - ``` - -2. [Activate](#profile-activation) the profile. -3. Define your samples in a normal Maven project in the `samples/` directory. - ### Code Formatting Code in this repo is formatted with @@ -110,30 +87,6 @@ To run formatting on your project, you can run: mvn com.coveo:fmt-maven-plugin:format ``` -### Profile Activation - -To include code samples when building and testing the project, enable the -`enable-samples` Maven profile. - -#### Command line - -To activate the Maven profile on the command line add `-Penable-samples` to your -Maven command. - -#### Maven `settings.xml` - -To activate the Maven profile in your `~/.m2/settings.xml` add an entry of -`enable-samples` following the instructions in [Active Profiles][2]. - -This method has the benefit of applying to all projects you build (and is -respected by IntelliJ IDEA) and is recommended if you are going to be -contributing samples to several projects. - -#### IntelliJ IDEA - -To activate the Maven Profile inside IntelliJ IDEA, follow the instructions in -[Activate Maven profiles][3] to activate `enable-samples`. - [1]: https://cloud.google.com/docs/authentication/getting-started#creating_a_service_account [2]: https://maven.apache.org/settings.html#Active_Profiles -[3]: https://www.jetbrains.com/help/idea/work-with-maven-profiles.html#activate_maven_profiles +[3]: https://github.com/GoogleCloudPlatform/java-docs-samples/blob/main/SAMPLE_FORMAT.md \ No newline at end of file diff --git a/README.md b/README.md index 8bfe942c18..b7643a88dc 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-bigquery - 2.4.0 + 2.4.1 ``` @@ -58,13 +58,13 @@ implementation 'com.google.cloud:google-cloud-bigquery' If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.4.0' +implementation 'com.google.cloud:google-cloud-bigquery:2.4.1' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.4.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.4.1" ``` ## Authentication diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 5ca5bc86e3..9ca71411f7 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -6,7 +6,7 @@ google-cloud-bigquery-parent com.google.cloud - 2.4.1 + 2.5.0 diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 3e4ec56c67..4eeeae45db 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery - 2.4.1 + 2.5.0 jar BigQuery https://github.com/googleapis/java-bigquery @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquery-parent - 2.4.1 + 2.5.0 google-cloud-bigquery diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineReason.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineReason.java new file mode 100644 index 0000000000..e682ba2461 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineReason.java @@ -0,0 +1,92 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.auto.value.AutoValue; +import java.io.Serializable; +import javax.annotation.Nullable; + +@AutoValue +public abstract class BiEngineReason implements Serializable { + + @AutoValue.Builder + public abstract static class Builder { + + /** + * High-level BI Engine reason for partial or disabled acceleration. + * + * @param code code or {@code null} for none + */ + public abstract Builder setCode(String code); + + /** + * Free form human-readable reason for partial or disabled acceleration. + * + * @param message message or {@code null} for none + */ + public abstract Builder setMessage(String message); + + /** Creates a {@code BiEngineReason} object. */ + public abstract BiEngineReason build(); + } + + /** + * High-level BI Engine reason for partial or disabled acceleration. + * + * @return value or {@code null} for none + */ + @Nullable + public abstract String getCode(); + + /** + * Free form human-readable reason for partial or disabled acceleration. + * + * @return value or {@code null} for none + */ + @Nullable + public abstract String getMessage(); + + public abstract Builder toBuilder(); + + public static Builder newBuilder() { + return new AutoValue_BiEngineReason.Builder(); + } + + com.google.api.services.bigquery.model.BiEngineReason toPb() { + com.google.api.services.bigquery.model.BiEngineReason biEngineReasonPb = + new com.google.api.services.bigquery.model.BiEngineReason(); + if (getCode() != null) { + biEngineReasonPb.setCode(getCode()); + } + if (getMessage() != null) { + biEngineReasonPb.setMessage(getMessage()); + } + return biEngineReasonPb; + } + + static BiEngineReason fromPb( + com.google.api.services.bigquery.model.BiEngineReason biEngineReasonPb) { + Builder builder = newBuilder(); + if (biEngineReasonPb.getCode() != null) { + builder.setCode(biEngineReasonPb.getCode()); + } + if (biEngineReasonPb.getMessage() != null) { + builder.setMessage(biEngineReasonPb.getMessage()); + } + return builder.build(); + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineStats.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineStats.java new file mode 100644 index 0000000000..34d6c43268 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BiEngineStats.java @@ -0,0 +1,101 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.api.services.bigquery.model.BiEngineStatistics; +import com.google.auto.value.AutoValue; +import java.io.Serializable; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +/** BIEngineStatistics contains query statistics specific to the use of BI Engine. */ +@AutoValue +public abstract class BiEngineStats implements Serializable { + + @AutoValue.Builder + public abstract static class Builder { + /** + * Specifies which mode of BI Engine acceleration was performed (if any). + * + * @param biEngineMode biEngineMode or {@code null} for none + */ + public abstract Builder setBiEngineMode(String biEngineMode); + + /** + * In case of DISABLED or PARTIAL bi_engine_mode, these contain the explanatory reasons as to + * why BI Engine could not accelerate. In case the full query was accelerated, this field is not + * populated. + * + * @param biEngineReasons biEngineReasons or {@code null} for none + */ + public abstract Builder setBiEngineReasons(List biEngineReasons); + + /** Creates a @code BiEngineStats} object. */ + public abstract BiEngineStats build(); + } + + /** + * Specifies which mode of BI Engine acceleration was performed (if any). + * + * @return value or {@code null} for none + */ + @Nullable + public abstract String getBiEngineMode(); + + /** + * In case of DISABLED or PARTIAL bi_engine_mode, these contain the explanatory reasons as to why + * BI Engine could not accelerate. In case the full query was accelerated, this field is not + * populated. + * + * @return value or {@code null} for none + */ + @Nullable + public abstract List getBiEngineReasons(); + + public abstract Builder toBuilder(); + + public static Builder newBuilder() { + return new AutoValue_BiEngineStats.Builder(); + } + + BiEngineStatistics toPb() { + BiEngineStatistics biEngineStatisticsPb = new BiEngineStatistics(); + if (getBiEngineMode() != null) { + biEngineStatisticsPb.setBiEngineMode(getBiEngineMode()); + } + if (getBiEngineReasons() != null) { + biEngineStatisticsPb.setBiEngineReasons( + getBiEngineReasons().stream().map(BiEngineReason::toPb).collect(Collectors.toList())); + } + return biEngineStatisticsPb; + } + + static BiEngineStats fromPb(BiEngineStatistics biEngineStatisticsPb) { + Builder builder = newBuilder(); + if (biEngineStatisticsPb.getBiEngineMode() != null) { + builder.setBiEngineMode(biEngineStatisticsPb.getBiEngineMode()); + } + if (biEngineStatisticsPb.getBiEngineReasons() != null) { + builder.setBiEngineReasons( + biEngineStatisticsPb.getBiEngineReasons().stream() + .map(BiEngineReason::fromPb) + .collect(Collectors.toList())); + } + return builder.build(); + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java index 999f8d2fe3..ab9fdabb39 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java @@ -321,6 +321,7 @@ public static class QueryStatistics extends JobStatistics { private static final long serialVersionUID = 7539354109226732353L; + private final BiEngineStats biEngineStats; private final Integer billingTier; private final Boolean cacheHit; private final String ddlOperationPerformed; @@ -402,6 +403,7 @@ public static StatementType[] values() { static final class Builder extends JobStatistics.Builder { + private BiEngineStats biEngineStats; private Integer billingTier; private Boolean cacheHit; private String ddlOperationPerformed; @@ -425,6 +427,10 @@ private Builder() {} private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { super(statisticsPb); if (statisticsPb.getQuery() != null) { + if (statisticsPb.getQuery().getBiEngineStatistics() != null) { + this.biEngineStats = + BiEngineStats.fromPb(statisticsPb.getQuery().getBiEngineStatistics()); + } this.billingTier = statisticsPb.getQuery().getBillingTier(); this.cacheHit = statisticsPb.getQuery().getCacheHit(); this.ddlOperationPerformed = statisticsPb.getQuery().getDdlOperationPerformed(); @@ -468,6 +474,11 @@ private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsP } } + Builder setBiEngineStats(BiEngineStats biEngineStats) { + this.biEngineStats = biEngineStats; + return self(); + } + Builder setBillingTier(Integer billingTier) { this.billingTier = billingTier; return self(); @@ -566,6 +577,7 @@ QueryStatistics build() { private QueryStatistics(Builder builder) { super(builder); + this.biEngineStats = builder.biEngineStats; this.billingTier = builder.billingTier; this.cacheHit = builder.cacheHit; this.ddlOperationPerformed = builder.ddlOperationPerformed; @@ -585,6 +597,11 @@ private QueryStatistics(Builder builder) { this.schema = builder.schema; } + /** Returns query statistics specific to the use of BI Engine. */ + public BiEngineStats getBiEngineStats() { + return biEngineStats; + } + /** Returns the billing tier for the job. */ public Integer getBillingTier() { return billingTier; @@ -701,6 +718,7 @@ public Schema getSchema() { @Override ToStringHelper toStringHelper() { return super.toStringHelper() + .add("biEngineStats", biEngineStats) .add("billingTier", billingTier) .add("cacheHit", cacheHit) .add("totalBytesBilled", totalBytesBilled) @@ -722,6 +740,7 @@ public final boolean equals(Object obj) { public final int hashCode() { return Objects.hash( baseHashCode(), + biEngineStats, billingTier, cacheHit, totalBytesBilled, @@ -733,6 +752,9 @@ public final int hashCode() { @Override com.google.api.services.bigquery.model.JobStatistics toPb() { JobStatistics2 queryStatisticsPb = new JobStatistics2(); + if (biEngineStats != null) { + queryStatisticsPb.setBiEngineStatistics(biEngineStats.toPb()); + } queryStatisticsPb.setBillingTier(billingTier); queryStatisticsPb.setCacheHit(cacheHit); queryStatisticsPb.setDdlOperationPerformed(ddlOperationPerformed); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java index 87fae41b58..0dad46e9e8 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/JobStatisticsTest.java @@ -36,6 +36,16 @@ public class JobStatisticsTest { + private static final BiEngineReason BI_ENGINE_REASON = + BiEngineReason.newBuilder() + .setMessage("Detected unsupported join type") + .setCode("UNSUPPORTED_SQL_TEXT") + .build(); + private static final BiEngineStats BI_ENGINE_STATS = + BiEngineStats.newBuilder() + .setBiEngineReasons(ImmutableList.of(BI_ENGINE_REASON)) + .setBiEngineMode("DISABLED") + .build(); private static final Integer BILLING_TIER = 42; private static final Boolean CACHE_HIT = true; private static final String DDL_OPERATION_PERFORMED = "SKIP"; @@ -154,6 +164,7 @@ public class JobStatisticsTest { .setCreationTimestamp(CREATION_TIME) .setEndTime(END_TIME) .setStartTime(START_TIME) + .setBiEngineStats(BI_ENGINE_STATS) .setBillingTier(BILLING_TIER) .setCacheHit(CACHE_HIT) .setDDLOperationPerformed(DDL_OPERATION_PERFORMED) @@ -246,6 +257,7 @@ public void testBuilder() { assertEquals(CREATION_TIME, QUERY_STATISTICS.getCreationTime()); assertEquals(START_TIME, QUERY_STATISTICS.getStartTime()); assertEquals(END_TIME, QUERY_STATISTICS.getEndTime()); + assertEquals(BI_ENGINE_STATS, QUERY_STATISTICS.getBiEngineStats()); assertEquals(BILLING_TIER, QUERY_STATISTICS.getBillingTier()); assertEquals(CACHE_HIT, QUERY_STATISTICS.getCacheHit()); assertEquals(DDL_OPERATION_PERFORMED, QUERY_STATISTICS.getDdlOperationPerformed()); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 8460bccb78..479a80e89f 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2900,6 +2900,14 @@ public void testQueryJob() throws InterruptedException, TimeoutException { assertTrue(bigquery.delete(destinationTable)); Job queryJob = bigquery.getJob(remoteJob.getJobId()); JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); + if (statistics.getBiEngineStats() != null) { + assertEquals(statistics.getBiEngineStats().getBiEngineMode(), "DISABLED"); + assertEquals( + statistics.getBiEngineStats().getBiEngineReasons().get(0).getCode(), "OTHER_REASON"); + assertEquals( + statistics.getBiEngineStats().getBiEngineReasons().get(0).getMessage(), + "Query output to destination table is not supported."); + } assertNotNull(statistics.getQueryPlan()); } diff --git a/pom.xml b/pom.xml index eda32fe314..f8ed583ed5 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquery-parent pom - 2.4.1 + 2.5.0 BigQuery Parent https://github.com/googleapis/java-bigquery @@ -14,7 +14,7 @@ com.google.cloud google-cloud-shared-config - 1.2.1 + 1.2.2 @@ -76,7 +76,7 @@ com.google.cloud google-cloud-bigquery - 2.4.1 + 2.5.0 @@ -107,7 +107,7 @@ org.mockito mockito-core - 4.0.0 + 4.1.0 test @@ -119,13 +119,13 @@ com.google.cloud google-cloud-datacatalog - 1.5.9 + 1.6.0 test com.google.api.grpc proto-google-cloud-datacatalog-v1 - 1.5.9 + 1.6.0 test diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index b94051f03e..81a4d65e81 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -45,7 +45,7 @@ com.google.cloud google-cloud-bigquery - 2.4.0 + 2.4.1 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index f073aaab25..897db4586e 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -44,7 +44,7 @@ com.google.cloud google-cloud-bigquery - 2.4.1 + 2.5.0 diff --git a/versions.txt b/versions.txt index 5e60b03b87..0b56dac06b 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-bigquery:2.4.1:2.4.1 \ No newline at end of file +google-cloud-bigquery:2.5.0:2.5.0 \ No newline at end of file