diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 658dc2f530..b87a7d0110 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -32,7 +32,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: persist-credentials: false @@ -67,6 +67,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@29b1f65c5e92e24fe6b6647da1eaabe529cec70f # v2.3.3 + uses: github/codeql-action/upload-sarif@6c089f53dd51dc3fc7e599c3cb5356453a52ca9e # v2.20.0 with: sarif_file: results.sarif diff --git a/CHANGELOG.md b/CHANGELOG.md index cceeee3b35..80bac768be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [2.27.1](https://github.com/googleapis/java-bigquery/compare/v2.27.0...v2.27.1) (2023-06-13) + + +### Dependencies + +* Update actions/checkout action to v3.5.3 ([#2746](https://github.com/googleapis/java-bigquery/issues/2746)) ([17f8438](https://github.com/googleapis/java-bigquery/commit/17f843880f5633b602de5221c26b830e7e304d2b)) +* Update dependency com.google.api.grpc:proto-google-cloud-bigqueryconnection-v1 to v2.21.0 ([#2741](https://github.com/googleapis/java-bigquery/issues/2741)) ([d665e52](https://github.com/googleapis/java-bigquery/commit/d665e523b2c393c17a734ff4714aeb85f8d61dd7)) +* Update dependency com.google.cloud:google-cloud-datacatalog-bom to v1.25.0 ([#2743](https://github.com/googleapis/java-bigquery/issues/2743)) ([5d38d23](https://github.com/googleapis/java-bigquery/commit/5d38d2375cedd29e35d75881a206cab3fdcdd6a5)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.11.0 ([#2738](https://github.com/googleapis/java-bigquery/issues/2738)) ([3b56445](https://github.com/googleapis/java-bigquery/commit/3b564458eef9df2173c47e26e2399a6a6cad6eee)) +* Update github/codeql-action action to v2.20.0 ([#2751](https://github.com/googleapis/java-bigquery/issues/2751)) ([42ae181](https://github.com/googleapis/java-bigquery/commit/42ae18134b972c1694a7e012d2f51c916e663c83)) +* Update github/codeql-action action to v2.3.6 ([#2712](https://github.com/googleapis/java-bigquery/issues/2712)) ([f043ed6](https://github.com/googleapis/java-bigquery/commit/f043ed61dacf4ea66eedaf0a6faada06057b7d50)) + ## [2.27.0](https://github.com/googleapis/java-bigquery/compare/v2.26.1...v2.27.0) (2023-05-30) diff --git a/README.md b/README.md index d26c0dbcf0..359e63f783 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ See https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google com.google.cloud libraries-bom - 26.15.0 + 26.16.0 pom import @@ -45,7 +45,7 @@ If you are using Maven without the BOM, add this to your dependencies: com.google.cloud google-cloud-bigquery - 2.26.1 + 2.27.0 ``` @@ -53,20 +53,20 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.15.0') +implementation platform('com.google.cloud:libraries-bom:26.16.0') implementation 'com.google.cloud:google-cloud-bigquery' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.26.1' +implementation 'com.google.cloud:google-cloud-bigquery:2.27.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.26.1" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.27.0" ``` @@ -189,6 +189,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-bigquery/tree | Load Json From Gcs Cmek | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/LoadJsonFromGcsCmek.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/LoadJsonFromGcsCmek.java) | | Load Json From Gcs Truncate | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/LoadJsonFromGcsTruncate.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/LoadJsonFromGcsTruncate.java) | | Load Local File | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/LoadLocalFile.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/LoadLocalFile.java) | +| Load Local File In Session | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/LoadLocalFileInSession.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/LoadLocalFileInSession.java) | | Load Orc From Gcs | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/LoadOrcFromGcs.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/LoadOrcFromGcs.java) | | Load Orc From Gcs Truncate | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/LoadOrcFromGcsTruncate.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/LoadOrcFromGcsTruncate.java) | | Load Parquet | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/LoadParquet.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/LoadParquet.java) | @@ -349,7 +350,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-bigquery/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigquery.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.26.1 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.27.0 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 1f8ff6a03c..d7ecf96cfc 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -6,7 +6,7 @@ google-cloud-bigquery-parent com.google.cloud - 2.27.0 + 2.27.1 diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 32757e5029..7c18b2e595 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery - 2.27.0 + 2.27.1 jar BigQuery https://github.com/googleapis/java-bigquery @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquery-parent - 2.27.0 + 2.27.1 google-cloud-bigquery diff --git a/pom.xml b/pom.xml index a7cedc04df..17592c2e18 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquery-parent pom - 2.27.0 + 2.27.1 BigQuery Parent https://github.com/googleapis/java-bigquery @@ -14,7 +14,7 @@ com.google.cloud google-cloud-shared-config - 1.5.5 + 1.5.6 @@ -55,7 +55,7 @@ github google-cloud-bigquery-parent v2-rev20230520-2.0.0 - 3.10.1 + 3.11.0 12.0.0 @@ -73,7 +73,7 @@ com.google.cloud google-cloud-bigquerystorage-bom - 2.37.2 + 2.38.0 pom import @@ -97,7 +97,7 @@ com.google.cloud google-cloud-datacatalog-bom - 1.24.0 + 1.25.0 pom import @@ -111,7 +111,7 @@ com.google.cloud google-cloud-bigquery - 2.27.0 + 2.27.1 @@ -155,19 +155,19 @@ com.google.cloud google-cloud-storage - 2.22.3 + 2.22.4 test com.google.cloud google-cloud-bigqueryconnection - 2.20.0 + 2.21.0 test com.google.api.grpc proto-google-cloud-bigqueryconnection-v1 - 2.20.0 + 2.21.0 test @@ -197,7 +197,7 @@ org.apache.maven.plugins maven-project-info-reports-plugin - 3.4.4 + 3.4.5 diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 241d1b12bf..f608c7f393 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -45,7 +45,7 @@ com.google.cloud google-cloud-bigquery - 2.26.1 + 2.27.0 @@ -63,13 +63,13 @@ com.google.cloud google-cloud-bigtable - 2.23.1 + 2.23.3 test com.google.cloud google-cloud-bigqueryconnection - 2.20.0 + 2.21.0 test diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index d48ceb280b..2a02c3c613 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -39,7 +39,7 @@ com.google.cloud libraries-bom - 26.15.0 + 26.16.0 pom import @@ -111,7 +111,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.1.0 + 3.1.2 **/*IT diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index eb260b372a..d5ecc32715 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -44,7 +44,7 @@ com.google.cloud google-cloud-bigquery - 2.27.0 + 2.27.1 @@ -61,13 +61,13 @@ com.google.cloud google-cloud-bigtable - 2.23.1 + 2.23.3 test com.google.cloud google-cloud-bigqueryconnection - 2.20.0 + 2.21.0 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 274942702d..974421e08e 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -47,7 +47,7 @@ com.google.cloud libraries-bom - 26.15.0 + 26.16.0 pom import @@ -79,13 +79,13 @@ com.google.cloud google-cloud-bigtable - 2.23.1 + 2.23.3 test com.google.cloud google-cloud-bigqueryconnection - 2.20.0 + 2.21.0 test diff --git a/samples/snippets/src/main/java/com/example/bigquery/LoadLocalFileInSession.java b/samples/snippets/src/main/java/com/example/bigquery/LoadLocalFileInSession.java new file mode 100644 index 0000000000..8efef5c405 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/bigquery/LoadLocalFileInSession.java @@ -0,0 +1,163 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.ConnectionProperty; +import com.google.cloud.bigquery.CsvOptions; +import com.google.cloud.bigquery.FormatOptions; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.JobId; +import com.google.cloud.bigquery.JobStatistics.LoadStatistics; +import com.google.cloud.bigquery.TableDataWriteChannel; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.WriteChannelConfiguration; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; + +public class LoadLocalFileInSession { + + public static void main(String[] args) throws IOException, InterruptedException { + // Use _SESSION if the table is a temporary table + String datasetName = "MY_DATASET_NAME"; + String tableName = "MY_TABLE_NAME"; + Path csvPath = FileSystems.getDefault().getPath(".", "my-data.csv"); + String sessionId = + createSessionForLoading(datasetName, tableName, csvPath, FormatOptions.csv()); + loadLocalFileInSession(datasetName, tableName, csvPath, FormatOptions.csv(), sessionId); + } + + // [START bigquery_load_from_file_create_session] + + public static String createSessionForLoading( + String datasetName, String tableName, Path csvPath, CsvOptions formatOptions) + throws IOException, InterruptedException { + LoadStatistics loadStatistics = null; + try { + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + TableId tableId = TableId.of(datasetName, tableName); + + // Enable createSession in the configuration + WriteChannelConfiguration writeChannelConfiguration = + WriteChannelConfiguration.newBuilder(tableId) + .setFormatOptions(formatOptions) + .setCreateSession(true) + .build(); + + // The location and JobName must be specified; other fields can be auto-detected. + String jobName = "jobId_" + UUID.randomUUID().toString(); + JobId jobId = JobId.newBuilder().setLocation("us").setJob(jobName).build(); + + // Imports a local file into a table. + try (TableDataWriteChannel writer = bigquery.writer(jobId, writeChannelConfiguration); + OutputStream stream = Channels.newOutputStream(writer)) { + Files.copy(csvPath, stream); + } + + // Get the Job created by the TableDataWriteChannel and wait for it to complete. + // Then retrieve the session ID + Job job = bigquery.getJob(jobId); + Job completedJob = job.waitFor(); + loadStatistics = completedJob.getStatistics(); + if (completedJob == null) { + System.out.println("Job not executed since it no longer exists."); + return ""; + } else if (completedJob.getStatus().getError() != null) { + System.out.println( + "BigQuery was unable to load local file to the table due to an error: \n" + + job.getStatus().getError()); + return ""; + } + + } catch (BigQueryException e) { + System.out.println("Local file not loaded. \n" + e.toString()); + } + return loadStatistics.getSessionInfo().getSessionId(); + } + + // [END bigquery_load_from_file_create_session] + + // [START bigquery_load_from_file_with_session] + public static void loadLocalFileInSession( + String datasetName, + String tableName, + Path csvPath, + FormatOptions formatOptions, + String sessionId) + throws IOException, InterruptedException { + try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + TableId tableId = TableId.of(datasetName, tableName); + + // Create ConnectionProperty with sessionID + ConnectionProperty sessionConnectionProperty = + ConnectionProperty.newBuilder().setKey("session_id").setValue(sessionId).build(); + + // Set the Connection Property with the SessionID in the Configuration. + WriteChannelConfiguration writeChannelConfiguration = + WriteChannelConfiguration.newBuilder(tableId) + .setFormatOptions(formatOptions) + .setConnectionProperties(ImmutableList.of(sessionConnectionProperty)) + .build(); + + // The location and JobName must be specified; other fields can be auto-detected. + String jobName = "jobId_" + UUID.randomUUID().toString(); + JobId jobId = JobId.newBuilder().setLocation("us").setJob(jobName).build(); + + // Imports a local file into a table. + try (TableDataWriteChannel writer = bigquery.writer(jobId, writeChannelConfiguration); + OutputStream stream = Channels.newOutputStream(writer)) { + Files.copy(csvPath, stream); + } + + // Get the Job created by the TableDataWriteChannel and wait for it to complete. + Job job = bigquery.getJob(jobId); + Job completedJob = job.waitFor(); + if (completedJob == null) { + System.out.println("Job not executed since it no longer exists."); + return; + } else if (completedJob.getStatus().getError() != null) { + System.out.println( + "BigQuery was unable to load local file to the table due to an error: \n" + + job.getStatus().getError()); + return; + } + + // Get output status + LoadStatistics stats = job.getStatistics(); + System.out.printf( + "Successfully loaded to Session %s. \n", stats.getSessionInfo().getSessionId()); + } catch (BigQueryException e) { + System.out.println("Local file not loaded. \n" + e.toString()); + } + } + // [END bigquery_load_from_file_with_session] +} diff --git a/samples/snippets/src/test/java/com/example/bigquery/LoadLocalFileInSessionIT.java b/samples/snippets/src/test/java/com/example/bigquery/LoadLocalFileInSessionIT.java new file mode 100644 index 0000000000..1bf6128784 --- /dev/null +++ b/samples/snippets/src/test/java/com/example/bigquery/LoadLocalFileInSessionIT.java @@ -0,0 +1,97 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertFalse; +import static junit.framework.TestCase.assertNotNull; + +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FormatOptions; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.Schema; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.util.UUID; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class LoadLocalFileInSessionIT { + private final Logger log = Logger.getLogger(this.getClass().getName()); + private String tableName; + private ByteArrayOutputStream bout; + private PrintStream out; + private PrintStream originalPrintStream; + + private static final String BIGQUERY_DATASET_NAME = System.getenv("BIGQUERY_DATASET_NAME"); + + private static void requireEnvVar(String varName) { + assertNotNull( + "Environment variable " + varName + " is required to perform these tests.", + System.getenv(varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnvVar("BIGQUERY_DATASET_NAME"); + } + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + originalPrintStream = System.out; + System.setOut(out); + tableName = "LOADLOCALFILETESTTABLE_" + UUID.randomUUID().toString().substring(0, 8); + Schema schema = + Schema.of( + Field.of("Name", LegacySQLTypeName.STRING), + Field.of("Age", LegacySQLTypeName.NUMERIC), + Field.of("Weight", LegacySQLTypeName.NUMERIC), + Field.of("IsMagic", LegacySQLTypeName.BOOLEAN)); + CreateTable.createTable(BIGQUERY_DATASET_NAME, tableName, schema); + } + + @After + public void tearDown() { + // Clean up + DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName); + // restores print statements in the original method + System.out.flush(); + System.setOut(originalPrintStream); + log.log(Level.INFO, "\n" + bout.toString()); + } + + @Test + public void loadLocalFileInSession() throws IOException, InterruptedException { + Path csvPath = FileSystems.getDefault().getPath("src/test/resources", "bigquery_noheader.csv"); + String sessionId = + LoadLocalFileInSession.createSessionForLoading( + BIGQUERY_DATASET_NAME, tableName, csvPath, FormatOptions.csv()); + assertFalse(sessionId.isEmpty()); + LoadLocalFileInSession.loadLocalFileInSession( + BIGQUERY_DATASET_NAME, tableName, csvPath, FormatOptions.csv(), sessionId); + assertThat(bout.toString()).contains("Successfully loaded to Session"); + } +} diff --git a/versions.txt b/versions.txt index 2427ff3366..b89a2104af 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-bigquery:2.27.0:2.27.0 \ No newline at end of file +google-cloud-bigquery:2.27.1:2.27.1 \ No newline at end of file