From ff2d22bf9081efa9c0df74acb745241c8beb27ed Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Wed, 22 May 2019 22:36:43 -0400 Subject: [PATCH 01/30] samples: Moving Bigtable samples into this repo (#1429) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Setup mvn package and copy in existing samples and tests * Fix style issues and ran tests * Updating pom with current versions * Adding default values for the system propertiesĖ† * surefire instead of failsafe --- .../com/m/examples/bigtable/HelloWorld.java | 187 +++++++++ .../bigtable/InstanceAdminExample.java | 215 ++++++++++ .../examples/bigtable/TableAdminExample.java | 385 ++++++++++++++++++ .../m/examples/bigtable/HelloWorldTest.java | 151 +++++++ .../bigtable/InstanceAdminExampleTest.java | 153 +++++++ .../bigtable/TableAdminExampleTest.java | 227 +++++++++++ 6 files changed, 1318 insertions(+) create mode 100644 samples/snippets/src/main/java/com/m/examples/bigtable/HelloWorld.java create mode 100644 samples/snippets/src/main/java/com/m/examples/bigtable/InstanceAdminExample.java create mode 100644 samples/snippets/src/main/java/com/m/examples/bigtable/TableAdminExample.java create mode 100644 samples/snippets/src/test/java/com/m/examples/bigtable/HelloWorldTest.java create mode 100644 samples/snippets/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java create mode 100644 samples/snippets/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java diff --git a/samples/snippets/src/main/java/com/m/examples/bigtable/HelloWorld.java b/samples/snippets/src/main/java/com/m/examples/bigtable/HelloWorld.java new file mode 100644 index 0000000000..36e1fc08f9 --- /dev/null +++ b/samples/snippets/src/main/java/com/m/examples/bigtable/HelloWorld.java @@ -0,0 +1,187 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package com.m.examples.bigtable; + +// [START bigtable_hw_imports_veneer] +import com.google.api.gax.rpc.NotFoundException; +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import java.io.IOException; + +// [END bigtable_hw_imports_veneer] + +/** + * An example of using Google Cloud Bigtable. + * + *

This example is a very simple "hello world" application, that illustrates how to create a new + * table, write to the table, read the data back, and delete the table. + * + *

+ */ +public class HelloWorld { + + private static final String COLUMN_FAMILY = "cf1"; + private static final String COLUMN_QUALIFIER = "greeting"; + private static final String ROW_KEY_PREFIX = "rowKey"; + private final String tableId; + private final BigtableDataClient dataClient; + private final BigtableTableAdminClient adminClient; + + public static void main(String[] args) throws Exception { + + if (args.length != 2) { + System.out.println("Missing required project id or instance id"); + return; + } + String projectId = args[0]; + String instanceId = args[1]; + + HelloWorld helloWorld = new HelloWorld(projectId, instanceId, "test-table"); + helloWorld.run(); + } + + public HelloWorld(String projectId, String instanceId, String tableId) throws IOException { + this.tableId = tableId; + + // [START bigtable_hw_connect_veneer] + // Creates the settings to configure a bigtable data client. + BigtableDataSettings settings = + BigtableDataSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId).build(); + + // Creates a bigtable data client. + dataClient = BigtableDataClient.create(settings); + + // Creates the settings to configure a bigtable table admin client. + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .build(); + + // Creates a bigtable table admin client. + adminClient = BigtableTableAdminClient.create(adminSettings); + // [END bigtable_hw_connect_veneer] + } + + public void run() throws Exception { + createTable(); + writeToTable(); + readSingleRow(); + readTable(); + deleteTable(); + dataClient.close(); + adminClient.close(); + } + + /** Demonstrates how to create a table. */ + public void createTable() { + // [START bigtable_hw_create_table_veneer] + // Checks if table exists, creates table if does not exist. + if (!adminClient.exists(tableId)) { + System.out.println("Creating table: " + tableId); + CreateTableRequest createTableRequest = + CreateTableRequest.of(tableId).addFamily(COLUMN_FAMILY); + adminClient.createTable(createTableRequest); + System.out.printf("Table %s created successfully%n", tableId); + } + // [END bigtable_hw_create_table_veneer] + } + + /** Demonstrates how to write some rows to a table. */ + public void writeToTable() { + // [START bigtable_hw_write_rows_veneer] + try { + System.out.println("\nWriting some greetings to the table"); + String[] greetings = {"Hello World!", "Hello Bigtable!", "Hello Java!"}; + for (int i = 0; i < greetings.length; i++) { + RowMutation rowMutation = + RowMutation.create(tableId, ROW_KEY_PREFIX + i) + .setCell(COLUMN_FAMILY, COLUMN_QUALIFIER, greetings[i]); + dataClient.mutateRow(rowMutation); + System.out.println(greetings[i]); + } + } catch (NotFoundException e) { + System.err.println("Failed to write to non-existent table: " + e.getMessage()); + } + // [END bigtable_hw_write_rows_veneer] + } + + /** Demonstrates how to read a single row from a table. */ + public void readSingleRow() { + // [START bigtable_hw_get_by_key_veneer] + try { + System.out.println("\nReading a single row by row key"); + Row row = dataClient.readRow(tableId, ROW_KEY_PREFIX + 0); + System.out.println("Row: " + row.getKey().toStringUtf8()); + for (RowCell cell : row.getCells()) { + System.out.printf( + "Family: %s Qualifier: %s Value: %s%n", + cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); + } + } catch (NotFoundException e) { + System.err.println("Failed to read from a non-existent table: " + e.getMessage()); + } + // [END bigtable_hw_get_by_key_veneer] + } + + /** Demonstrates how to read an entire table. */ + public void readTable() { + // [START bigtable_hw_scan_all_veneer] + try { + System.out.println("\nReading the entire table"); + Query query = Query.create(tableId); + ServerStream rowStream = dataClient.readRows(query); + for (Row r : rowStream) { + System.out.println("Row Key: " + r.getKey().toStringUtf8()); + for (RowCell cell : r.getCells()) { + System.out.printf( + "Family: %s Qualifier: %s Value: %s%n", + cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); + } + } + } catch (NotFoundException e) { + System.err.println("Failed to read a non-existent table: " + e.getMessage()); + } + // [END bigtable_hw_scan_all_veneer] + } + + /** Demonstrates how to delete a table. */ + public void deleteTable() { + // [START bigtable_hw_delete_table_veneer] + System.out.println("\nDeleting table: " + tableId); + try { + adminClient.deleteTable(tableId); + System.out.printf("Table %s deleted successfully%n", tableId); + } catch (NotFoundException e) { + System.err.println("Failed to delete a non-existent table: " + e.getMessage()); + } + // [END bigtable_hw_delete_table_veneer] + } +} diff --git a/samples/snippets/src/main/java/com/m/examples/bigtable/InstanceAdminExample.java b/samples/snippets/src/main/java/com/m/examples/bigtable/InstanceAdminExample.java new file mode 100644 index 0000000000..3b317d594c --- /dev/null +++ b/samples/snippets/src/main/java/com/m/examples/bigtable/InstanceAdminExample.java @@ -0,0 +1,215 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package com.m.examples.bigtable; + +import com.google.api.gax.rpc.AlreadyExistsException; +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.Cluster; +import com.google.cloud.bigtable.admin.v2.models.CreateClusterRequest; +import com.google.cloud.bigtable.admin.v2.models.CreateInstanceRequest; +import com.google.cloud.bigtable.admin.v2.models.Instance; +import com.google.cloud.bigtable.admin.v2.models.PartialListInstancesException; +import com.google.cloud.bigtable.admin.v2.models.StorageType; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * An example of using Google Cloud Bigtable. + * + *

This example demonstrates the usage of BigtableInstanceAdminClient to create, configure, and + * delete Cloud Bigtable Instances and Clusters. + * + *

+ */ +public class InstanceAdminExample { + + private static final String CLUSTER = "cluster"; + private final String clusterId; + private final String instanceId; + private final BigtableInstanceAdminClient adminClient; + + public static void main(String[] args) throws IOException { + + if (args.length != 1) { + System.out.println("Missing required project id"); + return; + } + String projectId = args[0]; + + InstanceAdminExample instanceAdmin = + new InstanceAdminExample(projectId, "ssd-instance", "ssd-cluster"); + instanceAdmin.run(); + } + + public InstanceAdminExample(String projectId, String instanceId, String clusterId) + throws IOException { + this.instanceId = instanceId; + this.clusterId = clusterId; + + // Creates the settings to configure a bigtable instance admin client. + BigtableInstanceAdminSettings instanceAdminSettings = + BigtableInstanceAdminSettings.newBuilder().setProjectId(projectId).build(); + + // Creates a bigtable instance admin client. + adminClient = BigtableInstanceAdminClient.create(instanceAdminSettings); + } + + public void run() { + createProdInstance(); + listInstances(); + getInstance(); + listClusters(); + addCluster(); + deleteCluster(); + deleteInstance(); + adminClient.close(); + } + + /** Demonstrates how to create a Production instance within a provided project. */ + public void createProdInstance() { + // Checks if instance exists, creates instance if does not exists. + if (!adminClient.exists(instanceId)) { + System.out.println("Instance does not exist, creating a PRODUCTION instance"); + // [START bigtable_create_prod_instance] + // Creates a Production Instance with the ID "ssd-instance", + // cluster id "ssd-cluster", 3 nodes and location "us-central1-f". + CreateInstanceRequest createInstanceRequest = + CreateInstanceRequest.of(instanceId) + .addCluster(clusterId, "us-central1-f", 3, StorageType.SSD) + .setType(Instance.Type.PRODUCTION) + .addLabel("department", "accounting"); + // Creates a production instance with the given request. + try { + Instance instance = adminClient.createInstance(createInstanceRequest); + System.out.printf("PRODUCTION type instance %s created successfully%n", instance.getId()); + } catch (Exception e) { + System.err.println("Failed to create instance: " + e.getMessage()); + throw e; + } + // [END bigtable_create_prod_instance] + } + } + + /** Demonstrates how to list all instances within a project. */ + public void listInstances() { + System.out.println("\nListing Instances"); + // [START bigtable_list_instances] + try { + List instances = adminClient.listInstances(); + for (Instance instance : instances) { + System.out.println(instance.getId()); + } + } catch (PartialListInstancesException e) { + System.err.println("Failed to list instances: " + e.getMessage()); + System.err.println("The following zones are unavailable: " + e.getUnavailableZones()); + System.err.println("But the following instances are reachable: " + e.getInstances()); + } + // [END bigtable_list_instances] + } + + /** Demonstrates how to get an instance. */ + public Instance getInstance() { + System.out.println("\nGet Instance"); + // [START bigtable_get_instance] + Instance instance = null; + try { + instance = adminClient.getInstance(instanceId); + System.out.println("Instance ID: " + instance.getId()); + System.out.println("Display Name: " + instance.getDisplayName()); + System.out.print("Labels: "); + Map labels = instance.getLabels(); + for (String key : labels.keySet()) { + System.out.printf("%s - %s", key, labels.get(key)); + } + System.out.println("\nState: " + instance.getState()); + System.out.println("Type: " + instance.getType()); + } catch (NotFoundException e) { + System.err.println("Failed to get non-existent instance: " + e.getMessage()); + } + // [END bigtable_get_instance] + return instance; + } + + /** Demonstrates how to list clusters within an instance. */ + public void listClusters() { + System.out.println("\nListing Clusters"); + // [START bigtable_get_clusters] + try { + List clusters = adminClient.listClusters(instanceId); + for (Cluster cluster : clusters) { + System.out.println(cluster.getId()); + } + } catch (NotFoundException e) { + System.err.println("Failed to list clusters from a non-existent instance: " + e.getMessage()); + } + // [END bigtable_get_clusters] + } + + /** Demonstrates how to delete an instance. */ + public void deleteInstance() { + System.out.println("\nDeleting Instance"); + // [START bigtable_delete_instance] + try { + adminClient.deleteInstance(instanceId); + System.out.println("Instance deleted: " + instanceId); + } catch (NotFoundException e) { + System.err.println("Failed to delete non-existent instance: " + e.getMessage()); + } + // [END bigtable_delete_instance] + } + + /** Demonstrates how to add a cluster to an instance. */ + public void addCluster() { + System.out.printf("%nAdding cluster: %s to instance: %s%n", CLUSTER, instanceId); + // [START bigtable_create_cluster] + try { + adminClient.createCluster( + CreateClusterRequest.of(instanceId, CLUSTER) + .setZone("us-central1-c") + .setServeNodes(3) + .setStorageType(StorageType.SSD)); + System.out.printf("Cluster: %s created successfully%n", CLUSTER); + } catch (AlreadyExistsException e) { + System.err.println("Failed to add cluster, already exists: " + e.getMessage()); + } + // [END bigtable_create_cluster] + } + + /** Demonstrates how to delete a cluster from an instance. */ + public void deleteCluster() { + System.out.printf("%nDeleting cluster: %s from instance: %s%n", CLUSTER, instanceId); + // [START bigtable_delete_cluster] + try { + adminClient.deleteCluster(instanceId, CLUSTER); + System.out.printf("Cluster: %s deleted successfully%n", CLUSTER); + } catch (NotFoundException e) { + System.err.println("Failed to delete a non-existent cluster: " + e.getMessage()); + } + // [END bigtable_delete_cluster] + } +} diff --git a/samples/snippets/src/main/java/com/m/examples/bigtable/TableAdminExample.java b/samples/snippets/src/main/java/com/m/examples/bigtable/TableAdminExample.java new file mode 100644 index 0000000000..56c47e19a3 --- /dev/null +++ b/samples/snippets/src/main/java/com/m/examples/bigtable/TableAdminExample.java @@ -0,0 +1,385 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package com.m.examples.bigtable; + +import static com.google.cloud.bigtable.admin.v2.models.GCRules.GCRULES; + +import com.google.api.gax.rpc.AlreadyExistsException; +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.models.GCRules.DurationRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.IntersectionRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.UnionRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.VersionRule; +import com.google.cloud.bigtable.admin.v2.models.ModifyColumnFamiliesRequest; +import com.google.cloud.bigtable.admin.v2.models.Table; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * An example of using Google Cloud Bigtable. + * + *

This example demonstrates the usage of BigtableTableAdminClient to create, configure and + * delete a Cloud Bigtable table. + * + *

    + *
  • creates table + *
  • lists all tables + *
  • gets table metadata + *
  • creates DurationRule + *
  • creates VersionRule + *
  • creates UnionRule + *
  • creates IntersectionRule + *
  • creates nested rule + *
  • lists column families + *
  • modifies column family rule + *
  • prints modified column family + *
  • deletes column family + *
  • deletes table + *
+ */ +public class TableAdminExample { + + private static final String COLUMN_FAMILY_1 = "cf1"; + private static final String COLUMN_FAMILY_2 = "cf2"; + private static final String COLUMN_FAMILY_3 = "cf3"; + private static final String COLUMN_FAMILY_4 = "cf4"; + private static final String COLUMN_FAMILY_5 = "cf5"; + private final String tableId; + private final BigtableTableAdminClient adminClient; + + public static void main(String[] args) throws IOException { + + if (args.length != 2) { + System.out.println("Missing required project id or instance id"); + return; + } + String projectId = args[0]; + String instanceId = args[1]; + + TableAdminExample tableAdmin = new TableAdminExample(projectId, instanceId, "test-table"); + tableAdmin.run(); + } + + public TableAdminExample(String projectId, String instanceId, String tableId) throws IOException { + this.tableId = tableId; + + // Creates the settings to configure a bigtable table admin client. + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .build(); + + // Creates a bigtable table admin client. + adminClient = BigtableTableAdminClient.create(adminSettings); + } + + public void run() { + createTable(); + listAllTables(); + getTableMeta(); + addFamilyWithMaxAgeRule(); + addFamilyWithMaxVersionsRule(); + addFamilyWithUnionRule(); + addFamilyWithIntersectionRule(); + addFamilyWithNestedRule(); + listColumnFamilies(); + modifyColumnFamilyRule(); + printModifiedColumnFamily(); + deleteColumnFamily(); + deleteTable(); + adminClient.close(); + } + + /** Demonstrates how to create a table with the specified configuration. */ + public void createTable() { + // [START bigtable_create_table] + // Checks if table exists, creates table if does not exist. + if (!adminClient.exists(tableId)) { + System.out.println("Table does not exist, creating table: " + tableId); + CreateTableRequest createTableRequest = CreateTableRequest.of(tableId).addFamily("cf"); + Table table = adminClient.createTable(createTableRequest); + System.out.printf("Table: %s created successfully%n", table.getId()); + } + // [END bigtable_create_table] + } + + /** Demonstrates how to list all tables within an instance. */ + public void listAllTables() { + System.out.println("\nListing tables in current instance"); + // [START bigtable_list_tables] + // Lists tables in the current instance. + try { + List tableIds = adminClient.listTables(); + for (String tableId : tableIds) { + System.out.println(tableId); + } + } catch (NotFoundException e) { + System.err.println("Failed to list tables from a non-existent instance: " + e.getMessage()); + } + // [END bigtable_list_tables] + } + + /** Demonstrates how to get a table's metadata. */ + public void getTableMeta() { + System.out.println("\nPrinting table metadata"); + // [START bigtable_get_table_metadata] + // Gets table metadata, and applies a view to the table fields. + try { + Table table = adminClient.getTable(tableId); + System.out.println("Table: " + table.getId()); + Collection columnFamilies = table.getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + System.out.printf( + "Column family: %s%nGC Rule: %s%n", + columnFamily.getId(), columnFamily.getGCRule().toString()); + } + } catch (NotFoundException e) { + System.err.println( + "Failed to retrieve table metadata for a non-existent table: " + e.getMessage()); + } + // [END bigtable_get_table_metadata] + } + + /** Demonstrates how to create a new instance of the DurationRule. */ + public void addFamilyWithMaxAgeRule() { + System.out.printf("%nCreating column family %s with max age GC rule%n", COLUMN_FAMILY_1); + // [START bigtable_create_family_gc_max_age] + // Creates a column family with GC policy : maximum age + // where age = current time minus cell timestamp + + // Defines the GC rule to retain data with max age of 5 days. + DurationRule maxAgeRule = GCRULES.maxAge(5, TimeUnit.DAYS); + + // Creates column family with given GC rule. + try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_1, maxAgeRule); + adminClient.modifyFamilies(columnFamiliesRequest); + System.out.println("Created column family: " + COLUMN_FAMILY_1); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_max_age] + } + + /** Demonstrates how to create a new instance of the VersionRule. */ + public void addFamilyWithMaxVersionsRule() { + System.out.printf("%nCreating column family %s with max versions GC rule%n", COLUMN_FAMILY_2); + // [START bigtable_create_family_gc_max_versions] + // Creates a column family with GC policy : most recent N versions + // where 1 = most recent version + + // Defines the GC policy to retain only the most recent 2 versions. + VersionRule versionRule = GCRULES.maxVersions(2); + + // Creates column family with given GC rule. + try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_2, versionRule); + adminClient.modifyFamilies(columnFamiliesRequest); + System.out.println("Created column family: " + COLUMN_FAMILY_2); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_max_versions] + } + + /** Demonstrates how to create a new instance of the UnionRule. */ + public void addFamilyWithUnionRule() { + System.out.printf("%nCreating column family %s with union GC rule%n", COLUMN_FAMILY_3); + // [START bigtable_create_family_gc_union] + // Creates a column family with GC policy to drop data that matches at least one condition. + + // Defines a list of GC rules to drop cells older than 5 days OR not the most recent + // version. + UnionRule unionRule = + GCRULES.union().rule(GCRULES.maxAge(5, TimeUnit.DAYS)).rule(GCRULES.maxVersions(1)); + + // Creates column family with given GC rule. + try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_3, unionRule); + adminClient.modifyFamilies(columnFamiliesRequest); + System.out.println("Created column family: " + COLUMN_FAMILY_3); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_union] + } + + /** Demonstrates how to create a new instance of the IntersectionRule. */ + public void addFamilyWithIntersectionRule() { + System.out.printf("%nCreating column family %s with intersection GC rule%n", COLUMN_FAMILY_4); + // [START bigtable_create_family_gc_intersection] + // Creates a column family with GC policy to drop data that matches all conditions. + + // Defines a GC rule to drop cells older than 5 days AND older than the most recent 2 versions. + DurationRule maxAgeRule = GCRULES.maxAge(5, TimeUnit.DAYS); + VersionRule versionRule = GCRULES.maxVersions(2); + IntersectionRule intersectionRule = GCRULES.intersection().rule(maxAgeRule).rule(versionRule); + + // Creates column family with given GC rule. + try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_4, intersectionRule); + adminClient.modifyFamilies(columnFamiliesRequest); + System.out.println("Created column family: " + COLUMN_FAMILY_4); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_intersection] + } + + /** Demonstrates how to create a nested rule using the IntersectionRule and UnionRule. */ + public void addFamilyWithNestedRule() { + System.out.printf("%nCreating column family %s with a nested GC rule%n", COLUMN_FAMILY_5); + // [START bigtable_create_family_gc_nested] + // Creates a nested GC rule: + // Drop cells that are either older than the 10 recent versions + // OR + // Drop cells that are older than a month AND older than the 2 recent versions + VersionRule versionRule1 = GCRULES.maxVersions(10); + VersionRule versionRule2 = GCRULES.maxVersions(2); + DurationRule maxAgeRule = GCRULES.maxAge(30, TimeUnit.DAYS); + IntersectionRule intersectionRule = GCRULES.intersection().rule(maxAgeRule).rule(versionRule2); + UnionRule unionRule = GCRULES.union().rule(intersectionRule).rule(versionRule1); + + // Creates column family with given GC rule. + try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to add a family + ModifyColumnFamiliesRequest columnFamiliesRequest = + ModifyColumnFamiliesRequest.of(tableId).addFamily(COLUMN_FAMILY_5, unionRule); + adminClient.modifyFamilies(columnFamiliesRequest); + System.out.println("Created column family: " + COLUMN_FAMILY_5); + } catch (AlreadyExistsException e) { + System.err.println( + "Failed to create column family with rule, already exists: " + e.getMessage()); + } + // [END bigtable_create_family_gc_nested] + } + + /** Demonstrates how to list a table's column families. */ + public void listColumnFamilies() { + System.out.println("\nPrinting ID and GC Rule for all column families"); + // [START bigtable_list_column_families] + // Lists all families in the table with GC rules. + try { + Table table = adminClient.getTable(tableId); + Collection columnFamilies = table.getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + System.out.printf( + "Column family: %s%nGC Rule: %s%n", + columnFamily.getId(), columnFamily.getGCRule().toString()); + } + } catch (NotFoundException e) { + System.err.println( + "Failed to list column families from a non-existent table: " + e.getMessage()); + } + // [END bigtable_list_column_families] + } + + /** Demonstrates how to modify a column family's rule. */ + public void modifyColumnFamilyRule() { + System.out.printf("%nUpdating column family %s GC rule%n", COLUMN_FAMILY_1); + // [START bigtable_update_gc_rule] + // Updates the column family metadata to update the GC rule. + // Updates a column family GC rule. + VersionRule versionRule = GCRULES.maxVersions(1); + try { + // ModifyColumnFamiliesRequest can be used both for adding and modifying families, here it is + // being used to modify a family + // Updates column family with given GC rule. + ModifyColumnFamiliesRequest updateRequest = + ModifyColumnFamiliesRequest.of(tableId).updateFamily(COLUMN_FAMILY_1, versionRule); + adminClient.modifyFamilies(updateRequest); + System.out.printf("Column family %s GC rule updated%n", COLUMN_FAMILY_1); + } catch (NotFoundException e) { + System.err.println("Failed to modify a non-existent column family: " + e.getMessage()); + } + // [END bigtable_update_gc_rule] + } + + /** Demonstrates how to print the modified column family. */ + public void printModifiedColumnFamily() { + System.out.printf("%nPrint updated GC rule for column family %s%n", COLUMN_FAMILY_1); + // [START bigtable_family_get_gc_rule] + try { + Table table = adminClient.getTable(tableId); + Collection columnFamilies = table.getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + if (columnFamily.getId().equals(COLUMN_FAMILY_1)) { + System.out.printf( + "Column family: %s%nGC Rule: %s%n", + columnFamily.getId(), columnFamily.getGCRule().toString()); + } + } + } catch (NotFoundException e) { + System.err.println("Failed to print a non-existent column family: " + e.getMessage()); + } + // [END bigtable_family_get_gc_rule] + } + + /** Demonstrates how to delete a column family. */ + public void deleteColumnFamily() { + System.out.println("\nDelete column family: " + COLUMN_FAMILY_2); + // [START bigtable_delete_family] + // Deletes a column family. + try { + ModifyColumnFamiliesRequest deleted = + ModifyColumnFamiliesRequest.of(tableId).dropFamily(COLUMN_FAMILY_2); + adminClient.modifyFamilies(deleted); + System.out.printf("Column family %s deleted successfully%n", COLUMN_FAMILY_2); + } catch (NotFoundException e) { + System.err.println("Failed to delete a non-existent column family: " + e.getMessage()); + } + // [END bigtable_delete_family] + } + + /** Demonstrates how to delete a table. */ + public void deleteTable() { + // [START bigtable_delete_table] + // Deletes the entire table. + System.out.println("\nDelete table: " + tableId); + try { + adminClient.deleteTable(tableId); + System.out.printf("Table: %s deleted successfully%n", tableId); + } catch (NotFoundException e) { + System.err.println("Failed to delete a non-existent table: " + e.getMessage()); + } + // [END bigtable_delete_table] + } +} diff --git a/samples/snippets/src/test/java/com/m/examples/bigtable/HelloWorldTest.java b/samples/snippets/src/test/java/com/m/examples/bigtable/HelloWorldTest.java new file mode 100644 index 0000000000..e19a1a17e6 --- /dev/null +++ b/samples/snippets/src/test/java/com/m/examples/bigtable/HelloWorldTest.java @@ -0,0 +1,151 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package com.m.examples.bigtable; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.Row; +import java.io.IOException; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Integration tests for {@link HelloWorld} */ +public class HelloWorldTest { + + private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; + private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String TABLE_PREFIX = "table"; + private static String tableId; + private static BigtableDataClient dataClient; + private static BigtableTableAdminClient adminClient; + private static String projectId; + private static String instanceId; + private HelloWorld helloWorld; + + @BeforeClass + public static void beforeClass() throws IOException { + projectId = System.getProperty(PROJECT_PROPERTY_NAME); + instanceId = System.getProperty(INSTANCE_PROPERTY_NAME); + if (projectId == null || instanceId == null) { + dataClient = null; + adminClient = null; + return; + } + BigtableDataSettings settings = + BigtableDataSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId).build(); + dataClient = BigtableDataClient.create(settings); + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .build(); + adminClient = BigtableTableAdminClient.create(adminSettings); + } + + @AfterClass + public static void afterClass() throws Exception { + garbageCollect(); + dataClient.close(); + adminClient.close(); + } + + @Before + public void setup() throws IOException { + if (adminClient == null || dataClient == null) { + throw new AssumptionViolatedException( + PROJECT_PROPERTY_NAME + + " or " + + INSTANCE_PROPERTY_NAME + + " property is not set, skipping integration tests."); + } + tableId = generateTableId(); + helloWorld = new HelloWorld(projectId, instanceId, tableId); + adminClient.createTable(CreateTableRequest.of(tableId).addFamily("cf1")); + } + + @After + public void after() { + if (adminClient.exists(tableId)) { + adminClient.deleteTable(tableId); + } + } + + @Test + public void testCreateAndDeleteTable() throws IOException { + // Creates a table. + String testTable = generateTableId(); + HelloWorld testHelloWorld = new HelloWorld(projectId, instanceId, testTable); + testHelloWorld.createTable(); + assertTrue(adminClient.exists(testTable)); + + // Deletes a table. + testHelloWorld.deleteTable(); + assertTrue(!adminClient.exists(testTable)); + } + + @Test + public void testWriteToTable() { + // Writes to a table. + helloWorld.writeToTable(); + Row row = dataClient.readRow(tableId, "rowKey0"); + assertNotNull(row); + } + + // TODO: add test for helloWorld.readSingleRow() + // TODO: add test for helloWorld.readTable() + + @Test + public void testRunDoesNotFail() throws Exception { + helloWorld.run(); + } + + private String generateTableId() { + return String.format( + "%s-%016x-%x", TABLE_PREFIX, System.currentTimeMillis(), new Random().nextLong()); + } + + private static void garbageCollect() { + Pattern timestampPattern = Pattern.compile(TABLE_PREFIX + "-([0-9a-f]+)-([0-9a-f]+)"); + for (String tableId : adminClient.listTables()) { + Matcher matcher = timestampPattern.matcher(tableId); + if (!matcher.matches()) { + continue; + } + String timestampStr = matcher.group(1); + long timestamp = Long.parseLong(timestampStr, 16); + if (System.currentTimeMillis() - timestamp < TimeUnit.MINUTES.toMillis(15)) { + continue; + } + System.out.println("\nGarbage collecting orphaned table: " + tableId); + adminClient.deleteTable(tableId); + } + } +} diff --git a/samples/snippets/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java b/samples/snippets/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java new file mode 100644 index 0000000000..8a92449ba7 --- /dev/null +++ b/samples/snippets/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java @@ -0,0 +1,153 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package com.m.examples.bigtable; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.Cluster; +import com.google.cloud.bigtable.admin.v2.models.CreateInstanceRequest; +import com.google.cloud.bigtable.admin.v2.models.Instance; +import com.google.cloud.bigtable.admin.v2.models.Instance.Type; +import com.google.cloud.bigtable.admin.v2.models.StorageType; +import java.io.IOException; +import java.util.Random; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Integration tests for {@link InstanceAdminExample} */ +public class InstanceAdminExampleTest { + + private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; + private static final String ID_PREFIX = "instanceadmin"; + private static final String CLUSTER = "cluster"; + private static String projectId; + private static BigtableInstanceAdminClient adminClient; + private String clusterId; + private String instanceId; + private InstanceAdminExample instanceAdmin; + + @BeforeClass + public static void beforeClass() throws IOException { + projectId = System.getProperty(PROJECT_PROPERTY_NAME); + if (projectId == null) { + adminClient = null; + return; + } + BigtableInstanceAdminSettings instanceAdminSettings = + BigtableInstanceAdminSettings.newBuilder().setProjectId(projectId).build(); + adminClient = BigtableInstanceAdminClient.create(instanceAdminSettings); + } + + @AfterClass + public static void afterClass() { + garbageCollect(); + adminClient.close(); + } + + @Before + public void setup() throws IOException { + if (adminClient == null) { + throw new AssumptionViolatedException( + PROJECT_PROPERTY_NAME + " property is not set, skipping integration tests."); + } + instanceId = generateId(); + clusterId = generateId(); + instanceAdmin = new InstanceAdminExample(projectId, instanceId, clusterId); + adminClient.createInstance( + CreateInstanceRequest.of(instanceId) + .addCluster(clusterId, "us-central1-f", 3, StorageType.SSD) + .setType(Type.PRODUCTION) + .addLabel("example", "instance_admin")); + } + + @After + public void after() { + if (adminClient.exists(instanceId)) { + adminClient.deleteInstance(instanceId); + } + } + + @Test + public void testCreateAndDeleteInstance() throws IOException { + // Creates an instance. + String testInstance = generateId(); + String testCluster = generateId(); + InstanceAdminExample testInstanceAdmin = + new InstanceAdminExample(projectId, testInstance, testCluster); + testInstanceAdmin.createProdInstance(); + assertTrue(adminClient.exists(testInstance)); + + // Deletes an instance. + testInstanceAdmin.deleteInstance(); + assertFalse(adminClient.exists(testInstance)); + } + + @Test + public void testGetInstance() { + // Gets an instance. + Instance instance = instanceAdmin.getInstance(); + assertNotNull(instance); + } + + @Test(expected = NotFoundException.class) + public void testAddAndDeleteCluster() { + // Adds a cluster. + instanceAdmin.addCluster(); + Cluster cluster = adminClient.getCluster(instanceId, CLUSTER); + assertNotNull(cluster); + + // Deletes a cluster. + instanceAdmin.deleteCluster(); + adminClient.getCluster(instanceId, CLUSTER); + } + + // TODO: add test for instanceAdmin.listInstances() + // TODO: and test for instanceAdmin.listClusters() + + @Test + public void testRunDoesNotFail() { + instanceAdmin.run(); + } + + private static String generateId() { + return String.format("%s-%x", ID_PREFIX, new Random().nextInt()); + } + + private static void garbageCollect() { + Pattern timestampPattern = Pattern.compile(ID_PREFIX + "-([0-9a-f]+)"); + System.out.println(); + for (Instance instance : adminClient.listInstances()) { + Matcher matcher = timestampPattern.matcher(instance.getId()); + if (!matcher.matches()) { + continue; + } + System.out.println("Garbage collecting orphaned table: " + instance); + adminClient.deleteInstance(instance.getId()); + } + } +} diff --git a/samples/snippets/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java b/samples/snippets/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java new file mode 100644 index 0000000000..d2163bf691 --- /dev/null +++ b/samples/snippets/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java @@ -0,0 +1,227 @@ +/* + * Copyright 2019 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package com.m.examples.bigtable; + +import static com.google.cloud.bigtable.admin.v2.models.GCRules.GCRULES; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.models.GCRules.DurationRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.GCRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.IntersectionRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.UnionRule; +import com.google.cloud.bigtable.admin.v2.models.GCRules.VersionRule; +import java.io.IOException; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Integration tests for {@link TableAdminExample} */ +public class TableAdminExampleTest { + + private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; + private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String TABLE_PREFIX = "table"; + private static BigtableTableAdminClient adminClient; + private static String instanceId; + private static String projectId; + private String tableId; + private TableAdminExample tableAdmin; + + @BeforeClass + public static void beforeClass() throws IOException { + projectId = System.getProperty(PROJECT_PROPERTY_NAME); + instanceId = System.getProperty(INSTANCE_PROPERTY_NAME); + if (projectId == null || instanceId == null) { + adminClient = null; + return; + } + BigtableTableAdminSettings adminSettings = + BigtableTableAdminSettings.newBuilder() + .setInstanceId(instanceId) + .setProjectId(projectId) + .build(); + adminClient = BigtableTableAdminClient.create(adminSettings); + } + + @AfterClass + public static void afterClass() { + garbageCollect(); + adminClient.close(); + } + + @Before + public void setup() throws IOException { + if (adminClient == null) { + throw new AssumptionViolatedException( + INSTANCE_PROPERTY_NAME + + " or " + + PROJECT_PROPERTY_NAME + + " property is not set, skipping integration tests."); + } + tableId = generateTableId(); + tableAdmin = new TableAdminExample(projectId, instanceId, tableId); + adminClient.createTable(CreateTableRequest.of(tableId).addFamily("cf")); + } + + @After + public void after() { + if (adminClient.exists(tableId)) { + adminClient.deleteTable(tableId); + } + } + + @Test + public void testCreateAndDeleteTable() throws IOException { + // Creates a table. + String testTable = generateTableId(); + TableAdminExample testTableAdmin = new TableAdminExample(projectId, instanceId, testTable); + testTableAdmin.createTable(); + assertTrue(adminClient.exists(testTable)); + + // Deletes a table. + testTableAdmin.deleteTable(); + assertFalse(adminClient.exists(testTable)); + } + + @Test + public void testCreateMaxAgeRuleAndModifyAndPrintColumnFamily() { + // Max age rule + tableAdmin.addFamilyWithMaxAgeRule(); + DurationRule maxAgeCondition = GCRULES.maxAge(5, TimeUnit.DAYS); + boolean maxAgeRule = ruleCheck(maxAgeCondition); + assertTrue(maxAgeRule); + + // Modifies cf1. + tableAdmin.modifyColumnFamilyRule(); + GCRule modifiedRule = GCRULES.maxVersions(1); + boolean maxVersionRule = ruleCheck(modifiedRule); + assertTrue(maxVersionRule); + } + + @Test + public void testCreateMaxVersionsRuleAndDeleteColumnFamily() { + // Max versions rule + tableAdmin.addFamilyWithMaxVersionsRule(); + VersionRule maxVersionCondition = GCRULES.maxVersions(2); + boolean maxVersionRule = ruleCheck(maxVersionCondition); + assertTrue(maxVersionRule); + + // Deletes cf2. + tableAdmin.deleteColumnFamily(); + boolean found = true; + List columnFamilies = adminClient.getTable(tableId).getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + if (columnFamily.equals("cf2")) { + found = false; + break; + } + } + assertTrue(found); + } + + @Test + public void testCreateUnionRule() { + // Union rule + tableAdmin.addFamilyWithUnionRule(); + DurationRule maxAgeRule = GCRULES.maxAge(5, TimeUnit.DAYS); + VersionRule versionRule = GCRULES.maxVersions(1); + UnionRule unionCondition = GCRULES.union().rule(maxAgeRule).rule(versionRule); + boolean unionRule = ruleCheck(unionCondition); + assertTrue(unionRule); + } + + @Test + public void testCreateIntersectionRule() { + // Intersection rule + tableAdmin.addFamilyWithIntersectionRule(); + DurationRule maxAgeRule = GCRULES.maxAge(5, TimeUnit.DAYS); + VersionRule versionRule = GCRULES.maxVersions(2); + IntersectionRule intersectionCondition = + GCRULES.intersection().rule(maxAgeRule).rule(versionRule); + boolean intersectionRule = ruleCheck(intersectionCondition); + assertTrue(intersectionRule); + } + + @Test + public void testCreateNestedRule() { + // Nested rule + tableAdmin.addFamilyWithNestedRule(); + VersionRule versionRule = GCRULES.maxVersions(10); + DurationRule maxAgeRule = GCRULES.maxAge(30, TimeUnit.DAYS); + VersionRule versionRule2 = GCRULES.maxVersions(2); + IntersectionRule intersectionRule = GCRULES.intersection().rule(maxAgeRule).rule(versionRule2); + UnionRule nestedCondition = GCRULES.union().rule(intersectionRule).rule(versionRule); + boolean nestedRule = ruleCheck(nestedCondition); + assertTrue(nestedRule); + } + + @Test + public void testRunDoesNotFail() { + tableAdmin.run(); + } + + // TODO: add test for tableAdmin.listAllTables() + // TODO: add test for tableAdmin.getTableMeta() + // TODO: add test for tableAdmin.listColumnFamilies() + + private boolean ruleCheck(GCRule condition) { + boolean found = false; + List columnFamilies = adminClient.getTable(tableId).getColumnFamilies(); + for (ColumnFamily columnFamily : columnFamilies) { + if (columnFamily.getGCRule().equals(condition)) { + found = true; + break; + } + } + return found; + } + + private String generateTableId() { + return String.format( + "%s-%016x-%x", TABLE_PREFIX, System.currentTimeMillis(), new Random().nextLong()); + } + + private static void garbageCollect() { + Pattern timestampPattern = Pattern.compile(TABLE_PREFIX + "-([0-9a-f]+)-([0-9a-f]+)"); + for (String tableId : adminClient.listTables()) { + Matcher matcher = timestampPattern.matcher(tableId); + if (!matcher.matches()) { + continue; + } + String timestampStr = matcher.group(1); + long timestamp = Long.parseLong(timestampStr, 16); + if (System.currentTimeMillis() - timestamp < TimeUnit.MINUTES.toMillis(10)) { + continue; + } + System.out.println("\nGarbage collecting orphaned table: " + tableId); + adminClient.deleteTable(tableId); + } + } +} From 925ee4f599604f4be0e41bd1bfc6460405fee6e0 Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Mon, 3 Jun 2019 15:48:31 -0400 Subject: [PATCH 02/30] samples: Writing quickstart + test for Bigtable (#1431) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Setup mvn package and copy in existing samples and tests * Fix style issues and ran tests * Writing quickstart + test for Bigtable * Updating pom with current versions * Adding default values for the system propertiesĖ† * surefire instead of failsafe * setup table for testing using cbt tool, retriggering test * Removing unnecessary code from quickstart test that was causing failures * cleaning up quickstart * Changing test variables to use GOOGLE_CLOUD_PROJECT and the instance environment variable --- .../com/m/examples/bigtable/Quickstart.java | 59 +++++++++++++ .../m/examples/bigtable/HelloWorldTest.java | 21 +++-- .../bigtable/InstanceAdminExampleTest.java | 14 ++-- .../m/examples/bigtable/QuickstartTest.java | 82 +++++++++++++++++++ .../bigtable/TableAdminExampleTest.java | 22 ++--- 5 files changed, 170 insertions(+), 28 deletions(-) create mode 100644 samples/snippets/src/main/java/com/m/examples/bigtable/Quickstart.java create mode 100644 samples/snippets/src/test/java/com/m/examples/bigtable/QuickstartTest.java diff --git a/samples/snippets/src/main/java/com/m/examples/bigtable/Quickstart.java b/samples/snippets/src/main/java/com/m/examples/bigtable/Quickstart.java new file mode 100644 index 0000000000..4d52061428 --- /dev/null +++ b/samples/snippets/src/main/java/com/m/examples/bigtable/Quickstart.java @@ -0,0 +1,59 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.m.examples.bigtable; + +// START [bigtable_quickstart_veneer] + +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; + +public class Quickstart { + + public static void quickstart(String projectId, String instanceId, String tableId) { + // String projectId = "my-project-id"; + // String instanceId = "my-instance-id"; + // String tableId = "my-table-id"; + + BigtableDataSettings settings = + BigtableDataSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId).build(); + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (BigtableDataClient dataClient = BigtableDataClient.create(settings)) { + try { + System.out.println("\nReading a single row by row key"); + Row row = dataClient.readRow(tableId, "r1"); + System.out.println("Row: " + row.getKey().toStringUtf8()); + for (RowCell cell : row.getCells()) { + System.out.printf( + "Family: %s Qualifier: %s Value: %s%n", + cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); + } + } catch (NotFoundException e) { + System.err.println("Failed to read from a non-existent table: " + e.getMessage()); + } + + } catch (Exception e) { + System.out.println("Error during functionName: \n" + e.toString()); + } + } +} +// END [bigtable_quickstart_veneer] diff --git a/samples/snippets/src/test/java/com/m/examples/bigtable/HelloWorldTest.java b/samples/snippets/src/test/java/com/m/examples/bigtable/HelloWorldTest.java index e19a1a17e6..522a99e014 100644 --- a/samples/snippets/src/test/java/com/m/examples/bigtable/HelloWorldTest.java +++ b/samples/snippets/src/test/java/com/m/examples/bigtable/HelloWorldTest.java @@ -40,8 +40,7 @@ /** Integration tests for {@link HelloWorld} */ public class HelloWorldTest { - private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; - private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String INSTANCE_PROPERTY_NAME = "BIGTABLE_TESTING_INSTANCE"; private static final String TABLE_PREFIX = "table"; private static String tableId; private static BigtableDataClient dataClient; @@ -50,10 +49,17 @@ public class HelloWorldTest { private static String instanceId; private HelloWorld helloWorld; + private static String requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + "System property '%s' is required to perform these tests.".format(varName)); + return System.getenv(varName); + } + @BeforeClass public static void beforeClass() throws IOException { - projectId = System.getProperty(PROJECT_PROPERTY_NAME); - instanceId = System.getProperty(INSTANCE_PROPERTY_NAME); + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); + instanceId = requireEnv(INSTANCE_PROPERTY_NAME); if (projectId == null || instanceId == null) { dataClient = null; adminClient = null; @@ -79,13 +85,6 @@ public static void afterClass() throws Exception { @Before public void setup() throws IOException { - if (adminClient == null || dataClient == null) { - throw new AssumptionViolatedException( - PROJECT_PROPERTY_NAME - + " or " - + INSTANCE_PROPERTY_NAME - + " property is not set, skipping integration tests."); - } tableId = generateTableId(); helloWorld = new HelloWorld(projectId, instanceId, tableId); adminClient.createTable(CreateTableRequest.of(tableId).addFamily("cf1")); diff --git a/samples/snippets/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java b/samples/snippets/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java index 8a92449ba7..b7bcee7d3a 100644 --- a/samples/snippets/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java +++ b/samples/snippets/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java @@ -42,7 +42,6 @@ /** Integration tests for {@link InstanceAdminExample} */ public class InstanceAdminExampleTest { - private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; private static final String ID_PREFIX = "instanceadmin"; private static final String CLUSTER = "cluster"; private static String projectId; @@ -51,9 +50,16 @@ public class InstanceAdminExampleTest { private String instanceId; private InstanceAdminExample instanceAdmin; + private static String requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + "System property '%s' is required to perform these tests.".format(varName)); + return System.getenv(varName); + } + @BeforeClass public static void beforeClass() throws IOException { - projectId = System.getProperty(PROJECT_PROPERTY_NAME); + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); if (projectId == null) { adminClient = null; return; @@ -71,10 +77,6 @@ public static void afterClass() { @Before public void setup() throws IOException { - if (adminClient == null) { - throw new AssumptionViolatedException( - PROJECT_PROPERTY_NAME + " property is not set, skipping integration tests."); - } instanceId = generateId(); clusterId = generateId(); instanceAdmin = new InstanceAdminExample(projectId, instanceId, clusterId); diff --git a/samples/snippets/src/test/java/com/m/examples/bigtable/QuickstartTest.java b/samples/snippets/src/test/java/com/m/examples/bigtable/QuickstartTest.java new file mode 100644 index 0000000000..929506859c --- /dev/null +++ b/samples/snippets/src/test/java/com/m/examples/bigtable/QuickstartTest.java @@ -0,0 +1,82 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.m.examples.bigtable; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Integration tests for {@link Quickstart} + */ +public class QuickstartTest { + + private static final String INSTANCE_PROPERTY_NAME = "BIGTABLE_TESTING_INSTANCE"; + private static final String TABLE_ID = "quickstart-table"; + private static String projectId; + private static String instanceId; + private ByteArrayOutputStream bout; + + private static String requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + "System property '%s' is required to perform these tests.".format(varName)); + return System.getenv(varName); + } + + @BeforeClass + public static void beforeClass() { + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); + instanceId = requireEnv(INSTANCE_PROPERTY_NAME); + } + + @Before + public void setupStream() { + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + } + + @Test + public void testQuickstart() { + try { + Quickstart.quickstart(projectId, instanceId, TABLE_ID); + } catch (Exception e) { + System.out.println("Failed to run quickstart."); + System.out.println(e); + } + + String output = bout.toString(); + assertThat(output, CoreMatchers.containsString("Reading a single row by row key")); + assertThat(output, CoreMatchers.containsString("Row: r1")); + assertThat( + output, CoreMatchers.containsString("Family: cf1 Qualifier: c1 Value: quickstart")); + } +} diff --git a/samples/snippets/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java b/samples/snippets/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java index d2163bf691..51ed37cf05 100644 --- a/samples/snippets/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java +++ b/samples/snippets/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java @@ -18,6 +18,7 @@ import static com.google.cloud.bigtable.admin.v2.models.GCRules.GCRULES; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; @@ -45,8 +46,7 @@ /** Integration tests for {@link TableAdminExample} */ public class TableAdminExampleTest { - private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; - private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + private static final String INSTANCE_PROPERTY_NAME = "BIGTABLE_TESTING_INSTANCE"; private static final String TABLE_PREFIX = "table"; private static BigtableTableAdminClient adminClient; private static String instanceId; @@ -54,10 +54,17 @@ public class TableAdminExampleTest { private String tableId; private TableAdminExample tableAdmin; + private static String requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + "System property '%s' is required to perform these tests.".format(varName)); + return System.getenv(varName); + } + @BeforeClass public static void beforeClass() throws IOException { - projectId = System.getProperty(PROJECT_PROPERTY_NAME); - instanceId = System.getProperty(INSTANCE_PROPERTY_NAME); + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); + instanceId = requireEnv(INSTANCE_PROPERTY_NAME); if (projectId == null || instanceId == null) { adminClient = null; return; @@ -78,13 +85,6 @@ public static void afterClass() { @Before public void setup() throws IOException { - if (adminClient == null) { - throw new AssumptionViolatedException( - INSTANCE_PROPERTY_NAME - + " or " - + PROJECT_PROPERTY_NAME - + " property is not set, skipping integration tests."); - } tableId = generateTableId(); tableAdmin = new TableAdminExample(projectId, instanceId, tableId); adminClient.createTable(CreateTableRequest.of(tableId).addFamily("cf")); From 8ad06313b0efe99bf64d8147cd60da1473204b16 Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Thu, 6 Jun 2019 18:54:27 -0400 Subject: [PATCH 03/30] samples: Fixing region tag in Bigtable quickstart (#1448) --- .../src/main/java/com/m/examples/bigtable/Quickstart.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/src/main/java/com/m/examples/bigtable/Quickstart.java b/samples/snippets/src/main/java/com/m/examples/bigtable/Quickstart.java index 4d52061428..14359d316b 100644 --- a/samples/snippets/src/main/java/com/m/examples/bigtable/Quickstart.java +++ b/samples/snippets/src/main/java/com/m/examples/bigtable/Quickstart.java @@ -16,7 +16,7 @@ package com.m.examples.bigtable; -// START [bigtable_quickstart_veneer] +// [START bigtable_quickstart_veneer] import com.google.api.gax.rpc.NotFoundException; import com.google.cloud.bigtable.data.v2.BigtableDataClient; @@ -56,4 +56,4 @@ public static void quickstart(String projectId, String instanceId, String tableI } } } -// END [bigtable_quickstart_veneer] +// [END bigtable_quickstart_veneer] From 7b856b9ed3114980700110b8a9b8c92a57f1d51e Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Tue, 18 Jun 2019 14:28:54 -0400 Subject: [PATCH 04/30] samples: Hbase client for Bigtable write samples (#1445) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Setup mvn package and copy in existing samples and tests * Fix style issues and ran tests * Writing quickstart + test for Bigtable * Updating pom with current versions * Adding default values for the system propertiesĖ† * surefire instead of failsafe * setup table for testing using cbt tool, retriggering test * Removing unnecessary code from quickstart test that was causing failures * cleaning up quickstart * Changing test variables to use GOOGLE_CLOUD_PROJECT and the instance environment variable * HBase write samples and test cleanups * rename to folder to snippets * Add copyright to pom.xml * Cleaning up some of the existing files * POM touchups --- .../com/m/examples/bigtable/Quickstart.java | 23 ++++++++----------- .../m/examples/bigtable/HelloWorldTest.java | 11 +++------ .../bigtable/InstanceAdminExampleTest.java | 6 +---- .../m/examples/bigtable/QuickstartTest.java | 23 ++++--------------- .../bigtable/TableAdminExampleTest.java | 10 +++----- 5 files changed, 22 insertions(+), 51 deletions(-) diff --git a/samples/snippets/src/main/java/com/m/examples/bigtable/Quickstart.java b/samples/snippets/src/main/java/com/m/examples/bigtable/Quickstart.java index 14359d316b..7ca481f1bd 100644 --- a/samples/snippets/src/main/java/com/m/examples/bigtable/Quickstart.java +++ b/samples/snippets/src/main/java/com/m/examples/bigtable/Quickstart.java @@ -38,21 +38,18 @@ public static void quickstart(String projectId, String instanceId, String tableI // once, and can be reused for multiple requests. After completing all of your requests, call // the "close" method on the client to safely clean up any remaining background resources. try (BigtableDataClient dataClient = BigtableDataClient.create(settings)) { - try { - System.out.println("\nReading a single row by row key"); - Row row = dataClient.readRow(tableId, "r1"); - System.out.println("Row: " + row.getKey().toStringUtf8()); - for (RowCell cell : row.getCells()) { - System.out.printf( - "Family: %s Qualifier: %s Value: %s%n", - cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); - } - } catch (NotFoundException e) { - System.err.println("Failed to read from a non-existent table: " + e.getMessage()); + System.out.println("\nReading a single row by row key"); + Row row = dataClient.readRow(tableId, "r1"); + System.out.println("Row: " + row.getKey().toStringUtf8()); + for (RowCell cell : row.getCells()) { + System.out.printf( + "Family: %s Qualifier: %s Value: %s%n", + cell.getFamily(), cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8()); } - + } catch (NotFoundException e) { + System.err.println("Failed to read from a non-existent table: " + e.getMessage()); } catch (Exception e) { - System.out.println("Error during functionName: \n" + e.toString()); + System.out.println("Error during quickstart: \n" + e.toString()); } } } diff --git a/samples/snippets/src/test/java/com/m/examples/bigtable/HelloWorldTest.java b/samples/snippets/src/test/java/com/m/examples/bigtable/HelloWorldTest.java index 522a99e014..985b37bbd7 100644 --- a/samples/snippets/src/test/java/com/m/examples/bigtable/HelloWorldTest.java +++ b/samples/snippets/src/test/java/com/m/examples/bigtable/HelloWorldTest.java @@ -40,7 +40,7 @@ /** Integration tests for {@link HelloWorld} */ public class HelloWorldTest { - private static final String INSTANCE_PROPERTY_NAME = "BIGTABLE_TESTING_INSTANCE"; + private static final String INSTANCE_ENV = "BIGTABLE_TESTING_INSTANCE"; private static final String TABLE_PREFIX = "table"; private static String tableId; private static BigtableDataClient dataClient; @@ -52,19 +52,14 @@ public class HelloWorldTest { private static String requireEnv(String varName) { assertNotNull( System.getenv(varName), - "System property '%s' is required to perform these tests.".format(varName)); + "Environment variable '%s' is required to perform these tests.".format(varName)); return System.getenv(varName); } @BeforeClass public static void beforeClass() throws IOException { projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); - instanceId = requireEnv(INSTANCE_PROPERTY_NAME); - if (projectId == null || instanceId == null) { - dataClient = null; - adminClient = null; - return; - } + instanceId = requireEnv(INSTANCE_ENV); BigtableDataSettings settings = BigtableDataSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId).build(); dataClient = BigtableDataClient.create(settings); diff --git a/samples/snippets/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java b/samples/snippets/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java index b7bcee7d3a..cf12dfffbe 100644 --- a/samples/snippets/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java +++ b/samples/snippets/src/test/java/com/m/examples/bigtable/InstanceAdminExampleTest.java @@ -53,17 +53,13 @@ public class InstanceAdminExampleTest { private static String requireEnv(String varName) { assertNotNull( System.getenv(varName), - "System property '%s' is required to perform these tests.".format(varName)); + "Environment variable '%s' is required to perform these tests.".format(varName)); return System.getenv(varName); } @BeforeClass public static void beforeClass() throws IOException { projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); - if (projectId == null) { - adminClient = null; - return; - } BigtableInstanceAdminSettings instanceAdminSettings = BigtableInstanceAdminSettings.newBuilder().setProjectId(projectId).build(); adminClient = BigtableInstanceAdminClient.create(instanceAdminSettings); diff --git a/samples/snippets/src/test/java/com/m/examples/bigtable/QuickstartTest.java b/samples/snippets/src/test/java/com/m/examples/bigtable/QuickstartTest.java index 929506859c..d648d05483 100644 --- a/samples/snippets/src/test/java/com/m/examples/bigtable/QuickstartTest.java +++ b/samples/snippets/src/test/java/com/m/examples/bigtable/QuickstartTest.java @@ -19,17 +19,9 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; -import com.google.cloud.bigtable.data.v2.BigtableDataClient; -import com.google.cloud.bigtable.data.v2.BigtableDataSettings; import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.io.PrintStream; import org.hamcrest.CoreMatchers; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.AssumptionViolatedException; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -39,7 +31,7 @@ */ public class QuickstartTest { - private static final String INSTANCE_PROPERTY_NAME = "BIGTABLE_TESTING_INSTANCE"; + private static final String INSTANCE_ENV = "BIGTABLE_TESTING_INSTANCE"; private static final String TABLE_ID = "quickstart-table"; private static String projectId; private static String instanceId; @@ -48,30 +40,25 @@ public class QuickstartTest { private static String requireEnv(String varName) { assertNotNull( System.getenv(varName), - "System property '%s' is required to perform these tests.".format(varName)); + "Environment variable '%s' is required to perform these tests.".format(varName)); return System.getenv(varName); } @BeforeClass public static void beforeClass() { projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); - instanceId = requireEnv(INSTANCE_PROPERTY_NAME); + instanceId = requireEnv(INSTANCE_ENV); } @Before - public void setupStream() { + public void setUp() { bout = new ByteArrayOutputStream(); System.setOut(new PrintStream(bout)); } @Test public void testQuickstart() { - try { - Quickstart.quickstart(projectId, instanceId, TABLE_ID); - } catch (Exception e) { - System.out.println("Failed to run quickstart."); - System.out.println(e); - } + Quickstart.quickstart(projectId, instanceId, TABLE_ID); String output = bout.toString(); assertThat(output, CoreMatchers.containsString("Reading a single row by row key")); diff --git a/samples/snippets/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java b/samples/snippets/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java index 51ed37cf05..887c54a27a 100644 --- a/samples/snippets/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java +++ b/samples/snippets/src/test/java/com/m/examples/bigtable/TableAdminExampleTest.java @@ -46,7 +46,7 @@ /** Integration tests for {@link TableAdminExample} */ public class TableAdminExampleTest { - private static final String INSTANCE_PROPERTY_NAME = "BIGTABLE_TESTING_INSTANCE"; + private static final String INSTANCE_ENV = "BIGTABLE_TESTING_INSTANCE"; private static final String TABLE_PREFIX = "table"; private static BigtableTableAdminClient adminClient; private static String instanceId; @@ -57,18 +57,14 @@ public class TableAdminExampleTest { private static String requireEnv(String varName) { assertNotNull( System.getenv(varName), - "System property '%s' is required to perform these tests.".format(varName)); + "Environment variable '%s' is required to perform these tests.".format(varName)); return System.getenv(varName); } @BeforeClass public static void beforeClass() throws IOException { projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); - instanceId = requireEnv(INSTANCE_PROPERTY_NAME); - if (projectId == null || instanceId == null) { - adminClient = null; - return; - } + instanceId = requireEnv(INSTANCE_ENV); BigtableTableAdminSettings adminSettings = BigtableTableAdminSettings.newBuilder() .setInstanceId(instanceId) From abb08617e2719fc1d58df2eb0ea647aaf224ccdd Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Thu, 18 Jul 2019 17:12:22 -0400 Subject: [PATCH 05/30] samples: Bigtable Write samples (#1513) * Bigtable Writes samples for Java Client * Adding increment comment * Correcting bytes written in previous tests --- .../java/com/example/bigtable/WriteBatch.java | 68 ++++++++++ .../example/bigtable/WriteConditionally.java | 65 ++++++++++ .../com/example/bigtable/WriteIncrement.java | 51 ++++++++ .../com/example/bigtable/WriteSimple.java | 62 ++++++++++ .../java/com/example/bigtable/WritesTest.java | 116 ++++++++++++++++++ 5 files changed, 362 insertions(+) create mode 100644 samples/snippets/src/main/java/com/example/bigtable/WriteBatch.java create mode 100644 samples/snippets/src/main/java/com/example/bigtable/WriteConditionally.java create mode 100644 samples/snippets/src/main/java/com/example/bigtable/WriteIncrement.java create mode 100644 samples/snippets/src/main/java/com/example/bigtable/WriteSimple.java create mode 100644 samples/snippets/src/test/java/com/example/bigtable/WritesTest.java diff --git a/samples/snippets/src/main/java/com/example/bigtable/WriteBatch.java b/samples/snippets/src/main/java/com/example/bigtable/WriteBatch.java new file mode 100644 index 0000000000..9b4e58a505 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/bigtable/WriteBatch.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigtable; + +// [START bigtable_writes_batch] + +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.BulkMutation; +import com.google.cloud.bigtable.data.v2.models.Mutation; +import com.google.protobuf.ByteString; + +public class WriteBatch { + private static final String COLUMN_FAMILY_NAME = "stats_summary"; + + public static void writeBatch(String projectId, String instanceId, String tableId) { + // String projectId = "my-project-id"; + // String instanceId = "my-instance-id"; + // String tableId = "mobile-time-series"; + + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + long timestamp = System.currentTimeMillis() * 1000; + ByteString one = ByteString.copyFrom(new byte[] {0, 0, 0, 0, 0, 0, 0, 1}); + + BulkMutation bulkMutation = + BulkMutation.create(tableId) + .add( + "tablet#a0b81f74#20190501", + Mutation.create() + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_wifi".getBytes()), + timestamp, + one) + .setCell(COLUMN_FAMILY_NAME, "os_build", timestamp, "12155.0.0-rc1")) + .add( + "tablet#a0b81f74#20190502", + Mutation.create() + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_wifi".getBytes()), + timestamp, + one) + .setCell(COLUMN_FAMILY_NAME, "os_build", timestamp, "12155.0.0-rc6")); + + dataClient.bulkMutateRows(bulkMutation); + + System.out.print("Successfully wrote 2 rows"); + } catch (Exception e) { + System.out.println("Error during WriteBatch: \n" + e.toString()); + } + } +} + +// [END bigtable_writes_batch] diff --git a/samples/snippets/src/main/java/com/example/bigtable/WriteConditionally.java b/samples/snippets/src/main/java/com/example/bigtable/WriteConditionally.java new file mode 100644 index 0000000000..7cc8fd2244 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/bigtable/WriteConditionally.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigtable; + +// [START bigtable_writes_conditional] + +import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS; + +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation; +import com.google.cloud.bigtable.data.v2.models.Filters.Filter; +import com.google.cloud.bigtable.data.v2.models.Mutation; + +public class WriteConditionally { + private static final String COLUMN_FAMILY_NAME = "stats_summary"; + + public static void writeConditionally(String projectId, String instanceId, String tableId) { + // String projectId = "my-project-id"; + // String instanceId = "my-instance-id"; + // String tableId = "mobile-time-series"; + + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + long timestamp = System.currentTimeMillis() * 1000; + + String rowKey = "phone#4c410523#20190501"; + + Mutation mutation = + Mutation.create().setCell(COLUMN_FAMILY_NAME, "os_name", timestamp, "android"); + + Filter filter = + FILTERS + .chain() + .filter(FILTERS.family().exactMatch(COLUMN_FAMILY_NAME)) + .filter(FILTERS.qualifier().exactMatch("os_build")) + .filter(FILTERS.value().regex("PQ2A\\..*")); + + ConditionalRowMutation conditionalRowMutation = + ConditionalRowMutation.create(tableId, rowKey).condition(filter).then(mutation); + + boolean success = dataClient.checkAndMutateRow(conditionalRowMutation); + + System.out.printf("Successfully updated row's os_name: %b", success); + + } catch (Exception e) { + System.out.println("Error during WriteConditionally: \n" + e.toString()); + e.printStackTrace(); + } + } +} + +// [END bigtable_writes_conditional] diff --git a/samples/snippets/src/main/java/com/example/bigtable/WriteIncrement.java b/samples/snippets/src/main/java/com/example/bigtable/WriteIncrement.java new file mode 100644 index 0000000000..ec6d756b62 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/bigtable/WriteIncrement.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigtable; + +// [START bigtable_writes_increment] + +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.ReadModifyWriteRow; +import com.google.cloud.bigtable.data.v2.models.Row; +import java.nio.charset.Charset; + +public class WriteIncrement { + private static final String COLUMN_FAMILY_NAME = "stats_summary"; + + public static void writeIncrement(String projectId, String instanceId, String tableId) { + // String projectId = "my-project-id"; + // String instanceId = "my-instance-id"; + // String tableId = "mobile-time-series"; + + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + // Get an existing row that has a cell with an incrementable value. A value can be incremented + // if it is encoded as a 64-bit big-endian signed integer. + String rowKey = "phone#4c410523#20190501"; + ReadModifyWriteRow mutation = + ReadModifyWriteRow.create(tableId, rowKey) + .increment(COLUMN_FAMILY_NAME, "connected_cell", -1); + Row success = dataClient.readModifyWriteRow(mutation); + + System.out.printf( + "Successfully updated row %s", success.getKey().toString(Charset.defaultCharset())); + } catch (Exception e) { + System.out.println("Error during WriteIncrement: \n" + e.toString()); + } + } +} + +// [END bigtable_writes_increment] diff --git a/samples/snippets/src/main/java/com/example/bigtable/WriteSimple.java b/samples/snippets/src/main/java/com/example/bigtable/WriteSimple.java new file mode 100644 index 0000000000..5067198921 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/bigtable/WriteSimple.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigtable; + +// [START bigtable_writes_simple] + +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.protobuf.ByteString; + +public class WriteSimple { + private static final String COLUMN_FAMILY_NAME = "stats_summary"; + + public static void writeSimple(String projectId, String instanceId, String tableId) { + // String projectId = "my-project-id"; + // String instanceId = "my-instance-id"; + // String tableId = "mobile-time-series"; + + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + long timestamp = System.currentTimeMillis() * 1000; + + String rowKey = "phone#4c410523#20190501"; + ByteString one = ByteString.copyFrom(new byte[] {0, 0, 0, 0, 0, 0, 0, 1}); + + RowMutation rowMutation = + RowMutation.create(tableId, rowKey) + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_cell".getBytes()), + timestamp, + one) + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_wifi".getBytes()), + timestamp, + one) + .setCell(COLUMN_FAMILY_NAME, "os_build", timestamp, "PQ2A.190405.003"); + + dataClient.mutateRow(rowMutation); + System.out.printf("Successfully wrote row %s", rowKey); + + } catch (Exception e) { + System.out.println("Error during WriteSimple: \n" + e.toString()); + } + } +} + +// [END bigtable_writes_simple] diff --git a/samples/snippets/src/test/java/com/example/bigtable/WritesTest.java b/samples/snippets/src/test/java/com/example/bigtable/WritesTest.java new file mode 100644 index 0000000000..afe70c4027 --- /dev/null +++ b/samples/snippets/src/test/java/com/example/bigtable/WritesTest.java @@ -0,0 +1,116 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigtable; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.UUID; +import org.hamcrest.CoreMatchers; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class WritesTest { + + private static final String INSTANCE_ENV = "BIGTABLE_TESTING_INSTANCE"; + private static final String TABLE_ID = + "mobile-time-series-" + UUID.randomUUID().toString().substring(0, 20); + private static final String COLUMN_FAMILY_NAME = "stats_summary"; + + private static String projectId; + private static String instanceId; + private ByteArrayOutputStream bout; + + private static String requireEnv(String varName) { + assertNotNull( + System.getenv(varName), + "Environment variable '%s' is required to perform these tests.".format(varName)); + return System.getenv(varName); + } + + @BeforeClass + public static void beforeClass() { + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); + instanceId = requireEnv(INSTANCE_ENV); + try (BigtableTableAdminClient adminClient = + BigtableTableAdminClient.create(projectId, instanceId)) { + CreateTableRequest createTableRequest = + CreateTableRequest.of(TABLE_ID).addFamily(COLUMN_FAMILY_NAME); + adminClient.createTable(createTableRequest); + } catch (Exception e) { + System.out.println("Error during beforeClass: \n" + e.toString()); + } + } + + @Before + public void setupStream() { + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + } + + @AfterClass + public static void afterClass() { + try (BigtableTableAdminClient adminClient = + BigtableTableAdminClient.create(projectId, instanceId)) { + adminClient.deleteTable(TABLE_ID); + } catch (Exception e) { + System.out.println("Error during afterClass: \n" + e.toString()); + } + } + + @Test + public void test1_WriteSimple() { + WriteSimple.writeSimple(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output, CoreMatchers.containsString("Successfully wrote row")); + } + + @Test + public void test2_WriteBatch() { + WriteBatch.writeBatch(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output, CoreMatchers.containsString("Successfully wrote 2 rows")); + } + + @Test + public void test3_WriteConditionally() { + WriteConditionally.writeConditionally(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output, CoreMatchers.containsString("Successfully updated row's os_name: true")); + } + + @Test + public void test4_WriteIncrement() { + WriteIncrement.writeIncrement(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat( + output, CoreMatchers.containsString("Successfully updated row phone#4c410523#20190501")); + } +} From cb4738b53a1e4b48dfaacc49bad4d19f356cf6ca Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Tue, 3 Dec 2019 14:46:46 -0500 Subject: [PATCH 06/30] samples: Read and Filter Snippets (#1747) * WIP for Bigtable Read and Filter snippets * Remaining Filter snippets WIP * All tests working, improved interleave label and condition, removed sink * Changing time to java.time Adding comments about filtering Handling exceptions Reformatting Reads print output other cleanups * Updating function declarations * Cleanup requireEnv function. --- .../java/com/example/bigtable/Filters.java | 459 +++++++++++ .../main/java/com/example/bigtable/Reads.java | 272 +++++++ .../example/bigtable/WriteConditionally.java | 4 +- .../com/example/bigtable/WriteIncrement.java | 4 +- .../com/example/bigtable/WriteSimple.java | 6 +- .../com/example/bigtable/FiltersTest.java | 745 ++++++++++++++++++ .../java/com/example/bigtable/ReadsTest.java | 343 ++++++++ .../java/com/example/bigtable/WritesTest.java | 8 +- 8 files changed, 1829 insertions(+), 12 deletions(-) create mode 100644 samples/snippets/src/main/java/com/example/bigtable/Filters.java create mode 100644 samples/snippets/src/main/java/com/example/bigtable/Reads.java create mode 100644 samples/snippets/src/test/java/com/example/bigtable/FiltersTest.java create mode 100644 samples/snippets/src/test/java/com/example/bigtable/ReadsTest.java diff --git a/samples/snippets/src/main/java/com/example/bigtable/Filters.java b/samples/snippets/src/main/java/com/example/bigtable/Filters.java new file mode 100644 index 0000000000..93367eeace --- /dev/null +++ b/samples/snippets/src/main/java/com/example/bigtable/Filters.java @@ -0,0 +1,459 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigtable; + +import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS; +// [START bigtable_filters_limit_row_sample] +// [START bigtable_filters_limit_row_regex] +// [START bigtable_filters_limit_cells_per_col] +// [START bigtable_filters_limit_cells_per_row] +// [START bigtable_filters_limit_cells_per_row_offset] +// [START bigtable_filters_limit_col_family_regex] +// [START bigtable_filters_limit_col_qualifier_regex] +// [START bigtable_filters_limit_col_range] +// [START bigtable_filters_limit_value_range] +// [START bigtable_filters_limit_value_regex] +// [START bigtable_filters_limit_timestamp_range] +// [START bigtable_filters_limit_block_all] +// [START bigtable_filters_limit_pass_all] +// [START bigtable_filters_modify_strip_value] +// [START bigtable_filters_modify_apply_label] +// [START bigtable_filters_composing_chain] +// [START bigtable_filters_composing_interleave] +// [START bigtable_filters_composing_condition] +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.Filters.Filter; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +public class Filters { + // [END bigtable_filters_limit_row_sample] + // [END bigtable_filters_limit_row_regex] + // [END bigtable_filters_limit_cells_per_col] + // [END bigtable_filters_limit_cells_per_row] + // [END bigtable_filters_limit_cells_per_row_offset] + // [END bigtable_filters_limit_col_family_regex] + // [END bigtable_filters_limit_col_qualifier_regex] + // [END bigtable_filters_limit_col_range] + // [END bigtable_filters_limit_value_range] + // [END bigtable_filters_limit_value_regex] + // [END bigtable_filters_limit_timestamp_range] + // [END bigtable_filters_limit_block_all] + // [END bigtable_filters_limit_pass_all] + // [END bigtable_filters_modify_strip_value] + // [END bigtable_filters_modify_apply_label] + // [END bigtable_filters_composing_chain] + // [END bigtable_filters_composing_interleave] + // [END bigtable_filters_composing_condition] + + // [START bigtable_filters_limit_row_sample] + public static void filterLimitRowSample() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitRowSample(projectId, instanceId, tableId); + } + + public static void filterLimitRowSample(String projectId, String instanceId, String tableId) { + // A filter that matches cells from a row with probability .5 + Filter filter = FILTERS.key().sample(.5); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_row_sample] + + // [START bigtable_filters_limit_row_regex] + public static void filterLimitRowRegex() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitRowRegex(projectId, instanceId, tableId); + } + + public static void filterLimitRowRegex(String projectId, String instanceId, String tableId) { + // A filter that matches cells from rows whose keys satisfy the given regex + Filter filter = FILTERS.key().regex(".*#20190501$"); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_row_regex] + + // [START bigtable_filters_limit_cells_per_col] + public static void filterLimitCellsPerCol() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitCellsPerCol(projectId, instanceId, tableId); + } + + public static void filterLimitCellsPerCol(String projectId, String instanceId, String tableId) { + // A filter that matches only the most recent 2 cells within each column + Filter filter = FILTERS.limit().cellsPerColumn(2); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_cells_per_col] + + // [START bigtable_filters_limit_cells_per_row] + public static void filterLimitCellsPerRow() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitCellsPerRow(projectId, instanceId, tableId); + } + + public static void filterLimitCellsPerRow(String projectId, String instanceId, String tableId) { + // A filter that matches the first 2 cells of each row + Filter filter = FILTERS.limit().cellsPerRow(2); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_cells_per_row] + + // [START bigtable_filters_limit_cells_per_row_offset] + public static void filterLimitCellsPerRowOffset() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitCellsPerRowOffset(projectId, instanceId, tableId); + } + + public static void filterLimitCellsPerRowOffset( + String projectId, String instanceId, String tableId) { + // A filter that skips the first 2 cells per row + Filter filter = FILTERS.offset().cellsPerRow(2); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_cells_per_row_offset] + + // [START bigtable_filters_limit_col_family_regex] + public static void filterLimitColFamilyRegex() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitColFamilyRegex(projectId, instanceId, tableId); + } + + public static void filterLimitColFamilyRegex( + String projectId, String instanceId, String tableId) { + // A filter that matches cells whose column family satisfies the given regex + Filter filter = FILTERS.family().regex("stats_.*$"); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_col_family_regex] + + // [START bigtable_filters_limit_col_qualifier_regex] + public static void filterLimitColQualifierRegex() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitColQualifierRegex(projectId, instanceId, tableId); + } + + public static void filterLimitColQualifierRegex( + String projectId, String instanceId, String tableId) { + // A filter that matches cells whose column qualifier satisfies the given regex + Filter filter = FILTERS.qualifier().regex("connected_.*$"); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_col_qualifier_regex] + + // [START bigtable_filters_limit_col_range] + public static void filterLimitColRange() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitColRange(projectId, instanceId, tableId); + } + + public static void filterLimitColRange(String projectId, String instanceId, String tableId) { + // A filter that matches cells whose column qualifiers are between data_plan_01gb and + // data_plan_10gb in the column family cell_plan + Filter filter = + FILTERS + .qualifier() + .rangeWithinFamily("cell_plan") + .startClosed("data_plan_01gb") + .endOpen("data_plan_10gb"); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_col_range] + + // [START bigtable_filters_limit_value_range] + public static void filterLimitValueRange() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitValueRange(projectId, instanceId, tableId); + } + + public static void filterLimitValueRange(String projectId, String instanceId, String tableId) { + // A filter that matches cells whose values are between the given values + Filter filter = FILTERS.value().range().startClosed("PQ2A.190405").endClosed("PQ2A.190406"); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_value_range] + + // [START bigtable_filters_limit_value_regex] + public static void filterLimitValueRegex() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitValueRegex(projectId, instanceId, tableId); + } + + public static void filterLimitValueRegex(String projectId, String instanceId, String tableId) { + // A filter that matches cells whose value satisfies the given regex + Filter filter = FILTERS.value().regex("PQ2A.*$"); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_value_regex] + + // [START bigtable_filters_limit_timestamp_range] + public static void filterLimitTimestampRange() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitTimestampRange(projectId, instanceId, tableId); + } + + public static void filterLimitTimestampRange( + String projectId, String instanceId, String tableId) { + // Get a time representing one hour ago + long timestamp = Instant.now().minus(1, ChronoUnit.HOURS).toEpochMilli() * 1000; + + // A filter that matches cells whose timestamp is from an hour ago or earlier + Filter filter = FILTERS.timestamp().range().startClosed(0L).endOpen(timestamp); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_timestamp_range] + + // [START bigtable_filters_limit_block_all] + public static void filterLimitBlockAll() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitBlockAll(projectId, instanceId, tableId); + } + + public static void filterLimitBlockAll(String projectId, String instanceId, String tableId) { + // A filter that does not match any cells + Filter filter = FILTERS.block(); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_block_all] + + // [START bigtable_filters_limit_pass_all] + public static void filterLimitPassAll() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterLimitPassAll(projectId, instanceId, tableId); + } + + public static void filterLimitPassAll(String projectId, String instanceId, String tableId) { + // A filter that matches all cells + Filter filter = FILTERS.pass(); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_limit_pass_all] + + // [START bigtable_filters_modify_strip_value] + public static void filterModifyStripValue() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterModifyStripValue(projectId, instanceId, tableId); + } + + public static void filterModifyStripValue(String projectId, String instanceId, String tableId) { + // A filter that replaces the outputted cell value with the empty string + Filter filter = FILTERS.value().strip(); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_modify_strip_value] + + // [START bigtable_filters_modify_apply_label] + public static void filterModifyApplyLabel() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterModifyApplyLabel(projectId, instanceId, tableId); + } + + public static void filterModifyApplyLabel(String projectId, String instanceId, String tableId) { + // A filter that applies the given label to the outputted cell + Filter filter = FILTERS.label("labelled"); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_modify_apply_label] + + // [START bigtable_filters_composing_chain] + public static void filterComposingChain() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterComposingChain(projectId, instanceId, tableId); + } + + public static void filterComposingChain(String projectId, String instanceId, String tableId) { + // A filter that selects one cell per column AND within the column family cell_plan + Filter filter = + FILTERS + .chain() + .filter(FILTERS.limit().cellsPerColumn(1)) + .filter(FILTERS.family().exactMatch("cell_plan")); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_composing_chain] + + // [START bigtable_filters_composing_interleave] + public static void filterComposingInterleave() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterComposingInterleave(projectId, instanceId, tableId); + } + + public static void filterComposingInterleave( + String projectId, String instanceId, String tableId) { + // A filter that matches cells with the value true OR with the column qualifier os_build + Filter filter = + FILTERS + .interleave() + .filter(FILTERS.value().exactMatch("true")) + .filter(FILTERS.qualifier().exactMatch("os_build")); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_composing_interleave] + + // [START bigtable_filters_composing_condition] + public static void filterComposingCondition() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + filterComposingCondition(projectId, instanceId, tableId); + } + + public static void filterComposingCondition(String projectId, String instanceId, String tableId) { + // A filter that applies the label passed-filter IF the cell has the column qualifier + // data_plan_10gb AND the value true, OTHERWISE applies the label filtered-out + Filter filter = + FILTERS + .condition( + FILTERS + .chain() + .filter(FILTERS.qualifier().exactMatch("data_plan_10gb")) + .filter(FILTERS.value().exactMatch("true"))) + .then(FILTERS.label("passed-filter")) + .otherwise(FILTERS.label("filtered-out")); + readFilter(projectId, instanceId, tableId, filter); + } + // [END bigtable_filters_composing_condition] + + // [START bigtable_filters_limit_row_sample] + // [START bigtable_filters_limit_row_regex] + // [START bigtable_filters_limit_cells_per_col] + // [START bigtable_filters_limit_cells_per_row] + // [START bigtable_filters_limit_cells_per_row_offset] + // [START bigtable_filters_limit_col_family_regex] + // [START bigtable_filters_limit_col_qualifier_regex] + // [START bigtable_filters_limit_col_range] + // [START bigtable_filters_limit_value_range] + // [START bigtable_filters_limit_value_regex] + // [START bigtable_filters_limit_timestamp_range] + // [START bigtable_filters_limit_block_all] + // [START bigtable_filters_limit_pass_all] + // [START bigtable_filters_modify_strip_value] + // [START bigtable_filters_modify_apply_label] + // [START bigtable_filters_composing_chain] + // [START bigtable_filters_composing_interleave] + // [START bigtable_filters_composing_condition] + private static void readFilter( + String projectId, String instanceId, String tableId, Filter filter) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + Query query = Query.create(tableId).filter(filter); + ServerStream rows = dataClient.readRows(query); + for (Row row : rows) { + printRow(row); + } + } catch (IOException e) { + System.out.println( + "Unable to intailize service client, as a network error occured: \n" + e.toString()); + } + } + + private static void printRow(Row row) { + System.out.printf("Reading data for %s%n", row.getKey().toStringUtf8()); + String colFamily = ""; + for (RowCell cell : row.getCells()) { + if (!cell.getFamily().equals(colFamily)) { + colFamily = cell.getFamily(); + System.out.printf("Column Family %s%n", colFamily); + } + String labels = + cell.getLabels().size() == 0 ? "" : " [" + String.join(",", cell.getLabels()) + "]"; + System.out.printf( + "\t%s: %s @%s%s%n", + cell.getQualifier().toStringUtf8(), + cell.getValue().toStringUtf8(), + cell.getTimestamp(), + labels); + } + System.out.println(); + } +} +// [END bigtable_filters_limit_row_sample] +// [END bigtable_filters_limit_row_regex] +// [END bigtable_filters_limit_cells_per_col] +// [END bigtable_filters_limit_cells_per_row] +// [END bigtable_filters_limit_cells_per_row_offset] +// [END bigtable_filters_limit_col_family_regex] +// [END bigtable_filters_limit_col_qualifier_regex] +// [END bigtable_filters_limit_col_range] +// [END bigtable_filters_limit_value_range] +// [END bigtable_filters_limit_value_regex] +// [END bigtable_filters_limit_timestamp_range] +// [END bigtable_filters_limit_block_all] +// [END bigtable_filters_limit_pass_all] +// [END bigtable_filters_modify_strip_value] +// [END bigtable_filters_modify_apply_label] +// [END bigtable_filters_composing_chain] +// [END bigtable_filters_composing_interleave] +// [END bigtable_filters_composing_condition] diff --git a/samples/snippets/src/main/java/com/example/bigtable/Reads.java b/samples/snippets/src/main/java/com/example/bigtable/Reads.java new file mode 100644 index 0000000000..1807fc2f81 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/bigtable/Reads.java @@ -0,0 +1,272 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigtable; + +// [START bigtable_reads_row] +// [START bigtable_reads_row_partial] +// [START bigtable_reads_rows] +// [START bigtable_reads_row_range] +// [START bigtable_reads_row_ranges] +// [START bigtable_reads_prefix] +// [START bigtable_reads_filter] + +import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS; + +import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.Filters; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.protobuf.ByteString; +import java.io.IOException; + +public class Reads { + // [END bigtable_reads_row] + // [END bigtable_reads_row_partial] + // [END bigtable_reads_rows] + // [END bigtable_reads_row_range] + // [END bigtable_reads_row_ranges] + // [END bigtable_reads_prefix] + // [END bigtable_reads_filter] + + // [START bigtable_reads_row] + public static void readRow() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + readRow(projectId, instanceId, tableId); + } + + public static void readRow(String projectId, String instanceId, String tableId) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + String rowkey = "phone#4c410523#20190501"; + + Row row = dataClient.readRow(tableId, rowkey); + printRow(row); + + } catch (IOException e) { + System.out.println( + "Unable to intailize service client, as a network error occured: \n" + e.toString()); + } + } + // [END bigtable_reads_row] + + // [START bigtable_reads_row_partial] + public static void readRowPartial() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + readRowPartial(projectId, instanceId, tableId); + } + + public static void readRowPartial(String projectId, String instanceId, String tableId) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + String rowkey = "phone#4c410523#20190501"; + Filters.Filter filter = + FILTERS + .chain() + .filter(FILTERS.family().exactMatch("stats_summary")) + .filter(FILTERS.qualifier().exactMatch("os_build")); + + Row row = dataClient.readRow(tableId, rowkey, filter); + printRow(row); + + } catch (IOException e) { + System.out.println( + "Unable to intailize service client, as a network error occured: \n" + e.toString()); + } + } + // [END bigtable_reads_row_partial] + + // [START bigtable_reads_rows] + public static void readRows() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + readRows(projectId, instanceId, tableId); + } + + public static void readRows(String projectId, String instanceId, String tableId) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + Query query = + Query.create(tableId).rowKey("phone#4c410523#20190501").rowKey("phone#4c410523#20190502"); + ServerStream rows = dataClient.readRows(query); + for (Row row : rows) { + printRow(row); + } + } catch (IOException e) { + System.out.println( + "Unable to intailize service client, as a network error occured: \n" + e.toString()); + } + } + // [END bigtable_reads_rows] + + // [START bigtable_reads_row_range] + public static void readRowRange() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + readRowRange(projectId, instanceId, tableId); + } + + public static void readRowRange(String projectId, String instanceId, String tableId) { + String start = "phone#4c410523#20190501"; + String end = "phone#4c410523#201906201"; + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + Query query = Query.create(tableId).range(start, end); + ServerStream rows = dataClient.readRows(query); + for (Row row : rows) { + printRow(row); + } + } catch (IOException e) { + System.out.println( + "Unable to intailize service client, as a network error occured: \n" + e.toString()); + } + } + // [END bigtable_reads_row_range] + + // [START bigtable_reads_row_ranges] + public static void readRowRanges() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + readRowRanges(projectId, instanceId, tableId); + } + + public static void readRowRanges(String projectId, String instanceId, String tableId) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + Query query = + Query.create(tableId) + .range("phone#4c410523#20190501", "phone#4c410523#20190601") + .range("phone#5c10102#20190501", "phone#5c10102#20190601"); + ServerStream rows = dataClient.readRows(query); + for (Row row : rows) { + printRow(row); + } + } catch (IOException e) { + System.out.println( + "Unable to intailize service client, as a network error occured: \n" + e.toString()); + } + } + // [END bigtable_reads_row_ranges] + + // [START bigtable_reads_prefix] + public static void readPrefix() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + readPrefix(projectId, instanceId, tableId); + } + + public static void readPrefix(String projectId, String instanceId, String tableId) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + Query query = Query.create(tableId).prefix("phone"); + ServerStream rows = dataClient.readRows(query); + for (Row row : rows) { + printRow(row); + } + } catch (IOException e) { + System.out.println( + "Unable to intailize service client, as a network error occured: \n" + e.toString()); + } + } + // [END bigtable_reads_prefix] + + // [START bigtable_reads_filter] + public static void readFilter() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project-id"; + String instanceId = "my-instance-id"; + String tableId = "mobile-time-series"; + readFilter(projectId, instanceId, tableId); + } + + public static void readFilter(String projectId, String instanceId, String tableId) { + Filters.Filter filter = FILTERS.value().regex("PQ2A.*"); + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + Query query = Query.create(tableId).filter(filter); + ServerStream rows = dataClient.readRows(query); + for (Row row : rows) { + printRow(row); + } + } catch (IOException e) { + System.out.println( + "Unable to intailize service client, as a network error occured: \n" + e.toString()); + } + } + // [END bigtable_reads_filter] + + // [START bigtable_reads_row] + // [START bigtable_reads_row_partial] + // [START bigtable_reads_rows] + // [START bigtable_reads_row_range] + // [START bigtable_reads_row_ranges] + // [START bigtable_reads_prefix] + // [START bigtable_reads_filter] + private static void printRow(Row row) { + System.out.printf("Reading data for %s%n", row.getKey().toStringUtf8()); + String colFamily = ""; + for (RowCell cell : row.getCells()) { + if (!cell.getFamily().equals(colFamily)) { + colFamily = cell.getFamily(); + System.out.printf("Column Family %s%n", colFamily); + } + System.out.printf( + "\t%s: %s @%s%n", + cell.getQualifier().toStringUtf8(), cell.getValue().toStringUtf8(), cell.getTimestamp()); + } + System.out.println(); + } +} +// [END bigtable_reads_row] +// [END bigtable_reads_row_partial] +// [END bigtable_reads_rows] +// [END bigtable_reads_row_range] +// [END bigtable_reads_row_ranges] +// [END bigtable_reads_prefix] +// [END bigtable_reads_filter] diff --git a/samples/snippets/src/main/java/com/example/bigtable/WriteConditionally.java b/samples/snippets/src/main/java/com/example/bigtable/WriteConditionally.java index 7cc8fd2244..ac01cb0c63 100644 --- a/samples/snippets/src/main/java/com/example/bigtable/WriteConditionally.java +++ b/samples/snippets/src/main/java/com/example/bigtable/WriteConditionally.java @@ -36,7 +36,7 @@ public static void writeConditionally(String projectId, String instanceId, Strin try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { long timestamp = System.currentTimeMillis() * 1000; - String rowKey = "phone#4c410523#20190501"; + String rowkey = "phone#4c410523#20190501"; Mutation mutation = Mutation.create().setCell(COLUMN_FAMILY_NAME, "os_name", timestamp, "android"); @@ -49,7 +49,7 @@ public static void writeConditionally(String projectId, String instanceId, Strin .filter(FILTERS.value().regex("PQ2A\\..*")); ConditionalRowMutation conditionalRowMutation = - ConditionalRowMutation.create(tableId, rowKey).condition(filter).then(mutation); + ConditionalRowMutation.create(tableId, rowkey).condition(filter).then(mutation); boolean success = dataClient.checkAndMutateRow(conditionalRowMutation); diff --git a/samples/snippets/src/main/java/com/example/bigtable/WriteIncrement.java b/samples/snippets/src/main/java/com/example/bigtable/WriteIncrement.java index ec6d756b62..0f91a13717 100644 --- a/samples/snippets/src/main/java/com/example/bigtable/WriteIncrement.java +++ b/samples/snippets/src/main/java/com/example/bigtable/WriteIncrement.java @@ -34,9 +34,9 @@ public static void writeIncrement(String projectId, String instanceId, String ta try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { // Get an existing row that has a cell with an incrementable value. A value can be incremented // if it is encoded as a 64-bit big-endian signed integer. - String rowKey = "phone#4c410523#20190501"; + String rowkey = "phone#4c410523#20190501"; ReadModifyWriteRow mutation = - ReadModifyWriteRow.create(tableId, rowKey) + ReadModifyWriteRow.create(tableId, rowkey) .increment(COLUMN_FAMILY_NAME, "connected_cell", -1); Row success = dataClient.readModifyWriteRow(mutation); diff --git a/samples/snippets/src/main/java/com/example/bigtable/WriteSimple.java b/samples/snippets/src/main/java/com/example/bigtable/WriteSimple.java index 5067198921..a782542c3e 100644 --- a/samples/snippets/src/main/java/com/example/bigtable/WriteSimple.java +++ b/samples/snippets/src/main/java/com/example/bigtable/WriteSimple.java @@ -33,11 +33,11 @@ public static void writeSimple(String projectId, String instanceId, String table try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { long timestamp = System.currentTimeMillis() * 1000; - String rowKey = "phone#4c410523#20190501"; + String rowkey = "phone#4c410523#20190501"; ByteString one = ByteString.copyFrom(new byte[] {0, 0, 0, 0, 0, 0, 0, 1}); RowMutation rowMutation = - RowMutation.create(tableId, rowKey) + RowMutation.create(tableId, rowkey) .setCell( COLUMN_FAMILY_NAME, ByteString.copyFrom("connected_cell".getBytes()), @@ -51,7 +51,7 @@ public static void writeSimple(String projectId, String instanceId, String table .setCell(COLUMN_FAMILY_NAME, "os_build", timestamp, "PQ2A.190405.003"); dataClient.mutateRow(rowMutation); - System.out.printf("Successfully wrote row %s", rowKey); + System.out.printf("Successfully wrote row %s", rowkey); } catch (Exception e) { System.out.println("Error during WriteSimple: \n" + e.toString()); diff --git a/samples/snippets/src/test/java/com/example/bigtable/FiltersTest.java b/samples/snippets/src/test/java/com/example/bigtable/FiltersTest.java new file mode 100644 index 0000000000..ab9c7af669 --- /dev/null +++ b/samples/snippets/src/test/java/com/example/bigtable/FiltersTest.java @@ -0,0 +1,745 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigtable; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertNotNull; + +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.BulkMutation; +import com.google.cloud.bigtable.data.v2.models.Mutation; +import com.google.protobuf.ByteString; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.UUID; +import java.util.logging.Filter; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +public class FiltersTest { + + private static final String INSTANCE_ENV = "BIGTABLE_TESTING_INSTANCE"; + private static final String TABLE_ID = + "mobile-time-series-" + UUID.randomUUID().toString().substring(0, 20); + private static final String COLUMN_FAMILY_NAME_STATS = "stats_summary"; + private static final String COLUMN_FAMILY_NAME_DATA = "cell_plan"; + private static final Instant CURRENT_TIME = Instant.now(); + private static final long TIMESTAMP = CURRENT_TIME.toEpochMilli() * 1000; + private static final long TIMESTAMP_MINUS_HR = + CURRENT_TIME.minus(1, ChronoUnit.HOURS).toEpochMilli() * 1000; + + private static String projectId; + private static String instanceId; + private ByteArrayOutputStream bout; + + private static String requireEnv(String varName) { + String value = System.getenv(varName); + assertNotNull(String.format("Environment variable '%s' is required to perform these tests.", varName), value); + return value; + } + + @BeforeClass + public static void beforeClass() throws IOException { + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); + instanceId = requireEnv(INSTANCE_ENV); + + try (BigtableTableAdminClient adminClient = + BigtableTableAdminClient.create(projectId, instanceId)) { + CreateTableRequest createTableRequest = + CreateTableRequest.of(TABLE_ID) + .addFamily(COLUMN_FAMILY_NAME_STATS) + .addFamily(COLUMN_FAMILY_NAME_DATA); + adminClient.createTable(createTableRequest); + + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + BulkMutation bulkMutation = + BulkMutation.create(TABLE_ID) + .add( + "phone#4c410523#20190501", + Mutation.create() + .setCell( + COLUMN_FAMILY_NAME_STATS, + ByteString.copyFrom("connected_cell".getBytes()), + TIMESTAMP, + 1) + .setCell( + COLUMN_FAMILY_NAME_STATS, + ByteString.copyFrom("connected_wifi".getBytes()), + TIMESTAMP, + 1) + .setCell(COLUMN_FAMILY_NAME_STATS, "os_build", TIMESTAMP, "PQ2A.190405.003") + .setCell( + COLUMN_FAMILY_NAME_DATA, "data_plan_01gb", TIMESTAMP_MINUS_HR, "true") + .setCell(COLUMN_FAMILY_NAME_DATA, "data_plan_01gb", TIMESTAMP, "false") + .setCell(COLUMN_FAMILY_NAME_DATA, "data_plan_05gb", TIMESTAMP, "true")) + .add( + "phone#4c410523#20190502", + Mutation.create() + .setCell( + COLUMN_FAMILY_NAME_STATS, + ByteString.copyFrom("connected_cell".getBytes()), + TIMESTAMP, + 1) + .setCell( + COLUMN_FAMILY_NAME_STATS, + ByteString.copyFrom("connected_wifi".getBytes()), + TIMESTAMP, + 1) + .setCell(COLUMN_FAMILY_NAME_STATS, "os_build", TIMESTAMP, "PQ2A.190405.004") + .setCell(COLUMN_FAMILY_NAME_DATA, "data_plan_05gb", TIMESTAMP, "true")) + .add( + "phone#4c410523#20190505", + Mutation.create() + .setCell( + COLUMN_FAMILY_NAME_STATS, + ByteString.copyFrom("connected_cell".getBytes()), + TIMESTAMP, + 0) + .setCell( + COLUMN_FAMILY_NAME_STATS, + ByteString.copyFrom("connected_wifi".getBytes()), + TIMESTAMP, + 1) + .setCell(COLUMN_FAMILY_NAME_STATS, "os_build", TIMESTAMP, "PQ2A.190406.000") + .setCell(COLUMN_FAMILY_NAME_DATA, "data_plan_05gb", TIMESTAMP, "true")) + .add( + "phone#5c10102#20190501", + Mutation.create() + .setCell( + COLUMN_FAMILY_NAME_STATS, + ByteString.copyFrom("connected_cell".getBytes()), + TIMESTAMP, + 1) + .setCell( + COLUMN_FAMILY_NAME_STATS, + ByteString.copyFrom("connected_wifi".getBytes()), + TIMESTAMP, + 1) + .setCell(COLUMN_FAMILY_NAME_STATS, "os_build", TIMESTAMP, "PQ2A.190401.002") + .setCell(COLUMN_FAMILY_NAME_DATA, "data_plan_10gb", TIMESTAMP, "true")) + .add( + "phone#5c10102#20190502", + Mutation.create() + .setCell( + COLUMN_FAMILY_NAME_STATS, + ByteString.copyFrom("connected_cell".getBytes()), + TIMESTAMP, + 1) + .setCell( + COLUMN_FAMILY_NAME_STATS, + ByteString.copyFrom("connected_wifi".getBytes()), + TIMESTAMP, + 0) + .setCell(COLUMN_FAMILY_NAME_STATS, "os_build", TIMESTAMP, "PQ2A.190406.000") + .setCell(COLUMN_FAMILY_NAME_DATA, "data_plan_10gb", TIMESTAMP, "true")); + + dataClient.bulkMutateRows(bulkMutation); + } + } catch (Exception e) { + System.out.println("Error during beforeClass: \n" + e.toString()); + throw (e); + } + } + + @Before + public void setupStream() { + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + } + + @AfterClass + public static void afterClass() throws IOException { + try (BigtableTableAdminClient adminClient = + BigtableTableAdminClient.create(projectId, instanceId)) { + adminClient.deleteTable(TABLE_ID); + } catch (Exception e) { + System.out.println("Error during afterClass: \n" + e.toString()); + throw (e); + } + } + + @Test + public void testFilterRowSample() { + Filters.filterLimitRowSample(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output).contains("Reading data for"); + } + + @Test + public void testFilterRowRegex() { + Filters.filterLimitRowRegex(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_01gb: false @%1$s\n" + + "\tdata_plan_01gb: true @%2$s\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190401.002 @%1$s", + TIMESTAMP, TIMESTAMP_MINUS_HR)); + } + + @Test + public void testFilterCellsPerCol() { + Filters.filterLimitCellsPerCol(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_01gb: false @%1$s\n" + + "\tdata_plan_01gb: true @%2$s\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.004 @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190401.002 @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s", + TIMESTAMP, TIMESTAMP_MINUS_HR)); + } + + @Test + public void testFilterCellsPerRow() { + Filters.filterLimitCellsPerRow(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_01gb: false @%1$s\n" + + "\tdata_plan_01gb: true @%2$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n", + TIMESTAMP, TIMESTAMP_MINUS_HR)); + } + + @Test + public void testFilterLimitCellsPerRowOffset() { + Filters.filterLimitCellsPerRowOffset(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family stats_summary\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.004 @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family stats_summary\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family stats_summary\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190401.002 @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family stats_summary\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s", + TIMESTAMP)); + } + + @Test + public void testFilterColFamilyRegex() { + Filters.filterLimitColFamilyRegex(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.004 @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190401.002 @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s", + TIMESTAMP)); + } + + @Test + public void testFilterColQualifierRegex() { + Filters.filterLimitColQualifierRegex(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s", + TIMESTAMP)); + } + + @Test + public void testFilterColRange() { + Filters.filterLimitColRange(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_01gb: false @%1$s\n" + + "\tdata_plan_01gb: true @%2$s\n" + + "\tdata_plan_05gb: true @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s", + TIMESTAMP, TIMESTAMP_MINUS_HR)); + } + + @Test + public void testFilterValueRange() { + Filters.filterLimitValueRange(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190405.004 @%1$s", + TIMESTAMP)); + } + + @Test + public void testFilterValueRegex() { + Filters.filterLimitValueRegex(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190405.004 @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190406.000 @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190401.002 @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190406.000 @%1$s", + TIMESTAMP)); + } + + @Test + public void testFilterTimestampRange() { + Filters.filterLimitTimestampRange(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_01gb: true @%s\n", + TIMESTAMP_MINUS_HR)); + } + + @Test + public void testFilterBlockAll() { + Filters.filterLimitBlockAll(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output).doesNotContain("Reading data for"); + } + + @Test + public void testFilterPassAll() { + Filters.filterLimitPassAll(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_01gb: false @%1$s\n" + + "\tdata_plan_01gb: true @%2$s\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.004 @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190401.002 @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s", + TIMESTAMP, TIMESTAMP_MINUS_HR)); + } + + @Test + public void testFilterStripValue() { + Filters.filterModifyStripValue(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_01gb: @%1$s\n" + + "\tdata_plan_01gb: @%2$s\n" + + "\tdata_plan_05gb: @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: @%1$s\n" + + "\tconnected_wifi: @%1$s\n" + + "\tos_build: @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: @%1$s\n" + + "\tconnected_wifi: @%1$s\n" + + "\tos_build: @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: @%1$s\n" + + "\tconnected_wifi: @%1$s\n" + + "\tos_build: @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: @%1$s\n" + + "\tconnected_wifi: @%1$s\n" + + "\tos_build: @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: @%1$s\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: @%1$s\n" + + "\tconnected_wifi: @%1$s\n" + + "\tos_build: @%1$s", + TIMESTAMP, TIMESTAMP_MINUS_HR)); + } + + @Test + public void testFilterApplyLabel() { + Filters.filterModifyApplyLabel(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_01gb: false @%1$s [labelled]\n" + + "\tdata_plan_01gb: true @%2$s [labelled]\n" + + "\tdata_plan_05gb: true @%1$s [labelled]\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" + + "\tos_build: PQ2A.190405.003 @%1$s [labelled]\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s [labelled]\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" + + "\tos_build: PQ2A.190405.004 @%1$s [labelled]\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s [labelled]\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s [labelled]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" + + "\tos_build: PQ2A.190406.000 @%1$s [labelled]\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s [labelled]\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" + + "\tos_build: PQ2A.190401.002 @%1$s [labelled]\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s [labelled]\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s [labelled]\n" + + "\tos_build: PQ2A.190406.000 @%1$s [labelled]", + TIMESTAMP, TIMESTAMP_MINUS_HR)); + } + + @Test + public void testFilterChain() { + Filters.filterComposingChain(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_01gb: false @%1$s\n" + + "\tdata_plan_05gb: true @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s\n", + TIMESTAMP)); + } + + @Test + public void testFilterInterleave() { + Filters.filterComposingInterleave(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_01gb: true @%2$s\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190405.004 @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190406.000 @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190401.002 @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190406.000 @%1$s", + TIMESTAMP, TIMESTAMP_MINUS_HR)); + } + + @Test + public void testFilterCondition() { + Filters.filterComposingCondition(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_01gb: false @%1$s [filtered-out]\n" + + "\tdata_plan_01gb: true @%2$s [filtered-out]\n" + + "\tdata_plan_05gb: true @%1$s [filtered-out]\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [filtered-out]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [filtered-out]\n" + + "\tos_build: PQ2A.190405.003 @%1$s [filtered-out]\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s [filtered-out]\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [filtered-out]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [filtered-out]\n" + + "\tos_build: PQ2A.190405.004 @%1$s [filtered-out]\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family cell_plan\n" + + "\tdata_plan_05gb: true @%1$s [filtered-out]\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s [filtered-out]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [filtered-out]\n" + + "\tos_build: PQ2A.190406.000 @%1$s [filtered-out]\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s [passed-filter]\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [passed-filter]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [passed-filter]\n" + + "\tos_build: PQ2A.190401.002 @%1$s [passed-filter]\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family cell_plan\n" + + "\tdata_plan_10gb: true @%1$s [passed-filter]\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [passed-filter]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s [passed-filter]\n" + + "\tos_build: PQ2A.190406.000 @%1$s [passed-filter]", + TIMESTAMP, TIMESTAMP_MINUS_HR)); + } +} diff --git a/samples/snippets/src/test/java/com/example/bigtable/ReadsTest.java b/samples/snippets/src/test/java/com/example/bigtable/ReadsTest.java new file mode 100644 index 0000000000..e7afadee07 --- /dev/null +++ b/samples/snippets/src/test/java/com/example/bigtable/ReadsTest.java @@ -0,0 +1,343 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigtable; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertNotNull; + +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; +import com.google.cloud.bigtable.admin.v2.models.Table; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.models.BulkMutation; +import com.google.cloud.bigtable.data.v2.models.Mutation; +import com.google.protobuf.ByteString; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.UUID; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class ReadsTest { + + private static final String INSTANCE_ENV = "BIGTABLE_TESTING_INSTANCE"; + private static final String TABLE_ID = + "mobile-time-series-" + UUID.randomUUID().toString().substring(0, 20); + private static final String COLUMN_FAMILY_NAME = "stats_summary"; + private static final long TIMESTAMP = System.currentTimeMillis() * 1000; + + private static String projectId; + private static String instanceId; + private ByteArrayOutputStream bout; + + private static String requireEnv(String varName) { + String value = System.getenv(varName); + assertNotNull(String.format("Environment variable '%s' is required to perform these tests.", varName), value); + return value; + } + + @BeforeClass + public static void beforeClass() throws IOException { + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); + instanceId = requireEnv(INSTANCE_ENV); + + try (BigtableTableAdminClient adminClient = + BigtableTableAdminClient.create(projectId, instanceId)) { + CreateTableRequest createTableRequest = + CreateTableRequest.of(TABLE_ID).addFamily(COLUMN_FAMILY_NAME); + adminClient.createTable(createTableRequest); + + try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { + BulkMutation bulkMutation = + BulkMutation.create(TABLE_ID) + .add( + "phone#4c410523#20190501", + Mutation.create() + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_cell".getBytes()), + TIMESTAMP, + 1) + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_wifi".getBytes()), + TIMESTAMP, + 1) + .setCell(COLUMN_FAMILY_NAME, "os_build", TIMESTAMP, "PQ2A.190405.003")) + .add( + "phone#4c410523#20190502", + Mutation.create() + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_cell".getBytes()), + TIMESTAMP, + 1) + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_wifi".getBytes()), + TIMESTAMP, + 1) + .setCell(COLUMN_FAMILY_NAME, "os_build", TIMESTAMP, "PQ2A.190405.004")) + .add( + "phone#4c410523#20190505", + Mutation.create() + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_cell".getBytes()), + TIMESTAMP, + 0) + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_wifi".getBytes()), + TIMESTAMP, + 1) + .setCell(COLUMN_FAMILY_NAME, "os_build", TIMESTAMP, "PQ2A.190406.000")) + .add( + "phone#5c10102#20190501", + Mutation.create() + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_cell".getBytes()), + TIMESTAMP, + 1) + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_wifi".getBytes()), + TIMESTAMP, + 1) + .setCell(COLUMN_FAMILY_NAME, "os_build", TIMESTAMP, "PQ2A.190401.002")) + .add( + "phone#5c10102#20190502", + Mutation.create() + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_cell".getBytes()), + TIMESTAMP, + 1) + .setCell( + COLUMN_FAMILY_NAME, + ByteString.copyFrom("connected_wifi".getBytes()), + TIMESTAMP, + 0) + .setCell(COLUMN_FAMILY_NAME, "os_build", TIMESTAMP, "PQ2A.190406.000")); + + dataClient.bulkMutateRows(bulkMutation); + } + } catch (Exception e) { + System.out.println("Error during beforeClass: \n" + e.toString()); + throw (e); + } + } + + @Before + public void setupStream() { + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + } + + @AfterClass + public static void afterClass() throws IOException { + try (BigtableTableAdminClient adminClient = + BigtableTableAdminClient.create(projectId, instanceId)) { + adminClient.deleteTable(TABLE_ID); + } catch (Exception e) { + System.out.println("Error during afterClass: \n" + e.toString()); + throw (e); + } + } + + @Test + public void testReadRow() { + Reads.readRow(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.003 @%1$s", + TIMESTAMP)); + } + + @Test + public void testReadRowPartial() { + Reads.readRowPartial(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190405.003 @%1$s", + TIMESTAMP)); + } + + @Test + public void testReadRows() { + Reads.readRows(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.004 @%1$s", + TIMESTAMP)); + } + + @Test + public void testReadRowRange() { + Reads.readRowRange(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.004 @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s", + TIMESTAMP)); + } + + @Test + public void testReadRowRanges() { + Reads.readRowRanges(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.004 @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190401.002 @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s", + TIMESTAMP)); + } + + @Test + public void testReadPrefix() { + Reads.readPrefix(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190405.004 @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tos_build: PQ2A.190401.002 @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family stats_summary\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s\n" + + "\tos_build: PQ2A.190406.000 @%1$s", + TIMESTAMP)); + } + + @Test + public void testReadFilter() { + Reads.readFilter(projectId, instanceId, TABLE_ID); + + String output = bout.toString(); + assertThat(output) + .contains( + String.format( + "Reading data for phone#4c410523#20190501\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190405.003 @%1$s\n\n" + + "Reading data for phone#4c410523#20190502\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190405.004 @%1$s\n\n" + + "Reading data for phone#4c410523#20190505\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190406.000 @%1$s\n\n" + + "Reading data for phone#5c10102#20190501\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190401.002 @%1$s\n\n" + + "Reading data for phone#5c10102#20190502\n" + + "Column Family stats_summary\n" + + "\tos_build: PQ2A.190406.000 @%1$s", + TIMESTAMP)); + } +} diff --git a/samples/snippets/src/test/java/com/example/bigtable/WritesTest.java b/samples/snippets/src/test/java/com/example/bigtable/WritesTest.java index afe70c4027..364b527a96 100644 --- a/samples/snippets/src/test/java/com/example/bigtable/WritesTest.java +++ b/samples/snippets/src/test/java/com/example/bigtable/WritesTest.java @@ -43,12 +43,10 @@ public class WritesTest { private static String projectId; private static String instanceId; private ByteArrayOutputStream bout; - private static String requireEnv(String varName) { - assertNotNull( - System.getenv(varName), - "Environment variable '%s' is required to perform these tests.".format(varName)); - return System.getenv(varName); + String value = System.getenv(varName); + assertNotNull(String.format("Environment variable '%s' is required to perform these tests.", varName), value); + return value; } @BeforeClass From a29f9f70d288575d002c044c9e5bb1267fbea3e7 Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Fri, 6 Dec 2019 16:51:50 -0500 Subject: [PATCH 07/30] samples: Bigtable: Using cleaner number input for snippets (#1787) --- .../src/main/java/com/example/bigtable/WriteBatch.java | 5 ++--- .../src/main/java/com/example/bigtable/WriteSimple.java | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/samples/snippets/src/main/java/com/example/bigtable/WriteBatch.java b/samples/snippets/src/main/java/com/example/bigtable/WriteBatch.java index 9b4e58a505..180793a10b 100644 --- a/samples/snippets/src/main/java/com/example/bigtable/WriteBatch.java +++ b/samples/snippets/src/main/java/com/example/bigtable/WriteBatch.java @@ -33,7 +33,6 @@ public static void writeBatch(String projectId, String instanceId, String tableI try (BigtableDataClient dataClient = BigtableDataClient.create(projectId, instanceId)) { long timestamp = System.currentTimeMillis() * 1000; - ByteString one = ByteString.copyFrom(new byte[] {0, 0, 0, 0, 0, 0, 0, 1}); BulkMutation bulkMutation = BulkMutation.create(tableId) @@ -44,7 +43,7 @@ public static void writeBatch(String projectId, String instanceId, String tableI COLUMN_FAMILY_NAME, ByteString.copyFrom("connected_wifi".getBytes()), timestamp, - one) + 1) .setCell(COLUMN_FAMILY_NAME, "os_build", timestamp, "12155.0.0-rc1")) .add( "tablet#a0b81f74#20190502", @@ -53,7 +52,7 @@ public static void writeBatch(String projectId, String instanceId, String tableI COLUMN_FAMILY_NAME, ByteString.copyFrom("connected_wifi".getBytes()), timestamp, - one) + 1) .setCell(COLUMN_FAMILY_NAME, "os_build", timestamp, "12155.0.0-rc6")); dataClient.bulkMutateRows(bulkMutation); diff --git a/samples/snippets/src/main/java/com/example/bigtable/WriteSimple.java b/samples/snippets/src/main/java/com/example/bigtable/WriteSimple.java index a782542c3e..5d7f4d5b2a 100644 --- a/samples/snippets/src/main/java/com/example/bigtable/WriteSimple.java +++ b/samples/snippets/src/main/java/com/example/bigtable/WriteSimple.java @@ -34,7 +34,6 @@ public static void writeSimple(String projectId, String instanceId, String table long timestamp = System.currentTimeMillis() * 1000; String rowkey = "phone#4c410523#20190501"; - ByteString one = ByteString.copyFrom(new byte[] {0, 0, 0, 0, 0, 0, 0, 1}); RowMutation rowMutation = RowMutation.create(tableId, rowkey) @@ -42,12 +41,12 @@ public static void writeSimple(String projectId, String instanceId, String table COLUMN_FAMILY_NAME, ByteString.copyFrom("connected_cell".getBytes()), timestamp, - one) + 1) .setCell( COLUMN_FAMILY_NAME, ByteString.copyFrom("connected_wifi".getBytes()), timestamp, - one) + 1) .setCell(COLUMN_FAMILY_NAME, "os_build", timestamp, "PQ2A.190405.003"); dataClient.mutateRow(rowMutation); From e7994fab18e9465624cc20904e9e7df4c3efec31 Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Tue, 14 Jan 2020 14:50:57 -0500 Subject: [PATCH 08/30] samples: bigtable hbase: read and filter snippets (#1947) * Read and filter snippets for Hbase Bigtable Reads done Filters WIP, issues with Pass and ColRange. Some nuances to sort out between HBase and other clients still * Filter snippets * Remove unused region tags Co-authored-by: Les Vogel --- .../main/java/com/example/bigtable/Filters.java | 6 +++--- .../src/main/java/com/example/bigtable/Reads.java | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/samples/snippets/src/main/java/com/example/bigtable/Filters.java b/samples/snippets/src/main/java/com/example/bigtable/Filters.java index 93367eeace..bb8d8cbd3f 100644 --- a/samples/snippets/src/main/java/com/example/bigtable/Filters.java +++ b/samples/snippets/src/main/java/com/example/bigtable/Filters.java @@ -75,8 +75,8 @@ public static void filterLimitRowSample() { } public static void filterLimitRowSample(String projectId, String instanceId, String tableId) { - // A filter that matches cells from a row with probability .5 - Filter filter = FILTERS.key().sample(.5); + // A filter that matches cells from a row with probability .75 + Filter filter = FILTERS.key().sample(.75); readFilter(projectId, instanceId, tableId, filter); } // [END bigtable_filters_limit_row_sample] @@ -415,7 +415,7 @@ private static void readFilter( } } catch (IOException e) { System.out.println( - "Unable to intailize service client, as a network error occured: \n" + e.toString()); + "Unable to initialize service client, as a network error occurred: \n" + e.toString()); } } diff --git a/samples/snippets/src/main/java/com/example/bigtable/Reads.java b/samples/snippets/src/main/java/com/example/bigtable/Reads.java index 1807fc2f81..778108725a 100644 --- a/samples/snippets/src/main/java/com/example/bigtable/Reads.java +++ b/samples/snippets/src/main/java/com/example/bigtable/Reads.java @@ -66,7 +66,7 @@ public static void readRow(String projectId, String instanceId, String tableId) } catch (IOException e) { System.out.println( - "Unable to intailize service client, as a network error occured: \n" + e.toString()); + "Unable to initialize service client, as a network error occurred: \n" + e.toString()); } } // [END bigtable_reads_row] @@ -97,7 +97,7 @@ public static void readRowPartial(String projectId, String instanceId, String ta } catch (IOException e) { System.out.println( - "Unable to intailize service client, as a network error occured: \n" + e.toString()); + "Unable to initialize service client, as a network error occurred: \n" + e.toString()); } } // [END bigtable_reads_row_partial] @@ -124,7 +124,7 @@ public static void readRows(String projectId, String instanceId, String tableId) } } catch (IOException e) { System.out.println( - "Unable to intailize service client, as a network error occured: \n" + e.toString()); + "Unable to initialize service client, as a network error occurred: \n" + e.toString()); } } // [END bigtable_reads_rows] @@ -153,7 +153,7 @@ public static void readRowRange(String projectId, String instanceId, String tabl } } catch (IOException e) { System.out.println( - "Unable to intailize service client, as a network error occured: \n" + e.toString()); + "Unable to initialize service client, as a network error occurred: \n" + e.toString()); } } // [END bigtable_reads_row_range] @@ -182,7 +182,7 @@ public static void readRowRanges(String projectId, String instanceId, String tab } } catch (IOException e) { System.out.println( - "Unable to intailize service client, as a network error occured: \n" + e.toString()); + "Unable to initialize service client, as a network error occurred: \n" + e.toString()); } } // [END bigtable_reads_row_ranges] @@ -208,7 +208,7 @@ public static void readPrefix(String projectId, String instanceId, String tableI } } catch (IOException e) { System.out.println( - "Unable to intailize service client, as a network error occured: \n" + e.toString()); + "Unable to initialize service client, as a network error occurred: \n" + e.toString()); } } // [END bigtable_reads_prefix] @@ -236,7 +236,7 @@ public static void readFilter(String projectId, String instanceId, String tableI } } catch (IOException e) { System.out.println( - "Unable to intailize service client, as a network error occured: \n" + e.toString()); + "Unable to initialize service client, as a network error occurred: \n" + e.toString()); } } // [END bigtable_reads_filter] From 76ffe70934ea9945453c5eb63d90f40be92a4495 Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Thu, 18 Jun 2020 13:37:50 -0400 Subject: [PATCH 09/30] samples: [bigtable] Update region tags for reads and filters (#3148) * Update read region tags * Update region tags for filters * Move region tags in filters * Move region tags out of class * Update tags for java * change generic to print * update java bigtable filters region tags * make into one sample * untrack hbase-test * change function to code * lint everything --- .../java/com/example/bigtable/Filters.java | 80 ++----------------- .../main/java/com/example/bigtable/Reads.java | 34 ++------ .../com/example/bigtable/FiltersTest.java | 64 ++++++++++----- .../java/com/example/bigtable/ReadsTest.java | 4 +- .../java/com/example/bigtable/WritesTest.java | 5 +- 5 files changed, 63 insertions(+), 124 deletions(-) diff --git a/samples/snippets/src/main/java/com/example/bigtable/Filters.java b/samples/snippets/src/main/java/com/example/bigtable/Filters.java index bb8d8cbd3f..cb6e6175d8 100644 --- a/samples/snippets/src/main/java/com/example/bigtable/Filters.java +++ b/samples/snippets/src/main/java/com/example/bigtable/Filters.java @@ -16,25 +16,10 @@ package com.example.bigtable; +// [START bigtable_filters_print] + import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS; -// [START bigtable_filters_limit_row_sample] -// [START bigtable_filters_limit_row_regex] -// [START bigtable_filters_limit_cells_per_col] -// [START bigtable_filters_limit_cells_per_row] -// [START bigtable_filters_limit_cells_per_row_offset] -// [START bigtable_filters_limit_col_family_regex] -// [START bigtable_filters_limit_col_qualifier_regex] -// [START bigtable_filters_limit_col_range] -// [START bigtable_filters_limit_value_range] -// [START bigtable_filters_limit_value_regex] -// [START bigtable_filters_limit_timestamp_range] -// [START bigtable_filters_limit_block_all] -// [START bigtable_filters_limit_pass_all] -// [START bigtable_filters_modify_strip_value] -// [START bigtable_filters_modify_apply_label] -// [START bigtable_filters_composing_chain] -// [START bigtable_filters_composing_interleave] -// [START bigtable_filters_composing_condition] + import com.google.api.gax.rpc.ServerStream; import com.google.cloud.bigtable.data.v2.BigtableDataClient; import com.google.cloud.bigtable.data.v2.models.Filters.Filter; @@ -45,26 +30,11 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; + public class Filters { - // [END bigtable_filters_limit_row_sample] - // [END bigtable_filters_limit_row_regex] - // [END bigtable_filters_limit_cells_per_col] - // [END bigtable_filters_limit_cells_per_row] - // [END bigtable_filters_limit_cells_per_row_offset] - // [END bigtable_filters_limit_col_family_regex] - // [END bigtable_filters_limit_col_qualifier_regex] - // [END bigtable_filters_limit_col_range] - // [END bigtable_filters_limit_value_range] - // [END bigtable_filters_limit_value_regex] - // [END bigtable_filters_limit_timestamp_range] - // [END bigtable_filters_limit_block_all] - // [END bigtable_filters_limit_pass_all] - // [END bigtable_filters_modify_strip_value] - // [END bigtable_filters_modify_apply_label] - // [END bigtable_filters_composing_chain] - // [END bigtable_filters_composing_interleave] - // [END bigtable_filters_composing_condition] + // Write your code here. + // [START_EXCLUDE] // [START bigtable_filters_limit_row_sample] public static void filterLimitRowSample() { // TODO(developer): Replace these variables before running the sample. @@ -383,25 +353,8 @@ public static void filterComposingCondition(String projectId, String instanceId, readFilter(projectId, instanceId, tableId, filter); } // [END bigtable_filters_composing_condition] + // [END_EXCLUDE] - // [START bigtable_filters_limit_row_sample] - // [START bigtable_filters_limit_row_regex] - // [START bigtable_filters_limit_cells_per_col] - // [START bigtable_filters_limit_cells_per_row] - // [START bigtable_filters_limit_cells_per_row_offset] - // [START bigtable_filters_limit_col_family_regex] - // [START bigtable_filters_limit_col_qualifier_regex] - // [START bigtable_filters_limit_col_range] - // [START bigtable_filters_limit_value_range] - // [START bigtable_filters_limit_value_regex] - // [START bigtable_filters_limit_timestamp_range] - // [START bigtable_filters_limit_block_all] - // [START bigtable_filters_limit_pass_all] - // [START bigtable_filters_modify_strip_value] - // [START bigtable_filters_modify_apply_label] - // [START bigtable_filters_composing_chain] - // [START bigtable_filters_composing_interleave] - // [START bigtable_filters_composing_condition] private static void readFilter( String projectId, String instanceId, String tableId, Filter filter) { // Initialize client that will be used to send requests. This client only needs to be created @@ -439,21 +392,4 @@ private static void printRow(Row row) { System.out.println(); } } -// [END bigtable_filters_limit_row_sample] -// [END bigtable_filters_limit_row_regex] -// [END bigtable_filters_limit_cells_per_col] -// [END bigtable_filters_limit_cells_per_row] -// [END bigtable_filters_limit_cells_per_row_offset] -// [END bigtable_filters_limit_col_family_regex] -// [END bigtable_filters_limit_col_qualifier_regex] -// [END bigtable_filters_limit_col_range] -// [END bigtable_filters_limit_value_range] -// [END bigtable_filters_limit_value_regex] -// [END bigtable_filters_limit_timestamp_range] -// [END bigtable_filters_limit_block_all] -// [END bigtable_filters_limit_pass_all] -// [END bigtable_filters_modify_strip_value] -// [END bigtable_filters_modify_apply_label] -// [END bigtable_filters_composing_chain] -// [END bigtable_filters_composing_interleave] -// [END bigtable_filters_composing_condition] +// [END bigtable_filters_print] diff --git a/samples/snippets/src/main/java/com/example/bigtable/Reads.java b/samples/snippets/src/main/java/com/example/bigtable/Reads.java index 778108725a..fc0762ac1a 100644 --- a/samples/snippets/src/main/java/com/example/bigtable/Reads.java +++ b/samples/snippets/src/main/java/com/example/bigtable/Reads.java @@ -16,13 +16,7 @@ package com.example.bigtable; -// [START bigtable_reads_row] -// [START bigtable_reads_row_partial] -// [START bigtable_reads_rows] -// [START bigtable_reads_row_range] -// [START bigtable_reads_row_ranges] -// [START bigtable_reads_prefix] -// [START bigtable_reads_filter] +// [START bigtable_reads_print] import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS; @@ -37,15 +31,9 @@ import java.io.IOException; public class Reads { - // [END bigtable_reads_row] - // [END bigtable_reads_row_partial] - // [END bigtable_reads_rows] - // [END bigtable_reads_row_range] - // [END bigtable_reads_row_ranges] - // [END bigtable_reads_prefix] - // [END bigtable_reads_filter] - // [START bigtable_reads_row] + // Write your code here. + // [START_EXCLUDE] public static void readRow() { // TODO(developer): Replace these variables before running the sample. String projectId = "my-project-id"; @@ -240,14 +228,8 @@ public static void readFilter(String projectId, String instanceId, String tableI } } // [END bigtable_reads_filter] + // [END_EXCLUDE] - // [START bigtable_reads_row] - // [START bigtable_reads_row_partial] - // [START bigtable_reads_rows] - // [START bigtable_reads_row_range] - // [START bigtable_reads_row_ranges] - // [START bigtable_reads_prefix] - // [START bigtable_reads_filter] private static void printRow(Row row) { System.out.printf("Reading data for %s%n", row.getKey().toStringUtf8()); String colFamily = ""; @@ -263,10 +245,4 @@ private static void printRow(Row row) { System.out.println(); } } -// [END bigtable_reads_row] -// [END bigtable_reads_row_partial] -// [END bigtable_reads_rows] -// [END bigtable_reads_row_range] -// [END bigtable_reads_row_ranges] -// [END bigtable_reads_prefix] -// [END bigtable_reads_filter] +// [END bigtable_reads_print] diff --git a/samples/snippets/src/test/java/com/example/bigtable/FiltersTest.java b/samples/snippets/src/test/java/com/example/bigtable/FiltersTest.java index ab9c7af669..429b5712c4 100644 --- a/samples/snippets/src/test/java/com/example/bigtable/FiltersTest.java +++ b/samples/snippets/src/test/java/com/example/bigtable/FiltersTest.java @@ -57,7 +57,9 @@ public class FiltersTest { private static String requireEnv(String varName) { String value = System.getenv(varName); - assertNotNull(String.format("Environment variable '%s' is required to perform these tests.", varName), value); + assertNotNull( + String.format("Environment variable '%s' is required to perform these tests.", varName), + value); return value; } @@ -597,36 +599,46 @@ public void testFilterApplyLabel() { + "\tdata_plan_01gb: true @%2$s [labelled]\n" + "\tdata_plan_05gb: true @%1$s [labelled]\n" + "Column Family stats_summary\n" - + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" - + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[labelled]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[labelled]\n" + "\tos_build: PQ2A.190405.003 @%1$s [labelled]\n\n" + "Reading data for phone#4c410523#20190502\n" + "Column Family cell_plan\n" + "\tdata_plan_05gb: true @%1$s [labelled]\n" + "Column Family stats_summary\n" - + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" - + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[labelled]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[labelled]\n" + "\tos_build: PQ2A.190405.004 @%1$s [labelled]\n\n" + "Reading data for phone#4c410523#20190505\n" + "Column Family cell_plan\n" + "\tdata_plan_05gb: true @%1$s [labelled]\n" + "Column Family stats_summary\n" - + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s [labelled]\n" - + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s " + + "[labelled]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[labelled]\n" + "\tos_build: PQ2A.190406.000 @%1$s [labelled]\n\n" + "Reading data for phone#5c10102#20190501\n" + "Column Family cell_plan\n" + "\tdata_plan_10gb: true @%1$s [labelled]\n" + "Column Family stats_summary\n" - + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" - + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[labelled]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[labelled]\n" + "\tos_build: PQ2A.190401.002 @%1$s [labelled]\n\n" + "Reading data for phone#5c10102#20190502\n" + "Column Family cell_plan\n" + "\tdata_plan_10gb: true @%1$s [labelled]\n" + "Column Family stats_summary\n" - + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [labelled]\n" - + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s [labelled]\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[labelled]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s " + + "[labelled]\n" + "\tos_build: PQ2A.190406.000 @%1$s [labelled]", TIMESTAMP, TIMESTAMP_MINUS_HR)); } @@ -709,36 +721,46 @@ public void testFilterCondition() { + "\tdata_plan_01gb: true @%2$s [filtered-out]\n" + "\tdata_plan_05gb: true @%1$s [filtered-out]\n" + "Column Family stats_summary\n" - + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [filtered-out]\n" - + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [filtered-out]\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[filtered-out]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[filtered-out]\n" + "\tos_build: PQ2A.190405.003 @%1$s [filtered-out]\n\n" + "Reading data for phone#4c410523#20190502\n" + "Column Family cell_plan\n" + "\tdata_plan_05gb: true @%1$s [filtered-out]\n" + "Column Family stats_summary\n" - + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [filtered-out]\n" - + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [filtered-out]\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[filtered-out]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[filtered-out]\n" + "\tos_build: PQ2A.190405.004 @%1$s [filtered-out]\n\n" + "Reading data for phone#4c410523#20190505\n" + "Column Family cell_plan\n" + "\tdata_plan_05gb: true @%1$s [filtered-out]\n" + "Column Family stats_summary\n" - + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s [filtered-out]\n" - + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [filtered-out]\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s " + + "[filtered-out]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[filtered-out]\n" + "\tos_build: PQ2A.190406.000 @%1$s [filtered-out]\n\n" + "Reading data for phone#5c10102#20190501\n" + "Column Family cell_plan\n" + "\tdata_plan_10gb: true @%1$s [passed-filter]\n" + "Column Family stats_summary\n" - + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [passed-filter]\n" - + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [passed-filter]\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[passed-filter]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[passed-filter]\n" + "\tos_build: PQ2A.190401.002 @%1$s [passed-filter]\n\n" + "Reading data for phone#5c10102#20190502\n" + "Column Family cell_plan\n" + "\tdata_plan_10gb: true @%1$s [passed-filter]\n" + "Column Family stats_summary\n" - + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s [passed-filter]\n" - + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s [passed-filter]\n" + + "\tconnected_cell: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0001 @%1$s " + + "[passed-filter]\n" + + "\tconnected_wifi: \u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000 @%1$s " + + "[passed-filter]\n" + "\tos_build: PQ2A.190406.000 @%1$s [passed-filter]", TIMESTAMP, TIMESTAMP_MINUS_HR)); } diff --git a/samples/snippets/src/test/java/com/example/bigtable/ReadsTest.java b/samples/snippets/src/test/java/com/example/bigtable/ReadsTest.java index e7afadee07..7b526c87ea 100644 --- a/samples/snippets/src/test/java/com/example/bigtable/ReadsTest.java +++ b/samples/snippets/src/test/java/com/example/bigtable/ReadsTest.java @@ -52,7 +52,9 @@ public class ReadsTest { private static String requireEnv(String varName) { String value = System.getenv(varName); - assertNotNull(String.format("Environment variable '%s' is required to perform these tests.", varName), value); + assertNotNull( + String.format("Environment variable '%s' is required to perform these tests.", varName), + value); return value; } diff --git a/samples/snippets/src/test/java/com/example/bigtable/WritesTest.java b/samples/snippets/src/test/java/com/example/bigtable/WritesTest.java index 364b527a96..b28a1ef74d 100644 --- a/samples/snippets/src/test/java/com/example/bigtable/WritesTest.java +++ b/samples/snippets/src/test/java/com/example/bigtable/WritesTest.java @@ -43,9 +43,12 @@ public class WritesTest { private static String projectId; private static String instanceId; private ByteArrayOutputStream bout; + private static String requireEnv(String varName) { String value = System.getenv(varName); - assertNotNull(String.format("Environment variable '%s' is required to perform these tests.", varName), value); + assertNotNull( + String.format("Environment variable '%s' is required to perform these tests.", varName), + value); return value; } From 8fdf63ec09a5bbc761ce21e7a4b80f89cb9b32fb Mon Sep 17 00:00:00 2001 From: Billy Jacobson Date: Wed, 24 Jun 2020 14:00:03 -0400 Subject: [PATCH 10/30] samples: fix region tag (#3238) Fixes #issue > It's a good idea to open an issue first for discussion. - [ ] I have followed [Sample Format Guide](https://github.com/GoogleCloudPlatform/java-docs-samples/blob/master/SAMPLE_FORMAT.md) - [ ] `pom.xml` parent set to latest `shared-configuration` - [ ] Appropriate changes to README are included in PR - [ ] API's need to be enabled to test (tell us) - [ ] Environment Variables need to be set (ask us to set them) - [x] **Tests** pass: `mvn clean verify` **required** - [x] **Lint** passes: `mvn -P lint checkstyle:check` **required** - [ ] **Static Analysis**: `mvn -P lint clean compile pmd:cpd-check spotbugs:check` **advisory only** - [x] Please **merge** this PR for me once it is approved. --- samples/snippets/src/main/java/com/example/bigtable/Reads.java | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/snippets/src/main/java/com/example/bigtable/Reads.java b/samples/snippets/src/main/java/com/example/bigtable/Reads.java index fc0762ac1a..989da4f62f 100644 --- a/samples/snippets/src/main/java/com/example/bigtable/Reads.java +++ b/samples/snippets/src/main/java/com/example/bigtable/Reads.java @@ -34,6 +34,7 @@ public class Reads { // Write your code here. // [START_EXCLUDE] + // [START bigtable_reads_row] public static void readRow() { // TODO(developer): Replace these variables before running the sample. String projectId = "my-project-id"; From 3e8c39f554c8ce532fc9a2f084876a9b3434ed02 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 21 Jul 2020 20:47:50 +0200 Subject: [PATCH 11/30] chore(deps): update dependency com.google.cloud:google-cloud-bigtable to v1.14.0 (#372) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [com.google.cloud:google-cloud-bigtable](https://togithub.com/googleapis/java-bigtable) | minor | `1.13.1` -> `1.14.0` | --- ### Release Notes
googleapis/java-bigtable ### [`v1.14.0`](https://togithub.com/googleapis/java-bigtable/blob/master/CHANGELOG.md#​1140-httpswwwgithubcomgoogleapisjava-bigtablecomparev1131v1140-2020-07-20) [Compare Source](https://togithub.com/googleapis/java-bigtable/compare/v1.13.1...v1.14.0) ##### Features - Cloud Bigtable Managed Backup Implementation ([#​305](https://www.github.com/googleapis/java-bigtable/issues/305)) ([9e3307a](https://www.github.com/googleapis/java-bigtable/commit/9e3307a4872d3bae3c04e7857a9eb4859151965e)) ##### [1.13.1](https://www.github.com/googleapis/java-bigtable/compare/v1.13.0...v1.13.1) (2020-07-10) ##### Reverts - Revert "fix: change app_profile_id to the correct header app_profile ([#​318](https://togithub.com/googleapis/java-bigtable/issues/318))" ([#​335](https://togithub.com/googleapis/java-bigtable/issues/335)) ([200cfac](https://www.github.com/googleapis/java-bigtable/commit/200cfac9b68be9882943765b06de7c0644daf53d)), closes [#​318](https://www.github.com/googleapis/java-bigtable/issues/318) [#​335](https://www.github.com/googleapis/java-bigtable/issues/335) ##### Dependencies - update autovalue.version to v1.7.3 ([#​341](https://www.github.com/googleapis/java-bigtable/issues/341)) ([071d0fe](https://www.github.com/googleapis/java-bigtable/commit/071d0feac03b5077b088a09244fe863aa111c231)) - update autovalue.version to v1.7.4 ([#​365](https://www.github.com/googleapis/java-bigtable/issues/365)) ([18ba73d](https://www.github.com/googleapis/java-bigtable/commit/18ba73da3e6f1035699dc885109f90ffb160cb4e)) - update dependency com.google.cloud:google-cloud-shared-dependencies to v0.6.0 ([#​325](https://www.github.com/googleapis/java-bigtable/issues/325)) ([6fc5b7b](https://www.github.com/googleapis/java-bigtable/commit/6fc5b7b1f13c5326001cc7bcecbf4918a8505d26)) - update dependency com.google.cloud:google-cloud-shared-dependencies to v0.7.0 ([#​326](https://www.github.com/googleapis/java-bigtable/issues/326)) ([2aab7a8](https://www.github.com/googleapis/java-bigtable/commit/2aab7a8ec8775ab67e0277fd1785ee4735e1521d)) - update dependency com.google.cloud:google-cloud-shared-dependencies to v0.8.1 ([#​343](https://www.github.com/googleapis/java-bigtable/issues/343)) ([f0314a1](https://www.github.com/googleapis/java-bigtable/commit/f0314a13841cff495cc3424805a0aaf19bcc71d0)) - update dependency com.google.cloud:google-cloud-shared-dependencies to v0.8.2 ([#​362](https://www.github.com/googleapis/java-bigtable/issues/362)) ([1c37f5d](https://www.github.com/googleapis/java-bigtable/commit/1c37f5df35e4a317ac339e5d5b3cd50acf8e8c5a)) - update dependency com.google.cloud:google-cloud-shared-dependencies to v0.8.3 ([#​366](https://www.github.com/googleapis/java-bigtable/issues/366)) ([ade8b36](https://www.github.com/googleapis/java-bigtable/commit/ade8b362d66378e6b1ba75c6c04558e4437080a9)) - update dependency com.google.errorprone:error_prone_annotations to v2.4.0 ([#​328](https://www.github.com/googleapis/java-bigtable/issues/328)) ([b9e5e83](https://www.github.com/googleapis/java-bigtable/commit/b9e5e832791b728d5931e1fe4546d01ffb4903d1)) - update protobuf.version to v3.12.2 ([#​320](https://www.github.com/googleapis/java-bigtable/issues/320)) ([d5cf1d4](https://www.github.com/googleapis/java-bigtable/commit/d5cf1d4a62f9782c18a2d7ebb48d137426f84d5b)) - use google-cloud-shared-dependencies in bigtable deps bom ([#​324](https://www.github.com/googleapis/java-bigtable/issues/324)) ([8f76fc8](https://www.github.com/googleapis/java-bigtable/commit/8f76fc8e7d326ed7837070cf496a2f20d4c7485e))
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#googleapis/java-bigtable). --- samples/install-without-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index b22bcd33e6..f1b8e7e961 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -29,7 +29,7 @@ com.google.cloud google-cloud-bigtable - 1.13.1 + 1.14.0 From 563bd47f76e20b308b8db1491f60618e8cfe5aee Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2020 15:31:37 -0400 Subject: [PATCH 12/30] chore: release 1.14.1-SNAPSHOT (#373) * updated versions.txt [ci skip] * updated samples/pom.xml [ci skip] * updated samples/install-without-bom/pom.xml [ci skip] * updated samples/snippets/pom.xml [ci skip] * updated google-cloud-bigtable-bom/pom.xml [ci skip] * updated google-cloud-bigtable-deps-bom/pom.xml [ci skip] * updated google-cloud-bigtable-emulator/pom.xml [ci skip] * updated google-cloud-bigtable/pom.xml [ci skip] * updated grpc-google-cloud-bigtable-admin-v2/pom.xml [ci skip] * updated grpc-google-cloud-bigtable-v2/pom.xml [ci skip] * updated pom.xml [ci skip] * updated proto-google-cloud-bigtable-admin-v2/pom.xml [ci skip] * updated proto-google-cloud-bigtable-v2/pom.xml [ci skip] * updated samples/snapshot/pom.xml Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- google-cloud-bigtable-bom/pom.xml | 14 +++++++------- google-cloud-bigtable-deps-bom/pom.xml | 2 +- google-cloud-bigtable-emulator/pom.xml | 8 ++++---- google-cloud-bigtable/pom.xml | 8 ++++---- grpc-google-cloud-bigtable-admin-v2/pom.xml | 8 ++++---- grpc-google-cloud-bigtable-v2/pom.xml | 8 ++++---- pom.xml | 2 +- proto-google-cloud-bigtable-admin-v2/pom.xml | 8 ++++---- proto-google-cloud-bigtable-v2/pom.xml | 8 ++++---- samples/snapshot/pom.xml | 2 +- versions.txt | 12 ++++++------ 11 files changed, 40 insertions(+), 40 deletions(-) diff --git a/google-cloud-bigtable-bom/pom.xml b/google-cloud-bigtable-bom/pom.xml index 3b77068e6d..52a799c063 100644 --- a/google-cloud-bigtable-bom/pom.xml +++ b/google-cloud-bigtable-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigtable-bom - 1.14.0 + 1.14.1-SNAPSHOT pom com.google.cloud @@ -72,32 +72,32 @@ com.google.cloud google-cloud-bigtable - 1.14.0 + 1.14.1-SNAPSHOT com.google.cloud google-cloud-bigtable-emulator - 0.123.0 + 0.123.1-SNAPSHOT com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 1.14.0 + 1.14.1-SNAPSHOT com.google.api.grpc grpc-google-cloud-bigtable-v2 - 1.14.0 + 1.14.1-SNAPSHOT com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 1.14.0 + 1.14.1-SNAPSHOT com.google.api.grpc proto-google-cloud-bigtable-v2 - 1.14.0 + 1.14.1-SNAPSHOT diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml index f8f21d02b1..16102e6c84 100644 --- a/google-cloud-bigtable-deps-bom/pom.xml +++ b/google-cloud-bigtable-deps-bom/pom.xml @@ -12,7 +12,7 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.0 + 1.14.1-SNAPSHOT pom diff --git a/google-cloud-bigtable-emulator/pom.xml b/google-cloud-bigtable-emulator/pom.xml index 031100b318..42884b7786 100644 --- a/google-cloud-bigtable-emulator/pom.xml +++ b/google-cloud-bigtable-emulator/pom.xml @@ -5,7 +5,7 @@ 4.0.0 google-cloud-bigtable-emulator - 0.123.0 + 0.123.1-SNAPSHOT Google Cloud Java - Bigtable Emulator https://github.com/googleapis/java-bigtable @@ -14,7 +14,7 @@ com.google.cloud google-cloud-bigtable-parent - 1.14.0 + 1.14.1-SNAPSHOT scm:git:git@github.com:googleapis/java-bigtable.git @@ -80,14 +80,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.0 + 1.14.1-SNAPSHOT pom import com.google.cloud google-cloud-bigtable-bom - 1.14.0 + 1.14.1-SNAPSHOT pom import diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index be20395a64..90f607726f 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-bigtable - 1.14.0 + 1.14.1-SNAPSHOT jar Google Cloud Bigtable https://github.com/googleapis/java-bigtable @@ -12,7 +12,7 @@ com.google.cloud google-cloud-bigtable-parent - 1.14.0 + 1.14.1-SNAPSHOT google-cloud-bigtable @@ -36,14 +36,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.0 + 1.14.1-SNAPSHOT pom import com.google.cloud google-cloud-bigtable-bom - 1.14.0 + 1.14.1-SNAPSHOT pom import diff --git a/grpc-google-cloud-bigtable-admin-v2/pom.xml b/grpc-google-cloud-bigtable-admin-v2/pom.xml index 6b8e663b9a..3a05f1209c 100644 --- a/grpc-google-cloud-bigtable-admin-v2/pom.xml +++ b/grpc-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 1.14.0 + 1.14.1-SNAPSHOT grpc-google-cloud-bigtable-admin-v2 GRPC library for grpc-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 1.14.0 + 1.14.1-SNAPSHOT @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.0 + 1.14.1-SNAPSHOT pom import com.google.cloud google-cloud-bigtable-bom - 1.14.0 + 1.14.1-SNAPSHOT pom import diff --git a/grpc-google-cloud-bigtable-v2/pom.xml b/grpc-google-cloud-bigtable-v2/pom.xml index 3574b668cf..ff675b8d81 100644 --- a/grpc-google-cloud-bigtable-v2/pom.xml +++ b/grpc-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 1.14.0 + 1.14.1-SNAPSHOT grpc-google-cloud-bigtable-v2 GRPC library for grpc-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 1.14.0 + 1.14.1-SNAPSHOT @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.0 + 1.14.1-SNAPSHOT pom import com.google.cloud google-cloud-bigtable-bom - 1.14.0 + 1.14.1-SNAPSHOT pom import diff --git a/pom.xml b/pom.xml index 9d0b57aa45..b39880dece 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ google-cloud-bigtable-parent pom - 1.14.0 + 1.14.1-SNAPSHOT Google Cloud Bigtable Parent https://github.com/googleapis/java-bigtable diff --git a/proto-google-cloud-bigtable-admin-v2/pom.xml b/proto-google-cloud-bigtable-admin-v2/pom.xml index 778237ef87..164d80f9f5 100644 --- a/proto-google-cloud-bigtable-admin-v2/pom.xml +++ b/proto-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 1.14.0 + 1.14.1-SNAPSHOT proto-google-cloud-bigtable-admin-v2 PROTO library for proto-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 1.14.0 + 1.14.1-SNAPSHOT @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.0 + 1.14.1-SNAPSHOT pom import com.google.cloud google-cloud-bigtable-bom - 1.14.0 + 1.14.1-SNAPSHOT pom import diff --git a/proto-google-cloud-bigtable-v2/pom.xml b/proto-google-cloud-bigtable-v2/pom.xml index 7f8628551c..7fccaf4a1a 100644 --- a/proto-google-cloud-bigtable-v2/pom.xml +++ b/proto-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 1.14.0 + 1.14.1-SNAPSHOT proto-google-cloud-bigtable-v2 PROTO library for proto-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 1.14.0 + 1.14.1-SNAPSHOT @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.0 + 1.14.1-SNAPSHOT pom import com.google.cloud google-cloud-bigtable-bom - 1.14.0 + 1.14.1-SNAPSHOT pom import diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index f5aa3a83e6..879ceb5de4 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-bigtable - 1.14.0 + 1.14.1-SNAPSHOT diff --git a/versions.txt b/versions.txt index 60222c86f4..c5e884ba98 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-bigtable:1.14.0:1.14.0 -grpc-google-cloud-bigtable-admin-v2:1.14.0:1.14.0 -grpc-google-cloud-bigtable-v2:1.14.0:1.14.0 -proto-google-cloud-bigtable-admin-v2:1.14.0:1.14.0 -proto-google-cloud-bigtable-v2:1.14.0:1.14.0 -google-cloud-bigtable-emulator:0.123.0:0.123.0 +google-cloud-bigtable:1.14.0:1.14.1-SNAPSHOT +grpc-google-cloud-bigtable-admin-v2:1.14.0:1.14.1-SNAPSHOT +grpc-google-cloud-bigtable-v2:1.14.0:1.14.1-SNAPSHOT +proto-google-cloud-bigtable-admin-v2:1.14.0:1.14.1-SNAPSHOT +proto-google-cloud-bigtable-v2:1.14.0:1.14.1-SNAPSHOT +google-cloud-bigtable-emulator:0.123.0:0.123.1-SNAPSHOT From 33bbcbfc0a2a0d8ae30c20968d0207f3e06b3761 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Mon, 3 Aug 2020 08:49:12 -0700 Subject: [PATCH 13/30] Add cloud-RAD for Java (#374) Co-authored-by: Jeff Ching Source-Author: Les Vogel Source-Date: Thu Jul 30 13:09:50 2020 -0700 Source-Repo: googleapis/synthtool Source-Sha: dd230c816f88d0141fcd0be83498986287220d1b Source-Link: https://github.com/googleapis/synthtool/commit/dd230c816f88d0141fcd0be83498986287220d1b --- .kokoro/release/publish_javadoc.cfg | 10 + .kokoro/release/publish_javadoc.sh | 25 ++- synth.metadata | 298 +++++++++++++++++++++++++++- 3 files changed, 329 insertions(+), 4 deletions(-) diff --git a/.kokoro/release/publish_javadoc.cfg b/.kokoro/release/publish_javadoc.cfg index 6f63399325..3707db3caa 100644 --- a/.kokoro/release/publish_javadoc.cfg +++ b/.kokoro/release/publish_javadoc.cfg @@ -1,14 +1,24 @@ # Format: //devtools/kokoro/config/proto/build.proto + +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/doc-templates/" + env_vars: { key: "STAGING_BUCKET" value: "docs-staging" } +env_vars: { + key: "STAGING_BUCKET_V2" + value: "docs-staging-v2-staging" + # Production will be at: docs-staging-v2 +} + env_vars: { key: "TRAMPOLINE_BUILD_FILE" value: "github/java-bigtable/.kokoro/release/publish_javadoc.sh" } + before_action { fetch_keystore { keystore_resource { diff --git a/.kokoro/release/publish_javadoc.sh b/.kokoro/release/publish_javadoc.sh index aea31758fc..3d624a7dcc 100755 --- a/.kokoro/release/publish_javadoc.sh +++ b/.kokoro/release/publish_javadoc.sh @@ -24,6 +24,11 @@ if [[ -z "${STAGING_BUCKET}" ]]; then exit 1 fi +if [[ -z "${STAGING_BUCKET_V2}" ]]; then + echo "Need to set STAGING_BUCKET_V2 environment variable" + exit 1 +fi + # work from the git root directory pushd $(dirname "$0")/../../ @@ -31,13 +36,13 @@ pushd $(dirname "$0")/../../ python3 -m pip install gcp-docuploader # compile all packages -mvn clean install -B -DskipTests=true +mvn clean install -B -q -DskipTests=true NAME=google-cloud-bigtable VERSION=$(grep ${NAME}: versions.txt | cut -d: -f3) # build the docs -mvn site -B +mvn site -B -q pushd target/site/apidocs @@ -53,3 +58,19 @@ python3 -m docuploader upload . \ --staging-bucket ${STAGING_BUCKET} popd + +# V2 +mvn clean site -B -q -Ddevsite.template="${KOKORO_GFILE_DIR}/java/" + +pushd target/devsite + +# create metadata +python3 -m docuploader create-metadata \ + --name ${NAME} \ + --version ${VERSION} \ + --language java + +# upload docs +python3 -m docuploader upload . \ + --credentials ${CREDENTIALS} \ + --staging-bucket ${STAGING_BUCKET_V2} diff --git a/synth.metadata b/synth.metadata index 86b0335063..645bcc5970 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-bigtable.git", - "sha": "b4edbd5cf420c652dec71ecccc236dbe823cfa4c" + "sha": "563bd47f76e20b308b8db1491f60618e8cfe5aee" } }, { @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "4f2c9f752a94042472fc03c5bd9e06e89817d2bd" + "sha": "dd230c816f88d0141fcd0be83498986287220d1b" } } ], @@ -42,5 +42,299 @@ "generator": "bazel" } } + ], + "generatedFiles": [ + ".github/CODEOWNERS", + ".github/ISSUE_TEMPLATE/feature_request.md", + ".github/ISSUE_TEMPLATE/support_request.md", + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/release-please.yml", + ".github/trusted-contribution.yml", + ".github/workflows/ci.yaml", + ".kokoro/build.bat", + ".kokoro/build.sh", + ".kokoro/coerce_logs.sh", + ".kokoro/common.cfg", + ".kokoro/common.sh", + ".kokoro/continuous/common.cfg", + ".kokoro/continuous/dependencies.cfg", + ".kokoro/continuous/integration.cfg", + ".kokoro/continuous/java11.cfg", + ".kokoro/continuous/java7.cfg", + ".kokoro/continuous/java8-osx.cfg", + ".kokoro/continuous/java8-win.cfg", + ".kokoro/continuous/java8.cfg", + ".kokoro/continuous/lint.cfg", + ".kokoro/continuous/propose_release.cfg", + ".kokoro/continuous/samples.cfg", + ".kokoro/dependencies.sh", + ".kokoro/linkage-monitor.sh", + ".kokoro/nightly/common.cfg", + ".kokoro/nightly/dependencies.cfg", + ".kokoro/nightly/java11.cfg", + ".kokoro/nightly/java7.cfg", + ".kokoro/nightly/java8-osx.cfg", + ".kokoro/nightly/java8-win.cfg", + ".kokoro/nightly/java8.cfg", + ".kokoro/nightly/lint.cfg", + ".kokoro/nightly/samples.cfg", + ".kokoro/populate-secrets.sh", + ".kokoro/presubmit/clirr.cfg", + ".kokoro/presubmit/common.cfg", + ".kokoro/presubmit/dependencies.cfg", + ".kokoro/presubmit/java11.cfg", + ".kokoro/presubmit/java7.cfg", + ".kokoro/presubmit/java8-osx.cfg", + ".kokoro/presubmit/java8-win.cfg", + ".kokoro/presubmit/java8.cfg", + ".kokoro/presubmit/linkage-monitor.cfg", + ".kokoro/presubmit/lint.cfg", + ".kokoro/presubmit/samples.cfg", + ".kokoro/release/bump_snapshot.cfg", + ".kokoro/release/common.cfg", + ".kokoro/release/common.sh", + ".kokoro/release/drop.cfg", + ".kokoro/release/drop.sh", + ".kokoro/release/promote.cfg", + ".kokoro/release/promote.sh", + ".kokoro/release/publish_javadoc.cfg", + ".kokoro/release/publish_javadoc.sh", + ".kokoro/release/snapshot.cfg", + ".kokoro/release/snapshot.sh", + ".kokoro/release/stage.cfg", + ".kokoro/release/stage.sh", + ".kokoro/trampoline.sh", + "CODE_OF_CONDUCT.md", + "LICENSE", + "codecov.yaml", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BaseBigtableInstanceAdminClient.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BaseBigtableInstanceAdminSettings.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BaseBigtableTableAdminClient.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BaseBigtableTableAdminSettings.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableInstanceAdminStub.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableInstanceAdminStubSettings.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableTableAdminStub.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/BigtableTableAdminStubSettings.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/GrpcBigtableInstanceAdminCallableFactory.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/GrpcBigtableInstanceAdminStub.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/GrpcBigtableTableAdminCallableFactory.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/GrpcBigtableTableAdminStub.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableStub.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableStubSettings.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/GrpcBigtableCallableFactory.java", + "google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/GrpcBigtableStub.java", + "google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BaseBigtableInstanceAdminClientTest.java", + "google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BaseBigtableTableAdminClientTest.java", + "google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/MockBigtableInstanceAdmin.java", + "google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/MockBigtableInstanceAdminImpl.java", + "google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/MockBigtableTableAdmin.java", + "google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/MockBigtableTableAdminImpl.java", + "grpc-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/BigtableInstanceAdminGrpc.java", + "grpc-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/BigtableTableAdminGrpc.java", + "grpc-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/BigtableGrpc.java", + "java.header", + "license-checks.xml", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/AppProfile.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/AppProfileName.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/AppProfileOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/Backup.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/BackupInfo.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/BackupInfoOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/BackupName.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/BackupOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/BigtableInstanceAdminProto.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/BigtableTableAdminProto.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CheckConsistencyRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CheckConsistencyRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CheckConsistencyResponse.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CheckConsistencyResponseOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/Cluster.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ClusterName.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ClusterOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ColumnFamily.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ColumnFamilyOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CommonProto.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateAppProfileRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateAppProfileRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateBackupMetadata.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateBackupMetadataOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateBackupRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateBackupRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateClusterMetadata.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateClusterMetadataOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateClusterRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateClusterRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateInstanceMetadata.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateInstanceMetadataOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateInstanceRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateInstanceRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateTableFromSnapshotMetadata.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateTableFromSnapshotMetadataOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateTableFromSnapshotRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateTableFromSnapshotRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateTableRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/CreateTableRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DeleteAppProfileRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DeleteAppProfileRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DeleteBackupRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DeleteBackupRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DeleteClusterRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DeleteClusterRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DeleteInstanceRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DeleteInstanceRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DeleteSnapshotRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DeleteSnapshotRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DeleteTableRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DeleteTableRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DropRowRangeRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/DropRowRangeRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GcRule.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GcRuleOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GenerateConsistencyTokenRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GenerateConsistencyTokenRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GenerateConsistencyTokenResponse.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GenerateConsistencyTokenResponseOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GetAppProfileRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GetAppProfileRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GetBackupRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GetBackupRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GetClusterRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GetClusterRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GetInstanceRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GetInstanceRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GetSnapshotRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GetSnapshotRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GetTableRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/GetTableRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/Instance.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/InstanceName.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/InstanceOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/InstanceProto.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListAppProfilesRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListAppProfilesRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListAppProfilesResponse.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListAppProfilesResponseOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListBackupsRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListBackupsRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListBackupsResponse.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListBackupsResponseOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListClustersRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListClustersRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListClustersResponse.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListClustersResponseOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListInstancesRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListInstancesRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListInstancesResponse.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListInstancesResponseOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListSnapshotsRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListSnapshotsRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListSnapshotsResponse.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListSnapshotsResponseOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListTablesRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListTablesRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListTablesResponse.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ListTablesResponseOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/LocationName.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ModifyColumnFamiliesRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ModifyColumnFamiliesRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/OperationProgress.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/OperationProgressOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/OptimizeRestoredTableMetadata.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/OptimizeRestoredTableMetadataOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/PartialUpdateInstanceRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/PartialUpdateInstanceRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/ProjectName.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/RestoreInfo.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/RestoreInfoOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/RestoreSourceType.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/RestoreTableMetadata.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/RestoreTableMetadataOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/RestoreTableRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/RestoreTableRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/Snapshot.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/SnapshotName.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/SnapshotOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/SnapshotTableMetadata.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/SnapshotTableMetadataOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/SnapshotTableRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/SnapshotTableRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/StorageType.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/Table.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/TableName.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/TableOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/TableProto.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/UpdateAppProfileMetadata.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/UpdateAppProfileMetadataOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/UpdateAppProfileRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/UpdateAppProfileRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/UpdateBackupRequest.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/UpdateBackupRequestOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/UpdateClusterMetadata.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/UpdateClusterMetadataOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/UpdateInstanceMetadata.java", + "proto-google-cloud-bigtable-admin-v2/src/main/java/com/google/bigtable/admin/v2/UpdateInstanceMetadataOrBuilder.java", + "proto-google-cloud-bigtable-admin-v2/src/main/proto/google/bigtable/admin/v2/bigtable_instance_admin.proto", + "proto-google-cloud-bigtable-admin-v2/src/main/proto/google/bigtable/admin/v2/bigtable_table_admin.proto", + "proto-google-cloud-bigtable-admin-v2/src/main/proto/google/bigtable/admin/v2/common.proto", + "proto-google-cloud-bigtable-admin-v2/src/main/proto/google/bigtable/admin/v2/instance.proto", + "proto-google-cloud-bigtable-admin-v2/src/main/proto/google/bigtable/admin/v2/table.proto", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/BigtableProto.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/Cell.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/CellOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/CheckAndMutateRowRequest.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/CheckAndMutateRowRequestOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/CheckAndMutateRowResponse.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/CheckAndMutateRowResponseOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/Column.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ColumnOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ColumnRange.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ColumnRangeOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/DataProto.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/Family.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/FamilyOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/MutateRowRequest.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/MutateRowRequestOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/MutateRowResponse.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/MutateRowResponseOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/MutateRowsRequest.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/MutateRowsRequestOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/MutateRowsResponse.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/MutateRowsResponseOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/Mutation.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/MutationOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ReadModifyWriteRowRequest.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ReadModifyWriteRowRequestOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ReadModifyWriteRowResponse.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ReadModifyWriteRowResponseOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ReadModifyWriteRule.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ReadModifyWriteRuleOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ReadRowsRequest.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ReadRowsRequestOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ReadRowsResponse.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ReadRowsResponseOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/Row.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/RowFilter.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/RowFilterOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/RowOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/RowRange.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/RowRangeOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/RowSet.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/RowSetOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/SampleRowKeysRequest.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/SampleRowKeysRequestOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/SampleRowKeysResponse.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/SampleRowKeysResponseOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/TableName.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/TimestampRange.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/TimestampRangeOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ValueRange.java", + "proto-google-cloud-bigtable-v2/src/main/java/com/google/bigtable/v2/ValueRangeOrBuilder.java", + "proto-google-cloud-bigtable-v2/src/main/proto/google/bigtable/v2/bigtable.proto", + "proto-google-cloud-bigtable-v2/src/main/proto/google/bigtable/v2/data.proto", + "renovate.json", + "samples/install-without-bom/pom.xml", + "samples/pom.xml", + "samples/snapshot/pom.xml", + "samples/snippets/pom.xml" ] } \ No newline at end of file From 950d224fff63ef8c87a6604c436d3e144f8f28b3 Mon Sep 17 00:00:00 2001 From: Weiran Fang <8175562+WeiranFang@users.noreply.github.com> Date: Wed, 5 Aug 2020 09:49:22 -0700 Subject: [PATCH 14/30] chore: Introspect peer IP address for directpath tests (#371) * Add peer IP check for directpath tests * remove atomic reference * fix lint * make failure message more explicit * string format fix * fix grpc-log-dir and summaryFile --- google-cloud-bigtable/pom.xml | 38 ++++++++ .../test_helpers/env/AbstractTestEnv.java | 4 + .../bigtable/test_helpers/env/CloudEnv.java | 93 +++++++++++++++++++ 3 files changed, 135 insertions(+) diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index 90f607726f..119624fd61 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -338,6 +338,44 @@ + + + bigtable-directpath-ipv4-it + + + + maven-failsafe-plugin + + + directpath-it + + integration-test + verify + + + false + + + cloud + ${bigtable.directpath-data-endpoint} + ${bigtable.directpath-admin-endpoint} + ${project.build.directory}/test-grpc-logs/directpath-ipv4-it + true + true + + + + com.google.cloud.bigtable.data.v2.it.*IT + + ${project.build.directory}/failsafe-reports/failsafe-summary-directpath-ipv4-it.xml + ${project.build.directory}/failsafe-reports/directpath-ipv4-it + + + + + + + diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java index 7f9bfe9b87..492e94e9bd 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java @@ -78,6 +78,10 @@ public boolean isDirectPathEnabled() { return Boolean.getBoolean("bigtable.attempt-directpath"); } + public boolean isDirectPathIpv4() { + return Boolean.getBoolean("bigtable.directpath-ipv4"); + } + public String getPrimaryZone() { return "us-central1-b"; } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/CloudEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/CloudEnv.java index bee4dfb9d2..87aab527e5 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/CloudEnv.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/CloudEnv.java @@ -15,6 +15,9 @@ */ package com.google.cloud.bigtable.test_helpers.env; +import com.google.api.core.ApiFunction; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; +import com.google.api.gax.rpc.TransportChannelProvider; import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; @@ -23,7 +26,20 @@ import com.google.cloud.bigtable.data.v2.BigtableDataSettings; import com.google.common.base.MoreObjects; import com.google.common.base.Strings; +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ForwardingClientCall.SimpleForwardingClientCall; +import io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener; +import io.grpc.Grpc; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; import javax.annotation.Nullable; /** @@ -37,6 +53,10 @@ * */ class CloudEnv extends AbstractTestEnv { + // IP address prefixes allocated for DirectPath backends. + public static final String DP_IPV6_PREFIX = "2001:4860:8040"; + public static final String DP_IPV4_PREFIX = "34.126"; + private static final String DATA_ENDPOINT_PROPERTY_NAME = "bigtable.data-endpoint"; private static final String ADMIN_ENDPOINT_PROPERTY_NAME = "bigtable.admin-endpoint"; @@ -81,6 +101,26 @@ private CloudEnv( dataSettings.stubSettings().setEndpoint(dataEndpoint); } + if (isDirectPathEnabled()) { + TransportChannelProvider channelProvider = + dataSettings.stubSettings().getTransportChannelProvider(); + InstantiatingGrpcChannelProvider defaultTransportProvider = + (InstantiatingGrpcChannelProvider) channelProvider; + InstantiatingGrpcChannelProvider instrumentedTransportChannelProvider = + defaultTransportProvider + .toBuilder() + .setChannelConfigurator( + new ApiFunction() { + @Override + public ManagedChannelBuilder apply(ManagedChannelBuilder builder) { + builder.intercept(directPathAddressCheckInterceptor()); + return builder; + } + }) + .build(); + dataSettings.stubSettings().setTransportChannelProvider(instrumentedTransportChannelProvider); + } + this.tableAdminSettings = BigtableTableAdminSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId); if (!Strings.isNullOrEmpty(adminEndpoint)) { @@ -153,4 +193,57 @@ private static String getRequiredProperty(String prop) { } return value; } + + /** + * Captures the request attributes "Grpc.TRANSPORT_ATTR_REMOTE_ADDR" when connection is + * established and verifies if the remote address is a DirectPath address. This is only used for + * DirectPath testing. {@link ClientCall#getAttributes()} + */ + private ClientInterceptor directPathAddressCheckInterceptor() { + return new ClientInterceptor() { + @Override + public ClientCall interceptCall( + MethodDescriptor method, CallOptions callOptions, Channel next) { + final ClientCall clientCall = next.newCall(method, callOptions); + return new SimpleForwardingClientCall(clientCall) { + @Override + public void start(Listener responseListener, Metadata headers) { + super.start( + new SimpleForwardingClientCallListener(responseListener) { + @Override + public void onHeaders(Metadata headers) { + // Check peer IP after connection is established. + SocketAddress remoteAddr = + clientCall.getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR); + if (!verifyRemoteAddress(remoteAddr)) { + throw new RuntimeException( + String.format( + "Synthetically aborting the current request because it did not adhere" + + " to the test environment's requirement for DirectPath." + + " Expected test to access DirectPath via %s," + + " but RPC was destined for %s", + isDirectPathIpv4() ? "ipv4" : "ipv6", remoteAddr.toString())); + } + super.onHeaders(headers); + } + }, + headers); + } + }; + } + }; + } + + private boolean verifyRemoteAddress(SocketAddress remoteAddr) { + if (remoteAddr instanceof InetSocketAddress) { + InetAddress inetAddress = ((InetSocketAddress) remoteAddr).getAddress(); + String addr = inetAddress.getHostAddress(); + if (isDirectPathIpv4()) { + return addr.startsWith(DP_IPV4_PREFIX); + } else { + return addr.startsWith(DP_IPV6_PREFIX); + } + } + return true; + } } From 37c38bd34405e0f4fc4b08f2a43f614bb869309e Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Thu, 6 Aug 2020 10:54:59 -0700 Subject: [PATCH 15/30] cleanup: removes unused kokoro config files (#379) * cleanup: removes unused kokoro config files Removes unused kokoro files from the java library template. We have stopped running some of these due to Github quota issues. * fix:reverts back samples.cfg files The files presubmit/samples.cfg and nightly/samples.cfg should remain in the java template repository. Co-authored-by: Jeffrey Rennie Source-Author: Thiago Nunes Source-Date: Thu Aug 6 09:48:58 2020 +1000 Source-Repo: googleapis/synthtool Source-Sha: 4530cc6ff080ef8aca258c1ec92c4db10a1bbfb4 Source-Link: https://github.com/googleapis/synthtool/commit/4530cc6ff080ef8aca258c1ec92c4db10a1bbfb4 --- .kokoro/continuous/dependencies.cfg | 12 ------ .kokoro/continuous/integration.cfg | 7 ---- .kokoro/continuous/java11.cfg | 7 ---- .kokoro/continuous/java7.cfg | 7 ---- .kokoro/continuous/java8-osx.cfg | 3 -- .kokoro/continuous/java8-win.cfg | 3 -- .kokoro/continuous/lint.cfg | 13 ------- .kokoro/continuous/propose_release.cfg | 53 -------------------------- .kokoro/continuous/samples.cfg | 31 --------------- .kokoro/nightly/dependencies.cfg | 12 ------ .kokoro/nightly/lint.cfg | 13 ------- synth.metadata | 19 ++------- 12 files changed, 4 insertions(+), 176 deletions(-) delete mode 100644 .kokoro/continuous/dependencies.cfg delete mode 100644 .kokoro/continuous/integration.cfg delete mode 100644 .kokoro/continuous/java11.cfg delete mode 100644 .kokoro/continuous/java7.cfg delete mode 100644 .kokoro/continuous/java8-osx.cfg delete mode 100644 .kokoro/continuous/java8-win.cfg delete mode 100644 .kokoro/continuous/lint.cfg delete mode 100644 .kokoro/continuous/propose_release.cfg delete mode 100644 .kokoro/continuous/samples.cfg delete mode 100644 .kokoro/nightly/dependencies.cfg delete mode 100644 .kokoro/nightly/lint.cfg diff --git a/.kokoro/continuous/dependencies.cfg b/.kokoro/continuous/dependencies.cfg deleted file mode 100644 index b74da3741e..0000000000 --- a/.kokoro/continuous/dependencies.cfg +++ /dev/null @@ -1,12 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/java8" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/java-bigtable/.kokoro/dependencies.sh" -} diff --git a/.kokoro/continuous/integration.cfg b/.kokoro/continuous/integration.cfg deleted file mode 100644 index 3b017fc80f..0000000000 --- a/.kokoro/continuous/integration.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/java8" -} diff --git a/.kokoro/continuous/java11.cfg b/.kokoro/continuous/java11.cfg deleted file mode 100644 index 709f2b4c73..0000000000 --- a/.kokoro/continuous/java11.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/java11" -} diff --git a/.kokoro/continuous/java7.cfg b/.kokoro/continuous/java7.cfg deleted file mode 100644 index cb24f44eea..0000000000 --- a/.kokoro/continuous/java7.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/java7" -} diff --git a/.kokoro/continuous/java8-osx.cfg b/.kokoro/continuous/java8-osx.cfg deleted file mode 100644 index 795232c40a..0000000000 --- a/.kokoro/continuous/java8-osx.cfg +++ /dev/null @@ -1,3 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -build_file: "java-bigtable/.kokoro/build.sh" diff --git a/.kokoro/continuous/java8-win.cfg b/.kokoro/continuous/java8-win.cfg deleted file mode 100644 index 8e491f24cc..0000000000 --- a/.kokoro/continuous/java8-win.cfg +++ /dev/null @@ -1,3 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -build_file: "java-bigtable/.kokoro/build.bat" diff --git a/.kokoro/continuous/lint.cfg b/.kokoro/continuous/lint.cfg deleted file mode 100644 index 6d323c8ae7..0000000000 --- a/.kokoro/continuous/lint.cfg +++ /dev/null @@ -1,13 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Configure the docker image for kokoro-trampoline. - -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/java8" -} - -env_vars: { - key: "JOB_TYPE" - value: "lint" -} \ No newline at end of file diff --git a/.kokoro/continuous/propose_release.cfg b/.kokoro/continuous/propose_release.cfg deleted file mode 100644 index 5fc40910b3..0000000000 --- a/.kokoro/continuous/propose_release.cfg +++ /dev/null @@ -1,53 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "java-bigtable/.kokoro/trampoline.sh" - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/node:10-user" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/java-bigtable/.kokoro/continuous/propose_release.sh" -} - -# tokens used by release-please to keep an up-to-date release PR. -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "github-magic-proxy-key-release-please" - } - } -} - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "github-magic-proxy-token-release-please" - } - } -} - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "github-magic-proxy-url-release-please" - } - } -} diff --git a/.kokoro/continuous/samples.cfg b/.kokoro/continuous/samples.cfg deleted file mode 100644 index fa7b493d0b..0000000000 --- a/.kokoro/continuous/samples.cfg +++ /dev/null @@ -1,31 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/java8" -} - -env_vars: { - key: "JOB_TYPE" - value: "samples" -} - -env_vars: { - key: "GCLOUD_PROJECT" - value: "gcloud-devel" -} - -env_vars: { - key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "keystore/73713_java_it_service_account" -} - -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "java_it_service_account" - } - } -} diff --git a/.kokoro/nightly/dependencies.cfg b/.kokoro/nightly/dependencies.cfg deleted file mode 100644 index b74da3741e..0000000000 --- a/.kokoro/nightly/dependencies.cfg +++ /dev/null @@ -1,12 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/java8" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/java-bigtable/.kokoro/dependencies.sh" -} diff --git a/.kokoro/nightly/lint.cfg b/.kokoro/nightly/lint.cfg deleted file mode 100644 index 6d323c8ae7..0000000000 --- a/.kokoro/nightly/lint.cfg +++ /dev/null @@ -1,13 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Configure the docker image for kokoro-trampoline. - -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/java8" -} - -env_vars: { - key: "JOB_TYPE" - value: "lint" -} \ No newline at end of file diff --git a/synth.metadata b/synth.metadata index 645bcc5970..e85c685d80 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,22 +4,22 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-bigtable.git", - "sha": "563bd47f76e20b308b8db1491f60618e8cfe5aee" + "sha": "950d224fff63ef8c87a6604c436d3e144f8f28b3" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "94fe3637559a257634d7b47a15bb8d976daff788", - "internalRef": "316039767" + "sha": "96ef637adf148d54236ad83fefa30c7f75f29737", + "internalRef": "325052820" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "dd230c816f88d0141fcd0be83498986287220d1b" + "sha": "4530cc6ff080ef8aca258c1ec92c4db10a1bbfb4" } } ], @@ -57,26 +57,15 @@ ".kokoro/common.cfg", ".kokoro/common.sh", ".kokoro/continuous/common.cfg", - ".kokoro/continuous/dependencies.cfg", - ".kokoro/continuous/integration.cfg", - ".kokoro/continuous/java11.cfg", - ".kokoro/continuous/java7.cfg", - ".kokoro/continuous/java8-osx.cfg", - ".kokoro/continuous/java8-win.cfg", ".kokoro/continuous/java8.cfg", - ".kokoro/continuous/lint.cfg", - ".kokoro/continuous/propose_release.cfg", - ".kokoro/continuous/samples.cfg", ".kokoro/dependencies.sh", ".kokoro/linkage-monitor.sh", ".kokoro/nightly/common.cfg", - ".kokoro/nightly/dependencies.cfg", ".kokoro/nightly/java11.cfg", ".kokoro/nightly/java7.cfg", ".kokoro/nightly/java8-osx.cfg", ".kokoro/nightly/java8-win.cfg", ".kokoro/nightly/java8.cfg", - ".kokoro/nightly/lint.cfg", ".kokoro/nightly/samples.cfg", ".kokoro/populate-secrets.sh", ".kokoro/presubmit/clirr.cfg", From f0765d1ab7088ed1b64ad008925569cda8ab3cc6 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Fri, 7 Aug 2020 08:29:23 -0700 Subject: [PATCH 16/30] build: update dependencies check to only check for runtime and compile scopes (#381) * change:Updated dependencies check to only use runtime & compile scope * Update dependencies.sh * feat: update dependencies check to only check for runtime and compile scopes Co-authored-by: Saleh Mostafa Co-authored-by: Jeffrey Rennie Source-Author: salehsquared Source-Date: Thu Aug 6 13:01:02 2020 -0400 Source-Repo: googleapis/synthtool Source-Sha: f8823dec98277a9516f2fb6fae9f58b3a59a23e1 Source-Link: https://github.com/googleapis/synthtool/commit/f8823dec98277a9516f2fb6fae9f58b3a59a23e1 --- .kokoro/dependencies.sh | 9 +++++---- synth.metadata | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.kokoro/dependencies.sh b/.kokoro/dependencies.sh index cee4f11e75..c91e5a5693 100755 --- a/.kokoro/dependencies.sh +++ b/.kokoro/dependencies.sh @@ -43,12 +43,13 @@ function completenessCheck() { # Output dep list with compile scope generated using the original pom # Running mvn dependency:list on Java versions that support modules will also include the module of the dependency. # This is stripped from the output as it is not present in the flattened pom. + # Only dependencies with 'compile' or 'runtime' scope are included from original dependency list. msg "Generating dependency list using original pom..." - mvn dependency:list -f pom.xml -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' | sed -e s/\\s--\\smodule.*// | grep -v ':test$' >.org-list.txt + mvn dependency:list -f pom.xml -DincludeScope=runtime -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' | sed -e s/\\s--\\smodule.*// >.org-list.txt - # Output dep list generated using the flattened pom (test scope deps are ommitted) + # Output dep list generated using the flattened pom (only 'compile' and 'runtime' scopes) msg "Generating dependency list using flattened pom..." - mvn dependency:list -f .flattened-pom.xml -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' >.new-list.txt + mvn dependency:list -f .flattened-pom.xml -DincludeScope=runtime -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' >.new-list.txt # Compare two dependency lists msg "Comparing dependency lists..." @@ -85,4 +86,4 @@ then else msg "Errors found. See log statements above." exit 1 -fi +fi \ No newline at end of file diff --git a/synth.metadata b/synth.metadata index e85c685d80..57ce0b11e2 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-bigtable.git", - "sha": "950d224fff63ef8c87a6604c436d3e144f8f28b3" + "sha": "37c38bd34405e0f4fc4b08f2a43f614bb869309e" } }, { @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "4530cc6ff080ef8aca258c1ec92c4db10a1bbfb4" + "sha": "f8823dec98277a9516f2fb6fae9f58b3a59a23e1" } } ], From 865649778097d838f27961a6e8df72dbb5f97419 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Fri, 7 Aug 2020 13:26:15 -0400 Subject: [PATCH 17/30] update kokoro configs --- .kokoro/nightly/samples.cfg | 5 +++++ .kokoro/presubmit/samples.cfg | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.kokoro/nightly/samples.cfg b/.kokoro/nightly/samples.cfg index f25429314f..b72bc63092 100644 --- a/.kokoro/nightly/samples.cfg +++ b/.kokoro/nightly/samples.cfg @@ -36,3 +36,8 @@ env_vars: { key: "ENABLE_BUILD_COP" value: "true" } + +env_vars: { + key: "BIGTABLE_TESTING_INSTANCE" + value: "instance" +} \ No newline at end of file diff --git a/.kokoro/presubmit/samples.cfg b/.kokoro/presubmit/samples.cfg index 01e0960047..1e677bfdff 100644 --- a/.kokoro/presubmit/samples.cfg +++ b/.kokoro/presubmit/samples.cfg @@ -30,4 +30,9 @@ env_vars: { env_vars: { key: "SECRET_MANAGER_KEYS" value: "java-docs-samples-service-account" +} + +env_vars: { + key: "BIGTABLE_TESTING_INSTANCE" + value: "instance" } \ No newline at end of file From 012fbc392618b53d14d28b8eb65a232de1765f3a Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Fri, 7 Aug 2020 16:06:52 -0400 Subject: [PATCH 18/30] update kokoro configs --- synth.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synth.py b/synth.py index a8d7565dea..799187531c 100644 --- a/synth.py +++ b/synth.py @@ -37,6 +37,8 @@ def main(): 'README.md', '.kokoro/presubmit/integration.cfg', '.kokoro/nightly/integration.cfg', + '.kokoro/presubmit/samples.cfg', + '.kokoro/nightly/samples.cfg', # todo remove once template is updated '.github/ISSUE_TEMPLATE/bug_report.md', 'CONTRIBUTING.md', From bdae336074d80815dcaaf8c71befafc0ed66c079 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 7 Aug 2020 23:34:08 +0200 Subject: [PATCH 19/30] deps: update dependency com.google.cloud:google-cloud-shared-dependencies to v0.8.6 (#377) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [com.google.cloud:google-cloud-shared-dependencies](https://togithub.com/googleapis/java-shared-dependencies) | patch | `0.8.3` -> `0.8.6` | --- ### Release Notes
googleapis/java-shared-dependencies ### [`v0.8.6`](https://togithub.com/googleapis/java-shared-dependencies/blob/master/CHANGELOG.md#​086-httpswwwgithubcomgoogleapisjava-shared-dependenciescomparev085v086-2020-08-07) [Compare Source](https://togithub.com/googleapis/java-shared-dependencies/compare/v0.8.5...v0.8.6) ### [`v0.8.5`](https://togithub.com/googleapis/java-shared-dependencies/blob/master/CHANGELOG.md#​085-httpswwwgithubcomgoogleapisjava-shared-dependenciescomparev084v085-2020-08-07) [Compare Source](https://togithub.com/googleapis/java-shared-dependencies/compare/v0.8.4...v0.8.5) ### [`v0.8.4`](https://togithub.com/googleapis/java-shared-dependencies/blob/master/CHANGELOG.md#​084-httpswwwgithubcomgoogleapisjava-shared-dependenciescomparev083v084-2020-08-04) [Compare Source](https://togithub.com/googleapis/java-shared-dependencies/compare/v0.8.3...v0.8.4)
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#googleapis/java-bigtable). --- google-cloud-bigtable-deps-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml index 16102e6c84..863743dd6d 100644 --- a/google-cloud-bigtable-deps-bom/pom.xml +++ b/google-cloud-bigtable-deps-bom/pom.xml @@ -79,7 +79,7 @@ com.google.cloud google-cloud-shared-dependencies - 0.8.3 + 0.8.6 pom import From b114840958e4841f939c889832f533029a1e0195 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 14 Aug 2020 04:26:08 +0200 Subject: [PATCH 20/30] chore(deps): update dependency com.google.cloud:libraries-bom to v9 (#387) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [com.google.cloud:libraries-bom](https://togithub.com/GoogleCloudPlatform/cloud-opensource-java) | major | `8.1.0` -> `9.0.0` | --- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#googleapis/java-bigtable). --- samples/snippets/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index f894c41f15..d3d5a12204 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 8.1.0 + 9.0.0 pom import From f9bf950a91a9264d539c5a8ad22e74a7fa3dc722 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 17 Aug 2020 19:02:30 +0200 Subject: [PATCH 21/30] chore(deps): update dependency com.google.cloud:libraries-bom to v9.1.0 (#389) --- samples/snippets/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index d3d5a12204..952bc3d8c2 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 9.0.0 + 9.1.0 pom import From 667a21da52bec37cf867db54e5cd445bf200a82c Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Thu, 20 Aug 2020 11:19:40 -0700 Subject: [PATCH 22/30] chore: update build cfgs (#393) * chore(java_templates): add lint/static analysis presubmit checks for samples * chore(java_templates): add lint/static analysis presubmit checks for samples * chore: fix trailing whitespace Source-Author: Jeff Ching Source-Date: Mon Aug 17 14:29:16 2020 -0700 Source-Repo: googleapis/synthtool Source-Sha: c3caf0704f25a0c365f1c315e804a30b87c62a75 Source-Link: https://github.com/googleapis/synthtool/commit/c3caf0704f25a0c365f1c315e804a30b87c62a75 * chore(java_templates): stop running pmd/spotbugs checks for samples This was creating too much noise. We will revisit with other options and/or tune these checks. Source-Author: Jeff Ching Source-Date: Wed Aug 19 12:26:49 2020 -0700 Source-Repo: googleapis/synthtool Source-Sha: 9602086c6c5b05db77950c7f7495a2a3868f3537 Source-Link: https://github.com/googleapis/synthtool/commit/9602086c6c5b05db77950c7f7495a2a3868f3537 --- .github/workflows/samples.yaml | 14 ++++++++++++++ synth.metadata | 7 +++---- 2 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/samples.yaml diff --git a/.github/workflows/samples.yaml b/.github/workflows/samples.yaml new file mode 100644 index 0000000000..a1d5007306 --- /dev/null +++ b/.github/workflows/samples.yaml @@ -0,0 +1,14 @@ +on: + pull_request: +name: samples +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - name: Run checkstyle + run: mvn -P lint --quiet --batch-mode checkstyle:check + working-directory: samples/snippets diff --git a/synth.metadata b/synth.metadata index 57ce0b11e2..d80f9ea77e 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-bigtable.git", - "sha": "37c38bd34405e0f4fc4b08f2a43f614bb869309e" + "sha": "f9bf950a91a9264d539c5a8ad22e74a7fa3dc722" } }, { @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "f8823dec98277a9516f2fb6fae9f58b3a59a23e1" + "sha": "9602086c6c5b05db77950c7f7495a2a3868f3537" } } ], @@ -51,6 +51,7 @@ ".github/release-please.yml", ".github/trusted-contribution.yml", ".github/workflows/ci.yaml", + ".github/workflows/samples.yaml", ".kokoro/build.bat", ".kokoro/build.sh", ".kokoro/coerce_logs.sh", @@ -66,7 +67,6 @@ ".kokoro/nightly/java8-osx.cfg", ".kokoro/nightly/java8-win.cfg", ".kokoro/nightly/java8.cfg", - ".kokoro/nightly/samples.cfg", ".kokoro/populate-secrets.sh", ".kokoro/presubmit/clirr.cfg", ".kokoro/presubmit/common.cfg", @@ -78,7 +78,6 @@ ".kokoro/presubmit/java8.cfg", ".kokoro/presubmit/linkage-monitor.cfg", ".kokoro/presubmit/lint.cfg", - ".kokoro/presubmit/samples.cfg", ".kokoro/release/bump_snapshot.cfg", ".kokoro/release/common.cfg", ".kokoro/release/common.sh", From a6e70a9274917b54f69325279fd4cc7fdb7c4572 Mon Sep 17 00:00:00 2001 From: Weiran Fang <8175562+WeiranFang@users.noreply.github.com> Date: Mon, 24 Aug 2020 12:59:43 -0700 Subject: [PATCH 23/30] chore(fix): directpath ipv4 connection should not fail for an ipv6-enabled test client (#388) * chore(fix): directpath ipv4 connection should not fail for an ipv6-enabled test client * rename bigtable-directpath-ipv4-it profile to bigtable-directpath-ipv4only-it * fix sys prop & method naming * fix err msg --- google-cloud-bigtable/pom.xml | 10 +++++----- .../bigtable/test_helpers/env/AbstractTestEnv.java | 4 ++-- .../cloud/bigtable/test_helpers/env/CloudEnv.java | 8 +++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index 119624fd61..309e4d51e8 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -340,7 +340,7 @@ - bigtable-directpath-ipv4-it + bigtable-directpath-ipv4only-it @@ -359,16 +359,16 @@ cloud ${bigtable.directpath-data-endpoint} ${bigtable.directpath-admin-endpoint} - ${project.build.directory}/test-grpc-logs/directpath-ipv4-it + ${project.build.directory}/test-grpc-logs/directpath-ipv4only-it true - true + true com.google.cloud.bigtable.data.v2.it.*IT - ${project.build.directory}/failsafe-reports/failsafe-summary-directpath-ipv4-it.xml - ${project.build.directory}/failsafe-reports/directpath-ipv4-it + ${project.build.directory}/failsafe-reports/failsafe-summary-directpath-ipv4only-it.xml + ${project.build.directory}/failsafe-reports/directpath-ipv4only-it diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java index 492e94e9bd..c86403deb6 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java @@ -78,8 +78,8 @@ public boolean isDirectPathEnabled() { return Boolean.getBoolean("bigtable.attempt-directpath"); } - public boolean isDirectPathIpv4() { - return Boolean.getBoolean("bigtable.directpath-ipv4"); + public boolean isDirectPathIpv4Only() { + return Boolean.getBoolean("bigtable.directpath-ipv4only"); } public String getPrimaryZone() { diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/CloudEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/CloudEnv.java index 87aab527e5..cee19b29ae 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/CloudEnv.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/CloudEnv.java @@ -222,7 +222,8 @@ public void onHeaders(Metadata headers) { + " to the test environment's requirement for DirectPath." + " Expected test to access DirectPath via %s," + " but RPC was destined for %s", - isDirectPathIpv4() ? "ipv4" : "ipv6", remoteAddr.toString())); + isDirectPathIpv4Only() ? "ipv4 only" : "ipv4 or ipv6", + remoteAddr.toString())); } super.onHeaders(headers); } @@ -238,10 +239,11 @@ private boolean verifyRemoteAddress(SocketAddress remoteAddr) { if (remoteAddr instanceof InetSocketAddress) { InetAddress inetAddress = ((InetSocketAddress) remoteAddr).getAddress(); String addr = inetAddress.getHostAddress(); - if (isDirectPathIpv4()) { + if (isDirectPathIpv4Only()) { return addr.startsWith(DP_IPV4_PREFIX); } else { - return addr.startsWith(DP_IPV6_PREFIX); + // For an ipv6-enabled VM, client could connect to either ipv4 or ipv6 balancer addresses. + return addr.startsWith(DP_IPV6_PREFIX) || addr.startsWith(DP_IPV4_PREFIX); } } return true; From a56a0f8c9caf675b68d02587b042e1feeb261ccb Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Wed, 26 Aug 2020 07:00:46 -0700 Subject: [PATCH 24/30] fix: temporarily disable reporting to unblock releases (#395) Source-Author: Stephanie Wang Source-Date: Tue Aug 25 13:05:26 2020 -0400 Source-Repo: googleapis/synthtool Source-Sha: 968465a1cad496e1292ef4584a054a35f756ff94 Source-Link: https://github.com/googleapis/synthtool/commit/968465a1cad496e1292ef4584a054a35f756ff94 --- .kokoro/release/stage.sh | 5 +++-- synth.metadata | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.kokoro/release/stage.sh b/.kokoro/release/stage.sh index 3c482cbc55..d19191fc89 100755 --- a/.kokoro/release/stage.sh +++ b/.kokoro/release/stage.sh @@ -16,8 +16,9 @@ set -eo pipefail # Start the releasetool reporter -python3 -m pip install gcp-releasetool -python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script +# Disable reporting due to issue observed with Kokoro blocking releases +# python3 -m pip install gcp-releasetool +# python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script source $(dirname "$0")/common.sh MAVEN_SETTINGS_FILE=$(realpath $(dirname "$0")/../../)/settings.xml diff --git a/synth.metadata b/synth.metadata index d80f9ea77e..a72f63a746 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-bigtable.git", - "sha": "f9bf950a91a9264d539c5a8ad22e74a7fa3dc722" + "sha": "a6e70a9274917b54f69325279fd4cc7fdb7c4572" } }, { @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "9602086c6c5b05db77950c7f7495a2a3868f3537" + "sha": "968465a1cad496e1292ef4584a054a35f756ff94" } } ], From 848affb3586fbff65bc7a7bb6720460343f70199 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Thu, 27 Aug 2020 07:28:50 -0700 Subject: [PATCH 25/30] build(java): switch to release-publish app for notifying GitHub of release status (#396) Source-Author: Jeff Ching Source-Date: Wed Aug 26 21:48:06 2020 -0700 Source-Repo: googleapis/synthtool Source-Sha: 019c7168faa0e56619f792693a8acdb30d6de19b Source-Link: https://github.com/googleapis/synthtool/commit/019c7168faa0e56619f792693a8acdb30d6de19b --- .kokoro/release/stage.cfg | 31 +++---------------------------- .kokoro/release/stage.sh | 5 ++--- synth.metadata | 4 ++-- 3 files changed, 7 insertions(+), 33 deletions(-) diff --git a/.kokoro/release/stage.cfg b/.kokoro/release/stage.cfg index 1bd3edcae4..2ab56e7b0a 100644 --- a/.kokoro/release/stage.cfg +++ b/.kokoro/release/stage.cfg @@ -13,32 +13,7 @@ action { } } -# Fetch the token needed for reporting release status to GitHub -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "yoshi-automation-github-key" - } - } -} - -# Fetch magictoken to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "releasetool-magictoken" - } - } -} - -# Fetch api key to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "magic-github-proxy-api-key" - } - } +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } diff --git a/.kokoro/release/stage.sh b/.kokoro/release/stage.sh index d19191fc89..3c482cbc55 100755 --- a/.kokoro/release/stage.sh +++ b/.kokoro/release/stage.sh @@ -16,9 +16,8 @@ set -eo pipefail # Start the releasetool reporter -# Disable reporting due to issue observed with Kokoro blocking releases -# python3 -m pip install gcp-releasetool -# python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script +python3 -m pip install gcp-releasetool +python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script source $(dirname "$0")/common.sh MAVEN_SETTINGS_FILE=$(realpath $(dirname "$0")/../../)/settings.xml diff --git a/synth.metadata b/synth.metadata index a72f63a746..4d2ddc577b 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-bigtable.git", - "sha": "a6e70a9274917b54f69325279fd4cc7fdb7c4572" + "sha": "a56a0f8c9caf675b68d02587b042e1feeb261ccb" } }, { @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "968465a1cad496e1292ef4584a054a35f756ff94" + "sha": "019c7168faa0e56619f792693a8acdb30d6de19b" } } ], From 23cb8d8fd04ede0f430850b5f761e37d0464367e Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 31 Aug 2020 22:24:44 +0200 Subject: [PATCH 26/30] chore(deps): update dependency com.google.cloud:libraries-bom to v10 (#400) --- samples/snippets/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 952bc3d8c2..0e34671195 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 9.1.0 + 10.0.0 pom import From 2298596dab8a1ef87c0f48d3abe8bc3955417eb1 Mon Sep 17 00:00:00 2001 From: yangnuoyu <46553752+yangnuoyu@users.noreply.github.com> Date: Mon, 31 Aug 2020 14:44:11 -0700 Subject: [PATCH 27/30] feat(deps): adopt flatten plugin and google-cloud-shared-dependencies (#350) Co-authored-by: kolea2 <45548808+kolea2@users.noreply.github.com> --- .gitignore | 2 ++ google-cloud-bigtable/pom.xml | 5 +++++ grpc-google-cloud-bigtable-admin-v2/pom.xml | 9 +++++++++ grpc-google-cloud-bigtable-v2/pom.xml | 9 +++++++++ proto-google-cloud-bigtable-admin-v2/pom.xml | 9 +++++++++ proto-google-cloud-bigtable-v2/pom.xml | 9 +++++++++ 6 files changed, 43 insertions(+) diff --git a/.gitignore b/.gitignore index be0ddcd12a..dbde6a740b 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,5 @@ api_key # Python utilities *.pyc artman-genfiles + +.flattened-pom.xml diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index 309e4d51e8..c2ef87ebfd 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -452,6 +452,11 @@ false + + + org.codehaus.mojo + flatten-maven-plugin + diff --git a/grpc-google-cloud-bigtable-admin-v2/pom.xml b/grpc-google-cloud-bigtable-admin-v2/pom.xml index 3a05f1209c..0e002197fe 100644 --- a/grpc-google-cloud-bigtable-admin-v2/pom.xml +++ b/grpc-google-cloud-bigtable-admin-v2/pom.xml @@ -90,4 +90,13 @@ + + + + + org.codehaus.mojo + flatten-maven-plugin + + + diff --git a/grpc-google-cloud-bigtable-v2/pom.xml b/grpc-google-cloud-bigtable-v2/pom.xml index ff675b8d81..6240b3ba6b 100644 --- a/grpc-google-cloud-bigtable-v2/pom.xml +++ b/grpc-google-cloud-bigtable-v2/pom.xml @@ -82,4 +82,13 @@ + + + + + org.codehaus.mojo + flatten-maven-plugin + + + diff --git a/proto-google-cloud-bigtable-admin-v2/pom.xml b/proto-google-cloud-bigtable-admin-v2/pom.xml index 164d80f9f5..30987be007 100644 --- a/proto-google-cloud-bigtable-admin-v2/pom.xml +++ b/proto-google-cloud-bigtable-admin-v2/pom.xml @@ -58,4 +58,13 @@ compile + + + + + org.codehaus.mojo + flatten-maven-plugin + + + \ No newline at end of file diff --git a/proto-google-cloud-bigtable-v2/pom.xml b/proto-google-cloud-bigtable-v2/pom.xml index 7fccaf4a1a..90067cd62a 100644 --- a/proto-google-cloud-bigtable-v2/pom.xml +++ b/proto-google-cloud-bigtable-v2/pom.xml @@ -53,4 +53,13 @@ compile + + + + + org.codehaus.mojo + flatten-maven-plugin + + + \ No newline at end of file From 08f7d84333c6a74bf03e0a57707b878a29400dd4 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 1 Sep 2020 16:07:02 +0200 Subject: [PATCH 28/30] deps: update dependency com.google.cloud:google-cloud-shared-dependencies to v0.9.0 (#402) --- google-cloud-bigtable-deps-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml index 863743dd6d..58fefb1a52 100644 --- a/google-cloud-bigtable-deps-bom/pom.xml +++ b/google-cloud-bigtable-deps-bom/pom.xml @@ -79,7 +79,7 @@ com.google.cloud google-cloud-shared-dependencies - 0.8.6 + 0.9.0 pom import From 6f1ead2097150a87cb9712bcf35c6eaa9d57440c Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 1 Sep 2020 17:27:11 -0400 Subject: [PATCH 29/30] feat: extend channel priming logic to also send fake requests (#398) * feat: extend channel priming logic to also send fake requests * finish * remove debug * copyright * remove debug code from test * fix deps * address some feedback * add additional error handling test + some comments --- google-cloud-bigtable/pom.xml | 4 + .../data/v2/BigtableDataSettings.java | 38 ++- .../data/v2/internal/RefreshChannel.java | 54 ---- .../data/v2/stub/BigtableChannelPrimer.java | 168 +++++++++++++ .../data/v2/stub/EnhancedBigtableStub.java | 141 +++++++---- .../v2/stub/EnhancedBigtableStubSettings.java | 38 ++- .../data/v2/internal/RefreshChannelTest.java | 38 --- .../v2/stub/BigtableChannelPrimerTest.java | 234 ++++++++++++++++++ .../v2/stub/EnhancedBigtableStubTest.java | 44 +++- .../v2/stub/metrics/MetricsTracerTest.java | 13 +- 10 files changed, 593 insertions(+), 179 deletions(-) delete mode 100644 google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/RefreshChannel.java create mode 100644 google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimer.java delete mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/RefreshChannelTest.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimerTest.java diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index c2ef87ebfd..6c562e5d16 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -87,6 +87,10 @@ com.google.api.grpc proto-google-iam-v1 + + com.google.auth + google-auth-library-credentials + com.google.auth google-auth-library-oauth2-http diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java index 437ebc653b..8dd0fa6d97 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java @@ -26,6 +26,7 @@ import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings; import com.google.common.base.Strings; import io.grpc.ManagedChannelBuilder; +import java.util.List; import java.util.logging.Logger; import javax.annotation.Nonnull; @@ -185,11 +186,20 @@ public String getAppProfileId() { } /** Gets if channels will gracefully refresh connections to Cloud Bigtable service */ - @BetaApi("This API depends on experimental gRPC APIs") + @BetaApi("Channel priming is not currently stable and may change in the future") public boolean isRefreshingChannel() { return stubSettings.isRefreshingChannel(); } + /** + * Gets the table ids that will be used to send warmup requests when {@link + * #isRefreshingChannel()} is enabled. + */ + @BetaApi("Channel priming is not currently stable and may change in the future") + public List getPrimingTableIds() { + return stubSettings.getPrimedTableIds(); + } + /** Returns the underlying RPC settings. */ public EnhancedBigtableStubSettings getStubSettings() { return stubSettings; @@ -307,18 +317,40 @@ public CredentialsProvider getCredentialsProvider() { * connections, which causes the client to renegotiate the gRPC connection in the request path, * which causes periodic spikes in latency */ - @BetaApi("This API depends on experimental gRPC APIs") + @BetaApi("Channel priming is not currently stable and may change in the future") public Builder setRefreshingChannel(boolean isRefreshingChannel) { stubSettings.setRefreshingChannel(isRefreshingChannel); return this; } /** Gets if channels will gracefully refresh connections to Cloud Bigtable service */ - @BetaApi("This API depends on experimental gRPC APIs") + @BetaApi("Channel priming is not currently stable and may change in the future") public boolean isRefreshingChannel() { return stubSettings.isRefreshingChannel(); } + /** + * Configure the tables that can be used to prime a channel during a refresh. + * + *

These tables work in conjunction with {@link #setRefreshingChannel(boolean)}. When a + * channel is refreshed, it will send a request to each table to warm up the serverside caches + * before admitting the new channel into the channel pool. + */ + @BetaApi("Channel priming is not currently stable and may change in the future") + public Builder setPrimingTableIds(String... tableIds) { + stubSettings.setPrimedTableIds(tableIds); + return this; + } + + /** + * Gets the table ids that will be used to send warmup requests when {@link + * #setRefreshingChannel(boolean)} is enabled. + */ + @BetaApi("Channel priming is not currently stable and may change in the future") + public List getPrimingTableIds() { + return stubSettings.getPrimedTableIds(); + } + /** * Returns the underlying settings for making RPC calls. The settings should be changed with * care. diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/RefreshChannel.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/RefreshChannel.java deleted file mode 100644 index e34ecd750d..0000000000 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/RefreshChannel.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.data.v2.internal; - -import com.google.api.core.BetaApi; -import com.google.api.core.InternalApi; -import com.google.api.gax.grpc.ChannelPrimer; -import io.grpc.ConnectivityState; -import io.grpc.ManagedChannel; -import java.util.concurrent.TimeUnit; - -/** - * Establish a connection to the Cloud Bigtable service on managedChannel - * - *

This class is considered an internal implementation detail and not meant to be used by - * applications. - */ -@BetaApi("This API depends on gRPC experimental API") -@InternalApi -public final class RefreshChannel implements ChannelPrimer { - - /** - * primeChannel establishes a connection to Cloud Bigtable service. This typically take less than - * 1s. In case of service failure, an upper limit of 10s prevents primeChannel from looping - * forever. - */ - @Override - public void primeChannel(ManagedChannel managedChannel) { - for (int i = 0; i < 10; i++) { - ConnectivityState connectivityState = managedChannel.getState(true); - if (connectivityState == ConnectivityState.READY) { - break; - } - try { - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) { - break; - } - } - } -} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimer.java new file mode 100644 index 0000000000..15be8f7309 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimer.java @@ -0,0 +1,168 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.stub; + +import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS; + +import com.google.api.core.ApiFuture; +import com.google.api.core.BetaApi; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.api.gax.core.InstantiatingExecutorProvider; +import com.google.api.gax.grpc.ChannelPrimer; +import com.google.api.gax.grpc.GrpcTransportChannel; +import com.google.api.gax.rpc.FixedTransportChannelProvider; +import com.google.auth.Credentials; +import com.google.cloud.bigtable.data.v2.models.Query; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.ByteString; +import io.grpc.ConnectivityState; +import io.grpc.ManagedChannel; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; +import org.threeten.bp.Duration; + +/** + * A channel warmer that ensures that a Bigtable channel is ready to be used before being added to + * the active {@link com.google.api.gax.grpc.ChannelPool}. + * + *

This implementation is subject to change in the future, but currently it will prime the + * channel by sending a ReadRow request for a hardcoded, non-existent row key. + */ +@BetaApi("Channel priming is not currently stable and might change in the future") +class BigtableChannelPrimer implements ChannelPrimer { + private static Logger LOG = Logger.getLogger(BigtableChannelPrimer.class.toString()); + + static ByteString PRIMING_ROW_KEY = ByteString.copyFromUtf8("nonexistent-priming-row"); + private static Duration PRIME_REQUEST_TIMEOUT = Duration.ofSeconds(30); + + private final EnhancedBigtableStubSettings settingsTemplate; + private final List tableIds; + + static BigtableChannelPrimer create( + Credentials credentials, + String projectId, + String instanceId, + String appProfileId, + List tableIds) { + EnhancedBigtableStubSettings.Builder builder = + EnhancedBigtableStubSettings.newBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .setAppProfileId(appProfileId) + .setCredentialsProvider(FixedCredentialsProvider.create(credentials)) + .setExecutorProvider( + InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(1).build()); + + // Disable retries for priming request + builder + .readRowSettings() + .setRetrySettings( + builder + .readRowSettings() + .getRetrySettings() + .toBuilder() + .setMaxAttempts(1) + .setJittered(false) + .setInitialRpcTimeout(PRIME_REQUEST_TIMEOUT) + .setMaxRpcTimeout(PRIME_REQUEST_TIMEOUT) + .setTotalTimeout(PRIME_REQUEST_TIMEOUT) + .build()); + return new BigtableChannelPrimer(builder.build(), tableIds); + } + + private BigtableChannelPrimer( + EnhancedBigtableStubSettings settingsTemplate, List tableIds) { + Preconditions.checkNotNull(settingsTemplate, "settingsTemplate can't be null"); + this.settingsTemplate = settingsTemplate; + this.tableIds = ImmutableList.copyOf(tableIds); + } + + @Override + public void primeChannel(ManagedChannel managedChannel) { + try { + primeChannelUnsafe(managedChannel); + } catch (IOException | RuntimeException e) { + LOG.warning( + String.format("Unexpected error while trying to prime a channel: %s", e.getMessage())); + } + } + + private void primeChannelUnsafe(ManagedChannel managedChannel) throws IOException { + if (tableIds.isEmpty()) { + waitForChannelReady(managedChannel); + } else { + sendPrimeRequests(managedChannel); + } + } + + private void waitForChannelReady(ManagedChannel managedChannel) { + for (int i = 0; i < 30; i++) { + ConnectivityState connectivityState = managedChannel.getState(true); + if (connectivityState == ConnectivityState.READY) { + break; + } + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + break; + } + } + } + + private void sendPrimeRequests(ManagedChannel managedChannel) throws IOException { + // Wrap the channel in a temporary stub + EnhancedBigtableStubSettings primingSettings = + settingsTemplate + .toBuilder() + .setTransportChannelProvider( + FixedTransportChannelProvider.create(GrpcTransportChannel.create(managedChannel))) + .build(); + + try (EnhancedBigtableStub stub = EnhancedBigtableStub.create(primingSettings)) { + Map> primeFutures = new HashMap<>(); + + // Prime all of the table ids in parallel + for (String tableId : tableIds) { + ApiFuture f = + stub.readRowCallable() + .futureCall(Query.create(tableId).rowKey(PRIMING_ROW_KEY).filter(FILTERS.block())); + + primeFutures.put(tableId, f); + } + + // Wait for all of the prime requests to complete. + for (Map.Entry> entry : primeFutures.entrySet()) { + try { + entry.getValue().get(); + } catch (Throwable e) { + if (e instanceof ExecutionException) { + e = e.getCause(); + } + LOG.warning( + String.format( + "Failed to prime channel for table: %s: %s", entry.getKey(), e.getMessage())); + } + } + } + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index 8d9d2fc70c..d729d6244d 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -20,10 +20,12 @@ import com.google.api.gax.batching.Batcher; import com.google.api.gax.batching.BatcherImpl; import com.google.api.gax.core.BackgroundResource; +import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.core.GaxProperties; import com.google.api.gax.grpc.GaxGrpcProperties; import com.google.api.gax.grpc.GrpcCallSettings; import com.google.api.gax.grpc.GrpcRawCallableFactory; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.retrying.ExponentialRetryAlgorithm; import com.google.api.gax.retrying.RetryAlgorithm; import com.google.api.gax.retrying.RetryingExecutorWithContext; @@ -38,6 +40,7 @@ import com.google.api.gax.tracing.SpanName; import com.google.api.gax.tracing.TracedServerStreamingCallable; import com.google.api.gax.tracing.TracedUnaryCallable; +import com.google.auth.Credentials; import com.google.bigtable.v2.BigtableGrpc; import com.google.bigtable.v2.CheckAndMutateRowRequest; import com.google.bigtable.v2.CheckAndMutateRowResponse; @@ -120,65 +123,93 @@ public class EnhancedBigtableStub implements AutoCloseable { public static EnhancedBigtableStub create(EnhancedBigtableStubSettings settings) throws IOException { - ClientContext clientContext = ClientContext.create(settings); + settings = finalizeSettings(settings, Tags.getTagger(), Stats.getStatsRecorder()); - return new EnhancedBigtableStub( - settings, clientContext, Tags.getTagger(), Stats.getStatsRecorder()); + return new EnhancedBigtableStub(settings, ClientContext.create(settings)); } - @InternalApi("Visible for testing") - public EnhancedBigtableStub( - EnhancedBigtableStubSettings settings, - ClientContext clientContext, - Tagger tagger, - StatsRecorder statsRecorder) { - this.settings = settings; + public static EnhancedBigtableStubSettings finalizeSettings( + EnhancedBigtableStubSettings settings, Tagger tagger, StatsRecorder stats) + throws IOException { + EnhancedBigtableStubSettings.Builder builder = settings.toBuilder(); + + // TODO: this implementation is on the cusp of unwieldy, if we end up adding more features + // consider splitting it up by feature. + + // Inject channel priming + if (settings.isRefreshingChannel()) { + // Fix the credentials so that they can be shared + Credentials credentials = null; + if (settings.getCredentialsProvider() != null) { + credentials = settings.getCredentialsProvider().getCredentials(); + } + builder.setCredentialsProvider(FixedCredentialsProvider.create(credentials)); + + // Inject the primer + InstantiatingGrpcChannelProvider transportProvider = + (InstantiatingGrpcChannelProvider) settings.getTransportChannelProvider(); + + builder.setTransportChannelProvider( + transportProvider + .toBuilder() + .setChannelPrimer( + BigtableChannelPrimer.create( + credentials, + settings.getProjectId(), + settings.getInstanceId(), + settings.getAppProfileId(), + settings.getPrimedTableIds())) + .build()); + } - this.clientContext = - clientContext - .toBuilder() - .setTracerFactory( - new CompositeTracerFactory( - ImmutableList.of( - // Add OpenCensus Tracing - new OpencensusTracerFactory( - ImmutableMap.builder() - // Annotate traces with the same tags as metrics - .put( - RpcMeasureConstants.BIGTABLE_PROJECT_ID.getName(), - settings.getProjectId()) - .put( - RpcMeasureConstants.BIGTABLE_INSTANCE_ID.getName(), - settings.getInstanceId()) - .put( - RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID.getName(), - settings.getAppProfileId()) - // Also annotate traces with library versions - .put("gax", GaxGrpcProperties.getGaxGrpcVersion()) - .put("grpc", GaxGrpcProperties.getGrpcVersion()) - .put( - "gapic", - GaxProperties.getLibraryVersion( - EnhancedBigtableStubSettings.class)) - .build()), - // Add OpenCensus Metrics - MetricsTracerFactory.create( - tagger, - statsRecorder, - ImmutableMap.builder() - .put( - RpcMeasureConstants.BIGTABLE_PROJECT_ID, - TagValue.create(settings.getProjectId())) - .put( - RpcMeasureConstants.BIGTABLE_INSTANCE_ID, - TagValue.create(settings.getInstanceId())) - .put( - RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID, - TagValue.create(settings.getAppProfileId())) - .build()), - // Add user configured tracer - clientContext.getTracerFactory()))) - .build(); + // Inject Opencensus instrumentation + builder.setTracerFactory( + new CompositeTracerFactory( + ImmutableList.of( + // Add OpenCensus Tracing + new OpencensusTracerFactory( + ImmutableMap.builder() + // Annotate traces with the same tags as metrics + .put( + RpcMeasureConstants.BIGTABLE_PROJECT_ID.getName(), + settings.getProjectId()) + .put( + RpcMeasureConstants.BIGTABLE_INSTANCE_ID.getName(), + settings.getInstanceId()) + .put( + RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID.getName(), + settings.getAppProfileId()) + // Also annotate traces with library versions + .put("gax", GaxGrpcProperties.getGaxGrpcVersion()) + .put("grpc", GaxGrpcProperties.getGrpcVersion()) + .put( + "gapic", + GaxProperties.getLibraryVersion(EnhancedBigtableStubSettings.class)) + .build()), + // Add OpenCensus Metrics + MetricsTracerFactory.create( + tagger, + stats, + ImmutableMap.builder() + .put( + RpcMeasureConstants.BIGTABLE_PROJECT_ID, + TagValue.create(settings.getProjectId())) + .put( + RpcMeasureConstants.BIGTABLE_INSTANCE_ID, + TagValue.create(settings.getInstanceId())) + .put( + RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID, + TagValue.create(settings.getAppProfileId())) + .build()), + // Add user configured tracer + settings.getTracerFactory()))); + + return builder.build(); + } + + public EnhancedBigtableStub(EnhancedBigtableStubSettings settings, ClientContext clientContext) { + this.settings = settings; + this.clientContext = clientContext; this.requestContext = RequestContext.create( settings.getProjectId(), settings.getInstanceId(), settings.getAppProfileId()); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java index 1906228a30..d843265d1e 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java @@ -28,7 +28,6 @@ import com.google.api.gax.rpc.StubSettings; import com.google.api.gax.rpc.TransportChannelProvider; import com.google.api.gax.rpc.UnaryCallSettings; -import com.google.cloud.bigtable.data.v2.internal.RefreshChannel; import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation; import com.google.cloud.bigtable.data.v2.models.KeyOffset; import com.google.cloud.bigtable.data.v2.models.Query; @@ -150,6 +149,7 @@ public class EnhancedBigtableStubSettings extends StubSettings primedTableIds; private final ServerStreamingCallSettings readRowsSettings; private final UnaryCallSettings readRowSettings; @@ -188,6 +188,7 @@ private EnhancedBigtableStubSettings(Builder builder) { instanceId = builder.instanceId; appProfileId = builder.appProfileId; isRefreshingChannel = builder.isRefreshingChannel; + primedTableIds = builder.primedTableIds; // Per method settings. readRowsSettings = builder.readRowsSettings.build(); @@ -226,6 +227,12 @@ public boolean isRefreshingChannel() { return isRefreshingChannel; } + /** Gets the tables that will be primed during a channel refresh. */ + @BetaApi("Channel priming is not currently stable and might change in the future") + public List getPrimedTableIds() { + return primedTableIds; + } + /** Returns a builder for the default ChannelProvider for this service. */ public static InstantiatingGrpcChannelProvider.Builder defaultGrpcTransportProviderBuilder() { return BigtableStubSettings.defaultGrpcTransportProviderBuilder() @@ -483,6 +490,7 @@ public static class Builder extends StubSettings.Builder primedTableIds; private final ServerStreamingCallSettings.Builder readRowsSettings; private final UnaryCallSettings.Builder readRowSettings; @@ -505,6 +513,7 @@ public static class Builder extends StubSettings.BuilderWhen enabled, this will wait for the connection to complete the SSL handshake. The effect + * can be enhanced by configuring table ids that can be used warm serverside caches using {@link + * #setPrimedTableIds(String...)}. * * @see com.google.cloud.bigtable.data.v2.BigtableDataSettings.Builder#setRefreshingChannel */ @@ -709,12 +723,25 @@ public Builder setRefreshingChannel(boolean isRefreshingChannel) { return this; } + /** Configures which tables will be primed when a connection is created. */ + @BetaApi("Channel priming is not currently stable and might change in the future") + public Builder setPrimedTableIds(String... tableIds) { + this.primedTableIds = ImmutableList.copyOf(tableIds); + return this; + } + /** Gets if channels will gracefully refresh connections to Cloud Bigtable service */ @BetaApi("This API depends on experimental gRPC APIs") public boolean isRefreshingChannel() { return isRefreshingChannel; } + /** Gets the tables that will be primed during a channel refresh. */ + @BetaApi("Channel priming is not currently stable and might change in the future") + public List getPrimedTableIds() { + return primedTableIds; + } + /** Returns the builder for the settings used for calls to readRows. */ public ServerStreamingCallSettings.Builder readRowsSettings() { return readRowsSettings; @@ -760,17 +787,10 @@ public EnhancedBigtableStubSettings build() { Preconditions.checkState(projectId != null, "Project id must be set"); Preconditions.checkState(instanceId != null, "Instance id must be set"); - // Set ChannelPrimer on TransportChannelProvider so channels will gracefully refresh - // connections to Cloud Bigtable service if (isRefreshingChannel) { Preconditions.checkArgument( getTransportChannelProvider() instanceof InstantiatingGrpcChannelProvider, "refreshingChannel only works with InstantiatingGrpcChannelProviders"); - InstantiatingGrpcChannelProvider.Builder channelBuilder = - ((InstantiatingGrpcChannelProvider) getTransportChannelProvider()) - .toBuilder() - .setChannelPrimer(new RefreshChannel()); - setTransportChannelProvider(channelBuilder.build()); } return new EnhancedBigtableStubSettings(this); } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/RefreshChannelTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/RefreshChannelTest.java deleted file mode 100644 index c41fa4d2a5..0000000000 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/RefreshChannelTest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.data.v2.internal; - -import io.grpc.ConnectivityState; -import io.grpc.ManagedChannel; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.Mockito; - -@RunWith(JUnit4.class) -public class RefreshChannelTest { - // RefreshChannel should establish connection to the server through managedChannel.getState(true) - @Test - public void testGetStateIsCalled() { - RefreshChannel refreshChannel = new RefreshChannel(); - ManagedChannel managedChannel = Mockito.mock(ManagedChannel.class); - - Mockito.doReturn(ConnectivityState.READY).when(managedChannel).getState(true); - - refreshChannel.primeChannel(managedChannel); - Mockito.verify(managedChannel, Mockito.atLeastOnce()).getState(true); - } -} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimerTest.java new file mode 100644 index 0000000000..42d13a7ab1 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/BigtableChannelPrimerTest.java @@ -0,0 +1,234 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.stub; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.core.ApiFunction; +import com.google.auth.oauth2.AccessToken; +import com.google.auth.oauth2.OAuth2Credentials; +import com.google.bigtable.v2.BigtableGrpc.BigtableImplBase; +import com.google.bigtable.v2.ReadRowsRequest; +import com.google.bigtable.v2.ReadRowsResponse; +import com.google.bigtable.v2.RowFilter; +import com.google.bigtable.v2.RowSet; +import com.google.common.collect.ImmutableList; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.grpc.ServerCall; +import io.grpc.ServerCall.Listener; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.net.ServerSocket; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.logging.Handler; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mockito; +import org.mockito.internal.stubbing.answers.ThrowsException; + +@RunWith(JUnit4.class) +public class BigtableChannelPrimerTest { + private static final String TOKEN_VALUE = "fake-token"; + + int port; + Server server; + FakeService fakeService; + MetadataInterceptor metadataInterceptor; + BigtableChannelPrimer primer; + ManagedChannel channel; + private LogHandler logHandler; + + @Before + public void setup() throws IOException { + try (ServerSocket ss = new ServerSocket(0)) { + port = ss.getLocalPort(); + } catch (IOException e) { + e.printStackTrace(); + } + + fakeService = new FakeService(); + metadataInterceptor = new MetadataInterceptor(); + server = + ServerBuilder.forPort(port).intercept(metadataInterceptor).addService(fakeService).build(); + server.start(); + + primer = + BigtableChannelPrimer.create( + OAuth2Credentials.create(new AccessToken(TOKEN_VALUE, null)), + "fake-project", + "fake-instance", + "fake-app-profile", + ImmutableList.of("table1", "table2")); + + channel = ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); + + logHandler = new LogHandler(); + Logger.getLogger(BigtableChannelPrimer.class.toString()).addHandler(logHandler); + } + + @After + public void teardown() { + Logger.getLogger(BigtableChannelPrimer.class.toString()).removeHandler(logHandler); + channel.shutdown(); + server.shutdown(); + } + + @Test + public void testCredentials() { + primer.primeChannel(channel); + + for (Metadata metadata : metadataInterceptor.metadataList) { + assertThat(metadata.get(Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER))) + .isEqualTo("Bearer " + TOKEN_VALUE); + } + channel.shutdown(); + } + + @Test + public void testRequests() { + final Queue requests = new ConcurrentLinkedQueue<>(); + + fakeService.readRowsCallback = + new ApiFunction() { + @Override + public ReadRowsResponse apply(ReadRowsRequest req) { + requests.add(req); + return ReadRowsResponse.getDefaultInstance(); + } + }; + primer.primeChannel(channel); + + assertThat(requests) + .containsExactly( + ReadRowsRequest.newBuilder() + .setTableName("projects/fake-project/instances/fake-instance/tables/table1") + .setAppProfileId("fake-app-profile") + .setRows(RowSet.newBuilder().addRowKeys(BigtableChannelPrimer.PRIMING_ROW_KEY)) + .setFilter(RowFilter.newBuilder().setBlockAllFilter(true).build()) + .setRowsLimit(1) + .build(), + ReadRowsRequest.newBuilder() + .setTableName("projects/fake-project/instances/fake-instance/tables/table2") + .setAppProfileId("fake-app-profile") + .setRows(RowSet.newBuilder().addRowKeys(BigtableChannelPrimer.PRIMING_ROW_KEY)) + .setFilter(RowFilter.newBuilder().setBlockAllFilter(true).build()) + .setRowsLimit(1) + .build()); + } + + @Test + public void testErrorsAreLogged() { + fakeService.readRowsCallback = + new ApiFunction() { + @Override + public ReadRowsResponse apply(ReadRowsRequest req) { + throw new StatusRuntimeException(Status.FAILED_PRECONDITION); + } + }; + primer.primeChannel(channel); + + assertThat(logHandler.logs).hasSize(2); + for (LogRecord log : logHandler.logs) { + assertThat(log.getMessage()).contains("FAILED_PRECONDITION"); + } + } + + @Test + public void testErrorsAreLoggedForBasic() { + BigtableChannelPrimer basicPrimer = + BigtableChannelPrimer.create( + OAuth2Credentials.create(new AccessToken(TOKEN_VALUE, null)), + "fake-project", + "fake-instance", + "fake-app-profile", + ImmutableList.of()); + + ManagedChannel channel = + Mockito.mock( + ManagedChannel.class, new ThrowsException(new UnsupportedOperationException())); + primer.primeChannel(channel); + + assertThat(logHandler.logs).hasSize(1); + for (LogRecord log : logHandler.logs) { + assertThat(log.getMessage()).contains("Unexpected"); + } + } + + private static class MetadataInterceptor implements ServerInterceptor { + ConcurrentLinkedQueue metadataList = new ConcurrentLinkedQueue<>(); + + @Override + public Listener interceptCall( + ServerCall serverCall, + Metadata metadata, + ServerCallHandler serverCallHandler) { + metadataList.add(metadata); + + return serverCallHandler.startCall(serverCall, metadata); + } + } + + static class FakeService extends BigtableImplBase { + private ApiFunction readRowsCallback = + new ApiFunction() { + @Override + public ReadRowsResponse apply(ReadRowsRequest readRowsRequest) { + return ReadRowsResponse.getDefaultInstance(); + } + }; + + @Override + public void readRows( + ReadRowsRequest request, StreamObserver responseObserver) { + + try { + responseObserver.onNext(readRowsCallback.apply(request)); + responseObserver.onCompleted(); + } catch (RuntimeException e) { + responseObserver.onError(e); + } + } + } + + private static class LogHandler extends Handler { + private ConcurrentLinkedQueue logs = new ConcurrentLinkedQueue<>(); + + @Override + public void publish(LogRecord record) { + logs.add(record); + } + + @Override + public void flush() {} + + @Override + public void close() throws SecurityException {} + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java index be2d9c2a0f..b823930fb6 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java @@ -18,14 +18,13 @@ import static com.google.common.truth.Truth.assertThat; import com.google.api.gax.core.NoCredentialsProvider; -import com.google.api.gax.grpc.testing.InProcessServer; -import com.google.api.gax.grpc.testing.LocalChannelProvider; import com.google.api.gax.rpc.ServerStreamingCallable; import com.google.bigtable.v2.BigtableGrpc; import com.google.bigtable.v2.ReadRowsRequest; import com.google.bigtable.v2.ReadRowsResponse; import com.google.bigtable.v2.RowSet; import com.google.cloud.bigtable.admin.v2.internal.NameUtil; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; import com.google.cloud.bigtable.data.v2.internal.RequestContext; import com.google.cloud.bigtable.data.v2.models.DefaultRowAdapter; import com.google.cloud.bigtable.data.v2.models.Query; @@ -34,8 +33,11 @@ import com.google.protobuf.ByteString; import com.google.protobuf.BytesValue; import com.google.protobuf.StringValue; +import io.grpc.Server; +import io.grpc.ServerBuilder; import io.grpc.stub.StreamObserver; import java.io.IOException; +import java.net.ServerSocket; import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; import org.junit.After; @@ -49,37 +51,40 @@ public class EnhancedBigtableStubTest { private static final String PROJECT_ID = "fake-project"; private static final String INSTANCE_ID = "fake-instance"; - private static final String FAKE_HOST_NAME = "fake-stub-host:123"; private static final String TABLE_NAME = NameUtil.formatTableName(PROJECT_ID, INSTANCE_ID, "fake-table"); private static final String APP_PROFILE_ID = "app-profile-id"; - private InProcessServer server; + private Server server; private FakeDataService fakeDataService; + private EnhancedBigtableStubSettings defaultSettings; private EnhancedBigtableStub enhancedBigtableStub; @Before public void setUp() throws IOException, IllegalAccessException, InstantiationException { + int port; + try (ServerSocket ss = new ServerSocket(0)) { + port = ss.getLocalPort(); + } fakeDataService = new FakeDataService(); - server = new InProcessServer<>(fakeDataService, FAKE_HOST_NAME); + server = ServerBuilder.forPort(port).addService(fakeDataService).build(); server.start(); - EnhancedBigtableStubSettings enhancedBigtableStubSettings = - EnhancedBigtableStubSettings.newBuilder() + defaultSettings = + BigtableDataSettings.newBuilderForEmulator(port) .setProjectId(PROJECT_ID) .setInstanceId(INSTANCE_ID) .setAppProfileId(APP_PROFILE_ID) .setCredentialsProvider(NoCredentialsProvider.create()) - .setEndpoint(FAKE_HOST_NAME) - .setTransportChannelProvider(LocalChannelProvider.create(FAKE_HOST_NAME)) - .build(); + .build() + .getStubSettings(); - enhancedBigtableStub = EnhancedBigtableStub.create(enhancedBigtableStubSettings); + enhancedBigtableStub = EnhancedBigtableStub.create(defaultSettings); } @After public void tearDown() { - server.stop(); + server.shutdown(); } @Test @@ -117,6 +122,21 @@ public void testCreateReadRowsRawCallable() throws InterruptedException { assertThat(fakeDataService.popLastRequest()).isEqualTo(expectedRequest2); } + @Test + public void testChannelPrimerConfigured() throws IOException { + EnhancedBigtableStubSettings settings = + defaultSettings + .toBuilder() + .setRefreshingChannel(true) + .setPrimedTableIds("table1", "table2") + .build(); + + try (EnhancedBigtableStub ignored = EnhancedBigtableStub.create(settings)) { + // priming will issue a request per table on startup + assertThat(fakeDataService.requests).hasSize(2); + } + } + private static class FakeDataService extends BigtableGrpc.BigtableImplBase { final BlockingQueue requests = Queues.newLinkedBlockingDeque(); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java index 9fd78c5cd5..49ebae81f2 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/MetricsTracerTest.java @@ -120,14 +120,11 @@ public void setUp() throws Exception { .setInstanceId(INSTANCE_ID) .setAppProfileId(APP_PROFILE_ID) .build(); - EnhancedBigtableStubSettings stubSettings = settings.getStubSettings(); - - stub = - new EnhancedBigtableStub( - stubSettings, - ClientContext.create(stubSettings), - Tags.getTagger(), - localStats.getStatsRecorder()); + EnhancedBigtableStubSettings stubSettings = + EnhancedBigtableStub.finalizeSettings( + settings.getStubSettings(), Tags.getTagger(), localStats.getStatsRecorder()); + + stub = new EnhancedBigtableStub(stubSettings, ClientContext.create(stubSettings)); } @After From afd4c89d1f925efa9dd190a5b83be3dc2b06b873 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2020 17:38:03 +0000 Subject: [PATCH 30/30] chore: release 1.15.0 (#401) :robot: I have created a release \*beep\* \*boop\* --- ## [1.15.0](https://www.github.com/googleapis/java-bigtable/compare/v1.14.0...v1.15.0) (2020-09-01) ### Features * extend channel priming logic to also send fake requests ([#398](https://www.github.com/googleapis/java-bigtable/issues/398)) ([6f1ead2](https://www.github.com/googleapis/java-bigtable/commit/6f1ead2097150a87cb9712bcf35c6eaa9d57440c)) * **deps:** adopt flatten plugin and google-cloud-shared-dependencies ([#350](https://www.github.com/googleapis/java-bigtable/issues/350)) ([2298596](https://www.github.com/googleapis/java-bigtable/commit/2298596dab8a1ef87c0f48d3abe8bc3955417eb1)) ### Bug Fixes * temporarily disable reporting to unblock releases ([#395](https://www.github.com/googleapis/java-bigtable/issues/395)) ([a56a0f8](https://www.github.com/googleapis/java-bigtable/commit/a56a0f8c9caf675b68d02587b042e1feeb261ccb)) ### Dependencies * update dependency com.google.cloud:google-cloud-shared-dependencies to v0.8.6 ([#377](https://www.github.com/googleapis/java-bigtable/issues/377)) ([bdae336](https://www.github.com/googleapis/java-bigtable/commit/bdae336074d80815dcaaf8c71befafc0ed66c079)) * update dependency com.google.cloud:google-cloud-shared-dependencies to v0.9.0 ([#402](https://www.github.com/googleapis/java-bigtable/issues/402)) ([08f7d84](https://www.github.com/googleapis/java-bigtable/commit/08f7d84333c6a74bf03e0a57707b878a29400dd4)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). --- CHANGELOG.md | 19 +++++++++++++++++++ README.md | 6 +++--- google-cloud-bigtable-bom/pom.xml | 14 +++++++------- google-cloud-bigtable-deps-bom/pom.xml | 2 +- google-cloud-bigtable-emulator/pom.xml | 8 ++++---- google-cloud-bigtable/pom.xml | 8 ++++---- grpc-google-cloud-bigtable-admin-v2/pom.xml | 8 ++++---- grpc-google-cloud-bigtable-v2/pom.xml | 8 ++++---- pom.xml | 2 +- proto-google-cloud-bigtable-admin-v2/pom.xml | 8 ++++---- proto-google-cloud-bigtable-v2/pom.xml | 8 ++++---- samples/snapshot/pom.xml | 2 +- versions.txt | 12 ++++++------ 13 files changed, 62 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d55124c9c..5b5c872aa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## [1.15.0](https://www.github.com/googleapis/java-bigtable/compare/v1.14.0...v1.15.0) (2020-09-01) + + +### Features + +* extend channel priming logic to also send fake requests ([#398](https://www.github.com/googleapis/java-bigtable/issues/398)) ([6f1ead2](https://www.github.com/googleapis/java-bigtable/commit/6f1ead2097150a87cb9712bcf35c6eaa9d57440c)) +* **deps:** adopt flatten plugin and google-cloud-shared-dependencies ([#350](https://www.github.com/googleapis/java-bigtable/issues/350)) ([2298596](https://www.github.com/googleapis/java-bigtable/commit/2298596dab8a1ef87c0f48d3abe8bc3955417eb1)) + + +### Bug Fixes + +* temporarily disable reporting to unblock releases ([#395](https://www.github.com/googleapis/java-bigtable/issues/395)) ([a56a0f8](https://www.github.com/googleapis/java-bigtable/commit/a56a0f8c9caf675b68d02587b042e1feeb261ccb)) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.8.6 ([#377](https://www.github.com/googleapis/java-bigtable/issues/377)) ([bdae336](https://www.github.com/googleapis/java-bigtable/commit/bdae336074d80815dcaaf8c71befafc0ed66c079)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.9.0 ([#402](https://www.github.com/googleapis/java-bigtable/issues/402)) ([08f7d84](https://www.github.com/googleapis/java-bigtable/commit/08f7d84333c6a74bf03e0a57707b878a29400dd4)) + ## [1.14.0](https://www.github.com/googleapis/java-bigtable/compare/v1.13.1...v1.14.0) (2020-07-20) diff --git a/README.md b/README.md index cf0ef9bcaa..90549e0575 100644 --- a/README.md +++ b/README.md @@ -16,16 +16,16 @@ If you are using Maven, add this to your pom.xml file com.google.cloud google-cloud-bigtable - 1.14.0 + 1.15.0 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.cloud:google-cloud-bigtable:1.14.0' +compile 'com.google.cloud:google-cloud-bigtable:1.15.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "1.14.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "1.15.0" ``` [//]: # ({x-version-update-end}) diff --git a/google-cloud-bigtable-bom/pom.xml b/google-cloud-bigtable-bom/pom.xml index 52a799c063..7bb0444ad0 100644 --- a/google-cloud-bigtable-bom/pom.xml +++ b/google-cloud-bigtable-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigtable-bom - 1.14.1-SNAPSHOT + 1.15.0 pom com.google.cloud @@ -72,32 +72,32 @@ com.google.cloud google-cloud-bigtable - 1.14.1-SNAPSHOT + 1.15.0 com.google.cloud google-cloud-bigtable-emulator - 0.123.1-SNAPSHOT + 0.124.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 1.14.1-SNAPSHOT + 1.15.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 1.14.1-SNAPSHOT + 1.15.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 1.14.1-SNAPSHOT + 1.15.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 1.14.1-SNAPSHOT + 1.15.0 diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml index 58fefb1a52..3c12565a0c 100644 --- a/google-cloud-bigtable-deps-bom/pom.xml +++ b/google-cloud-bigtable-deps-bom/pom.xml @@ -12,7 +12,7 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.1-SNAPSHOT + 1.15.0 pom diff --git a/google-cloud-bigtable-emulator/pom.xml b/google-cloud-bigtable-emulator/pom.xml index 42884b7786..feed747c3d 100644 --- a/google-cloud-bigtable-emulator/pom.xml +++ b/google-cloud-bigtable-emulator/pom.xml @@ -5,7 +5,7 @@ 4.0.0 google-cloud-bigtable-emulator - 0.123.1-SNAPSHOT + 0.124.0 Google Cloud Java - Bigtable Emulator https://github.com/googleapis/java-bigtable @@ -14,7 +14,7 @@ com.google.cloud google-cloud-bigtable-parent - 1.14.1-SNAPSHOT + 1.15.0 scm:git:git@github.com:googleapis/java-bigtable.git @@ -80,14 +80,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.1-SNAPSHOT + 1.15.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.14.1-SNAPSHOT + 1.15.0 pom import diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index 6c562e5d16..0ba7a1d644 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-bigtable - 1.14.1-SNAPSHOT + 1.15.0 jar Google Cloud Bigtable https://github.com/googleapis/java-bigtable @@ -12,7 +12,7 @@ com.google.cloud google-cloud-bigtable-parent - 1.14.1-SNAPSHOT + 1.15.0 google-cloud-bigtable @@ -36,14 +36,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.1-SNAPSHOT + 1.15.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.14.1-SNAPSHOT + 1.15.0 pom import diff --git a/grpc-google-cloud-bigtable-admin-v2/pom.xml b/grpc-google-cloud-bigtable-admin-v2/pom.xml index 0e002197fe..a27712ee6a 100644 --- a/grpc-google-cloud-bigtable-admin-v2/pom.xml +++ b/grpc-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 1.14.1-SNAPSHOT + 1.15.0 grpc-google-cloud-bigtable-admin-v2 GRPC library for grpc-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 1.14.1-SNAPSHOT + 1.15.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.1-SNAPSHOT + 1.15.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.14.1-SNAPSHOT + 1.15.0 pom import diff --git a/grpc-google-cloud-bigtable-v2/pom.xml b/grpc-google-cloud-bigtable-v2/pom.xml index 6240b3ba6b..bb8015da65 100644 --- a/grpc-google-cloud-bigtable-v2/pom.xml +++ b/grpc-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 1.14.1-SNAPSHOT + 1.15.0 grpc-google-cloud-bigtable-v2 GRPC library for grpc-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 1.14.1-SNAPSHOT + 1.15.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.1-SNAPSHOT + 1.15.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.14.1-SNAPSHOT + 1.15.0 pom import diff --git a/pom.xml b/pom.xml index b39880dece..ed134153c6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ google-cloud-bigtable-parent pom - 1.14.1-SNAPSHOT + 1.15.0 Google Cloud Bigtable Parent https://github.com/googleapis/java-bigtable diff --git a/proto-google-cloud-bigtable-admin-v2/pom.xml b/proto-google-cloud-bigtable-admin-v2/pom.xml index 30987be007..63726b23ff 100644 --- a/proto-google-cloud-bigtable-admin-v2/pom.xml +++ b/proto-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 1.14.1-SNAPSHOT + 1.15.0 proto-google-cloud-bigtable-admin-v2 PROTO library for proto-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 1.14.1-SNAPSHOT + 1.15.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.1-SNAPSHOT + 1.15.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.14.1-SNAPSHOT + 1.15.0 pom import diff --git a/proto-google-cloud-bigtable-v2/pom.xml b/proto-google-cloud-bigtable-v2/pom.xml index 90067cd62a..a84a85e002 100644 --- a/proto-google-cloud-bigtable-v2/pom.xml +++ b/proto-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 1.14.1-SNAPSHOT + 1.15.0 proto-google-cloud-bigtable-v2 PROTO library for proto-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 1.14.1-SNAPSHOT + 1.15.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.14.1-SNAPSHOT + 1.15.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.14.1-SNAPSHOT + 1.15.0 pom import diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 879ceb5de4..bb697bfcdc 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-bigtable - 1.14.1-SNAPSHOT + 1.15.0 diff --git a/versions.txt b/versions.txt index c5e884ba98..3918eab106 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-bigtable:1.14.0:1.14.1-SNAPSHOT -grpc-google-cloud-bigtable-admin-v2:1.14.0:1.14.1-SNAPSHOT -grpc-google-cloud-bigtable-v2:1.14.0:1.14.1-SNAPSHOT -proto-google-cloud-bigtable-admin-v2:1.14.0:1.14.1-SNAPSHOT -proto-google-cloud-bigtable-v2:1.14.0:1.14.1-SNAPSHOT -google-cloud-bigtable-emulator:0.123.0:0.123.1-SNAPSHOT +google-cloud-bigtable:1.15.0:1.15.0 +grpc-google-cloud-bigtable-admin-v2:1.15.0:1.15.0 +grpc-google-cloud-bigtable-v2:1.15.0:1.15.0 +proto-google-cloud-bigtable-admin-v2:1.15.0:1.15.0 +proto-google-cloud-bigtable-v2:1.15.0:1.15.0 +google-cloud-bigtable-emulator:0.124.0:0.124.0