Skip to content

Commit b68a1e9

Browse files
jsrinnnjsrinnn
authored
impl(bigquery): Table options and policies (#11544)
1 parent 6b3b8a2 commit b68a1e9

13 files changed

Lines changed: 445 additions & 31 deletions

‎google/cloud/bigquery/bigquery_rest.cmake‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ add_library(
1818
google_cloud_cpp_bigquery_rest # cmake-format: sort
1919
v2/minimal/internal/bigquery_http_response.cc
2020
v2/minimal/internal/bigquery_http_response.h
21+
v2/minimal/internal/common_options.h
2122
v2/minimal/internal/common_v2_resources.cc
2223
v2/minimal/internal/common_v2_resources.h
2324
v2/minimal/internal/dataset.cc
@@ -78,6 +79,10 @@ add_library(
7879
v2/minimal/internal/table.h
7980
v2/minimal/internal/table_constraints.cc
8081
v2/minimal/internal/table_constraints.h
82+
v2/minimal/internal/table_idempotency_policy.cc
83+
v2/minimal/internal/table_idempotency_policy.h
84+
v2/minimal/internal/table_options.cc
85+
v2/minimal/internal/table_options.h
8186
v2/minimal/internal/table_partition.cc
8287
v2/minimal/internal/table_partition.h
8388
v2/minimal/internal/table_request.cc
@@ -88,6 +93,7 @@ add_library(
8893
v2/minimal/internal/table_rest_stub.h
8994
v2/minimal/internal/table_rest_stub_factory.cc
9095
v2/minimal/internal/table_rest_stub_factory.h
96+
v2/minimal/internal/table_retry_policy.h
9197
v2/minimal/internal/table_schema.cc
9298
v2/minimal/internal/table_schema.h
9399
v2/minimal/internal/table_view.cc
@@ -203,6 +209,8 @@ function (bigquery_rest_define_tests)
203209
v2/minimal/internal/job_rest_stub_test.cc
204210
v2/minimal/internal/job_test.cc
205211
v2/minimal/internal/rest_stub_utils_test.cc
212+
v2/minimal/internal/table_idempotency_policy_test.cc
213+
v2/minimal/internal/table_options_test.cc
206214
v2/minimal/internal/table_request_test.cc
207215
v2/minimal/internal/table_response_test.cc
208216
v2/minimal/internal/table_rest_stub_test.cc

‎google/cloud/bigquery/bigquery_rest_unit_tests.bzl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ bigquery_rest_unit_tests = [
4040
"v2/minimal/internal/job_rest_stub_test.cc",
4141
"v2/minimal/internal/job_test.cc",
4242
"v2/minimal/internal/rest_stub_utils_test.cc",
43+
"v2/minimal/internal/table_idempotency_policy_test.cc",
44+
"v2/minimal/internal/table_options_test.cc",
4345
"v2/minimal/internal/table_request_test.cc",
4446
"v2/minimal/internal/table_response_test.cc",
4547
"v2/minimal/internal/table_rest_stub_test.cc",

‎google/cloud/bigquery/google_cloud_cpp_bigquery_rest.bzl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
google_cloud_cpp_bigquery_rest_hdrs = [
2020
"v2/minimal/internal/bigquery_http_response.h",
21+
"v2/minimal/internal/common_options.h",
2122
"v2/minimal/internal/common_v2_resources.h",
2223
"v2/minimal/internal/dataset.h",
2324
"v2/minimal/internal/dataset_client.h",
@@ -50,11 +51,14 @@ google_cloud_cpp_bigquery_rest_hdrs = [
5051
"v2/minimal/internal/rest_stub_utils.h",
5152
"v2/minimal/internal/table.h",
5253
"v2/minimal/internal/table_constraints.h",
54+
"v2/minimal/internal/table_idempotency_policy.h",
55+
"v2/minimal/internal/table_options.h",
5356
"v2/minimal/internal/table_partition.h",
5457
"v2/minimal/internal/table_request.h",
5558
"v2/minimal/internal/table_response.h",
5659
"v2/minimal/internal/table_rest_stub.h",
5760
"v2/minimal/internal/table_rest_stub_factory.h",
61+
"v2/minimal/internal/table_retry_policy.h",
5862
"v2/minimal/internal/table_schema.h",
5963
"v2/minimal/internal/table_view.h",
6064
]
@@ -89,6 +93,8 @@ google_cloud_cpp_bigquery_rest_srcs = [
8993
"v2/minimal/internal/rest_stub_utils.cc",
9094
"v2/minimal/internal/table.cc",
9195
"v2/minimal/internal/table_constraints.cc",
96+
"v2/minimal/internal/table_idempotency_policy.cc",
97+
"v2/minimal/internal/table_options.cc",
9298
"v2/minimal/internal/table_partition.cc",
9399
"v2/minimal/internal/table_request.cc",
94100
"v2/minimal/internal/table_response.cc",

‎google/cloud/bigquery/v2/minimal/internal/common_options.h‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Copyright 2023 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_BIGQUERY_V2_MINIMAL_INTERNAL_COMMON_OPTIONS_H
16+
#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_BIGQUERY_V2_MINIMAL_INTERNAL_COMMON_OPTIONS_H
17+
18+
#include "google/cloud/options.h"
19+
#include "google/cloud/version.h"
20+
#include <memory>
21+
#include <thread>
22+
23+
namespace google {
24+
namespace cloud {
25+
namespace bigquery_v2_minimal_internal {
26+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
27+
28+
auto constexpr kBackoffScaling = 2.0;
29+
std::size_t constexpr kConnectionPoolSize = 4;
30+
std::size_t constexpr kConnectionPoolSizeMax = 64;
31+
32+
inline std::size_t DefaultConnectionPoolSize() {
33+
// For better resource utilization and greater throughput, it is recommended
34+
// to calculate the default pool size based on cores(CPU) available. However,
35+
// as per C++11 documentation `std::thread::hardware_concurrency()` cannot be
36+
// fully relied upon. It is only a hint and the value can be 0 if it is not
37+
// well defined or not computable. Apart from CPU count, multiple channels
38+
// can be opened for each CPU to increase throughput. The pool size is also
39+
// capped so that servers with many cores do not create too many channels.
40+
int cpu_count = std::thread::hardware_concurrency();
41+
if (cpu_count == 0) return kConnectionPoolSize;
42+
return (std::min)(kConnectionPoolSizeMax, cpu_count * kConnectionPoolSize);
43+
}
44+
45+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
46+
} // namespace bigquery_v2_minimal_internal
47+
} // namespace cloud
48+
} // namespace google
49+
50+
#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_BIGQUERY_V2_MINIMAL_INTERNAL_COMMON_OPTIONS_H

‎google/cloud/bigquery/v2/minimal/internal/dataset_options.cc‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// limitations under the License.
1414

1515
#include "google/cloud/bigquery/v2/minimal/internal/dataset_options.h"
16+
#include "google/cloud/bigquery/v2/minimal/internal/common_options.h"
1617
#include "google/cloud/bigquery/v2/minimal/internal/dataset_retry_policy.h"
1718
#include "google/cloud/internal/populate_common_options.h"
1819
#include <memory>
@@ -23,18 +24,6 @@ namespace cloud {
2324
namespace bigquery_v2_minimal_internal {
2425
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
2526

26-
namespace {
27-
auto constexpr kBackoffScaling = 2.0;
28-
std::size_t constexpr kConnectionPoolSize = 4;
29-
std::size_t constexpr kConnectionPoolSizeMax = 64;
30-
31-
std::size_t DefaultConnectionPoolSize() {
32-
int cpu_count = std::thread::hardware_concurrency();
33-
if (cpu_count == 0) return kConnectionPoolSize;
34-
return (std::min)(kConnectionPoolSizeMax, cpu_count * kConnectionPoolSize);
35-
}
36-
} // namespace
37-
3827
Options DatasetDefaultOptions(Options options) {
3928
options = google::cloud::internal::PopulateCommonOptions(
4029
std::move(options), "GOOGLE_CLOUD_CPP_BIGQUERY_V2_DATASET_ENDPOINT", "",

‎google/cloud/bigquery/v2/minimal/internal/job_options.cc‎

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
// Implementation of internal interface for Bigquery V2 Job resource.
1616

1717
#include "google/cloud/bigquery/v2/minimal/internal/job_options.h"
18+
#include "google/cloud/bigquery/v2/minimal/internal/common_options.h"
1819
#include "google/cloud/bigquery/v2/minimal/internal/job_retry_policy.h"
1920
#include "google/cloud/internal/populate_common_options.h"
2021
#include <memory>
@@ -25,25 +26,6 @@ namespace cloud {
2526
namespace bigquery_v2_minimal_internal {
2627
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
2728

28-
namespace {
29-
auto constexpr kBackoffScaling = 2.0;
30-
std::size_t constexpr kConnectionPoolSize = 4;
31-
std::size_t constexpr kConnectionPoolSizeMax = 64;
32-
33-
std::size_t DefaultConnectionPoolSize() {
34-
// For better resource utilization and greater throughput, it is recommended
35-
// to calculate the default pool size based on cores(CPU) available. However,
36-
// as per C++11 documentation `std::thread::hardware_concurrency()` cannot be
37-
// fully relied upon. It is only a hint and the value can be 0 if it is not
38-
// well defined or not computable. Apart from CPU count, multiple channels
39-
// can be opened for each CPU to increase throughput. The pool size is also
40-
// capped so that servers with many cores do not create too many channels.
41-
int cpu_count = std::thread::hardware_concurrency();
42-
if (cpu_count == 0) return kConnectionPoolSize;
43-
return (std::min)(kConnectionPoolSizeMax, cpu_count * kConnectionPoolSize);
44-
}
45-
} // namespace
46-
4729
Options BigQueryJobDefaultOptions(Options options) {
4830
options = google::cloud::internal::PopulateCommonOptions(
4931
std::move(options), "GOOGLE_CLOUD_CPP_BIGQUERY_V2_JOB_ENDPOINT", "",

‎google/cloud/bigquery/v2/minimal/internal/table_idempotency_policy.cc‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// Copyright 2023 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "google/cloud/bigquery/v2/minimal/internal/table_idempotency_policy.h"
16+
#include <memory>
17+
18+
namespace google {
19+
namespace cloud {
20+
namespace bigquery_v2_minimal_internal {
21+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
22+
23+
using ::google::cloud::Idempotency;
24+
25+
TableIdempotencyPolicy::~TableIdempotencyPolicy() = default;
26+
27+
std::unique_ptr<TableIdempotencyPolicy> TableIdempotencyPolicy::clone() const {
28+
return std::make_unique<TableIdempotencyPolicy>(*this);
29+
}
30+
31+
Idempotency TableIdempotencyPolicy::GetTable(GetTableRequest const&) {
32+
return Idempotency::kIdempotent;
33+
}
34+
35+
Idempotency TableIdempotencyPolicy::ListTables(ListTablesRequest const&) {
36+
return Idempotency::kIdempotent;
37+
}
38+
39+
std::unique_ptr<TableIdempotencyPolicy> MakeDefaultTableIdempotencyPolicy() {
40+
return std::make_unique<TableIdempotencyPolicy>();
41+
}
42+
43+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
44+
} // namespace bigquery_v2_minimal_internal
45+
} // namespace cloud
46+
} // namespace google

‎google/cloud/bigquery/v2/minimal/internal/table_idempotency_policy.h‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright 2023 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_BIGQUERY_V2_MINIMAL_INTERNAL_TABLE_IDEMPOTENCY_POLICY_H
16+
#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_BIGQUERY_V2_MINIMAL_INTERNAL_TABLE_IDEMPOTENCY_POLICY_H
17+
18+
#include "google/cloud/bigquery/v2/minimal/internal/table_request.h"
19+
#include "google/cloud/idempotency.h"
20+
#include "google/cloud/internal/retry_policy.h"
21+
#include "google/cloud/version.h"
22+
#include <memory>
23+
24+
namespace google {
25+
namespace cloud {
26+
namespace bigquery_v2_minimal_internal {
27+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
28+
29+
class TableIdempotencyPolicy {
30+
public:
31+
virtual ~TableIdempotencyPolicy();
32+
33+
virtual std::unique_ptr<TableIdempotencyPolicy> clone() const;
34+
35+
virtual google::cloud::Idempotency GetTable(GetTableRequest const& request);
36+
37+
virtual google::cloud::Idempotency ListTables(
38+
ListTablesRequest const& request);
39+
};
40+
41+
std::unique_ptr<TableIdempotencyPolicy> MakeDefaultTableIdempotencyPolicy();
42+
43+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
44+
} // namespace bigquery_v2_minimal_internal
45+
} // namespace cloud
46+
} // namespace google
47+
48+
#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_BIGQUERY_V2_MINIMAL_INTERNAL_TABLE_IDEMPOTENCY_POLICY_H

‎google/cloud/bigquery/v2/minimal/internal/table_idempotency_policy_test.cc‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Copyright 2023 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "google/cloud/bigquery/v2/minimal/internal/table_idempotency_policy.h"
16+
#include "google/cloud/common_options.h"
17+
#include "google/cloud/testing_util/status_matchers.h"
18+
#include <gmock/gmock.h>
19+
20+
namespace google {
21+
namespace cloud {
22+
namespace bigquery_v2_minimal_internal {
23+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
24+
25+
TEST(TableIdempotencyPolicytTest, GetTable) {
26+
auto actual = MakeDefaultTableIdempotencyPolicy();
27+
auto expected = Idempotency::kIdempotent;
28+
29+
GetTableRequest request;
30+
EXPECT_EQ(actual->GetTable(request), expected);
31+
}
32+
33+
TEST(TableIdempotencyPolicytTest, ListTables) {
34+
auto actual = MakeDefaultTableIdempotencyPolicy();
35+
auto expected = Idempotency::kIdempotent;
36+
37+
ListTablesRequest request;
38+
EXPECT_EQ(actual->ListTables(request), expected);
39+
}
40+
41+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
42+
} // namespace bigquery_v2_minimal_internal
43+
} // namespace cloud
44+
} // namespace google

‎google/cloud/bigquery/v2/minimal/internal/table_options.cc‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Copyright 2023 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "google/cloud/bigquery/v2/minimal/internal/table_options.h"
16+
#include "google/cloud/bigquery/v2/minimal/internal/common_options.h"
17+
#include "google/cloud/internal/populate_common_options.h"
18+
#include <memory>
19+
#include <thread>
20+
21+
namespace google {
22+
namespace cloud {
23+
namespace bigquery_v2_minimal_internal {
24+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
25+
26+
Options TableDefaultOptions(Options options) {
27+
options = google::cloud::internal::PopulateCommonOptions(
28+
std::move(options), "GOOGLE_CLOUD_CPP_BIGQUERY_V2_TABLE_ENDPOINT", "",
29+
"GOOGLE_CLOUD_CPP_BIGQUERY_V2_TABLE_AUTHORITY",
30+
"bigquery.googleapis.com");
31+
32+
if (!options.has<TableRetryPolicyOption>()) {
33+
options.set<TableRetryPolicyOption>(
34+
TableLimitedTimeRetryPolicy(std::chrono::minutes(30)).clone());
35+
}
36+
if (!options.has<TableBackoffPolicyOption>()) {
37+
options.set<TableBackoffPolicyOption>(
38+
ExponentialBackoffPolicy(std::chrono::seconds(1),
39+
std::chrono::minutes(5), kBackoffScaling)
40+
.clone());
41+
}
42+
if (!options.has<TableIdempotencyPolicyOption>()) {
43+
options.set<TableIdempotencyPolicyOption>(
44+
MakeDefaultTableIdempotencyPolicy());
45+
}
46+
if (!options.has<TableConnectionPoolSizeOption>()) {
47+
options.set<TableConnectionPoolSizeOption>(DefaultConnectionPoolSize());
48+
}
49+
50+
return options;
51+
}
52+
53+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
54+
} // namespace bigquery_v2_minimal_internal
55+
} // namespace cloud
56+
} // namespace google

0 commit comments

Comments
 (0)