Skip to content

Commit 7a691da

Browse files
jsrinnnjsrinnn
authored
impl(bigquery): V2 GetJob api - Implementation (#10953)
* impl(bigquery): V2 GetJob api - implementation of Custom job class, parsing of http response, rest response, and rest stub implementation with unit tests
1 parent 8319c1f commit 7a691da

18 files changed

Lines changed: 976 additions & 40 deletions

‎google/cloud/bigquery/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ load(":bigquery_rest_unit_tests.bzl", "bigquery_rest_unit_tests")
121121
deps = [
122122
":google_cloud_cpp_bigquery_rest",
123123
"//google/cloud/testing_util:google_cloud_cpp_testing_private",
124+
"//google/cloud/testing_util:google_cloud_cpp_testing_rest_private",
124125
"@com_google_googletest//:gtest",
125126
],
126127
) for test in bigquery_rest_unit_tests]

‎google/cloud/bigquery/bigquery_rest.cmake‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@ add_library(
1818
google_cloud_cpp_bigquery_rest # cmake-format: sort
1919
v2/minimal/internal/bigquery_http_response.cc
2020
v2/minimal/internal/bigquery_http_response.h
21+
v2/minimal/internal/common_v2_resources.cc
22+
v2/minimal/internal/common_v2_resources.h
2123
v2/minimal/internal/job.h
24+
v2/minimal/internal/job_configuration.h
2225
v2/minimal/internal/job_idempotency_policy.cc
2326
v2/minimal/internal/job_idempotency_policy.h
2427
v2/minimal/internal/job_logging.cc
@@ -74,6 +77,7 @@ function (bigquery_rest_define_tests)
7477
set(bigquery_rest_unit_tests
7578
# cmake-format: sort
7679
v2/minimal/internal/bigquery_http_response_test.cc
80+
v2/minimal/internal/common_v2_resources_test.cc
7781
v2/minimal/internal/job_request_test.cc
7882
v2/minimal/internal/job_response_test.cc
7983
v2/minimal/internal/job_rest_stub_test.cc)

‎google/cloud/bigquery/bigquery_rest_unit_tests.bzl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
bigquery_rest_unit_tests = [
2020
"v2/minimal/internal/bigquery_http_response_test.cc",
21+
"v2/minimal/internal/common_v2_resources_test.cc",
2122
"v2/minimal/internal/job_request_test.cc",
2223
"v2/minimal/internal/job_response_test.cc",
2324
"v2/minimal/internal/job_rest_stub_test.cc",

‎google/cloud/bigquery/google_cloud_cpp_bigquery_rest.bzl‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@
1818

1919
google_cloud_cpp_bigquery_rest_hdrs = [
2020
"v2/minimal/internal/bigquery_http_response.h",
21+
"v2/minimal/internal/common_v2_resources.h",
2122
"v2/minimal/internal/job.h",
23+
"v2/minimal/internal/job_configuration.h",
2224
"v2/minimal/internal/job_idempotency_policy.h",
2325
"v2/minimal/internal/job_logging.h",
2426
"v2/minimal/internal/job_metadata.h",
@@ -33,6 +35,7 @@ google_cloud_cpp_bigquery_rest_hdrs = [
3335

3436
google_cloud_cpp_bigquery_rest_srcs = [
3537
"v2/minimal/internal/bigquery_http_response.cc",
38+
"v2/minimal/internal/common_v2_resources.cc",
3639
"v2/minimal/internal/job_idempotency_policy.cc",
3740
"v2/minimal/internal/job_logging.cc",
3841
"v2/minimal/internal/job_metadata.cc",

‎google/cloud/bigquery/v2/minimal/internal/bigquery_http_response.cc‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,32 @@
1313
// limitations under the License.
1414

1515
#include "google/cloud/bigquery/v2/minimal/internal/bigquery_http_response.h"
16+
#include "google/cloud/internal/make_status.h"
1617

1718
namespace google {
1819
namespace cloud {
1920
namespace bigquery_v2_minimal_internal {
2021
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
2122

23+
namespace rest = ::google::cloud::rest_internal;
24+
2225
StatusOr<BigQueryHttpResponse> BigQueryHttpResponse::BuildFromRestResponse(
23-
rest_internal::RestResponse& rest_response) {
26+
std::unique_ptr<rest::RestResponse> rest_response) {
2427
BigQueryHttpResponse response;
25-
auto payload =
26-
rest_internal::ReadAll(std::move(rest_response).ExtractPayload());
27-
if (!payload.ok()) return payload.status();
28+
if (rest_response == nullptr) {
29+
return internal::InvalidArgumentError(
30+
"RestResponse argument passed in is null", GCP_ERROR_INFO());
31+
}
32+
if (rest::IsHttpError(*rest_response)) {
33+
return rest::AsStatus(std::move(*rest_response));
34+
}
35+
response.http_status_code = rest_response->StatusCode();
36+
response.http_headers = rest_response->Headers();
37+
38+
auto payload = rest::ReadAll(std::move(*rest_response).ExtractPayload());
39+
if (!payload) return std::move(payload).status();
40+
41+
response.payload = std::move(*payload);
2842
return response;
2943
}
3044

‎google/cloud/bigquery/v2/minimal/internal/bigquery_http_response.h‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,15 @@ namespace cloud {
2424
namespace bigquery_v2_minimal_internal {
2525
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
2626

27-
// Parses the RestResponse and builds an BigQueryHttpResponse.
28-
struct BigQueryHttpResponse {
29-
// Builds BigqueryHttpResponse from RestResponse.
27+
class BigQueryHttpResponse {
28+
public:
29+
BigQueryHttpResponse() = default;
30+
// Parses the RestResponse and builds an BigQueryHttpResponse.
3031
static StatusOr<BigQueryHttpResponse> BuildFromRestResponse(
31-
rest_internal::RestResponse& rest_response);
32+
std::unique_ptr<rest_internal::RestResponse> rest_response);
3233

3334
rest_internal::HttpStatusCode http_status_code;
34-
std::unordered_map<std::string, std::string> http_headers;
35+
std::multimap<std::string, std::string> http_headers;
3536
std::string payload;
3637
};
3738

‎google/cloud/bigquery/v2/minimal/internal/bigquery_http_response_test.cc‎

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,85 @@
1313
// limitations under the License.
1414

1515
#include "google/cloud/bigquery/v2/minimal/internal/bigquery_http_response.h"
16+
#include "google/cloud/internal/http_payload.h"
17+
#include "google/cloud/internal/make_status.h"
18+
#include "google/cloud/internal/rest_response.h"
19+
#include "google/cloud/testing_util/mock_http_payload.h"
20+
#include "google/cloud/testing_util/mock_rest_response.h"
21+
#include "google/cloud/testing_util/status_matchers.h"
1622
#include <gmock/gmock.h>
1723

1824
namespace google {
1925
namespace cloud {
2026
namespace bigquery_v2_minimal_internal {
2127
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
2228

23-
TEST(BigQueryHttpResponseTest, BuildFromRestResponseSuccess) {}
29+
using ::google::cloud::rest_internal::HttpStatusCode;
30+
using ::google::cloud::testing_util::MakeMockHttpPayloadSuccess;
31+
using ::google::cloud::testing_util::MockHttpPayload;
32+
using ::google::cloud::testing_util::MockRestResponse;
33+
using ::google::cloud::testing_util::StatusIs;
34+
using ::testing::ByMove;
35+
using ::testing::Eq;
36+
using ::testing::HasSubstr;
37+
using ::testing::Return;
2438

25-
TEST(BigQueryHttpResponseTest, BuildFromRestResponseFailure) {}
39+
TEST(BigQueryHttpResponseTest, Success) {
40+
std::string job_response_payload = "My code my code my kingdom for code!";
41+
42+
auto mock_response = absl::make_unique<MockRestResponse>();
43+
EXPECT_CALL(*mock_response, StatusCode)
44+
.WillRepeatedly(Return(HttpStatusCode::kOk));
45+
EXPECT_CALL(*mock_response, Headers)
46+
.WillRepeatedly(Return(std::multimap<std::string, std::string>()));
47+
EXPECT_CALL(std::move(*mock_response), ExtractPayload)
48+
.WillRepeatedly(
49+
Return(ByMove(MakeMockHttpPayloadSuccess(job_response_payload))));
50+
51+
auto http_response =
52+
BigQueryHttpResponse::BuildFromRestResponse(std::move(mock_response));
53+
ASSERT_STATUS_OK(http_response);
54+
EXPECT_TRUE(http_response->http_headers.empty());
55+
EXPECT_THAT(http_response->payload, Eq(job_response_payload));
56+
}
57+
58+
TEST(BigQueryHttpResponseTest, HttpError) {
59+
auto mock_payload = absl::make_unique<MockHttpPayload>();
60+
auto mock_response = absl::make_unique<MockRestResponse>();
61+
EXPECT_CALL(*mock_response, StatusCode)
62+
.WillRepeatedly(Return(HttpStatusCode::kBadRequest));
63+
EXPECT_CALL(std::move(*mock_response), ExtractPayload)
64+
.WillOnce(Return(std::move(mock_payload)));
65+
auto http_response =
66+
BigQueryHttpResponse::BuildFromRestResponse(std::move(mock_response));
67+
EXPECT_THAT(http_response, StatusIs(StatusCode::kInvalidArgument,
68+
HasSubstr("Received HTTP status code")));
69+
}
70+
71+
TEST(BigQueryHttpResponseTest, PayloadError) {
72+
auto mock_payload = absl::make_unique<MockHttpPayload>();
73+
EXPECT_CALL(*mock_payload, Read).WillRepeatedly([](absl::Span<char> const&) {
74+
return internal::AbortedError("invalid payload", GCP_ERROR_INFO());
75+
});
76+
77+
auto mock_response = absl::make_unique<MockRestResponse>();
78+
EXPECT_CALL(*mock_response, StatusCode)
79+
.WillRepeatedly(Return(HttpStatusCode::kOk));
80+
EXPECT_CALL(std::move(*mock_response), ExtractPayload)
81+
.WillOnce(Return(std::move(mock_payload)));
82+
auto http_response =
83+
BigQueryHttpResponse::BuildFromRestResponse(std::move(mock_response));
84+
EXPECT_THAT(http_response,
85+
StatusIs(StatusCode::kAborted, HasSubstr("invalid payload")));
86+
}
87+
88+
TEST(BigQueryHttpResponseTest, NullPtr) {
89+
auto http_response = BigQueryHttpResponse::BuildFromRestResponse(nullptr);
90+
EXPECT_FALSE(http_response.ok());
91+
EXPECT_THAT(http_response,
92+
StatusIs(StatusCode::kInvalidArgument,
93+
HasSubstr("RestResponse argument passed in is null")));
94+
}
2695

2796
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
2897
} // namespace bigquery_v2_minimal_internal

‎google/cloud/bigquery/v2/minimal/internal/common_v2_resources.cc‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
// Copyright 2023 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "google/cloud/bigquery/v2/minimal/internal/common_v2_resources.h"
16+
17+
namespace google {
18+
namespace cloud {
19+
namespace bigquery_v2_minimal_internal {
20+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
21+
22+
// Bypass clang-tidy warnings for self-referential and recursive structures.
23+
24+
// NOLINTBEGIN
25+
struct QueryParameterType;
26+
27+
void to_json(nlohmann::json& j, QueryParameterStructType const& q) {
28+
if (q.type != nullptr) {
29+
j = nlohmann::json{
30+
{"name", q.name}, {"type", *q.type}, {"description", q.description}};
31+
} else {
32+
j = nlohmann::json{{"name", q.name}, {"description", q.description}};
33+
}
34+
}
35+
36+
void from_json(nlohmann::json const& j, QueryParameterStructType& q) {
37+
if (j.contains("name")) j.at("name").get_to(q.name);
38+
if (j.contains("type")) {
39+
if (q.type == nullptr) {
40+
q.type = std::make_shared<QueryParameterType>();
41+
}
42+
j.at("type").get_to(*q.type);
43+
}
44+
if (j.contains("description")) j.at("description").get_to(q.description);
45+
}
46+
47+
void to_json(nlohmann::json& j, QueryParameterType const& q) {
48+
if (q.array_type != nullptr) {
49+
j = nlohmann::json{{"type", q.type},
50+
{"array_type", *q.array_type},
51+
{"struct_types", q.struct_types}};
52+
} else {
53+
j = nlohmann::json{{"type", q.type}, {"struct_types", q.struct_types}};
54+
}
55+
}
56+
57+
void from_json(nlohmann::json const& j, QueryParameterType& q) {
58+
if (j.contains("type")) j.at("type").get_to(q.type);
59+
if (j.contains("array_type")) {
60+
if (q.array_type == nullptr) {
61+
q.array_type = std::make_shared<QueryParameterType>();
62+
}
63+
j.at("array_type").get_to(*q.array_type);
64+
}
65+
if (j.contains("struct_types")) j.at("struct_types").get_to(q.struct_types);
66+
}
67+
68+
void to_json(nlohmann::json& j, QueryParameterValue const& q) {
69+
j = nlohmann::json{{"value", q.value},
70+
{"array_values", q.array_values},
71+
{"struct_values", q.struct_values}};
72+
}
73+
74+
void from_json(nlohmann::json const& j, QueryParameterValue& q) {
75+
if (j.contains("value")) j.at("value").get_to(q.value);
76+
if (j.contains("array_values")) j.at("array_values").get_to(q.array_values);
77+
if (j.contains("struct_values"))
78+
j.at("struct_values").get_to(q.struct_values);
79+
}
80+
// NOLINTEND
81+
82+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
83+
} // namespace bigquery_v2_minimal_internal
84+
} // namespace cloud
85+
} // namespace google

‎google/cloud/bigquery/v2/minimal/internal/common_v2_resources.h‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
// Copyright 2023 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_BIGQUERY_V2_MINIMAL_INTERNAL_COMMON_V2_RESOURCES_H
16+
#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_BIGQUERY_V2_MINIMAL_INTERNAL_COMMON_V2_RESOURCES_H
17+
18+
#include "google/cloud/version.h"
19+
#include <nlohmann/json.hpp>
20+
#include <string>
21+
22+
namespace google {
23+
namespace cloud {
24+
namespace bigquery_v2_minimal_internal {
25+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
26+
27+
// Disabling clang-tidy here as the namespace is needed for using the
28+
// NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT.
29+
using namespace nlohmann::literals; // NOLINT
30+
31+
struct ErrorProto {
32+
std::string reason;
33+
std::string location;
34+
std::string message;
35+
};
36+
37+
struct TableReference {
38+
std::string project_id;
39+
std::string dataset_id;
40+
std::string table_id;
41+
};
42+
43+
struct DatasetReference {
44+
std::string dataset_id;
45+
std::string project_id;
46+
};
47+
48+
// Self referential and circular structures below is based on the bigquery v2
49+
// QueryParameterType and QueryParameterValue proto definitions. Disabling
50+
// clang-tidy warnings for these two types as we want to be close to the
51+
// protobuf definition as possible.
52+
53+
// NOLINTBEGIN
54+
struct QueryParameterType;
55+
56+
struct QueryParameterStructType {
57+
std::string name;
58+
std::shared_ptr<QueryParameterType> type;
59+
std::string description;
60+
};
61+
62+
struct QueryParameterType {
63+
std::string type;
64+
65+
std::shared_ptr<QueryParameterType> array_type;
66+
std::vector<QueryParameterStructType> struct_types;
67+
};
68+
69+
struct QueryParameterValue {
70+
std::string value;
71+
std::vector<QueryParameterValue> array_values;
72+
std::map<std::string, QueryParameterValue> struct_values;
73+
};
74+
// NOLINTEND
75+
76+
struct QueryParameter {
77+
std::string name;
78+
QueryParameterType parameter_type;
79+
QueryParameterValue parameter_value;
80+
};
81+
82+
struct ConnectionProperty {
83+
std::string key;
84+
std::string value;
85+
};
86+
87+
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(ErrorProto, reason, location,
88+
message);
89+
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(TableReference, project_id,
90+
dataset_id, table_id);
91+
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(DatasetReference, project_id,
92+
dataset_id);
93+
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(ConnectionProperty, key, value);
94+
95+
void to_json(nlohmann::json& j, QueryParameterType const& q);
96+
97+
void from_json(nlohmann::json const& j, QueryParameterType& q);
98+
99+
void to_json(nlohmann::json& j, QueryParameterValue const& q);
100+
101+
void from_json(nlohmann::json const& j, QueryParameterValue& q);
102+
103+
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(QueryParameter, name,
104+
parameter_type,
105+
parameter_value);
106+
107+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
108+
} // namespace bigquery_v2_minimal_internal
109+
} // namespace cloud
110+
} // namespace google
111+
112+
#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_BIGQUERY_V2_MINIMAL_INTERNAL_COMMON_V2_RESOURCES_H

0 commit comments

Comments
 (0)