Skip to content

Commit afa1e76

Browse files
coryancoryan
authored
docs: an example to extract Cloud Batch logs (#11935)
1 parent ceacd6b commit afa1e76

3 files changed

Lines changed: 192 additions & 2 deletions

File tree

‎examples/CMakeLists.txt‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,21 @@ if (spanner IN_LIST GOOGLE_CLOUD_CPP_ENABLE AND iam IN_LIST
5151
google_cloud_cpp_add_common_options(grpc_credential_types)
5252
endif ()
5353

54+
if (batch IN_LIST GOOGLE_CLOUD_CPP_ENABLE AND logging IN_LIST
55+
GOOGLE_CLOUD_CPP_ENABLE)
56+
add_executable(batch_logging batch_logging.cc)
57+
target_link_libraries(
58+
batch_logging PRIVATE google_cloud_cpp_testing google-cloud-cpp::batch
59+
google-cloud-cpp::logging)
60+
google_cloud_cpp_add_common_options(batch_logging)
61+
endif ()
62+
5463
# Label all the binaries as "tests", so they run in our CI builds.
5564
if (NOT BUILD_TESTING)
5665
return()
5766
endif ()
5867

59-
foreach (test gcs2cbt grpc_credential_types)
68+
foreach (test batch_logging gcs2cbt grpc_credential_types)
6069
if (NOT TARGET "${test}")
6170
continue()
6271
endif ()

‎examples/batch_logging.cc‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
// Copyright 2023 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// The #includes (with the extra blank line) are part of the code extracted into
16+
// the reference documentation. We want to highlight what includes are needed.
17+
// [START batch_job_logs]
18+
#include "google/cloud/batch/v1/batch_client.h"
19+
#include "google/cloud/logging/v2/logging_service_v2_client.h"
20+
#include "google/cloud/project.h"
21+
22+
// [END batch_job_logs]
23+
#include "google/cloud/common_options.h"
24+
#include "google/cloud/credentials.h"
25+
#include "google/cloud/internal/getenv.h"
26+
#include "google/cloud/internal/random.h"
27+
#include "google/cloud/internal/time_utils.h"
28+
#include "google/cloud/testing_util/example_driver.h"
29+
#include "absl/strings/match.h"
30+
#include <google/protobuf/text_format.h>
31+
#include <fstream>
32+
#include <iostream>
33+
#include <random>
34+
#include <string>
35+
#include <thread>
36+
#include <vector>
37+
38+
namespace {
39+
40+
void JobLogs(std::vector<std::string> const& argv) {
41+
if (argv.size() != 3) {
42+
throw google::cloud::testing_util::Usage{
43+
"job-logs <project-id> <location-id> <job-id>"};
44+
}
45+
// [START batch_job_logs]
46+
[](std::string const& project_id, std::string const& location_id,
47+
std::string const& job_id) {
48+
auto const name = "projects/" + project_id + "/locations/" + location_id +
49+
"/jobs/" + job_id;
50+
auto batch = google::cloud::batch_v1::BatchServiceClient(
51+
google::cloud::batch_v1::MakeBatchServiceConnection());
52+
auto job = batch.GetJob(name);
53+
if (!job) throw std::move(job).status();
54+
55+
auto logging = google::cloud::logging_v2::LoggingServiceV2Client(
56+
google::cloud::logging_v2::MakeLoggingServiceV2Connection());
57+
auto const project = google::cloud::Project(project_id);
58+
auto const log_name = "projects/" + project_id + "/logs/batch_task_logs";
59+
google::logging::v2::ListLogEntriesRequest request;
60+
request.mutable_resource_names()->Add(project.FullName());
61+
request.set_filter("logName=\"" + log_name +
62+
"\" labels.job_uid=" + job->uid());
63+
for (auto l : logging.ListLogEntries(request)) {
64+
if (!l) throw std::move(l).status();
65+
std::cout << l->text_payload() << "\n";
66+
}
67+
}
68+
// [END batch_job_logs]
69+
(argv.at(0), argv.at(1), argv.at(2));
70+
}
71+
72+
// Use the same value as in google/cloud/batch/samples/samples.cc so only one
73+
// of these samples needs to cleanup stale jobs.
74+
auto constexpr kJobPrefix = "batch-examples-";
75+
76+
google::cloud::batch::v1::Job CreateTestJob(
77+
google::cloud::batch_v1::BatchServiceClient client,
78+
std::string const& project_id, std::string const& location_id,
79+
std::string const& job_id) {
80+
google::cloud::batch::v1::CreateJobRequest request;
81+
request.set_parent("projects/" + project_id + "/locations/" + location_id);
82+
request.set_job_id(job_id);
83+
// Most of the job description is fixed in this example; use a string to
84+
// initialize it.
85+
auto constexpr kText = R"pb(
86+
task_groups {
87+
task_count: 4
88+
task_spec {
89+
compute_resource { cpu_milli: 500 memory_mib: 16 }
90+
max_retry_count: 2
91+
max_run_duration { seconds: 3600 }
92+
runnables {
93+
script {
94+
text: "echo Hello world! This is task ${BATCH_TASK_INDEX}. This job has a total of ${BATCH_TASK_COUNT} tasks."
95+
}
96+
}
97+
}
98+
}
99+
allocation_policy {
100+
instances {
101+
policy { machine_type: "e2-standard-4" provisioning_model: STANDARD }
102+
}
103+
}
104+
labels { key: "env" value: "testing" }
105+
labels { key: "type" value: "script" }
106+
logs_policy { destination: CLOUD_LOGGING }
107+
)pb";
108+
auto* job = request.mutable_job();
109+
if (!google::protobuf::TextFormat::ParseFromString(kText, job)) {
110+
throw std::runtime_error("Error parsing Job description");
111+
}
112+
// Create a client and issue the request.
113+
return client.CreateJob(request).value();
114+
}
115+
116+
void WaitForJob(google::cloud::batch_v1::BatchServiceClient client,
117+
std::string job_name) {
118+
std::cout << "\nWaiting for " << job_name << std::flush;
119+
// It takes about 60 seconds to finish a job, so waiting for about 5 minutes
120+
// seems enough.
121+
auto const polling_period = std::chrono::seconds(10);
122+
for (int i = 0; i != 30; ++i) {
123+
auto response = client.GetJob(job_name);
124+
auto done = [](auto state) {
125+
using google::cloud::batch::v1::JobStatus;
126+
return state == JobStatus::SUCCEEDED || state == JobStatus::FAILED;
127+
};
128+
if (response && done(response->status().state())) {
129+
std::cout << ".DONE\n";
130+
return;
131+
}
132+
std::cout << "." << std::flush;
133+
std::this_thread::sleep_for(polling_period);
134+
}
135+
std::cout << ".DONE (TIMEOUT)\n";
136+
throw std::runtime_error("Timeout waiting for job");
137+
}
138+
139+
void AutoRun(std::vector<std::string> const& argv) {
140+
namespace examples = ::google::cloud::testing_util;
141+
using ::google::cloud::internal::GetEnv;
142+
if (!argv.empty()) throw examples::Usage{"auto"};
143+
examples::CheckEnvironmentVariablesAreSet({
144+
"GOOGLE_CLOUD_PROJECT",
145+
"GOOGLE_CLOUD_CPP_TEST_REGION",
146+
});
147+
auto const project_id = GetEnv("GOOGLE_CLOUD_PROJECT").value();
148+
auto const location_id = GetEnv("GOOGLE_CLOUD_CPP_TEST_REGION").value();
149+
150+
auto generator = google::cloud::internal::DefaultPRNG(std::random_device{}());
151+
auto const job_id =
152+
kJobPrefix + google::cloud::internal::Sample(
153+
generator, 32, "abcdefghijklmnopqrstuvwxyz");
154+
155+
auto client = google::cloud::batch_v1::BatchServiceClient(
156+
google::cloud::batch_v1::MakeBatchServiceConnection());
157+
158+
// Create the job to drive this test.
159+
auto job = CreateTestJob(client, project_id, location_id, job_id);
160+
std::cout << "Created test job: " << job.name() << "\n";
161+
162+
// Wait until the job completes, otherwise the logs may be empty. The logs may
163+
// still be delayed, but this is less likely.
164+
WaitForJob(client, job.name());
165+
166+
std::cout << "Running JobLogs() test" << std::endl;
167+
JobLogs({project_id, location_id, job_id});
168+
169+
(void)client.DeleteJob(job.name()).get().value();
170+
std::cout << "Deleted test job: " << job.name() << "\n";
171+
}
172+
173+
} // namespace
174+
175+
int main(int argc, char* argv[]) { // NOLINT(bugprone-exception-escape)
176+
google::cloud::testing_util::Example example({
177+
{"job-logs", JobLogs},
178+
{"auto", AutoRun},
179+
});
180+
return example.Run(argc, argv);
181+
}

‎google/cloud/batch/v1/samples/samples.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -485,7 +485,7 @@ void AutoRun(std::vector<std::string> const& argv) {
485485
bool success = false;
486486
auto const name = "projects/" + project_id + "/locations/" + location_id +
487487
"/jobs/" + script_job_id;
488-
// It taskes about 60 seconds to finish a job, so waiting for about 5 minutes
488+
// It takes about 60 seconds to finish a job, so waiting for about 5 minutes
489489
// seems enough.
490490
auto const polling_period = std::chrono::seconds(10);
491491
for (int i = 0; i != 30; ++i) {

0 commit comments

Comments
 (0)