Skip to content

Commit d8d6945

Browse files
software-dovsoftware-dov
authored
Add tests and impl for Pages class and associated iterator (googleapis#55)
The Pages class is the next level up in the PaginatedResult abstraction hierarchy above PageResult. It provides: * a single constructor that takes a (copied) closure that retrieves successive pages from the API service. * iterator-returning begin() and end() methods. Includessome changes in PageResult: ElementAccessor is intended to be a lightweight, stateless, uniform wrapper, and isnow used purely as a default-constructed throwaway.
1 parent 2d92299 commit d8d6945

2 files changed

Lines changed: 271 additions & 25 deletions

File tree

‎gax/pagination.h‎

Lines changed: 196 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,39 @@
1616
#define GAPIC_GENERATOR_CPP_GAX_PAGINATION_H_
1717

1818
#include "gax/internal/invoke_result.h"
19+
#include "gax/status.h"
1920
#include <google/protobuf/repeated_field.h>
20-
#include <functional>
2121
#include <iterator>
2222
#include <string>
2323
#include <type_traits>
2424

2525
namespace google {
2626
namespace gax {
2727

28-
template <typename ElementType, typename PageType, typename ElementAccessor>
28+
/**
29+
* Wraps a 'page' message with a consistent interface that provides an iterator
30+
* over its repeated elements and an accessor for its next_page_token field.
31+
*
32+
* Picking a repeated field, deducing the type of its elements, and providing an
33+
* ElementAccessor functor that references the field is outside the scope of the
34+
* template.
35+
*
36+
* @tparam ElementType the type of the repeated elements in the page.
37+
* @tparam PageType the type of the wrapped message.
38+
* @tparam ElementAccessor a default-constructable functor that has an overload
39+
* of operator() that takes a PageType& and returns a mutable pointer to the
40+
* repeated field that contains ElementType.
41+
*
42+
* Note: if there is more than one repeated field in a PageType message, all but
43+
* one will be efectively hidden except through the RawPage accessor.
44+
*/
45+
template <
46+
typename ElementType, typename PageType, typename ElementAccessor,
47+
typename std::enable_if<
48+
gax::internal::is_invocable<ElementAccessor, PageType&>::value,
49+
int>::type = 0,
50+
typename std::enable_if<
51+
std::is_default_constructible<ElementAccessor>::value, int>::type = 0>
2952
class PageResult {
3053
using FieldIterator =
3154
typename std::remove_pointer<typename gax::internal::invoke_result_t<
@@ -40,8 +63,8 @@ class PageResult {
4063
using pointer = ElementType*;
4164
using reference = ElementType&;
4265

43-
reference operator*() const { return *current_; }
44-
pointer operator->() const { return &(*current_); }
66+
ElementType& operator*() const { return *current_; }
67+
ElementType* operator->() const { return &(*current_); }
4568
iterator& operator++() {
4669
++current_;
4770
return *this;
@@ -51,25 +74,186 @@ class PageResult {
5174
}
5275
bool operator!=(iterator const& rhs) const { return !(*this == rhs); }
5376

77+
private:
78+
friend PageResult;
5479
iterator(FieldIterator current) : current_(std::move(current)) {}
5580

5681
FieldIterator current_;
5782
};
58-
// Note: Since the constructor will eventually be private (with friend
59-
// access), we don't have to define all the variants for references.
60-
// We just want to take ownership of a page.
61-
PageResult(PageType&& raw_page, ElementAccessor accessor)
62-
: raw_page_(std::move(raw_page)), accessor_(accessor) {}
6383

64-
iterator begin() { return accessor_(raw_page_)->begin(); }
65-
iterator end() { return accessor_(raw_page_)->end(); }
84+
PageResult(PageType const& raw_page) : raw_page_(raw_page) {}
85+
PageResult(PageType&& raw_page) : raw_page_(std::move(raw_page)) {}
6686

87+
iterator begin() { return ElementAccessor{}(raw_page_)->begin(); }
88+
iterator end() { return ElementAccessor{}(raw_page_)->end(); }
89+
90+
// Const overloads
91+
iterator begin() const { return ElementAccessor{}(raw_page_)->cbegin(); }
92+
iterator end() const { return ElementAccessor{}(raw_page_)->cend(); }
93+
94+
/**
95+
* @brief Get the next_page_token for the page.
96+
*
97+
* @retun the token for the next page in the sequence
98+
* or the empty string if this is the last page.
99+
*/
67100
std::string NextPageToken() const { return raw_page_.next_page_token(); }
101+
102+
/**
103+
* @brief Get the underlying page message.
104+
*
105+
* This can be useful if the page has fields other than the repeated field or
106+
* next page token.
107+
*
108+
* @return a const reference to the underlying raw page.
109+
*/
68110
PageType const& RawPage() const { return raw_page_; }
69111

112+
// Note: the non-const variant is intended for internal use only.
113+
PageType& RawPage() { return raw_page_; }
114+
70115
private:
71116
PageType raw_page_;
72-
ElementAccessor accessor_;
117+
};
118+
119+
/**
120+
* Wraps a sequence of pages implied to be serially returned by a paginated API
121+
* method and provides an iterator that retrieves subsequent pages, usually via
122+
* synchronous rpc.
123+
*
124+
* Determining whether a protobuf message constitutes a valid PageType is
125+
* outside the scope of the template code.
126+
*
127+
* @par Example
128+
*
129+
* @code
130+
* ListElementsRequest request;
131+
* // Set up request to describe the right resource
132+
* // and optionally customize the initial page token or page size.
133+
*
134+
* class ElementsAccessor {
135+
* public:
136+
* protobuf::RepeatedPtrField<Element>*
137+
* operator()(ListElementsResponse* response) {
138+
* return response.mutable_elements();
139+
* }
140+
* };
141+
*
142+
* auto get_next_page = [request, stub](ListElementsResponse* response) mutable
143+
* {
144+
* gax::CallContext ctx;
145+
* gax::Status status = stub->ListElements(context, request, response);
146+
* request.set_next_page_token(response->next_page_token());
147+
* return status;
148+
* };
149+
*
150+
* // Only list up to 20 pages, even though there may be more.
151+
* Pages<EltType, ListElementsResponse, decltype(get_next_page),
152+
* ElementsAccessor> pages(std::move(get_next_page), 20);
153+
* for(auto& page : pages) {
154+
* // Do something with the page
155+
* }
156+
* @endcode
157+
*
158+
* @tparam ElementType the type of the repeated elements in the page.
159+
* @tparam PageType thye type of the wrapped message.
160+
* @tparam ElementAccessor a default-constructable functor that has an overload
161+
* of operator() that takes a PageType& and returns a mutable pointer to the
162+
* repeated field that contains ElementType.
163+
* @tparam NextPageRetriever a copy-constructable functor that has an overload
164+
* of operator() that takes a mutable PageType*, initiates an rpc that
165+
* overwrites the contents of the page with the next page, and returns a
166+
* gax::Status indicating the success or failure of the rpc.
167+
*
168+
* Note: the initial page request MUST be captured by value in the
169+
* NextPageRetriever functor so that calling begin() multiple times on a Pages
170+
* instance results in valid behavior.
171+
*/
172+
template <typename ElementType, typename PageType, typename ElementAccessor,
173+
typename NextPageRetriever,
174+
typename std::enable_if<
175+
gax::internal::is_invocable<NextPageRetriever, PageType*>::value,
176+
int>::type = 0,
177+
typename std::enable_if<
178+
std::is_copy_constructible<NextPageRetriever>::value, int>::type =
179+
0>
180+
class Pages {
181+
public:
182+
class iterator {
183+
public:
184+
using PageResultT = PageResult<ElementType, PageType, ElementAccessor>;
185+
using iterator_category = std::forward_iterator_tag;
186+
using value_type = PageResultT;
187+
using difference_type = std::ptrdiff_t;
188+
using pointer = PageResultT*;
189+
using reference = PageResultT&;
190+
191+
PageResultT const& operator*() const { return page_result_; }
192+
PageResultT const* operator->() const { return &page_result_; }
193+
iterator& operator++() {
194+
// Note: if the rpc fails, the page will be untouched,
195+
// i.e. will have an empty page token and element collection.
196+
// This invalidates any iterators on the PageResult.
197+
page_result_.RawPage().Clear();
198+
get_next_page_(&(page_result_.RawPage()));
199+
num_pages_++;
200+
return *this;
201+
}
202+
203+
// Just want to compare against end()
204+
bool operator==(iterator const& rhs) const {
205+
return num_pages_ == rhs.num_pages_ ||
206+
page_result_.NextPageToken() == rhs.page_result_.NextPageToken();
207+
}
208+
bool operator!=(iterator const& rhs) const { return !(*this == rhs); }
209+
210+
private:
211+
friend Pages;
212+
// Note: copying a message with many repeated elements is expensive.
213+
// Callers should move pages in when instantiating an iterator.
214+
iterator(PageType page_result, NextPageRetriever get_next_page,
215+
int num_pages)
216+
: page_result_(std::move(page_result)),
217+
get_next_page_(std::move(get_next_page)),
218+
num_pages_(num_pages) {}
219+
220+
PageResultT page_result_;
221+
NextPageRetriever get_next_page_;
222+
int num_pages_;
223+
};
224+
225+
/**
226+
* Create a new Pages instance using the provided page retrieval functor.
227+
* Iterators on the instance will yield a number of pages up to the page cap.
228+
*
229+
* @param get_next_page an instance of the page retrieval functor.
230+
* @param pages_cap the maximum number of pages to retrieve. A value of 0
231+
* (default) indicates no cap.
232+
*/
233+
Pages(NextPageRetriever get_next_page, int pages_cap = 0)
234+
: get_next_page_(std::move(get_next_page)), pages_cap_(pages_cap) {}
235+
236+
iterator begin() const {
237+
PageType page;
238+
// Copying the next-page lambda is necessary to start at the beginning.
239+
NextPageRetriever fresh_get_next_page_(get_next_page_);
240+
fresh_get_next_page_(&page);
241+
242+
return iterator(std::move(page), std::move(fresh_get_next_page_), 1);
243+
}
244+
245+
iterator end() const {
246+
return iterator{PageType{}, get_next_page_, pages_cap_};
247+
}
248+
249+
private:
250+
// Note: be sure to capture the initial page request by value so that calling
251+
// begin() multiple times is valid.
252+
// This means that whenever get_next_page_ is copied, i.e. whenever the user
253+
// calls Pages::begin(), a fresh copy of the initial request gets created,
254+
// which means that begin() _really_ starts at the beginning.
255+
NextPageRetriever get_next_page_;
256+
const int pages_cap_;
73257
};
74258

75259
} // namespace gax

‎gax/pagination_test.cc‎

Lines changed: 75 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
#include "gax/pagination.h"
1616
#include "google/longrunning/operations.pb.h"
17+
#include "gax/status.h"
1718
#include <google/protobuf/util/message_differencer.h>
1819
#include <gmock/gmock-matchers.h>
1920
#include <gtest/gtest.h>
@@ -25,33 +26,56 @@ namespace {
2526

2627
using namespace ::google;
2728

28-
// Protobuf messages are not equality comparible by default.
29-
// This complicates testing, so just define a comparison function here.
30-
bool Equal(longrunning::Operation const& lhs,
31-
longrunning::Operation const& rhs) {
32-
return protobuf::util::MessageDifferencer::Equals(lhs, rhs);
33-
}
29+
class OperationsAccessor {
30+
public:
31+
protobuf::RepeatedPtrField<longrunning::Operation>* operator()(
32+
longrunning::ListOperationsResponse& lor) const {
33+
return lor.mutable_operations();
34+
}
35+
};
36+
37+
class PageRetriever {
38+
public:
39+
// Start at 1 to count number of pages seen total, including the first.
40+
PageRetriever(int max_pages) : i_(1), max_pages_(max_pages) {}
41+
gax::Status operator()(longrunning::ListOperationsResponse* lor) {
42+
if (i_ < max_pages_) {
43+
std::stringstream ss;
44+
ss << "NextPage" << i_;
45+
lor->set_next_page_token(ss.str());
46+
i_++;
47+
} else {
48+
lor->clear_next_page_token();
49+
}
50+
51+
return gax::Status{};
52+
}
3453

35-
auto constexpr accessor = [](longrunning::ListOperationsResponse& lor) {
36-
return lor.mutable_operations();
54+
private:
55+
int i_;
56+
const int max_pages_;
3757
};
3858

59+
using TestPages =
60+
gax::Pages<longrunning::Operation, longrunning::ListOperationsResponse,
61+
OperationsAccessor, PageRetriever>;
62+
3963
using TestedPageResult =
4064
gax::PageResult<longrunning::Operation, longrunning::ListOperationsResponse,
41-
decltype(accessor)>;
65+
OperationsAccessor>;
4266

43-
TestedPageResult MakeTestedPageResult() {
67+
TestedPageResult MakeTestedPageResult(int num_pages = 10) {
4468
longrunning::ListOperationsResponse response;
4569
response.set_next_page_token("NextPage");
4670

47-
for (int i = 0; i < 10; i++) {
71+
for (int i = 0; i < num_pages; i++) {
4872
std::stringstream ss;
4973
ss << "TestOperation" << i;
5074
auto operation = response.add_operations();
5175
operation->set_name(ss.str());
5276
}
5377

54-
return TestedPageResult(std::move(response), accessor);
78+
return TestedPageResult(std::move(response));
5579
}
5680

5781
TEST(PageResult, RawPage) {
@@ -79,7 +103,7 @@ TEST(PageResult, BasicIteration) {
79103
// Note: cannot use EXPECT_EQ for the elements or on vectors constructed
80104
// from the respective iterators because messages do not define operator==
81105
// as a member function.
82-
EXPECT_TRUE(Equal(*prIt, *reIt));
106+
EXPECT_TRUE(protobuf::util::MessageDifferencer::Equals(*prIt, *reIt));
83107
}
84108
EXPECT_EQ(prIt, page_result.end());
85109
EXPECT_EQ(reIt, page_result.RawPage().operations().end());
@@ -93,4 +117,42 @@ TEST(PageResult, MoveIteration) {
93117
EXPECT_EQ(page_result.begin()->name(), "");
94118
}
95119

120+
TEST(Pages, Basic) {
121+
TestPages terminal(
122+
// The output param is pristine, which means its next_page_token
123+
// is empty.
124+
PageRetriever(0));
125+
126+
EXPECT_EQ(terminal.begin(), terminal.end());
127+
EXPECT_EQ(terminal.end()->NextPageToken(), "");
128+
}
129+
130+
TEST(Pages, Iteration) {
131+
int i = 1;
132+
TestPages pages(PageRetriever(10));
133+
for (auto const& p : pages) {
134+
std::stringstream ss;
135+
ss << "NextPage" << i;
136+
137+
EXPECT_EQ(p.NextPageToken(), ss.str());
138+
i++;
139+
}
140+
EXPECT_EQ(i, 10);
141+
}
142+
143+
TEST(Pages, PageCap) {
144+
int i = 1;
145+
TestPages pages(PageRetriever(10), 5);
146+
auto iter = pages.begin();
147+
for (; iter != pages.end(); ++iter) {
148+
std::stringstream ss;
149+
ss << "NextPage" << i;
150+
151+
EXPECT_EQ(iter->NextPageToken(), ss.str());
152+
i++;
153+
}
154+
EXPECT_EQ(i, 5);
155+
EXPECT_EQ(iter->NextPageToken(), "NextPage5");
156+
}
157+
96158
} // namespace

0 commit comments

Comments
 (0)