From 66cc1a039d7f1190dc96cde20a78c1d1cd40ff94 Mon Sep 17 00:00:00 2001 From: sumit-ql Date: Sun, 16 Sep 2018 19:32:24 +0530 Subject: [PATCH 1/8] Using RowSet in scan method.Using read_rows() instead of yield_rows() --- src/google/cloud/happybase/table.py | 19 ++++++++++++++----- unit_tests/test_table.py | 25 ++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/google/cloud/happybase/table.py b/src/google/cloud/happybase/table.py index b9c0715..9fbed69 100644 --- a/src/google/cloud/happybase/table.py +++ b/src/google/cloud/happybase/table.py @@ -35,6 +35,8 @@ from google.cloud.bigtable.row_filters import TimestampRange from google.cloud.bigtable.row_filters import TimestampRangeFilter from google.cloud.bigtable.table import Table as _LowLevelTable +from google.cloud.bigtable.row_set import RowSet +from google.cloud.bigtable.row_set import RowRange from google.cloud.happybase.batch import _get_column_pairs from google.cloud.happybase.batch import _WAL_SENTINEL @@ -235,7 +237,7 @@ def rows(self, rows, columns=None, timestamp=None, # versions == 1 since we only want the latest. filter_ = _filter_chain_helper(versions=1, timestamp=timestamp, filters=filters) - rows_generator = self._low_level_table.yield_rows( + rows_generator = self._low_level_table.read_rows( filter_=filter_) # NOTE: We could use max_loops = 1000 or some similar value to ensure # that the stream isn't open too long. @@ -380,10 +382,11 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, """ row_start, row_stop, filter_chain = _scan_filter_helper( row_start, row_stop, row_prefix, columns, timestamp, limit, kwargs) - - rows_generator = self._low_level_table.yield_rows( - start_key=row_start, end_key=row_stop, - limit=limit, filter_=filter_chain) + + row_set = _get_row_set_object(row_start, row_stop) + + rows_generator = self._low_level_table.read_rows( + row_set=row_set, limit=limit, filter_=filter_chain) for rowdata in rows_generator: curr_row_data = rowdata @@ -969,3 +972,9 @@ def _row_keys_filter_helper(row_keys): return filters[0] else: return RowFilterUnion(filters=filters) + + +def _get_row_set_object(row_start, row_stop): + row_set = RowSet() + row_set.add_row_range(RowRange(row_start, row_stop)) + return row_set \ No newline at end of file diff --git a/unit_tests/test_table.py b/unit_tests/test_table.py index 8edf317..04e06d7 100644 --- a/unit_tests/test_table.py +++ b/unit_tests/test_table.py @@ -594,6 +594,8 @@ def _scan_test_helper(self, row_limits=(None, None), row_prefix=None, import types from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT + from google.cloud.bigtable.row_set import RowSet + from google.cloud.bigtable.row_set import RowRange name = 'table-name' row_start, row_stop = row_limits @@ -618,9 +620,16 @@ def mock_columns_filter_helper(*args): def mock_filter_chain_helper(**kwargs): mock_filters.append(kwargs) return fake_filter - + + row_set = RowSet() + row_set.add_row_range(RowRange(row_start, row_stop)) + + def mock_get_row_set_object(*args): + return row_set + with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, - _columns_filter_helper=mock_columns_filter_helper): + _columns_filter_helper=mock_columns_filter_helper, + _get_row_set_object=mock_get_row_set_object): result = table.scan(row_start=row_start, row_stop=row_stop, row_prefix=row_prefix, columns=columns, filter=filter_, timestamp=timestamp, @@ -636,12 +645,22 @@ def mock_filter_chain_helper(**kwargs): if row_prefix: row_start = row_prefix row_stop = MUT._string_successor(row_prefix) + """ read_rows_kwargs = { 'end_key': row_stop, 'filter_': fake_filter, 'limit': limit, 'start_key': row_start, } + """ + + + read_rows_kwargs = { + 'row_set': row_set, + 'filter_': fake_filter, + 'limit': limit, + } + self.assertEqual(table._low_level_table.read_rows_calls, [ (read_rows_args, read_rows_kwargs), ]) @@ -1466,7 +1485,7 @@ def read_row(self, *args, **kwargs): self.read_row_calls.append((args, kwargs)) return self.read_row_result - def yield_rows(self, *args, **kwargs): + def read_rows(self, *args, **kwargs): self.read_rows_calls.append((args, kwargs)) rows_dict = self.read_rows_result.rows for row_key in sorted(rows_dict): From 506959a8715befd3192af7201c6c5818467514e5 Mon Sep 17 00:00:00 2001 From: sumit-ql Date: Sun, 16 Sep 2018 19:38:50 +0530 Subject: [PATCH 2/8] Cleanup --- src/google/cloud/happybase/table.py | 8 +++++--- unit_tests/test_table.py | 9 --------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/google/cloud/happybase/table.py b/src/google/cloud/happybase/table.py index 9fbed69..059adc3 100644 --- a/src/google/cloud/happybase/table.py +++ b/src/google/cloud/happybase/table.py @@ -975,6 +975,8 @@ def _row_keys_filter_helper(row_keys): def _get_row_set_object(row_start, row_stop): - row_set = RowSet() - row_set.add_row_range(RowRange(row_start, row_stop)) - return row_set \ No newline at end of file + """Return a RowSet object for the given row_start and row_stop + """ + row_set = RowSet() + row_set.add_row_range(RowRange(row_start, row_stop)) + return row_set \ No newline at end of file diff --git a/unit_tests/test_table.py b/unit_tests/test_table.py index 04e06d7..f325ee6 100644 --- a/unit_tests/test_table.py +++ b/unit_tests/test_table.py @@ -645,15 +645,6 @@ def mock_get_row_set_object(*args): if row_prefix: row_start = row_prefix row_stop = MUT._string_successor(row_prefix) - """ - read_rows_kwargs = { - 'end_key': row_stop, - 'filter_': fake_filter, - 'limit': limit, - 'start_key': row_start, - } - """ - read_rows_kwargs = { 'row_set': row_set, From 32add375d75b6d2ebc17956db84063f17db6c2e3 Mon Sep 17 00:00:00 2001 From: sumit-ql Date: Thu, 20 Sep 2018 18:40:38 +0530 Subject: [PATCH 3/8] Review comments implementations. --- src/google/cloud/happybase/table.py | 10 ++++----- unit_tests/test_table.py | 33 +++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/google/cloud/happybase/table.py b/src/google/cloud/happybase/table.py index 059adc3..95dad15 100644 --- a/src/google/cloud/happybase/table.py +++ b/src/google/cloud/happybase/table.py @@ -382,9 +382,9 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, """ row_start, row_stop, filter_chain = _scan_filter_helper( row_start, row_stop, row_prefix, columns, timestamp, limit, kwargs) - + row_set = _get_row_set_object(row_start, row_stop) - + rows_generator = self._low_level_table.read_rows( row_set=row_set, limit=limit, filter_=filter_chain) @@ -972,11 +972,11 @@ def _row_keys_filter_helper(row_keys): return filters[0] else: return RowFilterUnion(filters=filters) - + def _get_row_set_object(row_start, row_stop): """Return a RowSet object for the given row_start and row_stop - """ + """ row_set = RowSet() row_set.add_row_range(RowRange(row_start, row_stop)) - return row_set \ No newline at end of file + return row_set diff --git a/unit_tests/test_table.py b/unit_tests/test_table.py index f325ee6..f50caa4 100644 --- a/unit_tests/test_table.py +++ b/unit_tests/test_table.py @@ -620,13 +620,12 @@ def mock_columns_filter_helper(*args): def mock_filter_chain_helper(**kwargs): mock_filters.append(kwargs) return fake_filter - - row_set = RowSet() - row_set.add_row_range(RowRange(row_start, row_stop)) - + + fake_row_set = object() + def mock_get_row_set_object(*args): - return row_set - + return fake_row_set + with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, _columns_filter_helper=mock_columns_filter_helper, _get_row_set_object=mock_get_row_set_object): @@ -645,13 +644,13 @@ def mock_get_row_set_object(*args): if row_prefix: row_start = row_prefix row_stop = MUT._string_successor(row_prefix) - + read_rows_kwargs = { - 'row_set': row_set, + 'row_set': fake_row_set, 'filter_': fake_filter, 'limit': limit, } - + self.assertEqual(table._low_level_table.read_rows_calls, [ (read_rows_args, read_rows_kwargs), ]) @@ -1437,6 +1436,22 @@ def test_many_rows(self): self.assertEqual(result, expected_result) +class Test___get_row_set_object(unittest.TestCase): + + def _callFUT(self, *args, **kwargs): + from google.cloud.happybase.table import _get_row_set_object + return _get_row_set_object(*args, **kwargs) + + def test_row_set_object(self): + from google.cloud.bigtable.row_set import RowSet + + start_key = b"row_key2" + end_key = b"row_key9" + + row_set = self._callFUT(start_key, end_key) + self.assertIsInstance(row_set, RowSet) + + class _Connection(object): def __init__(self, instance): From 747386988f19962a75753124340995ac6dabf6df Mon Sep 17 00:00:00 2001 From: sumit-ql Date: Thu, 20 Sep 2018 19:21:31 +0530 Subject: [PATCH 4/8] lint fixes. --- unit_tests/test_table.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/unit_tests/test_table.py b/unit_tests/test_table.py index f50caa4..5b42a45 100644 --- a/unit_tests/test_table.py +++ b/unit_tests/test_table.py @@ -594,8 +594,6 @@ def _scan_test_helper(self, row_limits=(None, None), row_prefix=None, import types from google.cloud._testing import _Monkey from google.cloud.happybase import table as MUT - from google.cloud.bigtable.row_set import RowSet - from google.cloud.bigtable.row_set import RowRange name = 'table-name' row_start, row_stop = row_limits @@ -624,6 +622,7 @@ def mock_filter_chain_helper(**kwargs): fake_row_set = object() def mock_get_row_set_object(*args): + #pylint: disable=unused-argument return fake_row_set with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, From 5756fd7162faec1740e9b7fe4bf94d540f9c4fc4 Mon Sep 17 00:00:00 2001 From: sumit-ql Date: Thu, 20 Sep 2018 19:46:40 +0530 Subject: [PATCH 5/8] lint fixes --- unit_tests/test_table.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/unit_tests/test_table.py b/unit_tests/test_table.py index 5b42a45..23f05d7 100644 --- a/unit_tests/test_table.py +++ b/unit_tests/test_table.py @@ -621,8 +621,7 @@ def mock_filter_chain_helper(**kwargs): fake_row_set = object() - def mock_get_row_set_object(*args): - #pylint: disable=unused-argument + def mock_get_row_set_object(*args): #pylint: disable=unused-argument return fake_row_set with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, From 7633156fdc9d1ce9d02bae3064bf912adc99fd9d Mon Sep 17 00:00:00 2001 From: sumit-ql Date: Thu, 20 Sep 2018 20:09:15 +0530 Subject: [PATCH 6/8] lint fixes again. --- unit_tests/test_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/test_table.py b/unit_tests/test_table.py index 23f05d7..3bcf857 100644 --- a/unit_tests/test_table.py +++ b/unit_tests/test_table.py @@ -621,7 +621,7 @@ def mock_filter_chain_helper(**kwargs): fake_row_set = object() - def mock_get_row_set_object(*args): #pylint: disable=unused-argument + def mock_get_row_set_object(*args): # pylint: disable=unused-argument return fake_row_set with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, From 2d2aaa911d5ee2e05742b8b6dffbff52c68d5d80 Mon Sep 17 00:00:00 2001 From: sumit-ql Date: Fri, 21 Sep 2018 20:56:14 +0530 Subject: [PATCH 7/8] Using add_row_range_from_keys() --- src/google/cloud/happybase/table.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/google/cloud/happybase/table.py b/src/google/cloud/happybase/table.py index 95dad15..d2a0407 100644 --- a/src/google/cloud/happybase/table.py +++ b/src/google/cloud/happybase/table.py @@ -978,5 +978,6 @@ def _get_row_set_object(row_start, row_stop): """Return a RowSet object for the given row_start and row_stop """ row_set = RowSet() - row_set.add_row_range(RowRange(row_start, row_stop)) + row_set.add_row_range_from_keys(start_key=row_start, + end_key=row_stop) return row_set From 43b953fa7340158768e8b2a90fd5517077999e62 Mon Sep 17 00:00:00 2001 From: sumit-ql Date: Fri, 21 Sep 2018 21:18:14 +0530 Subject: [PATCH 8/8] lint fixes --- src/google/cloud/happybase/table.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/google/cloud/happybase/table.py b/src/google/cloud/happybase/table.py index d2a0407..8824cf2 100644 --- a/src/google/cloud/happybase/table.py +++ b/src/google/cloud/happybase/table.py @@ -36,7 +36,6 @@ from google.cloud.bigtable.row_filters import TimestampRangeFilter from google.cloud.bigtable.table import Table as _LowLevelTable from google.cloud.bigtable.row_set import RowSet -from google.cloud.bigtable.row_set import RowRange from google.cloud.happybase.batch import _get_column_pairs from google.cloud.happybase.batch import _WAL_SENTINEL