From 488c22b9492cda4870999acedd2c28d9341e5c3b Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Fri, 27 Jan 2017 08:25:58 -0800 Subject: [PATCH 01/11] Add global disable_list_exceptions variable to disable exceptions thrown during expensive operations. Raise ListError if mapreduce over a bucket is attempted --- Makefile | 2 ++ riak/__init__.py | 12 +++++++++--- riak/client/operations.py | 23 +++++++++++++++++++++-- riak/mapreduce.py | 17 ++++++++++------- riak/riak_error.py | 11 +++++++++++ riak/tests/base.py | 3 +++ riak/tests/test_kv.py | 34 ++++++++++++++++++++++++++++++++-- riak/tests/test_mapreduce.py | 9 ++++++++- riak/tests/yz_setup.py | 4 ++++ 9 files changed, 100 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 316389e4..166e4007 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,8 @@ DOCSDIR := $(PROJDIR)/docs PYPI_REPOSITORY ?= pypi +all: lint test + .PHONY: lint lint: $(PROJDIR)/.runner lint diff --git a/riak/__init__.py b/riak/__init__.py index de68354a..306cf7a0 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -19,7 +19,7 @@ operations, and run Linkwalking operations. """ -from riak.riak_error import RiakError, ConflictError +from riak.riak_error import RiakError, ConflictError, ListError from riak.client import RiakClient from riak.bucket import RiakBucket, BucketType from riak.table import Table @@ -30,11 +30,17 @@ __all__ = ['RiakBucket', 'Table', 'BucketType', 'RiakNode', 'RiakObject', 'RiakClient', 'RiakMapReduce', 'RiakKeyFilter', - 'RiakLink', 'RiakError', 'ConflictError', - 'ONE', 'ALL', 'QUORUM', 'key_filter'] + 'RiakLink', 'RiakError', 'ConflictError', 'ListError', + 'ONE', 'ALL', 'QUORUM', 'key_filter', + 'disable_list_exceptions'] ONE = "one" ALL = "all" QUORUM = "quorum" key_filter = RiakKeyFilter() + +""" +Set to true to allow listing operations +""" +disable_list_exceptions = False diff --git a/riak/client/operations.py b/riak/client/operations.py index 1acf06e1..0d507f12 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -13,11 +13,11 @@ # limitations under the License. import six - import riak.client.multi +from riak import ListError from riak.client.transport import RiakClientTransport, \ - retryable, retryableHttpOnly + retryable, retryableHttpOnly from riak.client.index_page import IndexPage from riak.datatypes import TYPES from riak.table import Table @@ -55,7 +55,11 @@ def get_buckets(self, transport, bucket_type=None, timeout=None): :rtype: list of :class:`RiakBucket ` instances """ + if not riak.disable_list_exceptions: + raise ListError() + _validate_timeout(timeout) + if bucket_type: bucketfn = self._bucket_type_bucket_builder else: @@ -100,6 +104,9 @@ def stream_buckets(self, bucket_type=None, timeout=None): ` instances """ + if not riak.disable_list_exceptions: + raise ListError() + _validate_timeout(timeout) if bucket_type: @@ -467,7 +474,11 @@ def get_keys(self, transport, bucket, timeout=None): :type timeout: int :rtype: list """ + if not riak.disable_list_exceptions: + raise ListError() + _validate_timeout(timeout) + return transport.get_keys(bucket, timeout=timeout) def stream_keys(self, bucket, timeout=None): @@ -503,6 +514,9 @@ def stream_keys(self, bucket, timeout=None): :type timeout: int :rtype: iterator """ + if not riak.disable_list_exceptions: + raise ListError() + _validate_timeout(timeout) def make_op(transport): @@ -678,10 +692,15 @@ def ts_stream_keys(self, table, timeout=None): :type timeout: int :rtype: iterator """ + if not riak.disable_list_exceptions: + raise ListError() + t = table if isinstance(t, six.string_types): t = Table(self, table) + _validate_timeout(timeout) + resource = self._acquire() transport = resource.object stream = transport.ts_stream_keys(t, timeout) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 7d2f690e..1b604663 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -17,9 +17,9 @@ from __future__ import print_function from collections import Iterable, namedtuple -from riak import RiakError from six import string_types, PY2 -from riak.bucket import RiakBucket + +import riak #: Links are just bucket/key/tag tuples, this class provides a @@ -128,8 +128,10 @@ def add_bucket(self, bucket, bucket_type=None): :type bucket_type: string, None :rtype: :class:`RiakMapReduce` """ + if not riak.disable_list_exceptions: + raise riak.ListError() self._input_mode = 'bucket' - if isinstance(bucket, RiakBucket): + if isinstance(bucket, riak.RiakBucket): if bucket.bucket_type.is_default(): self._inputs = {'bucket': bucket.name} else: @@ -308,14 +310,15 @@ def run(self, timeout=None): try: result = self._client.mapred(self._inputs, query, timeout) - except RiakError as e: + except riak.RiakError as e: if 'worker_startup_failed' in e.value: for phase in self._phases: if phase._language == 'erlang': if type(phase._function) is str: - raise RiakError('May have tried erlang strfun ' - 'when not allowed\n' - 'original error: ' + e.value) + raise riak.RiakError( + 'May have tried erlang strfun ' + 'when not allowed\n' + 'original error: ' + e.value) raise e # If the last phase is NOT a link phase, then return the result. diff --git a/riak/riak_error.py b/riak/riak_error.py index 97d0878c..4fe0ce05 100644 --- a/riak/riak_error.py +++ b/riak/riak_error.py @@ -36,3 +36,14 @@ class ConflictError(RiakError): """ def __init__(self, message='Object in conflict'): super(ConflictError, self).__init__(message) + + +class ListError(RiakError): + """ + Raised when a list operation is attempted and + riak.disable_list_exceptions is false. + """ + def __init__(self, message='Bucket and key list operations ' + 'are expensive and should not be ' + 'used in production.'): + super(ListError, self).__init__(message) diff --git a/riak/tests/base.py b/riak/tests/base.py index aa81c0da..9aaf4e69 100644 --- a/riak/tests/base.py +++ b/riak/tests/base.py @@ -15,6 +15,7 @@ # -*- coding: utf-8 -*- import logging import random +import riak from riak.client import RiakClient from riak.tests import HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS @@ -70,9 +71,11 @@ def create_client(cls, host=None, http_port=None, pb_port=None, **kwargs) def setUp(self): + riak.disable_list_exceptions = True self.bucket_name = self.randname() self.key_name = self.randname() self.client = self.create_client() def tearDown(self): + riak.disable_list_exceptions = False self.client.close() diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 56f55844..63206c95 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -20,7 +20,8 @@ from six import string_types, PY2, PY3 from time import sleep -from riak import ConflictError, RiakBucket, RiakError +from riak import ConflictError, RiakError, ListError +from riak import RiakClient, RiakBucket, BucketType from riak.resolver import default_resolver, last_written_resolver from riak.tests import RUN_KV, RUN_RESOLVE, PROTOCOL from riak.tests.base import IntegrationTestBase @@ -63,7 +64,6 @@ def tearDownModule(): class NotJsonSerializable(object): - def __init__(self, *args, **kwargs): self.args = list(args) self.kwargs = kwargs @@ -86,6 +86,36 @@ def __eq__(self, other): return True +class KVUnitTests(unittest.TestCase): + def test_list_keys_exception(self): + c = RiakClient() + bt = BucketType(c, 'test') + b = RiakBucket(c, 'test', bt) + with self.assertRaises(ListError): + b.get_keys() + + def test_stream_buckets_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + bs = [] + for bl in c.stream_buckets(): + bs.extend(bl) + + def test_stream_keys_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + ks = [] + for kl in c.stream_keys('test'): + ks.extend(kl) + + def test_ts_stream_keys_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + ks = [] + for kl in c.ts_stream_keys('test'): + ks.extend(kl) + + @unittest.skipUnless(RUN_KV, 'RUN_KV is 0') class BasicKVTests(IntegrationTestBase, unittest.TestCase, Comparison): def test_no_returnbody(self): diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index f8b90e98..bfdfc7dd 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -19,7 +19,7 @@ from six import PY2 from riak.mapreduce import RiakMapReduce -from riak import key_filter, RiakError +from riak import key_filter, RiakClient, RiakError, ListError from riak.tests import RUN_MAPREDUCE, RUN_SECURITY, RUN_YZ from riak.tests.base import IntegrationTestBase from riak.tests.test_yokozuna import wait_for_yz_index @@ -39,6 +39,13 @@ def tearDownModule(): yzTearDown(testrun_yz_mr) +class MapReduceUnitTests(unittest.TestCase): + def test_mapred_bucket_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + c.add('bucket') + + @unittest.skipUnless(RUN_MAPREDUCE, 'RUN_MAPREDUCE is 0') class LinkTests(IntegrationTestBase, unittest.TestCase): def test_store_and_get_links(self): diff --git a/riak/tests/yz_setup.py b/riak/tests/yz_setup.py index 78c44755..88a7daee 100644 --- a/riak/tests/yz_setup.py +++ b/riak/tests/yz_setup.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import riak from riak import RiakError from riak.tests import RUN_YZ @@ -21,6 +22,7 @@ def yzSetUp(*yzdata): if RUN_YZ: + riak.disable_list_exceptions = True c = IntegrationTestBase.create_client() for yz in yzdata: logging.debug("yzSetUp: %s", yz) @@ -43,6 +45,7 @@ def yzSetUp(*yzdata): def yzTearDown(c, *yzdata): if RUN_YZ: + riak.disable_list_exceptions = True c = IntegrationTestBase.create_client() for yz in yzdata: logging.debug("yzTearDown: %s", yz) @@ -57,3 +60,4 @@ def yzTearDown(c, *yzdata): for key in keys: b.delete(key) c.close() + riak.disable_list_exceptions = False From 4fee03a6c0fd51a368c0364adcc5408dafe9c003 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 9 Feb 2017 08:43:56 -0800 Subject: [PATCH 02/11] Add note about exceptions being raised for list operations --- RELNOTES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/RELNOTES.md b/RELNOTES.md index bd567fde..0cc58993 100644 --- a/RELNOTES.md +++ b/RELNOTES.md @@ -1,5 +1,9 @@ # Riak Python Client Release Notes +## [`2.8.0` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.8.0) + +* [Running expensive operations *now raise exceptions*](https://github.com/basho/riak-python-client/pull/518). You can disable these exceptions for development purposes but should not do so in production. + ## [`2.7.0` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.7.0) * Riak TS 1.5 support * Support for `head` parameter From b27163d4526fd6c8894e225e61f8d9de27223034 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Mon, 27 Feb 2017 12:54:35 -0800 Subject: [PATCH 03/11] Add a workaround for Python bug 19542 --- riak/client/__init__.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index ac33f8f2..7015b48f 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -275,8 +275,9 @@ def bucket(self, name, bucket_type='default'): raise TypeError('bucket_type must be a string ' 'or riak.bucket.BucketType') - return self._buckets.setdefault((bucket_type, name), - RiakBucket(self, name, bucket_type)) + b = RiakBucket(self, name, bucket_type) + return self._setdefault_handle_none( + self._buckets, (bucket_type, name), b) def bucket_type(self, name): """ @@ -291,12 +292,9 @@ def bucket_type(self, name): if not isinstance(name, string_types): raise TypeError('BucketType name must be a string') - if name in self._bucket_types: - return self._bucket_types[name] - else: - btype = BucketType(self, name) - self._bucket_types[name] = btype - return btype + btype = BucketType(self, name) + return self._setdefault_handle_none( + self._bucket_types, name, btype) def table(self, name): """ @@ -390,6 +388,16 @@ def _error_rate(node): else: return random.choice(good) + def _setdefault_handle_none(self, wvdict, key, value): + # TODO FIXME FUTURE + # This is a workaround for Python issue 19542 + # http://bugs.python.org/issue19542 + rv = wvdict.setdefault(key, value) + if rv is None: + return value + else: + return rv + @lazy_property def _multiget_pool(self): if self._multiget_pool_size: From b0ee21aec32a584f07d2c377d904352e884b5cfe Mon Sep 17 00:00:00 2001 From: Ali Riza Keles Date: Tue, 21 Mar 2017 16:52:57 +0300 Subject: [PATCH 04/11] fix missing str to byte operation of fl params in encode_search_query method of codecs.pbuf module --- riak/codecs/pbuf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/codecs/pbuf.py b/riak/codecs/pbuf.py index fe34ee15..0b4de2a6 100644 --- a/riak/codecs/pbuf.py +++ b/riak/codecs/pbuf.py @@ -556,9 +556,9 @@ def encode_search_query(self, req, **kwargs): req.op = kwargs['q.op'] if 'fl' in kwargs: if isinstance(kwargs['fl'], list): - req.fl.extend(kwargs['fl']) + req.fl.extend([str_to_bytes(fl) for fl in kwargs['fl']]) else: - req.fl.append(kwargs['fl']) + req.fl.append(str_to_bytes(kwargs['fl'])) if 'presort' in kwargs: req.presort = kwargs['presort'] From a190e871eabf2f42df40c62e888282848b9b8505 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Mon, 27 Mar 2017 16:29:29 -0700 Subject: [PATCH 05/11] 2.8.0 release is actually 3.0.0 --- RELNOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELNOTES.md b/RELNOTES.md index 0cc58993..6722c3ec 100644 --- a/RELNOTES.md +++ b/RELNOTES.md @@ -1,6 +1,6 @@ # Riak Python Client Release Notes -## [`2.8.0` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.8.0) +## [`3.0.0` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-3.0.0) * [Running expensive operations *now raise exceptions*](https://github.com/basho/riak-python-client/pull/518). You can disable these exceptions for development purposes but should not do so in production. From 2c01ebb72c76b8f27ec4d2965f59a7a7165f2bf2 Mon Sep 17 00:00:00 2001 From: Steven Joseph Date: Sat, 28 Oct 2017 11:53:33 +1100 Subject: [PATCH 06/11] Expose facet_counts in riak http results --- riak/codecs/http.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/riak/codecs/http.py b/riak/codecs/http.py index bfc91f2c..5da99961 100644 --- a/riak/codecs/http.py +++ b/riak/codecs/http.py @@ -229,6 +229,8 @@ def _normalize_json_search_response(self, json): if u'response' in json: result['num_found'] = json[u'response'][u'numFound'] result['max_score'] = float(json[u'response'][u'maxScore']) + if 'facet_counts' in json: + result['facet_counts'] = json[u'facet_counts'] docs = [] for doc in json[u'response'][u'docs']: resdoc = {} From 648f9056d47332df890232e85d497623608e2964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20A=2E=20Ceccon?= Date: Mon, 19 Mar 2018 15:05:41 +0100 Subject: [PATCH 07/11] Do not use deprecated argument `verbose` in namedtuple Argument `verbose` to nametuple constructor was deprecated in Python 3.3, and was removed in Python 3.7. --- riak/codecs/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/riak/codecs/__init__.py b/riak/codecs/__init__.py index 00324f14..b824fcc0 100644 --- a/riak/codecs/__init__.py +++ b/riak/codecs/__init__.py @@ -21,8 +21,7 @@ from riak.util import bytes_to_str Msg = collections.namedtuple('Msg', - ['msg_code', 'data', 'resp_code'], - verbose=False) + ['msg_code', 'data', 'resp_code']) class Codec(object): From 95e61a03ce6e51289aa184a618f7462ff7fd602f Mon Sep 17 00:00:00 2001 From: bryanhuntesl <31992054+bryanhuntesl@users.noreply.github.com> Date: Mon, 19 Mar 2018 18:25:02 +0000 Subject: [PATCH 08/11] disable spurious flake8 python style warnings --- .travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.sh b/.travis.sh index 497de259..40518170 100755 --- a/.travis.sh +++ b/.travis.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -o errexit -flake8 --exclude=riak/pb riak *.py +flake8 --ignore E722,E741 --exclude=riak/pb riak *.py sudo riak-admin security disable From c52b745a96115426205d7c8a6245902cf2441b3b Mon Sep 17 00:00:00 2001 From: bryanhuntesl <31992054+bryanhuntesl@users.noreply.github.com> Date: Mon, 19 Mar 2018 18:43:41 +0000 Subject: [PATCH 09/11] and again... add more exclusions.. --- .travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.sh b/.travis.sh index 40518170..739c66cd 100755 --- a/.travis.sh +++ b/.travis.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -o errexit -flake8 --ignore E722,E741 --exclude=riak/pb riak *.py +flake8 --ignore E123,E126,E226,E722,E741 --exclude=riak/pb riak *.py sudo riak-admin security disable From b3a7d2d2c2621efc3cbf7a339a2341ff4da882bd Mon Sep 17 00:00:00 2001 From: bryanhuntesl <31992054+bryanhuntesl@users.noreply.github.com> Date: Mon, 19 Mar 2018 18:54:49 +0000 Subject: [PATCH 10/11] Reduce test time - test only 2.7, 3.6, and nightly test only 2.7 (stable), 3.6 (3 series stable) and nightly - the matrix was too big - 3.2, 3.3, 3.4, and 3.5 are the least used. --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 90f86d41..7c46a5cd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,6 @@ dist: trusty language: python python: - '2.7' - - '3.3' - - '3.4' - - '3.5' - '3.6' - nightly addons: From 3a2bf990db0792a3ba00719db5fa1651c89ce758 Mon Sep 17 00:00:00 2001 From: Steven Joseph Date: Tue, 20 Mar 2018 10:11:43 +1100 Subject: [PATCH 11/11] Expose more result types from solr results --- riak/codecs/http.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/riak/codecs/http.py b/riak/codecs/http.py index 5da99961..b981b77a 100644 --- a/riak/codecs/http.py +++ b/riak/codecs/http.py @@ -226,11 +226,15 @@ def _normalize_json_search_response(self, json): same return value """ result = {} + if 'facet_counts' in json: + result['facet_counts'] = json[u'facet_counts'] + if 'grouped' in json: + result['grouped'] = json[u'grouped'] + if 'stats' in json: + result['stats'] = json[u'stats'] if u'response' in json: result['num_found'] = json[u'response'][u'numFound'] result['max_score'] = float(json[u'response'][u'maxScore']) - if 'facet_counts' in json: - result['facet_counts'] = json[u'facet_counts'] docs = [] for doc in json[u'response'][u'docs']: resdoc = {}