diff --git a/.travis.sh b/.travis.sh index 497de259..739c66cd 100755 --- a/.travis.sh +++ b/.travis.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -o errexit -flake8 --exclude=riak/pb riak *.py +flake8 --ignore E123,E126,E226,E722,E741 --exclude=riak/pb riak *.py sudo riak-admin security disable diff --git a/.travis.yml b/.travis.yml index 90f86d41..7c46a5cd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,6 @@ dist: trusty language: python python: - '2.7' - - '3.3' - - '3.4' - - '3.5' - '3.6' - nightly addons: diff --git a/Makefile b/Makefile index 316389e4..166e4007 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,8 @@ DOCSDIR := $(PROJDIR)/docs PYPI_REPOSITORY ?= pypi +all: lint test + .PHONY: lint lint: $(PROJDIR)/.runner lint diff --git a/RELNOTES.md b/RELNOTES.md index bd567fde..6722c3ec 100644 --- a/RELNOTES.md +++ b/RELNOTES.md @@ -1,5 +1,9 @@ # Riak Python Client Release Notes +## [`3.0.0` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-3.0.0) + +* [Running expensive operations *now raise exceptions*](https://github.com/basho/riak-python-client/pull/518). You can disable these exceptions for development purposes but should not do so in production. + ## [`2.7.0` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.7.0) * Riak TS 1.5 support * Support for `head` parameter diff --git a/riak/__init__.py b/riak/__init__.py index de68354a..306cf7a0 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -19,7 +19,7 @@ operations, and run Linkwalking operations. """ -from riak.riak_error import RiakError, ConflictError +from riak.riak_error import RiakError, ConflictError, ListError from riak.client import RiakClient from riak.bucket import RiakBucket, BucketType from riak.table import Table @@ -30,11 +30,17 @@ __all__ = ['RiakBucket', 'Table', 'BucketType', 'RiakNode', 'RiakObject', 'RiakClient', 'RiakMapReduce', 'RiakKeyFilter', - 'RiakLink', 'RiakError', 'ConflictError', - 'ONE', 'ALL', 'QUORUM', 'key_filter'] + 'RiakLink', 'RiakError', 'ConflictError', 'ListError', + 'ONE', 'ALL', 'QUORUM', 'key_filter', + 'disable_list_exceptions'] ONE = "one" ALL = "all" QUORUM = "quorum" key_filter = RiakKeyFilter() + +""" +Set to true to allow listing operations +""" +disable_list_exceptions = False diff --git a/riak/client/__init__.py b/riak/client/__init__.py index ac33f8f2..7015b48f 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -275,8 +275,9 @@ def bucket(self, name, bucket_type='default'): raise TypeError('bucket_type must be a string ' 'or riak.bucket.BucketType') - return self._buckets.setdefault((bucket_type, name), - RiakBucket(self, name, bucket_type)) + b = RiakBucket(self, name, bucket_type) + return self._setdefault_handle_none( + self._buckets, (bucket_type, name), b) def bucket_type(self, name): """ @@ -291,12 +292,9 @@ def bucket_type(self, name): if not isinstance(name, string_types): raise TypeError('BucketType name must be a string') - if name in self._bucket_types: - return self._bucket_types[name] - else: - btype = BucketType(self, name) - self._bucket_types[name] = btype - return btype + btype = BucketType(self, name) + return self._setdefault_handle_none( + self._bucket_types, name, btype) def table(self, name): """ @@ -390,6 +388,16 @@ def _error_rate(node): else: return random.choice(good) + def _setdefault_handle_none(self, wvdict, key, value): + # TODO FIXME FUTURE + # This is a workaround for Python issue 19542 + # http://bugs.python.org/issue19542 + rv = wvdict.setdefault(key, value) + if rv is None: + return value + else: + return rv + @lazy_property def _multiget_pool(self): if self._multiget_pool_size: diff --git a/riak/client/operations.py b/riak/client/operations.py index 1acf06e1..0d507f12 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -13,11 +13,11 @@ # limitations under the License. import six - import riak.client.multi +from riak import ListError from riak.client.transport import RiakClientTransport, \ - retryable, retryableHttpOnly + retryable, retryableHttpOnly from riak.client.index_page import IndexPage from riak.datatypes import TYPES from riak.table import Table @@ -55,7 +55,11 @@ def get_buckets(self, transport, bucket_type=None, timeout=None): :rtype: list of :class:`RiakBucket ` instances """ + if not riak.disable_list_exceptions: + raise ListError() + _validate_timeout(timeout) + if bucket_type: bucketfn = self._bucket_type_bucket_builder else: @@ -100,6 +104,9 @@ def stream_buckets(self, bucket_type=None, timeout=None): ` instances """ + if not riak.disable_list_exceptions: + raise ListError() + _validate_timeout(timeout) if bucket_type: @@ -467,7 +474,11 @@ def get_keys(self, transport, bucket, timeout=None): :type timeout: int :rtype: list """ + if not riak.disable_list_exceptions: + raise ListError() + _validate_timeout(timeout) + return transport.get_keys(bucket, timeout=timeout) def stream_keys(self, bucket, timeout=None): @@ -503,6 +514,9 @@ def stream_keys(self, bucket, timeout=None): :type timeout: int :rtype: iterator """ + if not riak.disable_list_exceptions: + raise ListError() + _validate_timeout(timeout) def make_op(transport): @@ -678,10 +692,15 @@ def ts_stream_keys(self, table, timeout=None): :type timeout: int :rtype: iterator """ + if not riak.disable_list_exceptions: + raise ListError() + t = table if isinstance(t, six.string_types): t = Table(self, table) + _validate_timeout(timeout) + resource = self._acquire() transport = resource.object stream = transport.ts_stream_keys(t, timeout) diff --git a/riak/codecs/__init__.py b/riak/codecs/__init__.py index 00324f14..b824fcc0 100644 --- a/riak/codecs/__init__.py +++ b/riak/codecs/__init__.py @@ -21,8 +21,7 @@ from riak.util import bytes_to_str Msg = collections.namedtuple('Msg', - ['msg_code', 'data', 'resp_code'], - verbose=False) + ['msg_code', 'data', 'resp_code']) class Codec(object): diff --git a/riak/codecs/http.py b/riak/codecs/http.py index bfc91f2c..b981b77a 100644 --- a/riak/codecs/http.py +++ b/riak/codecs/http.py @@ -226,6 +226,12 @@ def _normalize_json_search_response(self, json): same return value """ result = {} + if 'facet_counts' in json: + result['facet_counts'] = json[u'facet_counts'] + if 'grouped' in json: + result['grouped'] = json[u'grouped'] + if 'stats' in json: + result['stats'] = json[u'stats'] if u'response' in json: result['num_found'] = json[u'response'][u'numFound'] result['max_score'] = float(json[u'response'][u'maxScore']) diff --git a/riak/codecs/pbuf.py b/riak/codecs/pbuf.py index fe34ee15..0b4de2a6 100644 --- a/riak/codecs/pbuf.py +++ b/riak/codecs/pbuf.py @@ -556,9 +556,9 @@ def encode_search_query(self, req, **kwargs): req.op = kwargs['q.op'] if 'fl' in kwargs: if isinstance(kwargs['fl'], list): - req.fl.extend(kwargs['fl']) + req.fl.extend([str_to_bytes(fl) for fl in kwargs['fl']]) else: - req.fl.append(kwargs['fl']) + req.fl.append(str_to_bytes(kwargs['fl'])) if 'presort' in kwargs: req.presort = kwargs['presort'] diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 7d2f690e..1b604663 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -17,9 +17,9 @@ from __future__ import print_function from collections import Iterable, namedtuple -from riak import RiakError from six import string_types, PY2 -from riak.bucket import RiakBucket + +import riak #: Links are just bucket/key/tag tuples, this class provides a @@ -128,8 +128,10 @@ def add_bucket(self, bucket, bucket_type=None): :type bucket_type: string, None :rtype: :class:`RiakMapReduce` """ + if not riak.disable_list_exceptions: + raise riak.ListError() self._input_mode = 'bucket' - if isinstance(bucket, RiakBucket): + if isinstance(bucket, riak.RiakBucket): if bucket.bucket_type.is_default(): self._inputs = {'bucket': bucket.name} else: @@ -308,14 +310,15 @@ def run(self, timeout=None): try: result = self._client.mapred(self._inputs, query, timeout) - except RiakError as e: + except riak.RiakError as e: if 'worker_startup_failed' in e.value: for phase in self._phases: if phase._language == 'erlang': if type(phase._function) is str: - raise RiakError('May have tried erlang strfun ' - 'when not allowed\n' - 'original error: ' + e.value) + raise riak.RiakError( + 'May have tried erlang strfun ' + 'when not allowed\n' + 'original error: ' + e.value) raise e # If the last phase is NOT a link phase, then return the result. diff --git a/riak/riak_error.py b/riak/riak_error.py index 97d0878c..4fe0ce05 100644 --- a/riak/riak_error.py +++ b/riak/riak_error.py @@ -36,3 +36,14 @@ class ConflictError(RiakError): """ def __init__(self, message='Object in conflict'): super(ConflictError, self).__init__(message) + + +class ListError(RiakError): + """ + Raised when a list operation is attempted and + riak.disable_list_exceptions is false. + """ + def __init__(self, message='Bucket and key list operations ' + 'are expensive and should not be ' + 'used in production.'): + super(ListError, self).__init__(message) diff --git a/riak/tests/base.py b/riak/tests/base.py index aa81c0da..9aaf4e69 100644 --- a/riak/tests/base.py +++ b/riak/tests/base.py @@ -15,6 +15,7 @@ # -*- coding: utf-8 -*- import logging import random +import riak from riak.client import RiakClient from riak.tests import HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS @@ -70,9 +71,11 @@ def create_client(cls, host=None, http_port=None, pb_port=None, **kwargs) def setUp(self): + riak.disable_list_exceptions = True self.bucket_name = self.randname() self.key_name = self.randname() self.client = self.create_client() def tearDown(self): + riak.disable_list_exceptions = False self.client.close() diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 56f55844..63206c95 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -20,7 +20,8 @@ from six import string_types, PY2, PY3 from time import sleep -from riak import ConflictError, RiakBucket, RiakError +from riak import ConflictError, RiakError, ListError +from riak import RiakClient, RiakBucket, BucketType from riak.resolver import default_resolver, last_written_resolver from riak.tests import RUN_KV, RUN_RESOLVE, PROTOCOL from riak.tests.base import IntegrationTestBase @@ -63,7 +64,6 @@ def tearDownModule(): class NotJsonSerializable(object): - def __init__(self, *args, **kwargs): self.args = list(args) self.kwargs = kwargs @@ -86,6 +86,36 @@ def __eq__(self, other): return True +class KVUnitTests(unittest.TestCase): + def test_list_keys_exception(self): + c = RiakClient() + bt = BucketType(c, 'test') + b = RiakBucket(c, 'test', bt) + with self.assertRaises(ListError): + b.get_keys() + + def test_stream_buckets_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + bs = [] + for bl in c.stream_buckets(): + bs.extend(bl) + + def test_stream_keys_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + ks = [] + for kl in c.stream_keys('test'): + ks.extend(kl) + + def test_ts_stream_keys_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + ks = [] + for kl in c.ts_stream_keys('test'): + ks.extend(kl) + + @unittest.skipUnless(RUN_KV, 'RUN_KV is 0') class BasicKVTests(IntegrationTestBase, unittest.TestCase, Comparison): def test_no_returnbody(self): diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index f8b90e98..bfdfc7dd 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -19,7 +19,7 @@ from six import PY2 from riak.mapreduce import RiakMapReduce -from riak import key_filter, RiakError +from riak import key_filter, RiakClient, RiakError, ListError from riak.tests import RUN_MAPREDUCE, RUN_SECURITY, RUN_YZ from riak.tests.base import IntegrationTestBase from riak.tests.test_yokozuna import wait_for_yz_index @@ -39,6 +39,13 @@ def tearDownModule(): yzTearDown(testrun_yz_mr) +class MapReduceUnitTests(unittest.TestCase): + def test_mapred_bucket_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + c.add('bucket') + + @unittest.skipUnless(RUN_MAPREDUCE, 'RUN_MAPREDUCE is 0') class LinkTests(IntegrationTestBase, unittest.TestCase): def test_store_and_get_links(self): diff --git a/riak/tests/yz_setup.py b/riak/tests/yz_setup.py index 78c44755..88a7daee 100644 --- a/riak/tests/yz_setup.py +++ b/riak/tests/yz_setup.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import riak from riak import RiakError from riak.tests import RUN_YZ @@ -21,6 +22,7 @@ def yzSetUp(*yzdata): if RUN_YZ: + riak.disable_list_exceptions = True c = IntegrationTestBase.create_client() for yz in yzdata: logging.debug("yzSetUp: %s", yz) @@ -43,6 +45,7 @@ def yzSetUp(*yzdata): def yzTearDown(c, *yzdata): if RUN_YZ: + riak.disable_list_exceptions = True c = IntegrationTestBase.create_client() for yz in yzdata: logging.debug("yzTearDown: %s", yz) @@ -57,3 +60,4 @@ def yzTearDown(c, *yzdata): for key in keys: b.delete(key) c.close() + riak.disable_list_exceptions = False