From d960a16ebe432729ba89c1e0174173c95e831596 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Fri, 5 Aug 2011 17:11:00 +0200 Subject: [PATCH 001/118] Restored Python 2.6 compatibility. By way of the unittest2 package, so it needs to be installed to run the tests on Python 2.6. --- riak/tests/suite.py | 7 ++++++- riak/tests/test_all.py | 15 ++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/riak/tests/suite.py b/riak/tests/suite.py index bddfd45b..2e4735b7 100644 --- a/riak/tests/suite.py +++ b/riak/tests/suite.py @@ -1,6 +1,11 @@ -import unittest import riak.tests.test_server_test import os.path +import platform + +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest def additional_tests(): top_level = os.path.join(os.path.dirname(__file__), "../../") diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 7b5f5bdb..d2fa9913 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -8,7 +8,12 @@ import simplejson as json import os import random -import unittest +import platform + +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest import uuid import time @@ -829,7 +834,9 @@ def test_solr_search_with_params_from_bucket(self): bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage", wt="xml") - self.assertEquals(1, len(list(results.find("result").iter("doc")))) + result = results.find("result") + if not hasattr(result, "iter"): setattr(result, "iter", result.getiterator) + self.assertEquals(1, len(list(result.iter("doc")))) def test_solr_search_with_params(self): if SKIP_SEARCH: @@ -837,7 +844,9 @@ def test_solr_search_with_params(self): bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = self.client.solr().search("searchbucket", "username:roidrage", wt="xml") - self.assertEquals(1, len(list(results.find("result").iter("doc")))) + result = results.find("result") + if not hasattr(result, "iter"): setattr(result, "iter", result.getiterator) + self.assertEquals(1, len(list(result.iter("doc")))) def test_solr_search(self): if SKIP_SEARCH: From 02acb73cd5cf99adf0b1c53083502316b50fdf52 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Fri, 5 Aug 2011 10:49:58 -0500 Subject: [PATCH 002/118] Add SKIP_LUWAK and SKIP_SEARCH to the README. #43 --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index f322d5db..e934bdd6 100644 --- a/README.rst +++ b/README.rst @@ -26,6 +26,7 @@ To run the unit tests, execute:: python setup.py test +If you don't have `Luwak `_ or `Riak Search `_ enabled you can set the ``SKIP_LUWAK`` and ``SKIP_SEARCH`` environment variables to skip those tests. ======== Tutorial From 26f72096542c657e38548a779d9c251ed9f007e2 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Mon, 8 Aug 2011 17:34:17 -0500 Subject: [PATCH 003/118] Don't return the empty object (self) with siblings. Also, update THANKS file. --- THANKS | 2 ++ riak/riak_object.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/THANKS b/THANKS index f734f6c9..c8c1371a 100644 --- a/THANKS +++ b/THANKS @@ -12,3 +12,5 @@ Mark Erdmann Greg Nelson Mikhail Sobolev Eric Moritz +Brett Hoerner +Scott Lystig Fritchie diff --git a/riak/riak_object.py b/riak/riak_object.py index 3d552900..01fdae8c 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -461,7 +461,7 @@ def get_siblings(self, r=None): :type r: integer :rtype: array of RiakObject """ - a = [self] + a = [] for i in range(self.get_sibling_count()): a.append(self.get_sibling(i, r)) return a From 699ae7b05f7a21e292d19403f7ed2a87622ce7ff Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 19 Aug 2011 16:40:17 -0500 Subject: [PATCH 004/118] README changes: Python 2.7, setuptools, non-localhost env vars --- README.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e934bdd6..17565f96 100644 --- a/README.rst +++ b/README.rst @@ -16,18 +16,24 @@ Documentation for Riak is available at http://wiki.basho.com/How-Things-Work.htm Install ======= +The recommended version of Python for use with this client is Python 2.7. + You must have `Protocol Buffers`_ installed before you can install the Riak Client. From the Riak Python Client root directory, execute:: python setup.py install +There is an additional dependency on the Python package `setuptools`. Please install `setuptools` first, e.g. ``port install py27-setuptools`` for OS X and MacPorts. + Unit Test =========== -To run the unit tests, execute:: +To run the unit tests against a Riak server (with default TCP port configuration) on localhost, execute:: python setup.py test If you don't have `Luwak `_ or `Riak Search `_ enabled you can set the ``SKIP_LUWAK`` and ``SKIP_SEARCH`` environment variables to skip those tests. +If your Riak server isn't running on localhost, use the environment variables ``RIAK_TEST_HOST`` and ``RIAK_TEST_HTTP_PORT`` and ``RIAK_TEST_PB_PORT=8087`` to specify where to find the Riak server. + ======== Tutorial ======== From 251542abcf5fedab4465e517117018041d64d9e2 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 19 Aug 2011 17:09:35 -0500 Subject: [PATCH 005/118] Ignore *.egg files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a8d3d3a0..ef74d171 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ docs/_build build/ dist/ riak.egg-info/ +*.egg From d24d6e3d293ffd8e9b9a757652db58032c739bdb Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Tue, 30 Aug 2011 22:13:41 -0400 Subject: [PATCH 006/118] Relax some dependencies: * if something cannot be imported, then simply disable the transport. * add a future import to enable "with" statements in pbc.py --- riak/transports/http.py | 8 +++++++- riak/transports/pbc.py | 14 +++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index c719b8d1..57fba0cc 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -573,7 +573,10 @@ def httplib_request(cls, method, host, port, uri, headers, body=''): if response is not None: response.close() raise -import urllib3 +try: + import urllib3 +except ImportError: + urllib3 = None class RiakHttpPoolTransport(RiakHttpTransport): """ @@ -585,6 +588,9 @@ class RiakHttpPoolTransport(RiakHttpTransport): def __init__(self, host='127.0.0.1', port=8098, prefix='riak', mapred_prefix='mapred', client_id=None): + if urllib3 is None: + raise RiakError("this transport is not available (no urllib3)") + super(RiakHttpPoolTransport, self).__init__(host=host, port=port, prefix=prefix, diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index ae71c764..ce1a27b9 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -17,6 +17,8 @@ specific language governing permissions and limitations under the License. """ +from __future__ import with_statement + import socket, struct try: @@ -28,7 +30,11 @@ from riak.metadata import * from riak.mapreduce import RiakMapReduce, RiakLink from riak import RiakError -import riakclient_pb2 + +try: + import riakclient_pb2 +except ImportError: + riakclient_pb2 = None ## Protocol codes MSG_CODE_ERROR_RESP = 0 @@ -82,6 +88,9 @@ def __init__(self, host='127.0.0.1', port=8087, client_id=None): @param string host - Hostname or IP address (default '127.0.0.1') @param int port - Port number (default 8087) """ + if riakclient_pb2 is None: + raise RiakError("this transport is not available (no protobuf)") + super(RiakPbcTransport, self).__init__() self._host = host self._port = port @@ -492,6 +501,9 @@ def pbify_content(self, metadata, data, rpb_content) : class RiakPbcCachedTransport(RiakTransport): """Threadsafe pool of PBC connections, based on urllib3's pool [aka Queue]""" def __init__(self, host='127.0.0.1', port=8087, client_id=None, maxsize=0, block=False, timeout=None): + if riakclient_pb2 is None: + raise RiakError("this transport is not available (no protobuf)") + self.host = host self.port = port self.client_id = client_id From 562584d50f9b24bcc05d61f3c6befdb1b77de9e1 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 2 Sep 2011 23:55:55 +0200 Subject: [PATCH 007/118] Stop ignoring tmp_dir argument to TestServer --- riak/test_server.py | 2 +- riak/tests/test_server_test.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/riak/test_server.py b/riak/test_server.py index 0262aaa4..18a17430 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -71,7 +71,7 @@ def __init__(self, tmp_dir="/tmp/riak/test_server", bin_dir=os.path.expanduser("~/.riak/install/riak-0.14.2/bin"), vm_args=None, **options): self._lock = threading.Lock() - self.temp_dir = "/tmp/riak/test_server" + self.temp_dir = tmp_dir self.bin_dir = bin_dir self._prepared = False self._started = False diff --git a/riak/tests/test_server_test.py b/riak/tests/test_server_test.py index 6a2437b8..1c1307cb 100644 --- a/riak/tests/test_server_test.py +++ b/riak/tests/test_server_test.py @@ -40,6 +40,11 @@ def test_set_ring_state_dir(self): def test_set_default_tmp_dir(self): self.assertEquals(self.test_server.temp_dir, "/tmp/riak/test_server") + def test_set_non_default_tmp_dir(self): + tmp_dir = '/not/the/default/dir' + server = TestServer(tmp_dir=tmp_dir) + self.assertEquals(server.temp_dir, tmp_dir) + def suite(): suite = unittest.TestSuite() suite.addTest(TestServerTestCase()) From 758d87ffcff721b58026607e6fa118ae028fe3fe Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 22:45:55 -0400 Subject: [PATCH 008/118] Remove the unused get_value() method, which is just DICT.get() anyways. --- riak/transports/http.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 57fba0cc..fa752fc8 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -366,13 +366,6 @@ def post_request(self, uri=None, body=None, params=None, content_type="applicati # Utility functions used by Riak library. - @classmethod - def get_value(cls, key, array, defaultValue) : - if key in array: - return array[key] - else: - return defaultValue - def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : """ Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, From d68e46b16830d80e4ad8bba6bbeefbbf7e3f5f36 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 22:48:49 -0400 Subject: [PATCH 009/118] Remove pycurl: it does not provide anything beyond the builtin httplib. --- riak/transports/http.py | 61 ++--------------------------------------- 1 file changed, 2 insertions(+), 59 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index fa752fc8..1783bd7f 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -19,13 +19,7 @@ """ import urllib, re from cStringIO import StringIO -# Use pycurl as first choice, httplib as second choice. -try: - import pycurl - HAS_PYCURL = True -except ImportError: - import httplib - HAS_PYCURL = False +import httplib try: import json except ImportError: @@ -403,11 +397,7 @@ def http_request(cls, method, host, port, url, headers = None, obj = '') : """ if not headers: headers = {} - if HAS_PYCURL: - return cls.pycurl_request(method, host, port, url, headers, obj) - else: - return cls.httplib_request(method, host, port, url, headers, obj) - + return cls.httplib_request(method, host, port, url, headers, obj) @classmethod def httplib_request(cls, method, host, port, uri, headers = None, body=''): @@ -436,53 +426,6 @@ def httplib_request(cls, method, host, port, uri, headers = None, body=''): if response is not None: response.close() raise - - @classmethod - def pycurl_request(cls, method, host, port, uri, headers, body=''): - if not headers: - headers = {} - url = "http://" + host + ":" + str(port) + uri - # Set up Curl... - client = pycurl.Curl() - client.setopt(pycurl.URL, url) - client.setopt(pycurl.HTTPHEADER, cls.build_headers(headers)) - if method == 'GET': - client.setopt(pycurl.HTTPGET, 1) - elif method == 'POST': - client.setopt(pycurl.POST, 1) - client.setopt(pycurl.POSTFIELDS, body) - elif method == 'PUT': - client.setopt(pycurl.CUSTOMREQUEST, method) - client.setopt(pycurl.POSTFIELDS, body) - elif method == 'DELETE': - client.setopt(pycurl.CUSTOMREQUEST, method) - - # Capture the response headers... - response_headers_io = StringIO() - client.setopt(pycurl.HEADERFUNCTION, response_headers_io.write) - - # Capture the response body... - response_body_io = StringIO() - client.setopt(pycurl.WRITEFUNCTION, response_body_io.write) - - try: - # Run the request. - client.perform() - http_code = client.getinfo(pycurl.HTTP_CODE) - client.close() - - # Get the headers... - response_headers = cls.parse_http_headers(response_headers_io.getvalue()) - response_headers['http_code'] = http_code - - # Get the body... - response_body = response_body_io.getvalue() - - return response_headers, response_body - except: - if client is not None: client.close() - raise - @classmethod def build_headers(cls, headers): return ['%s: %s' % (header, value) for header, value in headers.iteritems()] From 0f6f01a5c29b998cc2e8f514de5dcb4d595bdc3b Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 23:04:06 -0400 Subject: [PATCH 010/118] Remove host/port parameters from various method signatures, and rely on self._host and self._port for the values. * riak/transports/http.py: (RiakHttpTransport.ping): remove host/port from .http_request() call (RiakHttpTransport.get, RiakHttpRequest.delete, RiakHttpRequest.get_keys, RiakHttpRequest.get_buckets, RiakHttpRequest.get_bucket_props, RiakHttpRequest.set_bucket_props, RiakHttpRequest.get_request, RiakHttpRequest.get_file, RiakHttpRequest.delete_file, RiakHttpRequest.post_request): remove host/port return values from build_rest_path() and avoid passing to .http_request() (RiakHttpTransport.put, RiakHttpTransport.store_file): remove host/port return values from build_rest_path(), and remove the host/port from from .do_put() (RiakHttpTransport.do_put): remove host/port from the signature, and avoid passing them to .http_request() (RiakHttpTransport.mapred): remove host/port localvars, and avoid passing to .http_request() (RiakHttpTransport.build_rest_path): only return the constructed path (RiakHttpTransport.http_request): move to an instance method, and use self._host and self._port rather than params (which are now eliminated from the func signature). (RiakHttpTransport.httplib_request, RiakHttpReuseTransport.httplib_request): remove host/port from the signature and use self._host and self._port instead. Switch to an instance method to support this. (RiakHttpPoolTransport.httplib_request): maintain existing functionality by storing the pool in the class, but this seems wrong. otherwise, in the main: eliminate the host/port parameters. --- riak/transports/http.py | 95 ++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 49 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 1783bd7f..2e9ec583 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -73,7 +73,7 @@ def ping(self) : """ Check server is alive over HTTP """ - response = self.http_request('GET', self._host, self._port, '/ping') + response = self.http_request('GET', '/ping') return(response is not None) and (response[1] == 'OK') @@ -84,9 +84,9 @@ def get(self, robj, r, vtag = None) : params = {'r' : r} if vtag is not None: params['vtag'] = vtag - host, port, url = self.build_rest_path(robj.get_bucket(), robj.get_key(), - params=params) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(robj.get_bucket(), robj.get_key(), + params=params) + response = self.http_request('GET', url) return self.parse_body(response, [200, 300, 404]) def put(self, robj, w = None, dw = None, return_body = True): @@ -95,8 +95,8 @@ def put(self, robj, w = None, dw = None, return_body = True): """ # Construct the URL... params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw} - host, port, url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), - params=params) + url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), + params=params) # Construct the headers... headers = MultiDict({'Accept' : 'text/plain, */*; q=0.5', @@ -114,13 +114,13 @@ def put(self, robj, w = None, dw = None, return_body = True): headers['X-Riak-Meta-%s' % key] = value content = robj.get_encoded_data() - return self.do_put(host, port, url, headers, content, return_body, key=robj.get_key()) + return self.do_put(url, headers, content, return_body, key=robj.get_key()) - def do_put(self, host, port, url, headers, content, return_body=False, key=None): + def do_put(self, url, headers, content, return_body=False, key=None): if key is None: - response = self.http_request('POST', host, port, url, headers, content) + response = self.http_request('POST', url, headers, content) else: - response = self.http_request('PUT', host, port, url, headers, content) + response = self.http_request('PUT', url, headers, content) if return_body: return self.parse_body(response, [200, 201, 300]) @@ -131,18 +131,18 @@ def do_put(self, host, port, url, headers, content, return_body=False, key=None) def delete(self, robj, rw): # Construct the URL... params = {'rw' : rw} - host, port, url = self.build_rest_path(robj.get_bucket(), robj.get_key(), - params=params) + url = self.build_rest_path(robj.get_bucket(), robj.get_key(), + params=params) # Run the operation.. - response = self.http_request('DELETE', host, port, url) + response = self.http_request('DELETE', url) self.check_http_code(response, [204, 404]) return self def get_keys(self, bucket): params = {'props' : 'True', 'keys' : 'true'} - host, port, url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(bucket, params=params) + response = self.http_request('GET', url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: @@ -153,8 +153,8 @@ def get_keys(self, bucket): def get_buckets(self): params = {'buckets': 'true'} - host, port, url = self.build_rest_path(None, params=params) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(None, params=params) + response = self.http_request('GET', url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: @@ -166,8 +166,8 @@ def get_buckets(self): def get_bucket_props(self, bucket): # Run the request... params = {'props' : 'True', 'keys' : 'False'} - host, port, url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(bucket, params=params) + response = self.http_request('GET', url) headers = response[0] encoded_props = response[1] @@ -182,12 +182,12 @@ def set_bucket_props(self, bucket, props): """ Set the properties on the bucket object given """ - host, port, url = self.build_rest_path(bucket) + url = self.build_rest_path(bucket) headers = {'Content-Type' : 'application/json'} content = json.dumps({'props' : props}) # Run the request... - response = self.http_request('PUT', host, port, url, headers, content) + response = self.http_request('PUT', url, headers, content) # Handle the response... if response is None: @@ -208,10 +208,8 @@ def mapred(self, inputs, query, timeout=None): content = json.dumps(job) # Do the request... - host = self._host - port = self._port url = "/" + self._mapred_prefix - response = self.http_request('POST', host, port, url, {}, content) + response = self.http_request('POST', url, {}, content) result = json.loads(response[1]) return result @@ -330,19 +328,19 @@ def add_links_for_riak_object(self, robject, headers): return headers def get_request(self, uri=None, params=None): - host, port, url = self.build_rest_path(bucket=None, params=params, prefix=uri) - return self.http_request('GET', host, port, url) + url = self.build_rest_path(bucket=None, params=params, prefix=uri) + return self.http_request('GET', url) def store_file(self, key, content_type="application/octet-stream", content=None): - host, port, url = self.build_rest_path(prefix='luwak', key=key) + url = self.build_rest_path(prefix='luwak', key=key) headers = {'Content-Type' : content_type, 'X-Riak-ClientId' : self._client_id} - return self.do_put(host, port, url, headers, content, key=key) + return self.do_put(url, headers, content, key=key) def get_file(self, key): - host, port, url = self.build_rest_path(prefix='luwak', key=key) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(prefix='luwak', key=key) + response = self.http_request('GET', url) result = self.parse_body(response, [200, 300, 404]) if result is not None: (vclock, data) = result @@ -350,13 +348,13 @@ def get_file(self, key): return body def delete_file(self, key): - host, port, url = self.build_rest_path(prefix='luwak', key=key) - response = self.http_request('DELETE', host, port, url) + url = self.build_rest_path(prefix='luwak', key=key) + response = self.http_request('DELETE', url) self.parse_body(response, [204, 404]) def post_request(self, uri=None, body=None, params=None, content_type="application/json"): - host, port, uri = self.build_rest_path(prefix=uri, params=params) - return self.http_request('POST', self._host, self._port, uri, {'Content-Type': content_type}, body) + uri = self.build_rest_path(prefix=uri, params=params) + return self.http_request('POST', uri, {'Content-Type': content_type}, body) # Utility functions used by Riak library. @@ -386,10 +384,9 @@ def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : path += '?' + s # Return. - return self._host, self._port, path + return path - @classmethod - def http_request(cls, method, host, port, url, headers = None, obj = '') : + def http_request(self, method, url, headers = None, obj = '') : """ Given a Method, URL, Headers, and Body, perform and HTTP request, and return an array of arity 2 containing an associative array of @@ -397,17 +394,16 @@ def http_request(cls, method, host, port, url, headers = None, obj = '') : """ if not headers: headers = {} - return cls.httplib_request(method, host, port, url, headers, obj) + return self.httplib_request(method, url, headers, obj) - @classmethod - def httplib_request(cls, method, host, port, uri, headers = None, body=''): + def httplib_request(self, method, uri, headers = None, body=''): if not headers: headers = {} # Run the request... client = None response = None try: - client = httplib.HTTPConnection(host, port) + client = httplib.HTTPConnection(self._host, self._port) client.request(method, uri, body, headers) response = client.getresponse() @@ -473,13 +469,12 @@ def __copy__(self): return RiakHttpReuseTransport(self._host, self._port, self._prefix, self._mapred_prefix) - @classmethod - def httplib_request(cls, method, host, port, uri, headers, body=''): + def httplib_request(self, method, uri, headers, body=''): # Run the request... client = None response = None try: - client = httplib.HTTPConnection(host, port) + client = httplib.HTTPConnection(self._host, self._port) #handle the connection myself, try to reuse sockets client.auto_open = 0 @@ -538,13 +533,15 @@ def __copy__(self): return RiakHttpPoolTransport(self._host, self._port, self._prefix, self._mapred_prefix) - @classmethod - def httplib_request(cls, method, host, port, uri, headers, body=''): + def httplib_request(self, method, uri, headers, body=''): try: - if cls.http_pool is None: - cls.http_pool = urllib3.connection_from_url('http://%s:%d' % (host, port), maxsize=10) + ### it seems wrong to put the pool into a *class* variable, + ### but this code is supporting backwards-compat where the + ### use of a class variable was the design. + if self.__class__.http_pool is None: + self.__class__.http_pool = urllib3.connection_from_url('http://%s:%d' % (self._host, self._port), maxsize=10) - response = cls.http_pool.urlopen(method, uri, body, headers) + response = self.http_pool.urlopen(method, uri, body, headers) response_headers = {'http_code': response.status} for key, value in response.getheaders().iteritems(): From cc9001dd6dc836accb00c2cc3ef3ab3ce1e05636 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 23:28:11 -0400 Subject: [PATCH 011/118] Remove unused populate_links() method. --- riak/riak_object.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 01fdae8c..d7010928 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -395,20 +395,6 @@ def populate(self, Result) : else: raise RiakError("do not know how to handle type " + str(type(Result))) - def populate_links(self, linkHeaders) : - """ - Private. - - :rtype: self - """ - for linkHeader in linkHeaders.strip().split(','): - linkHeader = linkHeader.strip() - matches = re.match("\<\/([^\/]+)\/([^\/]+)\/([^\/]+)\>; ?riaktag=\"([^\']+)\"", linkHeader) - if (matches is not None): - link = RiakLink(matches.group(2), matches.group(3), matches.group(4)) - self._links.append(link) - return self - def has_siblings(self): """ Return True if this object has siblings. From e21caf0ece03639480a6c15316332844ca62c805 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 23:32:00 -0400 Subject: [PATCH 012/118] Turn a seemingly random base-10 value into an understandable hex value. --- riak/transports/transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index b8245f0e..96069bfb 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -35,7 +35,7 @@ def make_random_client_id(self): Returns a random client identifier """ return 'py_%s' % base64.b64encode( - str(random.randint(1, 1073741824))) + str(random.randint(1, 0x40000000))) @classmethod def make_fixed_client_id(self): From 1a9c8e5a6bf4cc185fd5739bbaf8f946353c24ff Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 23:34:01 -0400 Subject: [PATCH 013/118] Fix docstring for RiakTransport.delete_file(). --- riak/transports/transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 96069bfb..7d0be45f 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -134,9 +134,9 @@ def get_file(self, key): """ raise RiakError("luwak not supported by this transport.") + def delete_file(self, key): """ Delete an object in luwak. key = the object's key """ - def delete_file(self, key): raise RiakError("luwak not supported by this transport.") From 3e196c5aa0dd1ed9cac4361ba1249df809ccf9fd Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 23:44:52 -0400 Subject: [PATCH 014/118] Fold .httplib_request() directly into .http_request(). * riak/transports/http.py: (RiakHttpTransport.http_request): adjust param names to URI and BODY to match the expections of the .httplib_request() method body. fold that method into http_request. alter header examinatino to "is None" so we don't (unnecessarily) fire on an empty dictionary. (RiakHttpTransport.httplib_request): folded into .http_request() (RiakHttpReuseTransport.httplib_request): renamed to .http_request() (RiakHttpPoolTransport.httplib_request): renamed to .http_request() --- riak/transports/http.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 2e9ec583..3b5d241e 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -386,18 +386,13 @@ def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : # Return. return path - def http_request(self, method, url, headers = None, obj = '') : + def http_request(self, method, uri, headers=None, body='') : """ Given a Method, URL, Headers, and Body, perform and HTTP request, - and return an array of arity 2 containing an associative array of - response headers and the response body. + and return a 2-tuple containing a dictionary of response headers + and the response body. """ - if not headers: - headers = {} - return self.httplib_request(method, url, headers, obj) - - def httplib_request(self, method, uri, headers = None, body=''): - if not headers: + if headers is None: headers = {} # Run the request... client = None @@ -469,7 +464,7 @@ def __copy__(self): return RiakHttpReuseTransport(self._host, self._port, self._prefix, self._mapred_prefix) - def httplib_request(self, method, uri, headers, body=''): + def http_request(self, method, uri, headers, body=''): # Run the request... client = None response = None @@ -533,7 +528,7 @@ def __copy__(self): return RiakHttpPoolTransport(self._host, self._port, self._prefix, self._mapred_prefix) - def httplib_request(self, method, uri, headers, body=''): + def http_request(self, method, uri, headers, body=''): try: ### it seems wrong to put the pool into a *class* variable, ### but this code is supporting backwards-compat where the From 34befab6f89c6baa089daaf575e4c195302016aa Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 7 Sep 2011 19:52:34 -0400 Subject: [PATCH 015/118] Build a connection management class to handle multiple connections to multiple hosts, passing them out for use. Both HTTP and bare sockets are supported. * riak/transports/connection.py: (ConnectionManager): new base class for managing connections (HTTPConnectionManager): manage HTTP connections (Socket): new connection type, for bare sockets (SocketConnectionManager): manage bare socket connections (NoHostsDefined): simple exception when we cannot open a conn * riak/transports/http.py: (RiakHttpTransport.__init__): use a connection manager (RiakHttpTransport.__copy__): disable for now (RiakHttpTransport.parse_body): can't print host/port right now (RiakHttpTransport.http_request): rebuild to use the connection manager. --- riak/transports/connection.py | 133 ++++++++++++++++++++++++++++++++++ riak/transports/http.py | 41 ++++++----- 2 files changed, 154 insertions(+), 20 deletions(-) create mode 100644 riak/transports/connection.py diff --git a/riak/transports/connection.py b/riak/transports/connection.py new file mode 100644 index 00000000..754cfd98 --- /dev/null +++ b/riak/transports/connection.py @@ -0,0 +1,133 @@ +# +# ### docco +# + +import httplib +import socket +import contextlib + + +class ConnectionManager(object): + + # Must be constructable with: connection_class(host, port) + # Must have two attribute: host and port + # Must have a close() method + connection_class = None + + def __init__(self, hostports=[]): + self.hostports = hostports[:] + self.conns = [ ] + + def add_hostport(self, host, port): + self.hostports.append((host, port)) + + def remove_host(self, host, port=None): + if port is None: + self.hostports = [(h, p) for h, p in self.hostports + if h != host] + else: + self.hostports.remove((host, port)) + + # just in case somebody wants a host/port combo and typos... + remove_hostport = remove_host + + def take(self): + if len(self.conns) == 0: + # RACE: in a multi-threaded environment, a conn might arrive in + # self.conns, but... no biggy. If we're bouncing up against + # needing a new connection, then we'll just create one. + return self._new_connection() + + # RACE: self.conns might empty out right now, so we need to protect + # our access to it. + try: + # round-robin: take from the front, we'll append when it comes back + return self.conns.pop(0) + except IndexError: + return self._new_connection() + + def giveback(self, conn): + # Connections using a host/port pair that is NOT in self.hostports + # should be ignored. Likely, remove_host() was called while this + # connection was borrowed for some work. + if (conn.host, conn.port) in self.hostports: + self.conns.append(conn) + else: + # Proactively close the connection. The caller won't know whether + # we put it into our list, or left the connection for the caller + # to deal with (and close). + conn.close() + + @contextlib.contextmanager + def withconn(self): + conn = self.take() + try: + yield conn + finally: + self.giveback(conn) + + def _new_connection(self): + if len(self.hostports) == 0: + raise NoHostsDefined() + + # Grab the first host/port combo. We'll put this at the end, so that + # we do a round-robin on the host/port pairs. + host, port = self.hostports[0] + conn = self.connection_class(host, port) + + if len(self.hostports) == 1: + # No rotation needed. + return conn + + # Be careful about rotating. We want to append before removing, so that + # we never hit a len==0 race condition. + self.hostports.append((host, port)) + + # RACE: another thread may have appended the same host/port pair. We + # will add another pair. Each thread will remove one, resulting in + # a correct state of a single pair in the list. + # RACE: another thread may get the host/port pair from hostports[0] + # before we have a chance to remove it. We don't need precision + # round-robin behavior; just something close. + # RACE: another thread may have removed hostports[0], but it will have + # placed another copy at the end. We have added a host/port pair, and + # will remove one, leaving the list in a correct state. + self.hostports.remove((host, port)) + + return conn + + +class HTTPConnectionManager(ConnectionManager): + connection_class = httplib.HTTPConnection + + +class Socket(object): + + def __init__(self, host, port): + self.host = host + self.port = port + + self.sock = None + + def maybe_connect(self): + if self.sock is None: + self.sock = s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + try: + s.connect((self.host, self.port)) + except: + self.close() + raise + + def close(self): + if self.sock is not None: + self.sock.close() + self.sock = None + + +class SocketConnectionManager(ConnectionManager): + connection_class = Socket + + +class NoHostsDefined(Exception): + pass diff --git a/riak/transports/http.py b/riak/transports/http.py index 3b5d241e..b5306d53 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -17,6 +17,8 @@ specific language governing permissions and limitations under the License. """ +from __future__ import with_statement + import urllib, re from cStringIO import StringIO import httplib @@ -30,6 +32,7 @@ from riak.mapreduce import RiakLink from riak import RiakError from riak.multidict import MultiDict +from connection import HTTPConnectionManager MAX_LINK_HEADER_SIZE = 8192 - 8 # substract length of "Link: " header string and newline @@ -51,8 +54,7 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', @param string client_id - client id to use for vector clocks """ super(RiakHttpTransport, self).__init__() - self._host = host - self._port = port + self._conns = HTTPConnectionManager([(host, port)]) self._prefix = prefix self._mapred_prefix = mapred_prefix self._client_id = client_id @@ -60,6 +62,9 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', self._client_id = self.make_random_client_id() def __copy__(self): + ### not implemented right now + raise Exception('not implemented') + ### we don't have _host and _port. will fix after some refactoring... return RiakHttpTransport(self._host, self._port, self._prefix, self._mapred_prefix) @@ -240,7 +245,8 @@ def parse_body(self, response, expected_statuses): # Check if the server is down(status==0) if not status: - m = 'Could not contact Riak Server: http://' + self._host + ':' + str(self._port) + '!' + ### we need the host/port that was used. + m = 'Could not contact Riak Server: http://$HOST:$PORT !' raise RiakError(m) # Verify that we got one of the expected statuses. Otherwise, raise an exception. @@ -395,27 +401,22 @@ def http_request(self, method, uri, headers=None, body='') : if headers is None: headers = {} # Run the request... - client = None - response = None - try: - client = httplib.HTTPConnection(self._host, self._port) - client.request(method, uri, body, headers) - response = client.getresponse() + with self._conns.withconn() as conn: + conn.request(method, uri, body, headers) + response = conn.getresponse() - # Get the response headers... - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value + try: + # Get the response headers... + response_headers = {'http_code': response.status} + for (key, value) in response.getheaders(): + response_headers[key.lower()] = value - # Get the body... - response_body = response.read() - response.close() + # Get the body... + response_body = response.read() + finally: + response.close() return response_headers, response_body - except: - if client is not None: client.close() - if response is not None: response.close() - raise @classmethod def build_headers(cls, headers): From 136448ad16bb9e0c212ea7f2f96564509534a033 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 7 Sep 2011 23:45:05 -0400 Subject: [PATCH 016/118] Add/expand some commentary in the ConnectionManager, and add code to open a connection to the Riak server(s) once its host/port is defined. When a host (and port) is removed from the hostport set, then remove connections. --- riak/transports/connection.py | 50 ++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/riak/transports/connection.py b/riak/transports/connection.py index 754cfd98..85b44c2b 100644 --- a/riak/transports/connection.py +++ b/riak/transports/connection.py @@ -15,12 +15,25 @@ class ConnectionManager(object): connection_class = None def __init__(self, hostports=[]): + # We want a private copy of this list: either to detach the argument + # default, or to detach from the caller's list. self.hostports = hostports[:] - self.conns = [ ] + + # Open a connection to each specified host/port. On single-threaded + # systems, this will create a round-robin across all specified servers. + # When multi-threaded, this will give us an initial set for all the + # threads to work with (and more will be created, according to demand). + self.conns = [self.connection_class(host, port) + for host, port in hostports] def add_hostport(self, host, port): self.hostports.append((host, port)) + # Open an initial connection. For single-threaded, this adds to the + # round-robin pool. On multi-threaded, it simply gives us an extra + # connectiong for the load-balancing across the servers. + self.conn.append(self.connection_class(host, port)) + def remove_host(self, host, port=None): if port is None: self.hostports = [(h, p) for h, p in self.hostports @@ -28,7 +41,27 @@ def remove_host(self, host, port=None): else: self.hostports.remove((host, port)) - # just in case somebody wants a host/port combo and typos... + # Now that the host/port pair has been removed from self.hostports, + # no connections on this pair will be added in .giveback(). Thus, the + # existing connections are all that may exist at this time. We'll + # snapshot the list, and look for offending connections, then try and + # remove them, being wary that race conditions may remove them before + # we can remove it. + for conn in self.conns[:]: + if conn.host == host: + try: + if port is None or conn.port == port: + self.conns.remove(conn) + + # If the connection was still present (no ValueError), then we + # should go ahead and close it down. + conn.close() + except ValueError: + # Another thread removed the connection. It won't be coming back, + # so we have nothing to do here. + pass + + # Just in case somebody uses a host/port combo and typos... remove_hostport = remove_host def take(self): @@ -80,17 +113,20 @@ def _new_connection(self): return conn # Be careful about rotating. We want to append before removing, so that - # we never hit a len==0 race condition. + # we never hit a len==0 race condition (which could prevent the creation + # of needed connections). self.hostports.append((host, port)) # RACE: another thread may have appended the same host/port pair. We - # will add another pair. Each thread will remove one, resulting in - # a correct state of a single pair in the list. + # will add another pair. Each thread will remove one (either [0], or + # one that had been appened), resulting in a correct state of a single + # pair in the list. # RACE: another thread may get the host/port pair from hostports[0] # before we have a chance to remove it. We don't need precision # round-robin behavior; just something close. - # RACE: another thread may have removed hostports[0], but it will have - # placed another copy at the end. We have added a host/port pair, and + # RACE: another thread may have removed hostports[0] (which we are + # also trying to remove), but it will have placed another copy at + # the end before doing so. We have also added a host/port pair, and # will remove one, leaving the list in a correct state. self.hostports.remove((host, port)) From bf39809b9d5cf5d43c0a4e9380be1e219fd3f24a Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Thu, 8 Sep 2011 04:06:10 -0400 Subject: [PATCH 017/118] Enable the test suite to run under earlier versions of Python, and when some dependencies (protobuf and urllib3) are missing. Adjust the existing SKIP_* tests to use the unittest "skip" mechanisms. --- riak/test_server.py | 11 +++++-- riak/tests/test_all.py | 66 ++++++++++++++++++++++-------------------- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/riak/test_server.py b/riak/test_server.py index 0262aaa4..feed739a 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -1,3 +1,5 @@ +from __future__ import with_statement + import os.path import threading import string @@ -8,6 +10,11 @@ from subprocess import Popen, PIPE from riak.util import deep_merge +try: + bytes +except NameError: + bytes = str + def erlang_config(hash, depth=1): def printable(item): k, v = item @@ -48,7 +55,7 @@ class TestServer: "ring_creation_size": 64 }, "riak_kv": { - "storage_backend": bytearray("riak_kv_test_backend"), + "storage_backend": bytes("riak_kv_test_backend"), "pb_ip": "127.0.0.1", "pb_port": 9002, "js_vm_count": 8, @@ -60,7 +67,7 @@ class TestServer: }, "riak_search": { "enabled": True, - "search_backend": bytearray("riak_search_test_backend") + "search_backend": bytes("riak_search_test_backend") }, "luwak": { "enabled": True diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index d2fa9913..73940fe9 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import with_statement import copy import cPickle @@ -24,12 +25,24 @@ from riak.mapreduce import RiakLink from riak.test_server import TestServer +try: + import riak.transports.riakclient_pb2 + HAVE_PROTO = True +except ImportError: + HAVE_PROTO = False +try: + import urllib3 + HAVE_HTTP_POOL = True +except ImportError: + HAVE_HTTP_POOL = False + HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) PB_HOST = os.environ.get('RIAK_TEST_PB_HOST', HOST) HTTP_PORT = int(os.environ.get('RIAK_TEST_HTTP_PORT', '8098')) PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '0')) +SKIP_LUWAK = int(os.environ.get('SKIP_LUWAK', '0')) USE_TEST_SERVER = int(os.environ.get('USE_TEST_SERVER', '0')) if USE_TEST_SERVER: @@ -40,7 +53,6 @@ test_server.prepare() test_server.start() -SKIP_LUWAK = int(os.environ.get('SKIP_LUWAK', '0')) class NotJsonSerializable(object): @@ -443,10 +455,8 @@ def test_store_of_missing_object(self): self.assertEqual(o.get_content_type(), "application/octet-stream") o.delete() - + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_search_integration(self): - if SKIP_SEARCH: - return True # Create some objects to search across... bucket = self.client.bucket("searchbucket") bucket.new("one", {"foo":"one", "bar":"red"}).store() @@ -698,6 +708,8 @@ class RiakPbcTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): + if not HAVE_PROTO: + self.skipTest('protobuf is unavailable') self.host = PB_HOST self.port = PB_PORT self.transport_class = RiakPbcTransport @@ -712,9 +724,12 @@ def test_uses_client_id_if_given(self): client_id = zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) # + class RiakPbcCachedTransportCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): + if not HAVE_PROTO: + self.skipTest('protobuf is unavailable') self.host = PB_HOST self.port = PB_PORT self.transport_class = RiakPbcCachedTransport @@ -779,10 +794,8 @@ def test_disable_search_commit_hook(self): bucket.disable_search() self.assertFalse(self.client.bucket("no_search_bucket").search_enabled()) + @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_store_file_with_luwak(self): - if SKIP_LUWAK: - return True - file = os.path.dirname(__file__) + "/test_all.py" with open(file, "r") as input_file: data = input_file.read() @@ -790,10 +803,8 @@ def test_store_file_with_luwak(self): key = uuid.uuid1().hex self.client.store_file(key, data) + @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_store_get_file_with_luwak(self): - if SKIP_LUWAK: - return True - file = os.path.dirname(__file__) + "/test_all.py" with open(file, "r") as input_file: data = input_file.read() @@ -804,10 +815,8 @@ def test_store_get_file_with_luwak(self): file = self.client.get_file(key) self.assertEquals(data, file) + @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_delete_file_with_luwak(self): - if SKIP_LUWAK: - return True - file = os.path.dirname(__file__) + "/test_all.py" with open(file, "r") as input_file: data = input_file.read() @@ -820,17 +829,15 @@ def test_delete_file_with_luwak(self): file = self.client.get_file(key) self.assertIsNone(file) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_from_bucket(self): - if SKIP_SEARCH: - return True bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage") self.assertEquals(1, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params_from_bucket(self): - if SKIP_SEARCH: - return True bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage", wt="xml") @@ -838,9 +845,8 @@ def test_solr_search_with_params_from_bucket(self): if not hasattr(result, "iter"): setattr(result, "iter", result.getiterator) self.assertEquals(1, len(list(result.iter("doc")))) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params(self): - if SKIP_SEARCH: - return True bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = self.client.solr().search("searchbucket", "username:roidrage", wt="xml") @@ -848,56 +854,52 @@ def test_solr_search_with_params(self): if not hasattr(result, "iter"): setattr(result, "iter", result.getiterator) self.assertEquals(1, len(list(result.iter("doc")))) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search(self): - if SKIP_SEARCH: - return True bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = self.client.solr().search("searchbucket", "username:roidrage") self.assertEquals(1, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_document_to_index(self): - if SKIP_SEARCH: - return True - self.client.solr().add("searchbucket", {"id": "doc", "username": "tony"}) results = self.client.solr().search("searchbucket", "username:tony") self.assertEquals("tony", results["response"]["docs"][0]["fields"]["username"]) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_multiple_documents_to_index(self): - if SKIP_SEARCH: - return True self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(2, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_id(self): - if SKIP_SEARCH: - return True self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", docs=["dizzy"]) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(1, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): - if SKIP_SEARCH: - return True self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", queries=["username:dizzy", "username:russell"]) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query_and_id(self): - if SKIP_SEARCH: - return True self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", docs=["dizzy"], queries=["username:russell"]) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results["response"]["docs"])) + class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): + if not HAVE_HTTP_POOL: + self.skipTest('urllib3 is unavailable') self.host = HTTP_HOST self.port = HTTP_PORT self.transport_class = RiakHttpPoolTransport From 303bad95a2b3f105d428733e7c45b5345ad21c6b Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 3 Sep 2011 20:40:13 +0800 Subject: [PATCH 018/118] change to python 2.5 string --- riak/transports/pbc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index ce1a27b9..365ce54f 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -367,8 +367,8 @@ def send_msg(self, msg_code, msg): pkt = self.encode_msg(msg_code, msg) sent_len = self._sock.send(pkt) if sent_len != len(pkt): - raise RiakError("PB socket returned short write {0} - expected {1}". - format(sent_len, len(pkt))) + raise RiakError("PB socket returned short write %d - expected %d"%\ + (sent_len, len(pkt)) def recv_msg(self): self.recv_pkt() From ca53521551788363c1d162520c9aae1e3d363515 Mon Sep 17 00:00:00 2001 From: Socrates Lee Date: Sat, 3 Sep 2011 22:40:19 +0800 Subject: [PATCH 019/118] change string format to 2.5 (cherry picked from commit 80931c36a3aba05ae161baf61971fab2039dc08d) --- riak/transports/pbc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 365ce54f..7d33975d 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -407,7 +407,7 @@ def recv_msg(self): msg = riakclient_pb2.RpbMapRedResp() msg.ParseFromString(self._inbuf[1:]) else: - raise Exception("unknown msg code {0}".format(msg_code)) + raise Exception("unknown msg code %s"%msg_code) return msg_code, msg @@ -415,8 +415,8 @@ def recv_pkt(self): nmsglen = self._sock.recv(4) if len(nmsglen) != 4: self._sock = None - raise RiakError("Socket returned short packet length {0} - expected 4". - format(nmsglen)) + raise RiakError("Socket returned short packet length %d - expected 4"%\ + nmsglen) msglen, = struct.unpack('!i', nmsglen) self._inbuf_len = msglen self._inbuf = '' @@ -426,8 +426,8 @@ def recv_pkt(self): if not recv_buf: break self._inbuf += recv_buf if len(self._inbuf) != self._inbuf_len: - raise RiakError("Socket returned short packet {0} - expected {1}". - format(len(self._inbuf), self._inbuf_len)) + raise RiakError("Socket returned short packet %d - expected %d"%\ + (len(self._inbuf), self._inbuf_len)) def decode_contents(self, rpb_contents): contents = [] From c18c22d89bbb5a3dab0d414bfeb0e8f68cbfab9d Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Fri, 9 Sep 2011 13:01:33 +0100 Subject: [PATCH 020/118] Fix syntax error from 303bad9 --- riak/transports/pbc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 7d33975d..7beb5267 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -368,7 +368,7 @@ def send_msg(self, msg_code, msg): sent_len = self._sock.send(pkt) if sent_len != len(pkt): raise RiakError("PB socket returned short write %d - expected %d"%\ - (sent_len, len(pkt)) + (sent_len, len(pkt))) def recv_msg(self): self.recv_pkt() From efa672c3cf5c1cf8f3dbafa572bff460ae7e5fa3 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 9 Sep 2011 21:23:07 -0400 Subject: [PATCH 021/118] Align the .http_request() method of the RiakHttpTransport subclasses with the superclass' signature. * riak/transports/http.py: (RiakHttpReuseTransport.http_request, RiakHttpPoolTransport): default the HEADERS param to None, and then set it to an empty dict when the caller doesn't set it --- riak/transports/http.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 3b5d241e..466cf229 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -464,7 +464,9 @@ def __copy__(self): return RiakHttpReuseTransport(self._host, self._port, self._prefix, self._mapred_prefix) - def http_request(self, method, uri, headers, body=''): + def http_request(self, method, uri, headers=None, body=''): + if headers is None: + headers = {} # Run the request... client = None response = None @@ -528,7 +530,9 @@ def __copy__(self): return RiakHttpPoolTransport(self._host, self._port, self._prefix, self._mapred_prefix) - def http_request(self, method, uri, headers, body=''): + def http_request(self, method, uri, headers={}, body=''): + if headers is None: + headers = {} try: ### it seems wrong to put the pool into a *class* variable, ### but this code is supporting backwards-compat where the From 54eae5aa23a24d923cdc8eff1dba28dedd24e3be Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 9 Sep 2011 21:52:39 -0400 Subject: [PATCH 022/118] Skip RiakHttpReuseTransport tests because that transport is broken (it is redundant, given the new RiakHttpTransport capabilities) --- riak/tests/test_all.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 73940fe9..f064bb0e 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -926,6 +926,8 @@ def test_set_client_id(self): class RiakHttpReuseTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): + ### RiakHttpReuseTransport does not work on this branch. + self.skipTest('RiakHttpReuseTransport is broken right now.') self.host = HTTP_HOST self.port = HTTP_PORT self.transport_class = RiakHttpReuseTransport From f2c347edf6f221e1d7ff5c297dc2f326cb84d19b Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Sat, 10 Sep 2011 23:18:32 -0400 Subject: [PATCH 023/118] Fix the test suite when invoked as "python test_all.py". In this case, __file__ has no directory component, so .dirname() returns '' and adding it to '/test_all.py' (or similar) will result in a reference to a file at th root of the filesystem (oops). Using os.path.join() properly handles this situation, and can increase portability. * riak/tests/test_all.py: (BaseTestCase.test_store_binary_object_from_file, BaseTestCase.test_store_file_with_luwak, BaseTestCase.test_store_get_file_with_luwak, BaseTestCase.test_delete_file_with_luwak): use os.path.join() to properly deal with variant invocations that affect the __file__ value. (BaseTestCase.test_store_binary_object_from_file_should_use_default_mimetype): use os.path.abspath(), os.pardir, and os.path.join() to correctly find the THANKS file. --- riak/tests/test_all.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 73940fe9..50914fa3 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -478,7 +478,8 @@ def test_search_integration(self): def test_store_binary_object_from_file(self): bucket = self.client.bucket('bucket') rand = str(self.randint()) - obj = bucket.new_binary_from_file('foo_from_file', os.path.dirname(__file__) + "/test_all.py") + filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') + obj = bucket.new_binary_from_file('foo_from_file', filepath) obj.store() obj = bucket.get_binary('foo_from_file') self.assertNotEqual(obj.get_data(), None) @@ -487,7 +488,9 @@ def test_store_binary_object_from_file(self): def test_store_binary_object_from_file_should_use_default_mimetype(self): bucket = self.client.bucket('bucket') rand = str(self.randint()) - obj = bucket.new_binary_from_file('foo_from_file', os.path.dirname(__file__) + '/../../THANKS') + filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), + os.pardir, os.pardir, 'THANKS') + obj = bucket.new_binary_from_file('foo_from_file', filepath) obj.store() obj = bucket.get_binary('foo_from_file') self.assertEqual(obj.get_content_type(), 'application/octet-stream') @@ -796,7 +799,7 @@ def test_disable_search_commit_hook(self): @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_store_file_with_luwak(self): - file = os.path.dirname(__file__) + "/test_all.py" + file = os.path.join(os.path.dirname(__file__), "test_all.py") with open(file, "r") as input_file: data = input_file.read() @@ -805,7 +808,7 @@ def test_store_file_with_luwak(self): @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_store_get_file_with_luwak(self): - file = os.path.dirname(__file__) + "/test_all.py" + file = os.path.join(os.path.dirname(__file__), "test_all.py") with open(file, "r") as input_file: data = input_file.read() @@ -817,7 +820,7 @@ def test_store_get_file_with_luwak(self): @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_delete_file_with_luwak(self): - file = os.path.dirname(__file__) + "/test_all.py" + file = os.path.join(os.path.dirname(__file__), "test_all.py") with open(file, "r") as input_file: data = input_file.read() From 18951435310a7a056cccc7e053a68bc09251a303 Mon Sep 17 00:00:00 2001 From: Reid Draper Date: Sun, 11 Sep 2011 12:48:17 +0100 Subject: [PATCH 024/118] Fix TypeError in exception text by using len(nmsglen) instead of nmsglen directly --- riak/transports/pbc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 7beb5267..07bf6f85 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -416,7 +416,7 @@ def recv_pkt(self): if len(nmsglen) != 4: self._sock = None raise RiakError("Socket returned short packet length %d - expected 4"%\ - nmsglen) + len(nmsglen)) msglen, = struct.unpack('!i', nmsglen) self._inbuf_len = msglen self._inbuf = '' From 1b4b74d172db61225786dd9acdea3d343a5b63da Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Sun, 11 Sep 2011 19:48:20 -0400 Subject: [PATCH 025/118] Fixes via @reiddraper. * riak/transports/connection.py: (ConnectionManager.add_hostport): fix typo: s/conn/conns/ (Connectionmanager.remove_host): tweak control flow to move conn.close() outside of the try/except. --- riak/transports/connection.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/riak/transports/connection.py b/riak/transports/connection.py index 85b44c2b..75669c6e 100644 --- a/riak/transports/connection.py +++ b/riak/transports/connection.py @@ -32,7 +32,7 @@ def add_hostport(self, host, port): # Open an initial connection. For single-threaded, this adds to the # round-robin pool. On multi-threaded, it simply gives us an extra # connectiong for the load-balancing across the servers. - self.conn.append(self.connection_class(host, port)) + self.conns.append(self.connection_class(host, port)) def remove_host(self, host, port=None): if port is None: @@ -48,18 +48,17 @@ def remove_host(self, host, port=None): # remove them, being wary that race conditions may remove them before # we can remove it. for conn in self.conns[:]: - if conn.host == host: + if conn.host == host and (port is None or conn.port == port): try: - if port is None or conn.port == port: - self.conns.remove(conn) - - # If the connection was still present (no ValueError), then we - # should go ahead and close it down. - conn.close() + self.conns.remove(conn) except ValueError: # Another thread removed the connection. It won't be coming back, # so we have nothing to do here. pass + else: + # If the connection was still present (no ValueError), then we + # should go ahead and close it down. + conn.close() # Just in case somebody uses a host/port combo and typos... remove_hostport = remove_host From dcb8678b8f460ce1b5d5d86e14d567a3bcbaa0d1 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 12 Sep 2011 21:08:52 -0400 Subject: [PATCH 026/118] Adjust for compatibility with Python 2.5 --- riak/util.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/riak/util.py b/riak/util.py index c2e154a0..39f1a210 100644 --- a/riak/util.py +++ b/riak/util.py @@ -1,8 +1,12 @@ -import collections +try: + from collections import Mapping +except ImportError: + # compatibility with Python 2.5 + Mapping = dict def quacks_like_dict(object): """Check if object is dict-like""" - return isinstance(object, collections.Mapping) + return isinstance(object, Mapping) def deep_merge(a, b): """Merge two deep dicts non-destructively From 6235c0b3c8e6fc49077e840fbe8db164e425d892 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 12 Sep 2011 21:45:00 -0400 Subject: [PATCH 027/118] Pass a ConnectionManager to the transport constructor. Create an appropriate CM and pass that to the transport. We also adjust the constructor signature and allow for arbitrary options to be passed to the transport (eg. Http takes a prefix and mapred_prefix). Some backwards compat code has been introduced in all transports (other than the basic RiakHttpTransport) to extract a host/port pair from the CM. This commit also re-enables testing of RiakHttpReuseTransport, even though it is busted and should go away "soon". --- riak/client.py | 25 ++++++++++++++------- riak/tests/test_all.py | 2 -- riak/transports/http.py | 48 ++++++++++++++++++++++------------------- riak/transports/pbc.py | 18 ++++++++++++++-- 4 files changed, 59 insertions(+), 34 deletions(-) diff --git a/riak/client.py b/riak/client.py index 867cd2e4..ced72420 100644 --- a/riak/client.py +++ b/riak/client.py @@ -27,6 +27,8 @@ from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduce from riak.search import RiakSearch +import riak.transports.connection + class RiakClient(object): """ @@ -53,14 +55,21 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', :param solr_transport_class: HTTP-based transport class for Solr interface queries :type transport_class: :class:`RiakHttpTransport` """ - if not transport_class: - self._transport = RiakHttpTransport(host, - port, - prefix, - mapred_prefix, - client_id) - else: - self._transport = transport_class(host, port, client_id=client_id) + if transport_class is None: + transport_class = RiakHttpTransport + + hostports = [ (host, port), ] + self._cm = transport_class.default_cm(hostports) + + ### we need to allow additional transport options. make this an + ### argument to __init__ ? + transport_options = { } + + self._transport = transport_class(self._cm, + prefix=prefix, + mapred_prefix=mapred_prefix, + client_id=client_id, + **transport_options) self._r = "default" self._w = "default" diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 37e69df5..50914fa3 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -929,8 +929,6 @@ def test_set_client_id(self): class RiakHttpReuseTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): - ### RiakHttpReuseTransport does not work on this branch. - self.skipTest('RiakHttpReuseTransport is broken right now.') self.host = HTTP_HOST self.port = HTTP_PORT self.transport_class = RiakHttpReuseTransport diff --git a/riak/transports/http.py b/riak/transports/http.py index 3bb71925..30ca5d58 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -42,9 +42,13 @@ class RiakHttpTransport(RiakTransport) : Riak. The Riak API uses HTTP, so there is no persistent connection, and the RiakClient object is extremely lightweight. """ - def __init__(self, host='127.0.0.1', port=8098, prefix='riak', - mapred_prefix='mapred', - client_id = None): + + # The ConnectionManager class that this transport prefers. + default_cm = HTTPConnectionManager + + def __init__(self, cm, + prefix='riak', mapred_prefix='mapred', client_id=None, + **unused_options): """ Construct a new RiakClient object. @param string host - Hostname or IP address (default '127.0.0.1') @@ -54,7 +58,7 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', @param string client_id - client id to use for vector clocks """ super(RiakHttpTransport, self).__init__() - self._conns = HTTPConnectionManager([(host, port)]) + self._conns = cm self._prefix = prefix self._mapred_prefix = mapred_prefix self._client_id = client_id @@ -451,18 +455,18 @@ class RiakHttpReuseTransport(RiakHttpTransport): Reuse sockets """ - def __init__(self, host='127.0.0.1', port=8098, prefix='riak', - mapred_prefix='mapred', - client_id=None): - super(RiakHttpReuseTransport, self).__init__(host=host, - port=port, - prefix=prefix, - mapred_prefix= + def __init__(self, cm, + prefix='riak', mapred_prefix='mapred', client_id=None, + **unused_options): + super(RiakHttpReuseTransport, self).__init__(cm, + prefix, mapred_prefix, - client_id=client_id) + client_id) + ### for backwards compat + self._host, self._port = cm.hostports[0] def __copy__(self): - return RiakHttpReuseTransport(self._host, self._port, self._prefix, + return RiakHttpReuseTransport(self._conns, self._prefix, self._mapred_prefix) def http_request(self, method, uri, headers=None, body=''): @@ -514,21 +518,21 @@ class RiakHttpPoolTransport(RiakHttpTransport): http_pool = None - def __init__(self, host='127.0.0.1', port=8098, prefix='riak', - mapred_prefix='mapred', - client_id=None): + def __init__(self, cm, + prefix='riak', mapred_prefix='mapred', client_id=None, + **unused_options): if urllib3 is None: raise RiakError("this transport is not available (no urllib3)") - super(RiakHttpPoolTransport, self).__init__(host=host, - port=port, - prefix=prefix, - mapred_prefix= + super(RiakHttpPoolTransport, self).__init__(cm, + prefix, mapred_prefix, - client_id=client_id) + client_id) + ### for backwards compat + self._host, self._port = cm.hostports[0] def __copy__(self): - return RiakHttpPoolTransport(self._host, self._port, self._prefix, + return RiakHttpPoolTransport(self._conns, self._prefix, self._mapred_prefix) def http_request(self, method, uri, headers={}, body=''): diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 07bf6f85..08d08a39 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -30,6 +30,7 @@ from riak.metadata import * from riak.mapreduce import RiakMapReduce, RiakLink from riak import RiakError +from connection import SocketConnectionManager try: import riakclient_pb2 @@ -82,7 +83,11 @@ class RiakPbcTransport(RiakTransport): 'quorum' : RIAKC_RW_QUORUM, 'one' : RIAKC_RW_ONE } - def __init__(self, host='127.0.0.1', port=8087, client_id=None): + + # The ConnectionManager class that this transport prefers. + default_cm = SocketConnectionManager + + def __init__(self, cm, client_id=None, **unused_options): """ Construct a new RiakPbcTransport object. @param string host - Hostname or IP address (default '127.0.0.1') @@ -92,6 +97,10 @@ def __init__(self, host='127.0.0.1', port=8087, client_id=None): raise RiakError("this transport is not available (no protobuf)") super(RiakPbcTransport, self).__init__() + + ### backwards compat. we don't use the ConnectionManager (yet). + host, port = cm.hostports[0] + self._host = host self._port = port self._client_id = client_id @@ -500,10 +509,15 @@ def pbify_content(self, metadata, data, rpb_content) : import contextlib class RiakPbcCachedTransport(RiakTransport): """Threadsafe pool of PBC connections, based on urllib3's pool [aka Queue]""" - def __init__(self, host='127.0.0.1', port=8087, client_id=None, maxsize=0, block=False, timeout=None): + def __init__(self, cm, + client_id=None, maxsize=0, block=False, timeout=None, + **unused_options): if riakclient_pb2 is None: raise RiakError("this transport is not available (no protobuf)") + ### backwards compat. we don't use the ConnectionManager (yet). + host, port = cm.hostports[0] + self.host = host self.port = port self.client_id = client_id From 9d90b0cbb008b1d77426f673f2330312d7a3360e Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Tue, 13 Sep 2011 03:36:27 -0400 Subject: [PATCH 028/118] Fix the transport creation in the search object. --- riak/search.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/riak/search.py b/riak/search.py index c01f9d3d..5e006c25 100644 --- a/riak/search.py +++ b/riak/search.py @@ -5,12 +5,12 @@ class RiakSearch: def __init__(self, client, transport_class=None, host="127.0.0.1", port=8098): - if not transport_class: - self._transport = RiakHttpTransport(host, - port, - "/solr") - else: - self._transport = transport_class(host, port, client_id=client_id) + if transport_class is None: + transport_class = RiakHttpTransport + + hostports = [ (host, port), ] + self._cm = transport_class.default_cm(hostports) + self._transport = transport_class(self._cm, prefix="/solr") self._client = client self._decoders = {"text/xml": ElementTree.fromstring} From d0de18b1100f03eb315c3c3eb2a3938c9aebe4cf Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Tue, 13 Sep 2011 03:50:39 -0400 Subject: [PATCH 029/118] First draft of a background thread to monitor connections. --- riak/transports/monitor.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 riak/transports/monitor.py diff --git a/riak/transports/monitor.py b/riak/transports/monitor.py new file mode 100644 index 00000000..6380eee1 --- /dev/null +++ b/riak/transports/monitor.py @@ -0,0 +1,29 @@ +import threading +import time + + +class Monitor(object): + + def __init__(self, cm, transport): + self._cm = cm + self._transport = transport + + self._stop_loop = False + self._thread = threading.Thread(target=self._run) + + self._periodic = 0.050 # 50 msec + + def start(self): + self._thread.start() + + def terminate(self): + self._stop_loop = True + self._thread.join() + + def _run(self): + while not self._stop_loop: + ### look for changes in the ring servers + + ### see if some offline servers came back + + time.sleep(self._periodic) From 7d4cdc4ff4f0ed2f2d121bd8905a34343e8b5c56 Mon Sep 17 00:00:00 2001 From: Rusty Klophaus Date: Fri, 9 Sep 2011 08:10:41 -0400 Subject: [PATCH 030/118] Add 2i query support. --- docs/tutorial.rst | 28 ++++ riak/client.py | 10 ++ riak/mapreduce.py | 33 ++++- riak/metadata.py | 1 + riak/riak_object.py | 20 ++- riak/tests/test_all.py | 62 ++++++++- riak/transports/http.py | 7 +- riak/transports/pbc.py | 10 ++ riak/transports/riakclient.proto | 23 +++- riak/transports/riakclient_pb2.py | 206 ++++++++++++++++++++++++------ 10 files changed, 352 insertions(+), 48 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 825e9ac3..f7a3cefa 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -387,3 +387,31 @@ tutorial, but usage of this feature looks like:: .. _`Riak Search`: http://wiki.basho.com/Riak-Search.html .. _Lucene: http://lucene.apache.org/ + +Using Secondary Indexes +======================= + +Secondary Indexes is a new feature available as of Riak 1.0. It +allows you to tag an object with index metadata, and then later find +the object by querying the metadata, returning a list of matching keys. + +Usage of this feature looks like:: + + import riak + + client = riak.RiakClient() + bucket = client.bucket('mybucket') + + # Store the object... + obj = bucket.new('mykey1', 'mydata') + obj.set_indexes({ + 'field1_bin': 'val1', + 'field2_int': 1001 + }) + obj.store() + + # Query the indexes. The return value is a list of ``RiakLink`` objects. + results = client.index('mybucket', 'field1_bin', 'val1').run() + + # Query the indexes using a range... + results = client.index('mybucket', 'field1_bin', 'val1', 'val5').run() diff --git a/riak/client.py b/riak/client.py index 867cd2e4..3ea9c7aa 100644 --- a/riak/client.py +++ b/riak/client.py @@ -265,6 +265,16 @@ def search(self, *args): mr = RiakMapReduce(self) return apply(mr.search, args) + def index(self, *args): + """ + Start assembling a Map/Reduce operation based on secondary + index query results. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.index, args) + def link(self, *args): """ Start assembling a Map/Reduce operation. A shortcut for :func:`RiakMapReduce.link`. diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 01020c97..c7181d15 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -63,8 +63,8 @@ def add_object(self, obj): def add_bucket_key_data(self, bucket, key, data) : if self._input_mode == 'bucket': raise Exception('Already added a bucket, can\'t add an object.') - elif self._input_mode == 'search': - raise Exception('Already added a search query, can\'t add an object.') + elif self._input_mode == 'query': + raise Exception('Already added a query, can\'t add an object.') else: self._inputs.append([bucket, key, data]) return self @@ -75,15 +75,15 @@ def add_bucket(self, bucket) : return self def add_key_filters(self, key_filters) : - if self._input_mode == 'search': - raise Exception('Key filters are not supported in search query.') + if self._input_mode == 'query': + raise Exception('Key filters are not supported in a query.') self._key_filters.extend(key_filters) return self def add_key_filter(self, *args) : - if self._input_mode == 'search': - raise Exception('Key filters are not supported in search query.') + if self._input_mode == 'query': + raise Exception('Key filters are not supported in a query.') self._key_filters.append(args) return self @@ -95,12 +95,31 @@ def search(self, bucket, query): @param bucket - The bucket over which to perform the search. @param query - The search query. """ - self._input_mode = 'search' + self._input_mode = 'query' self._inputs = {'module':'riak_search', 'function':'mapred_search', 'arg':[bucket, query]} return self + def index(self, bucket, index, startkey, endkey = None): + """ + Begin a map/reduce operation using a Secondary Index + query. + @param bucket - The bucket over which to perform the search. + @param query - The search query. + """ + self._input_mode = 'query' + + if endkey == None: + self._inputs = {'bucket': bucket, + 'index':index, + 'key':startkey } + else: + self._inputs = {'bucket':bucket, + 'index':index, + 'start':startkey, + 'end':endkey } + return self def link(self, bucket='_', tag='_', keep=False): """ diff --git a/riak/metadata.py b/riak/metadata.py index 5efad289..37c81b2a 100644 --- a/riak/metadata.py +++ b/riak/metadata.py @@ -25,3 +25,4 @@ MD_LASTMOD = "lastmod" MD_LASTMOD_USECS = "lastmod-usecs" MD_USERMETA = "usermeta" +MD_INDEX = "index" diff --git a/riak/riak_object.py b/riak/riak_object.py index d7010928..fe20dbdd 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -47,7 +47,7 @@ def __init__(self, client, bucket, key=None): self._encode_data = True self._vclock = None self._data = None - self._metadata = {MD_USERMETA: {}} + self._metadata = {MD_USERMETA: {}, MD_INDEX: {}} self._links = [] self._siblings = [] self._exists = False @@ -177,6 +177,24 @@ def set_usermeta(self, usermeta): self._metadata[MD_USERMETA] = usermeta return self + def get_indexes(self): + if MD_INDEX in self._metadata: + return self._metadata[MD_INDEX] + else: + return {} + + def set_indexes(self, indexes): + """ + Sets the field/value indexes under which this object will be + indexed. + + :param indexes: The field/value index data. + :type indexes: dict + :rtype: data + """ + self._metadata[MD_INDEX] = indexes + return self + def exists(self): """ Return True if the object exists, False otherwise. Allows you to diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 50914fa3..6825d24e 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -43,6 +43,7 @@ PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '0')) SKIP_LUWAK = int(os.environ.get('SKIP_LUWAK', '0')) +SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '0')) USE_TEST_SERVER = int(os.environ.get('USE_TEST_SERVER', '0')) if USE_TEST_SERVER: @@ -517,6 +518,66 @@ def test_list_buckets(self): buckets = self.client.get_buckets() self.assertTrue("list_bucket" in buckets) + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_secondary_index_store(self): + # Create a new object with indexes... + bucket = self.client.bucket('indexbucket') + rand = self.randint() + obj = bucket.new('mykey1', rand) + obj.set_indexes({ + 'field1_bin': 'val1', + 'field2_int': 1001 + }) + obj.store() + + # Retrieve the object, check that the correct indexes exist... + obj = bucket.get('mykey1') + self.assertEqual('val1', obj.get_indexes()['field1_bin']) + self.assertEqual(1001, int(obj.get_indexes()['field2_int'])) + + # Clean up... + bucket.get('mykey1').delete() + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_secondary_index_query(self): + bucket = self.client.bucket('indexbucket') + bucket.new('mykey1', 'data1').set_indexes({'field1_bin':'val1', 'field2_int':1001}).store() + bucket.new('mykey2', 'data2').set_indexes({'field1_bin':'val2', 'field2_int':1002}).store() + bucket.new('mykey3', 'data3').set_indexes({'field1_bin':'val3', 'field2_int':1003}).store() + bucket.new('mykey4', 'data4').set_indexes({'field1_bin':'val4', 'field2_int':1004}).store() + + # Test an equality query... + results = self.client.index('indexbucket', 'field1_bin', 'val2').run() + self.assertEquals(1, len(results)) + self.assertEquals('mykey2', results[0].get_key()) + + # Test a range query... + results = self.client.index('indexbucket', 'field1_bin', 'val2', 'val4').run() + vals = set() + for i in results: + vals.add(i.get_key()) + self.assertEquals(3, len(results)) + self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) + + # Test an equality query... + results = self.client.index('indexbucket', 'field2_int', 1002).run() + self.assertEquals(1, len(results)) + self.assertEquals('mykey2', results[0].get_key()) + + # Test a range query... + results = self.client.index('indexbucket', 'field2_int', 1002, 1004).run() + vals = set() + for i in results: + vals.add(i.get_key()) + self.assertEquals(3, len(results)) + self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) + + # Clean up... + bucket.get('mykey1').delete() + bucket.get('mykey2').delete() + bucket.get('mykey3').delete() + bucket.get('mykey4').delete() + class MapReduceAliasTestMixIn(object): """This tests the map reduce aliases""" @@ -897,7 +958,6 @@ def test_delete_documents_from_search_by_query_and_id(self): results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results["response"]["docs"])) - class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): diff --git a/riak/transports/http.py b/riak/transports/http.py index 466cf229..2b2a7681 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -113,6 +113,9 @@ def put(self, robj, w = None, dw = None, return_body = True): for key, value in robj.get_usermeta().iteritems(): headers['X-Riak-Meta-%s' % key] = value + for key, value in robj.get_indexes().iteritems(): + headers['X-Riak-Index-%s' % key] = value + content = robj.get_encoded_data() return self.do_put(url, headers, content, return_body, key=robj.get_key()) @@ -261,7 +264,7 @@ def parse_body(self, response, expected_statuses): # Parse the headers... vclock = None - metadata = {MD_USERMETA: {}} + metadata = {MD_USERMETA: {}, MD_INDEX: {}} links = [] for header, value in headers.iteritems(): if header == 'content-type': @@ -278,6 +281,8 @@ def parse_body(self, response, expected_statuses): metadata[MD_LASTMOD] = value elif header.startswith('x-riak-meta-'): metadata[MD_USERMETA][header.replace('x-riak-meta-', '')] = value + elif header.startswith('x-riak-index-'): + metadata[MD_INDEX][header.replace('x-riak-index-', '')] = value elif header == 'x-riak-vclock': vclock = value if links: diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 07bf6f85..9c94d769 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -471,6 +471,11 @@ def decode_content(self, rpb_content): usermeta[usermd.key] = usermd.value if len(usermeta) > 0: metadata[MD_USERMETA] = usermeta + indexes = {} + for index in rpb_content.indexes: + indexes[index.key] = index.value + if len(indexes) > 0: + metadata[MD_INDEX] = indexes return metadata, rpb_content.value def pbify_content(self, metadata, data, rpb_content) : @@ -488,6 +493,11 @@ def pbify_content(self, metadata, data, rpb_content) : pair = rpb_content.usermeta.add() pair.key = uk pair.value = uv + elif k == MD_INDEX: + for uk, uv in v.iteritems(): + pair = rpb_content.indexes.add() + pair.key = uk + pair.value = str(uv) elif k == MD_LINKS: for link in v: pb_link = rpb_content.links.add() diff --git a/riak/transports/riakclient.proto b/riak/transports/riakclient.proto index 6acd0842..ac82cb10 100644 --- a/riak/transports/riakclient.proto +++ b/riak/transports/riakclient.proto @@ -120,12 +120,19 @@ message RpbGetReq { required bytes bucket = 1; required bytes key = 2; optional uint32 r = 3; + optional uint32 pr = 4; + optional bool basic_quorum = 5; + optional bool notfound_ok = 6; + optional bytes if_modified = 7; // fail if the supplied vclock does not match + optional bool head = 8; // return everything but the value + optional bool deletedvclock = 9; // return the tombstone's vclock, if applicable } // Get Response - if the record was not found there will be no content/vclock message RpbGetResp { repeated RpbContent content = 1; optional bytes vclock = 2; // the opaque vector clock for the object + optional bool unchanged = 3; } @@ -133,18 +140,23 @@ message RpbGetResp { // the key will be returned. message RpbPutReq { required bytes bucket = 1; - required bytes key = 2; + optional bytes key = 2; optional bytes vclock = 3; required RpbContent content = 4; optional uint32 w = 5; optional uint32 dw = 6; optional bool return_body = 7; + optional uint32 pw = 8; + optional bool if_not_modified = 9; + optional bool if_none_match = 10; + optional bool return_head = 11; } -// Put response - same as get response +// Put response - same as get response with optional key if one was generated message RpbPutResp { repeated RpbContent content = 1; optional bytes vclock = 2; // the opaque vector clock for the object + optional bytes key = 3; // the key generated, if any } @@ -153,6 +165,12 @@ message RpbDelReq { required bytes bucket = 1; required bytes key = 2; optional uint32 rw = 3; + optional bytes vclock = 4; + optional uint32 r = 5; + optional uint32 w = 6; + optional uint32 pr = 7; + optional uint32 pw = 8; + optional uint32 dw = 9; } // Delete response - not defined, will return a RpbDelResp on success or RpbErrorResp on failure @@ -224,6 +242,7 @@ message RpbContent { optional uint32 last_mod = 7; optional uint32 last_mod_usecs = 8; repeated RpbPair usermeta = 9; // user metadata stored with the object + repeated RpbPair indexes = 10; // user metadata stored with the object } // Key/value pair - used for user metadata diff --git a/riak/transports/riakclient_pb2.py b/riak/transports/riakclient_pb2.py index 68f9b2e0..2715abff 100644 --- a/riak/transports/riakclient_pb2.py +++ b/riak/transports/riakclient_pb2.py @@ -10,7 +10,7 @@ DESCRIPTOR = descriptor.FileDescriptor( name='riakclient.proto', package='', - serialized_pb='\n\x10riakclient.proto\"/\n\x0cRpbErrorResp\x12\x0e\n\x06\x65rrmsg\x18\x01 \x02(\x0c\x12\x0f\n\x07\x65rrcode\x18\x02 \x02(\r\"\'\n\x12RpbGetClientIdResp\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"&\n\x11RpbSetClientIdReq\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"<\n\x14RpbGetServerInfoResp\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12\x16\n\x0eserver_version\x18\x02 \x01(\x0c\"3\n\tRpbGetReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\t\n\x01r\x18\x03 \x01(\r\":\n\nRpbGetResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\"\x82\x01\n\tRpbPutReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\x0e\n\x06vclock\x18\x03 \x01(\x0c\x12\x1c\n\x07\x63ontent\x18\x04 \x02(\x0b\x32\x0b.RpbContent\x12\t\n\x01w\x18\x05 \x01(\r\x12\n\n\x02\x64w\x18\x06 \x01(\r\x12\x13\n\x0breturn_body\x18\x07 \x01(\x08\":\n\nRpbPutResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\"4\n\tRpbDelReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\n\n\x02rw\x18\x03 \x01(\r\"%\n\x12RpbListBucketsResp\x12\x0f\n\x07\x62uckets\x18\x01 \x03(\x0c\" \n\x0eRpbListKeysReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"-\n\x0fRpbListKeysResp\x12\x0c\n\x04keys\x18\x01 \x03(\x0c\x12\x0c\n\x04\x64one\x18\x02 \x01(\x08\"!\n\x0fRpbGetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"2\n\x10RpbGetBucketResp\x12\x1e\n\x05props\x18\x01 \x02(\x0b\x32\x0f.RpbBucketProps\"A\n\x0fRpbSetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x1e\n\x05props\x18\x02 \x02(\x0b\x32\x0f.RpbBucketProps\"5\n\x0cRpbMapRedReq\x12\x0f\n\x07request\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x02(\x0c\">\n\rRpbMapRedResp\x12\r\n\x05phase\x18\x01 \x01(\r\x12\x10\n\x08response\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\"\xc9\x01\n\nRpbContent\x12\r\n\x05value\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x01(\x0c\x12\x0f\n\x07\x63harset\x18\x03 \x01(\x0c\x12\x18\n\x10\x63ontent_encoding\x18\x04 \x01(\x0c\x12\x0c\n\x04vtag\x18\x05 \x01(\x0c\x12\x17\n\x05links\x18\x06 \x03(\x0b\x32\x08.RpbLink\x12\x10\n\x08last_mod\x18\x07 \x01(\r\x12\x16\n\x0elast_mod_usecs\x18\x08 \x01(\r\x12\x1a\n\x08usermeta\x18\t \x03(\x0b\x32\x08.RpbPair\"%\n\x07RpbPair\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c\"3\n\x07RpbLink\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0b\n\x03tag\x18\x03 \x01(\x0c\"3\n\x0eRpbBucketProps\x12\r\n\x05n_val\x18\x01 \x01(\r\x12\x12\n\nallow_mult\x18\x02 \x01(\x08') + serialized_pb='\n\x10riakclient.proto\"/\n\x0cRpbErrorResp\x12\x0e\n\x06\x65rrmsg\x18\x01 \x02(\x0c\x12\x0f\n\x07\x65rrcode\x18\x02 \x02(\r\"\'\n\x12RpbGetClientIdResp\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"&\n\x11RpbSetClientIdReq\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"<\n\x14RpbGetServerInfoResp\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12\x16\n\x0eserver_version\x18\x02 \x01(\x0c\"\xa4\x01\n\tRpbGetReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\t\n\x01r\x18\x03 \x01(\r\x12\n\n\x02pr\x18\x04 \x01(\r\x12\x14\n\x0c\x62\x61sic_quorum\x18\x05 \x01(\x08\x12\x13\n\x0bnotfound_ok\x18\x06 \x01(\x08\x12\x13\n\x0bif_modified\x18\x07 \x01(\x0c\x12\x0c\n\x04head\x18\x08 \x01(\x08\x12\x15\n\rdeletedvclock\x18\t \x01(\x08\"M\n\nRpbGetResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\x12\x11\n\tunchanged\x18\x03 \x01(\x08\"\xd3\x01\n\tRpbPutReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0e\n\x06vclock\x18\x03 \x01(\x0c\x12\x1c\n\x07\x63ontent\x18\x04 \x02(\x0b\x32\x0b.RpbContent\x12\t\n\x01w\x18\x05 \x01(\r\x12\n\n\x02\x64w\x18\x06 \x01(\r\x12\x13\n\x0breturn_body\x18\x07 \x01(\x08\x12\n\n\x02pw\x18\x08 \x01(\r\x12\x17\n\x0fif_not_modified\x18\t \x01(\x08\x12\x15\n\rif_none_match\x18\n \x01(\x08\x12\x13\n\x0breturn_head\x18\x0b \x01(\x08\"G\n\nRpbPutResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\x12\x0b\n\x03key\x18\x03 \x01(\x0c\"~\n\tRpbDelReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\n\n\x02rw\x18\x03 \x01(\r\x12\x0e\n\x06vclock\x18\x04 \x01(\x0c\x12\t\n\x01r\x18\x05 \x01(\r\x12\t\n\x01w\x18\x06 \x01(\r\x12\n\n\x02pr\x18\x07 \x01(\r\x12\n\n\x02pw\x18\x08 \x01(\r\x12\n\n\x02\x64w\x18\t \x01(\r\"%\n\x12RpbListBucketsResp\x12\x0f\n\x07\x62uckets\x18\x01 \x03(\x0c\" \n\x0eRpbListKeysReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"-\n\x0fRpbListKeysResp\x12\x0c\n\x04keys\x18\x01 \x03(\x0c\x12\x0c\n\x04\x64one\x18\x02 \x01(\x08\"!\n\x0fRpbGetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"2\n\x10RpbGetBucketResp\x12\x1e\n\x05props\x18\x01 \x02(\x0b\x32\x0f.RpbBucketProps\"A\n\x0fRpbSetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x1e\n\x05props\x18\x02 \x02(\x0b\x32\x0f.RpbBucketProps\"5\n\x0cRpbMapRedReq\x12\x0f\n\x07request\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x02(\x0c\">\n\rRpbMapRedResp\x12\r\n\x05phase\x18\x01 \x01(\r\x12\x10\n\x08response\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\"\xe4\x01\n\nRpbContent\x12\r\n\x05value\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x01(\x0c\x12\x0f\n\x07\x63harset\x18\x03 \x01(\x0c\x12\x18\n\x10\x63ontent_encoding\x18\x04 \x01(\x0c\x12\x0c\n\x04vtag\x18\x05 \x01(\x0c\x12\x17\n\x05links\x18\x06 \x03(\x0b\x32\x08.RpbLink\x12\x10\n\x08last_mod\x18\x07 \x01(\r\x12\x16\n\x0elast_mod_usecs\x18\x08 \x01(\r\x12\x1a\n\x08usermeta\x18\t \x03(\x0b\x32\x08.RpbPair\x12\x19\n\x07indexes\x18\n \x03(\x0b\x32\x08.RpbPair\"%\n\x07RpbPair\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c\"3\n\x07RpbLink\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0b\n\x03tag\x18\x03 \x01(\x0c\"3\n\x0eRpbBucketProps\x12\r\n\x05n_val\x18\x01 \x01(\r\x12\x12\n\nallow_mult\x18\x02 \x01(\x08') @@ -169,6 +169,48 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='pr', full_name='RpbGetReq.pr', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='basic_quorum', full_name='RpbGetReq.basic_quorum', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='notfound_ok', full_name='RpbGetReq.notfound_ok', index=5, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='if_modified', full_name='RpbGetReq.if_modified', index=6, + number=7, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='head', full_name='RpbGetReq.head', index=7, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='deletedvclock', full_name='RpbGetReq.deletedvclock', index=8, + number=9, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -178,8 +220,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=212, - serialized_end=263, + serialized_start=213, + serialized_end=377, ) @@ -204,6 +246,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='unchanged', full_name='RpbGetResp.unchanged', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -213,8 +262,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=265, - serialized_end=323, + serialized_start=379, + serialized_end=456, ) @@ -234,7 +283,7 @@ options=None), descriptor.FieldDescriptor( name='key', full_name='RpbPutReq.key', index=1, - number=2, type=12, cpp_type=9, label=2, + number=2, type=12, cpp_type=9, label=1, has_default_value=False, default_value="", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -274,6 +323,34 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='pw', full_name='RpbPutReq.pw', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='if_not_modified', full_name='RpbPutReq.if_not_modified', index=8, + number=9, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='if_none_match', full_name='RpbPutReq.if_none_match', index=9, + number=10, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='return_head', full_name='RpbPutReq.return_head', index=10, + number=11, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -283,8 +360,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=326, - serialized_end=456, + serialized_start=459, + serialized_end=670, ) @@ -309,6 +386,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='key', full_name='RpbPutResp.key', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -318,8 +402,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=458, - serialized_end=516, + serialized_start=672, + serialized_end=743, ) @@ -351,6 +435,48 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='vclock', full_name='RpbDelReq.vclock', index=3, + number=4, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='r', full_name='RpbDelReq.r', index=4, + number=5, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='w', full_name='RpbDelReq.w', index=5, + number=6, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='pr', full_name='RpbDelReq.pr', index=6, + number=7, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='pw', full_name='RpbDelReq.pw', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='dw', full_name='RpbDelReq.dw', index=8, + number=9, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -360,8 +486,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=518, - serialized_end=570, + serialized_start=745, + serialized_end=871, ) @@ -388,8 +514,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=572, - serialized_end=609, + serialized_start=873, + serialized_end=910, ) @@ -416,8 +542,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=611, - serialized_end=643, + serialized_start=912, + serialized_end=944, ) @@ -451,8 +577,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=645, - serialized_end=690, + serialized_start=946, + serialized_end=991, ) @@ -479,8 +605,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=692, - serialized_end=725, + serialized_start=993, + serialized_end=1026, ) @@ -507,8 +633,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=727, - serialized_end=777, + serialized_start=1028, + serialized_end=1078, ) @@ -542,8 +668,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=779, - serialized_end=844, + serialized_start=1080, + serialized_end=1145, ) @@ -577,8 +703,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=846, - serialized_end=899, + serialized_start=1147, + serialized_end=1200, ) @@ -619,8 +745,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=901, - serialized_end=963, + serialized_start=1202, + serialized_end=1264, ) @@ -694,6 +820,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='indexes', full_name='RpbContent.indexes', index=9, + number=10, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -703,8 +836,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=966, - serialized_end=1167, + serialized_start=1267, + serialized_end=1495, ) @@ -738,8 +871,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=1169, - serialized_end=1206, + serialized_start=1497, + serialized_end=1534, ) @@ -780,8 +913,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=1208, - serialized_end=1259, + serialized_start=1536, + serialized_end=1587, ) @@ -815,8 +948,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=1261, - serialized_end=1312, + serialized_start=1589, + serialized_end=1640, ) @@ -827,6 +960,7 @@ _RPBSETBUCKETREQ.fields_by_name['props'].message_type = _RPBBUCKETPROPS _RPBCONTENT.fields_by_name['links'].message_type = _RPBLINK _RPBCONTENT.fields_by_name['usermeta'].message_type = _RPBPAIR +_RPBCONTENT.fields_by_name['indexes'].message_type = _RPBPAIR class RpbErrorResp(message.Message): __metaclass__ = reflection.GeneratedProtocolMessageType From 4a18c00646604fe793e2db9977ee43339e9147e1 Mon Sep 17 00:00:00 2001 From: Rusty Klophaus Date: Thu, 15 Sep 2011 09:21:28 -0400 Subject: [PATCH 031/118] Detect and return correct error message when MapReduce job fails. Also, correctly handle when there are no MapReduce results. AZ679 --- riak/mapreduce.py | 4 ++++ riak/transports/http.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index c7181d15..d2b956d1 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -234,6 +234,10 @@ def run(self, timeout=None): if not link_results_flag: return result + # If there are no results, then return an empty list. + if result == None: + return [] + # Otherwise, if the last phase IS a link phase, then convert the # results to RiakLink objects. a = [] diff --git a/riak/transports/http.py b/riak/transports/http.py index 2b2a7681..7c06d8c8 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -213,6 +213,12 @@ def mapred(self, inputs, query, timeout=None): # Do the request... url = "/" + self._mapred_prefix response = self.http_request('POST', url, {}, content) + + # Make sure the expected status code came back... + status = response[0]['http_code'] + if status != 200: + raise Exception('Error running MapReduce operation. Status: ' + str(status) + ' : ' + response[1]) + result = json.loads(response[1]) return result From 85e9d5460787d2ad6b66663e07b106f8cd71e05d Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Thu, 15 Sep 2011 16:40:57 -0400 Subject: [PATCH 032/118] Add a utility function for marking APIs/usage as deprecated. --- riak/util.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/riak/util.py b/riak/util.py index c2e154a0..0a0cf4cc 100644 --- a/riak/util.py +++ b/riak/util.py @@ -1,4 +1,5 @@ import collections +import warnings def quacks_like_dict(object): """Check if object is dict-like""" @@ -31,3 +32,6 @@ def deep_merge(a, b): current_dst[key] = current_src[key] return dst + +def deprecated(message, stacklevel=3): + warnings.warn(message, DeprecationWarning, stacklevel=stacklevel) From 9099cf72d196760dfc2a8c86192fed5d619f0487 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Thu, 15 Sep 2011 17:50:58 -0400 Subject: [PATCH 033/118] Add an API level to the transports, with deprecation warnings. * riak/transports/transport.py: (RiakTransport): document the 'api' classvar, but not enter a default. all subclasses must provide the default explicitly. * riak/transports/http.py: (RiakHttpTransport): notate the class is now API v2. * riak/transports/pbc.py: (RiakPbcTransport): notate the class is now API v2. * riak/client.py: (RiakClient.__init__): handle api < 2 transports * riak/search.py: (RiakSearch.__init__): if api < 2, then raise a deprecation warning. the old code would simply fail, so we are "allowed" to bail out. --- riak/client.py | 31 +++++++++++++++++++------------ riak/search.py | 14 +++++++++++--- riak/transports/http.py | 3 +++ riak/transports/pbc.py | 4 ++++ riak/transports/transport.py | 6 ++++++ 5 files changed, 43 insertions(+), 15 deletions(-) diff --git a/riak/client.py b/riak/client.py index ced72420..9b35b039 100644 --- a/riak/client.py +++ b/riak/client.py @@ -27,6 +27,7 @@ from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduce from riak.search import RiakSearch +from riak.util import deprecated import riak.transports.connection @@ -58,18 +59,24 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', if transport_class is None: transport_class = RiakHttpTransport - hostports = [ (host, port), ] - self._cm = transport_class.default_cm(hostports) - - ### we need to allow additional transport options. make this an - ### argument to __init__ ? - transport_options = { } - - self._transport = transport_class(self._cm, - prefix=prefix, - mapred_prefix=mapred_prefix, - client_id=client_id, - **transport_options) + api = getattr(transport_class, 'api', 1) + if api >= 2: + hostports = [ (host, port), ] + self._cm = transport_class.default_cm(hostports) + + ### we need to allow additional transport options. make this an + ### argument to __init__ ? + transport_options = { } + + self._transport = transport_class(self._cm, + prefix=prefix, + mapred_prefix=mapred_prefix, + client_id=client_id, + **transport_options) + else: + deprecated('please upgrade the transport to the new API') + self._cm = None + self._transport = transport_class(host, port, client_id=client_id) self._r = "default" self._w = "default" diff --git a/riak/search.py b/riak/search.py index 5e006c25..7fe2ab73 100644 --- a/riak/search.py +++ b/riak/search.py @@ -8,9 +8,17 @@ def __init__(self, client, transport_class=None, if transport_class is None: transport_class = RiakHttpTransport - hostports = [ (host, port), ] - self._cm = transport_class.default_cm(hostports) - self._transport = transport_class(self._cm, prefix="/solr") + api = getattr(transport_class, 'api', 1) + if api >= 2: + hostports = [ (host, port), ] + self._cm = transport_class.default_cm(hostports) + self._transport = transport_class(self._cm, prefix="/solr") + else: + # The old code which attempted to use api==1 would actually + # throw a NameError, so it was obviously never used. We will + # simply raise an error here, intead of a gentle warning. + raise DeprecationWarning('please upgrade the transport to the ' + 'new API') self._client = client self._decoders = {"text/xml": ElementTree.fromstring} diff --git a/riak/transports/http.py b/riak/transports/http.py index 30ca5d58..cf286a20 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -43,6 +43,9 @@ class RiakHttpTransport(RiakTransport) : connection, and the RiakClient object is extremely lightweight. """ + # We're using the new RiakTransport API + api = 2 + # The ConnectionManager class that this transport prefers. default_cm = HTTPConnectionManager diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 08d08a39..c28c2041 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -77,6 +77,10 @@ class RiakPbcTransport(RiakTransport): The RiakPbcTransport object holds a connection to the protocol buffers interface on the riak server. """ + + # We're using the new RiakTransport API + api = 2 + rw_names = { 'default' : RIAKC_RW_DEFAULT, 'all' : RIAKC_RW_ALL, diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 7d0be45f..5fb57be6 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -29,6 +29,12 @@ class RiakTransport(object): Class to encapsulate transport details """ + # Subclasses should specify their API level. + # * missing or 1: the API used up and through 1.3.x. + # * 2: the API introduced with 1.4.x + # + # api = 2 + @classmethod def make_random_client_id(self): """ From 2cb424989f6d3fac62affdc13cc9da1e7544514e Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Thu, 15 Sep 2011 21:44:54 -0400 Subject: [PATCH 034/118] First draft at fixing issue #53. Introduce transport.put_new() to deal with returning the key generated by the server (the signature for .put() has no place to return it). * riak/riak_object.py: (RiakObject.store): if the object does not (yet) have a key, then use the .put_new() method to save the object and get the new key. * riak/transports/transport.py: (RiakTransport.put_new): new abstract method for storing an object and allowing the server to generate the key * riak/transports/http.py: (RiakHttpTransport.put): factor the header creation out into ... (RiakHttpTransport.build_put_headers): ... this. (RiakHttpTransport.put_new): new method to store the new object and return the server-generated key * riak/transport/pbc.py: (RiakHttpTransport.put_new): add a draft stub for implementing put_new and returning the server-generated key (doesn't work). --- riak/riak_object.py | 16 +++++++--- riak/transports/http.py | 59 ++++++++++++++++++++++++------------ riak/transports/pbc.py | 10 ++++++ riak/transports/transport.py | 10 ++++++ 4 files changed, 72 insertions(+), 23 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index fe20dbdd..6bee76c5 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -309,11 +309,19 @@ def store(self, w=None, dw=None, return_body=True): w = self._bucket.get_w(w) dw = self._bucket.get_dw(w) - # Issue the get over our transport + # Issue the put over our transport t = self._client.get_transport() - Result = t.put(self, w, dw, return_body) - if Result is not None: - self.populate(Result) + + if self._key is None: + key, vclock, metadata = t.put_new(self, w, dw, return_body) + self._exists = True + self._key = key + self._vclock = vclock + self.set_metadata(metadata) + else: + Result = t.put(self, w, dw, return_body) + if Result is not None: + self.populate(Result) return self diff --git a/riak/transports/http.py b/riak/transports/http.py index 27a41aac..a28209c3 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -109,25 +109,7 @@ def put(self, robj, w = None, dw = None, return_body = True): params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw} url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), params=params) - - # Construct the headers... - headers = MultiDict({'Accept' : 'text/plain, */*; q=0.5', - 'Content-Type' : robj.get_content_type(), - 'X-Riak-ClientId' : self._client_id}) - - # Add the vclock if it exists... - if robj.vclock() is not None: - headers['X-Riak-Vclock'] = robj.vclock() - - # Create the header from metadata - links = self.add_links_for_riak_object(robj, headers) - - for key, value in robj.get_usermeta().iteritems(): - headers['X-Riak-Meta-%s' % key] = value - - for key, value in robj.get_indexes().iteritems(): - headers['X-Riak-Index-%s' % key] = value - + headers = self.build_put_headers(robj) content = robj.get_encoded_data() return self.do_put(url, headers, content, return_body, key=robj.get_key()) @@ -143,6 +125,22 @@ def do_put(self, url, headers, content, return_body=False, key=None): self.check_http_code(response, [204]) return None + def put_new(self, robj, w=None, dw=None, return_meta=True): + """Put a new object into the Riak store, returning its (new) key.""" + # Construct the URL... + params = {'returnbody' : str(return_meta).lower(), 'w' : w, 'dw' : dw} + url = self.build_rest_path(bucket=robj.get_bucket(), params=params) + headers = self.build_put_headers(robj) + content = robj.get_encoded_data() + response = self.http_request('POST', url, headers, content) + key = response[0]['location'] + if return_meta: + vclock, [(metadata, data)] = self.parse_body(response, [201]) + return key, vclock, metadata + else: + self.check_http_code(response, [201]) + return key, None, None + def delete(self, robj, rw): # Construct the URL... params = {'rw' : rw} @@ -410,6 +408,29 @@ def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : # Return. return path + def build_put_headers(self, robj): + """Build the headers for a POST/PUT request.""" + + # Construct the headers... + headers = MultiDict({'Accept' : 'text/plain, */*; q=0.5', + 'Content-Type' : robj.get_content_type(), + 'X-Riak-ClientId' : self._client_id}) + + # Add the vclock if it exists... + if robj.vclock() is not None: + headers['X-Riak-Vclock'] = robj.vclock() + + # Create the header from metadata + links = self.add_links_for_riak_object(robj, headers) + + for key, value in robj.get_usermeta().iteritems(): + headers['X-Riak-Meta-%s' % key] = value + + for key, value in robj.get_indexes().iteritems(): + headers['X-Riak-Index-%s' % key] = value + + return headers + def http_request(self, method, uri, headers=None, body='') : """ Given a Method, URL, Headers, and Body, perform and HTTP request, diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 0b0ffa1c..b1c3b97f 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -216,6 +216,16 @@ def put(self, robj, w = None, dw = None, return_body = True): contents.append(self.decode_content(c)) return resp.vclock, contents + def put_new(self, robj, w=None, dw=None, return_meta=True): + ### not sure about all this. just use put() for now. we need the + ### resp.key value from self.put(). maybe refactor. + response = self.put(robj, w, dw, return_meta) + if response is None: + return None, None, None + assert len(response[1]) == 1 + return None, response[0], response[1][0][0] + + def delete(self, robj, rw = None): """ Serialize get request and deserialize response diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 5fb57be6..4454f48d 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -75,6 +75,16 @@ def put(self, robj, w = None, dw = None, return_body = True): """ raise RiakError("not implemented") + def put_new(self, robj, w=None, dw=None, return_meta=True): + """Put a new object into the Riak store, returning its (new) key. + + If return_meta is False, then the vlock and metadata return values + will be None. + + @return (key, vclock, metadata) + """ + raise RiakError("not implemented") + def delete(self, robj, rw = None): """ Serialize delete request and deserialize response From b264f823d804101cdd32bb403932546ebcaac64f Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 16 Sep 2011 11:40:49 -0400 Subject: [PATCH 035/118] Test the retrieval of a new object's key, and its format. --- riak/tests/test_all.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6825d24e..ac94fd67 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -828,7 +828,11 @@ def test_generate_key(self): bucket = self.client.bucket('random_key_bucket') for key in bucket.get_keys(): bucket.get(str(key)).delete() - bucket.new(None, data={}).store() + o = bucket.new(None, data={}) + self.assertIsNone(o.get_key()) + o.store() + self.assertIsNotNone(o.get_key()) + self.assertNotIn('/', o.get_key()) self.assertEqual(len(bucket.get_keys()), 1) def test_too_many_link_headers_shouldnt_break_http(self): From 6cfd296a86c1b475101c179a45a7453b76dcbfd5 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 12 Sep 2011 21:08:52 -0400 Subject: [PATCH 036/118] Adjust for compatibility with Python 2.5 --- riak/util.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/riak/util.py b/riak/util.py index c2e154a0..39f1a210 100644 --- a/riak/util.py +++ b/riak/util.py @@ -1,8 +1,12 @@ -import collections +try: + from collections import Mapping +except ImportError: + # compatibility with Python 2.5 + Mapping = dict def quacks_like_dict(object): """Check if object is dict-like""" - return isinstance(object, collections.Mapping) + return isinstance(object, Mapping) def deep_merge(a, b): """Merge two deep dicts non-destructively From 225335a68f79a6e89118d16346d1c4e7ebcb8a63 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 16 Sep 2011 11:43:02 -0400 Subject: [PATCH 037/118] Remove any leftover objects from a failed test_store_of_missing_object. --- riak/tests/test_all.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 50914fa3..45a76bfc 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -92,6 +92,13 @@ def create_client(self, host=None, port=None, transport_class=None): def setUp(self): self.client = self.create_client() + # make sure these are not left over from a previous, failed run + bucket = self.client.bucket('bucket') + o = bucket.get('nonexistent_key_json') + o.delete() + o = bucket.get('nonexistent_key_binary') + o.delete() + def test_is_alive(self): self.assertTrue(self.client.is_alive()) From 127492c44ce63db9e21f3e136c45479c588ea789 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 16 Sep 2011 12:09:16 -0400 Subject: [PATCH 038/118] Fix extraction of the key from the Location: header. --- riak/transports/http.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index a28209c3..9e61d545 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -133,7 +133,9 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): headers = self.build_put_headers(robj) content = robj.get_encoded_data() response = self.http_request('POST', url, headers, content) - key = response[0]['location'] + location = response[0]['location'] + idx = location.rindex('/') + key = location[idx+1:] if return_meta: vclock, [(metadata, data)] = self.parse_body(response, [201]) return key, vclock, metadata From db03c077469d0d6f4f7fb80153806d56a72837bb Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 16 Sep 2011 12:23:03 -0400 Subject: [PATCH 039/118] Add RiakPbcCachedTransport.put_new() and some missing docstrings. --- riak/transports/pbc.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index b1c3b97f..cf7f0cdb 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -217,6 +217,13 @@ def put(self, robj, w = None, dw = None, return_body = True): return resp.vclock, contents def put_new(self, robj, w=None, dw=None, return_meta=True): + """Put a new object into the Riak store, returning its (new) key. + + If return_meta is False, then the vlock and metadata return values + will be None. + + @return (key, vclock, metadata) + """ ### not sure about all this. just use put() for now. we need the ### resp.key value from self.put(). maybe refactor. response = self.put(robj, w, dw, return_meta) @@ -604,6 +611,17 @@ def put(self, robj, w = None, dw = None, return_body = True): with self._get_connection_from_pool() as connection: return connection.put(robj, w, dw, return_body) + def put_new(self, robj, w=None, dw=None, return_meta=True): + """Put a new object into the Riak store, returning its (new) key. + + If return_meta is False, then the vlock and metadata return values + will be None. + + @return (key, vclock, metadata) + """ + with self._get_connection_from_pool() as connection: + return connection.put_new(robj, w, dw, return_meta) + def delete(self, robj, rw = None): """ Serialize delete request and deserialize response From 3f28c3087a867cc187dece98521d252995f54920 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 16 Sep 2011 12:30:03 -0400 Subject: [PATCH 040/118] Properly implement RiakPbcTransport.put_new() --- riak/transports/pbc.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index cf7f0cdb..ccd3d8d7 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -224,14 +224,30 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): @return (key, vclock, metadata) """ - ### not sure about all this. just use put() for now. we need the - ### resp.key value from self.put(). maybe refactor. - response = self.put(robj, w, dw, return_meta) - if response is None: - return None, None, None - assert len(response[1]) == 1 - return None, response[0], response[1][0][0] - + bucket = robj.get_bucket() + + req = riakclient_pb2.RpbPutReq() + req.w = self.translate_rw_val(w) + req.dw = self.translate_rw_val(dw) + if return_meta: + req.return_body = 1 + + req.bucket = bucket.get_name() + + self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) + + self.maybe_connect() + self.send_msg(MSG_CODE_PUT_REQ, req) + msg_code, resp = self.recv_msg() + if msg_code != MSG_CODE_PUT_RESP: + raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + if not resp: + raise RiakError("missing response object") + if len(resp.content) != 1: + raise RiakError("siblings were returned from object creation") + + metadata, content = self.decode_content(resp.content[0]) + return resp.key, resp.vclock, metadata def delete(self, robj, rw = None): """ From 75d8853289ad22749f035206497028a6e4616383 Mon Sep 17 00:00:00 2001 From: David Koblas Date: Fri, 16 Sep 2011 11:58:30 -0700 Subject: [PATCH 041/118] Fix the typo where w was used rather than dw as the default value for dw --- riak/riak_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index fe20dbdd..26d342af 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -307,7 +307,7 @@ def store(self, w=None, dw=None, return_body=True): """ # Use defaults if not specified... w = self._bucket.get_w(w) - dw = self._bucket.get_dw(w) + dw = self._bucket.get_dw(dw) # Issue the get over our transport t = self._client.get_transport() From aacedcc9580c4a0c5c8d84aea141310b30a3ad3b Mon Sep 17 00:00:00 2001 From: Jeffrey Massung Date: Mon, 26 Sep 2011 15:26:55 -0600 Subject: [PATCH 042/118] Perform a dummy 2i query to see if an exception detailing indexes_not_supported and, if so, don't perform the request of the test. --- riak/tests/test_all.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6825d24e..af397c3d 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -546,6 +546,13 @@ def test_secondary_index_query(self): bucket.new('mykey3', 'data3').set_indexes({'field1_bin':'val3', 'field2_int':1003}).store() bucket.new('mykey4', 'data4').set_indexes({'field1_bin':'val4', 'field2_int':1004}).store() + # Immediate test to see if 2i is even supported w/ the backend + try: + self.client.index('foo','bar_bin','baz').run() + except Exception as e: + if "indexes_not_supported" in str(e): + return True + # Test an equality query... results = self.client.index('indexbucket', 'field1_bin', 'val2').run() self.assertEquals(1, len(results)) From f62b7ef46fb45a58328a29e867fa5af2a908a169 Mon Sep 17 00:00:00 2001 From: Rusty Klophaus Date: Fri, 30 Sep 2011 10:12:00 -0400 Subject: [PATCH 043/118] Update 2i support to handle multi-valued index entries. AZ789 --- docs/tutorial.rst | 18 +++++--- riak/riak_index_entry.py | 62 ++++++++++++++++++++++++++++ riak/riak_object.py | 53 ++++++++++++++++++------ riak/tests/test_all.py | 89 +++++++++++++++++++++++++++++++++++----- riak/transports/http.py | 21 +++++++--- riak/transports/pbc.py | 12 +++--- 6 files changed, 216 insertions(+), 39 deletions(-) create mode 100644 riak/riak_index_entry.py diff --git a/docs/tutorial.rst b/docs/tutorial.rst index f7a3cefa..ab780da7 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -395,19 +395,20 @@ Secondary Indexes is a new feature available as of Riak 1.0. It allows you to tag an object with index metadata, and then later find the object by querying the metadata, returning a list of matching keys. -Usage of this feature looks like:: +Your Riak cluster must have Secondary Indexes enabled. See the Riak +documentation for details. + +Usage of this feature looks like: import riak client = riak.RiakClient() bucket = client.bucket('mybucket') - # Store the object... + # Create and store the object with indexes... obj = bucket.new('mykey1', 'mydata') - obj.set_indexes({ - 'field1_bin': 'val1', - 'field2_int': 1001 - }) + obj.add_index('field1_bin', 'val1') + obj.add_index('field2_int', 1001) obj.store() # Query the indexes. The return value is a list of ``RiakLink`` objects. @@ -415,3 +416,8 @@ Usage of this feature looks like:: # Query the indexes using a range... results = client.index('mybucket', 'field1_bin', 'val1', 'val5').run() + + # Remove an index entry... + obj = bucket.get('mykey1') + obj.remove_index('field1_bin', 'val1') + obj.store() diff --git a/riak/riak_index_entry.py b/riak/riak_index_entry.py new file mode 100644 index 00000000..0a26bb13 --- /dev/null +++ b/riak/riak_index_entry.py @@ -0,0 +1,62 @@ +""" +Copyright 2010 Rusty Klophaus +Copyright 2010 Justin Sheehy +Copyright 2009 Jay Baird + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +class RiakIndexEntry: + def __init__(self, field, value): + self._field = field + self._value = str(value) + + def get_field(self): + return self._field + + def get_value(self): + return self._value + + def __str__(self): + return "RiakIndexEntry(field = '%s', value='%s')" % (self._field, self._value) + + def __eq__(self, other): + if not isinstance(other, RiakIndexEntry): + return False + + return \ + self.get_field() == other.get_field() and \ + self.get_value() == other.get_value() + + def __cmp__(self, other): + if other == None: + raise TypeError("RiakIndexEntry cannot be compared to None") + + if not isinstance(other, RiakIndexEntry): + raise TypeError("RiakIndexEntry cannot be compared to %s" % other.__class__.__name__) + + if self.get_field() < other.get_field(): + return -1 + + if self.get_field() > other.get_field(): + return 1 + + if self.get_value() < other.get_value(): + return -1 + + if self.get_value() > other.get_value(): + return 1 + + return 0 diff --git a/riak/riak_object.py b/riak/riak_object.py index 26d342af..7ad05032 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -20,6 +20,7 @@ import types, copy, re from metadata import * from riak import RiakError +from riak.riak_index_entry import RiakIndexEntry class RiakObject(object): """ @@ -47,7 +48,7 @@ def __init__(self, client, bucket, key=None): self._encode_data = True self._vclock = None self._data = None - self._metadata = {MD_USERMETA: {}, MD_INDEX: {}} + self._metadata = {MD_USERMETA: {}, MD_INDEX: []} self._links = [] self._siblings = [] self._exists = False @@ -177,24 +178,50 @@ def set_usermeta(self, usermeta): self._metadata[MD_USERMETA] = usermeta return self - def get_indexes(self): - if MD_INDEX in self._metadata: - return self._metadata[MD_INDEX] - else: - return {} + def add_index(self, field, value): + """ + Tag this object with the specified field/value pair for indexing. - def set_indexes(self, indexes): + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: self """ - Sets the field/value indexes under which this object will be - indexed. + rie = RiakIndexEntry(field, value) + if not rie in self._metadata[MD_INDEX]: + self._metadata[MD_INDEX].append(rie) - :param indexes: The field/value index data. - :type indexes: dict - :rtype: data + return self + + def remove_index(self, field, value): + """ + Remove the specified field/value pair as an index on this object. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: self """ - self._metadata[MD_INDEX] = indexes + rie = RiakIndexEntry(field, value) + if rie in self._metadata[MD_INDEX]: + self._metadata[MD_INDEX].remove(rie) return self + def get_indexes(self, field = None): + """ + Get a list of the index entries for this object. If a field is provided, returns a list + + :param field: The index field. + :type field: string or None + :rtype: (array of RiakIndexEntry) or (array of string or integer) + """ + if field == None: + return self._metadata[MD_INDEX] + else: + return [x.get_value() for x in self._metadata[MD_INDEX] if x.get_field() == field] + def exists(self): """ Return True if the object exists, False otherwise. Allows you to diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index af397c3d..ba0d9ccb 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -22,6 +22,7 @@ from riak import RiakPbcTransport, RiakPbcCachedTransport from riak import RiakHttpTransport, RiakHttpPoolTransport, RiakHttpReuseTransport from riak import RiakKeyFilter, key_filter +from riak.riak_index_entry import RiakIndexEntry from riak.mapreduce import RiakLink from riak.test_server import TestServer @@ -524,16 +525,67 @@ def test_secondary_index_store(self): bucket = self.client.bucket('indexbucket') rand = self.randint() obj = bucket.new('mykey1', rand) - obj.set_indexes({ - 'field1_bin': 'val1', - 'field2_int': 1001 - }) + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_int', 1011) obj.store() # Retrieve the object, check that the correct indexes exist... obj = bucket.get('mykey1') - self.assertEqual('val1', obj.get_indexes()['field1_bin']) - self.assertEqual(1001, int(obj.get_indexes()['field2_int'])) + self.assertEqual(['val1a'], sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1011'], sorted(obj.get_indexes('field1_int'))) + + # Add more indexes and save... + obj.add_index('field1_bin', 'val1b') + obj.add_index('field1_int', 1012) + obj.store() + + # Retrieve the object, check that the correct indexes exist... + obj = bucket.get('mykey1') + self.assertEqual(['val1a', 'val1b'], sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1011', '1012'], sorted(obj.get_indexes('field1_int'))) + + # Check the get_indexes() function... + self.assertEqual([ + RiakIndexEntry('field1_bin', 'val1a'), + RiakIndexEntry('field1_bin', 'val1b'), + RiakIndexEntry('field1_int', 1011), + RiakIndexEntry('field1_int', 1012) + ], sorted(obj.get_indexes())) + + # Delete an index... + obj.remove_index('field1_bin', 'val1a') + obj.remove_index('field1_int', 1011) + obj.store() + + # Retrieve the object, check that the correct indexes exist... + obj = bucket.get('mykey1') + self.assertEqual(['val1b'], sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1012'], sorted(obj.get_indexes('field1_int'))) + + # Check duplicate entries... + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_int', 1011) + obj.add_index('field1_int', 1011) + obj.add_index('field1_int', 1011) + + self.assertEqual([ + RiakIndexEntry('field1_bin', 'val1a'), + RiakIndexEntry('field1_bin', 'val1b'), + RiakIndexEntry('field1_int', 1011), + RiakIndexEntry('field1_int', 1012) + ], sorted(obj.get_indexes())) + + obj.store() + obj = bucket.get('mykey1') + + self.assertEqual([ + RiakIndexEntry('field1_bin', 'val1a'), + RiakIndexEntry('field1_bin', 'val1b'), + RiakIndexEntry('field1_int', 1011), + RiakIndexEntry('field1_int', 1012) + ], sorted(obj.get_indexes())) # Clean up... bucket.get('mykey1').delete() @@ -541,10 +593,27 @@ def test_secondary_index_store(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_query(self): bucket = self.client.bucket('indexbucket') - bucket.new('mykey1', 'data1').set_indexes({'field1_bin':'val1', 'field2_int':1001}).store() - bucket.new('mykey2', 'data2').set_indexes({'field1_bin':'val2', 'field2_int':1002}).store() - bucket.new('mykey3', 'data3').set_indexes({'field1_bin':'val3', 'field2_int':1003}).store() - bucket.new('mykey4', 'data4').set_indexes({'field1_bin':'val4', 'field2_int':1004}).store() + + bucket.\ + new('mykey1', 'data1').\ + add_index('field1_bin', 'val1').\ + add_index('field2_int', 1001).\ + store() + bucket.\ + new('mykey2', 'data1').\ + add_index('field1_bin', 'val2').\ + add_index('field2_int', 1002).\ + store() + bucket.\ + new('mykey3', 'data1').\ + add_index('field1_bin', 'val3').\ + add_index('field2_int', 1003).\ + store() + bucket.\ + new('mykey4', 'data1').\ + add_index('field1_bin', 'val4').\ + add_index('field2_int', 1004).\ + store() # Immediate test to see if 2i is even supported w/ the backend try: diff --git a/riak/transports/http.py b/riak/transports/http.py index 7c06d8c8..456b91b5 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -17,7 +17,7 @@ specific language governing permissions and limitations under the License. """ -import urllib, re +import urllib, re, csv from cStringIO import StringIO import httplib try: @@ -29,6 +29,7 @@ from riak.metadata import * from riak.mapreduce import RiakLink from riak import RiakError +from riak.riak_index_entry import RiakIndexEntry from riak.multidict import MultiDict MAX_LINK_HEADER_SIZE = 8192 - 8 # substract length of "Link: " header string and newline @@ -113,8 +114,12 @@ def put(self, robj, w = None, dw = None, return_body = True): for key, value in robj.get_usermeta().iteritems(): headers['X-Riak-Meta-%s' % key] = value - for key, value in robj.get_indexes().iteritems(): - headers['X-Riak-Index-%s' % key] = value + for rie in robj.get_indexes(): + key = 'X-Riak-Index-%s' % rie.get_field() + if key in headers: + headers[key] += ", " + rie.get_value() + else: + headers[key] = rie.get_value() content = robj.get_encoded_data() return self.do_put(url, headers, content, return_body, key=robj.get_key()) @@ -270,7 +275,7 @@ def parse_body(self, response, expected_statuses): # Parse the headers... vclock = None - metadata = {MD_USERMETA: {}, MD_INDEX: {}} + metadata = {MD_USERMETA: {}, MD_INDEX: []} links = [] for header, value in headers.iteritems(): if header == 'content-type': @@ -288,7 +293,13 @@ def parse_body(self, response, expected_statuses): elif header.startswith('x-riak-meta-'): metadata[MD_USERMETA][header.replace('x-riak-meta-', '')] = value elif header.startswith('x-riak-index-'): - metadata[MD_INDEX][header.replace('x-riak-index-', '')] = value + field = header.replace('x-riak-index-', '') + reader = csv.reader([value], skipinitialspace=True) + for line in reader: + for token in line: + rie = RiakIndexEntry(field, token) + metadata[MD_INDEX].append(rie) + elif header == 'x-riak-vclock': vclock = value if links: diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 9c94d769..9b84076d 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -30,6 +30,7 @@ from riak.metadata import * from riak.mapreduce import RiakMapReduce, RiakLink from riak import RiakError +from riak.riak_index_entry import RiakIndexEntry try: import riakclient_pb2 @@ -471,9 +472,10 @@ def decode_content(self, rpb_content): usermeta[usermd.key] = usermd.value if len(usermeta) > 0: metadata[MD_USERMETA] = usermeta - indexes = {} + indexes = [] for index in rpb_content.indexes: - indexes[index.key] = index.value + rie = RiakIndexEntry(index.key, index.value) + indexes.append(rie) if len(indexes) > 0: metadata[MD_INDEX] = indexes return metadata, rpb_content.value @@ -494,10 +496,10 @@ def pbify_content(self, metadata, data, rpb_content) : pair.key = uk pair.value = uv elif k == MD_INDEX: - for uk, uv in v.iteritems(): + for rie in v: pair = rpb_content.indexes.add() - pair.key = uk - pair.value = str(uv) + pair.key = rie.get_field() + pair.value = rie.get_value() elif k == MD_LINKS: for link in v: pb_link = rpb_content.links.add() From 71fdc33f5f2bf6e41e87e8b2e3c255fbae91c701 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Tue, 4 Oct 2011 14:05:56 -0400 Subject: [PATCH 044/118] Add retry logic to the HTTP request method. This also closes connections whenever an error occurs, for safety. This ensures the HTTPConnection object returns to a usable state. --- riak/transports/http.py | 55 +++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 27a41aac..22578d62 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -22,6 +22,8 @@ import urllib, re from cStringIO import StringIO import httplib +import socket +import errno try: import json except ImportError: @@ -49,6 +51,9 @@ class RiakHttpTransport(RiakTransport) : # The ConnectionManager class that this transport prefers. default_cm = HTTPConnectionManager + # How many times to retry a request + RETRY_COUNT = 3 + def __init__(self, cm, prefix='riak', mapred_prefix='mapred', client_id=None, **unused_options): @@ -419,22 +424,40 @@ def http_request(self, method, uri, headers=None, body='') : if headers is None: headers = {} # Run the request... - with self._conns.withconn() as conn: - conn.request(method, uri, body, headers) - response = conn.getresponse() - - try: - # Get the response headers... - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value - - # Get the body... - response_body = response.read() - finally: - response.close() - - return response_headers, response_body + for retry in range(self.RETRY_COUNT): + with self._conns.withconn() as conn: + ### should probably build this try/except into a custom + ### contextmanager for the connection. + try: + conn.request(method, uri, body, headers) + response = conn.getresponse() + + try: + # Get the response headers... + response_headers = {'http_code': response.status} + for (key, value) in response.getheaders(): + response_headers[key.lower()] = value + + # Get the body... + response_body = response.read() + finally: + response.close() + + return response_headers, response_body + except socket.error, e: + conn.close() + if e[0] == errno.ECONNRESET: + # Grab another connection and try again. + continue + # Don't know how to handle this. + raise + except httplib.HTTPException: + # Just close the connection and try again. + conn.close() + continue + + # No luck, even with retrying. + raise RiakError("could not get a response") @classmethod def build_headers(cls, headers): From b36ecb34708f59cf755a48998ab18ca2e846aa5d Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 18 Oct 2011 16:10:58 +0200 Subject: [PATCH 045/118] Copy erl_src/* from Ruby client --- erl_src/riak_kv_test_backend.beam | Bin 7160 -> 5984 bytes erl_src/riak_kv_test_backend.erl | 602 ++++++++++++++++++++------ erl_src/riak_search_test_backend.beam | Bin 8484 -> 4440 bytes erl_src/riak_search_test_backend.erl | 6 +- 4 files changed, 478 insertions(+), 130 deletions(-) diff --git a/erl_src/riak_kv_test_backend.beam b/erl_src/riak_kv_test_backend.beam index f5a61ce9315fc57dd32fe28e7056f968f25d1a86..485d6b75110592ff546372210b389188cc490b74 100644 GIT binary patch literal 5984 zcmbVQTWlNGnV!)ZiX%!V9&sqi@g*L67|Ss&YZTHKN|AlUk;OQ^gchAf&5fkUkxYvs zY0gN#)GmzJ1r+Q)IJFnshdv0&1`90qA;`n-zHK3F7uaH36lnTTbRU8Z7TAZP2-*fH ziazxFheM0b1!zm)*E#1u|M}1NU(SDyX71(r2}v4R$efwEFyl2=C5iouBuNXgR<*QJ zSXnQ4j_VZ`OXU@(UJ11v*YSq^sdA&`6k65#a$&VpFD*N*{$LJ%FwLIo|wa9AeMq$-iZM1Iot<@{eQnl_>dYx9SR9}`4 z@2J*22b&bUMxj=9y#e||gAy<^$8%ZTxlYPFM{&JIv$x{Z9M56E)LUybR@Rz5o*P;! z)m%sMT5C?oX|)=yzIwweEH$tlYp!{$-ncF&+v{}u+Ug)TliEvu!SI2{cZ3C@>Kt7qp}9Cki&KmkgjoTY}>N& zGHkOSGn}v<$NgiV}OgGbfNS_2586C@Y%?2Y0M$*V6rFVEqB9(;28xf;1 zt$Em5&e#TQw(ezY@i1fOVCqO7;oAHVxt5HH_u);bnicG_cvunlJmKCK2!J26D(q7Lh|XatL{3lru@;Cb(h4eZ{+!(0*d3 zM1EEzFDLkc@|JLQ;j(ESm(>HBbyv7CQTDSog{;lDv-Vj(Yg7DLWUh(KH7Rp7h5q5XHlfogK)x80H82gwa|JW*M2HsioC)2=uo zVzvSP)Zz&`H3^)WUHlME&U3PQh_;TSGO=YT2u$Q)J2EH<>T_Y~lYkuq*B9Kdq_Fi_ znjeOP4l@JORDwDM3Kdm7OlT7n!6ShD93YRS`4K=q(hd0tK^8?MHU-YqMdUD5T9HqM z4OK(xv>j@2J1D%qA+fLlLo71H@{j*f7&9)bNBxryU$YHC_`aNFKIVf?0(44NliiTM5_GCt;%h)3BhbeI`lu$W zC(`^l0(1Pyiy^^})2(0#s@%wjq!QND!6W>nvgLc3MzVTRvrtp0gvO!p(&2 zomRHAoL?Ct$Fl5<-oyOrJl#$Np%da5-HbRsk>kDuk`4v%SXMb>qLNT8KZSiySh9Lr zv#hoyCb+oXBPW4oW|?F%7^MwawG(_+-&!U^XXz*-;Y_wnS$_r+laQE8@GK+*SlMlf z9ELAKVha156=sqGon*^JuwMfE1!T@St`)_cVdG)|Mm=k1O&9N>6i_uM@=o^}br6bx215YSTE3r|jRQ>-^?JXxXDvfHRQl~A1ilY)k zWOp*9?cvK%IR_J8dD22NYs0*YFi%#$0g){5zIul*g(853tFCq9l zC6eTCdY|CTvpBBx*XZ(4m`!CSmc*5T5*fms0#)CH$`@hCYmZ-LIxZJHVbrG+{B;(2 zNg(kkbgxqw_fr|5jInSX`p+&J2Spm?4q?E1<&B{7zlqq;)69RE#WxJNhH80;LL;kh zr1+aC)HE9cM7aM=I&NTaF+jfv}3JYc~n*%`t5MS@la zvJSHDm&iF_8Y*u1WiVfcIaeO1@I~u0S*?PdLwv8Mc@>6LeZy?L zp7UVXVAyzD7xA1a<2eKLD`gZm z8rD!>Ye{I#;bv#_IwVbHXmZJP*AA4`p<^DX8fjjKj(Qsv20&CNM7hobE5Vz2lM)Lt zE$injA4{`?^mol`dAg zNrN6Hn9(uNjRBFv>oG=Wul6{sCuxa;p1+ilMacX${W4PJzDxh_3?T%>m1zazmv)zI}m{IGrp! zh1(bSE!c6(x1(LOZ&BI#7L+V#vU>X@zrzMJZH6&a-Y@;9LKjB+klcpwofQ8T@)7@l z<-OKrG}77CCR+JEx(0}*6cGJ-LHOTHl0h8w6sRZY??Bt()BkqV9|T1~Bt!inP=`Mf z@JY`zAo@{%AE=|#=<4rBpRjcFkW3u(EQtD}Ali$5By$Kv@(IupPzS>}`W+0<2YlK$ z*(F2#j?8qx7iecdq|*k?f;u?Q1by1~EU2^g9Qq_r`X>TD^)G&*-@*S1_?`YYsDI`L z#X|ZI5c%{1h;%OneKb-9bOJ>CoCc*pB=Z8u3i^|1j|P2`9Rn#Kvh%s1PrL&`pL|KN zBANZ5y`T_iCukT%dl8-(i1z9S^??XG<%bHQAFUxC{d(rs>SW75gGlC85b3!Iq8|(W z?E?9lWQgy9NVhEC`UT14LBwx?NT)3SBK2Fe^B}_60MUGpGWy#8puH4~SJBeE+;c_W zLYoW5SJC?T*zd-Fh5pMRT7LyZdCq!w_r89<>Jgp=(Cat8UoIAlOU2@YD?cf|RQYfD zqn-Jk{}%n5#Baa*>Cb-S&VO|GzM1Fh?lm(G5XwNwi`HONf10osP^Aw0MkZ$S|pYoXUjN~cTr$8h_ zXMl2!WJq5+=-X&df(XORjOS4-FWjr)Z_kyUip194It}EyCjH~bpZ;NL_TN?S?ahDq z&v$o_0sUSF?5%fgMOhcCJA_D1FF zl@;mv@mKJ*&>eT3Qhj{ZS$ArUCVf&JZ&jPknuDgaGFGltiG{Dc1-DfuZM}^(uTpLO zRqK5#!oip26)BWDdg5q`lq&R@@k;L^&c-iGeWhq=s5Dv{^0#vvYpt@gBAxro;rb?g S@e}z`*g}4qS#&)#(w_j{AzNes literal 7160 zcmbVQ2UL^Uwodp136URu2)zbEl@>@u0t%r91Qh{$NkRx12ni$s#I9f&%ZOvaf?`3` z5fx-q22l}lY$&!-Y$%}EuwfrZefvc4a_^n>)_N~%eV%>x*=P524!_YO6a7den^)mc zVZ*~T$}AE|C!9ngg`28m;!I&?o=_uIYlJCcNv2ehs;kzBRT_G_R3TJLRe4esUCR*4 zWr|E)l~gU&=t)&_u_B$WlBUbl8mUT}sw)RVt(&Hlr^<1aR3eq-NrhTj9c89os#Gr3 zNGURfOk*TdXr$>6lL(Oe9kQPfg-8m&(;Ce^4hg)|?zq>&DoDpsV* zr9z2VtugtoQz(|ph02r+sYIiuDwHZ|w!A=Bs!}Oc{}PtVv2>|sfYwS%E`+FoC_s*a zyv4F}H8h!8D9y^&6i`xC%4`Y}q^LDOsb$lV*J(;@XwXoK1Z>gNsyv}~O{2<{Qqc@_ z>&cbM%-n446!huI6p%O#9j$(MQ@ai6)y)!XBvXyhN6et-IWh$fsQj#tmkflt`g~T8UBk)JqZ*J!OBWrTGFULDASK-a} zW_W9h56c*f zPr-~SR5lZM6ap8*RF{wvfFMAEJlA3e##@60t-+Y#7>N;% z5iHL&j6^d&g|V0*@mUy^$%g$TaT%yA!B{2u&y$pMe%&s-NBKC!a;3GUp<F}a;6F;4|A$o6KT28jc@&N@lYV%`r`zL} zbO+puOtUm#iue&2WS*a7nojU=2TBlt9ZrGbQ&9387?3v=CD1%T_dZ6(w5{=PiL?FRRls)Q6~QM(v!Tp(&`jDw)9efgc5g)p zeCR(qZr@+T`5#3b{?#DNUXmcw9HCcOFgo$+EXD{X4M-NS++lR);jUN^fc@i~;jWNs z0}}EEJoUs~FuDgEP7@O3&LsvT(_E3}5kwW4<_219pc&n@ZORhRZ3K{?UK1X$I1m;G z@^Ei3;jK0C{fOY+EHDAbk)u6+Oqy{Z?x{VmJlqS8jTeIn$A$yPCW4E5LG`^j6oQ@b z{qRL-+tU*kFus8Oy;;2oHh~Tj-3m@pZv%V(aMK4oabS~az675g&E^Q`4g!XN#pm=o z3Rf203zq$OI0qc%XdTraAWxjbp_}0(WSY~v2ila%q?vf){{8aekMhF#VUVk1(g=|c z>P<^yzpMm+F&|hP%)@zLjMqOaJd_m@$_iM3t_$pEArM*M{jd-OYMh{c!F;+?Zx};> zCC=EqQ!8d*&9tPE5aRTvBRv0TH ziHnCO5zK_B2nGY3(}R6vS{N`!}Gyx%ktCR&*M0zj*bSoEL05UBGOv7yxtG#Wo@3&1XocF$m zrhl$tQLf+uhBOa@xR@};@#)Oo3d94;5m>|dbjM!S2w>T>$+VGut!J-oBl;k1`>|I{B4j{a@6zzk?(LQK!U2+&u87u8uj|)Lm#Kote zl#5|&Cxec0=m3z6A%NUV8^PGeM}PP9@r{8- zKO}DgKr#$~X+PQm=stO?UK+{t*+sMqpijSRFOB%fAG8A z`6w>`Ub;^Y8j;VzfIi<4jr60nkX{vzLy8(H4O8L;#!sNCw5w zmxDn-I{=VQUw{n&$-4ts03?t0Sp$#^s-HOk$)MWf03?I-=>pIX^$=RuB05)rV(0eG zLwm;*7{C9va!?(H0FeI40HlwsYp~^gyi9}kW&@T@E0u^uA^?$fQ8Y{}8ty&toG*;N zXPbv#@bxASqjA$YuU40vmo&M#xTf#y+Ce(ibYcl_l}XZ)#8va2&a++`XA+kCCkp1OJNXadQ0Ro#Db4xp;{$D+VqsUnpBzU|Fu7rL{F8b6!_`k+x@zg{{&P`9F_X(xW*W@pWja=F!psNWRm-_wDDc+gCUYeh_e$%M|1t%gKTh}hDZlMiuEDt{Wi%UpE$jFfBy7Y=8 zdw!h}-$O}Cn*45$aP-)3H^)EuKo>^7V4R-(qG{#C<(uBMr)^F6*g5R1XQ{d~u&3mR zpZars`0lR3H-~-8Xg{y~X7cI9uQhcqnkpH==M}kYblVPlWz@||y;Cw}>mB-0#j1h@ z_gbTkzucR?V5{^{$G*>E=VB?V``qz}b7g;yzpokh+eN2sA7&WNe|Jl4y7bb;{jb(h zgQ{&#DUN?Ue(k4|0fLL)yswXU^*VH>SK}_=qwl6AHLVIUjI2Dl+{~Ktri|!Pr8F7M)K5Ky*So5V z_FnsWt;T<9Vh8mgdrj%m@z;8=u^nIPt*6hV+}Oi89AiB5lyPL`x!r4uPQ{C!2tB4R zc=N<&XBWr5b!>RDs7}L#q6BNyF0xBor z>X-Sm_HW;Azi^b%Gg58WT=z~x^PhCGejT^e*4UwGo?%5>uwJRS)SP_(^tyvvwtW1M zVU}LF!sap|U^W>qPj4zZ6ZJ5!DP#7?-@V@~d`QjBPd%93)*PAP@N=l|*PM<^#uZ5k zCeLE@LUeB%ovbhCCPkEd49&SFb@y5MKJL~_m-3L0sSBc-Uy4df%H!N)X@zaaPZl2Q zOgg=C^E|f828&gdmFKOSrFfUWZ};Lq3qINw--vOYIj837l*_lqiiYibymhRq>)aad zUOyi_AMc(F(-th}@ZAR$tLIynma`Ad92yv^~vUTs2KgaB! zzCif8qUrD4F6U!@E5<$)1$pehoTDfk<$5D=;xFGanX$Xoit^A&QD?u{fBrV*(^z#D zTm9_opX+9C&&dAcmAiOavY(gTvI_a2WqJjZSJWvU(RR)MtD*Dp>ZAL{2qxLyh`q8e zvu)K`_G7Bwo;S(gvexHs|8^;G4m0O#!@VkZI?LI5*%$KxM{-V{=^B2p*t)dMIq9#8 z#%Wh?omE!EujLEulOq18?S7GU_!PH5V(m0%M##n;Y$o10wkV}VM@2_sih|+T_A}HPZhG|ai z6USVO?2K>88gk9+RcxIVcf+6M^NTNYVjr*PPN$Zv{$o4gH8wnb454?(#ltbFxY#2r zcZo@(?(q@9MJG@5+Es<4B}cC|aN|8%lCIRmTODY%82n(!hOUuw5>8gs5d1q=onOe* zqc=n=FPy6z9)G9ns2h!SU7>JqIXq}I8_&;nxnsOEc;@`KjfT?})z(XL^cSw`S+{j3 zRs7txX;8$Wdd^MbSgS|#$%4e)&b+DK*Yy%Bmvmaa&D+J;8{Fvd!s^z73!Y7(dxoSD zY5L-tCt)LRwzurCStZ|2c5!aJ`8O>+wx}kpsOItWEsLuzo~ei}2*gTUI%aS?W>Bsi zeaqj|edoMaL|Ao4fK3A5vbLn9!8T`bx<7kSo2x-K|E~teA&7(@yrjJTDt)xrK z!!x#Q%X2%Id}_;)l;VW5Zd=1+cCB+xzI3~VqJPLP@znhjh8Ije3?ciK-4*W2c@rJ; z=9A<2%FD%Fmv5)O*zoL2Jum;!^N%mD*57q2II==1UOm*u_g+jS?#`-iYgv&yuc=P- zN8Q}E*^ZJ6TkhNXSn?)SF(*~kyqYU2AiY?7t(6+BG3wbq(b?Q$wC3i7v%04)XKoK_ zjjCPZyKUt(GhuwdwSx&+oL6;6cis2y+JAd#&-Iv%-5)m^lu^y^&X)~fd?+sZbSX~R zeyw(qp3KNk_GQC7mop8&1k^=ZjSF|PJKrXaEtq)jk;vojk;kP&olXo%4 z;pX_9$K|TU($jHD_ce}=mM`@zwla^Fl61n$B{zw*(@Reed*b))Jnq)g@$-|{ZZFr_ zUtRvxjdEq?J^U^ACVz$5+rWm1zx>!2Ptp{Pn9Hkgd6{qMMxuR**}u#6KG<29Z`*UV z>q@xh!qWQYa;M25YJ;>*DW<&Kr&od|yZ-r`!8304p~{r`CZE5qF6aqwbEhsGYqoFv z=H=JZ$6YwXEL=wJ_IhN!REp(HIEYnx+_$RD6g|o~G38`y=GuL$)`U+E-ZSHgbh}}t zA$D>@t?ASBcja}Bi7hR;lOG1nni8w5j(P0u`?S=9+W&Wq5)=#D zGY)U6=~;MAxvs1|VPAcspYFO5GGd~QF#FI2@92$Hw*q{-YNxu>Iw!Q^L+T^hoN#wmxIuiQHK?A*lSGVkRno;z0`s@(CL)N_Zo|8fn@d>prJ z=i|VvpFVj+zq;nWWv6@UIQDJptA|FhZx<7lKMhFR__u9}Y4f4^2Q~@Ww|7lep3Iz= z+%CMXYJL46-9XPTxc2<>2g+abM7x4-JsdIOMSkV)_g*&-UsIG`{?4j4u}U;&VRh*F zd&$oS?wIVeakI%-r-4G+Jz#)~{=?+ah;*`Y4HX>q`*xr{oc+=Taj!GX_>5us0#{C<0FQo7^R^sgpgJ*me` zsV)zxXX7{qlO`N}>vhkBTjS`#@ZB5l9`}12Zc(&z?AtvK+i5~^$^K@=U&HSGeTnv7 z^yK-I?Dwj!YnyXNJiqzZ6Z_qJj+Sn$89Mjp=x?{`yAGH(XD;g+)V$uRC4P_3f1F3**n}Z)wYh z>=bmbvacTba_D=!c5bk=@WV>IfTvuMTfuhWvFeLugQMLF3O}4tR)3u+89ys72-{-JFvk8PapKO`cX zT5D|IT5ZSI^*n?vc}}`IT@?G0Jp3g&M%nc#VSFGr{j6i<iTa3_A)u^U+1!~s1=M-{(ZUXfdLtcm;AJL3_zMtxoQh~}>;-)i@e`-qjP4mlv7 z#*DgQ^+&(*p|iqh|gync|sPwxd|=FDKf8)q0Q zfyc7f?>=rbZq+yFjyxH3R==Ap27_u8_zU%w>*9$EZkA% z7rk$CUiRRJ+kU}hag?i@f?8MY2%KHL84J18G9Nc^$R^J`(;X4kri^!piTEP= +%%
  • `ttl' - The time in seconds that an object should live before being expired.
    • +%%
  • `max_memory' - The amount of memory in megabytes to limit the backend to.
    • +%% +%% -module(riak_kv_test_backend). -behavior(riak_kv_backend). --behavior(gen_server). + +%% KV Backend API +-export([api_version/0, + start/2, + stop/1, + get/3, + put/5, + delete/4, + drop/1, + fold_buckets/4, + fold_keys/4, + fold_objects/4, + is_empty/1, + status/1, + callback/3, + reset/0]). + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -endif. --export([start/2,stop/1,get/2,put/3,list/1,list_bucket/2,delete/2, - is_empty/1, drop/1, fold/3, callback/3, reset/0]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). +-define(API_VERSION, 1). +-define(CAPABILITIES, [async_fold]). + +-record(state, {data_ref :: integer() | atom(), + time_ref :: integer() | atom(), + max_memory :: undefined | integer(), + used_memory=0 :: integer(), + ttl :: integer()}). +-type state() :: #state{}. +-type config() :: []. -% @type state() = term(). --record(state, {t, p}). +%% =================================================================== +%% Public API +%% =================================================================== -% @spec start(Partition :: integer(), Config :: proplist()) -> -% {ok, state()} | {{error, Reason :: term()}, state()} -start(Partition, _Config) -> - gen_server:start_link(?MODULE, [Partition], []). +%% TestServer reset -% @spec reset() -> ok | {error, timeout} +-spec reset() -> ok | {error, timeout}. reset() -> - Pids = lists:foldl(fun(Item, Acc) -> - case lists:prefix("test_backend", atom_to_list(Item)) of - true -> [whereis(Item)|Acc]; - _ -> Acc - end - end, [], registered()), - [gen_server:cast(Pid,{reset, self()})|| Pid <- Pids], - receive_reset(Pids). - -receive_reset([]) -> ok; -receive_reset(Pids) -> - receive - {reset, Pid} -> - receive_reset(lists:delete(Pid, Pids)) - after 1000 -> - {error, timeout} - end. + {ok, Ring} = riak_core_ring_manager:get_my_ring(), + [ ets:delete_all_objects(list_to_atom("kv" ++ integer_to_list(P))) || + P <- riak_core_ring:my_indices(Ring) ], + ok. -%% @private -init([Partition]) -> - PName = list_to_atom("test_backend" ++ integer_to_list(Partition)), - P = list_to_atom(integer_to_list(Partition)), - register(PName, self()), - {ok, #state{t=ets:new(P,[]), p=P}}. +%% KV Backend API -%% @private -handle_cast({reset,From}, State) -> - ets:delete_all_objects(State#state.t), - From ! {reset, self()}, - {noreply, State}; -handle_cast(_, State) -> {noreply, State}. +%% @doc Return the major version of the +%% current API and a capabilities list. +-spec api_version() -> {integer(), [atom()]}. +api_version() -> + {?API_VERSION, ?CAPABILITIES}. -%% @private -handle_call(stop,_From,State) -> {reply, srv_stop(State), State}; -handle_call({get,BKey},_From,State) -> {reply, srv_get(State,BKey), State}; -handle_call({put,BKey,Val},_From,State) -> - {reply, srv_put(State,BKey,Val),State}; -handle_call({delete,BKey},_From,State) -> {reply, srv_delete(State,BKey),State}; -handle_call(list,_From,State) -> {reply, srv_list(State), State}; -handle_call({list_bucket,Bucket},_From,State) -> - {reply, srv_list_bucket(State, Bucket), State}; -handle_call(is_empty, _From, State) -> - {reply, ets:info(State#state.t, size) =:= 0, State}; -handle_call(drop, _From, State) -> - ets:delete(State#state.t), - {reply, ok, State}; -handle_call({fold, Fun0, Acc}, _From, State) -> - Fun = fun({{B,K}, V}, AccIn) -> Fun0({B,K}, V, AccIn) end, - Reply = ets:foldl(Fun, Acc, State#state.t), - {reply, Reply, State}. - -% @spec stop(state()) -> ok | {error, Reason :: term()} -stop(SrvRef) -> gen_server:call(SrvRef,stop). -srv_stop(State) -> - true = ets:delete(State#state.t), +%% @doc Start the memory backend +-spec start(integer(), config()) -> {ok, state()}. +start(Partition, Config) -> + TTL = config_value(ttl, Config), + MemoryMB = config_value(max_memory, Config), + case MemoryMB of + undefined -> + MaxMemory = undefined, + TimeRef = undefined; + _ -> + MaxMemory = MemoryMB * 1024 * 1024, + TimeRef = ets:new(list_to_atom(integer_to_list(Partition)), [ordered_set]) + end, + DataRef = ets:new(list_to_atom("kv" ++ integer_to_list(Partition)), [named_table, public]), + {ok, #state{data_ref=DataRef, + max_memory=MaxMemory, + time_ref=TimeRef, + ttl=TTL}}. + +%% @doc Stop the memory backend +-spec stop(state()) -> ok. +stop(#state{data_ref=DataRef, + max_memory=MaxMemory, + time_ref=TimeRef}) -> + catch ets:delete(DataRef), + case MaxMemory of + undefined -> + ok; + _ -> + catch ets:delete(TimeRef) + end, ok. -% get(state(), riak_object:bkey()) -> -% {ok, Val :: binary()} | {error, Reason :: term()} -% key must be 160b -get(SrvRef, BKey) -> gen_server:call(SrvRef,{get,BKey}). -srv_get(State, BKey) -> - case ets:lookup(State#state.t,BKey) of - [] -> {error, notfound}; - [{BKey,Val}] -> {ok, Val}; - Err -> {error, Err} +%% @doc Retrieve an object from the memory backend +-spec get(riak_object:bucket(), riak_object:key(), state()) -> + {ok, any(), state()} | + {ok, not_found, state()} | + {error, term(), state()}. +get(Bucket, Key, State=#state{data_ref=DataRef, + ttl=TTL}) -> + case ets:lookup(DataRef, {Bucket, Key}) of + [] -> {error, not_found, State}; + [{{Bucket, Key}, {{ts, Timestamp}, Val}}] -> + case exceeds_ttl(Timestamp, TTL) of + true -> + delete(Bucket, Key, undefined, State), + {error, not_found, State}; + false -> + {ok, Val, State} + end; + [{{Bucket, Key}, Val}] -> + {ok, Val, State}; + Error -> + {error, Error, State} end. -% put(state(), riak_object:bkey(), Val :: binary()) -> -% ok | {error, Reason :: term()} -% key must be 160b -put(SrvRef, BKey, Val) -> gen_server:call(SrvRef,{put,BKey,Val}). -srv_put(State,BKey,Val) -> - true = ets:insert(State#state.t, {BKey,Val}), - ok. +%% @doc Insert an object into the memory backend. +%% NOTE: The memory backend does not currently +%% support secondary indexing and the _IndexSpecs +%% parameter is ignored. +-type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. +-spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> + {ok, state()} | + {error, term(), state()}. +put(Bucket, PrimaryKey, _IndexSpecs, Val, State=#state{data_ref=DataRef, + max_memory=MaxMemory, + time_ref=TimeRef, + ttl=TTL, + used_memory=UsedMemory}) -> + Now = now(), + case TTL of + undefined -> + Val1 = Val; + _ -> + Val1 = {{ts, Now}, Val} + end, + case do_put(Bucket, PrimaryKey, Val1, DataRef) of + {ok, Size} -> + %% If the memory is capped update timestamp table + %% and check if the memory usage is over the cap. + case MaxMemory of + undefined -> + UsedMemory1 = UsedMemory; + _ -> + time_entry(Bucket, PrimaryKey, Now, TimeRef), + Freed = trim_data_table(MaxMemory, + UsedMemory + Size, + DataRef, + TimeRef, + 0), + UsedMemory1 = UsedMemory + Size - Freed + end, + {ok, State#state{used_memory=UsedMemory1}}; + {error, Reason} -> + {error, Reason, State} + end. -% delete(state(), riak_object:bkey()) -> -% ok | {error, Reason :: term()} -% key must be 160b -delete(SrvRef, BKey) -> gen_server:call(SrvRef,{delete,BKey}). -srv_delete(State, BKey) -> - true = ets:delete(State#state.t, BKey), - ok. +%% @doc Delete an object from the memory backend +%% NOTE: The memory backend does not currently +%% support secondary indexing and the _IndexSpecs +%% parameter is ignored. +-spec delete(riak_object:bucket(), riak_object:key(), [index_spec()], state()) -> + {ok, state()}. +delete(Bucket, Key, _IndexSpecs, State=#state{data_ref=DataRef, + time_ref=TimeRef, + used_memory=UsedMemory}) -> + case TimeRef of + undefined -> + UsedMemory1 = UsedMemory; + _ -> + %% Lookup the object so we can delete its + %% entry from the time table and account + %% for the memory used. + [Object] = ets:lookup(DataRef, {Bucket, Key}), + case Object of + {_, {{ts, Timestamp}, _}} -> + ets:delete(TimeRef, Timestamp), + UsedMemory1 = UsedMemory - object_size(Object); + _ -> + UsedMemory1 = UsedMemory + end + end, + ets:delete(DataRef, {Bucket, Key}), + {ok, State#state{used_memory=UsedMemory1}}. + +%% @doc Fold over all the buckets. +-spec fold_buckets(riak_kv_backend:fold_buckets_fun(), + any(), + [], + state()) -> {ok, any()}. +fold_buckets(FoldBucketsFun, Acc, Opts, #state{data_ref=DataRef}) -> + FoldFun = fold_buckets_fun(FoldBucketsFun), + case lists:member(async_fold, Opts) of + true -> + BucketFolder = + fun() -> + {Acc0, _} = ets:foldl(FoldFun, {Acc, sets:new()}, DataRef), + Acc0 + end, + {async, BucketFolder}; + false -> + {Acc0, _} = ets:foldl(FoldFun, {Acc, sets:new()}, DataRef), + {ok, Acc0} + end. + +%% @doc Fold over all the keys for one or all buckets. +-spec fold_keys(riak_kv_backend:fold_keys_fun(), + any(), + [{atom(), term()}], + state()) -> {ok, term()} | {async, fun()}. +fold_keys(FoldKeysFun, Acc, Opts, #state{data_ref=DataRef}) -> + Bucket = proplists:get_value(bucket, Opts), + FoldFun = fold_keys_fun(FoldKeysFun, Bucket), + case lists:member(async_fold, Opts) of + true -> + {async, get_folder(FoldFun, Acc, DataRef)}; + false -> + Acc0 = ets:foldl(FoldFun, Acc, DataRef), + {ok, Acc0} + end. + +%% @doc Fold over all the objects for one or all buckets. +-spec fold_objects(riak_kv_backend:fold_objects_fun(), + any(), + [{atom(), term()}], + state()) -> {ok, any()} | {async, fun()}. +fold_objects(FoldObjectsFun, Acc, Opts, #state{data_ref=DataRef}) -> + Bucket = proplists:get_value(bucket, Opts), + FoldFun = fold_objects_fun(FoldObjectsFun, Bucket), + case lists:member(async_fold, Opts) of + true -> + {async, get_folder(FoldFun, Acc, DataRef)}; + false -> + Acc0 = ets:foldl(FoldFun, Acc, DataRef), + {ok, Acc0} + end. -% list(state()) -> [riak_object:bkey()] -list(SrvRef) -> gen_server:call(SrvRef,list). -srv_list(State) -> - MList = ets:match(State#state.t,{'$1','_'}), - list(MList,[]). -list([],Acc) -> Acc; -list([[K]|Rest],Acc) -> list(Rest,[K|Acc]). - -% list_bucket(term(), Bucket :: riak_object:bucket()) -> [Key :: binary()] -list_bucket(SrvRef, Bucket) -> - gen_server:call(SrvRef,{list_bucket, Bucket}). -srv_list_bucket(State, {filter, Bucket, Fun}) -> - MList = lists:filter(Fun, ets:match(State#state.t,{{Bucket,'$1'},'_'})), - list(MList,[]); -srv_list_bucket(State, Bucket) -> - case Bucket of - '_' -> MatchSpec = {{'$1','_'},'_'}; - _ -> MatchSpec = {{Bucket,'$1'},'_'} +%% @doc Delete all objects from this memory backend +-spec drop(state()) -> {ok, state()}. +drop(State=#state{data_ref=DataRef, + time_ref=TimeRef}) -> + ets:delete_all_objects(DataRef), + case TimeRef of + undefined -> + ok; + _ -> + ets:delete_all_objects(TimeRef) end, - MList = ets:match(State#state.t,MatchSpec), - list(MList,[]). + {ok, State}. -is_empty(SrvRef) -> gen_server:call(SrvRef, is_empty). +%% @doc Returns true if this memory backend contains any +%% non-tombstone values; otherwise returns false. +-spec is_empty(state()) -> boolean(). +is_empty(#state{data_ref=DataRef}) -> + ets:info(DataRef, size) =:= 0. -drop(SrvRef) -> gen_server:call(SrvRef, drop). +%% @doc Get the status information for this memory backend +-spec status(state()) -> [{atom(), term()}]. +status(#state{data_ref=DataRef, + time_ref=TimeRef}) -> + DataStatus = ets:info(DataRef), + case TimeRef of + undefined -> + [{data_table_status, DataStatus}]; + _ -> + TimeStatus = ets:info(TimeRef), + [{data_table_status, DataStatus}, + {time_table_status, TimeStatus}] + end. -fold(SrvRef, Fun, Acc0) -> gen_server:call(SrvRef, {fold, Fun, Acc0}, infinity). +%% @doc Register an asynchronous callback +-spec callback(reference(), any(), state()) -> {ok, state()}. +callback(_Ref, _Msg, State) -> + {ok, State}. -%% Ignore callbacks for other backends so multi backend works -callback(_State, _Ref, _Msg) -> - ok. +%% =================================================================== +%% Internal functions +%% =================================================================== + +%% @TODO Some of these implementations may be suboptimal. +%% Need to do some measuring and testing to refine the +%% implementations. %% @private -handle_info(_Msg, State) -> {noreply, State}. +%% Return a function to fold over the buckets on this backend +fold_buckets_fun(FoldBucketsFun) -> + fun({{Bucket, _}, _}, {Acc, BucketSet}) -> + case sets:is_element(Bucket, BucketSet) of + true -> + {Acc, BucketSet}; + false -> + {FoldBucketsFun(Bucket, Acc), + sets:add_element(Bucket, BucketSet)} + end + end. %% @private -terminate(_Reason, _State) -> ok. +%% Return a function to fold over keys on this backend +fold_keys_fun(FoldKeysFun, undefined) -> + fun({{Bucket, Key}, _}, Acc) -> + FoldKeysFun(Bucket, Key, Acc) + end; +fold_keys_fun(FoldKeysFun, Bucket) -> + fun({{B, Key}, _}, Acc) -> + case B =:= Bucket of + true -> + FoldKeysFun(Bucket, Key, Acc); + false -> + Acc + end + end. %% @private -code_change(_OldVsn, State, _Extra) -> {ok, State}. +%% Return a function to fold over keys on this backend +fold_objects_fun(FoldObjectsFun, undefined) -> + fun({{Bucket, Key}, Value}, Acc) -> + FoldObjectsFun(Bucket, Key, Value, Acc) + end; +fold_objects_fun(FoldObjectsFun, Bucket) -> + fun({{B, Key}, Value}, Acc) -> + case B =:= Bucket of + true -> + FoldObjectsFun(Bucket, Key, Value, Acc); + false -> + Acc + end + end. + +%% @private +get_folder(FoldFun, Acc, DataRef) -> + fun() -> + ets:foldl(FoldFun, Acc, DataRef) + end. + +%% @private +do_put(Bucket, Key, Val, Ref) -> + Object = {{Bucket, Key}, Val}, + true = ets:insert(Ref, Object), + {ok, object_size(Object)}. + +%% @private +config_value(Key, Config) -> + config_value(Key, Config, undefined). + +%% @private +config_value(Key, Config, Default) -> + case proplists:get_value(Key, Config) of + undefined -> + app_helper:get_env(memory_backend, Key, Default); + Value -> + Value + end. + +%% Check if this timestamp is past the ttl setting. +exceeds_ttl(Timestamp, TTL) -> + Diff = (timer:now_diff(now(), Timestamp) / 1000 / 1000), + Diff > TTL. + +%% @private +time_entry(Bucket, Key, Now, TimeRef) -> + ets:insert(TimeRef, {Now, {Bucket, Key}}). + +%% @private +%% @doc Dump some entries if the max memory size has +%% been breached. +trim_data_table(MaxMemory, UsedMemory, _, _, Freed) when + (UsedMemory - Freed) =< MaxMemory -> + Freed; +trim_data_table(MaxMemory, UsedMemory, DataRef, TimeRef, Freed) -> + %% Delete the oldest object + OldestSize = delete_oldest(DataRef, TimeRef), + trim_data_table(MaxMemory, + UsedMemory, + DataRef, + TimeRef, + Freed + OldestSize). + +%% @private +delete_oldest(DataRef, TimeRef) -> + OldestTime = ets:first(TimeRef), + case OldestTime of + '$end_of_table' -> + 0; + _ -> + OldestKey = ets:lookup_element(TimeRef, OldestTime, 2), + ets:delete(TimeRef, OldestTime), + case ets:lookup(DataRef, OldestKey) of + [] -> + delete_oldest(DataRef, TimeRef); + [Object] -> + ets:delete(DataRef, OldestKey), + object_size(Object) + end + end. + +%% @private +object_size(Object) -> + case Object of + {{Bucket, Key}, {{ts, _}, Val}} -> + ok; + {{Bucket, Key}, Val} -> + ok + end, + size(Bucket) + size(Key) + size(Val). + +%% =================================================================== +%% EUnit tests +%% =================================================================== -%% -%% Test -%% -ifdef(TEST). -% @private -simple_test() -> +simple_test_() -> riak_kv_backend:standard_test(?MODULE, []). --ifdef(EQC). +ttl_test_() -> + Config = [{ttl, 15}], + {ok, State} = start(42, Config), + + Bucket = <<"Bucket">>, + Key = <<"Key">>, + Value = <<"Value">>, + + [ + %% Put an object + ?_assertEqual({ok, State}, put(Bucket, Key, [], Value, State)), + %% Wait 1 second to access it + ?_assertEqual(ok, timer:sleep(1000)), + ?_assertEqual({ok, Value, State}, get(Bucket, Key, State)), + %% Wait 3 seconds and access it again + ?_assertEqual(ok, timer:sleep(3000)), + ?_assertEqual({ok, Value, State}, get(Bucket, Key, State)), + %% Wait 15 seconds and it should expire + {timeout, 30000, ?_assertEqual(ok, timer:sleep(15000))}, + %% This time it should be gone + ?_assertEqual({error, not_found, State}, get(Bucket, Key, State)) + ]. + %% @private -eqc_test() -> - ?assertEqual(true, backend_eqc:test(?MODULE, true)). +max_memory_test_() -> + %% Set max size to 1.5kb + Config = [{max_memory, 1.5 * (1 / 1024)}], + {ok, State} = start(42, Config), + + Bucket = <<"Bucket">>, + Key1 = <<"Key1">>, + Value1 = list_to_binary(string:copies("1", 1024)), + Key2 = <<"Key2">>, + Value2 = list_to_binary(string:copies("2", 1024)), + + %% Write Key1 to the datastore + {ok, State1} = put(Bucket, Key1, [], Value1, State), + timer:sleep(timer:seconds(1)), + %% Write Key2 to the datastore + {ok, State2} = put(Bucket, Key2, [], Value2, State1), + + [ + %% Key1 should be kicked out + ?_assertEqual({error, not_found, State2}, get(Bucket, Key1, State2)), + %% Key2 should still be present + ?_assertEqual({ok, Value2, State2}, get(Bucket, Key2, State2)) + ]. + +-ifdef(EQC). + +eqc_test_() -> + {spawn, + [{inorder, + [{setup, + fun setup/0, + fun cleanup/1, + [ + {timeout, 60000, + [?_assertEqual(true, + backend_eqc:test(?MODULE, true))]} + ]}]}]}. + +setup() -> + application:load(sasl), + application:set_env(sasl, sasl_error_logger, {file, "riak_kv_memory_backend_eqc_sasl.log"}), + error_logger:tty(false), + error_logger:logfile({open, "riak_kv_memory_backend_eqc.log"}), + ok. + +cleanup(_) -> + ok. -endif. % EQC + -endif. % TEST diff --git a/erl_src/riak_search_test_backend.beam b/erl_src/riak_search_test_backend.beam index ddef8d248ea4bc6c7e069a9c7442e48dcf4a9273..b20a76c9de3a21a0177d94e14ef95b5192d14fa3 100644 GIT binary patch delta 282 zcmZ4DbVG^5%|FPHfk7}}BgcIvMxV(qm|}Sa90XiBi&UYPI)KKb!2S{+3DpOUozU<2O}SY|dd| zU|`8$N?~BnFDS{(&npHAFo6V^lgm@G8H^^M7LZaaD9B0GFUm~J)=kdIOwB9NPc6!c zFD^<3sbkA8ElJ5Nf-053rF3$ifSNdGqEMn}qI#kzSYvU1X;E@&HiJLN4pFQcC;t>s O=45b8DlP#!iva*qB~XX} delta 4383 zcmV+)5#a9FBBVkHMo&^P001Ezu?XJ+0Wy>50$sCG1HA~791Iu&006Tz46*|dsxi*| zQ6wsIczJpvYpDsgllBiae+GkW00004XaZya2XAn6X>V?GYybcN2516g00wV$bYy9A zYXAggVrgz<019MfVs&RMk3WB?ClVQFk%Z*E_AVRCM1Zf9r$WB>taYXApnZewh9 zWMyOk4RU34b#iWBWpZ+Fa&u$=4{~L6b#iWBcVTjFX>Mn8YG?vvlf)1r7Y1PuVGdyv zVH0QqWB>+pZ*_8GWorOVlTHvC6k%&GWMy!3FLG&NYhQC^VRB<=FOw`0BY#0+b94Xz z5C4Nu002Npc$}46`)?dY5?*`#%=>|kLxA!WArLvv1EPdDTyVVDv6VQnF_GI*;zkO%n^m?tXN8{$V5RWgayLnIEmb!c-1pIT~*yPYy6Y9d%mi!cXjuSXIrh@ z-oRWi%;{Pfn8nbTKjY?dxqoA`ZC0+^48y>gu7|eSV&&ROj%&}h&gBl*g5v4GG3Sh$ zZ3e|zBeZLwF>My-Y|rYk1J`U(s5`A(N5!}5u5IQ1o<*t9bXmEsX?xb3cl^N09i&w0 z1Uq06d$H;Vp-^DwENIpq*fl#ebJ6cwXa=F#Dt@=sLca=9ifPBQ?0?00y2G+v8#u)S zu!n6{%#N6m>CJ#l5wYWye6v0JS@K=W%!{90PR+0@)o>|h+hzq~g|Z^!>rBQ7xhG!7 z08#61STpk~4K@Aij#qT+mQBsyGn-=ZU9eDK7r}I5ja?1HfeAuetlTBwyJCP*a%|9I z_iP>(GDE*&?grwj8GqPb$mY81V5*Yi*_KYd2a$Ucqq3=H#Cs9B7l@-tHbsc@D)Bx< z?z3_`0B3?74Zo_dvmc@RfvH!oKmZ1nnwai%ywIMp19Lxwi$-|>DGz|J1*PQ|IpKjc z!h=Y75TyIMZU;+-=LZ!=dN7UjH6(ovtWt%5stIT2J;ya)OMi~+DfV(9pu zQg-}m6fTgmNq>;V)Ml&djCZ&*Jl~44F0YI8E{gN6;tc+XI74Id9wOgsEQf4T9;UE3 zg{V_t3mR&tRP9bP?b>VbD^jmjXWNTxdsBVXo93f4NOeXL=8P)Lc_vIZWXYmaal)kx zdw+neKTzz=A2qZ0hlu=8k?%tteF4!IsD#W87p8N2KY#gHRqhf~ZfDeljGFU}Z3*?- z^|e02wLVhRb0tu}zj0mW(q&8=2IWFgaP zpCahfEPtzwFspS{{Nh~FLsBwy1UUj>XJhXovtUH!enatzsH%>&D4J;8r8Jpu;0ibB zhJvGRBI>5}9EXe|WHc)HFo}|m>IAot;Fi{RZ|V5kh`(*+b~~O6t}>K$ZtHYoNH+#_ zrf0fKkl9AbHD}nP_LAw=G>3i0OuCmgDhN5c&l$jkZBz7o;lnB~xlYRVrc>@9Z`9Un2W2MUOKpwr=k` z2)Y9?Yr2kEv(ZC$bhdYq;;z_JEmc#J>mGvc!LzDqS&lbjq-Ww+2>yzIi?e2+3v3|R zfPa*PO;OzmY>Umtv7ysWA!ovIQmFD1MjgfPV)= z-w+c`k$W3kEddrpF*vab09r=Nv0)U9(s1K1vJHuh`> zm?ok@n$!w?5AFUtTuayMm34aMY0dwOb(owk+RJju<=m|A2#Dr%rNahJS1vHBU zj%*$YC#aH%D}1k}oX2*GCSw^PYNTU2s`Y6Xs%I7ps0cO`bnsy)+t9mZlFBEQR6ZG} za+@qlVultat!kcR#Jng)>Yiji-GV+YRg^Y%DI|^(RqtELC7Tvh?{j+tvVVO{Xh4!L z_8LEv?y8te)3y=U2hQRYvQWh6!3#c3jvgM|0NpWE7@rs&9)Xx3A>2dl-9zo2vN7&K znR_&Q^+KWP=lX%McqC*l3)MCV$vyl#PbVdSy5x5#o$LY%dW(&dB~2+k#&Rv`lmNi>aaV zcF=`wp~ts{o{;4mOneq8&r*M$MK)*EVh0l)uJ6x*C0Kx-l?9Z{2*Wi@OFVmyWRj>m z=ZJ03#Wy1fk8>chkv-Zsun4gRO$pZNci~W2mX+#UlCXJkk|>$yHGeVA#{e04txyln z!wOgA3g=T+pbtgl$VKFhi|Ar>oeOjw3A`Y->_J_xD;KFJE+WB2)e|tWr(U%nI}NBh z726BB?JjB_x!4G0Y57aU@|UDP<+aZx6a9NB+JbC@G|xzf(KDBldYl;OvV7YRJ-@7( zq>q|}-{EkR^hI0mntvU?z(C>qs3{sk_i=0WVVWmnQYQKuWkN`v%>9FvWZD?7O4hMh zSEKE6)m>6Ctqju{^#YsR?_6ciJTt!J=HYDZx_643_!&4TO5=DJ$&u4i=P>ssxu@gLFiM=LK z-pTaZ*^qORmy$^)=OmMJlF2#Avoe^zVYy~M`l?NCHM*bYVHqcTSsWc*cft{_7! zS_ayJ-+4!+0%2wPQwldl-N^5~Q@ID;ya=$b{ES&_|EbHP*glQ91}4K%17da%uEUwUCwIZ_5ECFU?@%FNsn zFn@>A&e2?$!`0{1k~SwkaEnS>tc+I7u;b5V7qdCNn7JfFT(x$kPmp5fB5zmskegD> zwx(zlGYV@xenuIeRRQNXUxOUXD@1{c5(^cez0$|W$MLGG5{39Sc=cUw%4=X8%SQ#d zdW5T|-tyrNA!h4E`YYpeWn}D;p8_VYAb)(*!bd)_U^TmX1?0<0t$YQ1PZen8E6~bU zpl%R?)S49NH7UrvdL?;M%gFiKU&;AelbpwkfG{QJ>+)$mJ?H09+IgDu^SJuFn)CBp z<@~%{bv`@i=k=UlAknp;rg-`U$@vB3y`X3CLQ2kWO`&dJwxm!uFbh$xsS{^88-KDP zSww3r5?vS7t_6hjb$IKzl?5MrlGxzDlY4HRm<{YPJUIj-1Mqy zS@q8{S7Di*ftJ;m#EFyyfQI=ioPX$qw3U8?@GU>Geybf>|EZt9zvD6f9ay2UfTq9$ zYUT&X^}$vHqS{!akU@l(er>xLUQyW@_neqDUZpAI@gDKos5|h)%Xg$I6h>> zv4!$g8;-Kka9LfZl|JvfS%;u&=&zwN$ztiUA1mUPnS@!fV}Dgdos5Z?^0>f8VJFyVl#R$G(_^kF4Tx1$ z{8cp|R(U|IvVd4+;;*uRSmgn+$|PJ>17eliWi{Q}Ix5N{detlDk~du`ugjlE;1esi z^M<`3SCNH-L6R)x#($Es@Dp5^&Pg?>wokbpPZd$pfSHsmyiy()*nfzH*He`;&#)70 zG|EQgHR)3&rq)!M*VLF=<1w|yVrq@ayvAZ`jmOj)6KhS4sWonlH6^CvYQIK53{NQo z&p4B3>4Eu7)mbNFeoA>ghnc?6RzuAo^>!vqGd0g zg%g2ix@5PpPP^mJf(6xXC+J7#=MZQ7g^QlMGDUsDb?r&^(xQVAC-;p zk31{*4(Q(sA5``y<#_)SeMKD6{-oqjGMVwp_hA7){0%jTHySTxGTvNlNJ%!bLvKSK{25#Ep}VI3FIMs&u$t<> zaMgcNRhP>BitGH^^z*-8GrjPeLYE@WZ=Ba}>EZxz6D8ZEX5U0HH}zxbruffiyE2^| zOEuGP%B456O@F_sf5y>72JU(>0R(0JRQbD-JHK1>-7L8edr8-Ni3BgzwO+D!vn{%| zFQv9Gv$cJRHBQd`M?c{Hp@{#7mHY>QdOH4Z@ {ok, Ring} = riak_core_ring_manager:get_my_ring(), - [ ets:delete_all_objects(list_to_atom(integer_to_list(P))) || + [ ets:delete_all_objects(list_to_atom("rs" ++ integer_to_list(P))) || P <- riak_core_ring:my_indices(Ring) ], riak_search_config:clear(), ok. start(Partition, _Config) -> - Table = ets:new(list_to_atom(integer_to_list(Partition)), + Table = ets:new(list_to_atom("rs" ++ integer_to_list(Partition)), [named_table, public, ordered_set]), {ok, #state{partition=Partition, table=Table}}. From 2887c93177d818bf4c54baf787071d9beb6eaed1 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 18 Oct 2011 16:14:11 +0200 Subject: [PATCH 046/118] Make TestServer work again Pass search_backend and storage_backend as atoms Set platform_data_dir to point to the temp dir --- riak/test_server.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/riak/test_server.py b/riak/test_server.py index b15d343e..57025387 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -15,6 +15,16 @@ except NameError: bytes = str +class Atom(object): + def __init__(self, s): + self.str = s + + def __str__(self): + return str(self.str) + + def __repr__(self): + return repr(self.str) + def erlang_config(hash, depth=1): def printable(item): k, v = item @@ -55,7 +65,7 @@ class TestServer: "ring_creation_size": 64 }, "riak_kv": { - "storage_backend": bytes("riak_kv_test_backend"), + "storage_backend": Atom("riak_kv_test_backend"), "pb_ip": "127.0.0.1", "pb_port": 9002, "js_vm_count": 8, @@ -67,7 +77,7 @@ class TestServer: }, "riak_search": { "enabled": True, - "search_backend": bytes("riak_search_test_backend") + "search_backend": Atom("riak_search_test_backend") }, "luwak": { "enabled": True @@ -92,6 +102,7 @@ def __init__(self, tmp_dir="/tmp/riak/test_server", self.app_config[key] = deep_merge(self.app_config[key], value) self.app_config["riak_core"]["ring_state_dir"] = os.path.join(self.temp_dir, "data", "ring") + self.app_config["riak_core"]["platform_data_dir"] = self.temp_dir def prepare(self): if not self._prepared: From 17b85b93334e8038ff0920f56344b78362f701bb Mon Sep 17 00:00:00 2001 From: Ian Plosker Date: Tue, 18 Oct 2011 12:16:01 -0400 Subject: [PATCH 047/118] Fix issue #77 incorrectly formatted long headers. https://github.com/basho/riak-python-client/issues/77 --- riak/transports/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 456b91b5..f4159648 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -339,7 +339,7 @@ def add_links_for_riak_object(self, robject, headers): for link in links: header = self.to_link_header(link) if len(current_header + header) > MAX_LINK_HEADER_SIZE: - headers.setdefault('Link', []).append(current_header) + headers.add('Link', current_header) current_header = '' if current_header != '': header = ', ' + header From f0c6fc9bd8d151311276d795071a875daa42380b Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 19 Oct 2011 07:55:15 +0200 Subject: [PATCH 048/118] Fix TestServer.recycle() Adding the Atom class broke recycle(), because it depends on being able to compare the {storage,search}_backend settings to a couple of strings. Fix this by adding __cmp__ and __eq__ methods to the Atom class so these comparisons work again. --- riak/test_server.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/riak/test_server.py b/riak/test_server.py index 57025387..c60bf9ab 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -25,6 +25,12 @@ def __str__(self): def __repr__(self): return repr(self.str) + def __eq__(self, other): + return self.str == other + + def __cmp__(self, other): + return cmp(self.str, other) + def erlang_config(hash, depth=1): def printable(item): k, v = item From adaf3bdd7003701010c5a1398e654590a986d833 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 19 Oct 2011 13:50:51 +0200 Subject: [PATCH 049/118] Add a bunch of missing stuff to the tarball --- MANIFEST.in | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..e691aa45 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include docs/* +include erl_src/* +include THANKS +include README.rst +include LICENSE +include RELEASE_NOTES.md From bb25b0c0d27dca8082b78a8f3e60acd66b96d0b5 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sun, 4 Sep 2011 20:42:03 +0200 Subject: [PATCH 050/118] Move erl_src under riak/'s namespace. This gets erl_src installed by setup.py install which makes TestServer work outside of a SCM checkout. --- {erl_src => riak/erl_src}/riak_kv_test_backend.beam | Bin {erl_src => riak/erl_src}/riak_kv_test_backend.erl | 0 .../erl_src}/riak_search_test_backend.beam | Bin .../erl_src}/riak_search_test_backend.erl | 0 riak/test_server.py | 2 +- setup.py | 4 +++- 6 files changed, 4 insertions(+), 2 deletions(-) rename {erl_src => riak/erl_src}/riak_kv_test_backend.beam (100%) rename {erl_src => riak/erl_src}/riak_kv_test_backend.erl (100%) rename {erl_src => riak/erl_src}/riak_search_test_backend.beam (100%) rename {erl_src => riak/erl_src}/riak_search_test_backend.erl (100%) diff --git a/erl_src/riak_kv_test_backend.beam b/riak/erl_src/riak_kv_test_backend.beam similarity index 100% rename from erl_src/riak_kv_test_backend.beam rename to riak/erl_src/riak_kv_test_backend.beam diff --git a/erl_src/riak_kv_test_backend.erl b/riak/erl_src/riak_kv_test_backend.erl similarity index 100% rename from erl_src/riak_kv_test_backend.erl rename to riak/erl_src/riak_kv_test_backend.erl diff --git a/erl_src/riak_search_test_backend.beam b/riak/erl_src/riak_search_test_backend.beam similarity index 100% rename from erl_src/riak_search_test_backend.beam rename to riak/erl_src/riak_search_test_backend.beam diff --git a/erl_src/riak_search_test_backend.erl b/riak/erl_src/riak_search_test_backend.erl similarity index 100% rename from erl_src/riak_search_test_backend.erl rename to riak/erl_src/riak_search_test_backend.erl diff --git a/riak/test_server.py b/riak/test_server.py index b15d343e..403360fb 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -44,7 +44,7 @@ class TestServer: "-smp": "enable", "-env ERL_MAX_PORTS": 4096, "-env ERL_FULLSWEEP_AFTER": 10, - "-pa": os.path.abspath(os.path.join(os.path.dirname(__file__), "../erl_src")) + "-pa": os.path.abspath(os.path.join(os.path.dirname(__file__), "erl_src")) } APP_CONFIG_DEFAULTS = { diff --git a/setup.py b/setup.py index 7814cde2..ae14401b 100755 --- a/setup.py +++ b/setup.py @@ -23,10 +23,12 @@ def make_pb(): install_requires = ['protobuf>=2.3.0', 'urllib3>=0.4.0'], dependency_links = ["http://downloads.basho.com/support"], package_data = { - '' : ['*.proto'] + '' : ['*.proto'], + 'riak' : ['erl_src/*'] }, description='Python client for Riak', zip_safe=True, + include_package_data=True, license='Apache 2', platforms='Platform Independent', author='Basho Technologies', From 4031652267c3f1c6677a3cf5e41364ea3adb991e Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 20 Oct 2011 10:35:46 +0200 Subject: [PATCH 051/118] Temporary, simple fix for issue #32 https://issues.basho.com/show_bug.cgi?id=649 has more details. Eventually, we want to allow unicode keys and bucket names, but we have yet to decide how to do that. Until we figure that out, this patch will at least let us use the search interface as long as we only use ascii-encodable keys and bucket names. --- riak/bucket.py | 8 ++++++-- riak/mapreduce.py | 4 +++- riak/riak_object.py | 4 +++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index cef43271..69c044ee 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -38,7 +38,9 @@ def __init__(self, client, name): :param name: The bucket name :type name: string """ - if isinstance(name, unicode): + try: + name.encode('ascii') + except UnicodeEncodeError: raise TypeError('Unicode bucket names are not supported.') self._client = client @@ -210,7 +212,9 @@ def new(self, key, data=None, content_type='application/json'): :type data: object :rtype: :class:`RiakObject ` """ - if isinstance(data, unicode): + try: + data.encode('ascii') + except: raise TypeError('Unicode data values are not supported.') obj = RiakObject(self._client, self, key) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index d2b956d1..0a3ddf30 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -320,7 +320,9 @@ def __init__(self, type, function, language, keep, arg): @param mixed arg - Additional value to pass into the map or reduce function. """ - if isinstance(function, unicode): + try: + function.encode('ascii') + except UnicodeEncodeError: raise TypeError('Unicode encoded functions are not supported.') self._type = type diff --git a/riak/riak_object.py b/riak/riak_object.py index 7ad05032..7e99174b 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -39,7 +39,9 @@ def __init__(self, client, bucket, key=None): is generated by the server when :func:`store` is called. :type key: string """ - if isinstance(key, unicode): + try: + key.encode('ascii') + except UnicodeEncodeError: raise TypeError('Unicode keys are not supported.') self._client = client From a7ab12423a5671d338077bb8e72219beaea08829 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 20 Oct 2011 10:56:19 +0200 Subject: [PATCH 052/118] Only attempt encoding to ascii if it is a type of string. --- riak/bucket.py | 6 ++++-- riak/mapreduce.py | 3 ++- riak/riak_object.py | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 69c044ee..b1af10b7 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -39,7 +39,8 @@ def __init__(self, client, name): :type name: string """ try: - name.encode('ascii') + if isinstance(name, basestring): + name.encode('ascii') except UnicodeEncodeError: raise TypeError('Unicode bucket names are not supported.') @@ -213,7 +214,8 @@ def new(self, key, data=None, content_type='application/json'): :rtype: :class:`RiakObject ` """ try: - data.encode('ascii') + if isinstance(data, basestring): + data.encode('ascii') except: raise TypeError('Unicode data values are not supported.') diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 0a3ddf30..3b7ce54d 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -321,7 +321,8 @@ def __init__(self, type, function, language, keep, arg): reduce function. """ try: - function.encode('ascii') + if isinstance(function, basestring): + function.encode('ascii') except UnicodeEncodeError: raise TypeError('Unicode encoded functions are not supported.') diff --git a/riak/riak_object.py b/riak/riak_object.py index 7e99174b..6b03d217 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -40,7 +40,8 @@ def __init__(self, client, bucket, key=None): :type key: string """ try: - key.encode('ascii') + if isinstance(key, basestring): + key.encode('ascii') except UnicodeEncodeError: raise TypeError('Unicode keys are not supported.') From 09c1951d5a454b659a2e316dc1e93196ec55e71a Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 26 Oct 2011 16:14:17 -0400 Subject: [PATCH 053/118] Fix RiakPbcCachedTransport to use the new API. --- riak/transports/pbc.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 63239904..a7a56a43 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -558,6 +558,13 @@ def pbify_content(self, metadata, data, rpb_content) : import contextlib class RiakPbcCachedTransport(RiakTransport): """Threadsafe pool of PBC connections, based on urllib3's pool [aka Queue]""" + + # We're using the new RiakTransport API + api = 2 + + # The ConnectionManager class that this transport prefers. + default_cm = SocketConnectionManager + def __init__(self, cm, client_id=None, maxsize=0, block=False, timeout=None, **unused_options): @@ -566,6 +573,7 @@ def __init__(self, cm, ### backwards compat. we don't use the ConnectionManager (yet). host, port = cm.hostports[0] + self._cm = cm self.host = host self.port = port @@ -579,7 +587,7 @@ def __init__(self, cm, def _new_connection(self): """New PBC connection""" - return RiakPbcTransport(self.host, self.port, self.client_id) + return RiakPbcTransport(self._cm, self.client_id) def _get_connection(self): connection = None From 14d3dbc1bcffa44377f426121faaf4fbf13cdb3b Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Sun, 6 Nov 2011 18:05:55 -0500 Subject: [PATCH 054/118] Collapse the usage of .maybe_connect() into the send methods. --- riak/transports/pbc.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index a7a56a43..22444d29 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -125,7 +125,6 @@ def ping(self): Ping the remote server @return boolean """ - self.maybe_connect() self.send_msg_code(MSG_CODE_PING_REQ) msg_code, msg = self.recv_msg() if msg_code == MSG_CODE_PING_RESP: @@ -137,7 +136,6 @@ def get_client_id(self): """ Get the client id used by this connection """ - self.maybe_connect() self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ) msg_code, resp = self.recv_msg() if msg_code == MSG_CODE_GET_CLIENT_ID_RESP: @@ -152,7 +150,6 @@ def set_client_id(self, client_id): req = riakclient_pb2.RpbSetClientIdReq() req.client_id = client_id - self.maybe_connect() self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req) msg_code, resp = self.recv_msg() if msg_code == MSG_CODE_SET_CLIENT_ID_RESP: @@ -175,7 +172,6 @@ def get(self, robj, r = None, vtag = None): req.bucket = bucket.get_name() req.key = robj.get_key() - self.maybe_connect() self.send_msg(MSG_CODE_GET_REQ, req) msg_code, resp = self.recv_msg() if msg_code == MSG_CODE_GET_RESP: @@ -206,7 +202,6 @@ def put(self, robj, w = None, dw = None, return_body = True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - self.maybe_connect() self.send_msg(MSG_CODE_PUT_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_PUT_RESP: @@ -237,7 +232,6 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - self.maybe_connect() self.send_msg(MSG_CODE_PUT_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_PUT_RESP: @@ -262,7 +256,6 @@ def delete(self, robj, rw = None): req.bucket = bucket.get_name() req.key = robj.get_key() - self.maybe_connect() self.send_msg(MSG_CODE_DEL_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_DEL_RESP: @@ -276,7 +269,6 @@ def get_keys(self, bucket): req = riakclient_pb2.RpbListKeysReq() req.bucket = bucket.get_name() - self.maybe_connect() self.send_msg(MSG_CODE_LIST_KEYS_REQ, req) keys = [] while True: @@ -296,7 +288,6 @@ def get_buckets(self): """ Serialize bucket listing request and deserialize response """ - self.maybe_connect() self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_LIST_BUCKETS_RESP: @@ -310,7 +301,6 @@ def get_bucket_props(self, bucket): req = riakclient_pb2.RpbGetBucketReq() req.bucket = bucket.get_name() - self.maybe_connect() self.send_msg(MSG_CODE_GET_BUCKET_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_GET_BUCKET_RESP: @@ -337,7 +327,6 @@ def set_bucket_props(self, bucket, props): if 'allow_mult' in props: req.props.allow_mult = props['allow_mult'] - self.maybe_connect() self.send_msg(MSG_CODE_SET_BUCKET_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_SET_BUCKET_RESP: @@ -357,7 +346,6 @@ def mapred(self, inputs, query, timeout=None): req.request = content req.content_type = "application/json" - self.maybe_connect() self.send_msg(MSG_CODE_MAPRED_REQ, req) # dictionary of phase results - each content should be an encoded array @@ -401,6 +389,7 @@ def maybe_connect(self): self.set_client_id(self._client_id) def send_msg_code(self, msg_code): + self.maybe_connect() pkt = struct.pack("!iB", 1, msg_code) self._sock.send(pkt) @@ -411,6 +400,7 @@ def encode_msg(self, msg_code, msg): return hdr + str def send_msg(self, msg_code, msg): + self.maybe_connect() pkt = self.encode_msg(msg_code, msg) sent_len = self._sock.send(pkt) if sent_len != len(pkt): From 2b03743aaf9766e63034bf2047e0460abeba3db6 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Sun, 6 Nov 2011 18:16:29 -0500 Subject: [PATCH 055/118] Fold recv_msg() into the send_msg_code() method. --- riak/transports/pbc.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 22444d29..f833f7b4 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -125,8 +125,7 @@ def ping(self): Ping the remote server @return boolean """ - self.send_msg_code(MSG_CODE_PING_REQ) - msg_code, msg = self.recv_msg() + msg_code, msg = self.send_msg_code(MSG_CODE_PING_REQ) if msg_code == MSG_CODE_PING_RESP: return 1 else: @@ -136,8 +135,7 @@ def get_client_id(self): """ Get the client id used by this connection """ - self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ) if msg_code == MSG_CODE_GET_CLIENT_ID_RESP: return resp.client_id else: @@ -288,8 +286,7 @@ def get_buckets(self): """ Serialize bucket listing request and deserialize response """ - self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ) if msg_code != MSG_CODE_LIST_BUCKETS_RESP: raise RiakError("unexpected protocol buffer message code: %d"%msg_code) return resp.buckets @@ -392,6 +389,7 @@ def send_msg_code(self, msg_code): self.maybe_connect() pkt = struct.pack("!iB", 1, msg_code) self._sock.send(pkt) + return self.recv_msg() def encode_msg(self, msg_code, msg): str = msg.SerializeToString() From 65b5dafae02c960c605028e5f2d6bf001ca36d33 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Sun, 6 Nov 2011 18:27:39 -0500 Subject: [PATCH 056/118] Shuffle recv_msg() into the return value for send_msg(). Additionally, to support the methods where multiple recv_msg() calls are made (multiple responses), a new method is introduced which does the upfront request in preparation for the following receives. --- riak/transports/pbc.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index f833f7b4..9d36ce9a 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -148,8 +148,7 @@ def set_client_id(self, client_id): req = riakclient_pb2.RpbSetClientIdReq() req.client_id = client_id - self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req) if msg_code == MSG_CODE_SET_CLIENT_ID_RESP: return True else: @@ -170,8 +169,7 @@ def get(self, robj, r = None, vtag = None): req.bucket = bucket.get_name() req.key = robj.get_key() - self.send_msg(MSG_CODE_GET_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_GET_REQ, req) if msg_code == MSG_CODE_GET_RESP: contents = [] for c in resp.content: @@ -200,8 +198,7 @@ def put(self, robj, w = None, dw = None, return_body = True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - self.send_msg(MSG_CODE_PUT_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req) if msg_code != MSG_CODE_PUT_RESP: raise RiakError("unexpected protocol buffer message code: %d"%msg_code) if resp is not None: @@ -230,8 +227,7 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - self.send_msg(MSG_CODE_PUT_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req) if msg_code != MSG_CODE_PUT_RESP: raise RiakError("unexpected protocol buffer message code: %d"%msg_code) if not resp: @@ -254,8 +250,7 @@ def delete(self, robj, rw = None): req.bucket = bucket.get_name() req.key = robj.get_key() - self.send_msg(MSG_CODE_DEL_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_DEL_REQ, req) if msg_code != MSG_CODE_DEL_RESP: raise RiakError("unexpected protocol buffer message code: %d"%msg_code) return self @@ -267,7 +262,7 @@ def get_keys(self, bucket): req = riakclient_pb2.RpbListKeysReq() req.bucket = bucket.get_name() - self.send_msg(MSG_CODE_LIST_KEYS_REQ, req) + self.send_msg_multi(MSG_CODE_LIST_KEYS_REQ, req) keys = [] while True: msg_code, resp = self.recv_msg() @@ -298,8 +293,7 @@ def get_bucket_props(self, bucket): req = riakclient_pb2.RpbGetBucketReq() req.bucket = bucket.get_name() - self.send_msg(MSG_CODE_GET_BUCKET_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_GET_BUCKET_REQ, req) if msg_code != MSG_CODE_GET_BUCKET_RESP: raise RiakError("unexpected protocol buffer message code: %d"%msg_code) props = {} @@ -324,8 +318,7 @@ def set_bucket_props(self, bucket, props): if 'allow_mult' in props: req.props.allow_mult = props['allow_mult'] - self.send_msg(MSG_CODE_SET_BUCKET_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_SET_BUCKET_REQ, req) if msg_code != MSG_CODE_SET_BUCKET_RESP: raise RiakError("unexpected protocol buffer message code: %d"%msg_code) @@ -343,7 +336,7 @@ def mapred(self, inputs, query, timeout=None): req.request = content req.content_type = "application/json" - self.send_msg(MSG_CODE_MAPRED_REQ, req) + self.send_msg_multi(MSG_CODE_MAPRED_REQ, req) # dictionary of phase results - each content should be an encoded array # which is appended to the result for that phase. @@ -398,6 +391,10 @@ def encode_msg(self, msg_code, msg): return hdr + str def send_msg(self, msg_code, msg): + self.send_msg_multi(msg_code, msg) + return self.recv_msg() + + def send_msg_multi(self, msg_code, msg): self.maybe_connect() pkt = self.encode_msg(msg_code, msg) sent_len = self._sock.send(pkt) From 7a07e0328d88c1916d0a6217cf01c99877cf9275 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Sun, 6 Nov 2011 18:33:01 -0500 Subject: [PATCH 057/118] Fix the __copy__ method. Pass the ConnectionManager to the copy. --- riak/transports/pbc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 9d36ce9a..0c726da8 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -106,6 +106,7 @@ def __init__(self, cm, client_id=None, **unused_options): ### backwards compat. we don't use the ConnectionManager (yet). host, port = cm.hostports[0] + self._cm = cm self._host = host self._port = port self._client_id = client_id @@ -118,7 +119,7 @@ def translate_rw_val(self, rw): return val def __copy__(self): - return RiakPbcTransport(self._host, self._port) + return RiakPbcTransport(self._cm, self._client_id) def ping(self): """ From 9ef7b2aae153aa1f5eeea13f2ee4a36c0e74508b Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 9 Nov 2011 14:28:27 -0500 Subject: [PATCH 058/118] Consolidate checking of the response's message code. --- riak/transports/pbc.py | 74 ++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 42 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 0c726da8..e050135f 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -126,7 +126,8 @@ def ping(self): Ping the remote server @return boolean """ - msg_code, msg = self.send_msg_code(MSG_CODE_PING_REQ) + # An expected response code of None implies "any response is valid". + msg_code, msg = self.send_msg_code(MSG_CODE_PING_REQ, None) if msg_code == MSG_CODE_PING_RESP: return 1 else: @@ -136,11 +137,9 @@ def get_client_id(self): """ Get the client id used by this connection """ - msg_code, resp = self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ) - if msg_code == MSG_CODE_GET_CLIENT_ID_RESP: - return resp.client_id - else: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ, + MSG_CODE_GET_CLIENT_ID_RESP) + return resp.client_id def set_client_id(self, client_id): """ @@ -149,11 +148,9 @@ def set_client_id(self, client_id): req = riakclient_pb2.RpbSetClientIdReq() req.client_id = client_id - msg_code, resp = self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req) - if msg_code == MSG_CODE_SET_CLIENT_ID_RESP: - return True - else: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req, + MSG_CODE_SET_CLIENT_ID_RESP) + return True def get(self, robj, r = None, vtag = None): """ @@ -170,7 +167,8 @@ def get(self, robj, r = None, vtag = None): req.bucket = bucket.get_name() req.key = robj.get_key() - msg_code, resp = self.send_msg(MSG_CODE_GET_REQ, req) + # An expected response code of None implies "any response is valid". + msg_code, resp = self.send_msg(MSG_CODE_GET_REQ, req, None) if msg_code == MSG_CODE_GET_RESP: contents = [] for c in resp.content: @@ -199,9 +197,8 @@ def put(self, robj, w = None, dw = None, return_body = True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req) - if msg_code != MSG_CODE_PUT_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req, + MSG_CODE_PUT_RESP) if resp is not None: contents = [] for c in resp.content: @@ -228,9 +225,8 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req) - if msg_code != MSG_CODE_PUT_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req, + MSG_CODE_PUT_RESP) if not resp: raise RiakError("missing response object") if len(resp.content) != 1: @@ -251,9 +247,8 @@ def delete(self, robj, rw = None): req.bucket = bucket.get_name() req.key = robj.get_key() - msg_code, resp = self.send_msg(MSG_CODE_DEL_REQ, req) - if msg_code != MSG_CODE_DEL_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_DEL_REQ, req, + MSG_CODE_DEL_RESP) return self def get_keys(self, bucket): @@ -266,9 +261,7 @@ def get_keys(self, bucket): self.send_msg_multi(MSG_CODE_LIST_KEYS_REQ, req) keys = [] while True: - msg_code, resp = self.recv_msg() - if msg_code != MSG_CODE_LIST_KEYS_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.recv_msg(MSG_CODE_LIST_KEYS_RESP) for key in resp.keys: keys.append(key) @@ -282,9 +275,8 @@ def get_buckets(self): """ Serialize bucket listing request and deserialize response """ - msg_code, resp = self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ) - if msg_code != MSG_CODE_LIST_BUCKETS_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ, + MSG_CODE_LIST_BUCKETS_RESP) return resp.buckets def get_bucket_props(self, bucket): @@ -294,9 +286,8 @@ def get_bucket_props(self, bucket): req = riakclient_pb2.RpbGetBucketReq() req.bucket = bucket.get_name() - msg_code, resp = self.send_msg(MSG_CODE_GET_BUCKET_REQ, req) - if msg_code != MSG_CODE_GET_BUCKET_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_GET_BUCKET_REQ, req, + MSG_CODE_GET_BUCKET_RESP) props = {} if resp.props.HasField('n_val'): props['n_val'] = resp.props.n_val @@ -319,10 +310,8 @@ def set_bucket_props(self, bucket, props): if 'allow_mult' in props: req.props.allow_mult = props['allow_mult'] - msg_code, resp = self.send_msg(MSG_CODE_SET_BUCKET_REQ, req) - if msg_code != MSG_CODE_SET_BUCKET_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) - + msg_code, resp = self.send_msg(MSG_CODE_SET_BUCKET_REQ, req, + MSG_CODE_SET_BUCKET_RESP) return self def mapred(self, inputs, query, timeout=None): @@ -343,9 +332,7 @@ def mapred(self, inputs, query, timeout=None): # which is appended to the result for that phase. result = {} while True: - msg_code, resp = self.recv_msg() - if msg_code != MSG_CODE_MAPRED_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.recv_msg(MSG_CODE_MAPRED_RESP) if resp.HasField("phase") and resp.HasField("response"): content = json.loads(resp.response) if resp.phase in result: @@ -379,11 +366,11 @@ def maybe_connect(self): if self._client_id: self.set_client_id(self._client_id) - def send_msg_code(self, msg_code): + def send_msg_code(self, msg_code, expect): self.maybe_connect() pkt = struct.pack("!iB", 1, msg_code) self._sock.send(pkt) - return self.recv_msg() + return self.recv_msg(expect) def encode_msg(self, msg_code, msg): str = msg.SerializeToString() @@ -391,9 +378,9 @@ def encode_msg(self, msg_code, msg): hdr = struct.pack("!iB", 1 + slen, msg_code) return hdr + str - def send_msg(self, msg_code, msg): + def send_msg(self, msg_code, msg, expect): self.send_msg_multi(msg_code, msg) - return self.recv_msg() + return self.recv_msg(expect) def send_msg_multi(self, msg_code, msg): self.maybe_connect() @@ -403,7 +390,7 @@ def send_msg_multi(self, msg_code, msg): raise RiakError("PB socket returned short write %d - expected %d"%\ (sent_len, len(pkt))) - def recv_msg(self): + def recv_msg(self, expect): self.recv_pkt() msg_code, = struct.unpack("B", self._inbuf[:1]) if msg_code == MSG_CODE_ERROR_RESP: @@ -441,6 +428,9 @@ def recv_msg(self): msg.ParseFromString(self._inbuf[1:]) else: raise Exception("unknown msg code %s"%msg_code) + if expect and msg_code != expect: + raise RiakError("unexpected protocol buffer message code: %d" + % msg_code) return msg_code, msg From 240476c2c755c7e444c58d6c72ff7a3886939459 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 9 Nov 2011 14:45:50 -0500 Subject: [PATCH 059/118] Move reading responses into send_msg_multi. This also required rejiggering the three send functions a bit to rely on a new .send_pkt() which also consolidates connection creation and writing to the socket. --- riak/transports/pbc.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index e050135f..916df890 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -258,16 +258,12 @@ def get_keys(self, bucket): req = riakclient_pb2.RpbListKeysReq() req.bucket = bucket.get_name() - self.send_msg_multi(MSG_CODE_LIST_KEYS_REQ, req) keys = [] - while True: - msg_code, resp = self.recv_msg(MSG_CODE_LIST_KEYS_RESP) - + def _handle_response(resp): for key in resp.keys: keys.append(key) - - if resp.HasField("done") and resp.done: - break + self.send_msg_multi(MSG_CODE_LIST_KEYS_REQ, req, + MSG_CODE_LIST_KEYS_RESP, _handle_response) return keys @@ -326,22 +322,18 @@ def mapred(self, inputs, query, timeout=None): req.request = content req.content_type = "application/json" - self.send_msg_multi(MSG_CODE_MAPRED_REQ, req) - # dictionary of phase results - each content should be an encoded array # which is appended to the result for that phase. result = {} - while True: - msg_code, resp = self.recv_msg(MSG_CODE_MAPRED_RESP) + def _handle_response(resp): if resp.HasField("phase") and resp.HasField("response"): content = json.loads(resp.response) if resp.phase in result: result[resp.phase] += content else: result[resp.phase] = content - - if resp.HasField("done") and resp.done: - break; + self.send_msg_multi(MSG_CODE_MAPRED_REQ, req, MSG_CODE_MAPRED_RESP, + _handle_response) # If a single result - return the same as the HTTP interface does # otherwise return all the phase information @@ -367,9 +359,7 @@ def maybe_connect(self): self.set_client_id(self._client_id) def send_msg_code(self, msg_code, expect): - self.maybe_connect() - pkt = struct.pack("!iB", 1, msg_code) - self._sock.send(pkt) + self.send_pkt(struct.pack("!iB", 1, msg_code)) return self.recv_msg(expect) def encode_msg(self, msg_code, msg): @@ -379,12 +369,19 @@ def encode_msg(self, msg_code, msg): return hdr + str def send_msg(self, msg_code, msg, expect): - self.send_msg_multi(msg_code, msg) + self.send_pkt(self.encode_msg(msg_code, msg)) return self.recv_msg(expect) - def send_msg_multi(self, msg_code, msg): + def send_msg_multi(self, msg_code, msg, expect, handler): + self.send_pkt(self.encode_msg(msg_code, msg)) + while True: + msg_code, resp = self.recv_msg(expect) + handler(resp) + if resp.HasField("done") and resp.done: + break + + def send_pkt(self, pkt): self.maybe_connect() - pkt = self.encode_msg(msg_code, msg) sent_len = self._sock.send(pkt) if sent_len != len(pkt): raise RiakError("PB socket returned short write %d - expected %d"%\ From a7c05b01d013d17aeaf104b671aa2ed7ba021c97 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 9 Nov 2011 16:41:53 -0500 Subject: [PATCH 060/118] Create a new FactoryConnectionManager to simplify CM setup. This change makes it possible to avoid a CM subclass just to specify the connection class. --- riak/transports/connection.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/riak/transports/connection.py b/riak/transports/connection.py index 75669c6e..44dfad53 100644 --- a/riak/transports/connection.py +++ b/riak/transports/connection.py @@ -5,6 +5,7 @@ import httplib import socket import contextlib +import functools class ConnectionManager(object): @@ -132,10 +133,6 @@ def _new_connection(self): return conn -class HTTPConnectionManager(ConnectionManager): - connection_class = httplib.HTTPConnection - - class Socket(object): def __init__(self, host, port): @@ -160,8 +157,18 @@ def close(self): self.sock = None -class SocketConnectionManager(ConnectionManager): - connection_class = Socket +class FactoryConnectionManager(ConnectionManager): + + def __init__(self, connection_class, hostports=[]): + self.connection_class = connection_class + ConnectionManager.__init__(self, hostports) + + +def cm_using(connection_class): + return functools.partial(FactoryConnectionManager, connection_class) + +HTTPConnectionManager = cm_using(httplib.HTTPConnection) +SocketConnectionManager = cm_using(Socket) class NoHostsDefined(Exception): From 07226cd050a7d93d9cb97e1660d9d0cd6ac88f84 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 9 Nov 2011 17:43:18 -0500 Subject: [PATCH 061/118] Make the protobuf transport use the ConnectionManager This uses a subclass of the generic connection.Socket since we need to record the connection-stateful value for client_id. There are certainly issues around using variant client_id values in a pooled environment (as noted by the old CachedTransport, and documented within the set_client_id method here). The send/recv sections now use cm.withconn() to acquire and use a connection, which is then passed down into the methods which do the actual send/recv of the bits. This also switches over to sock.sendall() to try and avoid short writes. The host/port has been stripped out, since that is now managed within the connection manager. --- riak/transports/pbc.py | 122 +++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 53 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 916df890..e70aa9c2 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -26,12 +26,12 @@ except ImportError: import simplejson as json -from transport import RiakTransport +from riak.transports.transport import RiakTransport from riak.metadata import * from riak.mapreduce import RiakMapReduce, RiakLink from riak import RiakError from riak.riak_index_entry import RiakIndexEntry -from connection import SocketConnectionManager +from riak.transports import connection try: import riakclient_pb2 @@ -71,6 +71,24 @@ RIAKC_RW_DEFAULT = 4294967291 +class SocketWithId(connection.Socket): + def __init__(self, host, port): + connection.Socket.__init__(self, host, port) + self.last_client_id = None + + def maybe_connect(self): + # If we're going to establish a new connection, then reset the last + # client_id used on this connection. + if self.sock is None: + self.last_client_id = None + + connection.Socket.maybe_connect(self) + + def send(self, pkt): + self.sock.sendall(pkt) + + def recv(self, want_len): + return self.sock.recv(want_len) class RiakPbcTransport(RiakTransport): @@ -90,27 +108,19 @@ class RiakPbcTransport(RiakTransport): } # The ConnectionManager class that this transport prefers. - default_cm = SocketConnectionManager + default_cm = connection.cm_using(SocketWithId) def __init__(self, cm, client_id=None, **unused_options): """ Construct a new RiakPbcTransport object. - @param string host - Hostname or IP address (default '127.0.0.1') - @param int port - Port number (default 8087) """ if riakclient_pb2 is None: raise RiakError("this transport is not available (no protobuf)") super(RiakPbcTransport, self).__init__() - ### backwards compat. we don't use the ConnectionManager (yet). - host, port = cm.hostports[0] - self._cm = cm - self._host = host - self._port = port self._client_id = client_id - self._sock = None def translate_rw_val(self, rw): val = self.rw_names.get(rw) @@ -150,6 +160,18 @@ def set_client_id(self, client_id): msg_code, resp = self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req, MSG_CODE_SET_CLIENT_ID_RESP) + + # Using different client_id values across connections is a bad idea + # since you never know which connection you might use for a given + # API call. Setting the client_id manually (rather than as part of + # the transport construction) can be error-prone since the connection + # could drop and be reinstated using self._client_id. + # + # To minimize the potential impact of variant client_id values across + # connections, we'll store this new client_id and use it for all + # future connections. + self._client_id = client_id + return True def get(self, robj, r = None, vtag = None): @@ -344,23 +366,10 @@ def _handle_response(resp): else: return result - - def maybe_connect(self): - if self._sock is None: - self._sock = s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - try: - s.connect((self._host, self._port)) - except: - self._sock = None - raise - - if self._client_id: - self.set_client_id(self._client_id) - def send_msg_code(self, msg_code, expect): - self.send_pkt(struct.pack("!iB", 1, msg_code)) - return self.recv_msg(expect) + with self._cm.withconn() as conn: + self.send_pkt(conn, struct.pack("!iB", 1, msg_code)) + return self.recv_msg(conn, expect) def encode_msg(self, msg_code, msg): str = msg.SerializeToString() @@ -369,26 +378,37 @@ def encode_msg(self, msg_code, msg): return hdr + str def send_msg(self, msg_code, msg, expect): - self.send_pkt(self.encode_msg(msg_code, msg)) - return self.recv_msg(expect) + with self._cm.withconn() as conn: + self.send_pkt(conn, self.encode_msg(msg_code, msg)) + if msg_code == MSG_CODE_SET_CLIENT_ID_REQ: + conn.last_client_id = self._client_id + return self.recv_msg(conn, expect) def send_msg_multi(self, msg_code, msg, expect, handler): - self.send_pkt(self.encode_msg(msg_code, msg)) - while True: - msg_code, resp = self.recv_msg(expect) - handler(resp) - if resp.HasField("done") and resp.done: - break - - def send_pkt(self, pkt): - self.maybe_connect() - sent_len = self._sock.send(pkt) - if sent_len != len(pkt): - raise RiakError("PB socket returned short write %d - expected %d"%\ - (sent_len, len(pkt))) - - def recv_msg(self, expect): - self.recv_pkt() + with self._cm.withconn() as conn: + self.send_pkt(conn, self.encode_msg(msg_code, msg)) + while True: + msg_code, resp = self.recv_msg(conn, expect) + handler(resp) + if resp.HasField("done") and resp.done: + break + + def send_pkt(self, conn, pkt): + conn.maybe_connect() + + # If the last client_id used on this connection is different than our + # client_id, then set a new ID on the connection. + if conn.last_client_id != self._client_id: + req = riakclient_pb2.RpbSetClientIdReq() + req.client_id = self._client_id + conn.send(self.encode_msg(MSG_CODE_SET_CLIENT_ID_REQ, req)) + conn.last_client_id = self._client_id + self.recv_msg(conn, MSG_CODE_SET_CLIENT_ID_RESP) + + conn.send(pkt) + + def recv_msg(self, conn, expect): + self.recv_pkt(conn) msg_code, = struct.unpack("B", self._inbuf[:1]) if msg_code == MSG_CODE_ERROR_RESP: msg = riakclient_pb2.RpbErrorResp() @@ -431,10 +451,9 @@ def recv_msg(self, expect): return msg_code, msg - def recv_pkt(self): - nmsglen = self._sock.recv(4) + def recv_pkt(self, conn): + nmsglen = conn.recv(4) if len(nmsglen) != 4: - self._sock = None raise RiakError("Socket returned short packet length %d - expected 4"%\ len(nmsglen)) msglen, = struct.unpack('!i', nmsglen) @@ -442,7 +461,7 @@ def recv_pkt(self): self._inbuf = '' while len(self._inbuf) < msglen: want_len = min(8192, msglen - len(self._inbuf)) - recv_buf = self._sock.recv(want_len) + recv_buf = conn.recv(want_len) if not recv_buf: break self._inbuf += recv_buf if len(self._inbuf) != self._inbuf_len: @@ -536,7 +555,7 @@ class RiakPbcCachedTransport(RiakTransport): api = 2 # The ConnectionManager class that this transport prefers. - default_cm = SocketConnectionManager + default_cm = connection.cm_using(SocketWithId) def __init__(self, cm, client_id=None, maxsize=0, block=False, timeout=None, @@ -545,11 +564,8 @@ def __init__(self, cm, raise RiakError("this transport is not available (no protobuf)") ### backwards compat. we don't use the ConnectionManager (yet). - host, port = cm.hostports[0] self._cm = cm - self.host = host - self.port = port self.client_id = client_id self.block = block self.timeout = timeout From 215f3c67e916883dcbeec791cf63347af90ad86b Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 9 Nov 2011 18:14:07 -0500 Subject: [PATCH 062/118] Add license headers. --- riak/transports/connection.py | 20 +++++++++++++++++--- riak/transports/monitor.py | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/riak/transports/connection.py b/riak/transports/connection.py index 75669c6e..6aa14029 100644 --- a/riak/transports/connection.py +++ b/riak/transports/connection.py @@ -1,6 +1,20 @@ -# -# ### docco -# +""" +Copyright 2011 Greg Stein + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" import httplib import socket diff --git a/riak/transports/monitor.py b/riak/transports/monitor.py index 6380eee1..c6223518 100644 --- a/riak/transports/monitor.py +++ b/riak/transports/monitor.py @@ -1,3 +1,21 @@ +""" +Copyright 2011 Greg Stein + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + import threading import time From c2252b2209c3bf4fbf7e2702ac082e916601d438 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 17 Nov 2011 16:10:50 +0100 Subject: [PATCH 063/118] Only catch UnicodeEncodeError. --- riak/bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/bucket.py b/riak/bucket.py index b1af10b7..b556c9db 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -216,7 +216,7 @@ def new(self, key, data=None, content_type='application/json'): try: if isinstance(data, basestring): data.encode('ascii') - except: + except UnicodeEncodeError: raise TypeError('Unicode data values are not supported.') obj = RiakObject(self._client, self, key) From 69c1cb0b5f40a31a6d2fc8b584c68a60578d5765 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 18 Nov 2011 16:38:24 +0100 Subject: [PATCH 064/118] Unit tests Adjust unit tests so that they verify the patch, and then adjust the patch so that the unit tests pass :) --- riak/bucket.py | 8 ++++---- riak/mapreduce.py | 4 ++-- riak/riak_object.py | 4 ++-- riak/tests/test_all.py | 27 +++++++++++++++++++++------ 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index b556c9db..83818fa6 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -40,8 +40,8 @@ def __init__(self, client, name): """ try: if isinstance(name, basestring): - name.encode('ascii') - except UnicodeEncodeError: + name = name.encode('ascii') + except UnicodeError: raise TypeError('Unicode bucket names are not supported.') self._client = client @@ -215,8 +215,8 @@ def new(self, key, data=None, content_type='application/json'): """ try: if isinstance(data, basestring): - data.encode('ascii') - except UnicodeEncodeError: + data = data.encode('ascii') + except UnicodeError: raise TypeError('Unicode data values are not supported.') obj = RiakObject(self._client, self, key) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 3b7ce54d..fa7d4f2b 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -322,8 +322,8 @@ def __init__(self, type, function, language, keep, arg): """ try: if isinstance(function, basestring): - function.encode('ascii') - except UnicodeEncodeError: + function = function.encode('ascii') + except UnicodeError: raise TypeError('Unicode encoded functions are not supported.') self._type = type diff --git a/riak/riak_object.py b/riak/riak_object.py index 6b03d217..f91fb318 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -41,8 +41,8 @@ def __init__(self, client, bucket, key=None): """ try: if isinstance(key, basestring): - key.encode('ascii') - except UnicodeEncodeError: + key = key.encode('ascii') + except UnicodeError: raise TypeError('Unicode keys are not supported.') self._client = client diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6377b685..1e3e7864 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -115,12 +115,20 @@ def test_store_and_get(self): self.assertEqual(obj.get_key(), 'foo') self.assertEqual(obj.get_data(), rand) - #unicode input should raise a TypeError, - #to avoid issues further down the line - self.assertRaises(TypeError, self.client.bucket, u'bucket') + # unicode objects are fine, as long as they don't + # contain any non-ASCII chars + self.client.bucket(u'bucket') + self.assertRaises(TypeError, self.client.bucket, u'búcket') + self.assertRaises(TypeError, self.client.bucket, 'búcket') + + bucket.get(u'foo') + self.assertRaises(TypeError, bucket.get, u'føø') + self.assertRaises(TypeError, bucket.get, 'føø') + self.assertRaises(TypeError, bucket.new, u'foo', 'éå') + self.assertRaises(TypeError, bucket.new, u'foo', 'éå') + self.assertRaises(TypeError, bucket.new, 'foo', u'éå') self.assertRaises(TypeError, bucket.new, 'foo', u'éå') - self.assertRaises(TypeError, bucket.get, u'foo') def test_binary_store_and_get(self): bucket = self.client.bucket('bucket') @@ -275,9 +283,16 @@ def test_javascript_source_map(self): "function (v) { return [JSON.parse(v.values[0].data)]; }").run() self.assertEqual(result, [2]) - #test unicode function + # test ASCII-encodable unicode is accepted + mr.map(u"function (v) { return [JSON.parse(v.values[0].data)]; }") + + # test non-ASCII-encodable unicode is rejected + self.assertRaises(TypeError, mr.map, + u"function (v) { /* æ */ return [JSON.parse(v.values[0].data)]; }") + + # test non-ASCII-encodable string is rejected self.assertRaises(TypeError, mr.map, - u"function (v) { return [JSON.parse(v.values[0].data)]; }") + "function (v) { /* æ */ return [JSON.parse(v.values[0].data)]; }") def test_javascript_named_map(self): # Create the object... From bd74b800c2717601396322acd58b83cbadeed2ad Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Mon, 21 Nov 2011 10:36:13 +0000 Subject: [PATCH 065/118] Merge pull request #89 from gstein/deprecate Deprecate old transports These transports are obsoleted by the new ConnectionManager. --- riak/tests/test_all.py | 79 +--------------------- riak/transports/http.py | 108 +++--------------------------- riak/transports/pbc.py | 144 ++-------------------------------------- setup.py | 4 +- 4 files changed, 23 insertions(+), 312 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 8d8bee3c..439c5b29 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -19,8 +19,8 @@ import time from riak import RiakClient -from riak import RiakPbcTransport, RiakPbcCachedTransport -from riak import RiakHttpTransport, RiakHttpPoolTransport, RiakHttpReuseTransport +from riak import RiakPbcTransport +from riak import RiakHttpTransport from riak import RiakKeyFilter, key_filter from riak.riak_index_entry import RiakIndexEntry from riak.mapreduce import RiakLink @@ -31,11 +31,6 @@ HAVE_PROTO = True except ImportError: HAVE_PROTO = False -try: - import urllib3 - HAVE_HTTP_POOL = True -except ImportError: - HAVE_HTTP_POOL = False HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) @@ -872,26 +867,6 @@ def test_uses_client_id_if_given(self): self.assertEqual(zero_client_id, c.get_client_id()) # -class RiakPbcCachedTransportCase(BaseTestCase, MapReduceAliasTestMixIn, - unittest.TestCase): - def setUp(self): - if not HAVE_PROTO: - self.skipTest('protobuf is unavailable') - self.host = PB_HOST - self.port = PB_PORT - self.transport_class = RiakPbcCachedTransport - super(RiakPbcCachedTransportCase, self).setUp() - - def test_uses_client_id_if_given(self): - self.host = PB_HOST - self.port = PB_PORT - zero_client_id = "\0\0\0\0" - c = RiakClient(PB_HOST, PB_PORT, - transport_class = RiakPbcCachedTransport, - client_id = zero_client_id) - self.assertEqual(zero_client_id, c.get_client_id()) # - - class RiakHttpTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): @@ -1045,56 +1020,6 @@ def test_delete_documents_from_search_by_query_and_id(self): results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results["response"]["docs"])) -class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): - - def setUp(self): - if not HAVE_HTTP_POOL: - self.skipTest('urllib3 is unavailable') - self.host = HTTP_HOST - self.port = HTTP_PORT - self.transport_class = RiakHttpPoolTransport - super(RiakHttpPoolTransportTestCase, self).setUp() - - def test_no_returnbody(self): - bucket = self.client.bucket("bucket") - o = bucket.new("foo", "bar").store(return_body=False) - self.assertEqual(o.vclock(), None) - - def test_generate_key(self): - # Ensure that Riak generates a random key when - # the key passed to bucket.new() is None. - bucket = self.client.bucket('random_key_bucket') - for key in bucket.get_keys(): - bucket.get(str(key)).delete() - bucket.new(None, data={}).store() - self.assertEqual(len(bucket.get_keys()), 1) - - def test_set_client_id(self): - self.client.set_client_id("Client") - self.assertEqual(self.client.get_transport().get_client_id(), "Client") - -class RiakHttpReuseTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): - - def setUp(self): - self.host = HTTP_HOST - self.port = HTTP_PORT - self.transport_class = RiakHttpReuseTransport - super(RiakHttpReuseTransportTestCase, self).setUp() - - def test_no_returnbody(self): - bucket = self.client.bucket("bucket") - o = bucket.new("foo", "bar").store(return_body=False) - self.assertEqual(o.vclock(), None) - - def test_generate_key(self): - # Ensure that Riak generates a random key when - # the key passed to bucket.new() is None. - bucket = self.client.bucket('random_key_bucket') - for key in bucket.get_keys(): - bucket.get(str(key)).delete() - bucket.new(None, data={}).store() - self.assertEqual(len(bucket.get_keys()), 1) - class RiakTestFilter(unittest.TestCase): def test_simple(self): diff --git a/riak/transports/http.py b/riak/transports/http.py index 493eb27d..e90a8b6a 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -36,6 +36,7 @@ from riak.riak_index_entry import RiakIndexEntry from riak.multidict import MultiDict from connection import HTTPConnectionManager +import riak.util MAX_LINK_HEADER_SIZE = 8192 - 8 # substract length of "Link: " header string and newline @@ -519,111 +520,24 @@ def parse_http_headers(cls, headers) : retVal[key] = value return retVal -import socket class RiakHttpReuseTransport(RiakHttpTransport): - """ - Reuse sockets - """ - + "Deprecated transport." def __init__(self, cm, prefix='riak', mapred_prefix='mapred', client_id=None, **unused_options): - super(RiakHttpReuseTransport, self).__init__(cm, - prefix, - mapred_prefix, - client_id) - ### for backwards compat - self._host, self._port = cm.hostports[0] + RiakHttpTransport.__init__(self, cm, prefix, mapred_prefix, + client_id, **unused_options) + riak.util.deprecated('please use RiakHttpTransport instead', + stacklevel=4) - def __copy__(self): - return RiakHttpReuseTransport(self._conns, self._prefix, - self._mapred_prefix) - - def http_request(self, method, uri, headers=None, body=''): - if headers is None: - headers = {} - # Run the request... - client = None - response = None - try: - client = httplib.HTTPConnection(self._host, self._port) - - #handle the connection myself, try to reuse sockets - client.auto_open = 0 - client.connect() - client.sock.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, - client.sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) | 1) - - client.request(method, uri, body, headers) - response = client.getresponse() - - # Get the response headers... - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value - - # Get the body... - response_body = response.read() - response.close() - - #close, this does not make any difference - client.close() - - return response_headers, response_body - except: - if client is not None: client.close() - if response is not None: response.close() - raise - -try: - import urllib3 -except ImportError: - urllib3 = None class RiakHttpPoolTransport(RiakHttpTransport): - """ - Use HTTP pool - """ - - http_pool = None - + "Deprecated transport." def __init__(self, cm, prefix='riak', mapred_prefix='mapred', client_id=None, **unused_options): - if urllib3 is None: - raise RiakError("this transport is not available (no urllib3)") - - super(RiakHttpPoolTransport, self).__init__(cm, - prefix, - mapred_prefix, - client_id) - ### for backwards compat - self._host, self._port = cm.hostports[0] - - def __copy__(self): - return RiakHttpPoolTransport(self._conns, self._prefix, - self._mapred_prefix) - - def http_request(self, method, uri, headers={}, body=''): - if headers is None: - headers = {} - try: - ### it seems wrong to put the pool into a *class* variable, - ### but this code is supporting backwards-compat where the - ### use of a class variable was the design. - if self.__class__.http_pool is None: - self.__class__.http_pool = urllib3.connection_from_url('http://%s:%d' % (self._host, self._port), maxsize=10) - - response = self.http_pool.urlopen(method, uri, body, headers) - - response_headers = {'http_code': response.status} - for key, value in response.getheaders().iteritems(): - response_headers[key.lower()] = value - - response_body = response.data - - return response_headers, response_body - except: - raise + RiakHttpTransport.__init__(self, cm, prefix, mapred_prefix, + client_id, **unused_options) + riak.util.deprecated('please use RiakHttpTransport instead', + stacklevel=4) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index e70aa9c2..3ca3d210 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -32,6 +32,8 @@ from riak import RiakError from riak.riak_index_entry import RiakIndexEntry from riak.transports import connection +from connection import SocketConnectionManager +import riak.util try: import riakclient_pb2 @@ -546,143 +548,11 @@ def pbify_content(self, metadata, data, rpb_content) : pb_link.tag = link.get_tag() rpb_content.value = data -from Queue import Empty, Full, Queue -import contextlib -class RiakPbcCachedTransport(RiakTransport): - """Threadsafe pool of PBC connections, based on urllib3's pool [aka Queue]""" - - # We're using the new RiakTransport API - api = 2 - - # The ConnectionManager class that this transport prefers. - default_cm = connection.cm_using(SocketWithId) - +class RiakPbcCachedTransport(RiakPbcTransport): + "Deprecated transport." def __init__(self, cm, client_id=None, maxsize=0, block=False, timeout=None, **unused_options): - if riakclient_pb2 is None: - raise RiakError("this transport is not available (no protobuf)") - - ### backwards compat. we don't use the ConnectionManager (yet). - self._cm = cm - - self.client_id = client_id - self.block = block - self.timeout = timeout - - self.pool = Queue(maxsize) - # Fill the queue up so that doing get() on it will block properly (check Queue#get) - [self.pool.put(None) for _ in xrange(maxsize)] - - def _new_connection(self): - """New PBC connection""" - return RiakPbcTransport(self._cm, self.client_id) - - def _get_connection(self): - connection = None - try: - connection = self.pool.get(block=self.block, timeout=self.timeout) - except Empty: - pass - return connection or self._new_connection() - - def _put_connection(self, connection): - try: - self.pool.put(connection, block=False) - except Full: - pass - - @contextlib.contextmanager - def _get_connection_from_pool(self): - """checkout conn, try operation, put conn back in pool""" - connection = self._get_connection() - try: - yield connection - finally: - self._put_connection(connection) - - def ping(self): - """ - Ping the remote server - @return boolean - """ - with self._get_connection_from_pool() as connection: - return connection.ping() - - def get(self, robj, r = None, vtag = None): - """ - Serialize get request and deserialize response - @return (vclock=None, [(metadata, value)]=None) - """ - with self._get_connection_from_pool() as connection: - return connection.get(robj, r, vtag) - - def put(self, robj, w = None, dw = None, return_body = True): - """ - Serialize put request and deserialize response - if 'content' - is true, retrieve the updated metadata/content - @return (vclock=None, [(metadata, value)]=None) - """ - with self._get_connection_from_pool() as connection: - return connection.put(robj, w, dw, return_body) - - def put_new(self, robj, w=None, dw=None, return_meta=True): - """Put a new object into the Riak store, returning its (new) key. - - If return_meta is False, then the vlock and metadata return values - will be None. - - @return (key, vclock, metadata) - """ - with self._get_connection_from_pool() as connection: - return connection.put_new(robj, w, dw, return_meta) - - def delete(self, robj, rw = None): - """ - Serialize delete request and deserialize response - @return true - """ - with self._get_connection_from_pool() as connection: - return connection.delete(robj, rw) - - def get_buckets(self): - """ - Serialize bucket listing request and deserialize response - """ - with self._get_connection_from_pool() as connection: - return connection.get_buckets() - - def get_bucket_props(self, bucket) : - """ - Serialize get bucket property request and deserialize response - @return dict() - """ - with self._get_connection_from_pool() as connection: - return connection.get_bucket_props(bucket) - - def set_bucket_props(self, bucket, props) : - """ - Serialize set bucket property request and deserialize response - bucket = bucket object - props = dictionary of properties - @return boolean - """ - with self._get_connection_from_pool() as connection: - return connection.set_bucket_props(bucket, props) - - def mapred(self, inputs, query, timeout = None) : - """ - Serialize map/reduce request - """ - with self._get_connection_from_pool() as connection: - return connection.mapred(inputs, query, timeout) - - def set_client_id(self, client_id): - """Mmm, this can turn ugly if you use different id for different objects in the pool""" - with self._get_connection_from_pool() as connection: - return connection.set_client_id(client_id) - - def get_client_id(self): - """see set_client_id notes, you can do wrong with this""" - with self._get_connection_from_pool() as connection: - return connection.get_client_id() + RiakPbcTransport.__init__(self, cm, client_id, **unused_options) + riak.util.deprecated('please use RiakPbcTransport instead', + stacklevel=4) diff --git a/setup.py b/setup.py index ae14401b..0af5d88a 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,9 @@ def make_pb(): name='riak', version='1.3.0', packages = find_packages(), - install_requires = ['protobuf>=2.3.0', 'urllib3>=0.4.0'], + extras_require = { + 'protobuf': ['protobuf>=2.3.0'], + }, dependency_links = ["http://downloads.basho.com/support"], package_data = { '' : ['*.proto'], From 54e9577c92e8926822dbfa199f1f2af7c75ff111 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Mon, 16 Jan 2012 21:18:30 -0500 Subject: [PATCH 066/118] Tutorial rst syntax fix. --- docs/tutorial.rst | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index ab780da7..f1fe1994 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -398,26 +398,26 @@ the object by querying the metadata, returning a list of matching keys. Your Riak cluster must have Secondary Indexes enabled. See the Riak documentation for details. -Usage of this feature looks like: +Usage of this feature looks like:: - import riak + import riak - client = riak.RiakClient() - bucket = client.bucket('mybucket') + client = riak.RiakClient() + bucket = client.bucket('mybucket') - # Create and store the object with indexes... - obj = bucket.new('mykey1', 'mydata') - obj.add_index('field1_bin', 'val1') - obj.add_index('field2_int', 1001) - obj.store() + # Create and store the object with indexes... + obj = bucket.new('mykey1', 'mydata') + obj.add_index('field1_bin', 'val1') + obj.add_index('field2_int', 1001) + obj.store() - # Query the indexes. The return value is a list of ``RiakLink`` objects. - results = client.index('mybucket', 'field1_bin', 'val1').run() + # Query the indexes. The return value is a list of ``RiakLink`` objects. + results = client.index('mybucket', 'field1_bin', 'val1').run() - # Query the indexes using a range... - results = client.index('mybucket', 'field1_bin', 'val1', 'val5').run() + # Query the indexes using a range... + results = client.index('mybucket', 'field1_bin', 'val1', 'val5').run() - # Remove an index entry... - obj = bucket.get('mykey1') - obj.remove_index('field1_bin', 'val1') - obj.store() + # Remove an index entry... + obj = bucket.get('mykey1') + obj.remove_index('field1_bin', 'val1') + obj.store() From be908fcf5aaa578f1375359b7b31b8d08e0ae7a4 Mon Sep 17 00:00:00 2001 From: Jeffrey Massung Date: Wed, 25 Jan 2012 15:45:57 -0700 Subject: [PATCH 067/118] Pull the check for 2i outside the tests and call it at the start of each 2i test. --- riak/tests/test_all.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 8c14b84b..edd56e0d 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -536,8 +536,21 @@ def test_list_buckets(self): buckets = self.client.get_buckets() self.assertTrue("list_bucket" in buckets) + def is_2i_supported(self): + # Immediate test to see if 2i is even supported w/ the backend + try: + self.client.index('foo','bar_bin','baz').run() + return True + except Exception as e: + if "indexes_not_supported" in str(e): + return False + raise e # re-raise to fail the test + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_store(self): + if not self.is_2i_supported(): + return True + # Create a new object with indexes... bucket = self.client.bucket('indexbucket') rand = self.randint() @@ -609,6 +622,9 @@ def test_secondary_index_store(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_query(self): + if not self.is_2i_supported(): + return True + bucket = self.client.bucket('indexbucket') bucket.\ @@ -632,13 +648,6 @@ def test_secondary_index_query(self): add_index('field2_int', 1004).\ store() - # Immediate test to see if 2i is even supported w/ the backend - try: - self.client.index('foo','bar_bin','baz').run() - except Exception as e: - if "indexes_not_supported" in str(e): - return True - # Test an equality query... results = self.client.index('indexbucket', 'field1_bin', 'val2').run() self.assertEquals(1, len(results)) From d77fd766f1b10bfe95d796a9c55080f67928cd91 Mon Sep 17 00:00:00 2001 From: Jeffrey Massung Date: Wed, 25 Jan 2012 16:14:18 -0700 Subject: [PATCH 068/118] Indexes are supported if the server error message isn't that they are not. --- riak/tests/test_all.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index edd56e0d..cd00fe8c 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -540,11 +540,10 @@ def is_2i_supported(self): # Immediate test to see if 2i is even supported w/ the backend try: self.client.index('foo','bar_bin','baz').run() - return True except Exception as e: if "indexes_not_supported" in str(e): return False - raise e # re-raise to fail the test + raise True # re-raise to fail the test @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_store(self): From 3d71cef488d620d0dea432864e106a55d093278b Mon Sep 17 00:00:00 2001 From: Jeffrey Massung Date: Wed, 25 Jan 2012 16:17:13 -0700 Subject: [PATCH 069/118] Handling extra exit condition - success. --- riak/tests/test_all.py | 1 + 1 file changed, 1 insertion(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index cd00fe8c..acb35022 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -540,6 +540,7 @@ def is_2i_supported(self): # Immediate test to see if 2i is even supported w/ the backend try: self.client.index('foo','bar_bin','baz').run() + return True except Exception as e: if "indexes_not_supported" in str(e): return False From 3a74100f78ca011a733be97efa7f7855e8cff8a3 Mon Sep 17 00:00:00 2001 From: Jeffrey Massung Date: Wed, 25 Jan 2012 17:31:18 -0700 Subject: [PATCH 070/118] Fixed a typo.. raise -> return. --- riak/tests/test_all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index acb35022..6de3f2d4 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -544,7 +544,7 @@ def is_2i_supported(self): except Exception as e: if "indexes_not_supported" in str(e): return False - raise True # re-raise to fail the test + return True # it failed, but is supported! @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_store(self): From d00d4f58077c58c22526b466a80b642f9af644bf Mon Sep 17 00:00:00 2001 From: Jeffrey Massung Date: Mon, 30 Jan 2012 15:35:38 -0700 Subject: [PATCH 071/118] Removed hack check for 2i enabled. --- riak/tests/test_all.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 8c14b84b..61bb6b16 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -632,13 +632,6 @@ def test_secondary_index_query(self): add_index('field2_int', 1004).\ store() - # Immediate test to see if 2i is even supported w/ the backend - try: - self.client.index('foo','bar_bin','baz').run() - except Exception as e: - if "indexes_not_supported" in str(e): - return True - # Test an equality query... results = self.client.index('indexbucket', 'field1_bin', 'val2').run() self.assertEquals(1, len(results)) From 24027b50cf6d2ebd5b3bfe13af37ec8c871cbfc2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 7 Feb 2012 16:28:01 -0500 Subject: [PATCH 072/118] Require unittest2 on Python 2.6. --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0af5d88a..7bcb9360 100755 --- a/setup.py +++ b/setup.py @@ -2,9 +2,9 @@ import glob import os import subprocess +import platform from setuptools import setup, find_packages - def make_docs(): if not os.path.exists('docs'): os.mkdir('docs') @@ -16,10 +16,16 @@ def make_pb(): subprocess.call(['protoc', '--python_out=.', './riak/transports/riakclient.proto']) if __name__ == "__main__": + if platform.python_version() < '2.7': + test_require = ["unittest2"] + else: + test_require = [] + setup( name='riak', version='1.3.0', packages = find_packages(), + requires = test_require, extras_require = { 'protobuf': ['protobuf>=2.3.0'], }, From bc0decf25ba621c610aa4a341636c1a60a179fbc Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 7 Feb 2012 16:33:34 -0500 Subject: [PATCH 073/118] Remove deprecated transports. --- riak/__init__.py | 4 ++-- riak/transports/__init__.py | 4 ++-- riak/transports/http.py | 22 ---------------------- riak/transports/pbc.py | 9 --------- 4 files changed, 4 insertions(+), 35 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index d8577d1d..8a334910 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -41,8 +41,8 @@ def __str__(self): from client import RiakClient from mapreduce import RiakMapReduce, RiakMapReducePhase, RiakLinkPhase,\ RiakKeyFilter -from transports.pbc import RiakPbcTransport, RiakPbcCachedTransport -from transports.http import RiakHttpTransport, RiakHttpReuseTransport, RiakHttpPoolTransport +from transports.pbc import RiakPbcTransport +from transports.http import RiakHttpTransport ONE = "one" ALL = "all" diff --git a/riak/transports/__init__.py b/riak/transports/__init__.py index f970fed9..55b7da3f 100644 --- a/riak/transports/__init__.py +++ b/riak/transports/__init__.py @@ -1,4 +1,4 @@ -from http import RiakHttpTransport, RiakHttpReuseTransport, RiakHttpPoolTransport -from pbc import RiakPbcTransport, RiakPbcCachedTransport +from http import RiakHttpTransport +from pbc import RiakPbcTransport diff --git a/riak/transports/http.py b/riak/transports/http.py index e90a8b6a..92d534b4 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -519,25 +519,3 @@ def parse_http_headers(cls, headers) : else: retVal[key] = value return retVal - - -class RiakHttpReuseTransport(RiakHttpTransport): - "Deprecated transport." - def __init__(self, cm, - prefix='riak', mapred_prefix='mapred', client_id=None, - **unused_options): - RiakHttpTransport.__init__(self, cm, prefix, mapred_prefix, - client_id, **unused_options) - riak.util.deprecated('please use RiakHttpTransport instead', - stacklevel=4) - - -class RiakHttpPoolTransport(RiakHttpTransport): - "Deprecated transport." - def __init__(self, cm, - prefix='riak', mapred_prefix='mapred', client_id=None, - **unused_options): - RiakHttpTransport.__init__(self, cm, prefix, mapred_prefix, - client_id, **unused_options) - riak.util.deprecated('please use RiakHttpTransport instead', - stacklevel=4) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 3ca3d210..150bf58d 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -547,12 +547,3 @@ def pbify_content(self, metadata, data, rpb_content) : pb_link.key = link.get_key() pb_link.tag = link.get_tag() rpb_content.value = data - -class RiakPbcCachedTransport(RiakPbcTransport): - "Deprecated transport." - def __init__(self, cm, - client_id=None, maxsize=0, block=False, timeout=None, - **unused_options): - RiakPbcTransport.__init__(self, cm, client_id, **unused_options) - riak.util.deprecated('please use RiakPbcTransport instead', - stacklevel=4) From c7bb6ab16666f4802c6384d564bb74e1079e6d31 Mon Sep 17 00:00:00 2001 From: Gregory Burd Date: Sat, 15 Oct 2011 17:39:35 -0400 Subject: [PATCH 074/118] Unquote items stored in the link header. Without this links with characters that must be encoded are never decoded. The unit test: python -m unittest riak.tests.test_all.RiakHttpPoolTransportTestCase did not check the content of the links, only that the right number of links are returned so the "foo3" bucket with a link containing special characters "tag2!@#%^&*)" results in the following when retrieved "tag2%21%40%23%25%5E%26%2A%29" --- riak/tests/test_all.py | 11 +++++++++++ riak/transports/http.py | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6de3f2d4..e708298d 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -440,6 +440,17 @@ def test_store_and_get_links(self): obj = bucket.get("foo") links = obj.get_links() self.assertEqual(len(links), 3) + for l in links: + if (l.get_key() == "foo1"): + self.assertEqual(l.get_tag(), "") + next + if (l.get_key() == "foo2"): + self.assertEqual(l.get_tag(), "tag") + next + if (l.get_key() == "foo3"): + self.assertEqual(l.get_tag(), "tag2!@#%^&*)") + next + self.assertEqual("unknown key", l.get_key()) def test_link_walking(self): # Create the object... diff --git a/riak/transports/http.py b/riak/transports/http.py index e90a8b6a..c0e38bfb 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -343,7 +343,9 @@ def parse_links(self, links, linkHeaders): linkHeader = linkHeader.strip() matches = re.match("; ?riaktag=\"([^\']+)\"", linkHeader) if matches is not None: - link = RiakLink(matches.group(2), matches.group(3), matches.group(4)) + link = RiakLink(urllib.unquote_plus(matches.group(2)), + urllib.unquote_plus(matches.group(3)), + urllib.unquote_plus(matches.group(4))) links.append(link) return self From f152ac1483cd5bb2330c00b1185dfbfacb09fdc0 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 11:56:37 -0800 Subject: [PATCH 075/118] Allow transport specific options to be passed through RiakClient. Added a new optional parameter transport_options, which is optionally provided and passed through to the transport class constructor. Allows custom settings to be passed through. --- riak/client.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/riak/client.py b/riak/client.py index bf1e2fcc..722f953d 100644 --- a/riak/client.py +++ b/riak/client.py @@ -39,7 +39,8 @@ class RiakClient(object): """ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', mapred_prefix='mapred', transport_class=None, - client_id=None, solr_transport_class=None): + client_id=None, solr_transport_class=None, + transport_options=None): """ Construct a new ``RiakClient`` object. @@ -55,6 +56,8 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', :type transport_class: :class:`RiakTransport` :param solr_transport_class: HTTP-based transport class for Solr interface queries :type transport_class: :class:`RiakHttpTransport` + :param transport_options: Optional key-value args to pass to the transport constuctor + :type transport_options: dict """ if transport_class is None: transport_class = RiakHttpTransport @@ -64,9 +67,10 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', hostports = [ (host, port), ] self._cm = transport_class.default_cm(hostports) - ### we need to allow additional transport options. make this an - ### argument to __init__ ? - transport_options = { } + # If no transport options are provided, then default to the + # empty dict, otherwise just pass through what we are provided. + if transport_options is None: + transport_options = {} self._transport = transport_class(self._cm, prefix=prefix, From e90f5c3ddc241475a57faf386d71ccbde3c42a91 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 11:42:04 -0800 Subject: [PATCH 076/118] Fixing PEP8 compliance, styling, and imports. Changed some code to be more PEP8 compliant in terms of style guides. Reworked the imports to be nicer, and avoid the splat-style import. This allows pyflakes to pickup any missing imports. --- riak/transports/pbc.py | 60 +++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 150bf58d..7a238ea1 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -19,20 +19,31 @@ """ from __future__ import with_statement -import socket, struct +import errno +import socket +import struct try: import json except ImportError: import simplejson as json -from riak.transports.transport import RiakTransport -from riak.metadata import * -from riak.mapreduce import RiakMapReduce, RiakLink +from riak.metadata import ( + MD_CHARSET, + MD_CTYPE, + MD_ENCODING, + MD_INDEX, + MD_LASTMOD, + MD_LASTMOD_USECS, + MD_LINKS, + MD_USERMETA, + MD_VTAG, + ) from riak import RiakError +from riak.mapreduce import RiakLink from riak.riak_index_entry import RiakIndexEntry from riak.transports import connection -from connection import SocketConnectionManager +from riak.transports.transport import RiakTransport import riak.util try: @@ -103,10 +114,10 @@ class RiakPbcTransport(RiakTransport): api = 2 rw_names = { - 'default' : RIAKC_RW_DEFAULT, - 'all' : RIAKC_RW_ALL, - 'quorum' : RIAKC_RW_QUORUM, - 'one' : RIAKC_RW_ONE + 'default': RIAKC_RW_DEFAULT, + 'all': RIAKC_RW_ALL, + 'quorum': RIAKC_RW_QUORUM, + 'one': RIAKC_RW_ONE } # The ConnectionManager class that this transport prefers. @@ -176,7 +187,7 @@ def set_client_id(self, client_id): return True - def get(self, robj, r = None, vtag = None): + def get(self, robj, r=None, vtag=None): """ Serialize get request and deserialize response """ @@ -201,7 +212,7 @@ def get(self, robj, r = None, vtag = None): else: return 0 - def put(self, robj, w = None, dw = None, return_body = True): + def put(self, robj, w=None, dw=None, return_body=True): """ Serialize get request and deserialize response """ @@ -259,7 +270,7 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): metadata, content = self.decode_content(resp.content[0]) return resp.key, resp.vclock, metadata - def delete(self, robj, rw = None): + def delete(self, robj, rw=None): """ Serialize get request and deserialize response """ @@ -316,14 +327,14 @@ def get_bucket_props(self, bucket): return props - def set_bucket_props(self, bucket, props): """ Serialize set bucket property request and deserialize response """ req = riakclient_pb2.RpbSetBucketReq() req.bucket = bucket.get_name() - if not 'n_val' in props and not 'allow_mult' in props: return self + if not 'n_val' in props and not 'allow_mult' in props: + return self if 'n_val' in props: req.props.n_val = props['n_val'] @@ -336,7 +347,7 @@ def set_bucket_props(self, bucket, props): def mapred(self, inputs, query, timeout=None): # Construct the job, optionally set the timeout... - job = {'inputs':inputs, 'query':query} + job = {'inputs': inputs, 'query': query} if timeout is not None: job['timeout'] = timeout @@ -446,29 +457,29 @@ def recv_msg(self, conn, expect): msg = riakclient_pb2.RpbMapRedResp() msg.ParseFromString(self._inbuf[1:]) else: - raise Exception("unknown msg code %s"%msg_code) + raise Exception("unknown msg code %s" % msg_code) if expect and msg_code != expect: raise RiakError("unexpected protocol buffer message code: %d" % msg_code) return msg_code, msg - def recv_pkt(self, conn): nmsglen = conn.recv(4) if len(nmsglen) != 4: - raise RiakError("Socket returned short packet length %d - expected 4"%\ - len(nmsglen)) + raise RiakError("Socket returned short packet length %d - expected 4" + % len(nmsglen)) msglen, = struct.unpack('!i', nmsglen) self._inbuf_len = msglen self._inbuf = '' while len(self._inbuf) < msglen: want_len = min(8192, msglen - len(self._inbuf)) recv_buf = conn.recv(want_len) - if not recv_buf: break + if not recv_buf: + break self._inbuf += recv_buf if len(self._inbuf) != self._inbuf_len: - raise RiakError("Socket returned short packet %d - expected %d"%\ - (len(self._inbuf), self._inbuf_len)) + raise RiakError("Socket returned short packet %d - expected %d" + % (len(self._inbuf), self._inbuf_len)) def decode_contents(self, rpb_contents): contents = [] @@ -520,10 +531,10 @@ def decode_content(self, rpb_content): metadata[MD_INDEX] = indexes return metadata, rpb_content.value - def pbify_content(self, metadata, data, rpb_content) : + def pbify_content(self, metadata, data, rpb_content): # Convert the broken out fields, building up # pbmetadata for any unknown ones - for k,v in metadata.iteritems(): + for k, v in metadata.iteritems(): if k == MD_CTYPE: rpb_content.content_type = v elif k == MD_CHARSET: @@ -547,3 +558,4 @@ def pbify_content(self, metadata, data, rpb_content) : pb_link.key = link.get_key() pb_link.tag = link.get_tag() rpb_content.value = data + From 5fb87576f21053c277b3d1f851177b534a1ac2c1 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 11:47:39 -0800 Subject: [PATCH 077/118] Alphabetize the imports for pbc transport --- riak/transports/pbc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 7a238ea1..327bcb90 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -28,6 +28,8 @@ except ImportError: import simplejson as json +from riak import RiakError +from riak.mapreduce import RiakLink from riak.metadata import ( MD_CHARSET, MD_CTYPE, @@ -39,8 +41,6 @@ MD_USERMETA, MD_VTAG, ) -from riak import RiakError -from riak.mapreduce import RiakLink from riak.riak_index_entry import RiakIndexEntry from riak.transports import connection from riak.transports.transport import RiakTransport From c4e68488eff40ea4f012638de75e73215ccad7f9 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 11:55:53 -0800 Subject: [PATCH 078/118] PEP8 fixes and import re-ordering for riak client. Fix some minor styling to be PEP8 compliant. Re-order the imports. --- riak/client.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/riak/client.py b/riak/client.py index 722f953d..5e78fe27 100644 --- a/riak/client.py +++ b/riak/client.py @@ -23,12 +23,11 @@ except ImportError: import simplejson as json -from riak.transports import RiakHttpTransport from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduce from riak.search import RiakSearch +from riak.transports import RiakHttpTransport from riak.util import deprecated -import riak.transports.connection class RiakClient(object): @@ -86,10 +85,10 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', self._w = "default" self._dw = "default" self._rw = "default" - self._encoders = {'application/json':json.dumps, - 'text/json':json.dumps} - self._decoders = {'application/json':json.loads, - 'text/json':json.loads} + self._encoders = {'application/json': json.dumps, + 'text/json': json.dumps} + self._decoders = {'application/json': json.loads, + 'text/json': json.loads} self._solr = None self._host = host self._port = port @@ -288,7 +287,7 @@ def search(self, *args): def index(self, *args): """ Start assembling a Map/Reduce operation based on secondary - index query results. + index query results. :rtype: :class:`RiakMapReduce` """ From 602b7f205d762706764b8619c6daa68831e6daf7 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 13:01:52 -0800 Subject: [PATCH 079/118] Remove the import, moved to appropriate branch --- riak/transports/pbc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 327bcb90..21a47b72 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -19,7 +19,6 @@ """ from __future__ import with_statement -import errno import socket import struct From 6bb7d8dfd57e27862e749d705b402d09b362bde7 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 11:44:04 -0800 Subject: [PATCH 080/118] Improvements to SocketWithId SocketWithId will now uses super() to invoke methods from the parent class, since it is a new style class. send() and recv() now both catch any errors that indicate the socket is closed, and close the internal socket to prevent it from being re-used. --- riak/transports/pbc.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 21a47b72..e2ec1a35 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -82,10 +82,22 @@ RIAKC_RW_ALL = 4294967292 RIAKC_RW_DEFAULT = 4294967291 +# These are a specific set of socket errors +# that could be raised on send/recv that indicate +# that the socket is closed or reset, and is not +# usable. On seeing any of these errors, the socket +# should be closed, and the connection re-established. +CONN_CLOSED_ERRORS = ( + errno.EHOSTUNREACH, + errno.ECONNRESET, + errno.EBADF, + errno.EPIPE + ) + class SocketWithId(connection.Socket): def __init__(self, host, port): - connection.Socket.__init__(self, host, port) + super(SocketWithId, self).__init__(host, port) self.last_client_id = None def maybe_connect(self): @@ -93,14 +105,27 @@ def maybe_connect(self): # client_id used on this connection. if self.sock is None: self.last_client_id = None - - connection.Socket.maybe_connect(self) + super(SocketWithId, self).maybe_connect() def send(self, pkt): - self.sock.sendall(pkt) + try: + self.sock.sendall(pkt) + except socket.error, e: + # If the socket is in a bad state, close it and allow it + # to re-connect on the next try + if e[0] in CONN_CLOSED_ERRORS: + self.close() + raise def recv(self, want_len): - return self.sock.recv(want_len) + try: + return self.sock.recv(want_len) + except socket.error, e: + # If the socket is in a bad state, close it and allow it + # to re-connect on the next try + if e[0] in CONN_CLOSED_ERRORS: + self.close() + raise class RiakPbcTransport(RiakTransport): From e8a798dbaa69ace40efc08f30ea4b71ff2f779b4 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 11:58:52 -0800 Subject: [PATCH 081/118] Make RiakPbcTransport more robust to connection errors. Modified RiakPbcTransport to take an optional retries parameter, which is defaulted to 1 for backwards compatibility. send_pkt will make up to 'retries' attempts to send in the face of any error which indicates the connection has been closed/reset (CONN_CLOSED_ERRORS). --- riak/transports/pbc.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index e2ec1a35..810cd958 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -147,7 +147,7 @@ class RiakPbcTransport(RiakTransport): # The ConnectionManager class that this transport prefers. default_cm = connection.cm_using(SocketWithId) - def __init__(self, cm, client_id=None, **unused_options): + def __init__(self, cm, client_id=None, retries=1, **unused_options): """ Construct a new RiakPbcTransport object. """ @@ -158,6 +158,7 @@ def __init__(self, cm, client_id=None, **unused_options): self._cm = cm self._client_id = client_id + self._retries = retries def translate_rw_val(self, rw): val = self.rw_names.get(rw) @@ -431,18 +432,30 @@ def send_msg_multi(self, msg_code, msg, expect, handler): break def send_pkt(self, conn, pkt): - conn.maybe_connect() - - # If the last client_id used on this connection is different than our - # client_id, then set a new ID on the connection. - if conn.last_client_id != self._client_id: - req = riakclient_pb2.RpbSetClientIdReq() - req.client_id = self._client_id - conn.send(self.encode_msg(MSG_CODE_SET_CLIENT_ID_REQ, req)) - conn.last_client_id = self._client_id - self.recv_msg(conn, MSG_CODE_SET_CLIENT_ID_RESP) - - conn.send(pkt) + attempt, e = 0, None + for attempt in xrange(self._retries): + try: + conn.maybe_connect() + + # If the last client_id used on this connection is different than our + # client_id, then set a new ID on the connection. + if conn.last_client_id != self._client_id: + req = riakclient_pb2.RpbSetClientIdReq() + req.client_id = self._client_id + conn.send(self.encode_msg(MSG_CODE_SET_CLIENT_ID_REQ, req)) + conn.last_client_id = self._client_id + self.recv_msg(conn, MSG_CODE_SET_CLIENT_ID_RESP) + + conn.send(pkt) + except socket.error, e: + # If this is some unknown socket error bail out + # instead of retrying + if e[0] not in CONN_CLOSED_ERRORS: + raise + + # Max attempts reached, raise whatever exception we are getting + if attempt + 1 == self.retries and e is not None: + raise e def recv_msg(self, conn, expect): self.recv_pkt(conn) From 30068063852d554cdb73abdaa0ee5eab704e32bf Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 12:10:59 -0800 Subject: [PATCH 082/118] Rename the 'retries' argument to RiakPbcTransport to 'max_attempts' It is much more clear as max_attempts, since previously if retries was 3, there would be a total of 3 attempts, not 1 attempt and 3 retries as the name would indicate. --- riak/transports/pbc.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 810cd958..afd7f1fe 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -147,7 +147,7 @@ class RiakPbcTransport(RiakTransport): # The ConnectionManager class that this transport prefers. default_cm = connection.cm_using(SocketWithId) - def __init__(self, cm, client_id=None, retries=1, **unused_options): + def __init__(self, cm, client_id=None, max_attempts=1, **unused_options): """ Construct a new RiakPbcTransport object. """ @@ -158,7 +158,7 @@ def __init__(self, cm, client_id=None, retries=1, **unused_options): self._cm = cm self._client_id = client_id - self._retries = retries + self._max_attempts = max_attempts def translate_rw_val(self, rw): val = self.rw_names.get(rw) @@ -432,8 +432,10 @@ def send_msg_multi(self, msg_code, msg, expect, handler): break def send_pkt(self, conn, pkt): - attempt, e = 0, None - for attempt in xrange(self._retries): + attempt = 0 + e = None + for attempt in xrange(self._max_attempts): + e = None try: conn.maybe_connect() @@ -454,7 +456,7 @@ def send_pkt(self, conn, pkt): raise # Max attempts reached, raise whatever exception we are getting - if attempt + 1 == self.retries and e is not None: + if attempt + 1 == self._max_attempts and e is not None: raise e def recv_msg(self, conn, expect): From 0b5bc9d12388da640aa0e674f8912f625d42fdd0 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 12:32:04 -0800 Subject: [PATCH 083/118] Handle implicit socket close on SocketWithId If we do a blocking read with non-zero request size, and we get a 0 byte response, assume the socket is closed since this indicates EOF. --- riak/transports/pbc.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index afd7f1fe..c19de917 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -119,7 +119,15 @@ def send(self, pkt): def recv(self, want_len): try: - return self.sock.recv(want_len) + res = self.sock.recv(want_len) + + # Assume the socket is closed if no data is + # returned on a blocking read. + if len(res) == 0 and want_len > 0: + self.close() + + return res + except socket.error, e: # If the socket is in a bad state, close it and allow it # to re-connect on the next try From 99262c1904ca7404f75d870541b7ef4e84cfd143 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 13:02:42 -0800 Subject: [PATCH 084/118] Import errno --- riak/transports/pbc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index c19de917..77c5ddcb 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -19,6 +19,7 @@ """ from __future__ import with_statement +import errno import socket import struct From b0029413f4fde156a1185270654a3f014ccaf253 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 15:08:57 -0800 Subject: [PATCH 085/118] Adding test to verify behavior on underlying socket being closed Try to verify the retry logic works properly by closing the underlying socket. --- riak/tests/test_all.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6de3f2d4..814e784e 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -890,6 +890,33 @@ def test_uses_client_id_if_given(self): client_id = zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) # + def test_close_underlying_socket(self): + c = RiakClient(PB_HOST, PB_PORT, transport_class = RiakPbcTransport) + + bucket = self.client.bucket('bucket_test_close') + rand = self.randint() + obj = bucket.new('foo', rand) + obj.store() + obj = bucket.get('foo') + self.assertTrue(obj.exists()) + self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') + self.assertEqual(obj.get_key(), 'foo') + self.assertEqual(obj.get_data(), rand) + + # Close the underlying socket. This gets a bit sketchy, + # since we are reaching into the internals, but there is + # no other way to get at the socket + conns = c._cm.conns + for conn in conns: + if conn.sock is not None: + conn.sock.close() + + obj = bucket.get('foo') + self.assertTrue(obj.exists()) + self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') + self.assertEqual(obj.get_key(), 'foo') + self.assertEqual(obj.get_data(), rand) + class RiakHttpTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): From e701404150df88525fd63ce87085d586ce856567 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 16:21:13 -0800 Subject: [PATCH 086/118] Added critical 'break' statement Need to ensure we break after the first successful send over the PBC socket. --- riak/transports/pbc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 77c5ddcb..06496faf 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -458,6 +458,7 @@ def send_pkt(self, conn, pkt): self.recv_msg(conn, MSG_CODE_SET_CLIENT_ID_RESP) conn.send(pkt) + break except socket.error, e: # If this is some unknown socket error bail out # instead of retrying From 427d40ed2c758cf41cf3b490a35264ea79a5c734 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 16:21:50 -0800 Subject: [PATCH 087/118] Added 2 new unit tests for PBC changes Added unit tests to test the behavior when the max_attempts are 1 and 2. In the first case we expect the operation to fail, while with retries it should succeed. --- riak/tests/test_all.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 814e784e..b6ee3b17 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -9,6 +9,7 @@ import simplejson as json import os import random +import socket import platform if platform.python_version() < '2.7': @@ -890,10 +891,10 @@ def test_uses_client_id_if_given(self): client_id = zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) # - def test_close_underlying_socket(self): + def test_close_underlying_socket_fails(self): c = RiakClient(PB_HOST, PB_PORT, transport_class = RiakPbcTransport) - bucket = self.client.bucket('bucket_test_close') + bucket = c.bucket('bucket_test_close') rand = self.randint() obj = bucket.new('foo', rand) obj.store() @@ -907,14 +908,36 @@ def test_close_underlying_socket(self): # since we are reaching into the internals, but there is # no other way to get at the socket conns = c._cm.conns - for conn in conns: - if conn.sock is not None: - conn.sock.close() + conns[0].sock.close() - obj = bucket.get('foo') + # This shoud fail with a socket error now + self.assertRaises(socket.error, bucket.get, 'foo') + + def test_close_underlying_socket_retry(self): + c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport, + transport_options={"max_attempts": 2}) + + bucket = c.bucket('bucket_test_close') + rand = self.randint() + obj = bucket.new('barbaz', rand) + obj.store() + obj = bucket.get('barbaz') self.assertTrue(obj.exists()) self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') - self.assertEqual(obj.get_key(), 'foo') + self.assertEqual(obj.get_key(), 'barbaz') + self.assertEqual(obj.get_data(), rand) + + # Close the underlying socket. This gets a bit sketchy, + # since we are reaching into the internals, but there is + # no other way to get at the socket + conns = c._cm.conns + conns[0].sock.close() + + # This should work, since we have a retry + obj = bucket.get('barbaz') + self.assertTrue(obj.exists()) + self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') + self.assertEqual(obj.get_key(), 'barbaz') self.assertEqual(obj.get_data(), rand) From 735eb0bf127b2eeb14582cd116c5bca1a9d724bb Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 16:42:50 -0800 Subject: [PATCH 088/118] Fix RiakObject.get_content_type() when content type not yet set. This method currently raises a KeyError instead of appropriately returning the default content types. --- riak/riak_object.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index c2e48f0e..f6e12bbe 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -214,7 +214,7 @@ def remove_index(self, field, value): def get_indexes(self, field = None): """ - Get a list of the index entries for this object. If a field is provided, returns a list + Get a list of the index entries for this object. If a field is provided, returns a list :param field: The index field. :type field: string or None @@ -244,7 +244,13 @@ def get_content_type(self): :rtype: string """ - return self._metadata[MD_CTYPE] + try: + return self._metadata[MD_CTYPE] + except KeyError: + if self._encode_data: + return "application/json" + else: + return "application/octet-stream" def set_content_type(self, content_type): """ From 8ff2f9eadb287c3516425d1819f0cd374c596d04 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Wed, 15 Feb 2012 09:55:15 -0500 Subject: [PATCH 089/118] Addressed basho/riak-python-client#83 Provided a set_links method. --- riak/riak_object.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index c2e48f0e..1c40382a 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -214,7 +214,7 @@ def remove_index(self, field, value): def get_indexes(self, field = None): """ - Get a list of the index entries for this object. If a field is provided, returns a list + Get a list of the index entries for this object. If a field is provided, returns a list :param field: The index field. :type field: string or None @@ -257,6 +257,27 @@ def set_content_type(self, content_type): self._metadata[MD_CTYPE] = content_type return self + def set_links(self, links): + """ + Replaces all links to a RiakObject + + :param links: An iterable of 2-item tuples, consisting of (RiakObject, tag). This could also be an iterable of + just a RiakObject, instead of the tuple, then a tag of None would be used. Lastly, it could also be an + iterable of RiakLink. They have tags built-in. + """ + new_links = [] + for item in links: + if isinstance(item, RiakLink): + link = item + elif isinstance(item, RiakObject): + link = RiakLink(item._bucket._name, item._key, None) + else: + link = RiakLink(item[0]._bucket._name, item._key, item[1]) + new_links.append(link) + + self._metadata[MD_LINKS] = new_links + return self + def add_link(self, obj, tag=None): """ Add a link to a RiakObject. From 8a05a2203bafb651157735229f223e7958e701a9 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Wed, 15 Feb 2012 10:24:27 -0500 Subject: [PATCH 090/118] Added unittest and fixed a small bug Last commit had a small bug with it, also added unittest for set_links. Signed-off-by: Shuhao Wu --- riak/riak_object.py | 2 +- riak/tests/test_all.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 1c40382a..11cc29a9 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -272,7 +272,7 @@ def set_links(self, links): elif isinstance(item, RiakObject): link = RiakLink(item._bucket._name, item._key, None) else: - link = RiakLink(item[0]._bucket._name, item._key, item[1]) + link = RiakLink(item[0]._bucket._name, item[0]._key, item[1]) new_links.append(link) self._metadata[MD_LINKS] = new_links diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6de3f2d4..4a367a15 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -441,6 +441,21 @@ def test_store_and_get_links(self): links = obj.get_links() self.assertEqual(len(links), 3) + def test_set_links(self): + # Create the object + bucket = self.client.bucket("bucket") + bucket.new("foo", 2).set_links([bucket.new("foo1"), + (bucket.new("foo2"), "tag"), + RiakLink("bucket", "foo2", "tag2")]).store() + obj = bucket.get("foo") + links = sorted(obj.get_links(), key=lambda x: x.get_key()) + self.assertEqual(len(links), 3) + self.assertEqual(links[0].get_key(), "foo1") + self.assertEqual(links[1].get_key(), "foo2") + self.assertEqual(links[1].get_tag(), "tag") + self.assertEqual(links[2].get_key(), "foo2") + self.assertEqual(links[2].get_tag(), "tag2") + def test_link_walking(self): # Create the object... bucket = self.client.bucket("bucket") From a0c48a57510ff60a40f039a1edc6d8292158be8b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 15 Feb 2012 13:30:19 -0500 Subject: [PATCH 091/118] Adjust link-matching spec to accommodate 1.0+ URLs and fix the unit test. --- riak/tests/test_all.py | 16 +++++++--------- riak/transports/http.py | 9 +++++---- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index e708298d..540486c4 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -432,25 +432,23 @@ def test_map_reduce_from_object(self): def test_store_and_get_links(self): # Create the object... bucket = self.client.bucket("bucket") - bucket.new("foo", 2) \ + bucket.new_binary("test_store_and_get_links", '2') \ .add_link(bucket.new("foo1")) \ .add_link(bucket.new("foo2"), "tag") \ .add_link(bucket.new("foo3"), "tag2!@#%^&*)") \ .store() - obj = bucket.get("foo") + obj = bucket.get("test_store_and_get_links") links = obj.get_links() self.assertEqual(len(links), 3) for l in links: if (l.get_key() == "foo1"): - self.assertEqual(l.get_tag(), "") - next - if (l.get_key() == "foo2"): + self.assertEqual(l.get_tag(), "bucket") + elif (l.get_key() == "foo2"): self.assertEqual(l.get_tag(), "tag") - next - if (l.get_key() == "foo3"): + elif (l.get_key() == "foo3"): self.assertEqual(l.get_tag(), "tag2!@#%^&*)") - next - self.assertEqual("unknown key", l.get_key()) + else: + self.assertEqual("unknown key", l.get_key()) def test_link_walking(self): # Create the object... diff --git a/riak/transports/http.py b/riak/transports/http.py index c0e38bfb..81ad330f 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -131,7 +131,7 @@ def do_put(self, url, headers, content, return_body=False, key=None): else: self.check_http_code(response, [204]) return None - + def put_new(self, robj, w=None, dw=None, return_meta=True): """Put a new object into the Riak store, returning its (new) key.""" # Construct the URL... @@ -341,7 +341,8 @@ def parse_links(self, links, linkHeaders): """ for linkHeader in linkHeaders.strip().split(','): linkHeader = linkHeader.strip() - matches = re.match("; ?riaktag=\"([^\']+)\"", linkHeader) + matches = re.match("; ?riaktag=\"([^\']+)\"", linkHeader) or \ + re.match("; ?riaktag=\"([^\']+)\"", linkHeader) if matches is not None: link = RiakLink(urllib.unquote_plus(matches.group(2)), urllib.unquote_plus(matches.group(3)), @@ -393,7 +394,7 @@ def delete_file(self, key): def post_request(self, uri=None, body=None, params=None, content_type="application/json"): uri = self.build_rest_path(prefix=uri, params=params) - return self.http_request('POST', uri, {'Content-Type': content_type}, body) + return self.http_request('POST', uri, {'Content-Type': content_type}, body) # Utility functions used by Riak library. @@ -449,7 +450,7 @@ def build_put_headers(self, robj): headers[key] += ", " + rie.get_value() else: headers[key] = rie.get_value() - + return headers def http_request(self, method, uri, headers=None, body='') : From 0045932d87827ee6dd204a6c99790f6565e58a17 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Mon, 27 Feb 2012 16:21:17 -0500 Subject: [PATCH 092/118] Added a way to efficiently set_links Before, there's no actual way to set_links without having the function iterate through the entire iterable. Now with an all_link=True or False, we could now set links directly to an list. Signed-off-by: Shuhao Wu --- riak/riak_object.py | 8 +++++++- riak/tests/test_all.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 11cc29a9..fbbedee8 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -257,14 +257,20 @@ def set_content_type(self, content_type): self._metadata[MD_CTYPE] = content_type return self - def set_links(self, links): + def set_links(self, links, all_link=False): """ Replaces all links to a RiakObject :param links: An iterable of 2-item tuples, consisting of (RiakObject, tag). This could also be an iterable of just a RiakObject, instead of the tuple, then a tag of None would be used. Lastly, it could also be an iterable of RiakLink. They have tags built-in. + :param all_link: A boolean indicates if links is all RiakLink object + This speeds up the operation. """ + if all_link: + self._metadata[MD_LINKS] = links + return self + new_links = [] for item in links: if isinstance(item, RiakLink): diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 4a367a15..e766cd45 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -456,6 +456,16 @@ def test_set_links(self): self.assertEqual(links[2].get_key(), "foo2") self.assertEqual(links[2].get_tag(), "tag2") + def test_set_links_all_links(self): + bucket = self.client.bucket("bucket") + foo1 = bucket.new("foo", 1) + foo2 = bucket.new("foo2", 2).store() + links = [RiakLink("bucket", "foo2")] + foo1.set_links(links, True) + links = foo1.get_links() + self.assertEqual(len(links), 1) + self.assertEqual(links[0].get_key(), "foo2") + def test_link_walking(self): # Create the object... bucket = self.client.bucket("bucket") From 4d3016b146a25453bcb65567ee004d12beda2031 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Mon, 27 Feb 2012 19:08:05 -0500 Subject: [PATCH 093/118] Added a set_indexes function like set_links Realized that I needed that for my code, I implemented this with this as well. set_links takes an iterable of 2 item tuples that's (field, value). It will set the indexes similarly to set_links Signed-off-by: Shuhao Wu --- riak/riak_object.py | 16 ++++++++++++++++ riak/tests/test_all.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/riak/riak_object.py b/riak/riak_object.py index fbbedee8..1a34fa45 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -212,6 +212,22 @@ def remove_index(self, field, value): self._metadata[MD_INDEX].remove(rie) return self + def set_indexes(self, indexes): + """ + Sets indexes once and for all. Currenly supports an iterable of 2 item tuples, + (field, value) + + :param indexes: iterable of 2 item tuples consisting the field and value. + :rtype: self + """ + new_indexes = [] + for field, value in indexes: + rie = RiakIndexEntry(field, value) + new_indexes.append(rie) + self._metadata[MD_INDEX] = new_indexes + + return self + def get_indexes(self, field = None): """ Get a list of the index entries for this object. If a field is provided, returns a list diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index e766cd45..f6f1da64 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -645,6 +645,22 @@ def test_secondary_index_store(self): # Clean up... bucket.get('mykey1').delete() + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_set_indexes(self): + if not self.is_2i_supported(): + return True + + bucket = self.client.bucket('indexbucket') + foo = bucket.new('foo', 1) + foo.set_indexes((('field1_bin', 'test'), ('field2_int', 1337))).store() + result = self.client.index('indexbucket', 'field2_int', 1337).run() + self.assertEqual(1, len(result)) + self.assertEqual('foo', result[0].get_key()) + + result = self.client.index('indexbucket', 'field1_bin', 'test').run() + self.assertEqual(1, len(result)) + self.assertEqual('foo', result[0].get_key()) + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_query(self): if not self.is_2i_supported(): From 1c3ae3c06868de01c903d308eb5705e77e8124a8 Mon Sep 17 00:00:00 2001 From: William Kral Date: Mon, 5 Mar 2012 16:35:38 -0800 Subject: [PATCH 094/118] Fixed a path conflict with PLATFORM_DATA_DIR introduce in riak 1.x --- riak/test_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/riak/test_server.py b/riak/test_server.py index f4645e3a..3922125d 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -200,6 +200,7 @@ def write_riak_script(self): line = re.sub("(RUNNER_USER=)(.*)", r'\1', line) line = re.sub("(RUNNER_LOG_DIR=)(.*)", r'\1%s' % self._temp_log, line) line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % self._temp_pipe, line) + line = re.sub("(PLATFORM_DATA_DIR=)(.*)", r'\1%s' % self.temp_dir) if string.strip(line) == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}": line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join(self.bin_dir, "..")) From 88ba5a924144a8c307ae4a4feb56b269bd1e81d3 Mon Sep 17 00:00:00 2001 From: William Kral Date: Tue, 6 Mar 2012 01:01:48 -0800 Subject: [PATCH 095/118] Fixed it properly in my live version but missed the line argument here --- riak/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/test_server.py b/riak/test_server.py index 3922125d..56715fbe 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -200,7 +200,7 @@ def write_riak_script(self): line = re.sub("(RUNNER_USER=)(.*)", r'\1', line) line = re.sub("(RUNNER_LOG_DIR=)(.*)", r'\1%s' % self._temp_log, line) line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % self._temp_pipe, line) - line = re.sub("(PLATFORM_DATA_DIR=)(.*)", r'\1%s' % self.temp_dir) + line = re.sub("(PLATFORM_DATA_DIR=)(.*)", r'\1%s' % self.temp_dir, line) if string.strip(line) == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}": line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join(self.bin_dir, "..")) From 2f890af19335023f9ae76a4b2eb189d7581e1112 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 6 Mar 2012 16:29:16 -0800 Subject: [PATCH 096/118] Adding support for if_none_match --- riak/riak_object.py | 9 ++++++--- riak/transports/http.py | 12 +++++++----- riak/transports/pbc.py | 8 ++++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index f6e12bbe..3f6acaea 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -324,7 +324,7 @@ def get_links(self): else: return [] - def store(self, w=None, dw=None, return_body=True): + def store(self, w=None, dw=None, return_body=True, if_none_match=False): """ Store the object in Riak. When this operation completes, the object could contain new metadata and possibly new data if Riak @@ -339,6 +339,9 @@ def store(self, w=None, dw=None, return_body=True): :type dw: integer :param return_body: if the newly stored object should be retrieved :type return_body: bool + :param if_none_match: Should the object be stored only if there is no + key previously defined + :type if_none_match: bool :rtype: self """ # Use defaults if not specified... @@ -349,13 +352,13 @@ def store(self, w=None, dw=None, return_body=True): t = self._client.get_transport() if self._key is None: - key, vclock, metadata = t.put_new(self, w, dw, return_body) + key, vclock, metadata = t.put_new(self, w, dw, return_body, if_none_match) self._exists = True self._key = key self._vclock = vclock self.set_metadata(metadata) else: - Result = t.put(self, w, dw, return_body) + Result = t.put(self, w, dw, return_body, if_none_match) if Result is not None: self.populate(Result) diff --git a/riak/transports/http.py b/riak/transports/http.py index 92d534b4..c00af22a 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -108,7 +108,7 @@ def get(self, robj, r, vtag = None) : response = self.http_request('GET', url) return self.parse_body(response, [200, 300, 404]) - def put(self, robj, w = None, dw = None, return_body = True): + def put(self, robj, w = None, dw = None, return_body = True, if_none_match=False): """ Serialize put request and deserialize response """ @@ -117,6 +117,8 @@ def put(self, robj, w = None, dw = None, return_body = True): url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), params=params) headers = self.build_put_headers(robj) + if if_none_match: + headers["If-None-Match"] = "1" content = robj.get_encoded_data() return self.do_put(url, headers, content, return_body, key=robj.get_key()) @@ -131,8 +133,8 @@ def do_put(self, url, headers, content, return_body=False, key=None): else: self.check_http_code(response, [204]) return None - - def put_new(self, robj, w=None, dw=None, return_meta=True): + + def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): """Put a new object into the Riak store, returning its (new) key.""" # Construct the URL... params = {'returnbody' : str(return_meta).lower(), 'w' : w, 'dw' : dw} @@ -391,7 +393,7 @@ def delete_file(self, key): def post_request(self, uri=None, body=None, params=None, content_type="application/json"): uri = self.build_rest_path(prefix=uri, params=params) - return self.http_request('POST', uri, {'Content-Type': content_type}, body) + return self.http_request('POST', uri, {'Content-Type': content_type}, body) # Utility functions used by Riak library. @@ -447,7 +449,7 @@ def build_put_headers(self, robj): headers[key] += ", " + rie.get_value() else: headers[key] = rie.get_value() - + return headers def http_request(self, method, uri, headers=None, body='') : diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 06496faf..60499a4e 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -246,7 +246,7 @@ def get(self, robj, r=None, vtag=None): else: return 0 - def put(self, robj, w=None, dw=None, return_body=True): + def put(self, robj, w=None, dw=None, return_body=True, if_none_match=False): """ Serialize get request and deserialize response """ @@ -257,6 +257,8 @@ def put(self, robj, w=None, dw=None, return_body=True): req.dw = self.translate_rw_val(dw) if return_body: req.return_body = 1 + if if_none_match: + req.if_none_match = 1 req.bucket = bucket.get_name() req.key = robj.get_key() @@ -274,7 +276,7 @@ def put(self, robj, w=None, dw=None, return_body=True): contents.append(self.decode_content(c)) return resp.vclock, contents - def put_new(self, robj, w=None, dw=None, return_meta=True): + def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): """Put a new object into the Riak store, returning its (new) key. If return_meta is False, then the vlock and metadata return values @@ -289,6 +291,8 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): req.dw = self.translate_rw_val(dw) if return_meta: req.return_body = 1 + if if_none_match: + req.if_none_match = 1 req.bucket = bucket.get_name() From a586849036f5acd5ffc300e9adf2abc2abb5c04f Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 6 Mar 2012 16:53:39 -0800 Subject: [PATCH 097/118] Tweaking the Http transport --- riak/transports/http.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index c00af22a..e207df72 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -118,7 +118,7 @@ def put(self, robj, w = None, dw = None, return_body = True, if_none_match=False params=params) headers = self.build_put_headers(robj) if if_none_match: - headers["If-None-Match"] = "1" + headers["If-None-Match"] = "*" content = robj.get_encoded_data() return self.do_put(url, headers, content, return_body, key=robj.get_key()) @@ -140,6 +140,8 @@ def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): params = {'returnbody' : str(return_meta).lower(), 'w' : w, 'dw' : dw} url = self.build_rest_path(bucket=robj.get_bucket(), params=params) headers = self.build_put_headers(robj) + if if_none_match: + headers["If-None-Match"] = "*" content = robj.get_encoded_data() response = self.http_request('POST', url, headers, content) location = response[0]['location'] From 02738ab2625f8cca0bb6aa8b4bf94b4acd3056b1 Mon Sep 17 00:00:00 2001 From: Brian Roach Date: Fri, 16 Mar 2012 14:52:35 -0600 Subject: [PATCH 098/118] Tiny doc fix --- riak/riak_object.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index cd9c8af7..e43aee30 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -214,7 +214,7 @@ def remove_index(self, field, value): def set_indexes(self, indexes): """ - Sets indexes once and for all. Currenly supports an iterable of 2 item tuples, + Replaces all indexes on a Riak object. Currenly supports an iterable of 2 item tuples, (field, value) :param indexes: iterable of 2 item tuples consisting the field and value. @@ -286,7 +286,7 @@ def set_links(self, links, all_link=False): :param links: An iterable of 2-item tuples, consisting of (RiakObject, tag). This could also be an iterable of just a RiakObject, instead of the tuple, then a tag of None would be used. Lastly, it could also be an iterable of RiakLink. They have tags built-in. - :param all_link: A boolean indicates if links is all RiakLink object + :param all_link: A boolean indicates if links are all RiakLink objects This speeds up the operation. """ if all_link: From f622f01a40309b284d9f2bf39de9f5d7c2371cb4 Mon Sep 17 00:00:00 2001 From: Sreejith Kesavan Date: Wed, 28 Mar 2012 14:53:11 +0530 Subject: [PATCH 099/118] Never fail without a reason --- riak/transports/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 301c7465..2d0d84f4 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -244,7 +244,7 @@ def mapred(self, inputs, query, timeout=None): def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] if not status in expected_statuses: - m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) + m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) + ':' + response[1] raise Exception(m) def parse_body(self, response, expected_statuses): From 7f1129776bb7250f0328d887c6edb31300a46dd1 Mon Sep 17 00:00:00 2001 From: Sreejith Kesavan Date: Wed, 28 Mar 2012 14:57:06 +0530 Subject: [PATCH 100/118] Removing ambiguous code. Already checked in check_http_code. --- riak/transports/http.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 2d0d84f4..e25be682 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -272,11 +272,6 @@ def parse_body(self, response, expected_statuses): m = 'Could not contact Riak Server: http://$HOST:$PORT !' raise RiakError(m) - # Verify that we got one of the expected statuses. Otherwise, raise an exception. - if not status in expected_statuses: - m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) - raise RiakError(m) - # If 404(Not Found), then clear the object. if status == 404: return None From 871d1495370c8940765bf9fecdabab6925e0d983 Mon Sep 17 00:00:00 2001 From: Sreejith Kesavan Date: Wed, 28 Mar 2012 15:01:44 +0530 Subject: [PATCH 101/118] Fixing string format --- riak/transports/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index e25be682..1fb7147e 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -244,7 +244,7 @@ def mapred(self, inputs, query, timeout=None): def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] if not status in expected_statuses: - m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) + ':' + response[1] + m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) + ' : ' + response[1] raise Exception(m) def parse_body(self, response, expected_statuses): From 13de673ec98a41666f90cd3f3c00fce69920b265 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 28 Mar 2012 17:44:40 -0400 Subject: [PATCH 102/118] Added test for if_none_match functionality. --- riak/tests/test_all.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index b6ee3b17..4e743890 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -222,6 +222,20 @@ def test_rw_settings(self): bucket.set_rw("one") self.assertEqual(bucket.get_rw(), "one") + def test_if_none_match(self): + bucket = self.client.bucket('if_none_match_test') + obj = bucket.get('obj') + obj.delete() + + obj.reload() + self.assertFalse(obj.exists()) + obj.set_data(["first store"]) + obj.store() + + obj.set_data(["second store"]) + with self.assertRaises(Exception): + obj.store(if_none_match=True) + def test_siblings(self): # Set up the bucket, clear any existing object... bucket = self.client.bucket('multiBucket') From 8f012cb2e6d39f2f91fcdf6e1ea33e2179bc7668 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 28 Mar 2012 18:33:34 -0400 Subject: [PATCH 103/118] PR/PW step 1: add accessors to bucket and client. --- riak/bucket.py | 55 +++++++++++++++++++++++++++++++++++++++++- riak/client.py | 42 ++++++++++++++++++++++++++++++++ riak/tests/test_all.py | 11 +++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/riak/bucket.py b/riak/bucket.py index 83818fa6..3cf1e3af 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -50,6 +50,8 @@ def __init__(self, client, name): self._w = None self._dw = None self._rw = None + self._pr = None + self._pw = None self._encoders = {} self._decoders = {} @@ -159,6 +161,57 @@ def set_rw(self, rw): self._rw = rw return self + def get_pr(self, pr=None): + """ + Get the PR-value for this bucket, if it is set, otherwise return + the PR-value for the client. + + :rtype: integer + """ + if (pr is not None): + return pr + if (self._pr is not None): + return self._pr + return self._client.get_pr() + + def set_pr(self, pr): + """ + Set the PR-value for this bucket. See :func:`set_r` for more + information. + + :param pr: The new PR-value + :type pr: integer + :rtype: self + """ + self._pr = pr + return self + + + def get_pw(self, pw=None): + """ + Get the PW-value for this bucket, if it is set, otherwise return + the PW-value for the client. + + :rtype: integer + """ + if (pw is not None): + return pw + if (self._pw is not None): + return self._pw + return self._client.get_pw() + + def set_pw(self, pw): + """ + Set the PW-value for this bucket. See :func:`set_r` for more + information. + + :param pw: The new PR-value + :type pw: integer + :rtype: self + """ + self._pw = pw + return self + def get_encoder(self, content_type): """ Get the encoding function for the provided content type for this bucket. @@ -428,7 +481,7 @@ def new_binary_from_file(self, key, filename): if not mimetype: mimetype = 'application/octet-stream' return self.new_binary(key, binary_data, mimetype) - + def search_enabled(self): """ Returns True if the search precommit hook is enabled for this bucket. diff --git a/riak/client.py b/riak/client.py index 5e78fe27..e53f2347 100644 --- a/riak/client.py +++ b/riak/client.py @@ -85,6 +85,8 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', self._w = "default" self._dw = "default" self._rw = "default" + self._pr = "default" + self._pw = "default" self._encoders = {'application/json': json.dumps, 'text/json': json.dumps} self._decoders = {'application/json': json.loads, @@ -182,6 +184,46 @@ def set_rw(self, rw): self._rw = rw return self + def get_pr(self): + """ + Get the PR-value setting for this ``RiakClient``. (default 0) + + :rtype: integer + """ + return self._pr + + def set_pr(self, pr): + """ + Set the PR-value for this ``RiakClient`` instance. See :func:`set_r` for a + description of how these values are used. + + :param pr: The PR value. + :type pr: integer + :rtype: self + """ + self._pr = pr + return self + + def get_pw(self): + """ + Get the PW-value setting for this ``RiakClient``. (default 0) + + :rtype: integer + """ + return self._pr + + def set_pw(self, pw): + """ + Set the PW-value for this ``RiakClient`` instance. See :func:`set_r` for a + description of how these values are used. + + :param pw: The W value. + :type pw: integer + :rtype: self + """ + self._pr = pr + return self + def get_client_id(self): """ Get the ``client_id`` for this ``RiakClient`` instance. diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 70281e5b..17610406 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -222,6 +222,17 @@ def test_rw_settings(self): bucket.set_rw("one") self.assertEqual(bucket.get_rw(), "one") + def test_primary_quora(self): + bucket = self.client.bucket('primary_quora') + self.assertEqual(bucket.get_pr(), "default") + self.assertEqual(bucket.get_pw(), "default") + + bucket.set_pr(1) + self.assertEqual(bucket.get_pr(), 1) + + bucket.set_pw("quorum") + self.assertEqual(bucket.get_pw(), "quorum") + def test_if_none_match(self): bucket = self.client.bucket('if_none_match_test') obj = bucket.get('obj') From 7a9bec2b8783b2ae8b0c0c029886c028c5e31695 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 28 Mar 2012 19:36:05 -0400 Subject: [PATCH 104/118] PR/PW step 1.5: Exclude query params that are 'None'. --- riak/tests/test_all.py | 2 ++ riak/transports/http.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 17610406..0a434d1c 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -1169,6 +1169,8 @@ def test_delete_documents_from_search_by_query_and_id(self): results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results["response"]["docs"])) + def test_build_rest_path_excludes_empty_query_params(self): + self.assertEquals(self.client.get_transport().build_rest_path(bucket=self.client.bucket("foo"), key="bar", params={'r': None}), "/riak/foo/bar?") class RiakTestFilter(unittest.TestCase): def test_simple(self): diff --git a/riak/transports/http.py b/riak/transports/http.py index 14468e34..e1e85184 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -418,8 +418,9 @@ def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : if params is not None: s = '' for key in params.keys(): - if s != '': s += '&' - s += urllib.quote_plus(key) + '=' + urllib.quote_plus(str(params[key])) + if params[key] is not None: + if s != '': s += '&' + s += urllib.quote_plus(key) + '=' + urllib.quote_plus(str(params[key])) path += '?' + s # Return. From d3baaebbdef4f8a846f4921ba22c5768be6e4f2d Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 28 Mar 2012 19:47:03 -0400 Subject: [PATCH 105/118] PR/PW step 2: Add support for primary quora. * return_meta/return_body are now consistent across Transport.put()/put_new(). * riak_object.store() supports pw. * riak_bucket.get() and riak_bucket.get_binary() support pr. * riak_object.delete() supports r, pr, w, dw, pw. * Some internal method calls were changed to use named arguments where necessary. --- riak/bucket.py | 14 ++++++++--- riak/riak_object.py | 41 +++++++++++++++++++++++++------ riak/transports/http.py | 19 +++++++------- riak/transports/pbc.py | 22 +++++++++++++---- riak/transports/riakclient_pb2.py | 23 ++++++++++++++++- setup.py | 11 +++------ 6 files changed, 95 insertions(+), 35 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 3cf1e3af..64c492c0 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -297,7 +297,7 @@ def new_binary(self, key, data, content_type='application/octet-stream'): obj._encode_data = False return obj - def get(self, key, r=None): + def get(self, key, r=None, pr=None): """ Retrieve a JSON-encoded object from Riak. @@ -305,14 +305,17 @@ def get(self, key, r=None): :type key: string :param r: R-Value of the request (defaults to bucket's R) :type r: integer + :param pr: PR-Value of the request (defaults to bucket's PR) + :type pr: integer :rtype: :class:`RiakObject ` """ obj = RiakObject(self._client, self, key) obj._encode_data = True r = self.get_r(r) - return obj.reload(r) + pr = self.get_pr(pr) + return obj.reload(r=r, pr=pr) - def get_binary(self, key, r=None): + def get_binary(self, key, r=None, pr=None): """ Retrieve a binary/string object from Riak. @@ -320,12 +323,15 @@ def get_binary(self, key, r=None): :type key: string :param r: R-Value of the request (defaults to bucket's R) :type r: integer + :param pr: PR-Value of the request (defaults to bucket's PR) + :type pr: integer :rtype: :class:`RiakObject ` """ obj = RiakObject(self._client, self, key) obj._encode_data = False r = self.get_r(r) - return obj.reload(r) + pr = self.get_pr(pr) + return obj.reload(r=r, pr=pr) def set_n_val(self, nval): """ diff --git a/riak/riak_object.py b/riak/riak_object.py index b0d8a784..3a3351a6 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -367,7 +367,7 @@ def get_links(self): else: return [] - def store(self, w=None, dw=None, return_body=True, if_none_match=False): + def store(self, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """ Store the object in Riak. When this operation completes, the object could contain new metadata and possibly new data if Riak @@ -380,6 +380,9 @@ def store(self, w=None, dw=None, return_body=True, if_none_match=False): :param dw: DW-value, wait for this many partitions to confirm the write before returning to client. :type dw: integer + :param pw: PW-value, require this many primary partitions to be available + before performing the put + :type pw: integer :param return_body: if the newly stored object should be retrieved :type return_body: bool :param if_none_match: Should the object be stored only if there is no @@ -390,25 +393,26 @@ def store(self, w=None, dw=None, return_body=True, if_none_match=False): # Use defaults if not specified... w = self._bucket.get_w(w) dw = self._bucket.get_dw(dw) + pw = self._bucket.get_pw(pw) # Issue the put over our transport t = self._client.get_transport() if self._key is None: - key, vclock, metadata = t.put_new(self, w, dw, return_body, if_none_match) + key, vclock, metadata = t.put_new(self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) self._exists = True self._key = key self._vclock = vclock self.set_metadata(metadata) else: - Result = t.put(self, w, dw, return_body, if_none_match) + Result = t.put(self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) if Result is not None: self.populate(Result) return self - def reload(self, r=None, vtag=None): + def reload(self, r=None, pr=None, vtag=None): """ Reload the object from Riak. When this operation completes, the object could contain new metadata and a new value, if the object @@ -421,8 +425,9 @@ def reload(self, r=None, vtag=None): """ # Do the request... r = self._bucket.get_r(r) + pr = self._bucket.get_pr(pr) t = self._client.get_transport() - Result = t.get(self, r, vtag) + Result = t.get(self, r=r, pr=pr, vtag=vtag) self.clear() if Result is not None: @@ -431,19 +436,39 @@ def reload(self, r=None, vtag=None): return self - def delete(self, rw=None): + def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Delete this object from Riak. :param rw: RW-value. Wait until this many partitions have - deleted the object before responding. + deleted the object before responding. (deprecated in Riak 1.0+, use R/W/DW) :type rw: integer + :param r: R-value, wait for this many partitions to read object + before performing the put + :type r: integer + :param w: W-value, wait for this many partitions to respond + before returning to client. + :type w: integer + :param dw: DW-value, wait for this many partitions to + confirm the write before returning to client. + :type dw: integer + :param pr: PR-value, require this many primary partitions to be available + before performing the read that precedes the put + :type pr: integer + :param pr: PW-value, require this many primary partitions to be available + before performing the put + :type pw: integer :rtype: self """ # Use defaults if not specified... rw = self._bucket.get_rw(rw) + r = self._bucket.get_r(r) + w = self._bucket.get_w(w) + dw = self._bucket.get_dw(dw) + pr = self._bucket.get_pr(pr) + pw = self._bucket.get_pw(pw) t = self._client.get_transport() - Result = t.delete(self, rw) + Result = t.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) self.clear() return self diff --git a/riak/transports/http.py b/riak/transports/http.py index e1e85184..c77955b6 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -96,11 +96,11 @@ def ping(self) : return(response is not None) and (response[1] == 'OK') - def get(self, robj, r, vtag = None) : + def get(self, robj, r = None, pr = None, vtag = None) : """ Get a bucket/key from the server """ - params = {'r' : r} + params = {'r' : r, 'pr': pr} if vtag is not None: params['vtag'] = vtag url = self.build_rest_path(robj.get_bucket(), robj.get_key(), @@ -108,12 +108,12 @@ def get(self, robj, r, vtag = None) : response = self.http_request('GET', url) return self.parse_body(response, [200, 300, 404]) - def put(self, robj, w = None, dw = None, return_body = True, if_none_match=False): + def put(self, robj, w = None, dw = None, pw = None, return_body = True, if_none_match=False): """ Serialize put request and deserialize response """ # Construct the URL... - params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw} + params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw, 'pw' : pw } url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), params=params) headers = self.build_put_headers(robj) @@ -134,10 +134,10 @@ def do_put(self, url, headers, content, return_body=False, key=None): self.check_http_code(response, [204]) return None - def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): + def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """Put a new object into the Riak store, returning its (new) key.""" # Construct the URL... - params = {'returnbody' : str(return_meta).lower(), 'w' : w, 'dw' : dw} + params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw, 'pw' : pw} url = self.build_rest_path(bucket=robj.get_bucket(), params=params) headers = self.build_put_headers(robj) if if_none_match: @@ -147,18 +147,19 @@ def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): location = response[0]['location'] idx = location.rindex('/') key = location[idx+1:] - if return_meta: + if return_body: vclock, [(metadata, data)] = self.parse_body(response, [201]) return key, vclock, metadata else: self.check_http_code(response, [201]) return key, None, None - def delete(self, robj, rw): + def delete(self, robj, rw=None, r = None, w = None, dw = None, pr = None, pw = None): # Construct the URL... - params = {'rw' : rw} + params = {'rw' : rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} url = self.build_rest_path(robj.get_bucket(), robj.get_key(), params=params) + # TODO: Send vclock of robj if it exists # Run the operation.. response = self.http_request('DELETE', url) self.check_http_code(response, [204, 404]) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 60499a4e..886558a9 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -221,7 +221,7 @@ def set_client_id(self, client_id): return True - def get(self, robj, r=None, vtag=None): + def get(self, robj, r=None, pr=None, vtag=None): """ Serialize get request and deserialize response """ @@ -232,6 +232,7 @@ def get(self, robj, r=None, vtag=None): req = riakclient_pb2.RpbGetReq() req.r = self.translate_rw_val(r) + req.pr = self.translate_rw_val(pr) req.bucket = bucket.get_name() req.key = robj.get_key() @@ -246,7 +247,7 @@ def get(self, robj, r=None, vtag=None): else: return 0 - def put(self, robj, w=None, dw=None, return_body=True, if_none_match=False): + def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """ Serialize get request and deserialize response """ @@ -255,6 +256,8 @@ def put(self, robj, w=None, dw=None, return_body=True, if_none_match=False): req = riakclient_pb2.RpbPutReq() req.w = self.translate_rw_val(w) req.dw = self.translate_rw_val(dw) + req.pw = self.translate_rw_val(pw) + if return_body: req.return_body = 1 if if_none_match: @@ -276,7 +279,7 @@ def put(self, robj, w=None, dw=None, return_body=True, if_none_match=False): contents.append(self.decode_content(c)) return resp.vclock, contents - def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): + def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """Put a new object into the Riak store, returning its (new) key. If return_meta is False, then the vlock and metadata return values @@ -289,7 +292,9 @@ def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): req = riakclient_pb2.RpbPutReq() req.w = self.translate_rw_val(w) req.dw = self.translate_rw_val(dw) - if return_meta: + req.pw = self.translate_rw_val(pw) + + if return_body: req.return_body = 1 if if_none_match: req.if_none_match = 1 @@ -308,7 +313,7 @@ def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): metadata, content = self.decode_content(resp.content[0]) return resp.key, resp.vclock, metadata - def delete(self, robj, rw=None): + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Serialize get request and deserialize response """ @@ -316,6 +321,13 @@ def delete(self, robj, rw=None): req = riakclient_pb2.RpbDelReq() req.rw = self.translate_rw_val(rw) + req.r = self.translate_rw_val(r) + req.w = self.translate_rw_val(w) + req.dw = self.translate_rw_val(dw) + req.pr = self.translate_rw_val(pr) + req.pw = self.translate_rw_val(pw) + + # TODO: Set the vclock if present req.bucket = bucket.get_name() req.key = robj.get_key() diff --git a/riak/transports/riakclient_pb2.py b/riak/transports/riakclient_pb2.py index 2715abff..e607fe72 100644 --- a/riak/transports/riakclient_pb2.py +++ b/riak/transports/riakclient_pb2.py @@ -7,6 +7,7 @@ # @@protoc_insertion_point(imports) + DESCRIPTOR = descriptor.FileDescriptor( name='riakclient.proto', package='', @@ -952,7 +953,6 @@ serialized_end=1640, ) - _RPBGETRESP.fields_by_name['content'].message_type = _RPBCONTENT _RPBPUTREQ.fields_by_name['content'].message_type = _RPBCONTENT _RPBPUTRESP.fields_by_name['content'].message_type = _RPBCONTENT @@ -961,6 +961,27 @@ _RPBCONTENT.fields_by_name['links'].message_type = _RPBLINK _RPBCONTENT.fields_by_name['usermeta'].message_type = _RPBPAIR _RPBCONTENT.fields_by_name['indexes'].message_type = _RPBPAIR +DESCRIPTOR.message_types_by_name['RpbErrorResp'] = _RPBERRORRESP +DESCRIPTOR.message_types_by_name['RpbGetClientIdResp'] = _RPBGETCLIENTIDRESP +DESCRIPTOR.message_types_by_name['RpbSetClientIdReq'] = _RPBSETCLIENTIDREQ +DESCRIPTOR.message_types_by_name['RpbGetServerInfoResp'] = _RPBGETSERVERINFORESP +DESCRIPTOR.message_types_by_name['RpbGetReq'] = _RPBGETREQ +DESCRIPTOR.message_types_by_name['RpbGetResp'] = _RPBGETRESP +DESCRIPTOR.message_types_by_name['RpbPutReq'] = _RPBPUTREQ +DESCRIPTOR.message_types_by_name['RpbPutResp'] = _RPBPUTRESP +DESCRIPTOR.message_types_by_name['RpbDelReq'] = _RPBDELREQ +DESCRIPTOR.message_types_by_name['RpbListBucketsResp'] = _RPBLISTBUCKETSRESP +DESCRIPTOR.message_types_by_name['RpbListKeysReq'] = _RPBLISTKEYSREQ +DESCRIPTOR.message_types_by_name['RpbListKeysResp'] = _RPBLISTKEYSRESP +DESCRIPTOR.message_types_by_name['RpbGetBucketReq'] = _RPBGETBUCKETREQ +DESCRIPTOR.message_types_by_name['RpbGetBucketResp'] = _RPBGETBUCKETRESP +DESCRIPTOR.message_types_by_name['RpbSetBucketReq'] = _RPBSETBUCKETREQ +DESCRIPTOR.message_types_by_name['RpbMapRedReq'] = _RPBMAPREDREQ +DESCRIPTOR.message_types_by_name['RpbMapRedResp'] = _RPBMAPREDRESP +DESCRIPTOR.message_types_by_name['RpbContent'] = _RPBCONTENT +DESCRIPTOR.message_types_by_name['RpbPair'] = _RPBPAIR +DESCRIPTOR.message_types_by_name['RpbLink'] = _RPBLINK +DESCRIPTOR.message_types_by_name['RpbBucketProps'] = _RPBBUCKETPROPS class RpbErrorResp(message.Message): __metaclass__ = reflection.GeneratedProtocolMessageType diff --git a/setup.py b/setup.py index 7bcb9360..c97fb055 100755 --- a/setup.py +++ b/setup.py @@ -16,20 +16,15 @@ def make_pb(): subprocess.call(['protoc', '--python_out=.', './riak/transports/riakclient.proto']) if __name__ == "__main__": + requires = ['protobuf(==2.4.1)'] if platform.python_version() < '2.7': - test_require = ["unittest2"] - else: - test_require = [] + requires.append("unittest2") setup( name='riak', version='1.3.0', packages = find_packages(), - requires = test_require, - extras_require = { - 'protobuf': ['protobuf>=2.3.0'], - }, - dependency_links = ["http://downloads.basho.com/support"], + requires = requires, package_data = { '' : ['*.proto'], 'riak' : ['erl_src/*'] From 6f0cb113ee94ca20eabb5430eea1cc70923474e8 Mon Sep 17 00:00:00 2001 From: Sreejith K Date: Thu, 29 Mar 2012 12:50:03 +0530 Subject: [PATCH 106/118] Remove all indexes for a field --- riak/riak_object.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index b0d8a784..d86b95f2 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -197,7 +197,7 @@ def add_index(self, field, value): return self - def remove_index(self, field, value): + def remove_index(self, field=None, value=None): """ Remove the specified field/value pair as an index on this object. @@ -207,11 +207,22 @@ def remove_index(self, field, value): :type value: string or integer :rtype: self """ - rie = RiakIndexEntry(field, value) - if rie in self._metadata[MD_INDEX]: - self._metadata[MD_INDEX].remove(rie) + if not field and not value: + ries = self._metadata[MD_INDEX] + elif field and not value: + ries = [x for x in self._metadata[MD_INDEX] if x.get_field() == field] + elif field and value: + ries = [RiakIndexEntry(field, value)] + else: + raise Exception("Cannot pass value without a field name while removing index") + + for rie in ries: + if rie in self._metadata[MD_INDEX]: + self._metadata[MD_INDEX].remove(rie) return self + remove_indexes = remove_index + def set_indexes(self, indexes): """ Replaces all indexes on a Riak object. Currenly supports an iterable of 2 item tuples, From 434dd805ca1b3c2e7ff7f13e92ec749c0197e7ff Mon Sep 17 00:00:00 2001 From: Sreejith K Date: Thu, 29 Mar 2012 12:53:42 +0530 Subject: [PATCH 107/118] Using a copy of Index metadata --- riak/riak_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index d86b95f2..59a18c82 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -208,7 +208,7 @@ def remove_index(self, field=None, value=None): :rtype: self """ if not field and not value: - ries = self._metadata[MD_INDEX] + ries = self._metadata[MD_INDEX][:] elif field and not value: ries = [x for x in self._metadata[MD_INDEX] if x.get_field() == field] elif field and value: From 7fe0524065b41092dcb67472e2161292a04091b8 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 10:04:50 -0400 Subject: [PATCH 108/118] Move test_generate_key to base test case (supported by both transports) and resolve key counts problem. Fixes #96. --- riak/tests/test_all.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 70281e5b..d3dbdb21 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -126,6 +126,19 @@ def test_store_and_get(self): self.assertRaises(TypeError, bucket.new, 'foo', u'éå') self.assertRaises(TypeError, bucket.new, 'foo', u'éå') + def test_generate_key(self): + # Ensure that Riak generates a random key when + # the key passed to bucket.new() is None. + bucket = self.client.bucket('random_key_bucket') + existing_keys = bucket.get_keys() + o = bucket.new(None, data={}) + self.assertIsNone(o.get_key()) + o.store() + self.assertIsNotNone(o.get_key()) + self.assertNotIn('/', o.get_key()) + self.assertNotIn(o.get_key(), existing_keys) + self.assertEqual(len(bucket.get_keys()), len(existing_keys) + 1) + def test_binary_store_and_get(self): bucket = self.client.bucket('bucket') # Store as binary, retrieve as binary, then compare... @@ -1018,19 +1031,6 @@ def test_no_returnbody(self): o = bucket.new("foo", "bar").store(return_body=False) self.assertEqual(o.vclock(), None) - def test_generate_key(self): - # Ensure that Riak generates a random key when - # the key passed to bucket.new() is None. - bucket = self.client.bucket('random_key_bucket') - for key in bucket.get_keys(): - bucket.get(str(key)).delete() - o = bucket.new(None, data={}) - self.assertIsNone(o.get_key()) - o.store() - self.assertIsNotNone(o.get_key()) - self.assertNotIn('/', o.get_key()) - self.assertEqual(len(bucket.get_keys()), 1) - def test_too_many_link_headers_shouldnt_break_http(self): bucket = self.client.bucket("bucket") o = bucket.new("lots_of_links", "My god, it's full of links!") From ea666fbcd29b37be39f342a7cd9dfee58b73ebbf Mon Sep 17 00:00:00 2001 From: Sreejith K Date: Thu, 29 Mar 2012 20:06:21 +0530 Subject: [PATCH 109/118] Tests for remove_indexes --- riak/tests/test_all.py | 50 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 70281e5b..5c50d647 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -685,6 +685,56 @@ def test_set_indexes(self): self.assertEqual(1, len(result)) self.assertEqual('foo', result[0].get_key()) + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_remove_indexes(self): + if not self.is_2i_supported(): + return True + + bucket = self.client.bucket('indexbucket') + bar = bucket.new('bar', 1).add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + result = self.client.index('indexbucket', 'bar_int', 1).run() + self.assertEqual(1, len(result)) + self.assertEqual(3, len(bar.get_indexes())) + self.assertEqual(2, len(bar.get_indexes('bar_int'))) + + # remove all indexes + bar = bar.remove_indexes().store() + result = self.client.index('indexbucket', 'bar_int', 1).run() + self.assertEqual(0, len(result)) + result = self.client.index('indexbucket', 'baz_bin', 'baz').run() + self.assertEqual(0, len(result)) + self.assertEqual(0, len(bar.get_indexes())) + self.assertEqual(0, len(bar.get_indexes('bar_int'))) + self.assertEqual(0, len(bar.get_indexes('baz_bin'))) + + # add index again + bar = bar.add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + # remove all index with field='bar_int' + bar = bar.remove_index(field='bar_int').store() + result = self.client.index('indexbucket', 'bar_int', 1).run() + self.assertEqual(0, len(result)) + result = self.client.index('indexbucket', 'bar_int', 2).run() + self.assertEqual(0, len(result)) + result = self.client.index('indexbucket', 'baz_bin', 'baz').run() + self.assertEqual(1, len(result)) + self.assertEqual(1, len(bar.get_indexes())) + self.assertEqual(0, len(bar.get_indexes('bar_int'))) + self.assertEqual(1, len(bar.get_indexes('baz_bin'))) + + # add index again + bar = bar.add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + # remove an index field value pair + bar = bar.remove_index(field='bar_int', value=2).store() + result = self.client.index('indexbucket', 'bar_int', 1).run() + self.assertEqual(1, len(result)) + result = self.client.index('indexbucket', 'bar_int', 2).run() + self.assertEqual(0, len(result)) + result = self.client.index('indexbucket', 'baz_bin', 'baz').run() + self.assertEqual(1, len(result)) + self.assertEqual(2, len(bar.get_indexes())) + self.assertEqual(1, len(bar.get_indexes('bar_int'))) + self.assertEqual(1, len(bar.get_indexes('baz_bin'))) + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_query(self): if not self.is_2i_supported(): From 16d5ca34bb14ab3be4548bcdbb59609e8e0fc721 Mon Sep 17 00:00:00 2001 From: Sreejith Kesavan Date: Thu, 29 Mar 2012 23:42:37 +0530 Subject: [PATCH 110/118] Result in RiakObject.populate doesn't have MD_INDEX field when using RiakPbcTransport. --- riak/riak_object.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/riak/riak_object.py b/riak/riak_object.py index 59a18c82..190d0c81 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -500,6 +500,8 @@ def populate(self, Result) : if len(contents) > 0: (metadata, data) = contents.pop(0) self._exists = True + if not metadata.has_key(MD_INDEX): + metadata[MD_INDEX] = [] self.set_metadata(metadata) self.set_encoded_data(data) # Create objects for all siblings From e68c188fd50f420576003f8896d9c7cb3ce1e945 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 14:44:40 -0400 Subject: [PATCH 111/118] Add Travis-CI support. --- .travis.yml | 10 ++++++++++ README.rst | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..bce57023 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: python +python: + - "2.6" + - "2.7" +install: python setup.py develop +script: python setup.py test +# TODO: get search enabled on Travis' Riak install, or fix the test server +env: "SKIP_LUWAK=1 SKIP_SEARCH=1" +notifications: + email: clients@basho.com diff --git a/README.rst b/README.rst index 17565f96..75a0bc81 100644 --- a/README.rst +++ b/README.rst @@ -2,6 +2,7 @@ Python Client for Riak ======================== +.. image:: https://secure.travis-ci.org/basho/riak-python-client.png?branch=master Documentation ============== @@ -11,7 +12,7 @@ The documentation source is found in `docs/ subdirectory `_ and can be built with `Sphinx `_. -Documentation for Riak is available at http://wiki.basho.com/How-Things-Work.html +Documentation for Riak is available at http://wiki.basho.com/Riak.html Install ======= From bcda5d70f3d56f85cf9ed895283d36f76a7c0296 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 15:06:03 -0400 Subject: [PATCH 112/118] Manually install protobuf using setup.py. --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bce57023..40b19756 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,9 @@ language: python python: - "2.6" - "2.7" -install: python setup.py develop +install: + - python setup.py easy_install protobuf=2.4.1 + - python setup.py develop script: python setup.py test # TODO: get search enabled on Travis' Riak install, or fix the test server env: "SKIP_LUWAK=1 SKIP_SEARCH=1" From 7a6931987bd4698546b87a8cb2338c5bf955c20b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 15:31:30 -0400 Subject: [PATCH 113/118] Fixing dependencies and build script again. --- .travis.yml | 6 ++---- setup.py | 10 ++++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 40b19756..d4c4ec76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,8 @@ language: python python: - "2.6" - "2.7" -install: - - python setup.py easy_install protobuf=2.4.1 - - python setup.py develop -script: python setup.py test +install: ./setup.py develop +script: ./setup.py test # TODO: get search enabled on Travis' Riak install, or fix the test server env: "SKIP_LUWAK=1 SKIP_SEARCH=1" notifications: diff --git a/setup.py b/setup.py index c97fb055..3de069a2 100755 --- a/setup.py +++ b/setup.py @@ -16,15 +16,17 @@ def make_pb(): subprocess.call(['protoc', '--python_out=.', './riak/transports/riakclient.proto']) if __name__ == "__main__": - requires = ['protobuf(==2.4.1)'] + install_requires = {'protobuf': ['>= 2.4.0', '< 2.5.0'] } + tests_require = [] if platform.python_version() < '2.7': - requires.append("unittest2") + tests_require.append("unittest2") setup( name='riak', version='1.3.0', packages = find_packages(), - requires = requires, + install_requires = install_requires, + tests_require = tests_require, package_data = { '' : ['*.proto'], 'riak' : ['erl_src/*'] @@ -35,7 +37,7 @@ def make_pb(): license='Apache 2', platforms='Platform Independent', author='Basho Technologies', - author_email='riak@basho.com', + author_email='clients@basho.com', test_suite='riak.tests.suite', url='https://github.com/basho/riak-python-client' ) From c935fc2b1706eed683ecdfaeb257b436103f0d9f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 15:53:51 -0400 Subject: [PATCH 114/118] Update build status image to link to the build status page. [ci skip] --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 75a0bc81..62cf297e 100644 --- a/README.rst +++ b/README.rst @@ -3,6 +3,7 @@ Python Client for Riak ======================== .. image:: https://secure.travis-ci.org/basho/riak-python-client.png?branch=master + :target: http://travis-ci.org/basho/riak-python-client Documentation ============== From 28b83f2b45b04500591d908af9232775a464d4f5 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 17:12:16 -0400 Subject: [PATCH 115/118] Missed this spot where arguments were positional. --- riak/riak_object.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 1eefd6c8..6a36a905 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -557,7 +557,7 @@ def get_sibling_count(self): """ return len(self._siblings) - def get_sibling(self, i, r=None): + def get_sibling(self, i, r=None, pr=None): """ Retrieve a sibling by sibling number. @@ -573,11 +573,12 @@ def get_sibling(self, i, r=None): else: # Use defaults if not specified. r = self._bucket.get_r(r) + pr = self._bucket.get_pr(pr) # Run the request... vtag = self._siblings[i] obj = RiakObject(self._client, self._bucket, self._key) - obj.reload(r, vtag) + obj.reload(r=r, pr=pr, vtag=vtag) # And make sure it knows who it's siblings are self._siblings[i] = obj From a5902a46ad04220e9309ddae46fd28f85a3b17f7 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 17:25:11 -0400 Subject: [PATCH 116/118] Now HTTP and PBC both fail in the same place in test_siblings. --- riak/tests/test_all.py | 6 +++--- riak/transports/pbc.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index efe9f718..9bfe99a4 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -264,7 +264,7 @@ def test_siblings(self): # Set up the bucket, clear any existing object... bucket = self.client.bucket('multiBucket') bucket.set_allow_multiples(True) - obj = bucket.get('foo') + obj = bucket.get_binary('foo') obj.delete() obj.reload() @@ -281,9 +281,9 @@ def test_siblings(self): if randval not in vals: break - other_obj = other_bucket.new('foo', randval) + other_obj = other_bucket.new_binary('foo', str(randval)) other_obj.store() - vals.add(randval) + vals.add(str(randval)) # Make sure the object has itself plus four siblings... obj.reload() diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 886558a9..0538c4f0 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -245,7 +245,7 @@ def get(self, robj, r=None, pr=None, vtag=None): contents.append(self.decode_content(c)) return resp.vclock, contents else: - return 0 + return None def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """ From 0f794c94503f3d543e6f4f5e5393636bb215553c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 17:35:15 -0400 Subject: [PATCH 117/118] Use proper behavior for sibling generation in test_siblings. --- riak/tests/test_all.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 9bfe99a4..3ae08a4e 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -265,11 +265,10 @@ def test_siblings(self): bucket = self.client.bucket('multiBucket') bucket.set_allow_multiples(True) obj = bucket.get_binary('foo') - obj.delete() - - obj.reload() - self.assertFalse(obj.exists()) - self.assertEqual(obj.get_data(), None) + # Even if it previously existed, let's store a base resolved version + # from which we can diverge by sending a stale vclock. + obj.set_data('start') + obj.store() # Store the same object five times... vals = set() @@ -282,6 +281,7 @@ def test_siblings(self): break other_obj = other_bucket.new_binary('foo', str(randval)) + other_obj._vclock = obj._vclock other_obj.store() vals.add(str(randval)) @@ -304,9 +304,6 @@ def test_siblings(self): self.assertEqual(obj.get_sibling_count(), 0) self.assertEqual(obj.get_data(), obj3.get_data()) - # Clean up for next test... - obj.delete() - def test_javascript_source_map(self): # Create the object... bucket = self.client.bucket("bucket") From 9f3013839c02a3303465fd5c9466e123b310b4e2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 30 Mar 2012 15:18:08 -0400 Subject: [PATCH 118/118] Release 1.4.0. --- RELEASE_NOTES.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ THANKS | 44 ++++++++++++++++++++++++++++++++++++-------- setup.py | 8 ++++++-- 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 3dc10669..fc181517 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,50 @@ # Riak Python Client Release Notes +## 1.4.0 Feature Release - 2012-03-30 + +Release 1.4.0 is a feature release comprising over 117 individual +commits. + +Noteworthy features: + +* Python 2.6 and 2.7 are supported. On 2.6, the unittest2 package is + required to run the test suite. +* Google's official protobuf package (2.4.1 or later) is now a + dependency. The package from downloads.basho.com/support is no + longer necessary. +* Travis-CI is enabled on the client. Go to + http://travis-ci.org/basho/riak-python-client for build status. +* Riak 1.0+ features, namely secondary indexes and primary quora + (PR/PW), are supported. +* `if_none_match` is a valid request option when storing objects, and + will prevent the write when set to `True` if the key already exists. +* Links can be set wholesale using the `set_links()` method. +* Transport-specific options can be passed through when creating a + `Client` object. +* A connection manager was added that will (when manipulated manually) + allow connections to multiple Riak nodes. This will be fully + integrated in a future release. + +Noteworthy bugfixes: + +* Links now use the proper URL-encoding in HTTP headers, preventing + problems with explosion from multiple encoding passes. +* Many fixes were applied to make the Protocol Buffers transport more + stable. +* `RiakObject.get_content_type()` will behave properly when content + type is not set. +* Deprecated transport classes were removed since their functionality + had folded into the primary transports. +* A temporary fix was made for unicode bucket/key names which raises + an error when they are used and cannot be coerced to ASCII. +* The Erlang sources/beams for the TestServer are now included in the + package. +* MapReduce failures will now produce a more useful error message and + be handled properly when no results are returned. + +There are lots of other great fixes from our wonderful +community. [Check them out!](https://github.com/basho/riak-python-client/compare/1.3.0...1.4.0) + ## 1.3.0 Feature Release - 2011-08-04 Release 1.3.0 is a feature release bringing a slew of updates. diff --git a/THANKS b/THANKS index c8c1371a..16927ec4 100644 --- a/THANKS +++ b/THANKS @@ -1,16 +1,44 @@ The following people have contributed to the Riak Python client: +Andrew Thompson Andy Gross -Justin Sheehy -Rusty Klophaus +Armon Dadgar +Brett Hoerner +Brian Roach +Bryan Fink +Daniel Lindsley +Daniel Néri +Daniel Reverri +David Koblas +Dmitry Rozhkov +Eric Florenzano +Eric Moritz +Filip de Waard +Gilles Devaux +Greg Nelson +Greg Stein +Gregory Burd +Ian Plosker Jayson Baird +Jeffrey Massung Jon Meredith -Eric Florenzano -Silas Sewell -Matt Heitzenroder +Josip Lisec +Justin Sheehy +Kevin Smith Mark Erdmann -Greg Nelson +Mark Phillips +Mathias Meyer +Matt Heitzenroder Mikhail Sobolev -Eric Moritz -Brett Hoerner +Reid Draper +Russell Brown +Rusty Klophaus Scott Lystig Fritchie +Sean Cribbs +Shuhao Wu +Silas Sewell +Socrates Lee +Soren Hansen +Sreejith Kesavan +Timothée Peignier +William Kral diff --git a/setup.py b/setup.py index 3de069a2..50980edc 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def make_pb(): setup( name='riak', - version='1.3.0', + version='1.4.0', packages = find_packages(), install_requires = install_requires, tests_require = tests_require, @@ -39,5 +39,9 @@ def make_pb(): author='Basho Technologies', author_email='clients@basho.com', test_suite='riak.tests.suite', - url='https://github.com/basho/riak-python-client' + url='https://github.com/basho/riak-python-client', + classifiers = ['License :: OSI Approved :: Apache Software License', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Topic :: Database'] )