From f4b43b29f7acfee6641e0fec6762c0c37f952619 Mon Sep 17 00:00:00 2001 From: Josip Lisec Date: Thu, 6 Jan 2011 20:55:09 +0100 Subject: [PATCH 0001/1060] Added support for key filters and bucket listing --- riak/client.py | 8 ++++++++ riak/mapreduce.py | 33 ++++++++++++++++++++++++++++++--- riak/tests/test_all.py | 15 +++++++++++++++ riak/transports/http.py | 26 ++++++++++++++++++++------ riak/transports/pbc.py | 21 +++++++++++++++++---- riak/transports/transport.py | 9 ++++++++- 6 files changed, 98 insertions(+), 14 deletions(-) diff --git a/riak/client.py b/riak/client.py index 91bdb099..756db971 100644 --- a/riak/client.py +++ b/riak/client.py @@ -213,6 +213,14 @@ def set_decoder(self, content_type, decoder): self._decoders[content_type] = decoder return self + def get_buckets(self): + """ + Get the list of buckets. + NOTE: Do not use this in production, as it requires traversing through + all keys stored in a cluster. + """ + return self._transport.get_buckets() + def bucket(self, name): """ Get the bucket by the specified name. Since buckets always exist, diff --git a/riak/mapreduce.py b/riak/mapreduce.py index a73d69e9..c4704cb6 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -19,6 +19,7 @@ """ import urllib from riak_object import RiakObject +from bucket import RiakBucket class RiakMapReduce(object): """ @@ -34,6 +35,7 @@ def __init__(self, client): self._client = client self._phases = [] self._inputs = [] + self._key_filters = [] self._input_mode = None def add(self, arg1, arg2=None, arg3=None): @@ -72,19 +74,33 @@ def add_bucket(self, bucket) : self._inputs = bucket return self + def add_key_filters(self, key_filters) : + if self._input_mode == 'search': + raise Exception('Key filters are not supported in search query.') + + self._key_filters.extend(key_filters) + return self + + def add_key_filter(self, *args) : + if self._input_mode == 'search': + raise Exception('Key filters are not supported in search query.') + + self._key_filters.append(args) + return self + def search(self, bucket, query): """ - Begin a map/reduce operation using a Search. This command will + Begin a map/reduce operation using a Search. This command will return an error unless executed against a Riak Search cluster. @param bucket - The bucket over which to perform the search. @param query - The search query. """ self._input_mode = 'search' - self._inputs = {'module':'riak_search', + self._inputs = {'module':'riak_search', 'function':'mapred_search', 'arg':[bucket, query]} return self - + def link(self, bucket='_', tag='_', keep=False): """ @@ -180,6 +196,17 @@ def run(self, timeout=None): if phase._keep: keep_flag = True query.append(phase.to_array()) + if (len(self._key_filters) > 0): + bucket_name = None + if (type(self._inputs) == str): + bucket_name = self._inputs + elif (type(self._inputs) == RiakBucket): + bucket_name = self._inputs.get_name() + + if (bucket_name is not None): + self._inputs = {'bucket': bucket_name, + 'key_filters': self._key_filters} + t = self._client.get_transport() result = t.mapred(self._inputs, query, timeout) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index bfbb5833..a9ca20c2 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -298,6 +298,21 @@ def test_javascript_arg_map_reduce(self): .run() self.assertEqual(result, [10]) + def test_key_filters(self): + bucket = self.client.bucket("kftest") + bucket.new("basho-20101215", 1).store() + bucket.new("google-20110103", 2).store() + bucket.new("yahoo-20090613", 3).store() + + result = self.client \ + .add("kftest") \ + .add_key_filters([["tokenize", "-", 2]]) \ + .add_key_filter("ends_with", "0613") \ + .map("function (v, keydata) { return [v.key]; }") \ + .run() + + self.assertEqual(result, ["yahoo-20090613"]) + def test_erlang_map_reduce(self): # Create the object... bucket = self.client.bucket("bucket") diff --git a/riak/transports/http.py b/riak/transports/http.py index 115cc017..3e290c0a 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -63,7 +63,7 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', self._client_id = self.make_random_client_id() def __copy__(self): - return RiakHttpTransport(self._host, self._port, self._prefix, + return RiakHttpTransport(self._host, self._port, self._prefix, self._mapred_prefix) """ @@ -134,14 +134,25 @@ def get_keys(self, bucket): host, port, url = self.build_rest_path(bucket, None, None, params) response = self.http_request('GET', host, port, url) - headers = response[0] - encoded_props = response[1] + headers, encoded_props = response[0:2] if (headers['http_code'] == 200): props = json.loads(encoded_props) return props['keys'] else: raise Exception('Error getting bucket properties.') - + + def get_buckets(self): + params = {'buckets': 'true'} + host, port, url = self.build_rest_path(None, None, None, params) + response = self.http_request('GET', host, port, url) + + headers, encoded_props = response[0:2] + if (headers['http_code'] == 200): + props = json.loads(encoded_props) + return prop['buckets'] + else: + raise Exception('Error getting buckets.') + def get_bucket_props(self, bucket, keys=False): # Run the request... params = {'props' : 'True', 'keys' : 'False'} @@ -178,7 +189,7 @@ def set_bucket_props(self, bucket, props): raise Exception('Error setting bucket properties.') return True - def mapred(self, inputs, query, timeout=None): + def mapred(self, inputs, query, key_filters=None, timeout=None): # Construct the job, optionally set the timeout... job = {'inputs':inputs, 'query':query} if timeout is not None: @@ -311,7 +322,10 @@ def build_rest_path(self, bucket, key=None, spec=None, params=None) : # Build 'http://hostname:port/prefix/bucket' path = '' path += '/' + self._prefix - path += '/' + urllib.quote_plus(bucket._name) + + # Add '.../bucket' + if (bucket is not None): + path += '/' + urllib.quote_plus(bucket._name) # Add '.../key' if (key is not None): diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 448027e8..690b1a45 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -74,7 +74,7 @@ class RiakPbcTransport(RiakTransport): 'default' : RIAKC_RW_DEFAULT, 'all' : RIAKC_RW_ALL, 'quorum' : RIAKC_RW_QUORUM, - 'one' : RIAKC_RW_ONE + 'one' : RIAKC_RW_ONE } def __init__(self, host='127.0.0.1', port=8087, client_id=None): """ @@ -128,7 +128,7 @@ def set_client_id(self, client_id): """ req = riakclient_pb2.RpbSetClientIdReq() req.client_id = client_id - + self.maybe_connect() self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req) msg_code, resp = self.recv_msg() @@ -170,7 +170,7 @@ def put(self, robj, w = None, dw = None, return_body = True): Serialize get request and deserialize response """ bucket = robj.get_bucket() - + req = riakclient_pb2.RpbPutReq() req.w = self.translate_rw_val(w) req.dw = self.translate_rw_val(dw) @@ -214,7 +214,7 @@ def delete(self, robj, rw = None): if msg_code != MSG_CODE_DEL_RESP: raise RiakError("unexpected protocol buffer message code: ", msg_code) return self - + def get_keys(self, bucket): """ Lists all keys within a bucket. @@ -238,6 +238,19 @@ def get_keys(self, bucket): return keys + def get_buckets(self): + """ + Serialize bucket listing request and deserialize response + """ + req = riakclient_pb2.RpbListBucketsReq() + + self.maybe_connect() + self.send_msg(MSG_CODE_LIST_KEYS_REQ, req) + msg_code, resp = self.recv_msg() + if msg_code != MSG_CODE_LIST_BUCKETS_RESP: + raise RiakError("unexpected protocol buffer message code: ", msg_code) + return resp.buckets + def get_bucket_props(self, bucket): """ Serialize bucket property request and deserialize response diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 056326a4..ebc2f22f 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -17,7 +17,7 @@ specific language governing permissions and limitations under the License. """ -from riak import RiakError +from riak import RiakError import base64 import random import threading @@ -76,6 +76,13 @@ def delete(self, robj, rw = None): """ raise RiakError("not implemented") + def get_buckets(self) : + """ + Serialize get buckets request and deserialize response + @return dict() + """ + raise RiakError("not implemented") + def get_bucket_props(self, bucket) : """ Serialize get bucket property request and deserialize response From c619cb74b40300a3f601ed356e2895d9b3231077 Mon Sep 17 00:00:00 2001 From: Josip Lisec Date: Thu, 6 Jan 2011 21:09:08 +0100 Subject: [PATCH 0002/1060] Clean up --- riak/transports/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 3e290c0a..6af6a4cd 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -189,7 +189,7 @@ def set_bucket_props(self, bucket, props): raise Exception('Error setting bucket properties.') return True - def mapred(self, inputs, query, key_filters=None, timeout=None): + def mapred(self, inputs, query, timeout=None): # Construct the job, optionally set the timeout... job = {'inputs':inputs, 'query':query} if timeout is not None: From d1e28d38ec8545ac84548e5dff5fd9025c556634 Mon Sep 17 00:00:00 2001 From: Josip Lisec Date: Wed, 19 Jan 2011 12:14:43 -0800 Subject: [PATCH 0003/1060] Fixed minor typo/major bug, thanks @h3lls --- riak/transports/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 6af6a4cd..3137e413 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -149,7 +149,7 @@ def get_buckets(self): headers, encoded_props = response[0:2] if (headers['http_code'] == 200): props = json.loads(encoded_props) - return prop['buckets'] + return props['buckets'] else: raise Exception('Error getting buckets.') From 275ca93cc860452e973bf9f943eff0a122e1579a Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Mon, 18 Apr 2011 15:33:04 +0200 Subject: [PATCH 0004/1060] Add more tests for list buckets and key filters. --- riak/tests/test_all.py | 13 ++++++++++++- riak/transports/pbc.py | 7 ++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 24baa944..441a38d4 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -325,6 +325,12 @@ def test_key_filters(self): self.assertEqual(result, ["yahoo-20090613"]) + def test_key_filters_with_search_query(self): + mapreduce = self.client \ + .search("kftest", "query") + self.assertRaises(Exception, mapreduce.add_key_filters, [["tokenize", "-", 2]]) + self.assertRaises(Exception, mapreduce.add_key_filter, "ends_with", "0613") + def test_erlang_map_reduce(self): # Create the object... bucket = self.client.bucket("bucket") @@ -406,7 +412,7 @@ def test_search_integration(self): bucket.new("four", {"foo":"four", "bar":"orange"}).store() bucket.new("five", {"foo":"five", "bar":"yellow"}).store() - # Run some operations... + # Run some operations... results = self.client.search("searchbucket", "foo:one OR foo:two").run() if (len(results) == 0): print "\n\nNot running test \"testSearchIntegration()\".\n" @@ -416,6 +422,11 @@ def test_search_integration(self): results = self.client.search("searchbucket", "(foo:one OR foo:two OR foo:three OR foo:four) AND (NOT bar:green)").run() self.assertEqual(len(results), 3) + def test_list_buckets(self): + bucket = self.client.bucket("list_bucket") + bucket.new("one", {"foo":"one", "bar":"red"}).store() + buckets = self.client.get_buckets() + self.assertTrue("list_bucket" in buckets) class RiakPbcTransportTestCase(BaseTestCase, unittest.TestCase): diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 74815ce0..4f25beff 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -242,10 +242,8 @@ def get_buckets(self): """ Serialize bucket listing request and deserialize response """ - req = riakclient_pb2.RpbListBucketsReq() - self.maybe_connect() - self.send_msg(MSG_CODE_LIST_KEYS_REQ, req) + self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_LIST_BUCKETS_RESP: raise RiakError("unexpected protocol buffer message code: ", msg_code) @@ -383,6 +381,9 @@ def recv_msg(self): elif msg_code == MSG_CODE_LIST_KEYS_RESP: msg = riakclient_pb2.RpbListKeysResp() msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_LIST_BUCKETS_RESP: + msg = riakclient_pb2.RpbListBucketsResp() + msg.ParseFromString(self._inbuf[1:]) elif msg_code == MSG_CODE_GET_BUCKET_RESP: msg = riakclient_pb2.RpbGetBucketResp() msg.ParseFromString(self._inbuf[1:]) From a8a5101c712f61a2a704e4c7f1e2569e720b7d8d Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Tue, 19 Apr 2011 14:51:16 +0200 Subject: [PATCH 0005/1060] Add proper user metadata support. --- riak/riak_object.py | 21 ++++++++++++++++++++- riak/tests/test_all.py | 10 ++++++++++ riak/transports/http.py | 7 +++++-- riak/transports/pbc.py | 2 +- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index a04e81fe..9c7aa589 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -47,7 +47,7 @@ def __init__(self, client, bucket, key=None): self._encode_data = True self._vclock = None self._data = None - self._metadata = {} + self._metadata = {MD_USERMETA: {}} self._links = [] self._siblings = [] self._exists = False @@ -158,6 +158,25 @@ def set_metadata(self, metadata): self._metadata = metadata return self + def get_usermeta(self): + if MD_USERMETA in self._metadata: + return self._metadata[MD_USERMETA] + else: + return {} + + def set_usermeta(self, usermeta): + """ + Sets the custom user metadata on this object. This doesn't include things + like content type and links, but only user-defined meta attributes stored + with the Riak object. + + :param userdata: The user metadata to store. + :type userdata: dict + :rtype: data + """ + self._metadata[MD_USERMETA] = usermeta + return self + def exists(self): """ Return True if the object exists, False otherwise. Allows you to diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6bd18719..dbbf0941 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -419,6 +419,16 @@ def test_store_binary_object_from_file_should_use_default_mimetype(self): obj = bucket.get_binary('foo_from_file') self.assertEqual(obj.get_content_type(), 'application/octet-stream') + def test_store_metadata(self): + bucket = self.client.bucket('bucket') + rand = self.randint() + obj = bucket.new('fooster', rand) + obj.set_usermeta({'custom': 'some metadata'}) + obj.store() + obj = bucket.get('fooster') + print(obj.get_usermeta()) + self.assertEqual('some metadata', obj.get_usermeta()['custom']) + def test_store_binary_object_from_file_should_fail_if_file_not_found(self): bucket = self.client.bucket('bucket') rand = str(self.randint()) diff --git a/riak/transports/http.py b/riak/transports/http.py index 0af3b277..44a0d660 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -112,6 +112,9 @@ def put(self, robj, w = None, dw = None, return_body = True): if headers['Link'] != '': headers['Link'] += ', ' headers['Link'] += self.to_link_header(link) + for key, value in robj.get_usermeta().iteritems(): + headers['X-Riak-Meta-%s' % key] = value + content = robj.get_encoded_data() # Run the operation. @@ -247,7 +250,7 @@ def parse_body(self, response, expected_statuses): # Parse the headers... vclock = None - metadata = {} + metadata = {MD_USERMETA: {}} links = [] for header, value in headers.iteritems(): if header == 'content-type': @@ -263,7 +266,7 @@ def parse_body(self, response, expected_statuses): elif header == 'last-modified': metadata[MD_LASTMOD] = value elif header.startswith('x-riak-meta-'): - metadata[MD_USERMETA][header] = value + metadata[MD_USERMETA][header.replace('x-riak-meta-', '')] = value elif header == 'x-riak-vclock': vclock = value if links != []: diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 98f18bd1..6f0ff21b 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -456,7 +456,7 @@ def pbify_content(self, metadata, data, rpb_content) : elif k == MD_ENCODING: rpb_content.charset = v elif k == MD_USERMETA: - for uk, uv in v: + for uk, uv in v.iteritems(): pair = rpb_content.usermeta.add() pair.key = uk pair.value = uv From 41bbe6ab578dd2e14f2412c4d9e6f783e97a5c67 Mon Sep 17 00:00:00 2001 From: Mikhail Sobolev Date: Thu, 21 Apr 2011 22:17:32 +0300 Subject: [PATCH 0006/1060] added sensible parameters for nosetests --- .gitignore | 1 + setup.cfg | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 setup.cfg diff --git a/.gitignore b/.gitignore index e2a2f0ab..de422718 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ docs/_build .*.swp +.coverage diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..40bf678c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[nosetests] +verbosity=2 +with-coverage=1 +cover-package=riak +cover-erase=1 From 5461faf21a9ba3080a90be1fc9cf626638f07e4d Mon Sep 17 00:00:00 2001 From: Mark Erdmann Date: Mon, 25 Apr 2011 18:11:05 -0700 Subject: [PATCH 0007/1060] Let Riak assign a random key This change makes it possible to pass None as the key argument to bucket.new(). I'm not sure if you'll want to change the request method to POST, but it seemed like the simplest way to make this work. An alternative would be to change the request method to POST whenever the key is None, or to refactor the HTTP transport so that a post method is called instead. --- riak/tests/test_all.py | 9 +++++++++ riak/transports/http.py | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 06f05272..46109644 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -458,6 +458,15 @@ def test_no_returnbody(self): o = bucket.new("foo", "bar").store(return_body=False) self.assertEqual(o.vclock(), None) + def test_generate_key(self): + # Ensure that Riak generates a random key when + # the key passed to bucket.new() is None. + bucket = self.client.bucket('random_key_bucket') + for key in bucket.get_keys(): + bucket.get(str(key)).delete() + bucket.new(None, data={}).store() + self.assertEqual(len(bucket.get_keys()), 1) + if __name__ == '__main__': unittest.main() diff --git a/riak/transports/http.py b/riak/transports/http.py index d9d0bd3c..cf4aeeea 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -115,9 +115,9 @@ def put(self, robj, w = None, dw = None, return_body = True): content = robj.get_encoded_data() # Run the operation. - response = self.http_request('PUT', host, port, url, headers, content) + response = self.http_request('POST', host, port, url, headers, content) if return_body: - return self.parse_body(response, [200, 300]) + return self.parse_body(response, [200, 201, 300]) else: self.check_http_code(response, [204]) return None From e4ee24ff341d3917c2ace04b29cf762cfc517bae Mon Sep 17 00:00:00 2001 From: Mikhail Sobolev Date: Tue, 26 Apr 2011 22:13:08 +0300 Subject: [PATCH 0008/1060] ignore build artefacts --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index de422718..a8d3d3a0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ docs/_build .*.swp .coverage + +build/ +dist/ +riak.egg-info/ From 1da36bcd3d52a09455bd9109048e676ac12bae35 Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Tue, 3 May 2011 20:14:49 -0400 Subject: [PATCH 0009/1060] Added a collection of map reduce functions that map to the built-in Javascript versions --- riak/mapreduce.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index ae1829c1..88dc7c8d 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -201,6 +201,52 @@ def run(self, timeout=None): return a + ## + # Start Shortcuts to built-ins + ## + def map_values(self, options=None): + return self.map("Riak.mapValues", options=options) + + def map_values_json(self, options=None): + return self.map("Riak.mapValuesJson", options=options) + + def reduce_sum(self, options=None): + return self.reduce("Riak.reduceSum", options=options) + + def reduce_min(self, options=None): + return self.reduce("Riak.reduceMin", options=options) + + def reduce_max(self, options=None): + return self.reduce("Riak.reduceMax", options=options) + + def reduce_sort(self, js_function, options=None): + if options is None: + options = dict() + + options['arg'] = js_function + return self.reduce("Riak.reduceSort", options=options) + + def reduce_numeric_sort(self, options=None): + return self.reduce("Riak.reduceNumericSort", options=options) + + def reduce_limit(self, limit, options=None): + if options is None: + options = dict() + + options['arg'] = limit + return self.reduce("Riak.reduceLimit", options=options) + + def reduce_slice(self, start_end, options=None): + if options is None: + options = dict() + + options['arg'] = start_end + return self.reduce("Riak.reduceSlice", options=options) + + def filter_not_found(self, options=None): + return self.reduce("Riak.filterNotFound", options=options) + + class RiakMapReducePhase(object): """ The RiakMapReducePhase holds information about a Map phase or @@ -383,3 +429,4 @@ def isEqual(self, link): """ is_equal = (self._bucket == link._bucket) and (self._key == link._key) and (self.get_tag() == link.get_tag()) return is_equal + From c899ba06acbe7e8d2605ae1d3d8462f75a95f519 Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Wed, 4 May 2011 11:40:43 -0400 Subject: [PATCH 0010/1060] Added a number of aliases for the riak_kv mapred_builtins.js module. This allows for a more declaritive syntax that will make functional programmers smile: results = client.add("bucket").map_values_json().reduce_sort().reduce_limit(10) --- riak/mapreduce.py | 16 ++-- riak/tests/test_all.py | 194 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 203 insertions(+), 7 deletions(-) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 88dc7c8d..453af398 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -219,11 +219,13 @@ def reduce_min(self, options=None): def reduce_max(self, options=None): return self.reduce("Riak.reduceMax", options=options) - def reduce_sort(self, js_function, options=None): + def reduce_sort(self, js_cmp=None, options=None): if options is None: options = dict() - options['arg'] = js_function + if js_cmp: + options['arg'] = js_cmp + return self.reduce("Riak.reduceSort", options=options) def reduce_numeric_sort(self, options=None): @@ -234,13 +236,17 @@ def reduce_limit(self, limit, options=None): options = dict() options['arg'] = limit - return self.reduce("Riak.reduceLimit", options=options) + # reduceLimit is broken in riak_kv + code="""function(value, arg) { + return value.slice(0, arg); + }""" + return self.reduce(code, options=options) - def reduce_slice(self, start_end, options=None): + def reduce_slice(self, start, end, options=None): if options is None: options = dict() - options['arg'] = start_end + options['arg'] = [start, end] return self.reduce("Riak.reduceSlice", options=options) def filter_not_found(self, options=None): diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 06f05272..996bf197 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -426,8 +426,198 @@ def test_store_binary_object_from_file_should_fail_if_file_not_found(self): obj = bucket.get_binary('not_found_from_file') self.assertEqual(obj.get_data(), None) +class MapReduceAliasTestMixIn(object): + """This tests the map reduce aliases""" -class RiakPbcTransportTestCase(BaseTestCase, unittest.TestCase): + def test_map_values(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new_binary('one', data='value_1').store() + bucket.new_binary('two', data='value_2').store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values().run() + + # Sort the result so that we can have a consistent + # expected value + result.sort() + + self.assertEqual(result, ["value_1", "value_2"]) + + def test_map_values_json(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data={'val': 'value_1'}).store() + bucket.new('two', data={'val': 'value_2'}).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().run() + + # Sort the result so that we can have a consistent + # expected value + result.sort(key=lambda x: x['val']) + + self.assertEqual(result, [{'val': "value_1"}, {'val': "value_2"}]) + + def test_reduce_sum(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().reduce_sum().run() + + self.assertEqual(result, [3]) + + def test_reduce_min(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().reduce_min().run() + + self.assertEqual(result, [1]) + + def test_reduce_max(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().reduce_max().run() + + self.assertEqual(result, [2]) + + def test_reduce_sort(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data="value1").store() + bucket.new('two', data="value2").store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().reduce_sort().run() + + self.assertEqual(result, ["value1","value2"]) + + def test_reduce_sort_custom(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data="value1").store() + bucket.new('two', data="value2").store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().reduce_sort("""function(x,y) { + if(x == y) return 0; + return x > y ? -1 : 1; + }""").run() + + self.assertEqual(result, ["value2","value1"]) + + def test_reduce_numeric_sort(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().reduce_numeric_sort().run() + + self.assertEqual(result, [1,2]) + + def test_reduce_limit(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json()\ + .reduce_numeric_sort()\ + .reduce_limit(1).run() + + self.assertEqual(result, [1]) + + def test_reduce_slice(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json()\ + .reduce_numeric_sort()\ + .reduce_slice(1,2).run() + + self.assertEqual(result, [2]) + + def test_filter_not_found(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Make sure "three" does not exist + bucket.get('three').delete() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two')\ + .add('bucket', 'three') + + # Use the map_values alias + result = mr.map_values_json()\ + .filter_not_found()\ + .run() + + self.assertEqual(sorted(result), [1,2]) + + +class RiakPbcTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, + unittest.TestCase): def setUp(self): self.host = PB_HOST @@ -445,7 +635,7 @@ def test_uses_client_id_if_given(self): self.assertEqual(zero_client_id, c.get_client_id()) # -class RiakHttpTransportTestCase(BaseTestCase, unittest.TestCase): +class RiakHttpTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): self.host = HTTP_HOST From bd9b069124ef6b8568147cf2df3eb5379876c8da Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Wed, 25 May 2011 10:01:27 +0200 Subject: [PATCH 0011/1060] Use PUT when key is set, and POST only when it isn't. POST could be used in both scenarios, but relying on PUT for known keys is more consistent with other clients. --- riak/transports/http.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 3f8057f0..64e30dae 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -115,7 +115,11 @@ def put(self, robj, w = None, dw = None, return_body = True): content = robj.get_encoded_data() # Run the operation. - response = self.http_request('POST', host, port, url, headers, content) + if robj.get_key() is None: + response = self.http_request('POST', host, port, url, headers, content) + else: + response = self.http_request('PUT', host, port, url, headers, content) + if return_body: return self.parse_body(response, [200, 201, 300]) else: From 0684ee2ae031fc1a0baa007904d568407199e0d3 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Wed, 25 May 2011 10:10:50 +0200 Subject: [PATCH 0012/1060] Updated THANKS file. Credit where credit is due. --- THANKS | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/THANKS b/THANKS index a060ebae..edd19c3f 100644 --- a/THANKS +++ b/THANKS @@ -1,4 +1,4 @@ -The following people have contributed to Riak: +The following people have contributed to the Riak Python client: Andy Gross Justin Sheehy @@ -7,4 +7,7 @@ Jayson Baird Jon Meredith Eric Florenzano Silas Sewell - +Matt Heitzenroder +Mark Erdmann +Greg Nelson +Mikhail Sobolev From 180245cb2aad84fee614be1b8eacf13646caeff1 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Wed, 25 May 2011 11:23:18 +0200 Subject: [PATCH 0013/1060] Add Eric Moritz. --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index edd19c3f..f734f6c9 100644 --- a/THANKS +++ b/THANKS @@ -11,3 +11,4 @@ Matt Heitzenroder Mark Erdmann Greg Nelson Mikhail Sobolev +Eric Moritz From 49c577c2179d745d575504d866fb3e18fe6668be Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Sat, 4 Jun 2011 00:17:50 -0400 Subject: [PATCH 0014/1060] Added a special F class that lets people chain key filters easier --- riak/__init__.py | 2 +- riak/mapreduce.py | 38 +++++++++++++++++++++++++++++ riak/tests/test_all.py | 55 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/riak/__init__.py b/riak/__init__.py index 841e4e27..aa8c707a 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -39,7 +39,7 @@ def __str__(self): from riak_object import RiakObject from bucket import RiakBucket from client import RiakClient -from mapreduce import RiakMapReduce, RiakMapReducePhase, RiakLinkPhase +from mapreduce import RiakMapReduce, RiakMapReducePhase, RiakLinkPhase, F from transports.pbc import RiakPbcTransport from transports.http import RiakHttpTransport diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 9bc61f3a..335c6dcb 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -463,3 +463,41 @@ def isEqual(self, link): is_equal = (self._bucket == link._bucket) and (self._key == link._key) and (self.get_tag() == link.get_tag()) return is_equal +class F(object): + def __init__(self, *args): + if args: + self._filters = [list(args)] + else: + self._filters = [] + + def __add__(self, other): + f = F() + f._filters = self._filters + other._filters + return f + + def _bool_op(self, op, other): + # If the current filter is an and, append the other's + # filters onto the filter + if(self._filters and self._filters[0][0] == op): + f = F() + f._filters.extend(self._filters) + f._filters[0].append(other._filters) + return f + # Otherwise just create a new F() object with an and + return F(op, self._filters, other._filters) + + def __and__(self, other): + return self._bool_op("and", other) + + def __or__(self, other): + return self._bool_op("or", other) + + def __repr__(self): + return str(self._filters) + + def __getattr__(self, name): + def function(*args): + args1 = [name] + list(args) + other = F(*args1) + return self + other + return function diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 1a58ee60..f67bdaca 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -13,6 +13,7 @@ from riak import RiakClient from riak import RiakPbcTransport from riak import RiakHttpTransport +from riak import F HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) @@ -685,5 +686,59 @@ def test_generate_key(self): self.assertEqual(len(bucket.get_keys()), 1) +class RiakTestFilter(unittest.TestCase): + def test_simple(self): + f1 = F("tokenize", "-", 1) + self.assertEqual(f1._filters, [["tokenize", "-", 1]]) + + def test_add(self): + f1 = F("tokenize", "-", 1) + f2 = F("eq", "2005") + f3 = f1 + f2 + self.assertEqual(f3._filters, [["tokenize", "-", 1], ["eq", "2005"]]) + + def test_and(self): + f1 = F("starts_with", "2005-") + f2 = F("ends_with", "-01") + f3 = f1 & f2 + self.assertEqual(f3._filters, [["and", [["starts_with", "2005-"]], [["ends_with", "-01"]]]]) + + def test_multi_and(self): + f1 = F("starts_with", "2005-") + f2 = F("ends_with", "-01") + f3 = F("matches", "-11-") + f4 = f1 & f2 & f3 + self.assertEqual(f4._filters, [["and", + [["starts_with", "2005-"]], + [["ends_with", "-01"]], + [["matches", "-11-"]], + ]]) + + def test_or(self): + f1 = F("starts_with", "2005-") + f2 = F("ends_with", "-01") + f3 = f1 | f2 + self.assertEqual(f3._filters, [["or", [["starts_with", "2005-"]], [["ends_with", "-01"]]]]) + + def test_multi_or(self): + f1 = F("starts_with", "2005-") + f2 = F("ends_with", "-01") + f3 = F("matches", "-11-") + f4 = f1 | f2 | f3 + self.assertEqual(f4._filters, [["or", + [["starts_with", "2005-"]], + [["ends_with", "-01"]], + [["matches", "-11-"]], + ]]) + + def test_chaining(self): + f1 = F().tokenize("-", 1).eq("2005") + f2 = F().tokenize("-", 2).eq("05") + f3 = f1 & f2 + self.assertEqual(f3._filters, [["and", + [["tokenize", "-", 1], ["eq", "2005"]], + [["tokenize", "-", 2], ["eq", "05"]] + ]]) + if __name__ == '__main__': unittest.main() From bf69d64512711b18ad319412413b25a09e0d3598 Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Mon, 6 Jun 2011 19:44:53 -0400 Subject: [PATCH 0015/1060] Added an instance of the F class to serve as the root of filter chains --- riak/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/__init__.py b/riak/__init__.py index aa8c707a..44b17c7e 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -47,6 +47,6 @@ def __str__(self): ALL = "all" QUORUM = "quorum" - +f = F() From dbb5bbc899c9a8df4470c57b1a3ffb67ef1b3a3a Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Mon, 6 Jun 2011 19:45:36 -0400 Subject: [PATCH 0016/1060] Added a __iter__ function so that F instances can be used with add_key_filters. --- riak/mapreduce.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 335c6dcb..4b10c504 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -501,3 +501,6 @@ def function(*args): other = F(*args1) return self + other return function + + def __iter__(self): + return iter(self._filters) From 14999863fd16e7d9daa8632048c239baa7500931 Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Mon, 6 Jun 2011 19:46:16 -0400 Subject: [PATCH 0017/1060] Used the F class's iterator function instead of accessing the private _filters Added a map reduce test using riak.f --- riak/tests/test_all.py | 57 +++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index f67bdaca..e85040ef 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -13,7 +13,7 @@ from riak import RiakClient from riak import RiakPbcTransport from riak import RiakHttpTransport -from riak import F +from riak import F, f HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) @@ -326,6 +326,28 @@ def test_key_filters(self): self.assertEqual(result, ["yahoo-20090613"]) + def test_key_filters_f_chain(self): + bucket = self.client.bucket("kftest") + bucket.new("basho-20101215", 1).store() + bucket.new("google-20110103", 2).store() + bucket.new("yahoo-20090613", 3).store() + + + # compose a chain of key filters using f as the root of + # two filters ANDed together to ensure that f can be the root + # of multiple chains + filters = f.tokenize("-", 1).eq("yahoo") \ + & f.tokenize("-", 2).ends_with("0613") + + result = self.client \ + .add("kftest") \ + .add_key_filters(filters) \ + .map("function (v, keydata) { return [v.key]; }") \ + .run() + + self.assertEqual(result, ["yahoo-20090613"]) + + def test_key_filters_with_search_query(self): mapreduce = self.client \ .search("kftest", "query") @@ -695,50 +717,51 @@ def test_add(self): f1 = F("tokenize", "-", 1) f2 = F("eq", "2005") f3 = f1 + f2 - self.assertEqual(f3._filters, [["tokenize", "-", 1], ["eq", "2005"]]) + self.assertEqual(list(f3), [["tokenize", "-", 1], ["eq", "2005"]]) def test_and(self): f1 = F("starts_with", "2005-") f2 = F("ends_with", "-01") f3 = f1 & f2 - self.assertEqual(f3._filters, [["and", [["starts_with", "2005-"]], [["ends_with", "-01"]]]]) + self.assertEqual(list(f3), [["and", [["starts_with", "2005-"]], [["ends_with", "-01"]]]]) def test_multi_and(self): f1 = F("starts_with", "2005-") f2 = F("ends_with", "-01") f3 = F("matches", "-11-") f4 = f1 & f2 & f3 - self.assertEqual(f4._filters, [["and", + self.assertEqual(list(f4), [["and", [["starts_with", "2005-"]], [["ends_with", "-01"]], [["matches", "-11-"]], ]]) - + def test_or(self): f1 = F("starts_with", "2005-") f2 = F("ends_with", "-01") f3 = f1 | f2 - self.assertEqual(f3._filters, [["or", [["starts_with", "2005-"]], [["ends_with", "-01"]]]]) + self.assertEqual(list(f3), [["or", [["starts_with", "2005-"]], + [["ends_with", "-01"]]]]) def test_multi_or(self): f1 = F("starts_with", "2005-") f2 = F("ends_with", "-01") f3 = F("matches", "-11-") f4 = f1 | f2 | f3 - self.assertEqual(f4._filters, [["or", - [["starts_with", "2005-"]], - [["ends_with", "-01"]], - [["matches", "-11-"]], - ]]) + self.assertEqual(list(f4), [["or", + [["starts_with", "2005-"]], + [["ends_with", "-01"]], + [["matches", "-11-"]], + ]]) def test_chaining(self): - f1 = F().tokenize("-", 1).eq("2005") - f2 = F().tokenize("-", 2).eq("05") + f1 = f.tokenize("-", 1).eq("2005") + f2 = f.tokenize("-", 2).eq("05") f3 = f1 & f2 - self.assertEqual(f3._filters, [["and", - [["tokenize", "-", 1], ["eq", "2005"]], - [["tokenize", "-", 2], ["eq", "05"]] - ]]) + self.assertEqual(list(f3), [["and", + [["tokenize", "-", 1], ["eq", "2005"]], + [["tokenize", "-", 2], ["eq", "05"]] + ]]) if __name__ == '__main__': unittest.main() From b6ff80b58adcb190c25a8383a86582737c503eef Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Mon, 6 Jun 2011 19:47:12 -0400 Subject: [PATCH 0018/1060] Added some spartan documentation on how to use key filters in python --- README.rst | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/README.rst b/README.rst index f0e5397b..545874dd 100644 --- a/README.rst +++ b/README.rst @@ -414,3 +414,56 @@ tutorial, but usage of this feature looks like:: .. _`Riak Search`: http://wiki.basho.com/Riak-Search.html .. _Lucene: http://lucene.apache.org/ + +Using Key Filters +================== + +`Key filters`_ are a way to pre-process MapReduce inputs from a full +bucket query simply by examining the key — without loading the object +first. This is especially useful if your keys are composed of +domain-specific information that can be analyzed at query-time. + +To illustrate this, let’s contrive an example. Let’s say we’re storing +customer invoices with a key constructed from the customer name and +the date, in a bucket called “invoices”. Here are some sample keys:: + + basho-20101215 + google-20110103 + yahoo-20090613 + +To query all invoices for a given customer:: + + import riak + + client = riak.RiakClient() + + query = client.add("invoices") + query.add_key_filter("tokenize", "-", 1) + query.add_key_filter("eq", "google") + + query.map("""function(v) { + var data = JSON.parse(v.values[0].data); + return [[v.key, data]]; + }""") + + +More complex key filters can be built using riak.f:: + + import riak + from riak import f + + client = riak.RiakClient() + + # Query basho's orders for 2010 + filters = f.tokenize("-", 1).eq("basho")\ + & f.tokenize("-", 2).starts_with("2010") + + query = client.add("invoices") + query = query.add_key_filters(filters) + + query.map("""function(v) { + var data = JSON.parse(v.values[0].data); + return [[v.key, data]]; + }""") + +.. _`Key filters`: http://wiki.basho.com/Key-Filters.html From 9e34a78b5222bc335328dabd6b73b3c27d66cf24 Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Mon, 6 Jun 2011 19:55:49 -0400 Subject: [PATCH 0019/1060] Added the version key filters were added to Riak --- README.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 545874dd..95ae78c4 100644 --- a/README.rst +++ b/README.rst @@ -418,10 +418,11 @@ tutorial, but usage of this feature looks like:: Using Key Filters ================== -`Key filters`_ are a way to pre-process MapReduce inputs from a full -bucket query simply by examining the key — without loading the object -first. This is especially useful if your keys are composed of -domain-specific information that can be analyzed at query-time. +`Key filters`_ are a new feature available as of Riak 0.14. They are +a way to pre-process MapReduce inputs from a full bucket query simply +by examining the key — without loading the object first. This is +especially useful if your keys are composed of domain-specific +information that can be analyzed at query-time. To illustrate this, let’s contrive an example. Let’s say we’re storing customer invoices with a key constructed from the customer name and From 59983b8898bdd0b610b16966b05c30f5b92d5e1b Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Mon, 6 Jun 2011 20:08:44 -0400 Subject: [PATCH 0020/1060] Added a number of riak.f examples to show it's flexibility --- README.rst | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 95ae78c4..60627144 100644 --- a/README.rst +++ b/README.rst @@ -448,23 +448,27 @@ To query all invoices for a given customer:: }""") -More complex key filters can be built using riak.f:: +Alternatively, you can use riak.f to build key filters:: - import riak - from riak import f + ... + query = client.add("invoices") + query.add_key_filters(f.tokenize("-", 1).eq("google")) + ... - client = riak.RiakClient() +Boolean operators can be used with riak.f instances:: # Query basho's orders for 2010 filters = f.tokenize("-", 1).eq("basho")\ & f.tokenize("-", 2).starts_with("2010") - query = client.add("invoices") - query = query.add_key_filters(filters) - - query.map("""function(v) { - var data = JSON.parse(v.values[0].data); - return [[v.key, data]]; - }""") +Alternatively filters can be added together in order to produce very +complex filters:: + + # Query invoices for basho or google + filters = f.tokenize("-", 1) + (f.eq("basho") | f.eq("google")) + + # This is the same as the following key filters + [['tokenize', '-', 1], ['or', [['eq', 'google']], [['eq', 'yahoo']]]] + .. _`Key filters`: http://wiki.basho.com/Key-Filters.html From 2d0f7e1d37abb7d6f42cc52b634ea3a8c1762aee Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Mon, 6 Jun 2011 20:11:25 -0400 Subject: [PATCH 0021/1060] Removed the useless dots --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index 60627144..3a889f9e 100644 --- a/README.rst +++ b/README.rst @@ -450,10 +450,7 @@ To query all invoices for a given customer:: Alternatively, you can use riak.f to build key filters:: - ... - query = client.add("invoices") query.add_key_filters(f.tokenize("-", 1).eq("google")) - ... Boolean operators can be used with riak.f instances:: From 2fedb98d265003cc07f8707662f038988a389d4d Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Mon, 6 Jun 2011 20:13:54 -0400 Subject: [PATCH 0022/1060] Reworded the + example --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 3a889f9e..d4df68d0 100644 --- a/README.rst +++ b/README.rst @@ -458,8 +458,8 @@ Boolean operators can be used with riak.f instances:: filters = f.tokenize("-", 1).eq("basho")\ & f.tokenize("-", 2).starts_with("2010") -Alternatively filters can be added together in order to produce very -complex filters:: +Filters can be combined using the + operator to produce very complex +filters:: # Query invoices for basho or google filters = f.tokenize("-", 1) + (f.eq("basho") | f.eq("google")) From 92cc6391e38ed92be0b4835f10d7eae2d539d99c Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Wed, 8 Jun 2011 18:54:34 -0400 Subject: [PATCH 0023/1060] As per the the pull request comment thread I've Renamed F to RiakKeyFilter to match convention and renamed f to key_filter to favor the explicit. --- riak/__init__.py | 5 +++-- riak/mapreduce.py | 12 ++++++------ riak/tests/test_all.py | 36 ++++++++++++++++++------------------ 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index 44b17c7e..8a334910 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -39,7 +39,8 @@ def __str__(self): from riak_object import RiakObject from bucket import RiakBucket from client import RiakClient -from mapreduce import RiakMapReduce, RiakMapReducePhase, RiakLinkPhase, F +from mapreduce import RiakMapReduce, RiakMapReducePhase, RiakLinkPhase,\ + RiakKeyFilter from transports.pbc import RiakPbcTransport from transports.http import RiakHttpTransport @@ -47,6 +48,6 @@ def __str__(self): ALL = "all" QUORUM = "quorum" -f = F() +key_filter = RiakKeyFilter() diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 4b10c504..01020c97 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -463,7 +463,7 @@ def isEqual(self, link): is_equal = (self._bucket == link._bucket) and (self._key == link._key) and (self.get_tag() == link.get_tag()) return is_equal -class F(object): +class RiakKeyFilter(object): def __init__(self, *args): if args: self._filters = [list(args)] @@ -471,7 +471,7 @@ def __init__(self, *args): self._filters = [] def __add__(self, other): - f = F() + f = RiakKeyFilter() f._filters = self._filters + other._filters return f @@ -479,12 +479,12 @@ def _bool_op(self, op, other): # If the current filter is an and, append the other's # filters onto the filter if(self._filters and self._filters[0][0] == op): - f = F() + f = RiakKeyFilter() f._filters.extend(self._filters) f._filters[0].append(other._filters) return f - # Otherwise just create a new F() object with an and - return F(op, self._filters, other._filters) + # Otherwise just create a new RiakKeyFilter() object with an and + return RiakKeyFilter(op, self._filters, other._filters) def __and__(self, other): return self._bool_op("and", other) @@ -498,7 +498,7 @@ def __repr__(self): def __getattr__(self, name): def function(*args): args1 = [name] + list(args) - other = F(*args1) + other = RiakKeyFilter(*args1) return self + other return function diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index e85040ef..7be67244 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -13,7 +13,7 @@ from riak import RiakClient from riak import RiakPbcTransport from riak import RiakHttpTransport -from riak import F, f +from riak import RiakKeyFilter, key_filter HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) @@ -336,8 +336,8 @@ def test_key_filters_f_chain(self): # compose a chain of key filters using f as the root of # two filters ANDed together to ensure that f can be the root # of multiple chains - filters = f.tokenize("-", 1).eq("yahoo") \ - & f.tokenize("-", 2).ends_with("0613") + filters = key_filter.tokenize("-", 1).eq("yahoo") \ + & key_filter.tokenize("-", 2).ends_with("0613") result = self.client \ .add("kftest") \ @@ -710,25 +710,25 @@ def test_generate_key(self): class RiakTestFilter(unittest.TestCase): def test_simple(self): - f1 = F("tokenize", "-", 1) + f1 = RiakKeyFilter("tokenize", "-", 1) self.assertEqual(f1._filters, [["tokenize", "-", 1]]) def test_add(self): - f1 = F("tokenize", "-", 1) - f2 = F("eq", "2005") + f1 = RiakKeyFilter("tokenize", "-", 1) + f2 = RiakKeyFilter("eq", "2005") f3 = f1 + f2 self.assertEqual(list(f3), [["tokenize", "-", 1], ["eq", "2005"]]) def test_and(self): - f1 = F("starts_with", "2005-") - f2 = F("ends_with", "-01") + f1 = RiakKeyFilter("starts_with", "2005-") + f2 = RiakKeyFilter("ends_with", "-01") f3 = f1 & f2 self.assertEqual(list(f3), [["and", [["starts_with", "2005-"]], [["ends_with", "-01"]]]]) def test_multi_and(self): - f1 = F("starts_with", "2005-") - f2 = F("ends_with", "-01") - f3 = F("matches", "-11-") + f1 = RiakKeyFilter("starts_with", "2005-") + f2 = RiakKeyFilter("ends_with", "-01") + f3 = RiakKeyFilter("matches", "-11-") f4 = f1 & f2 & f3 self.assertEqual(list(f4), [["and", [["starts_with", "2005-"]], @@ -737,16 +737,16 @@ def test_multi_and(self): ]]) def test_or(self): - f1 = F("starts_with", "2005-") - f2 = F("ends_with", "-01") + f1 = RiakKeyFilter("starts_with", "2005-") + f2 = RiakKeyFilter("ends_with", "-01") f3 = f1 | f2 self.assertEqual(list(f3), [["or", [["starts_with", "2005-"]], [["ends_with", "-01"]]]]) def test_multi_or(self): - f1 = F("starts_with", "2005-") - f2 = F("ends_with", "-01") - f3 = F("matches", "-11-") + f1 = RiakKeyFilter("starts_with", "2005-") + f2 = RiakKeyFilter("ends_with", "-01") + f3 = RiakKeyFilter("matches", "-11-") f4 = f1 | f2 | f3 self.assertEqual(list(f4), [["or", [["starts_with", "2005-"]], @@ -755,8 +755,8 @@ def test_multi_or(self): ]]) def test_chaining(self): - f1 = f.tokenize("-", 1).eq("2005") - f2 = f.tokenize("-", 2).eq("05") + f1 = key_filter.tokenize("-", 1).eq("2005") + f2 = key_filter.tokenize("-", 2).eq("05") f3 = f1 & f2 self.assertEqual(list(f3), [["and", [["tokenize", "-", 1], ["eq", "2005"]], From 1da89217bbdf3462c30c8d537c8e9c56ebadc2d0 Mon Sep 17 00:00:00 2001 From: Eric Moritz Date: Wed, 8 Jun 2011 18:58:25 -0400 Subject: [PATCH 0024/1060] Update the documentation to use key_filter instead of f --- README.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index d4df68d0..b8578d18 100644 --- a/README.rst +++ b/README.rst @@ -448,21 +448,21 @@ To query all invoices for a given customer:: }""") -Alternatively, you can use riak.f to build key filters:: +Alternatively, you can use riak.key_filter to build key filters:: - query.add_key_filters(f.tokenize("-", 1).eq("google")) + query.add_key_filters(key_filter.tokenize("-", 1).eq("google")) Boolean operators can be used with riak.f instances:: # Query basho's orders for 2010 - filters = f.tokenize("-", 1).eq("basho")\ - & f.tokenize("-", 2).starts_with("2010") + filters = key_filter.tokenize("-", 1).eq("basho")\ + & key_filter.tokenize("-", 2).starts_with("2010") Filters can be combined using the + operator to produce very complex filters:: # Query invoices for basho or google - filters = f.tokenize("-", 1) + (f.eq("basho") | f.eq("google")) + filters = key_filter.tokenize("-", 1) + (key_filter.eq("basho") | key_filter.eq("google")) # This is the same as the following key filters [['tokenize', '-', 1], ['or', [['eq', 'google']], [['eq', 'yahoo']]]] From eb6aa0ba774ea645bb197acb0f516f5a71f01042 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Wed, 22 Jun 2011 10:15:32 +0200 Subject: [PATCH 0025/1060] Add release notes for 1.2.2. --- RELEASE_NOTES.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 RELEASE_NOTES.md diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md new file mode 100644 index 00000000..6a471f22 --- /dev/null +++ b/RELEASE_NOTES.md @@ -0,0 +1,19 @@ +# Riak Python Client Release Notes + +## 1.2.2 Patch Release - 2011-06-22 + +Release 1.2.2 is a minor patch release. + +Noteworthy fixes and improvements: + +* #29: Add an nicer API for using key filters with MapReduce (Eric Moritz) +* #13 and #24: Let Riak generate a key when none is specified (Mark Erdmann) +* #28: Function aliases for the Riak built-in MapReduce functions (Eric Moritz) +* #20: Add a convenience method to create Riak object directly from file (Ana Nelson) +* #16: Support return\_body parameter when creating a new object (Stefan Praszalowicz, Andy Gross) +* #17: Storing an object fails when it doesn't exist in Riak (Eric Moritz, Andy Gross) +* #18: Ensure that a default content type is set when none specified (Andy Gross) +* #22: Fix user meta data support (Mathias Meyer) +* #23: Fix links to the wiki (Mikhail Sobolev) +* #25: Enable support for code coverage when running tests (Mikhail Sobolev) +* #26: Debian packaging (Dmitry Rozhkov) From 7926acab89b0aace4f9b0210b4cf8efdfb51ccea Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 23 Jun 2011 09:43:21 +0200 Subject: [PATCH 0026/1060] Bump package version to 1.2.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ee9776b5..e5b06a04 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def make_pb(): if __name__ == "__main__": setup( name='riak', - version='1.2.1', + version='1.2.2', packages = find_packages(), install_requires = ['protobuf>=2.3.0'], dependency_links = ["http://downloads.basho.com/support"], From 427eea425b7576705d98923529e1dea0a696c01e Mon Sep 17 00:00:00 2001 From: gilles Date: Tue, 28 Jun 2011 09:27:47 -0700 Subject: [PATCH 0027/1060] new transports: - connection reuse HTTP - pool HTTP - pool PBC (thread safe) Some cosmetics --- riak/__init__.py | 4 +- riak/client.py | 4 +- riak/tests/test_all.py | 66 +++++++++++- riak/transports/__init__.py | 4 +- riak/transports/http.py | 204 ++++++++++++++++++++++++++--------- riak/transports/pbc.py | 152 ++++++++++++++++++++++---- riak/transports/transport.py | 11 +- setup.py | 2 +- 8 files changed, 361 insertions(+), 86 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index 8a334910..b3ad1575 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -41,8 +41,8 @@ def __str__(self): from client import RiakClient from mapreduce import RiakMapReduce, RiakMapReducePhase, RiakLinkPhase,\ RiakKeyFilter -from transports.pbc import RiakPbcTransport -from transports.http import RiakHttpTransport +from transports.pbc import RiakPbcTransport, RiakPbcPoolTransport +from transports.http import RiakHttpTransport, RiakHttpReuseTransport, RiakHttpPoolTransport ONE = "one" ALL = "all" diff --git a/riak/client.py b/riak/client.py index 756db971..8f335fb2 100644 --- a/riak/client.py +++ b/riak/client.py @@ -25,7 +25,7 @@ from riak.transports import RiakHttpTransport from riak.bucket import RiakBucket -from riak.mapreduce import RiakMapReduce, RiakLink +from riak.mapreduce import RiakMapReduce class RiakClient(object): """ @@ -71,7 +71,7 @@ def get_transport(self): """ Get the transport instance the client is using for it's connection. """ - return self._transport; + return self._transport def get_r(self): """ diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 70a17021..8018817d 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -8,11 +8,10 @@ import simplejson as json import os import random -import sys import unittest from riak import RiakClient -from riak import RiakPbcTransport -from riak import RiakHttpTransport +from riak import RiakPbcTransport, RiakPbcPoolTransport +from riak import RiakHttpTransport, RiakHttpPoolTransport, RiakHttpReuseTransport from riak import RiakKeyFilter, key_filter HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') @@ -694,6 +693,23 @@ def test_uses_client_id_if_given(self): client_id = zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) # +class RiakPbcPoolTransportCase(BaseTestCase, MapReduceAliasTestMixIn, + unittest.TestCase): + def setUp(self): + self.host = PB_HOST + self.port = PB_PORT + self.transport_class = RiakPbcPoolTransport + super(RiakPbcPoolTransportCase, self).setUp() + + def test_uses_client_id_if_given(self): + self.host = PB_HOST + self.port = PB_PORT + zero_client_id = "\0\0\0\0" + c = RiakClient(PB_HOST, PB_PORT, + transport_class = RiakPbcPoolTransport, + client_id = zero_client_id) + self.assertEqual(zero_client_id, c.get_client_id()) # + class RiakHttpTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): @@ -717,6 +733,50 @@ def test_generate_key(self): bucket.new(None, data={}).store() self.assertEqual(len(bucket.get_keys()), 1) +class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): + + def setUp(self): + self.host = HTTP_HOST + self.port = HTTP_PORT + self.transport_class = RiakHttpPoolTransport + super(RiakHttpPoolTransportTestCase, self).setUp() + + def test_no_returnbody(self): + bucket = self.client.bucket("bucket") + o = bucket.new("foo", "bar").store(return_body=False) + self.assertEqual(o.vclock(), None) + + def test_generate_key(self): + # Ensure that Riak generates a random key when + # the key passed to bucket.new() is None. + bucket = self.client.bucket('random_key_bucket') + for key in bucket.get_keys(): + bucket.get(str(key)).delete() + bucket.new(None, data={}).store() + self.assertEqual(len(bucket.get_keys()), 1) + +class RiakHttpReuseTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): + + def setUp(self): + self.host = HTTP_HOST + self.port = HTTP_PORT + self.transport_class = RiakHttpReuseTransport + super(RiakHttpReuseTransportTestCase, self).setUp() + + def test_no_returnbody(self): + bucket = self.client.bucket("bucket") + o = bucket.new("foo", "bar").store(return_body=False) + self.assertEqual(o.vclock(), None) + + def test_generate_key(self): + # Ensure that Riak generates a random key when + # the key passed to bucket.new() is None. + bucket = self.client.bucket('random_key_bucket') + for key in bucket.get_keys(): + bucket.get(str(key)).delete() + bucket.new(None, data={}).store() + self.assertEqual(len(bucket.get_keys()), 1) + class RiakTestFilter(unittest.TestCase): def test_simple(self): diff --git a/riak/transports/__init__.py b/riak/transports/__init__.py index 55b7da3f..ca0cd5bf 100644 --- a/riak/transports/__init__.py +++ b/riak/transports/__init__.py @@ -1,4 +1,4 @@ -from http import RiakHttpTransport -from pbc import RiakPbcTransport +from http import RiakHttpTransport, RiakHttpReuseTransport, RiakHttpPoolTransport +from pbc import RiakPbcTransport, RiakPbcPoolTransport diff --git a/riak/transports/http.py b/riak/transports/http.py index bcfea524..9c8121a8 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -66,10 +66,10 @@ def __copy__(self): return RiakHttpTransport(self._host, self._port, self._prefix, self._mapred_prefix) - """ - Check server is alive over HTTP - """ def ping(self) : + """ + Check server is alive over HTTP + """ response = self.http_request('GET', self._host, self._port, '/ping') return(response is not None) and (response[1] == 'OK') @@ -82,7 +82,7 @@ def get(self, robj, r, vtag = None) : if vtag is not None: params['vtag'] = vtag host, port, url = self.build_rest_path(robj.get_bucket(), robj.get_key(), - None, params) + params=params) response = self.http_request('GET', host, port, url) return self.parse_body(response, [200, 300, 404]) @@ -93,7 +93,7 @@ def put(self, robj, w = None, dw = None, return_body = True): # Construct the URL... params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw} host, port, url = self.build_rest_path(robj.get_bucket(), robj.get_key(), - None, params) + params=params) # Construct the headers... headers = {'Accept' : 'text/plain, */*; q=0.5', @@ -101,12 +101,12 @@ def put(self, robj, w = None, dw = None, return_body = True): 'X-Riak-ClientId' : self._client_id} # Add the vclock if it exists... - if (robj.vclock() is not None): + if robj.vclock() is not None: headers['X-Riak-Vclock'] = robj.vclock() # Create the header from metadata links = robj.get_links() - if links != []: + if links: headers['Link'] = '' for link in links: if headers['Link'] != '': headers['Link'] += ', ' @@ -133,7 +133,7 @@ def delete(self, robj, rw): # Construct the URL... params = {'rw' : rw} host, port, url = self.build_rest_path(robj.get_bucket(), robj.get_key(), - None, params) + params=params) # Run the operation.. response = self.http_request('DELETE', host, port, url) self.check_http_code(response, [204, 404]) @@ -142,11 +142,11 @@ def delete(self, robj, rw): def get_keys(self, bucket): params = {'props' : 'True', 'keys' : 'true'} - host, port, url = self.build_rest_path(bucket, None, None, params) + host, port, url = self.build_rest_path(bucket, params=params) response = self.http_request('GET', host, port, url) headers, encoded_props = response[0:2] - if (headers['http_code'] == 200): + if headers['http_code'] == 200: props = json.loads(encoded_props) return props['keys'] else: @@ -154,25 +154,25 @@ def get_keys(self, bucket): def get_buckets(self): params = {'buckets': 'true'} - host, port, url = self.build_rest_path(None, None, None, params) + host, port, url = self.build_rest_path(None, params=params) response = self.http_request('GET', host, port, url) headers, encoded_props = response[0:2] - if (headers['http_code'] == 200): + if headers['http_code'] == 200: props = json.loads(encoded_props) return props['buckets'] else: raise Exception('Error getting buckets.') - def get_bucket_props(self, bucket, keys=False): + def get_bucket_props(self, bucket): # Run the request... params = {'props' : 'True', 'keys' : 'False'} - host, port, url = self.build_rest_path(bucket, None, None, params) + host, port, url = self.build_rest_path(bucket, params=params) response = self.http_request('GET', host, port, url) headers = response[0] encoded_props = response[1] - if (headers['http_code'] == 200): + if headers['http_code'] == 200: props = json.loads(encoded_props) return props['props'] else: @@ -191,12 +191,12 @@ def set_bucket_props(self, bucket, props): response = self.http_request('PUT', host, port, url, headers, content) # Handle the response... - if (response is None): + if response is None: raise Exception('Error setting bucket properties.') # Check the response value... status = response[0]['http_code'] - if (status != 204): + if status != 204: raise Exception('Error setting bucket properties.') return True @@ -219,7 +219,7 @@ def mapred(self, inputs, query, timeout=None): def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] - if (not status in expected_statuses): + if not status in expected_statuses: m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) raise Exception(m) @@ -231,7 +231,7 @@ def parse_body(self, response, expected_statuses): @return self """ # If no response given, then return. - if (response is None): + if response is None: return self # Make sure expected code came back @@ -243,21 +243,21 @@ def parse_body(self, response, expected_statuses): status = headers['http_code'] # Check if the server is down(status==0) - if (status == 0): + if not status: m = 'Could not contact Riak Server: http://' + self._host + ':' + str(self._port) + '!' raise RiakError(m) # Verify that we got one of the expected statuses. Otherwise, raise an exception. - if (not status in expected_statuses): + if not status in expected_statuses: m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) raise RiakError(m) # If 404(Not Found), then clear the object. - if (status == 404): + if status == 404: return None # If 300(Siblings), then return the list of siblings - elif (status == 300): + elif status == 300: # Parse and get rid of 'Siblings:' string in element 0 siblings = data.strip().split('\n') siblings.pop(0) @@ -284,10 +284,10 @@ def parse_body(self, response, expected_statuses): metadata[MD_USERMETA][header.replace('x-riak-meta-', '')] = value elif header == 'x-riak-vclock': vclock = value - if links != []: + if links: metadata[MD_LINKS] = links - return (vclock, [(metadata, data)]) + return vclock, [(metadata, data)] def to_link_header(self, link): """ @@ -308,24 +308,23 @@ def parse_links(self, links, linkHeaders) : """ for linkHeader in linkHeaders.strip().split(','): linkHeader = linkHeader.strip() - matches = re.match("\<\/([^\/]+)\/([^\/]+)\/([^\/]+)\>; ?riaktag=\"([^\']+)\"", linkHeader) - if (matches is not None): + matches = re.match("; ?riaktag=\"([^\']+)\"", linkHeader) + if matches is not None: link = RiakLink(matches.group(2), matches.group(3), matches.group(4)) links.append(link) return self - """ - Utility functions used by Riak library. - """ + #Utility functions used by Riak library. + @classmethod - def get_value(self, key, array, defaultValue) : - if (key in array): + def get_value(cls, key, array, defaultValue) : + if key in array: return array[key] else: return defaultValue - def build_rest_path(self, bucket, key=None, spec=None, params=None) : + def build_rest_path(self, bucket, key=None, params=None) : """ Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, construct and return a URL. @@ -335,18 +334,18 @@ def build_rest_path(self, bucket, key=None, spec=None, params=None) : path += '/' + self._prefix # Add '.../bucket' - if (bucket is not None): + if bucket is not None: path += '/' + urllib.quote_plus(bucket._name) # Add '.../key' - if (key is not None): + if key is not None: path += '/' + urllib.quote_plus(key) # Add query parameters. - if (params is not None): + if params is not None: s = '' for key in params.keys(): - if (s != ''): s += '&' + if s != '': s += '&' s += urllib.quote_plus(key) + '=' + urllib.quote_plus(str(params[key])) path += '?' + s @@ -354,20 +353,24 @@ def build_rest_path(self, bucket, key=None, spec=None, params=None) : return self._host, self._port, path @classmethod - def http_request(self, method, host, port, url, headers = {}, obj = '') : + def http_request(cls, method, host, port, url, headers = None, obj = '') : """ Given a Method, URL, Headers, and Body, perform and HTTP request, and return an array of arity 2 containing an associative array of response headers and the response body. """ + if not headers: + headers = {} if HAS_PYCURL: - return self.pycurl_request(method, host, port, url, headers, obj) + return cls.pycurl_request(method, host, port, url, headers, obj) else: - return self.httplib_request(method, host, port, url, headers, obj) + return cls.httplib_request(method, host, port, url, headers, obj) @classmethod - def httplib_request(self, method, host, port, uri, headers={}, body=''): + def httplib_request(cls, method, host, port, uri, headers = None, body=''): + if not headers: + headers = {} # Run the request... client = None response = None @@ -377,8 +380,7 @@ def httplib_request(self, method, host, port, uri, headers={}, body=''): response = client.getresponse() # Get the response headers... - response_headers = {} - response_headers['http_code'] = response.status + response_headers = {'http_code': response.status} for (key, value) in response.getheaders(): response_headers[key.lower()] = value @@ -394,12 +396,14 @@ def httplib_request(self, method, host, port, uri, headers={}, body=''): @classmethod - def pycurl_request(self, method, host, port, uri, headers={}, body=''): + def pycurl_request(cls, method, host, port, uri, headers, body=''): + if not headers: + headers = {} url = "http://" + host + ":" + str(port) + uri # Set up Curl... client = pycurl.Curl() client.setopt(pycurl.URL, url) - client.setopt(pycurl.HTTPHEADER, self.build_headers(headers)) + client.setopt(pycurl.HTTPHEADER, cls.build_headers(headers)) if method == 'GET': client.setopt(pycurl.HTTPGET, 1) elif method == 'POST': @@ -426,7 +430,7 @@ def pycurl_request(self, method, host, port, uri, headers={}, body=''): client.close() # Get the headers... - response_headers = self.parse_http_headers(response_headers_io.getvalue()) + response_headers = cls.parse_http_headers(response_headers_io.getvalue()) response_headers['http_code'] = http_code # Get the body... @@ -434,18 +438,18 @@ def pycurl_request(self, method, host, port, uri, headers={}, body=''): return response_headers, response_body except: - if (client is not None) : client.close() + if client is not None: client.close() raise @classmethod - def build_headers(self, headers): + def build_headers(cls, headers): headers1 = [] for key in headers.keys(): headers1.append('%s: %s' % (key, headers[key])) return headers1 @classmethod - def parse_http_headers(self, headers) : + def parse_http_headers(cls, headers) : """ Parse an HTTP Header string into an asssociative array of response headers. @@ -454,10 +458,10 @@ def parse_http_headers(self, headers) : fields = headers.split("\n") for field in fields: matches = re.match("([^:]+):(.+)", field) - if (matches is None): continue + if matches is None: continue key = matches.group(1).lower() value = matches.group(2).strip() - if (key in retVal.keys()): + if key in retVal.keys(): if isinstance(retVal[key], list): retVal[key].append(value) else: @@ -465,3 +469,101 @@ def parse_http_headers(self, headers) : else: retVal[key] = value return retVal + +import socket + +class RiakHttpReuseTransport(RiakHttpTransport): + """ + Reuse sockets + """ + + def __init__(self, host='127.0.0.1', port=8098, prefix='riak', + mapred_prefix='mapred', + client_id=None): + super(RiakHttpReuseTransport, self).__init__(host=host, + port=port, + prefix=prefix, + mapred_prefix= + mapred_prefix, + client_id=client_id) + + def __copy__(self): + return RiakHttpReuseTransport(self._host, self._port, self._prefix, + self._mapred_prefix) + + @classmethod + def httplib_request(cls, method, host, port, uri, headers, body=''): + # Run the request... + client = None + response = None + try: + client = httplib.HTTPConnection(host, port) + + #handle the connection myself, try to reuse sockets + client.auto_open = 0 + client.connect() + client.sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR, + client.sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) | 1) + + client.request(method, uri, body, headers) + response = client.getresponse() + + # Get the response headers... + response_headers = {'http_code': response.status} + for (key, value) in response.getheaders(): + response_headers[key.lower()] = value + + # Get the body... + response_body = response.read() + response.close() + + #close, this does not make any difference + client.close() + + return response_headers, response_body + except: + if client is not None: client.close() + if response is not None: response.close() + raise + +import urllib3 + +class RiakHttpPoolTransport(RiakHttpTransport): + """ + Use HTTP pool + """ + + http_pool = None + + def __init__(self, host='127.0.0.1', port=8098, prefix='riak', + mapred_prefix='mapred', + client_id=None): + super(RiakHttpPoolTransport, self).__init__(host=host, + port=port, + prefix=prefix, + mapred_prefix= + mapred_prefix, + client_id=client_id) + + def __copy__(self): + return RiakHttpPoolTransport(self._host, self._port, self._prefix, + self._mapred_prefix) + + @classmethod + def httplib_request(cls, method, host, port, uri, headers, body=''): + try: + if cls.http_pool is None: + cls.http_pool = urllib3.connection_from_url('http://%s:%d' % (host, port), maxsize=10) + + response = cls.http_pool.urlopen(method, uri, body, headers) + + response_headers = {'http_code': response.status} + for key, value in response.getheaders().iteritems(): + response_headers[key.lower()] = value + + response_body = response.data + + return response_headers, response_body + except: + raise \ No newline at end of file diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 00524adb..4257427e 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -120,7 +120,7 @@ def get_client_id(self): if msg_code == MSG_CODE_GET_CLIENT_ID_RESP: return resp.client_id else: - raise RiakError("unexpected protocol buffer message code: ", msg_code) + raise RiakError("unexpected protocol buffer message code: %d"%msg_code) def set_client_id(self, client_id): """ @@ -135,7 +135,7 @@ def set_client_id(self, client_id): if msg_code == MSG_CODE_SET_CLIENT_ID_RESP: return True else: - raise RiakError("unexpected protocol buffer message code: ", msg_code) + raise RiakError("unexpected protocol buffer message code: %d"%msg_code) def get(self, robj, r = None, vtag = None): """ @@ -159,12 +159,10 @@ def get(self, robj, r = None, vtag = None): contents = [] for c in resp.content: contents.append(self.decode_content(c)) - return (resp.vclock, contents) + return resp.vclock, contents else: return 0 - return 0 - def put(self, robj, w = None, dw = None, return_body = True): """ Serialize get request and deserialize response @@ -174,7 +172,7 @@ def put(self, robj, w = None, dw = None, return_body = True): req = riakclient_pb2.RpbPutReq() req.w = self.translate_rw_val(w) req.dw = self.translate_rw_val(dw) - if return_body == True: + if return_body: req.return_body = 1 req.bucket = bucket.get_name() @@ -189,12 +187,12 @@ def put(self, robj, w = None, dw = None, return_body = True): self.send_msg(MSG_CODE_PUT_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_PUT_RESP: - raise RiakError("unexpected protocol buffer message code: ", msg_code) + raise RiakError("unexpected protocol buffer message code: %d"%msg_code) if resp is not None: contents = [] for c in resp.content: contents.append(self.decode_content(c)) - return (resp.vclock, contents) + return resp.vclock, contents def delete(self, robj, rw = None): """ @@ -212,7 +210,7 @@ def delete(self, robj, rw = None): self.send_msg(MSG_CODE_DEL_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_DEL_RESP: - raise RiakError("unexpected protocol buffer message code: ", msg_code) + raise RiakError("unexpected protocol buffer message code: %d"%msg_code) return self def get_keys(self, bucket): @@ -228,13 +226,13 @@ def get_keys(self, bucket): while True: msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_LIST_KEYS_RESP: - raise RiakError("unexpected protocol buffer message code: ", msg_code) + raise RiakError("unexpected protocol buffer message code: %d"%msg_code) for key in resp.keys: keys.append(key) if resp.HasField("done") and resp.done: - break; + break return keys @@ -246,7 +244,7 @@ def get_buckets(self): self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_LIST_BUCKETS_RESP: - raise RiakError("unexpected protocol buffer message code: ", msg_code) + raise RiakError("unexpected protocol buffer message code: %d"%msg_code) return resp.buckets def get_bucket_props(self, bucket): @@ -260,7 +258,7 @@ def get_bucket_props(self, bucket): self.send_msg(MSG_CODE_GET_BUCKET_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_GET_BUCKET_RESP: - raise RiakError("unexpected protocol buffer message code: ", msg_code) + raise RiakError("unexpected protocol buffer message code: %d"%msg_code) props = {} if resp.props.HasField('n_val'): props['n_val'] = resp.props.n_val @@ -285,7 +283,7 @@ def set_bucket_props(self, bucket, props): self.send_msg(MSG_CODE_SET_BUCKET_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_SET_BUCKET_RESP: - raise RiakError("unexpected protocol buffer message code: ", msg_code) + raise RiakError("unexpected protocol buffer message code: %d"%msg_code) return self @@ -310,8 +308,7 @@ def mapred(self, inputs, query, timeout=None): while True: msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_MAPRED_RESP: - raise RiakError("unexpected protocol buffer message code: ", - msg_code) + raise RiakError("unexpected protocol buffer message code: %d"%msg_code) if resp.HasField("phase") and resp.HasField("response"): content = json.loads(resp.response) if resp.phase in result: @@ -324,7 +321,7 @@ def mapred(self, inputs, query, timeout=None): # If a single result - return the same as the HTTP interface does # otherwise return all the phase information - if len(result) == 0: + if not len(result): return None elif len(result) == 1: return result[max(result.keys())] @@ -399,7 +396,7 @@ def recv_msg(self): def recv_pkt(self): nmsglen = self._sock.recv(4) - if (len(nmsglen) != 4): + if len(nmsglen) != 4: raise RiakError("Socket returned short packet length {0} - expected 4". format(nmsglen)) msglen, = struct.unpack('!i', nmsglen) @@ -445,7 +442,7 @@ def decode_content(self, rpb_content): else: tag = None links.append(RiakLink(bucket, key, tag)) - if links != []: + if links: metadata[MD_LINKS] = links if rpb_content.HasField("last_mod"): metadata[MD_LASTMOD] = rpb_content.last_mod @@ -456,10 +453,9 @@ def decode_content(self, rpb_content): usermeta[usermd.key] = usermd.value if len(usermeta) > 0: metadata[MD_USERMETA] = usermeta - return (metadata, rpb_content.value) + return metadata, rpb_content.value def pbify_content(self, metadata, data, rpb_content) : - pbmetadata = {} # Convert the broken out fields, building up # pbmetadata for any unknown ones for k,v in metadata.iteritems(): @@ -481,3 +477,117 @@ def pbify_content(self, metadata, data, rpb_content) : pb_link.key = link.get_key() pb_link.tag = link.get_tag() rpb_content.value = data + +from Queue import Empty, Full, Queue +class RiakPbcPoolTransport(RiakTransport): + """Threadsafe pool of PBC connections, based on urllib3's pool [aka Queue]""" + def __init__(self, host='127.0.0.1', port=8087, client_id=None, maxsize=0, block=False, timeout=None): + self.host = host + self.port = port + self.client_id = client_id + self.block = block + self.timeout = timeout + + self.pool = Queue(None) + [self.pool.put(None) for _ in xrange(maxsize)] + + self.num_connections = 0 + self.num_requests = 0 + + def _new_conn(self): + """New PBC connection""" + self.num_connections += 1 + return RiakPbcTransport(self.host, self.port, self.client_id) + + def _get_conn(self): + conn = None + try: + conn = self.pool.get(block=self.block, timeout=self.timeout) + except Empty: + pass + return conn or self._new_conn() + + def _put_conn(self, conn): + try: + self.pool.put(conn, block=False) + except Full: + self.num_connections -= 1 + + def _make_call(self, function): + """checkout conn, try operation, put conn back in pool""" + self.num_requests += 1 + try: + conn = self._get_conn() + rv = function(conn) + self._put_conn(conn) + except Exception: + self.num_connections -= 1 + #re-raise leave caller decide what to do + raise + return rv + + def ping(self): + """ + Ping the remote server + @return boolean + """ + return self._make_call(lambda conn: conn.ping()) + + def get(self, robj, r = None, vtag = None): + """ + Serialize get request and deserialize response + @return (vclock=None, [(metadata, value)]=None) + """ + return self._make_call(lambda conn: conn.get(robj, r, vtag)) + + def put(self, robj, w = None, dw = None, return_body = True): + """ + Serialize put request and deserialize response - if 'content' + is true, retrieve the updated metadata/content + @return (vclock=None, [(metadata, value)]=None) + """ + return self._make_call(lambda conn: conn.put(robj, w, dw, return_body)) + + def delete(self, robj, rw = None): + """ + Serialize delete request and deserialize response + @return true + """ + return self._make_call(lambda conn: conn.delete(robj, rw)) + + def get_buckets(self): + """ + Serialize bucket listing request and deserialize response + """ + return self._make_call(lambda conn: conn.get_buckets()) + + def get_bucket_props(self, bucket) : + """ + Serialize get bucket property request and deserialize response + @return dict() + """ + return self._make_call(lambda conn: conn.get_bucket_props(bucket)) + + def set_bucket_props(self, bucket, props) : + """ + Serialize set bucket property request and deserialize response + bucket = bucket object + props = dictionary of properties + @return boolean + """ + return self._make_call(lambda conn: conn.set_bucket_props(bucket, props)) + + def mapred(self, inputs, query, timeout = None) : + """ + Serialize map/reduce request + """ + return self._make_call(lambda conn: conn.mapred(inputs, query, timeout)) + + def set_client_id(self, client_id): + """Mmm, this can turn ugly if you use different id for different objects in the pool""" + return self._make_call(lambda conn: conn.set_client_id(client_id)) + + def get_client_id(self): + """see set_client_id notes, you can do wrong with this""" + return self._make_call(lambda conn: conn.get_client_id()) + \ No newline at end of file diff --git a/riak/transports/transport.py b/riak/transports/transport.py index ebc2f22f..a9bcffb5 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -31,17 +31,17 @@ class RiakTransport(object): @classmethod def make_random_client_id(self): - ''' + """ Returns a random client identifier - ''' + """ return 'py_%s' % base64.b64encode( str(random.randint(1, 1073741824))) @classmethod def make_fixed_client_id(self): - ''' + """ Returns a unique identifier for the current machine/process/thread. - ''' + """ machine = platform.node() process = os.getpid() thread = threading.currentThread().getName() @@ -108,3 +108,6 @@ def mapred(self, inputs, query, timeout = None) : def set_client_id(self, client_id): raise RiakError("not implemented") + def get_client_id(self, client_id): + raise RiakError("not implemented") + diff --git a/setup.py b/setup.py index e5b06a04..3284f1a7 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ def make_pb(): name='riak', version='1.2.2', packages = find_packages(), - install_requires = ['protobuf>=2.3.0'], + install_requires = ['protobuf>=2.3.0', 'urllib3>=0.4.0'], dependency_links = ["http://downloads.basho.com/support"], package_data = { '' : ['*.proto'] From 360ef8eab662a08be5cdcc297ed2faac0b2cf154 Mon Sep 17 00:00:00 2001 From: gilles Date: Thu, 30 Jun 2011 09:26:38 -0700 Subject: [PATCH 0028/1060] let Queue decide if it's bounded or not --- riak/transports/pbc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 4257427e..68d2eb00 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -488,7 +488,8 @@ def __init__(self, host='127.0.0.1', port=8087, client_id=None, maxsize=0, block self.block = block self.timeout = timeout - self.pool = Queue(None) + self.pool = Queue(maxsize) + # Fill the queue up so that doing get() on it will block properly (check Queue#get) [self.pool.put(None) for _ in xrange(maxsize)] self.num_connections = 0 From 94f8605cec490d68b88abc01496de123ce5cd7a8 Mon Sep 17 00:00:00 2001 From: gilles Date: Thu, 30 Jun 2011 10:26:34 -0700 Subject: [PATCH 0029/1060] remove counters (inc not thread safe) comments in RiakTransport --- riak/transports/pbc.py | 15 +++++++++------ riak/transports/transport.py | 8 +++++++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 68d2eb00..3d379109 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -492,12 +492,14 @@ def __init__(self, host='127.0.0.1', port=8087, client_id=None, maxsize=0, block # Fill the queue up so that doing get() on it will block properly (check Queue#get) [self.pool.put(None) for _ in xrange(maxsize)] - self.num_connections = 0 - self.num_requests = 0 + #not thread safe: http://29a.ch/2009/2/20/atomic-get-and-increment-in-python + #TODO replace with thread safe + #self.num_connections = 0 + #self.num_requests = 0 def _new_conn(self): """New PBC connection""" - self.num_connections += 1 + #self.num_connections += 1 return RiakPbcTransport(self.host, self.port, self.client_id) def _get_conn(self): @@ -512,17 +514,18 @@ def _put_conn(self, conn): try: self.pool.put(conn, block=False) except Full: - self.num_connections -= 1 + pass + #self.num_connections -= 1 def _make_call(self, function): """checkout conn, try operation, put conn back in pool""" - self.num_requests += 1 + #self.num_requests += 1 try: conn = self._get_conn() rv = function(conn) self._put_conn(conn) except Exception: - self.num_connections -= 1 + #self.num_connections -= 1 #re-raise leave caller decide what to do raise return rv diff --git a/riak/transports/transport.py b/riak/transports/transport.py index a9bcffb5..8b080c75 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -106,8 +106,14 @@ def mapred(self, inputs, query, timeout = None) : raise RiakError("not implemented") def set_client_id(self, client_id): + """ + TODO Only used for PBC transports, should it be here Or in a new PBCTransport base class? + """ raise RiakError("not implemented") - def get_client_id(self, client_id): + def get_client_id(self): + """ + TODO Only used for PBC transports, should it be here Or in a new PBCTransport base class? + """ raise RiakError("not implemented") From c0e0f715cc79032cf12d4e78fb9c1c62a617ec84 Mon Sep 17 00:00:00 2001 From: gilles Date: Thu, 30 Jun 2011 14:00:56 -0700 Subject: [PATCH 0030/1060] removed the counter altogehter. The best way to know how many connections are in the pool is to call poo.qsize() --- riak/transports/pbc.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 3d379109..5f56bfc1 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -492,14 +492,8 @@ def __init__(self, host='127.0.0.1', port=8087, client_id=None, maxsize=0, block # Fill the queue up so that doing get() on it will block properly (check Queue#get) [self.pool.put(None) for _ in xrange(maxsize)] - #not thread safe: http://29a.ch/2009/2/20/atomic-get-and-increment-in-python - #TODO replace with thread safe - #self.num_connections = 0 - #self.num_requests = 0 - def _new_conn(self): """New PBC connection""" - #self.num_connections += 1 return RiakPbcTransport(self.host, self.port, self.client_id) def _get_conn(self): @@ -515,18 +509,14 @@ def _put_conn(self, conn): self.pool.put(conn, block=False) except Full: pass - #self.num_connections -= 1 def _make_call(self, function): """checkout conn, try operation, put conn back in pool""" - #self.num_requests += 1 try: conn = self._get_conn() rv = function(conn) self._put_conn(conn) except Exception: - #self.num_connections -= 1 - #re-raise leave caller decide what to do raise return rv From 0b5fe1c021ec06ef305bdc9442bffb048215dff2 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Fri, 1 Jul 2011 16:15:31 +0200 Subject: [PATCH 0031/1060] Allow setting the client_id for HTTP transports. --- riak/tests/test_all.py | 6 ++++-- riak/transports/http.py | 8 +++++++- riak/transports/transport.py | 5 +++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 8018817d..c4b4a814 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -445,7 +445,6 @@ def test_search_integration(self): self.assertEqual(len(results), 3) def test_store_binary_object_from_file(self): - print __file__ bucket = self.client.bucket('bucket') rand = str(self.randint()) obj = bucket.new_binary_from_file('foo_from_file', os.path.dirname(__file__) + "/test_all.py") @@ -469,7 +468,6 @@ def test_store_metadata(self): obj.set_usermeta({'custom': 'some metadata'}) obj.store() obj = bucket.get('fooster') - print(obj.get_usermeta()) self.assertEqual('some metadata', obj.get_usermeta()['custom']) def test_store_binary_object_from_file_should_fail_if_file_not_found(self): @@ -755,6 +753,10 @@ def test_generate_key(self): bucket.new(None, data={}).store() self.assertEqual(len(bucket.get_keys()), 1) + def test_set_client_id(self): + self.client.set_client_id("Client") + self.assertEqual(self.client.get_transport().get_client_id(), "Client") + class RiakHttpReuseTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): diff --git a/riak/transports/http.py b/riak/transports/http.py index 9c8121a8..991ceaa9 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -66,6 +66,12 @@ def __copy__(self): return RiakHttpTransport(self._host, self._port, self._prefix, self._mapred_prefix) + def set_client_id(self, client_id): + self._client_id = client_id + + def get_client_id(self): + return self._client_id + def ping(self) : """ Check server is alive over HTTP @@ -566,4 +572,4 @@ def httplib_request(cls, method, host, port, uri, headers, body=''): return response_headers, response_body except: - raise \ No newline at end of file + raise diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 8b080c75..6d39dae5 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -107,13 +107,14 @@ def mapred(self, inputs, query, timeout = None) : def set_client_id(self, client_id): """ - TODO Only used for PBC transports, should it be here Or in a new PBCTransport base class? + Set the client id. This overrides the default, random client id, which is automatically + generated when none is specified in when creating the transport object. """ raise RiakError("not implemented") def get_client_id(self): """ - TODO Only used for PBC transports, should it be here Or in a new PBCTransport base class? + Fetch the client id for the transport. """ raise RiakError("not implemented") From 1bed294c1d9ea2640c3d40f505f7cd88e7e34404 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Fri, 1 Jul 2011 16:21:07 +0200 Subject: [PATCH 0032/1060] Rename RiakPbcPoolTransport to RiakPbcCachedTransport. It's more of a connection cache than a pool. --- riak/__init__.py | 2 +- riak/tests/test_all.py | 10 +++++----- riak/transports/__init__.py | 2 +- riak/transports/pbc.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index b3ad1575..d8577d1d 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -41,7 +41,7 @@ def __str__(self): from client import RiakClient from mapreduce import RiakMapReduce, RiakMapReducePhase, RiakLinkPhase,\ RiakKeyFilter -from transports.pbc import RiakPbcTransport, RiakPbcPoolTransport +from transports.pbc import RiakPbcTransport, RiakPbcCachedTransport from transports.http import RiakHttpTransport, RiakHttpReuseTransport, RiakHttpPoolTransport ONE = "one" diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index c4b4a814..4dc97e03 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -10,7 +10,7 @@ import random import unittest from riak import RiakClient -from riak import RiakPbcTransport, RiakPbcPoolTransport +from riak import RiakPbcTransport, RiakPbcCachedTransport from riak import RiakHttpTransport, RiakHttpPoolTransport, RiakHttpReuseTransport from riak import RiakKeyFilter, key_filter @@ -691,20 +691,20 @@ def test_uses_client_id_if_given(self): client_id = zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) # -class RiakPbcPoolTransportCase(BaseTestCase, MapReduceAliasTestMixIn, +class RiakPbcCachedTransportCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): self.host = PB_HOST self.port = PB_PORT - self.transport_class = RiakPbcPoolTransport - super(RiakPbcPoolTransportCase, self).setUp() + self.transport_class = RiakPbcCachedTransport + super(RiakPbcCachedTransportCase, self).setUp() def test_uses_client_id_if_given(self): self.host = PB_HOST self.port = PB_PORT zero_client_id = "\0\0\0\0" c = RiakClient(PB_HOST, PB_PORT, - transport_class = RiakPbcPoolTransport, + transport_class = RiakPbcCachedTransport, client_id = zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) # diff --git a/riak/transports/__init__.py b/riak/transports/__init__.py index ca0cd5bf..f970fed9 100644 --- a/riak/transports/__init__.py +++ b/riak/transports/__init__.py @@ -1,4 +1,4 @@ from http import RiakHttpTransport, RiakHttpReuseTransport, RiakHttpPoolTransport -from pbc import RiakPbcTransport, RiakPbcPoolTransport +from pbc import RiakPbcTransport, RiakPbcCachedTransport diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 5f56bfc1..eec68084 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -479,7 +479,7 @@ def pbify_content(self, metadata, data, rpb_content) : rpb_content.value = data from Queue import Empty, Full, Queue -class RiakPbcPoolTransport(RiakTransport): +class RiakPbcCachedTransport(RiakTransport): """Threadsafe pool of PBC connections, based on urllib3's pool [aka Queue]""" def __init__(self, host='127.0.0.1', port=8087, client_id=None, maxsize=0, block=False, timeout=None): self.host = host @@ -584,4 +584,4 @@ def set_client_id(self, client_id): def get_client_id(self): """see set_client_id notes, you can do wrong with this""" return self._make_call(lambda conn: conn.get_client_id()) - \ No newline at end of file + From 8ff2c9d0db3b690e1eb6c5e314583d35caa6b8ed Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Mon, 4 Jul 2011 16:15:44 +0200 Subject: [PATCH 0033/1060] First draft of Ripple's test server port. --- riak/test_server.py | 120 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 riak/test_server.py diff --git a/riak/test_server.py b/riak/test_server.py new file mode 100644 index 00000000..137e3b63 --- /dev/null +++ b/riak/test_server.py @@ -0,0 +1,120 @@ +import os.path +import threading +import string +import re +import random + +class TestServer: + VM_ARGS_DEFAULTS = { + "-name": "riaktest%d@127.0.0.1" % random.randint(0, 100000), + "-setcookie": "%d_%d" % (random.randint(0, 100000), random.randint(0, 100000)), + "+K": True, + "+A": 64, + "-smp": "enable", + "-env ERL_MAX_PORTS": 4096, + "-env ERL_FULLSWEEP_AFTER": 10, + "-pa": os.path.join("../../../erl_src", __file__) + } + + APP_CONFIG_DEFAULTS = { + "riak_core": { + "web_ip": "127.0.0.1", + "web_port": 9000, + "handoff_port": 9001, + "ring_creation_size": 64 + }, + "riak_kv": { + "storage_backend": "riak_kv_test_backend", + "pb_ip": "127.0.0.1", + "pb_port": 9002, + "js_vm_count": 8, + "js_max_vm_mem": 8, + "js_thread_stack": 16, + "riak_kv_stat": True, + "map_cache_size": 0, + "vnode_cache_entries": 0 + }, + "riak_search": { + "enabled": True, + "search_backend": "riak_search_test_backend" + }, + "luwak": { + "enabled": True + } + } + + def __init__(self, tmp_dir="/tmp/riak/test_server", + bin_dir=os.path.expanduser("~/.riak/install/riak-0.14.2/bin")): + self.lock = threading.Lock() + self.temp_dir = "/tmp/riak/test_server" + self.bin_dir = bin_dir + self._prepared = False + + def prepare(self): + if not self._prepared: + self.create_temp_directories() + self._riak_script = os.path.join(self._temp_bin, "riak") + self.write_riak_script() + self.write_vm_args() + self.write_app_config() + self._prepared = True + + def create_temp_directories(self): + directories = ["bin", "etc", "log", "data", "pipe"] + for directory in directories: + dir = os.path.normpath(os.path.join(self.temp_dir, directory)) + if not os.path.exists(dir): + os.makedirs(dir) + setattr(self, "_temp_%s" % directory, dir) + + def write_riak_script(self): + temp_bin_file = open(self._riak_script, "wb") + riak_file = open(os.path.join(self.bin_dir, "riak"), "r") + for line in riak_file.readlines(): + line = re.sub("(RUNNER_SCRIPT_DIR=)(.*)", r'\1%s' % self._temp_bin, line) + line = re.sub("(RUNNER_ETC_DIR=)(.*)", r'\1%s' % self._temp_etc, line) + line = re.sub("(RUNNER_USER=)(.*)", r'\1', line) + line = re.sub("(RUNNER_LOG_DIR=)(.*)", r'\1%s' % self._temp_log, line) + line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % self._temp_pipe, line) + + if string.strip(line) == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}": + line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join("..", self.bin_dir)) + + temp_bin_file.write(line) + + os.fchmod(temp_bin_file.fileno(), 0755) + temp_bin_file.close() + riak_file.close() + + def write_vm_args(self): + with open(os.path.join(self._temp_etc, "vm.args"), 'wb') as vm_args: + for arg, value in self.__class__.VM_ARGS_DEFAULTS.items(): + vm_args.write("%s %s\n" % (arg, value)) + + def write_app_config(self): + with open(os.path.join(self._temp_etc, "app.config"), "wb") as app_config: + app_config.write(erlang_config(self.__class__.APP_CONFIG_DEFAULTS)) + + +def erlang_config(hash, depth=1): + def printable(item): + k, v = item + if isinstance(v, str): + p = '"%s"' % v + elif isinstance(v, dict): + p = erlang_config(v, depth + 1) + elif isinstance(v, bool): + p = ("%s" % v).lower() + else: + p = "%s" % v + + return "{%s, %s}" % (k, p) + + padding = ' ' * depth + parent_padding = ' ' * (depth-1) + values = (",\n%s" % padding).join(map(printable, hash.items())) + return "[\n%s%s\n%s]" % (padding, values, parent_padding) + + +if __name__ == "__main__": + TestServer().prepare() From 01c8ba055f426f4ae338b1657731a5cff170dbe2 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Mon, 4 Jul 2011 17:43:01 +0200 Subject: [PATCH 0034/1060] Starting Riak works now with the test backends. --- riak/test_server.py | 92 +++++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 33 deletions(-) diff --git a/riak/test_server.py b/riak/test_server.py index 137e3b63..5e24ea55 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -3,17 +3,38 @@ import string import re import random +from subprocess import Popen, PIPE + +def erlang_config(hash, depth=1): + def printable(item): + k, v = item + if isinstance(v, str): + p = '"%s"' % v + elif isinstance(v, dict): + p = erlang_config(v, depth + 1) + elif isinstance(v, bool): + p = ("%s" % v).lower() + else: + p = "%s" % v + + return "{%s, %s}" % (k, p) + + padding = ' ' * depth + parent_padding = ' ' * (depth-1) + values = (",\n%s" % padding).join(map(printable, hash.items())) + return "[\n%s%s\n%s]" % (padding, values, parent_padding) + class TestServer: VM_ARGS_DEFAULTS = { "-name": "riaktest%d@127.0.0.1" % random.randint(0, 100000), "-setcookie": "%d_%d" % (random.randint(0, 100000), random.randint(0, 100000)), - "+K": True, + "+K": "true", "+A": 64, "-smp": "enable", "-env ERL_MAX_PORTS": 4096, "-env ERL_FULLSWEEP_AFTER": 10, - "-pa": os.path.join("../../../erl_src", __file__) + "-pa": os.path.abspath(os.path.join(os.path.dirname(__file__), "../erl_src")) } APP_CONFIG_DEFAULTS = { @@ -24,7 +45,7 @@ class TestServer: "ring_creation_size": 64 }, "riak_kv": { - "storage_backend": "riak_kv_test_backend", + "storage_backend": bytearray("riak_kv_test_backend"), "pb_ip": "127.0.0.1", "pb_port": 9002, "js_vm_count": 8, @@ -36,7 +57,7 @@ class TestServer: }, "riak_search": { "enabled": True, - "search_backend": "riak_search_test_backend" + "search_backend": bytearray("riak_search_test_backend") }, "luwak": { "enabled": True @@ -45,18 +66,20 @@ class TestServer: def __init__(self, tmp_dir="/tmp/riak/test_server", bin_dir=os.path.expanduser("~/.riak/install/riak-0.14.2/bin")): - self.lock = threading.Lock() + self._lock = threading.Lock() self.temp_dir = "/tmp/riak/test_server" self.bin_dir = bin_dir self._prepared = False + self._started = False + self.vm_args = self.__class__.VM_ARGS_DEFAULTS def prepare(self): if not self._prepared: self.create_temp_directories() self._riak_script = os.path.join(self._temp_bin, "riak") - self.write_riak_script() - self.write_vm_args() - self.write_app_config() + self.__write_riak_script() + self.__write_vm_args() + self.__write_app_config() self._prepared = True def create_temp_directories(self): @@ -67,7 +90,27 @@ def create_temp_directories(self): os.makedirs(dir) setattr(self, "_temp_%s" % directory, dir) - def write_riak_script(self): + def start(self): + if self._prepared and not self._started: + print("Starting...") + with self._lock: + self._server = Popen([self._riak_script, "console"], stdin=PIPE, stdout=PIPE, stderr=PIPE) + self._server.stdin.write("\n") + self._server.stdin.flush() + self.wait_for_erlang_prompt() + + def wait_for_erlang_prompt(self): + prompted = False + buffer = "" + while not prompted: + line = self._server.stdout.read(1) + if len(line) > 0: + buffer += line + if re.search(r"\(%s\)\d+>" % self.vm_args["-name"], buffer): + print("Started...") + prompted = True + + def __write_riak_script(self): temp_bin_file = open(self._riak_script, "wb") riak_file = open(os.path.join(self.bin_dir, "riak"), "r") for line in riak_file.readlines(): @@ -78,7 +121,7 @@ def write_riak_script(self): line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % self._temp_pipe, line) if string.strip(line) == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}": - line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join("..", self.bin_dir)) + line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join(self.bin_dir, "..")) temp_bin_file.write(line) @@ -86,35 +129,18 @@ def write_riak_script(self): temp_bin_file.close() riak_file.close() - def write_vm_args(self): + def __write_vm_args(self): with open(os.path.join(self._temp_etc, "vm.args"), 'wb') as vm_args: for arg, value in self.__class__.VM_ARGS_DEFAULTS.items(): vm_args.write("%s %s\n" % (arg, value)) - def write_app_config(self): + def __write_app_config(self): with open(os.path.join(self._temp_etc, "app.config"), "wb") as app_config: app_config.write(erlang_config(self.__class__.APP_CONFIG_DEFAULTS)) + app_config.write(".") -def erlang_config(hash, depth=1): - def printable(item): - k, v = item - if isinstance(v, str): - p = '"%s"' % v - elif isinstance(v, dict): - p = erlang_config(v, depth + 1) - elif isinstance(v, bool): - p = ("%s" % v).lower() - else: - p = "%s" % v - - return "{%s, %s}" % (k, p) - - padding = ' ' * depth - parent_padding = ' ' * (depth-1) - values = (",\n%s" % padding).join(map(printable, hash.items())) - return "[\n%s%s\n%s]" % (padding, values, parent_padding) - - if __name__ == "__main__": - TestServer().prepare() + server = TestServer() + server.prepare() + server.start() From 1a0dee41b2190128b1ab9ccb05043686b74a62ed Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Mon, 4 Jul 2011 17:53:05 +0200 Subject: [PATCH 0035/1060] Add ring_state_dir. --- riak/test_server.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/riak/test_server.py b/riak/test_server.py index 5e24ea55..76eca6b0 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -3,6 +3,7 @@ import string import re import random +import time from subprocess import Popen, PIPE def erlang_config(hash, depth=1): @@ -72,6 +73,8 @@ def __init__(self, tmp_dir="/tmp/riak/test_server", self._prepared = False self._started = False self.vm_args = self.__class__.VM_ARGS_DEFAULTS + self.app_config = self.__class__.APP_CONFIG_DEFAULTS + self.app_config["riak_core"]["ring_state_dir"] = os.path.join(self.temp_dir, "data", "ring") def prepare(self): if not self._prepared: @@ -92,12 +95,13 @@ def create_temp_directories(self): def start(self): if self._prepared and not self._started: - print("Starting...") with self._lock: self._server = Popen([self._riak_script, "console"], stdin=PIPE, stdout=PIPE, stderr=PIPE) self._server.stdin.write("\n") self._server.stdin.flush() self.wait_for_erlang_prompt() + while True: + print(self._server.stderr.read(1)) def wait_for_erlang_prompt(self): prompted = False @@ -111,23 +115,21 @@ def wait_for_erlang_prompt(self): prompted = True def __write_riak_script(self): - temp_bin_file = open(self._riak_script, "wb") - riak_file = open(os.path.join(self.bin_dir, "riak"), "r") - for line in riak_file.readlines(): - line = re.sub("(RUNNER_SCRIPT_DIR=)(.*)", r'\1%s' % self._temp_bin, line) - line = re.sub("(RUNNER_ETC_DIR=)(.*)", r'\1%s' % self._temp_etc, line) - line = re.sub("(RUNNER_USER=)(.*)", r'\1', line) - line = re.sub("(RUNNER_LOG_DIR=)(.*)", r'\1%s' % self._temp_log, line) - line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % self._temp_pipe, line) - - if string.strip(line) == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}": - line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join(self.bin_dir, "..")) - - temp_bin_file.write(line) - - os.fchmod(temp_bin_file.fileno(), 0755) - temp_bin_file.close() - riak_file.close() + with open(self._riak_script, "wb") as temp_bin_file, open(os.path.join(self.bin_dir, "riak"), "r") as riak_file: + + for line in riak_file.readlines(): + line = re.sub("(RUNNER_SCRIPT_DIR=)(.*)", r'\1%s' % self._temp_bin, line) + line = re.sub("(RUNNER_ETC_DIR=)(.*)", r'\1%s' % self._temp_etc, line) + line = re.sub("(RUNNER_USER=)(.*)", r'\1', line) + line = re.sub("(RUNNER_LOG_DIR=)(.*)", r'\1%s' % self._temp_log, line) + line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % self._temp_pipe, line) + + if string.strip(line) == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}": + line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join(self.bin_dir, "..")) + + temp_bin_file.write(line) + + os.fchmod(temp_bin_file.fileno(), 0755) def __write_vm_args(self): with open(os.path.join(self._temp_etc, "vm.args"), 'wb') as vm_args: From 9bdb94be873f7bd493ef20776cebf198ecc6bd3e Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Mon, 4 Jul 2011 18:11:03 +0200 Subject: [PATCH 0036/1060] Add stop and cleanup methods. --- riak/test_server.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/riak/test_server.py b/riak/test_server.py index 76eca6b0..f514ef98 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -3,7 +3,7 @@ import string import re import random -import time +import shutil from subprocess import Popen, PIPE def erlang_config(hash, depth=1): @@ -100,8 +100,22 @@ def start(self): self._server.stdin.write("\n") self._server.stdin.flush() self.wait_for_erlang_prompt() - while True: - print(self._server.stderr.read(1)) + self._started = True + + def stop(self): + if self._started: + with self._lock: + self._server.stdin.write("init:stop().\n") + self._server.stdin.flush() + self._server.wait() + self._started = False + + def cleanup(self): + if self._started: + self.stop() + + shutil.rmtree(self.temp_dir, True) + self._prepared = False def wait_for_erlang_prompt(self): prompted = False @@ -133,12 +147,12 @@ def __write_riak_script(self): def __write_vm_args(self): with open(os.path.join(self._temp_etc, "vm.args"), 'wb') as vm_args: - for arg, value in self.__class__.VM_ARGS_DEFAULTS.items(): + for arg, value in self.vm_args.items(): vm_args.write("%s %s\n" % (arg, value)) def __write_app_config(self): with open(os.path.join(self._temp_etc, "app.config"), "wb") as app_config: - app_config.write(erlang_config(self.__class__.APP_CONFIG_DEFAULTS)) + app_config.write(erlang_config(self.app_config)) app_config.write(".") @@ -146,3 +160,5 @@ def __write_app_config(self): server = TestServer() server.prepare() server.start() + server.stop() + server.cleanup() From 2649a0dcde4b15a19527496b9cfa42df328c76f0 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Mon, 4 Jul 2011 18:36:42 +0200 Subject: [PATCH 0037/1060] Add recycle() method. --- riak/test_server.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/riak/test_server.py b/riak/test_server.py index f514ef98..b2c4ac09 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -62,6 +62,13 @@ class TestServer: }, "luwak": { "enabled": True + }, + "sasl": { + "sasl_error_logger": bytearray('{file, "log/sasl-error.log"}'), + "errlog_type": bytearray("error"), + "error_logger_mf_dir": "log/sasl", + "error_logger_mf_maxbytes": 10485760, + "error_logger_mf_maxfiles": 5 } } @@ -117,13 +124,46 @@ def cleanup(self): shutil.rmtree(self.temp_dir, True) self._prepared = False + def recycle(self): + if self._started: + with self._lock: + stdin = self._server.stdin + if self.app_config["riak_kv"]["storage_backend"] == "riak_kv_test_backend": + stdin.write("riak_kv_test_backend:reset().\n") + stdin.flush() + self.wait_for_erlang_prompt() + + if self.app_config["riak_search"]["enabled"]: + stdin.write("riak_search_test_backend:reset().\n") + stdin.flush() + self.wait_for_erlang_prompt() + else: + stdin.write("init:restart().\n") + stdin.flush() + self.wait_for_erlang_prompt() + self.wait_for_startup() + + def wait_for_startup(self): + listening = False + while not listening: + try: + s = socket.create_connection((self.app_config["riak_core"]["web_ip"], self.app_config["riak_core"]["web_port"]), 1.0) + except socket.error, (value, message): + pass + else: + listening = True + + def wait_for_erlang_prompt(self): + print("Waiting") prompted = False buffer = "" while not prompted: line = self._server.stdout.read(1) if len(line) > 0: buffer += line + if len(buffer) % 100 == 0: + print(buffer) if re.search(r"\(%s\)\d+>" % self.vm_args["-name"], buffer): print("Started...") prompted = True @@ -160,5 +200,6 @@ def __write_app_config(self): server = TestServer() server.prepare() server.start() + server.recycle() server.stop() server.cleanup() From 31dc8adefe460ffc9b20a1f5cb7eb94949f5fcf6 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Tue, 5 Jul 2011 09:37:50 +0200 Subject: [PATCH 0038/1060] Remove now unneeded prints. --- riak/test_server.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/riak/test_server.py b/riak/test_server.py index b2c4ac09..cdc73e86 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -4,6 +4,7 @@ import re import random import shutil +import time from subprocess import Popen, PIPE def erlang_config(hash, depth=1): @@ -155,17 +156,13 @@ def wait_for_startup(self): def wait_for_erlang_prompt(self): - print("Waiting") prompted = False buffer = "" while not prompted: line = self._server.stdout.read(1) if len(line) > 0: buffer += line - if len(buffer) % 100 == 0: - print(buffer) if re.search(r"\(%s\)\d+>" % self.vm_args["-name"], buffer): - print("Started...") prompted = True def __write_riak_script(self): @@ -200,6 +197,10 @@ def __write_app_config(self): server = TestServer() server.prepare() server.start() + print("Started...") + time.sleep(20) + print("Recycling...") server.recycle() + time.sleep(20) server.stop() server.cleanup() From db4f49753ed83c9920b725ebf231a1640a9b821a Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Tue, 5 Jul 2011 09:40:42 +0200 Subject: [PATCH 0039/1060] Remove sasl configuration. --- riak/test_server.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/riak/test_server.py b/riak/test_server.py index cdc73e86..29985c4f 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -63,13 +63,6 @@ class TestServer: }, "luwak": { "enabled": True - }, - "sasl": { - "sasl_error_logger": bytearray('{file, "log/sasl-error.log"}'), - "errlog_type": bytearray("error"), - "error_logger_mf_dir": "log/sasl", - "error_logger_mf_maxbytes": 10485760, - "error_logger_mf_maxfiles": 5 } } From 5b3606acb0a53eccdff13b45d704c630a617808f Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Wed, 6 Jul 2011 17:58:27 +0200 Subject: [PATCH 0040/1060] Add test backends for the test server. --- erl_src/riak_kv_test_backend.beam | Bin 0 -> 7160 bytes erl_src/riak_kv_test_backend.erl | 174 +++++++++++++++++++++++++ erl_src/riak_search_test_backend.beam | Bin 0 -> 8484 bytes erl_src/riak_search_test_backend.erl | 175 ++++++++++++++++++++++++++ 4 files changed, 349 insertions(+) create mode 100644 erl_src/riak_kv_test_backend.beam create mode 100644 erl_src/riak_kv_test_backend.erl create mode 100644 erl_src/riak_search_test_backend.beam create mode 100644 erl_src/riak_search_test_backend.erl diff --git a/erl_src/riak_kv_test_backend.beam b/erl_src/riak_kv_test_backend.beam new file mode 100644 index 0000000000000000000000000000000000000000..f5a61ce9315fc57dd32fe28e7056f968f25d1a86 GIT binary patch literal 7160 zcmbVQ2UL^Uwodp136URu2)zbEl@>@u0t%r91Qh{$NkRx12ni$s#I9f&%ZOvaf?`3` z5fx-q22l}lY$&!-Y$%}EuwfrZefvc4a_^n>)_N~%eV%>x*=P524!_YO6a7den^)mc zVZ*~T$}AE|C!9ngg`28m;!I&?o=_uIYlJCcNv2ehs;kzBRT_G_R3TJLRe4esUCR*4 zWr|E)l~gU&=t)&_u_B$WlBUbl8mUT}sw)RVt(&Hlr^<1aR3eq-NrhTj9c89os#Gr3 zNGURfOk*TdXr$>6lL(Oe9kQPfg-8m&(;Ce^4hg)|?zq>&DoDpsV* zr9z2VtugtoQz(|ph02r+sYIiuDwHZ|w!A=Bs!}Oc{}PtVv2>|sfYwS%E`+FoC_s*a zyv4F}H8h!8D9y^&6i`xC%4`Y}q^LDOsb$lV*J(;@XwXoK1Z>gNsyv}~O{2<{Qqc@_ z>&cbM%-n446!huI6p%O#9j$(MQ@ai6)y)!XBvXyhN6et-IWh$fsQj#tmkflt`g~T8UBk)JqZ*J!OBWrTGFULDASK-a} zW_W9h56c*f zPr-~SR5lZM6ap8*RF{wvfFMAEJlA3e##@60t-+Y#7>N;% z5iHL&j6^d&g|V0*@mUy^$%g$TaT%yA!B{2u&y$pMe%&s-NBKC!a;3GUp<F}a;6F;4|A$o6KT28jc@&N@lYV%`r`zL} zbO+puOtUm#iue&2WS*a7nojU=2TBlt9ZrGbQ&9387?3v=CD1%T_dZ6(w5{=PiL?FRRls)Q6~QM(v!Tp(&`jDw)9efgc5g)p zeCR(qZr@+T`5#3b{?#DNUXmcw9HCcOFgo$+EXD{X4M-NS++lR);jUN^fc@i~;jWNs z0}}EEJoUs~FuDgEP7@O3&LsvT(_E3}5kwW4<_219pc&n@ZORhRZ3K{?UK1X$I1m;G z@^Ei3;jK0C{fOY+EHDAbk)u6+Oqy{Z?x{VmJlqS8jTeIn$A$yPCW4E5LG`^j6oQ@b z{qRL-+tU*kFus8Oy;;2oHh~Tj-3m@pZv%V(aMK4oabS~az675g&E^Q`4g!XN#pm=o z3Rf203zq$OI0qc%XdTraAWxjbp_}0(WSY~v2ila%q?vf){{8aekMhF#VUVk1(g=|c z>P<^yzpMm+F&|hP%)@zLjMqOaJd_m@$_iM3t_$pEArM*M{jd-OYMh{c!F;+?Zx};> zCC=EqQ!8d*&9tPE5aRTvBRv0TH ziHnCO5zK_B2nGY3(}R6vS{N`!}Gyx%ktCR&*M0zj*bSoEL05UBGOv7yxtG#Wo@3&1XocF$m zrhl$tQLf+uhBOa@xR@};@#)Oo3d94;5m>|dbjM!S2w>T>$+VGut!J-oBl;k1`>|I{B4j{a@6zzk?(LQK!U2+&u87u8uj|)Lm#Kote zl#5|&Cxec0=m3z6A%NUV8^PGeM}PP9@r{8- zKO}DgKr#$~X+PQm=stO?UK+{t*+sMqpijSRFOB%fAG8A z`6w>`Ub;^Y8j;VzfIi<4jr60nkX{vzLy8(H4O8L;#!sNCw5w zmxDn-I{=VQUw{n&$-4ts03?t0Sp$#^s-HOk$)MWf03?I-=>pIX^$=RuB05)rV(0eG zLwm;*7{C9va!?(H0FeI40HlwsYp~^gyi9}kW&@T@E0u^uA^?$fQ8Y{}8ty&toG*;N zXPbv#@bxASqjA$YuU40vmo&M#xTf#y+Ce(ibYcl_l}XZ)#8va2&a++`XA+kCCkp1OJNXadQ0Ro#Db4xp;{$D+VqsUnpBzU|Fu7rL{F8b6!_`k+x@zg{{&P`9F_X(xW*W@pWja=F!psNWRm-_wDDc+gCUYeh_e$%M|1t%gKTh}hDZlMiuEDt{Wi%UpE$jFfBy7Y=8 zdw!h}-$O}Cn*45$aP-)3H^)EuKo>^7V4R-(qG{#C<(uBMr)^F6*g5R1XQ{d~u&3mR zpZars`0lR3H-~-8Xg{y~X7cI9uQhcqnkpH==M}kYblVPlWz@||y;Cw}>mB-0#j1h@ z_gbTkzucR?V5{^{$G*>E=VB?V``qz}b7g;yzpokh+eN2sA7&WNe|Jl4y7bb;{jb(h zgQ{&#DUN?Ue(k4|0fLL)yswXU^*VH>SK}_=qwl6AHLVIUjI2Dl+{~Ktri|!Pr8F7M)K5Ky*So5V z_FnsWt;T<9Vh8mgdrj%m@z;8=u^nIPt*6hV+}Oi89AiB5lyPL`x!r4uPQ{C!2tB4R zc=N<&XBWr5b!>RDs7}L#q6BNyF0xBor z>X-Sm_HW;Azi^b%Gg58WT=z~x^PhCGejT^e*4UwGo?%5>uwJRS)SP_(^tyvvwtW1M zVU}LF!sap|U^W>qPj4zZ6ZJ5!DP#7?-@V@~d`QjBPd%93)*PAP@N=l|*PM<^#uZ5k zCeLE@LUeB%ovbhCCPkEd49&SFb@y5MKJL~_m-3L0sSBc-Uy4df%H!N)X@zaaPZl2Q zOgg=C^E|f828&gdmFKOSrFfUWZ};Lq3qINw--vOYIj837l*_lqiiYibymhRq>)aad zUOyi_AMc(F(-th}@ZAR$tLIynma`Ad92yv^~vUTs2KgaB! zzCif8qUrD4F6U!@E5<$)1$pehoTDfk<$5D=;xFGanX$Xoit^A&QD?u{fBrV*(^z#D zTm9_opX+9C&&dAcmAiOavY(gTvI_a2WqJjZSJWvU(RR)MtD*Dp>ZAL{2qxLyh`q8e zvu)K`_G7Bwo;S(gvexHs|8^;G4m0O#!@VkZI?LI5*%$KxM{-V{=^B2p*t)dMIq9#8 z#%Wh?omE!EujLEulOq18?S7GU_!PH5V(m0%M##n;Y$o10wkV}VM@2_sih|+T_A}HPZhG|ai z6USVO?2K>88gk9+RcxIVcf+6M^NTNYVjr*PPN$Zv{$o4gH8wnb454?(#ltbFxY#2r zcZo@(?(q@9MJG@5+Es<4B}cC|aN|8%lCIRmTODY%82n(!hOUuw5>8gs5d1q=onOe* zqc=n=FPy6z9)G9ns2h!SU7>JqIXq}I8_&;nxnsOEc;@`KjfT?})z(XL^cSw`S+{j3 zRs7txX;8$Wdd^MbSgS|#$%4e)&b+DK*Yy%Bmvmaa&D+J;8{Fvd!s^z73!Y7(dxoSD zY5L-tCt)LRwzurCStZ|2c5!aJ`8O>+wx}kpsOItWEsLuzo~ei}2*gTUI%aS?W>Bsi zeaqj|edoMaL|Ao4fK3A5vbLn9!8T`bx<7kSo2x-K|E~teA&7(@yrjJTDt)xrK z!!x#Q%X2%Id}_;)l;VW5Zd=1+cCB+xzI3~VqJPLP@znhjh8Ije3?ciK-4*W2c@rJ; z=9A<2%FD%Fmv5)O*zoL2Jum;!^N%mD*57q2II==1UOm*u_g+jS?#`-iYgv&yuc=P- zN8Q}E*^ZJ6TkhNXSn?)SF(*~kyqYU2AiY?7t(6+BG3wbq(b?Q$wC3i7v%04)XKoK_ zjjCPZyKUt(GhuwdwSx&+oL6;6cis2y+JAd#&-Iv%-5)m^lu^y^&X)~fd?+sZbSX~R zeyw(qp3KNk_GQC7mop8&1k^=ZjSF|PJKrXaEtq)jk;vojk;kP&olXo%4 z;pX_9$K|TU($jHD_ce}=mM`@zwla^Fl61n$B{zw*(@Reed*b))Jnq)g@$-|{ZZFr_ zUtRvxjdEq?J^U^ACVz$5+rWm1zx>!2Ptp{Pn9Hkgd6{qMMxuR**}u#6KG<29Z`*UV z>q@xh!qWQYa;M25YJ;>*DW<&Kr&od|yZ-r`!8304p~{r`CZE5qF6aqwbEhsGYqoFv z=H=JZ$6YwXEL=wJ_IhN!REp(HIEYnx+_$RD6g|o~G38`y=GuL$)`U+E-ZSHgbh}}t zA$D>@t?ASBcja}Bi7hR;lOG1nni8w5j(P0u`?S=9+W&Wq5)=#D zGY)U6=~;MAxvs1|VPAcspYFO5GGd~QF#FI2@92$Hw*q{-YNxu>Iw!Q^L+T^hoN#wmxIuiQHK?A*lSGVkRno;z0`s@(CL)N_Zo|8fn@d>prJ z=i|VvpFVj+zq;nWWv6@UIQDJptA|FhZx<7lKMhFR__u9}Y4f4^2Q~@Ww|7lep3Iz= z+%CMXYJL46-9XPTxc2<>2g+abM7x4-JsdIOMSkV)_g*&-UsIG`{?4j4u}U;&VRh*F zd&$oS?wIVeakI%-r-4G+Jz#)~{=?+ah;*`Y4HX>q`*xr{oc+=Taj!GX_>5us0#{C<0FQo7^R^sgpgJ*me` zsV)zxXX7{qlO`N}>vhkBTjS`#@ZB5l9`}12Zc(&z?AtvK+i5~^$^K@=U&HSGeTnv7 z^yK-I?Dwj!YnyXNJiqzZ6Z_qJj+Sn$89Mjp=x?{`yAGH(XD;g+)V$uRC4P_3f1F3**n}Z)wYh z>=bmbvacTba_D=!c5bk=@WV>IfTvuMTfuhWvFeLugQMLF3O}4tR)3u+89ys72-{-JFvk8PapKO`cX zT5D|IT5ZSI^*n?vc}}`IT@?G0Jp3g&M%nc#VSFGr{j6i<iTa3_A)u^U+1!~s1=M-{(ZUXfdLtcm;AJL3_zMtxoQh~}>;-)i@e`-qjP4mlv7 z#*DgQ^+&(*p|iqh|gync|sPwxd|=FDKf8)q0Q zfyc7f?>=rbZq+yFjyxH3R==Ap27_u8_zU%w>*9$EZkA% z7rk$CUiRRJ+kU}hag?i@f?8MY2%KHL84J18G9Nc^$R^J`(;X4kri^!piTEP= +% {ok, state()} | {{error, Reason :: term()}, state()} +start(Partition, _Config) -> + gen_server:start_link(?MODULE, [Partition], []). + +% @spec reset() -> ok | {error, timeout} +reset() -> + Pids = lists:foldl(fun(Item, Acc) -> + case lists:prefix("test_backend", atom_to_list(Item)) of + true -> [whereis(Item)|Acc]; + _ -> Acc + end + end, [], registered()), + [gen_server:cast(Pid,{reset, self()})|| Pid <- Pids], + receive_reset(Pids). + +receive_reset([]) -> ok; +receive_reset(Pids) -> + receive + {reset, Pid} -> + receive_reset(lists:delete(Pid, Pids)) + after 1000 -> + {error, timeout} + end. + +%% @private +init([Partition]) -> + PName = list_to_atom("test_backend" ++ integer_to_list(Partition)), + P = list_to_atom(integer_to_list(Partition)), + register(PName, self()), + {ok, #state{t=ets:new(P,[]), p=P}}. + +%% @private +handle_cast({reset,From}, State) -> + ets:delete_all_objects(State#state.t), + From ! {reset, self()}, + {noreply, State}; +handle_cast(_, State) -> {noreply, State}. + +%% @private +handle_call(stop,_From,State) -> {reply, srv_stop(State), State}; +handle_call({get,BKey},_From,State) -> {reply, srv_get(State,BKey), State}; +handle_call({put,BKey,Val},_From,State) -> + {reply, srv_put(State,BKey,Val),State}; +handle_call({delete,BKey},_From,State) -> {reply, srv_delete(State,BKey),State}; +handle_call(list,_From,State) -> {reply, srv_list(State), State}; +handle_call({list_bucket,Bucket},_From,State) -> + {reply, srv_list_bucket(State, Bucket), State}; +handle_call(is_empty, _From, State) -> + {reply, ets:info(State#state.t, size) =:= 0, State}; +handle_call(drop, _From, State) -> + ets:delete(State#state.t), + {reply, ok, State}; +handle_call({fold, Fun0, Acc}, _From, State) -> + Fun = fun({{B,K}, V}, AccIn) -> Fun0({B,K}, V, AccIn) end, + Reply = ets:foldl(Fun, Acc, State#state.t), + {reply, Reply, State}. + +% @spec stop(state()) -> ok | {error, Reason :: term()} +stop(SrvRef) -> gen_server:call(SrvRef,stop). +srv_stop(State) -> + true = ets:delete(State#state.t), + ok. + +% get(state(), riak_object:bkey()) -> +% {ok, Val :: binary()} | {error, Reason :: term()} +% key must be 160b +get(SrvRef, BKey) -> gen_server:call(SrvRef,{get,BKey}). +srv_get(State, BKey) -> + case ets:lookup(State#state.t,BKey) of + [] -> {error, notfound}; + [{BKey,Val}] -> {ok, Val}; + Err -> {error, Err} + end. + +% put(state(), riak_object:bkey(), Val :: binary()) -> +% ok | {error, Reason :: term()} +% key must be 160b +put(SrvRef, BKey, Val) -> gen_server:call(SrvRef,{put,BKey,Val}). +srv_put(State,BKey,Val) -> + true = ets:insert(State#state.t, {BKey,Val}), + ok. + +% delete(state(), riak_object:bkey()) -> +% ok | {error, Reason :: term()} +% key must be 160b +delete(SrvRef, BKey) -> gen_server:call(SrvRef,{delete,BKey}). +srv_delete(State, BKey) -> + true = ets:delete(State#state.t, BKey), + ok. + +% list(state()) -> [riak_object:bkey()] +list(SrvRef) -> gen_server:call(SrvRef,list). +srv_list(State) -> + MList = ets:match(State#state.t,{'$1','_'}), + list(MList,[]). +list([],Acc) -> Acc; +list([[K]|Rest],Acc) -> list(Rest,[K|Acc]). + +% list_bucket(term(), Bucket :: riak_object:bucket()) -> [Key :: binary()] +list_bucket(SrvRef, Bucket) -> + gen_server:call(SrvRef,{list_bucket, Bucket}). +srv_list_bucket(State, {filter, Bucket, Fun}) -> + MList = lists:filter(Fun, ets:match(State#state.t,{{Bucket,'$1'},'_'})), + list(MList,[]); +srv_list_bucket(State, Bucket) -> + case Bucket of + '_' -> MatchSpec = {{'$1','_'},'_'}; + _ -> MatchSpec = {{Bucket,'$1'},'_'} + end, + MList = ets:match(State#state.t,MatchSpec), + list(MList,[]). + +is_empty(SrvRef) -> gen_server:call(SrvRef, is_empty). + +drop(SrvRef) -> gen_server:call(SrvRef, drop). + +fold(SrvRef, Fun, Acc0) -> gen_server:call(SrvRef, {fold, Fun, Acc0}, infinity). + +%% Ignore callbacks for other backends so multi backend works +callback(_State, _Ref, _Msg) -> + ok. + +%% @private +handle_info(_Msg, State) -> {noreply, State}. + +%% @private +terminate(_Reason, _State) -> ok. + +%% @private +code_change(_OldVsn, State, _Extra) -> {ok, State}. + +%% +%% Test +%% +-ifdef(TEST). + +% @private +simple_test() -> + riak_kv_backend:standard_test(?MODULE, []). + +-ifdef(EQC). +%% @private +eqc_test() -> + ?assertEqual(true, backend_eqc:test(?MODULE, true)). + +-endif. % EQC +-endif. % TEST diff --git a/erl_src/riak_search_test_backend.beam b/erl_src/riak_search_test_backend.beam new file mode 100644 index 0000000000000000000000000000000000000000..ddef8d248ea4bc6c7e069a9c7442e48dcf4a9273 GIT binary patch literal 8484 zcmb_gdpuMB|KFUk8LKfR_c4GXwmt4|CQo2a@P%7!8 zn=Vq4lyv>Lm#%crr+)7v`t`m1{`;NBwaGEbLKlQ)E+ z3-~fM0bdc5m^MyRl8UDMD;E~52qiormoL}+Q#@BHi4zKxxniJAS*~Eq6e@g~m@N@Z z6-pF*AT~xJjS&mw3U&Agp0I(X5MLpOB>WV3hk}oiE2K%vKsrBF8Rn9s6Zo-$u1lghBaW-)Qe5*4vjDoRd52_{RR>VhcmVjy1_%V0j>Icznjnxwbp!&9K)BE# zR~kfQKnMZ@!jr6MgityOLgVo$JkB4DAksr2O#}f+_JfdgCCxoVWUx9OPlr%wJQGU*hEPS6q{=qN%q9*-u*;*$`nNkAYFJsk)pLoBQ|i@-v` zvUdYIWYwjQ+R?8#{1HNvn(x^DGMfQpft4TR0vHhbjE2y zUJ!xgf+G-I0pMU#Kroq99D!toBT!7uaT?xCICsGWBu0a7N&p)%#NEM2j3(fLCRYdn z5E7#WJT?F;(_C=VnXUk21``p+U=q1jxanaaIl)O9C}1I-Yf2|k=@jotQ~$6V1>}z# z{SFvS0w#k(U||h?$rKPCtl{_&j*}9LMf4#l5$L8+HkpVfp1{ZARg(z5WD?jxU_s++ zT2M9-fhJy{V@<}34>Nr-IT&qVD=7nsnV}w=L5ITOpi)qD`M7OiR660eZNCHTg45A* z1t2rDK#6M+L40%wSWv=R^l*qy3ZHbw8-#d#Hy)!0e4wzrVGm4MSTkQT6IhO&IpGi1 z942e|P?RXX=BdQ&BF1Fu!!o%62jF%Mb+k|6g6OJ+A?9hQ<cH_W{?ml1_`f4hd`1bI%Fy!g2z$=g(kkE;*217 z91)2zf^CKC8-qt;h`{P>A`y)T)r%s~fnF_0nT#_8e8yHlD3&}IkI_V8On^2AppB$X zCeumdWeS9oU2wBdt^i~P3S=DxXMGmPI-X31yNMPHOVQ9mlQ9GroH^tQ@Vf!J;7ozh zGf6~XF$658&Gj$HY;G8I)Y5cIuhXSOT=iYDPyJ~pAn#v?$mXH4{aj6FO6Z5HDIw+PB_3#dS)2T)0);T$2b?C6A}g1(8- zL1LW15*>lWIMX#@iY^l40#g99E36(6i5NX3#tje|WF%&`3(gG~?KW;SXsUS7REc4L zNC89&ok0PWPQl9=6k;ZWLM{Odpx3z(h@jW)Wf1UnE;x@mK&T@!>JXP1b@&b!ocm>0 z03zcuqL*^i4@S5zzt&UWl?~??&`76=^Y?T6#8Bg{W3(){GP z|EtVUzC;q}-k`q`SmB_fP<;qW1Xv~s(11FK1(%U#ED-Er1$@C$z}^%8r#(vlX^$s} zCKEgqbQhc_h^8k5&h^RjhXS|9@$wBM!iPA<3ov^FX2$qQ#RM}SF!Of7F~L?Q6s!LG zko)uK3I_+>|MT(XJ8^umKuTHg@kOLiaV(NK&c~Yq7aklCAN+0v94dR3_js=*GFir~ za33axG;y4fG5WDcj4#*-;pqrC#HMhN$4@$p0a#+erRxX0^aC~M?~Dt8z`KhVH2LxZ z%H0qYzA|{x0qX%&oF7Dh5Qs2^EF8QiA^a1Qz#W+ihDHEj1%U#9FBJgr?u0-BU;tDB z;7bhvyjG3V;W2JvJO$jR0w@6x0JJ8?FwZmq_=3l}0I*D$UmpPGhsQJi7{j)}bRvN9 z#2DtW0Dx_TD*$}K z^4$Rbv=f#I^LqkJj=lbEJjw41=##owU_43phxvU{;rJtt0Khuw0I)n*H#~+f=Lx`c z8vq&rJhlXY-+f`(q=`8^HUxnA;QZ+Uz-w^+;P}G6Dg(gzn2a~Gk;SbR>W|ZM&K|UQZ!1Gp>66FAUO0a)6Jxkm;T*wnn&PEU$iO`$Qw&q$m3RnfnOp)-kPz$T1aB$p=3xO@>}u3ZH9E|lAW|12bS-uz{Ju`~((UbKV1uI+gIB>DJg z;y(>Lxs3bg{9pd#umyiVfPi>$mDJc(Q|eq}zs)r7np@qcBLcyehmoGatr}TFM-~Pgx$f;{zE1g4pL5RR zU8$WEx9Er`6wz48>GZ!29ZO06oF`a(^zqXRQRLPai)L(VzO&78 zGhK;05}K89NL4W4evK9rm}{*_-Ek+-xq&(OU1enD<(5;fMa=GvRl}B>8Y~COaZ4jD zhv%;*6iFAI-BMPNs;42%Jfv1nq3GRLES+Nf27g2~jux=yA6i{oCH1?$*CkTyF5$D| zS$Qx0%BPjToDpgOm3VR19J~_sSNP}F>Q~7Zk8_5;RD>=YOVF|{to1K!z881+WnRh| z%Is$wYi*C`j@oVKx#IM&mG^cG?4|f6`)Um2S+=Il407V#*b}z&(th>y)fq=TgJ0b;qSbxgkUbV$ z^phNc9lHc^XtNhH=1D*!M$~PWuy6tsvo;Qp7OYK$|Lk%@20YYDLO@4 zecq~$VRs1Uw|vJxL@TZH&~rN6y8xf&qlmu~ie8zwdy{d+0G~?OyYkZF81YGE&uy7I z{Tn*J6^WWSv>(;mJC7NUl*OGA-8$-+aUI#50Npw|pUyMxNZ4gE)M0%%pL{quVyq|X zE;H0YfpSq|3%GAh>SDe7S25q`R39ohHLEQC@qX%d>2s{p(E7D|B_=PcKIm`AGvQG! zzmRi^3tpm57e3!|KSlcD%cJb3#QtS&6>7&EHYZDl6KpMf((E^4-p76UC}t^41O00NINg3L@*;nJJ3; zl%k$vU12s6Va3CSYu{X~J0y%OaN_P0^en*q*qD)97q?qYOzg_oylzg&6IF5Ut*bSa z9dnHn0`6tG9B7HF9SZW^9&0L66~EAW$jc1xNYoAKm@kPTRX}@D2Le17-c`jsGMPz_ z-F7jk#iM8_(QES+;njh=qkTsccSb!6t$sHDlj{xSq2$9jsgwS02J4l{X-55qfs<;S z(Y8vtL&Jl$em-tZuw~idw;I_W76H18j?!#w%<;#cJE` zp4&Ut%&32|ezz63#Laru+OlazRFBqKKc`0qzIvBO_fAl5q!Ogt%;E;~40KEPWAIB8 zlXFkFZGE*V_|u996{j!j?r_E%UBvCCjJ>FhRu|POn?0VjYU^+Yn)wV*m{XLlx7Y?b z-P_)|28&qad$Wd3njTH5M}K!H{8EiFm`3bdz4>LbmNwL7C~^EGMsLG*Zx1^uidw%n zN02QlXs3O7*pI;V?Ri^TuJft!Jmw;P^U^n|S{T0B-KQ51YuMaKG^N!n8m$;_c!aW;>>bzm=y$wJU?->BV-RbUjPn9NHGx zQLpg;eY;1iHu%cWMKzB(1%`%?5l`&HXY86`V7O)*(fBd*^%LgnowfFFytb!P(?9y9 z*I4Xv&w0~9b(u?A?7Z@HZuQZcgSR#B=?+y_mz=phG>zS+`AcExYbUw?Bs-jRx95lh;*xR->W(zYaN|iUaVlLBy0%xZe@QSP52D1~} znw?0ho8-jWBKK~m6POy6IjZ+xr2HMca7*dSsNxf8ON(c1AEepezJ}0AJsLdq_GRpX zKI~xF{M}Q1`mX!*<>wA@6g$-v2C9w3`d41j$E~yvn%kOO^yws!YYr?qMRdbeyH0zg zcJ{Sxg5lXN3}5&)@XE;6eV+ZN+RO<5q+4{a6=zc0{5G^xwiYihpD|Rs;t}4x>B#nd zo66Qb`#Y)RgU{l(Wd=z;8u;C{`* z1$$r1I-^7FkE(a*d{o-;_-z!wt@lxg*7Jlo?hA8i?#WBtZHZ8qq|*2C!t2&Y$I?P| zkNe-D3GaWp_kE`7MrX70s>uyA(+lsVW;B$=%s#l1Pdt2Fo%-4eYMh0=TJ&wvz4x6d z)cPMM5T|$XF2oh*qj!2!I$|wmT@3Y8-Lh8GT6yW0{66E0d!p(sE{0xL-{MsjrB6N* zJf)>muSfl=Q>p5(*ZmxHq`4t)XKG<`&&4ygg=~)FDLv9>Em_X4xyy$gPMQ8qoQpxNfv0JOC??fnx{g|ES z>r}`)$)K^t?A8 z`-6|EIu@;- zKeTG`KzK`XO_cRs(zAYp$a|UTR~CQfxyQSD9kOp)Jzw0SNc?Sb^$9qVdIRTsK3%zoeT^iv|ep1I;k#N@wmjABhrZz>8Mbi zaC?4i+ON~PY)a}D*9Ve3+EbBVH{%1&yYfwYB+VzWV$X+{4W?dZv$D9o@lBd#d-t)T zb6TQjstZhcDl?|w*Y~Yb_zIjeElY;Ve99t=Zbis1hJ7j_|8&Jss|}W)xRnsIoE`2G zs$y<7K#pr^eXL2|aQAk5)V9l=gZGLZ%{`KG0~Q_0o8Nk(w~en|=W>4ASyN`zsb%}; z6>OAEbu{cCs3jjxMJ%CF^Skj(~%M}+R~ zgM%U6X=S6MXTCLp-^Kl7jZw)SZi9m#%VUV`r*>DRenZ{(s@!*ex8NNk;I?#5r6jg~ zU$7HdE#%lig1|+u);Q8zWUU#k-PyM|yjU7y;hVeE-X!mZF7@k{_z@co|E3#r|N3sR zOHYKp@_j{&Jf^Mud%{NA=5uXM!HD4o1G|nu^OTR@4a;qYi@~xH-a|yDH+;;}3@tB0sh{EMRZnxn@h})+;%iQ5oyMT9ps2J6XCa^PP=n?W)5D zkv|XaYHI4W|6#DOvL>VI0+cX-zW(vze7AVSt3by#fA6KIjE34rGXfa055^-atD7x0 z-PqvzV8imAdD}HRzhE1kJ$TJEStAj>rHj6%+jf0!hDL^fgthI z0@HZqA2WlzD~?sCl^qB@Qhi$2Dq%rn&Q_CGH)z+e=LQzCALv`P+ZgK=)V3!@Mpu=3 zylsfwuv;{Aywc$Y=S8iNRnL~~>!O(z+D9X8OG59&e%eM!FSqUCMtJYc%h@(^=7MkU zhs8P9Ecq2|bNCp#Zq~s)W`WC&h!~5Y5$C7TkJM%?%^eVRQzq-1(_8lQ=+iibZb^W}vhV!O{ z+|Ag&*}5^*P`fKT>p@tlL)!_0nA+moM|%-wR&_so`-6Mi7WLYny2?q&ztIT)Uc0AD z5Y5RyNCS_fSyfL$ldq@xJPWAJxkT_{&DWsLNOSJ0fV8Ms(!c@1Zx^z6Rg~&|U*~Ii zgm7#)sv?;ju!nEAGmEfB(7-!gGxyvp*Rtg3Co?ddcD-oZpRS*d2ZfBZvWHvQ-(HAz z-q^A9+oE}MW9|_>{aTEwxAqs5xo@(mOtjy1`2A@4;nCUqWwQllfe)`X7M{?K>>TpJ zwrN{*4k>@MY<_hq-)1_cp4@4^LMOeN|}aZ zI*Z>|o~!y{H-E(FMA^%xq|VhL5nEgS**HAb{b95zhI0Ph4#rlK+tTs}(ic65B!bza zeR0pc?VUbyUsQc*6P$C{sUJw|khbi1s+-evaP`%8Z5 z?9_Ds2g`F%cAck|)TY$gG}PPeueYl^wK9h`_x3+5&*wiXvwvs{(N wXZ?a%m#spq6D*7z163APEq%GNVAqO(omCh%Avyk$-S(eWKaFAocRL{d2a_KoWB>pF literal 0 HcmV?d00001 diff --git a/erl_src/riak_search_test_backend.erl b/erl_src/riak_search_test_backend.erl new file mode 100644 index 00000000..79b9da99 --- /dev/null +++ b/erl_src/riak_search_test_backend.erl @@ -0,0 +1,175 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. +%% +%% ------------------------------------------------------------------- + +-module(riak_search_test_backend). +-behavior(riak_search_backend). + +-export([ + reset/0, + start/2, + stop/1, + index/2, + delete/2, + stream/6, + range/8, + info/5, + fold/3, + is_empty/1, + drop/1 + ]). +-export([ + stream_results/3 + ]). + +-include("riak_search.hrl"). + +-record(state, {partition, table}). + +reset() -> + {ok, Ring} = riak_core_ring_manager:get_my_ring(), + [ ets:delete_all_objects(list_to_atom(integer_to_list(P))) || + P <- riak_core_ring:my_indices(Ring) ], + riak_search_config:clear(), + ok. + +start(Partition, _Config) -> + Table = ets:new(list_to_atom(integer_to_list(Partition)), + [named_table, public, ordered_set]), + {ok, #state{partition=Partition, table=Table}}. + +stop(State) -> + maybe_delete(State). + +index(IFTVPKList, #state{table=Table}=State) -> + lists:foreach( + fun({I, F, T, V, P, K}) -> + Key = {b(I), b(F), b(T), b(V)}, + case ets:lookup(Table, Key) of + [{_, _, ExistingKeyClock}] -> + if ExistingKeyClock > K -> + %% stored data is newer + ok; + true -> + %% stored data is older + ets:update_element(Table, Key, + [{2, P},{3, K}]) + end; + [] -> + ets:insert(Table, {Key, P, K}) + end + end, + IFTVPKList), + {reply, {indexed, node()}, State}. + +delete(IFTVKList, State) -> + Table = State#state.table, + lists:foreach(fun(IFTVK) -> delete_fun(IFTVK, Table) end, IFTVKList), + {reply, {deleted, node()}, State}. + +delete_fun({I, F, T, V, K}, Table) -> + Key = {b(I), b(F), b(T), b(V)}, + case ets:lookup(Table, Key) of + [{Key, _Props, ExistingKeyClock}] -> + if ExistingKeyClock > K -> + %% stored data is newer + ok; + true -> + %% stored data is older + ets:delete(Table, Key) + end; + [] -> + ok + end; +delete_fun({I, F, T, V, _P, K}, Table) -> + %% copied idea from merge_index_backend + %% other operations include Props, though delete shouldn't + delete_fun({I, F, T, V, K}, Table). + +info(Index, Field, Term, Sender, State) -> + Count = ets:select_count(State#state.table, + [{{{b(Index), b(Field), b(Term), '_'}, + '_', '_'}, + [],[true]}]), + riak_search_backend:info_response(Sender, [{Term, node(), Count}]), + noreply. + +-define(STREAM_SIZE, 100). + +range(Index, Field, StartTerm, EndTerm, _Size, FilterFun, Sender, State) -> + ST = b(StartTerm), + ET = b(EndTerm), + spawn(riak_search_ets_backend, stream_results, + [Sender, + FilterFun, + ets:select(State#state.table, + [{{{b(Index), b(Field), '$1', '$2'}, '$3', '_'}, + [{'>=', '$1', ST}, {'=<', '$1', ET}], + [{{'$2', '$3'}}]}], + ?STREAM_SIZE)]), + noreply. + +stream(Index, Field, Term, FilterFun, Sender, State) -> + spawn(riak_search_ets_backend, stream_results, + [Sender, + FilterFun, + ets:select(State#state.table, + [{{{b(Index), b(Field), b(Term), '$1'}, '$2', '_'}, + [], [{{'$1', '$2'}}]}], + ?STREAM_SIZE)]), + noreply. + +stream_results(Sender, FilterFun, {Results0, Continuation}) -> + case lists:filter(fun({V,P}) -> FilterFun(V, P) end, Results0) of + [] -> + ok; + Results -> + riak_search_backend:response_results(Sender, Results) + end, + stream_results(Sender, FilterFun, ets:select(Continuation)); +stream_results(Sender, _, '$end_of_table') -> + riak_search_backend:response_done(Sender). + +fold(FoldFun, Acc, State) -> + Fun = fun({{I,F,T,V},P,K}, {OuterAcc, {{I,{F,T}},InnerAcc}}) -> + %% same IFT, just accumulate doc/props/clock + {OuterAcc, {{I,{F,T}},[{V,P,K}|InnerAcc]}}; + ({{I,F,T,V},P,K}, {OuterAcc, {FoldKey, VPKList}}) -> + %% finished a string of IFT, send it off + %% (sorted order is assumed) + NewOuterAcc = FoldFun(FoldKey, VPKList, OuterAcc), + {NewOuterAcc, {{I,{F,T}},[{V,P,K}]}}; + ({{I,F,T,V},P,K}, {OuterAcc, undefined}) -> + %% first round through the fold - just start building + {OuterAcc, {{I,{F,T}},[{V,P,K}]}} + end, + {OuterAcc0, Final} = ets:foldl(Fun, {Acc, undefined}, State#state.table), + OuterAcc = case Final of + {FoldKey, VPKList} -> + %% one last IFT to send off + FoldFun(FoldKey, VPKList, OuterAcc0); + undefined -> + %% this partition was empty + OuterAcc0 + end, + {reply, OuterAcc, State}. + +is_empty(State) -> + 0 == ets:info(State#state.table, size). + +drop(State) -> + maybe_delete(State). + +maybe_delete(State) -> + case lists:member(State#state.table, ets:all()) of + true -> + ets:delete(State#state.table), + ok; + false -> + ok + end. + +b(Binary) when is_binary(Binary) -> Binary; +b(List) when is_list(List) -> iolist_to_binary(List). From ac046c124e173421deceaf9c2cdc5615e58cd0d7 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 7 Jul 2011 12:31:39 +0200 Subject: [PATCH 0041/1060] Fix bug with too many link headers. Having too many link headers would explode a single Link header to more than 8192 bytes, which is the boundary mochiweb uses. The HTTP transport will now split it up into multiple header, therefore had to introduce a dictionary that allows multiple values for a single keys, while still maintaining a dict like interface. --- riak/multidict.py | 201 ++++++++++++++++++++++++++++++++++++++++ riak/tests/test_all.py | 13 +++ riak/transports/http.py | 36 +++++-- 3 files changed, 240 insertions(+), 10 deletions(-) create mode 100644 riak/multidict.py diff --git a/riak/multidict.py b/riak/multidict.py new file mode 100644 index 00000000..5336803e --- /dev/null +++ b/riak/multidict.py @@ -0,0 +1,201 @@ +# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) +# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php +from UserDict import DictMixin + +class MultiDict(DictMixin): + + """ + An ordered dictionary that can have multiple values for each key. + Adds the methods getall, getone, mixed, and add to the normal + dictionary interface. + """ + + def __init__(self, *args, **kw): + if len(args) > 1: + raise TypeError( + "MultiDict can only be called with one positional argument") + if args: + if hasattr(args[0], 'iteritems'): + items = list(args[0].iteritems()) + elif hasattr(args[0], 'items'): + items = args[0].items() + else: + items = list(args[0]) + self._items = items + else: + self._items = [] + self._items.extend(kw.iteritems()) + + def __getitem__(self, key): + for k, v in self._items: + if k == key: + return v + raise KeyError(repr(key)) + + def __setitem__(self, key, value): + try: + del self[key] + except KeyError: + pass + self._items.append((key, value)) + + def add(self, key, value): + """ + Add the key and value, not overwriting any previous value. + """ + self._items.append((key, value)) + + def getall(self, key): + """ + Return a list of all values matching the key (may be an empty list) + """ + result = [] + for k, v in self._items: + if key == k: + result.append(v) + return result + + def getone(self, key): + """ + Get one value matching the key, raising a KeyError if multiple + values were found. + """ + v = self.getall(key) + if not v: + raise KeyError('Key not found: %r' % key) + if len(v) > 1: + raise KeyError('Multiple values match %r: %r' % (key, v)) + return v[0] + + def mixed(self): + """ + Returns a dictionary where the values are either single + values, or a list of values when a key/value appears more than + once in this dictionary. This is similar to the kind of + dictionary often used to represent the variables in a web + request. + """ + result = {} + multi = {} + for key, value in self._items: + if key in result: + # We do this to not clobber any lists that are + # *actual* values in this dictionary: + if key in multi: + result[key].append(value) + else: + result[key] = [result[key], value] + multi[key] = None + else: + result[key] = value + return result + + def dict_of_lists(self): + """ + Returns a dictionary where each key is associated with a + list of values. + """ + result = {} + for key, value in self._items: + if key in result: + result[key].append(value) + else: + result[key] = [value] + return result + + def __delitem__(self, key): + items = self._items + found = False + for i in range(len(items)-1, -1, -1): + if items[i][0] == key: + del items[i] + found = True + if not found: + raise KeyError(repr(key)) + + def __contains__(self, key): + for k, v in self._items: + if k == key: + return True + return False + + has_key = __contains__ + + def clear(self): + self._items = [] + + def copy(self): + return MultiDict(self) + + def setdefault(self, key, default=None): + for k, v in self._items: + if key == k: + return v + self._items.append((key, default)) + return default + + def pop(self, key, *args): + if len(args) > 1: + raise TypeError, "pop expected at most 2 arguments, got "\ + + repr(1 + len(args)) + for i in range(len(self._items)): + if self._items[i][0] == key: + v = self._items[i][1] + del self._items[i] + return v + if args: + return args[0] + else: + raise KeyError(repr(key)) + + def popitem(self): + return self._items.pop() + + def update(self, other=None, **kwargs): + if other is None: + pass + elif hasattr(other, 'items'): + self._items.extend(other.items()) + elif hasattr(other, 'keys'): + for k in other.keys(): + self._items.append((k, other[k])) + else: + for k, v in other: + self._items.append((k, v)) + if kwargs: + self.update(kwargs) + + def __repr__(self): + items = ', '.join(['(%r, %r)' % v for v in self._items]) + return '%s([%s])' % (self.__class__.__name__, items) + + def __len__(self): + return len(self._items) + + ## + ## All the iteration: + ## + + def keys(self): + return [k for k, v in self._items] + + def iterkeys(self): + for k, v in self._items: + yield k + + __iter__ = iterkeys + + def items(self): + return self._items[:] + + def iteritems(self): + return iter(self._items) + + def values(self): + return [v for k, v in self._items] + + def itervalues(self): + for k, v in self._items: + yield v + + diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 4dc97e03..15ad7655 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -13,6 +13,7 @@ from riak import RiakPbcTransport, RiakPbcCachedTransport from riak import RiakHttpTransport, RiakHttpPoolTransport, RiakHttpReuseTransport from riak import RiakKeyFilter, key_filter +from riak.mapreduce import RiakLink HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) @@ -731,6 +732,18 @@ def test_generate_key(self): bucket.new(None, data={}).store() self.assertEqual(len(bucket.get_keys()), 1) + def test_too_many_link_headers_shouldnt_break_http(self): + bucket = self.client.bucket("bucket") + o = bucket.new("lots_of_links", "My god, it's full of links!") + for i in range(0, 400): + link = RiakLink("other", "key%d" % i, "next") + o.add_link(link) + + o.store() + stored_object = bucket.get("lots_of_links") + self.assertEqual(len(stored_object.get_links()), 400) + + class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): diff --git a/riak/transports/http.py b/riak/transports/http.py index 991ceaa9..680e203e 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -35,6 +35,9 @@ from riak.metadata import * from riak.mapreduce import RiakLink from riak import RiakError +from riak.multidict import MultiDict + +MAX_LINK_HEADER_SIZE = 8192 - 6 # substract length of "Link: " header string class RiakHttpTransport(RiakTransport) : """ @@ -102,21 +105,16 @@ def put(self, robj, w = None, dw = None, return_body = True): params=params) # Construct the headers... - headers = {'Accept' : 'text/plain, */*; q=0.5', - 'Content-Type' : robj.get_content_type(), - 'X-Riak-ClientId' : self._client_id} + headers = MultiDict({'Accept' : 'text/plain, */*; q=0.5', + 'Content-Type' : robj.get_content_type(), + 'X-Riak-ClientId' : self._client_id}) # Add the vclock if it exists... if robj.vclock() is not None: headers['X-Riak-Vclock'] = robj.vclock() # Create the header from metadata - links = robj.get_links() - if links: - headers['Link'] = '' - for link in links: - if headers['Link'] != '': headers['Link'] += ', ' - headers['Link'] += self.to_link_header(link) + links = self.add_links_for_riak_object(robj, headers) for key, value in robj.get_usermeta().iteritems(): headers['X-Riak-Meta-%s' % key] = value @@ -307,7 +305,7 @@ def to_link_header(self, link): header += urllib.quote_plus(link.get_tag()) + '"' return header - def parse_links(self, links, linkHeaders) : + def parse_links(self, links, linkHeaders): """ Private. @return self @@ -320,6 +318,24 @@ def parse_links(self, links, linkHeaders) : links.append(link) return self + def add_links_for_riak_object(self, robject, headers): + links = robject.get_links() + if links: + current_header = '' + for link in links: + header = self.to_link_header(link) + if len(current_header + header) > MAX_LINK_HEADER_SIZE: + headers.setdefault('Link', []).append(current_header) + current_header = '' + + if current_header != '': header = ', ' + header + current_header += header + + headers.add('Link', current_header) + + return headers + + #Utility functions used by Riak library. From d89682b0468948db31d13f0b1a664833e6a51044 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 7 Jul 2011 12:39:30 +0200 Subject: [PATCH 0042/1060] Adapt build_headers() for better MultiDict usage. --- riak/transports/http.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 680e203e..f31c60a1 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -465,10 +465,7 @@ def pycurl_request(cls, method, host, port, uri, headers, body=''): @classmethod def build_headers(cls, headers): - headers1 = [] - for key in headers.keys(): - headers1.append('%s: %s' % (key, headers[key])) - return headers1 + return ['%s: %s' % (header, value) for header, value in headers.iteritems()] @classmethod def parse_http_headers(cls, headers) : From ec47a7d531983fdef69f451b8ef4d07829bcb1cd Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 7 Jul 2011 15:46:57 +0200 Subject: [PATCH 0043/1060] Add option to run TestServer when running test suite. --- riak/tests/test_all.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 4dc97e03..537b0360 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -13,6 +13,7 @@ from riak import RiakPbcTransport, RiakPbcCachedTransport from riak import RiakHttpTransport, RiakHttpPoolTransport, RiakHttpReuseTransport from riak import RiakKeyFilter, key_filter +from riak.test_server import TestServer HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) @@ -20,6 +21,16 @@ HTTP_PORT = int(os.environ.get('RIAK_TEST_HTTP_PORT', '8098')) PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '0')) +USE_TEST_SERVER = int(os.environ.get('USE_TEST_SERVER', '0')) + +if USE_TEST_SERVER: + HTTP_PORT = 9000 + PB_PORT = 9002 + test_server = TestServer() + test_server.cleanup() + test_server.prepare() + test_server.start() + class NotJsonSerializable(object): From 146cbb86597569ae152da87e2fd7b87c32977c9e Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 7 Jul 2011 17:03:55 +0200 Subject: [PATCH 0044/1060] Stop using function names that enable name mangling. --- riak/test_server.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/riak/test_server.py b/riak/test_server.py index 29985c4f..bf5a0017 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -81,9 +81,9 @@ def prepare(self): if not self._prepared: self.create_temp_directories() self._riak_script = os.path.join(self._temp_bin, "riak") - self.__write_riak_script() - self.__write_vm_args() - self.__write_app_config() + self.write_riak_script() + self.write_vm_args() + self.write_app_config() self._prepared = True def create_temp_directories(self): @@ -158,7 +158,7 @@ def wait_for_erlang_prompt(self): if re.search(r"\(%s\)\d+>" % self.vm_args["-name"], buffer): prompted = True - def __write_riak_script(self): + def write_riak_script(self): with open(self._riak_script, "wb") as temp_bin_file, open(os.path.join(self.bin_dir, "riak"), "r") as riak_file: for line in riak_file.readlines(): @@ -175,12 +175,12 @@ def __write_riak_script(self): os.fchmod(temp_bin_file.fileno(), 0755) - def __write_vm_args(self): + def write_vm_args(self): with open(os.path.join(self._temp_etc, "vm.args"), 'wb') as vm_args: for arg, value in self.vm_args.items(): vm_args.write("%s %s\n" % (arg, value)) - def __write_app_config(self): + def write_app_config(self): with open(os.path.join(self._temp_etc, "app.config"), "wb") as app_config: app_config.write(erlang_config(self.app_config)) app_config.write(".") From da11cb145fb3f023756bfe114ec71b95b235dab8 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 7 Jul 2011 17:07:32 +0200 Subject: [PATCH 0045/1060] Respect newline in link header size. --- riak/transports/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index f31c60a1..7b4712cd 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -37,7 +37,7 @@ from riak import RiakError from riak.multidict import MultiDict -MAX_LINK_HEADER_SIZE = 8192 - 6 # substract length of "Link: " header string +MAX_LINK_HEADER_SIZE = 8192 - 8 # substract length of "Link: " header string and newline class RiakHttpTransport(RiakTransport) : """ From b3f5b8c5c415a628c7e570514e045d29a75252e5 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Fri, 8 Jul 2011 12:17:54 +0200 Subject: [PATCH 0046/1060] Start fleshing out test suite for TestServer. --- riak/test_server.py | 15 ++++++++--- riak/tests/test_server_test.py | 48 ++++++++++++++++++++++++++++++++++ riak/util.py | 33 +++++++++++++++++++++++ 3 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 riak/tests/test_server_test.py create mode 100644 riak/util.py diff --git a/riak/test_server.py b/riak/test_server.py index bf5a0017..1283da70 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -6,6 +6,7 @@ import shutil import time from subprocess import Popen, PIPE +from riak.util import deep_merge def erlang_config(hash, depth=1): def printable(item): @@ -67,14 +68,22 @@ class TestServer: } def __init__(self, tmp_dir="/tmp/riak/test_server", - bin_dir=os.path.expanduser("~/.riak/install/riak-0.14.2/bin")): + bin_dir=os.path.expanduser("~/.riak/install/riak-0.14.2/bin"), + vm_args=None, **options): self._lock = threading.Lock() self.temp_dir = "/tmp/riak/test_server" self.bin_dir = bin_dir self._prepared = False self._started = False - self.vm_args = self.__class__.VM_ARGS_DEFAULTS - self.app_config = self.__class__.APP_CONFIG_DEFAULTS + self.vm_args = self.VM_ARGS_DEFAULTS.copy() + if vm_args is not None: + self.vm_args = deep_merge(self.vm_args, vm_args) + + self.app_config = self.APP_CONFIG_DEFAULTS.copy() + for key, value in options.items(): + if key in self.app_config: + self.app_config[key] = deep_merge(self.app_config[key], value) + self.app_config["riak_core"]["ring_state_dir"] = os.path.join(self.temp_dir, "data", "ring") def prepare(self): diff --git a/riak/tests/test_server_test.py b/riak/tests/test_server_test.py new file mode 100644 index 00000000..6a2437b8 --- /dev/null +++ b/riak/tests/test_server_test.py @@ -0,0 +1,48 @@ +from riak.test_server import TestServer +import unittest + +class TestServerTestCase(unittest.TestCase): + def setUp(self): + self.test_server = TestServer() + + def tearDown(self): + pass + + def test_options_defaults(self): + self.assertEquals(self.test_server.app_config["riak_core"]["handoff_port"], 9001) + self.assertEquals(self.test_server.app_config["riak_kv"]["pb_ip"], "127.0.0.1") + + def test_merge_riak_core_options(self): + self.test_server = TestServer(riak_core={"handoff_port": 10000}) + self.assertEquals(self.test_server.app_config["riak_core"]["handoff_port"], 10000) + + def test_merge_luwak_options(self): + self.test_server = TestServer(luwak={"enabled": False}) + self.assertEquals(self.test_server.app_config["luwak"]["enabled"], False) + + def test_merge_riak_search_options(self): + self.test_server = TestServer(riak_search={"search_backend": "riak_search_backend"}) + self.assertEquals(self.test_server.app_config["riak_search"]["search_backend"], + "riak_search_backend") + + def test_merge_riak_kv_options(self): + self.test_server = TestServer(riak_kv={"pb_ip": "192.168.2.1"}) + self.assertEquals(self.test_server.app_config["riak_kv"]["pb_ip"], "192.168.2.1") + + def test_merge_vmargs(self): + self.test_server = TestServer(vm_args={"-P": 65000}) + self.assertEquals(self.test_server.vm_args["-P"], 65000) + + def test_set_ring_state_dir(self): + self.assertEquals(self.test_server.app_config["riak_core"]["ring_state_dir"], + "/tmp/riak/test_server/data/ring") + + def test_set_default_tmp_dir(self): + self.assertEquals(self.test_server.temp_dir, "/tmp/riak/test_server") + +def suite(): + suite = unittest.TestSuite() + suite.addTest(TestServerTestCase()) + return suite + + diff --git a/riak/util.py b/riak/util.py new file mode 100644 index 00000000..c2e154a0 --- /dev/null +++ b/riak/util.py @@ -0,0 +1,33 @@ +import collections + +def quacks_like_dict(object): + """Check if object is dict-like""" + return isinstance(object, collections.Mapping) + +def deep_merge(a, b): + """Merge two deep dicts non-destructively + + Uses a stack to avoid maximum recursion depth exceptions + + >>> a = {'a': 1, 'b': {1: 1, 2: 2}, 'd': 6} + >>> b = {'c': 3, 'b': {2: 7}, 'd': {'z': [1, 2, 3]}} + >>> c = merge(a, b) + >>> from pprint import pprint; pprint(c) + {'a': 1, 'b': {1: 1, 2: 7}, 'c': 3, 'd': {'z': [1, 2, 3]}} + """ + assert quacks_like_dict(a), quacks_like_dict(b) + dst = a.copy() + + stack = [(dst, b)] + while stack: + current_dst, current_src = stack.pop() + for key in current_src: + if key not in current_dst: + current_dst[key] = current_src[key] + else: + if quacks_like_dict(current_src[key]) and quacks_like_dict(current_dst[key]) : + stack.append((current_dst[key], current_src[key])) + else: + current_dst[key] = current_src[key] + return dst + From 2bc7f2b2a32570387ebb976f8320f2ddab003f64 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Fri, 8 Jul 2011 12:18:27 +0200 Subject: [PATCH 0047/1060] Add test suite for all tests. Easier to split out tests in the future this way. --- riak/tests/suite.py | 11 +++++++++++ setup.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 riak/tests/suite.py diff --git a/riak/tests/suite.py b/riak/tests/suite.py new file mode 100644 index 00000000..bddfd45b --- /dev/null +++ b/riak/tests/suite.py @@ -0,0 +1,11 @@ +import unittest +import riak.tests.test_server_test +import os.path + +def additional_tests(): + top_level = os.path.join(os.path.dirname(__file__), "../../") + start_dir = os.path.dirname(__file__) + suite = unittest.TestSuite() + suite.addTest(unittest.TestLoader().discover(start_dir, + top_level_dir=top_level)) + return suite diff --git a/setup.py b/setup.py index 3284f1a7..7286f091 100755 --- a/setup.py +++ b/setup.py @@ -31,6 +31,6 @@ def make_pb(): platforms='Platform Independent', author='Basho Technologies', author_email='riak@basho.com', - test_suite='riak.tests.test_all', + test_suite='riak.tests.suite', url='https://github.com/basho/riak-python-client' ) From f1eba0b1f7d51a6eb4637d1f919e9dc7498321c2 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Mon, 11 Jul 2011 22:38:15 +0200 Subject: [PATCH 0048/1060] Add support to enable/disable search commit hook on buckets. --- riak/bucket.py | 16 ++++++++++++++++ riak/tests/test_all.py | 16 ++++++++++++++++ riak/transports/pbc.py | 2 ++ 3 files changed, 34 insertions(+) diff --git a/riak/bucket.py b/riak/bucket.py index 58163af6..784bc49e 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -27,6 +27,8 @@ class RiakBucket(object): objects within the bucket. """ + SEARCH_PRECOMMIT_HOOK = {"mod": "riak_search_kv_hook", "fun": "precommit"} + def __init__(self, client, name): """ Returns a new ``RiakBucket`` instance. @@ -420,3 +422,17 @@ def new_binary_from_file(self, key, filename): if not mimetype: mimetype = 'application/octet-stream' return self.new_binary(key, binary_data, mimetype) + + def search_enabled(self): + return (self.SEARCH_PRECOMMIT_HOOK in (self.get_property("precommit") or [])) + + def enable_search(self): + if self.SEARCH_PRECOMMIT_HOOK not in (self.get_property("precommit") or []): + self.set_properties({"precommit": list(self.get_property("precommit") or []) + list([self.SEARCH_PRECOMMIT_HOOK])}) + return True + + def disable_search(self): + if self.SEARCH_PRECOMMIT_HOOK in (self.get_property("precommit") or []): + self.set_properties({"precommit": self.get_property("precommit").remove(self.SEARCH_PRECOMMIT_HOOK)}) + return True + diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 15ad7655..ee17dd12 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -743,6 +743,22 @@ def test_too_many_link_headers_shouldnt_break_http(self): stored_object = bucket.get("lots_of_links") self.assertEqual(len(stored_object.get_links()), 400) + def test_bucket_search_enabled(self): + bucket = self.client.bucket("unsearch_bucket") + self.assertFalse(bucket.search_enabled()) + + def test_enable_search_commit_hook(self): + bucket = self.client.bucket("search_bucket") + bucket.enable_search() + self.assertTrue(self.client.bucket("search_bucket").search_enabled()) + + def test_disable_search_commit_hook(self): + bucket = self.client.bucket("no_search_bucket") + bucket.enable_search() + self.assertTrue(self.client.bucket("no_search_bucket").search_enabled()) + bucket.disable_search() + self.assertFalse(self.client.bucket("no_search_bucket").search_enabled()) + class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index eec68084..abde6316 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -274,6 +274,8 @@ def set_bucket_props(self, bucket, props): """ req = riakclient_pb2.RpbSetBucketReq() req.bucket = bucket.get_name() + if not 'n_val' in props and not 'allow_mult' in props: return self + if 'n_val' in props: req.props.n_val = props['n_val'] if 'allow_mult' in props: From dc7a0d6e75b97e340d82209f63c38bdeaf90b0c3 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Mon, 11 Jul 2011 22:52:51 +0200 Subject: [PATCH 0049/1060] No explicit list conversion required here. --- riak/bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/bucket.py b/riak/bucket.py index 784bc49e..180923b7 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -428,7 +428,7 @@ def search_enabled(self): def enable_search(self): if self.SEARCH_PRECOMMIT_HOOK not in (self.get_property("precommit") or []): - self.set_properties({"precommit": list(self.get_property("precommit") or []) + list([self.SEARCH_PRECOMMIT_HOOK])}) + self.set_properties({"precommit": (self.get_property("precommit") or []) + [self.SEARCH_PRECOMMIT_HOOK]}) return True def disable_search(self): From 7740e887828a4fe3a0a453b5450dadb2d9827efa Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Tue, 12 Jul 2011 09:49:35 +0200 Subject: [PATCH 0050/1060] Clean up code, add comments for search indexing methods on buckets. --- riak/bucket.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 180923b7..7febead2 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -424,15 +424,30 @@ def new_binary_from_file(self, key, filename): return self.new_binary(key, binary_data, mimetype) def search_enabled(self): - return (self.SEARCH_PRECOMMIT_HOOK in (self.get_property("precommit") or [])) + """ + Returns True if the search precommit hook is enabled for this bucket. + """ + return self.SEARCH_PRECOMMIT_HOOK in (self.get_property("precommit") or []) def enable_search(self): - if self.SEARCH_PRECOMMIT_HOOK not in (self.get_property("precommit") or []): - self.set_properties({"precommit": (self.get_property("precommit") or []) + [self.SEARCH_PRECOMMIT_HOOK]}) + """ + Enable search for this bucket by installing the precommit hook to + index objects in it. + """ + precommit_hooks = self.get_property("precommit") or [] + if self.SEARCH_PRECOMMIT_HOOK not in precommit_hooks: + self.set_properties({"precommit": + precommit_hooks + [self.SEARCH_PRECOMMIT_HOOK]}) return True def disable_search(self): - if self.SEARCH_PRECOMMIT_HOOK in (self.get_property("precommit") or []): - self.set_properties({"precommit": self.get_property("precommit").remove(self.SEARCH_PRECOMMIT_HOOK)}) + """ + Disable search for this bucket by removing the precommit hook to + index objects in it. + """ + precommit_hooks = self.get_property("precommit") or [] + if self.SEARCH_PRECOMMIT_HOOK in precommit_hooks: + precommit_hooks.remove(self.SEARCH_PRECOMMIT_HOOK) + self.set_properties({"precommit": precommit_hooks}) return True From 91df36f2f58a4276c1c645eacbb3eb689fd7d824 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Tue, 12 Jul 2011 10:06:30 +0200 Subject: [PATCH 0051/1060] Update README. --- README.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.rst b/README.rst index b8578d18..5d038704 100644 --- a/README.rst +++ b/README.rst @@ -412,6 +412,16 @@ tutorial, but usage of this feature looks like:: # John Doe # Anna Body +You can enable and disable search for specific buckets through convenience +methods that install/remove the precommit hook + + bucket = client.bucket('search') + + if bucket.search_enabled(): + bucket.disable_search() + else: + bucket.enable_search() + .. _`Riak Search`: http://wiki.basho.com/Riak-Search.html .. _Lucene: http://lucene.apache.org/ From cfaa9f1b070255d3f28a08378556d66f2c82b4e6 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Tue, 12 Jul 2011 12:43:31 +0200 Subject: [PATCH 0052/1060] WIP: Add luwak support. --- riak/client.py | 3 +++ riak/tests/test_all.py | 9 +++++++++ riak/transports/http.py | 21 ++++++++++++++------- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/riak/client.py b/riak/client.py index 8f335fb2..58130765 100644 --- a/riak/client.py +++ b/riak/client.py @@ -284,3 +284,6 @@ def reduce(self, *args): """ mr = RiakMapReduce(self) return apply(mr.reduce, args) + + def store_file(self, filename, data, content_type="application/octet-stream"): + self._transport.store_file(filename, content_type=content_type, content=data) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 15ad7655..ceaa61c4 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -9,6 +9,8 @@ import os import random import unittest +import uuid + from riak import RiakClient from riak import RiakPbcTransport, RiakPbcCachedTransport from riak import RiakHttpTransport, RiakHttpPoolTransport, RiakHttpReuseTransport @@ -743,6 +745,13 @@ def test_too_many_link_headers_shouldnt_break_http(self): stored_object = bucket.get("lots_of_links") self.assertEqual(len(stored_object.get_links()), 400) + def test_store_file_with_luwak(self): + file = os.path.dirname(__file__) + "/test_all.py" + with open(file, "r") as input_file: + data = input_file.read() + + key = uuid.uuid1().hex + self.client.store_file(key, data) class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): diff --git a/riak/transports/http.py b/riak/transports/http.py index 7b4712cd..3a23ff7c 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -101,7 +101,7 @@ def put(self, robj, w = None, dw = None, return_body = True): """ # Construct the URL... params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw} - host, port, url = self.build_rest_path(robj.get_bucket(), robj.get_key(), + host, port, url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), params=params) # Construct the headers... @@ -120,9 +120,10 @@ def put(self, robj, w = None, dw = None, return_body = True): headers['X-Riak-Meta-%s' % key] = value content = robj.get_encoded_data() + return self.do_put(host, port, url, headers, content, return_body, key=robj.get_key()) - # Run the operation. - if robj.get_key() is None: + def do_put(self, host, port, url, headers, content, return_body=False, key=None): + if key is None: response = self.http_request('POST', host, port, url, headers, content) else: response = self.http_request('PUT', host, port, url, headers, content) @@ -191,7 +192,7 @@ def set_bucket_props(self, bucket, props): headers = {'Content-Type' : 'application/json'} content = json.dumps({'props' : props}) - #Run the request... + # Run the request... response = self.http_request('PUT', host, port, url, headers, content) # Handle the response... @@ -336,8 +337,14 @@ def add_links_for_riak_object(self, robject, headers): return headers + def store_file(self, key, content_type="application/octet-stream", content=None): + host, port, url = self.build_rest_path(prefix='luwak', key=key) + headers = {'Content-Type' : content_type, + 'X-Riak-ClientId' : self._client_id} + + return self.do_put(host, port, url, headers, content, key=key) - #Utility functions used by Riak library. + # Utility functions used by Riak library. @classmethod def get_value(cls, key, array, defaultValue) : @@ -346,14 +353,14 @@ def get_value(cls, key, array, defaultValue) : else: return defaultValue - def build_rest_path(self, bucket, key=None, params=None) : + def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : """ Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, construct and return a URL. """ # Build 'http://hostname:port/prefix/bucket' path = '' - path += '/' + self._prefix + path += '/' + (prefix or self._prefix) # Add '.../bucket' if bucket is not None: From ba882ae40da61621d437dfbaeb0701edb84b41b1 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Tue, 12 Jul 2011 16:09:52 +0200 Subject: [PATCH 0053/1060] Add support to get and delete luwak objects. --- riak/client.py | 9 +++++++++ riak/tests/test_all.py | 25 +++++++++++++++++++++++++ riak/transports/http.py | 14 ++++++++++++++ riak/transports/pbc.py | 2 +- riak/transports/transport.py | 8 ++++++++ 5 files changed, 57 insertions(+), 1 deletion(-) diff --git a/riak/client.py b/riak/client.py index 58130765..51f653b3 100644 --- a/riak/client.py +++ b/riak/client.py @@ -286,4 +286,13 @@ def reduce(self, *args): return apply(mr.reduce, args) def store_file(self, filename, data, content_type="application/octet-stream"): + """ + Store data in luwak using filename as the key + """ self._transport.store_file(filename, content_type=content_type, content=data) + + def get_file(self, filename): + return self._transport.get_file(filename) + + def delete_file(self, filename): + self._transport.delete_file(filename) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index ceaa61c4..d36bd98d 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -10,6 +10,7 @@ import random import unittest import uuid +import time from riak import RiakClient from riak import RiakPbcTransport, RiakPbcCachedTransport @@ -753,6 +754,30 @@ def test_store_file_with_luwak(self): key = uuid.uuid1().hex self.client.store_file(key, data) + def test_store_get_file_with_luwak(self): + file = os.path.dirname(__file__) + "/test_all.py" + with open(file, "r") as input_file: + data = input_file.read() + + key = uuid.uuid1().hex + self.client.store_file(key, data) + time.sleep(1) + file = self.client.get_file(key) + self.assertEquals(data, file) + + def test_delete_file_with_luwak(self): + file = os.path.dirname(__file__) + "/test_all.py" + with open(file, "r") as input_file: + data = input_file.read() + + key = uuid.uuid1().hex + self.client.store_file(key, data) + time.sleep(1) + self.client.delete_file(key) + time.sleep(1) + file = self.client.get_file(key) + self.assertIsNone(file) + class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): diff --git a/riak/transports/http.py b/riak/transports/http.py index 3a23ff7c..821b5420 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -344,6 +344,20 @@ def store_file(self, key, content_type="application/octet-stream", content=None) return self.do_put(host, port, url, headers, content, key=key) + def get_file(self, key): + host, port, url = self.build_rest_path(prefix='luwak', key=key) + response = self.http_request('GET', host, port, url) + result = self.parse_body(response, [200, 300, 404]) + if result is not None: + (vclock, data) = result + (headers, body) = data.pop() + return body + + def delete_file(self, key): + host, port, url = self.build_rest_path(prefix='luwak', key=key) + response = self.http_request('DELETE', host, port, url) + self.parse_body(response, [204, 404]) + # Utility functions used by Riak library. @classmethod diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index eec68084..7e12f0ab 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -584,4 +584,4 @@ def set_client_id(self, client_id): def get_client_id(self): """see set_client_id notes, you can do wrong with this""" return self._make_call(lambda conn: conn.get_client_id()) - + diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 6d39dae5..0710eff5 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -118,3 +118,11 @@ def get_client_id(self): """ raise RiakError("not implemented") + def store_file(self, key, content_type="application/octet-stream", content=None): + raise RiakError("luwak not supported by this transport.") + + def get_file(self, key): + raise RiakError("luwak not supported by this transport.") + + def delete_file(self, key): + raise RiakError("luwak not supported by this transport.") From d8311b38d9120bada40645f8d4cdb304a2f3764e Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Tue, 12 Jul 2011 16:12:54 +0200 Subject: [PATCH 0054/1060] Add switch to disable luwak tests. --- riak/tests/test_all.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index d36bd98d..8a7edb1e 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -24,6 +24,7 @@ HTTP_PORT = int(os.environ.get('RIAK_TEST_HTTP_PORT', '8098')) PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '0')) +SKIP_LUWAK = int(os.environ.get('SKIP_LUWAK', '0')) class NotJsonSerializable(object): @@ -747,6 +748,9 @@ def test_too_many_link_headers_shouldnt_break_http(self): self.assertEqual(len(stored_object.get_links()), 400) def test_store_file_with_luwak(self): + if SKIP_LUWAK: + return True + file = os.path.dirname(__file__) + "/test_all.py" with open(file, "r") as input_file: data = input_file.read() @@ -755,6 +759,9 @@ def test_store_file_with_luwak(self): self.client.store_file(key, data) def test_store_get_file_with_luwak(self): + if SKIP_LUWAK: + return True + file = os.path.dirname(__file__) + "/test_all.py" with open(file, "r") as input_file: data = input_file.read() @@ -766,6 +773,9 @@ def test_store_get_file_with_luwak(self): self.assertEquals(data, file) def test_delete_file_with_luwak(self): + if SKIP_LUWAK: + return True + file = os.path.dirname(__file__) + "/test_all.py" with open(file, "r") as input_file: data = input_file.read() From 93859c05a9ca4998276cbaa2e53d6998fd979c21 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Tue, 12 Jul 2011 16:31:10 +0200 Subject: [PATCH 0055/1060] Add documentation for luwak methods. --- README.rst | 17 +++++++++++++++++ riak/transports/transport.py | 14 ++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/README.rst b/README.rst index b8578d18..c595bbd8 100644 --- a/README.rst +++ b/README.rst @@ -469,3 +469,20 @@ filters:: .. _`Key filters`: http://wiki.basho.com/Key-Filters.html + +Luwak for Large File Storage +============================ + +If your Riak installation has Luwak support enabled, you can use the client to +interact with it, storing, fetching and deleting files. + + client = riak.RiakClient() + + client.store_file('image.jpg', 'lots of data', content_type="image/jpeg") + + # Returns just the data stored in luwak + client.get_file('image.jpg') + + client.delete_file('image.jpg') + +.. _`Luwak`: http://wiki.basho.com/Luwak.html diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 0710eff5..b8245f0e 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -119,10 +119,24 @@ def get_client_id(self): raise RiakError("not implemented") def store_file(self, key, content_type="application/octet-stream", content=None): + """ + Store a large piece of data in luwak. + key = the key/filename for the object + content_type = the object's content type + content = the object's data + """ raise RiakError("luwak not supported by this transport.") def get_file(self, key): + """ + Get an object from luwak. + key = the object's key + """ raise RiakError("luwak not supported by this transport.") + """ + Delete an object in luwak. + key = the object's key + """ def delete_file(self, key): raise RiakError("luwak not supported by this transport.") From 347b397d43777c0cb819bb9aaf4fe93305681e9b Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Tue, 12 Jul 2011 16:35:35 +0200 Subject: [PATCH 0056/1060] Needs more colons in the README. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index c595bbd8..7d661433 100644 --- a/README.rst +++ b/README.rst @@ -474,7 +474,7 @@ Luwak for Large File Storage ============================ If your Riak installation has Luwak support enabled, you can use the client to -interact with it, storing, fetching and deleting files. +interact with it, storing, fetching and deleting files:: client = riak.RiakClient() From 91383468b84ee28b0e07757776179736a85c5210 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Wed, 13 Jul 2011 18:16:46 +0200 Subject: [PATCH 0057/1060] Add support for search through Solr interface. --- riak/bucket.py | 4 ++++ riak/client.py | 15 ++++++++++++++- riak/tests/test_all.py | 15 +++++++++++++++ riak/transports/http.py | 14 ++++++++++++-- 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 58163af6..ef845d0a 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -420,3 +420,7 @@ def new_binary_from_file(self, key, filename): if not mimetype: mimetype = 'application/octet-stream' return self.new_binary(key, binary_data, mimetype) + + def search(self, query, **params): + return self._client.solr_transport.search(self, query, **params) + diff --git a/riak/client.py b/riak/client.py index 8f335fb2..33234fba 100644 --- a/riak/client.py +++ b/riak/client.py @@ -35,7 +35,7 @@ class RiakClient(object): """ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', mapred_prefix='mapred', transport_class=None, - client_id=None): + client_id=None, solr_transport_class=None): """ Construct a new ``RiakClient`` object. @@ -49,6 +49,8 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', :type mapred_prefix: string :param transport_class: transport class to use :type transport_class: :class:`RiakTransport` + :param solr_transport_class: HTTP-based transport class for Solr interface queries + :type transport_class: :class:`RiakHttpTransport` """ if not transport_class: self._transport = RiakHttpTransport(host, @@ -58,6 +60,17 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', client_id) else: self._transport = transport_class(host, port, client_id=client_id) + + if not solr_transport_class: + self.solr_transport = RiakHttpTransport(host, + port, + prefix, + mapred_prefix, + client_id) + else: + self.solr_transport = solr_transport_class(host, port, client_id=client_id) + + self._r = "default" self._w = "default" self._dw = "default" diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 15ad7655..e1afe4a3 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -743,6 +743,21 @@ def test_too_many_link_headers_shouldnt_break_http(self): stored_object = bucket.get("lots_of_links") self.assertEqual(len(stored_object.get_links()), 400) + def test_solr_search(self): + if SKIP_SEARCH: + return True + bucket = self.client.bucket('searchbucket') + bucket.new("user", {"username": "roidrage"}).store() + results = bucket.search("username:roidrage") + self.assertEquals(1, len(results["response"]["docs"])) + + def test_solr_search_with_params(self): + if SKIP_SEARCH: + return True + bucket = self.client.bucket('searchbucket') + bucket.new("user", {"username": "roidrage"}).store() + results = bucket.search("username:roidrage", wt="xml") + self.assertRegexpMatches(results, r' Date: Wed, 13 Jul 2011 23:38:04 +0200 Subject: [PATCH 0058/1060] Move search API into a separate class. --- riak/bucket.py | 2 +- riak/client.py | 20 ++++++++++---------- riak/search.py | 31 +++++++++++++++++++++++++++++++ riak/tests/test_all.py | 21 +++++++++++++++++++-- riak/transports/http.py | 12 +++--------- 5 files changed, 64 insertions(+), 22 deletions(-) create mode 100644 riak/search.py diff --git a/riak/bucket.py b/riak/bucket.py index ef845d0a..c86a73e4 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -422,5 +422,5 @@ def new_binary_from_file(self, key, filename): return self.new_binary(key, binary_data, mimetype) def search(self, query, **params): - return self._client.solr_transport.search(self, query, **params) + return self._client.solr().search(self._name, query, **params) diff --git a/riak/client.py b/riak/client.py index 33234fba..e0b8057e 100644 --- a/riak/client.py +++ b/riak/client.py @@ -26,6 +26,7 @@ from riak.transports import RiakHttpTransport from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduce +from riak.search import RiakSearch class RiakClient(object): """ @@ -61,16 +62,6 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', else: self._transport = transport_class(host, port, client_id=client_id) - if not solr_transport_class: - self.solr_transport = RiakHttpTransport(host, - port, - prefix, - mapred_prefix, - client_id) - else: - self.solr_transport = solr_transport_class(host, port, client_id=client_id) - - self._r = "default" self._w = "default" self._dw = "default" @@ -79,6 +70,9 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', 'text/json':json.dumps} self._decoders = {'application/json':json.loads, 'text/json':json.loads} + self._solr = None + self._host = host + self._port = port def get_transport(self): """ @@ -297,3 +291,9 @@ def reduce(self, *args): """ mr = RiakMapReduce(self) return apply(mr.reduce, args) + + def solr(self): + if self._solr is None: + self._solr = RiakSearch(self) + + return self._solr diff --git a/riak/search.py b/riak/search.py new file mode 100644 index 00000000..a2b18019 --- /dev/null +++ b/riak/search.py @@ -0,0 +1,31 @@ +from riak.transports import RiakHttpTransport + +class RiakSearch: + def __init__(self, client, transport_class=None, + host="127.0.0.1", port=8098, client_id=None): + if not transport_class: + self._transport = RiakHttpTransport(host, + port, + "/solr", + client_id) + else: + self._transport = transport_class(host, port, client_id=client_id) + + self._client = client + + def add(self, doc): + pass + + def delete(self, doc): + pass + + def search(self, index, query, **params): + options = {'q': query, 'wt': 'json'} + options.update(params) + headers, results = self._transport.search(index, options) + decoder = self._client.get_decoder(headers['content-type']) + + if decoder: + return decoder(results) + else: + return results diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index e1afe4a3..815d52f5 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -743,7 +743,7 @@ def test_too_many_link_headers_shouldnt_break_http(self): stored_object = bucket.get("lots_of_links") self.assertEqual(len(stored_object.get_links()), 400) - def test_solr_search(self): + def test_solr_search_from_bucket(self): if SKIP_SEARCH: return True bucket = self.client.bucket('searchbucket') @@ -751,7 +751,7 @@ def test_solr_search(self): results = bucket.search("username:roidrage") self.assertEquals(1, len(results["response"]["docs"])) - def test_solr_search_with_params(self): + def test_solr_search_with_params_from_bucket(self): if SKIP_SEARCH: return True bucket = self.client.bucket('searchbucket') @@ -759,6 +759,23 @@ def test_solr_search_with_params(self): results = bucket.search("username:roidrage", wt="xml") self.assertRegexpMatches(results, r' Date: Thu, 14 Jul 2011 00:02:03 +0200 Subject: [PATCH 0059/1060] Add decoder for XML search results. --- riak/search.py | 20 ++++++++++++++------ riak/tests/test_all.py | 6 ++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/riak/search.py b/riak/search.py index a2b18019..2ea73515 100644 --- a/riak/search.py +++ b/riak/search.py @@ -1,4 +1,5 @@ from riak.transports import RiakHttpTransport +from xml.etree import ElementTree class RiakSearch: def __init__(self, client, transport_class=None, @@ -12,7 +13,18 @@ def __init__(self, client, transport_class=None, self._transport = transport_class(host, port, client_id=client_id) self._client = client + self._decoders = {"text/xml": ElementTree.fromstring} + def get_decoder(self, content_type): + decoder = self._client.get_decoder(content_type) or self._decoders[content_type] + if not decoder: + decoder = self.decode + + return decoder + + def decode(self, data): + return data + def add(self, doc): pass @@ -23,9 +35,5 @@ def search(self, index, query, **params): options = {'q': query, 'wt': 'json'} options.update(params) headers, results = self._transport.search(index, options) - decoder = self._client.get_decoder(headers['content-type']) - - if decoder: - return decoder(results) - else: - return results + decoder = self.get_decoder(headers['content-type']) + return decoder(results) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 815d52f5..98ba8c1a 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -9,6 +9,8 @@ import os import random import unittest +from xml.dom.minidom import parse, parseString + from riak import RiakClient from riak import RiakPbcTransport, RiakPbcCachedTransport from riak import RiakHttpTransport, RiakHttpPoolTransport, RiakHttpReuseTransport @@ -757,7 +759,7 @@ def test_solr_search_with_params_from_bucket(self): bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage", wt="xml") - self.assertRegexpMatches(results, r' Date: Thu, 14 Jul 2011 00:07:08 +0200 Subject: [PATCH 0060/1060] No client_id required for search, hand in host and port. --- riak/client.py | 2 +- riak/search.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/riak/client.py b/riak/client.py index e0b8057e..672e4bf2 100644 --- a/riak/client.py +++ b/riak/client.py @@ -294,6 +294,6 @@ def reduce(self, *args): def solr(self): if self._solr is None: - self._solr = RiakSearch(self) + self._solr = RiakSearch(self, host=self._host, port=self._port) return self._solr diff --git a/riak/search.py b/riak/search.py index 2ea73515..40239c69 100644 --- a/riak/search.py +++ b/riak/search.py @@ -3,12 +3,11 @@ class RiakSearch: def __init__(self, client, transport_class=None, - host="127.0.0.1", port=8098, client_id=None): + host="127.0.0.1", port=8098): if not transport_class: self._transport = RiakHttpTransport(host, port, - "/solr", - client_id) + "/solr") else: self._transport = transport_class(host, port, client_id=client_id) From 783dc0a67da5b1c5f59bab3bfcc28fcedc748ab8 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 14 Jul 2011 10:03:19 +0200 Subject: [PATCH 0061/1060] Add support to add documents to the Riak Search index directly. --- riak/search.py | 20 ++++++++++++++++++-- riak/tests/test_all.py | 15 +++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/riak/search.py b/riak/search.py index 40239c69..2f111a12 100644 --- a/riak/search.py +++ b/riak/search.py @@ -1,5 +1,6 @@ from riak.transports import RiakHttpTransport from xml.etree import ElementTree +from xml.dom.minidom import Document class RiakSearch: def __init__(self, client, transport_class=None, @@ -24,8 +25,23 @@ def get_decoder(self, content_type): def decode(self, data): return data - def add(self, doc): - pass + def add(self, index, *docs): + xml = Document() + root = xml.createElement('add') + for doc in docs: + doc_element = xml.createElement('doc') + for key, value in doc.iteritems(): + field = xml.createElement('field') + field.setAttribute("name", key) + text = xml.createTextNode(value) + field.appendChild(text) + doc_element.appendChild(field) + root.appendChild(doc_element) + xml.appendChild(root) + + url = "/solr/%s/update" % index + host, port, url = self._transport.build_rest_path(bucket=None, prefix=url) + headers, response = self._transport.http_request('POST', host, port, url, {'Content-Type': 'text/xml'}, xml.toxml()) def delete(self, doc): pass diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 98ba8c1a..5242d32a 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -777,6 +777,21 @@ def test_solr_search(self): results = self.client.solr().search("searchbucket", "username:roidrage") self.assertEquals(1, len(results["response"]["docs"])) + def test_add_document_to_index(self): + if SKIP_SEARCH: + return True + + self.client.solr().add("searchbucket", {"id": "doc", "username": "tony"}) + results = self.client.solr().search("searchbucket", "username:tony") + self.assertEquals("tony", results["response"]["docs"][0]["fields"]["username"]) + + def test_add_multiple_documents_to_index(self): + if SKIP_SEARCH: + return True + self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) + results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") + self.assertEquals(2, len(results["response"]["docs"])) + class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): From 2361ec9b8b765a6938d42499a9680ba78e6f9138 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 14 Jul 2011 10:12:14 +0200 Subject: [PATCH 0062/1060] Add post_request method to HTTP transport. Abstracts some details of the HTTP transport a tiny bit, but just enough to allow for simpler external usage. --- riak/search.py | 4 ++-- riak/transports/http.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/riak/search.py b/riak/search.py index 2f111a12..995483ca 100644 --- a/riak/search.py +++ b/riak/search.py @@ -40,8 +40,8 @@ def add(self, index, *docs): xml.appendChild(root) url = "/solr/%s/update" % index - host, port, url = self._transport.build_rest_path(bucket=None, prefix=url) - headers, response = self._transport.http_request('POST', host, port, url, {'Content-Type': 'text/xml'}, xml.toxml()) + self._transport.post_request(uri=url, body=xml.toxml(), + content_type="text/xml") def delete(self, doc): pass diff --git a/riak/transports/http.py b/riak/transports/http.py index 39524e85..8a029309 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -340,6 +340,8 @@ def add_links_for_riak_object(self, robject, headers): return headers + def post_request(self, uri=None, body=None, content_type="application/json"): + return self.http_request('POST', self._host, self._port, uri, {'Content-Type': content_type}, body) #Utility functions used by Riak library. From 94908f9ec332895f6c77a316ca34ca611ef0da57 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 14 Jul 2011 10:17:12 +0200 Subject: [PATCH 0063/1060] Support params in post_request. --- riak/transports/http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 8a029309..ce264397 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -340,7 +340,8 @@ def add_links_for_riak_object(self, robject, headers): return headers - def post_request(self, uri=None, body=None, content_type="application/json"): + def post_request(self, uri=None, body=None, params=None, content_type="application/json"): + host, port, uri = self.build_rest_path(prefix=uri, params=params) return self.http_request('POST', self._host, self._port, uri, {'Content-Type': content_type}, body) #Utility functions used by Riak library. @@ -352,7 +353,7 @@ def get_value(cls, key, array, defaultValue) : else: return defaultValue - def build_rest_path(self, bucket, key=None, params=None, prefix=None) : + def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : """ Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, construct and return a URL. From dcb80af7a34165435c1bdf40a0a7ddaa0343c3b6 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 14 Jul 2011 10:32:22 +0200 Subject: [PATCH 0064/1060] Remove HTTP search in favour of get_request. Similar semantics to post_request. --- riak/search.py | 3 ++- riak/transports/http.py | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/riak/search.py b/riak/search.py index 995483ca..a9ef69ba 100644 --- a/riak/search.py +++ b/riak/search.py @@ -49,6 +49,7 @@ def delete(self, doc): def search(self, index, query, **params): options = {'q': query, 'wt': 'json'} options.update(params) - headers, results = self._transport.search(index, options) + uri = "/solr/%s/select" % index + headers, results = self._transport.get_request(uri, options) decoder = self.get_decoder(headers['content-type']) return decoder(results) diff --git a/riak/transports/http.py b/riak/transports/http.py index ce264397..67de8d25 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -220,11 +220,6 @@ def mapred(self, inputs, query, timeout=None): result = json.loads(response[1]) return result - def search(self, index, options): - prefix = "/solr/%s/select" % index - host, port, url = self.build_rest_path(bucket=None, params=options, prefix=prefix) - return self.http_request('GET', host, port, url) - def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] if not status in expected_statuses: @@ -339,6 +334,9 @@ def add_links_for_riak_object(self, robject, headers): return headers + def get_request(self, uri=None, params=None): + host, port, url = self.build_rest_path(bucket=None, params=params, prefix=uri) + return self.http_request('GET', host, port, url) def post_request(self, uri=None, body=None, params=None, content_type="application/json"): host, port, uri = self.build_rest_path(prefix=uri, params=params) From fea6e96e60f06597b1fda14f6436d33d44827244 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 14 Jul 2011 18:25:03 +0200 Subject: [PATCH 0065/1060] Add support to delete docs through Solr interface. Works for both IDs and queries. --- riak/search.py | 29 +++++++++++++++++++++++++++-- riak/tests/test_all.py | 23 +++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/riak/search.py b/riak/search.py index a9ef69ba..c01f9d3d 100644 --- a/riak/search.py +++ b/riak/search.py @@ -43,8 +43,31 @@ def add(self, index, *docs): self._transport.post_request(uri=url, body=xml.toxml(), content_type="text/xml") - def delete(self, doc): - pass + index = add + + def delete(self, index, docs=None, queries=None): + xml = Document() + root = xml.createElement('delete') + if docs: + for doc in docs: + doc_element = xml.createElement('id') + text = xml.createTextNode(doc) + doc_element.appendChild(text) + root.appendChild(doc_element) + if queries: + for query in queries: + query_element = xml.createElement('query') + text = xml.createTextNode(query) + query_element.appendChild(text) + root.appendChild(query_element) + + xml.appendChild(root) + + url = "/solr/%s/update" % index + self._transport.post_request(uri=url, body=xml.toxml(), + content_type="text/xml") + + remove = delete def search(self, index, query, **params): options = {'q': query, 'wt': 'json'} @@ -53,3 +76,5 @@ def search(self, index, query, **params): headers, results = self._transport.get_request(uri, options) decoder = self.get_decoder(headers['content-type']) return decoder(results) + + select = search diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 5242d32a..05de7c17 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -792,6 +792,29 @@ def test_add_multiple_documents_to_index(self): results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(2, len(results["response"]["docs"])) + def test_delete_documents_from_search_by_id(self): + if SKIP_SEARCH: + return True + self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) + self.client.solr().delete("searchbucket", docs=["dizzy"]) + results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") + self.assertEquals(1, len(results["response"]["docs"])) + + def test_delete_documents_from_search_by_query(self): + if SKIP_SEARCH: + return True + self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) + self.client.solr().delete("searchbucket", queries=["username:dizzy", "username:russell"]) + results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") + self.assertEquals(0, len(results["response"]["docs"])) + + def test_delete_documents_from_search_by_query_and_id(self): + if SKIP_SEARCH: + return True + self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) + self.client.solr().delete("searchbucket", docs=["dizzy"], queries=["username:russell"]) + results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") + self.assertEquals(0, len(results["response"]["docs"])) class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): From 374dc90798d6e1c4c1c201b49664d7b5c3859bfc Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 28 Jul 2011 11:49:27 +0200 Subject: [PATCH 0066/1060] Add documentation on the test server. --- README.rst | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/README.rst b/README.rst index b8578d18..95a6def4 100644 --- a/README.rst +++ b/README.rst @@ -469,3 +469,56 @@ filters:: .. _`Key filters`: http://wiki.basho.com/Key-Filters.html + +Test Server +=========== + +The client includes a Riak test server that can be used to start a Riak instance +on demand for testing purposes in your application. It uses in-memory storage +backends for both Riak KV and Riak Search and is therefore reasonably fast for a +testing setup. The in-memory setups also make it easier to wipe all data in the +instance without having to list and delete all keys manually. The original code +comes from Ripple, as do the file system implementations. + +The server needs a local Riak installation, of which it uses only the installed +Erlang libraries and the configuration files to generate and run a temporary +server in a different directory. + +By default, the HTTP port is set to 9000 and the Protocol Buffers interface +listens on port 9001. + +To use it, simply point it to your local Riak installation, and the rest is done +automagically:: + + from riak import TestServer + + server = TestServer(bin_dir="/usr/local/riak/0.14.2/bin") + server.prepare() + server.start() + +The server is started as an external process, with communication going through +the Erlang console. That allows it to easily wipe the in-memory backends used by +Riak and Riak Search. You can use the recycle() method to clean up the server:: + + server.recycle() + +To change the default configuration, you can specify additional arguments for +the Erlang VM. Let's raise the maximum number of processes to 1000000, just for +fun:: + + server = TestServer(vm_args={"+P": "1000000"}) + +You can also change the default configuration used to generate the app.config +file for the Riak instance. The format of the attributes follows the convention +of the app.config file itself, using a dict with keys for every section in the +configuration file, so "riak_core", "riak_kv", and so on. These in turn are also +dicts, following the same key-value format of the app.config file. + +So to change the default HTTP port to 8080, you can do the following:: + + server = TestServer(riak_core={"http_port": 8080}) + +The server should shut down properly when you stop the Python process, but if +you only need it for a subset of your tests, just stop the server:: + + server.stop() From dcc57ddef024073250bc002d3ff31e8d51987a28 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 28 Jul 2011 12:41:37 +0200 Subject: [PATCH 0067/1060] Add documentation for Solr search. --- README.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/README.rst b/README.rst index b8578d18..63f18b40 100644 --- a/README.rst +++ b/README.rst @@ -412,6 +412,42 @@ tutorial, but usage of this feature looks like:: # John Doe # Anna Body +Search using the Solr Interface +------------------------------- + +The search as outlined above goes through Riak's MapReduce facilities to find +and fetch objects. Sometimes you either want to go through the Solr-like +interface Riak Search offers, e.g. to index and search documents without storing +them in Riak KV and relying on the pre-commit hook to index. + +Using the Solr interface also allows you to specify sort and limit parameters, +which, using the search based on MapReduce, you'd have to do that with reduce +functions. + +You can index documents into search indexes as simple Python dicts, which need +to have an attribute named "id":: + + client = riak.RiakClient() + client.solr().add("user", {"id": "anna", "first_name": "Anna"}) + +To search for documents, specify the index and a query string:: + + client = riak.RiakClient() + client.solr().search("user", "first_name:Anna") + +Additionally you can specify all the parameters supported by the Solr +interface:: + + client.solr().search("user", "Anna", wt="json", df="first_name") + +The search interface supports both XML and JSON, parsing both result formats +into dicts. + +You can also remove documents from the index again, using either a list of +document ids or queries:: + + client.solr().delete("user", docs=["anna"], queries=["first_name:Anna"]) + .. _`Riak Search`: http://wiki.basho.com/Riak-Search.html .. _Lucene: http://lucene.apache.org/ From 2c0158e88b7fcabf28cf947ed56d56634e860086 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 28 Jul 2011 14:11:33 +0200 Subject: [PATCH 0068/1060] Add link to Solr search syntax/parameters. --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 63f18b40..26eb911c 100644 --- a/README.rst +++ b/README.rst @@ -450,6 +450,7 @@ document ids or queries:: .. _`Riak Search`: http://wiki.basho.com/Riak-Search.html .. _Lucene: http://lucene.apache.org/ +.. _`Riak Search - Querying via the Solr Interface`: http://wiki.basho.com/Riak-Search---Querying.html#Querying-via-the-Solr-Interface Using Key Filters ================== From 0e8ef20cba036297e869628bd8115696ad77514f Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Fri, 29 Jul 2011 15:10:36 +0200 Subject: [PATCH 0069/1060] Fix typos. --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 95a6def4..bdb48aad 100644 --- a/README.rst +++ b/README.rst @@ -490,7 +490,7 @@ listens on port 9001. To use it, simply point it to your local Riak installation, and the rest is done automagically:: - from riak import TestServer + from riak.test_server import TestServer server = TestServer(bin_dir="/usr/local/riak/0.14.2/bin") server.prepare() @@ -516,7 +516,7 @@ dicts, following the same key-value format of the app.config file. So to change the default HTTP port to 8080, you can do the following:: - server = TestServer(riak_core={"http_port": 8080}) + server = TestServer(riak_core={"web_port": 8080}) The server should shut down properly when you stop the Python process, but if you only need it for a subset of your tests, just stop the server:: From 6e69f546a983fbe88dc746d1ac2588160db77655 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Fri, 29 Jul 2011 16:06:09 +0200 Subject: [PATCH 0070/1060] Add notes on cleanup() and Riak version. --- README.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index bdb48aad..30f147a4 100644 --- a/README.rst +++ b/README.rst @@ -478,11 +478,12 @@ on demand for testing purposes in your application. It uses in-memory storage backends for both Riak KV and Riak Search and is therefore reasonably fast for a testing setup. The in-memory setups also make it easier to wipe all data in the instance without having to list and delete all keys manually. The original code -comes from Ripple, as do the file system implementations. +comes from Ripple_, as do the file system implementations. The server needs a local Riak installation, of which it uses only the installed Erlang libraries and the configuration files to generate and run a temporary -server in a different directory. +server in a different directory. Make sure you run the most recent stable +version of Riak, and not a development snapshot, where your mileage may vary. By default, the HTTP port is set to 9000 and the Protocol Buffers interface listens on port 9001. @@ -522,3 +523,9 @@ The server should shut down properly when you stop the Python process, but if you only need it for a subset of your tests, just stop the server:: server.stop() + +If you plan on repeatedly running the test server, either in multiple test +suites or in subsequent test runs, be sure to call cleanup() before starting or +after stopping it. + +.. _Ripple: https://github.com/seancribbs/ripple From ac554f336cc0271de8b4c9a0d50577fc33e22f2c Mon Sep 17 00:00:00 2001 From: Reid Draper Date: Mon, 1 Aug 2011 19:28:01 -0400 Subject: [PATCH 0071/1060] fix bug connection not returing to pool during exception --- riak/transports/pbc.py | 43 +++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index abde6316..949d4615 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -481,6 +481,7 @@ def pbify_content(self, metadata, data, rpb_content) : rpb_content.value = data from Queue import Empty, Full, Queue +import contextlib class RiakPbcCachedTransport(RiakTransport): """Threadsafe pool of PBC connections, based on urllib3's pool [aka Queue]""" def __init__(self, host='127.0.0.1', port=8087, client_id=None, maxsize=0, block=False, timeout=None): @@ -512,29 +513,30 @@ def _put_conn(self, conn): except Full: pass - def _make_call(self, function): + @contextlib.contextmanager + def _get_connection_from_pool(self): """checkout conn, try operation, put conn back in pool""" + conn = self._get_conn() try: - conn = self._get_conn() - rv = function(conn) + yield conn + finally: self._put_conn(conn) - except Exception: - raise - return rv def ping(self): """ Ping the remote server @return boolean """ - return self._make_call(lambda conn: conn.ping()) + with self._get_connection_from_pool() as conn: + return conn.ping() def get(self, robj, r = None, vtag = None): """ Serialize get request and deserialize response @return (vclock=None, [(metadata, value)]=None) """ - return self._make_call(lambda conn: conn.get(robj, r, vtag)) + with self._get_connection_from_pool() as conn: + return conn.get(robj, r, vtag) def put(self, robj, w = None, dw = None, return_body = True): """ @@ -542,27 +544,30 @@ def put(self, robj, w = None, dw = None, return_body = True): is true, retrieve the updated metadata/content @return (vclock=None, [(metadata, value)]=None) """ - return self._make_call(lambda conn: conn.put(robj, w, dw, return_body)) + with self._get_connection_from_pool() as conn: + return conn.put(robj, w, dw, return_body) def delete(self, robj, rw = None): """ Serialize delete request and deserialize response @return true """ - return self._make_call(lambda conn: conn.delete(robj, rw)) - + with self._get_connection_from_pool() as conn: + return conn.delete(robj, rw) def get_buckets(self): """ Serialize bucket listing request and deserialize response """ - return self._make_call(lambda conn: conn.get_buckets()) + with self._get_connection_from_pool() as conn: + return conn.get_buckets() def get_bucket_props(self, bucket) : """ Serialize get bucket property request and deserialize response @return dict() """ - return self._make_call(lambda conn: conn.get_bucket_props(bucket)) + with self._get_connection_from_pool() as conn: + return conn.get_bucket_props(bucket) def set_bucket_props(self, bucket, props) : """ @@ -571,19 +576,23 @@ def set_bucket_props(self, bucket, props) : props = dictionary of properties @return boolean """ - return self._make_call(lambda conn: conn.set_bucket_props(bucket, props)) + with self._get_connection_from_pool() as conn: + return conn.set_bucket_props(bucket, props) def mapred(self, inputs, query, timeout = None) : """ Serialize map/reduce request """ - return self._make_call(lambda conn: conn.mapred(inputs, query, timeout)) + with self._get_connection_from_pool() as conn: + return conn.mapred(inputs, query, timeout) def set_client_id(self, client_id): """Mmm, this can turn ugly if you use different id for different objects in the pool""" - return self._make_call(lambda conn: conn.set_client_id(client_id)) + with self._get_connection_from_pool() as conn: + return conn.set_client_id(client_id) def get_client_id(self): """see set_client_id notes, you can do wrong with this""" - return self._make_call(lambda conn: conn.get_client_id()) + with self._get_connection_from_pool() as conn: + return conn.get_client_id() From ee59062c0f38349babe56751b41fd943e55f0bfe Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Tue, 2 Aug 2011 09:51:19 +0200 Subject: [PATCH 0072/1060] Fix syntax for Python 2.6. --- riak/test_server.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/riak/test_server.py b/riak/test_server.py index 1283da70..0262aaa4 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -168,21 +168,21 @@ def wait_for_erlang_prompt(self): prompted = True def write_riak_script(self): - with open(self._riak_script, "wb") as temp_bin_file, open(os.path.join(self.bin_dir, "riak"), "r") as riak_file: - - for line in riak_file.readlines(): - line = re.sub("(RUNNER_SCRIPT_DIR=)(.*)", r'\1%s' % self._temp_bin, line) - line = re.sub("(RUNNER_ETC_DIR=)(.*)", r'\1%s' % self._temp_etc, line) - line = re.sub("(RUNNER_USER=)(.*)", r'\1', line) - line = re.sub("(RUNNER_LOG_DIR=)(.*)", r'\1%s' % self._temp_log, line) - line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % self._temp_pipe, line) + with open(self._riak_script, "wb") as temp_bin_file: + with open(os.path.join(self.bin_dir, "riak"), "r") as riak_file: + for line in riak_file.readlines(): + line = re.sub("(RUNNER_SCRIPT_DIR=)(.*)", r'\1%s' % self._temp_bin, line) + line = re.sub("(RUNNER_ETC_DIR=)(.*)", r'\1%s' % self._temp_etc, line) + line = re.sub("(RUNNER_USER=)(.*)", r'\1', line) + line = re.sub("(RUNNER_LOG_DIR=)(.*)", r'\1%s' % self._temp_log, line) + line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % self._temp_pipe, line) - if string.strip(line) == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}": - line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join(self.bin_dir, "..")) + if string.strip(line) == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}": + line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join(self.bin_dir, "..")) - temp_bin_file.write(line) + temp_bin_file.write(line) - os.fchmod(temp_bin_file.fileno(), 0755) + os.fchmod(temp_bin_file.fileno(), 0755) def write_vm_args(self): with open(os.path.join(self._temp_etc, "vm.args"), 'wb') as vm_args: From eaf2e5598d313a49c981b44fde5a88a02d18c93b Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Tue, 2 Aug 2011 23:17:32 -0500 Subject: [PATCH 0073/1060] Reset the ProtocolBuffer transport socket upon a connection error or broken pipe. --- riak/transports/pbc.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index abde6316..85eda2ea 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -334,7 +334,13 @@ def mapred(self, inputs, query, timeout=None): def maybe_connect(self): if self._sock is None: self._sock = s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect((self._host, self._port)) + + try: + s.connect((self._host, self._port)) + except: + self._sock = None + raise + if self._client_id: self.set_client_id(self._client_id) @@ -399,6 +405,7 @@ def recv_msg(self): def recv_pkt(self): nmsglen = self._sock.recv(4) if len(nmsglen) != 4: + self._sock = None raise RiakError("Socket returned short packet length {0} - expected 4". format(nmsglen)) msglen, = struct.unpack('!i', nmsglen) From ffc73dd7e6b143430988255fe5f626b2ba75fac5 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Wed, 3 Aug 2011 11:36:36 +0200 Subject: [PATCH 0074/1060] Use connection as a name instead of conn. --- riak/transports/pbc.py | 61 +++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 949d4615..1d7b476e 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -495,48 +495,48 @@ def __init__(self, host='127.0.0.1', port=8087, client_id=None, maxsize=0, block # Fill the queue up so that doing get() on it will block properly (check Queue#get) [self.pool.put(None) for _ in xrange(maxsize)] - def _new_conn(self): + def _new_connection(self): """New PBC connection""" return RiakPbcTransport(self.host, self.port, self.client_id) - def _get_conn(self): - conn = None + def _get_connection(self): + connection = None try: - conn = self.pool.get(block=self.block, timeout=self.timeout) + connection = self.pool.get(block=self.block, timeout=self.timeout) except Empty: pass - return conn or self._new_conn() + return connection or self._new_connection() - def _put_conn(self, conn): + def _put_connection(self, connection): try: - self.pool.put(conn, block=False) + self.pool.put(connection, block=False) except Full: pass @contextlib.contextmanager def _get_connection_from_pool(self): """checkout conn, try operation, put conn back in pool""" - conn = self._get_conn() + connection = self._get_connection() try: - yield conn + yield connection finally: - self._put_conn(conn) + self._put_connection(connection) def ping(self): """ Ping the remote server @return boolean """ - with self._get_connection_from_pool() as conn: - return conn.ping() + with self._get_connection_from_pool() as connection: + return connection.ping() def get(self, robj, r = None, vtag = None): """ Serialize get request and deserialize response @return (vclock=None, [(metadata, value)]=None) """ - with self._get_connection_from_pool() as conn: - return conn.get(robj, r, vtag) + with self._get_connection_from_pool() as connection: + return connection.get(robj, r, vtag) def put(self, robj, w = None, dw = None, return_body = True): """ @@ -544,30 +544,31 @@ def put(self, robj, w = None, dw = None, return_body = True): is true, retrieve the updated metadata/content @return (vclock=None, [(metadata, value)]=None) """ - with self._get_connection_from_pool() as conn: - return conn.put(robj, w, dw, return_body) + with self._get_connection_from_pool() as connection: + return connection.put(robj, w, dw, return_body) def delete(self, robj, rw = None): """ Serialize delete request and deserialize response @return true """ - with self._get_connection_from_pool() as conn: - return conn.delete(robj, rw) + with self._get_connection_from_pool() as connection: + return connection.delete(robj, rw) + def get_buckets(self): """ Serialize bucket listing request and deserialize response """ - with self._get_connection_from_pool() as conn: - return conn.get_buckets() + with self._get_connection_from_pool() as connection: + return connection.get_buckets() def get_bucket_props(self, bucket) : """ Serialize get bucket property request and deserialize response @return dict() """ - with self._get_connection_from_pool() as conn: - return conn.get_bucket_props(bucket) + with self._get_connection_from_pool() as connection: + return connection.get_bucket_props(bucket) def set_bucket_props(self, bucket, props) : """ @@ -576,23 +577,23 @@ def set_bucket_props(self, bucket, props) : props = dictionary of properties @return boolean """ - with self._get_connection_from_pool() as conn: - return conn.set_bucket_props(bucket, props) + with self._get_connection_from_pool() as connection: + return connection.set_bucket_props(bucket, props) def mapred(self, inputs, query, timeout = None) : """ Serialize map/reduce request """ - with self._get_connection_from_pool() as conn: - return conn.mapred(inputs, query, timeout) + with self._get_connection_from_pool() as connection: + return connection.mapred(inputs, query, timeout) def set_client_id(self, client_id): """Mmm, this can turn ugly if you use different id for different objects in the pool""" - with self._get_connection_from_pool() as conn: - return conn.set_client_id(client_id) + with self._get_connection_from_pool() as connection: + return connection.set_client_id(client_id) def get_client_id(self): """see set_client_id notes, you can do wrong with this""" - with self._get_connection_from_pool() as conn: - return conn.get_client_id() + with self._get_connection_from_pool() as connection: + return connection.get_client_id() From a0b2e8085a3ee0f2f99030fda56728dda5047a5a Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Wed, 3 Aug 2011 11:51:15 +0200 Subject: [PATCH 0075/1060] Update README to mention HTTP only. --- README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 7d661433..a627214b 100644 --- a/README.rst +++ b/README.rst @@ -474,7 +474,10 @@ Luwak for Large File Storage ============================ If your Riak installation has Luwak support enabled, you can use the client to -interact with it, storing, fetching and deleting files:: +interact with it, storing, fetching and deleting files. Note that Luwak is HTTP +only and will always use the settings provided for the HTTP transport. If you +mix Luwak with normal Riak usage through the Protocol Buffers interface, it's +best to use multiple client objects for each separate use case:: client = riak.RiakClient() From cae6b77f2197784d3d6b978eeafe97946467a9db Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Wed, 3 Aug 2011 13:11:41 +0200 Subject: [PATCH 0076/1060] Improve Luwak example in README. --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a627214b..3eeb2e03 100644 --- a/README.rst +++ b/README.rst @@ -481,7 +481,8 @@ best to use multiple client objects for each separate use case:: client = riak.RiakClient() - client.store_file('image.jpg', 'lots of data', content_type="image/jpeg") + image = open('hulk.jpg', 'rb') + client.store_file('image.jpg', image.read(), content_type="image/jpeg") # Returns just the data stored in luwak client.get_file('image.jpg') From 459fe42d49bf7edf33a76854a093c15bb967db5f Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 4 Aug 2011 12:15:53 +0200 Subject: [PATCH 0077/1060] Update changelog for 1.3.0 release. --- RELEASE_NOTES.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 6a471f22..3dc10669 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,33 @@ # Riak Python Client Release Notes +## 1.3.0 Feature Release - 2011-08-04 + +Release 1.3.0 is a feature release bringing a slew of updates. + +Noteworthy features: + +* #37: Support for the Riak Search HTTP Interface (Mathias Meyer) +* #36: Support to store large files in Luwak (Mathias Meyer) +* #35: Convenience methods to enable, disable and check search indexing + on Riak buckets (Mathias Meyer) +* #34: Port of Ripple's test server to Python, allows faster testing + thanks to an in-memory Riak instance (Mathias Meyer) +* #31: New transports: A Protocol Buffers connection cache + (riak.transports.pbc.RiakPbcCacheTransport), a transport to reuse the + underlying TCP connections by setting SO_REUSEADDR on the socket + (riak.transports.http.RiakHttpReuseTransport), and one that tries to + reuse connections to the same host (riak.transports.http.RiakHttpPoolTransport) + (Gilles Devaux) + +Fixes: + +* #33: Respect maximum link header size when using HTTP. Link header is now + split up into multiple headers when it exceeds the maximum size of 8192 bytes. + (Mathias Meyer) +* #41: Connections potentially not returned to the protocol buffers connection + pool. (Reid Draper) +* #42: Reset protocol buffer connection up on connection error (Brett Hoerner) + ## 1.2.2 Patch Release - 2011-06-22 Release 1.2.2 is a minor patch release. From 986f6e024b6982e910b422382649f8b6f8e33e3c Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Thu, 4 Aug 2011 15:16:09 +0200 Subject: [PATCH 0078/1060] Bump version 1.3.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7286f091..7814cde2 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def make_pb(): if __name__ == "__main__": setup( name='riak', - version='1.2.2', + version='1.3.0', packages = find_packages(), install_requires = ['protobuf>=2.3.0', 'urllib3>=0.4.0'], dependency_links = ["http://downloads.basho.com/support"], From d960a16ebe432729ba89c1e0174173c95e831596 Mon Sep 17 00:00:00 2001 From: Mathias Meyer Date: Fri, 5 Aug 2011 17:11:00 +0200 Subject: [PATCH 0079/1060] Restored Python 2.6 compatibility. By way of the unittest2 package, so it needs to be installed to run the tests on Python 2.6. --- riak/tests/suite.py | 7 ++++++- riak/tests/test_all.py | 15 ++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/riak/tests/suite.py b/riak/tests/suite.py index bddfd45b..2e4735b7 100644 --- a/riak/tests/suite.py +++ b/riak/tests/suite.py @@ -1,6 +1,11 @@ -import unittest import riak.tests.test_server_test import os.path +import platform + +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest def additional_tests(): top_level = os.path.join(os.path.dirname(__file__), "../../") diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 7b5f5bdb..d2fa9913 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -8,7 +8,12 @@ import simplejson as json import os import random -import unittest +import platform + +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest import uuid import time @@ -829,7 +834,9 @@ def test_solr_search_with_params_from_bucket(self): bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage", wt="xml") - self.assertEquals(1, len(list(results.find("result").iter("doc")))) + result = results.find("result") + if not hasattr(result, "iter"): setattr(result, "iter", result.getiterator) + self.assertEquals(1, len(list(result.iter("doc")))) def test_solr_search_with_params(self): if SKIP_SEARCH: @@ -837,7 +844,9 @@ def test_solr_search_with_params(self): bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = self.client.solr().search("searchbucket", "username:roidrage", wt="xml") - self.assertEquals(1, len(list(results.find("result").iter("doc")))) + result = results.find("result") + if not hasattr(result, "iter"): setattr(result, "iter", result.getiterator) + self.assertEquals(1, len(list(result.iter("doc")))) def test_solr_search(self): if SKIP_SEARCH: From 02acb73cd5cf99adf0b1c53083502316b50fdf52 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Fri, 5 Aug 2011 10:49:58 -0500 Subject: [PATCH 0080/1060] Add SKIP_LUWAK and SKIP_SEARCH to the README. #43 --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index f322d5db..e934bdd6 100644 --- a/README.rst +++ b/README.rst @@ -26,6 +26,7 @@ To run the unit tests, execute:: python setup.py test +If you don't have `Luwak `_ or `Riak Search `_ enabled you can set the ``SKIP_LUWAK`` and ``SKIP_SEARCH`` environment variables to skip those tests. ======== Tutorial From 26f72096542c657e38548a779d9c251ed9f007e2 Mon Sep 17 00:00:00 2001 From: Brett Hoerner Date: Mon, 8 Aug 2011 17:34:17 -0500 Subject: [PATCH 0081/1060] Don't return the empty object (self) with siblings. Also, update THANKS file. --- THANKS | 2 ++ riak/riak_object.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/THANKS b/THANKS index f734f6c9..c8c1371a 100644 --- a/THANKS +++ b/THANKS @@ -12,3 +12,5 @@ Mark Erdmann Greg Nelson Mikhail Sobolev Eric Moritz +Brett Hoerner +Scott Lystig Fritchie diff --git a/riak/riak_object.py b/riak/riak_object.py index 3d552900..01fdae8c 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -461,7 +461,7 @@ def get_siblings(self, r=None): :type r: integer :rtype: array of RiakObject """ - a = [self] + a = [] for i in range(self.get_sibling_count()): a.append(self.get_sibling(i, r)) return a From 699ae7b05f7a21e292d19403f7ed2a87622ce7ff Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 19 Aug 2011 16:40:17 -0500 Subject: [PATCH 0082/1060] README changes: Python 2.7, setuptools, non-localhost env vars --- README.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e934bdd6..17565f96 100644 --- a/README.rst +++ b/README.rst @@ -16,18 +16,24 @@ Documentation for Riak is available at http://wiki.basho.com/How-Things-Work.htm Install ======= +The recommended version of Python for use with this client is Python 2.7. + You must have `Protocol Buffers`_ installed before you can install the Riak Client. From the Riak Python Client root directory, execute:: python setup.py install +There is an additional dependency on the Python package `setuptools`. Please install `setuptools` first, e.g. ``port install py27-setuptools`` for OS X and MacPorts. + Unit Test =========== -To run the unit tests, execute:: +To run the unit tests against a Riak server (with default TCP port configuration) on localhost, execute:: python setup.py test If you don't have `Luwak `_ or `Riak Search `_ enabled you can set the ``SKIP_LUWAK`` and ``SKIP_SEARCH`` environment variables to skip those tests. +If your Riak server isn't running on localhost, use the environment variables ``RIAK_TEST_HOST`` and ``RIAK_TEST_HTTP_PORT`` and ``RIAK_TEST_PB_PORT=8087`` to specify where to find the Riak server. + ======== Tutorial ======== From 251542abcf5fedab4465e517117018041d64d9e2 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 19 Aug 2011 17:09:35 -0500 Subject: [PATCH 0083/1060] Ignore *.egg files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a8d3d3a0..ef74d171 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ docs/_build build/ dist/ riak.egg-info/ +*.egg From d24d6e3d293ffd8e9b9a757652db58032c739bdb Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Tue, 30 Aug 2011 22:13:41 -0400 Subject: [PATCH 0084/1060] Relax some dependencies: * if something cannot be imported, then simply disable the transport. * add a future import to enable "with" statements in pbc.py --- riak/transports/http.py | 8 +++++++- riak/transports/pbc.py | 14 +++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index c719b8d1..57fba0cc 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -573,7 +573,10 @@ def httplib_request(cls, method, host, port, uri, headers, body=''): if response is not None: response.close() raise -import urllib3 +try: + import urllib3 +except ImportError: + urllib3 = None class RiakHttpPoolTransport(RiakHttpTransport): """ @@ -585,6 +588,9 @@ class RiakHttpPoolTransport(RiakHttpTransport): def __init__(self, host='127.0.0.1', port=8098, prefix='riak', mapred_prefix='mapred', client_id=None): + if urllib3 is None: + raise RiakError("this transport is not available (no urllib3)") + super(RiakHttpPoolTransport, self).__init__(host=host, port=port, prefix=prefix, diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index ae71c764..ce1a27b9 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -17,6 +17,8 @@ specific language governing permissions and limitations under the License. """ +from __future__ import with_statement + import socket, struct try: @@ -28,7 +30,11 @@ from riak.metadata import * from riak.mapreduce import RiakMapReduce, RiakLink from riak import RiakError -import riakclient_pb2 + +try: + import riakclient_pb2 +except ImportError: + riakclient_pb2 = None ## Protocol codes MSG_CODE_ERROR_RESP = 0 @@ -82,6 +88,9 @@ def __init__(self, host='127.0.0.1', port=8087, client_id=None): @param string host - Hostname or IP address (default '127.0.0.1') @param int port - Port number (default 8087) """ + if riakclient_pb2 is None: + raise RiakError("this transport is not available (no protobuf)") + super(RiakPbcTransport, self).__init__() self._host = host self._port = port @@ -492,6 +501,9 @@ def pbify_content(self, metadata, data, rpb_content) : class RiakPbcCachedTransport(RiakTransport): """Threadsafe pool of PBC connections, based on urllib3's pool [aka Queue]""" def __init__(self, host='127.0.0.1', port=8087, client_id=None, maxsize=0, block=False, timeout=None): + if riakclient_pb2 is None: + raise RiakError("this transport is not available (no protobuf)") + self.host = host self.port = port self.client_id = client_id From 562584d50f9b24bcc05d61f3c6befdb1b77de9e1 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 2 Sep 2011 23:55:55 +0200 Subject: [PATCH 0085/1060] Stop ignoring tmp_dir argument to TestServer --- riak/test_server.py | 2 +- riak/tests/test_server_test.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/riak/test_server.py b/riak/test_server.py index 0262aaa4..18a17430 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -71,7 +71,7 @@ def __init__(self, tmp_dir="/tmp/riak/test_server", bin_dir=os.path.expanduser("~/.riak/install/riak-0.14.2/bin"), vm_args=None, **options): self._lock = threading.Lock() - self.temp_dir = "/tmp/riak/test_server" + self.temp_dir = tmp_dir self.bin_dir = bin_dir self._prepared = False self._started = False diff --git a/riak/tests/test_server_test.py b/riak/tests/test_server_test.py index 6a2437b8..1c1307cb 100644 --- a/riak/tests/test_server_test.py +++ b/riak/tests/test_server_test.py @@ -40,6 +40,11 @@ def test_set_ring_state_dir(self): def test_set_default_tmp_dir(self): self.assertEquals(self.test_server.temp_dir, "/tmp/riak/test_server") + def test_set_non_default_tmp_dir(self): + tmp_dir = '/not/the/default/dir' + server = TestServer(tmp_dir=tmp_dir) + self.assertEquals(server.temp_dir, tmp_dir) + def suite(): suite = unittest.TestSuite() suite.addTest(TestServerTestCase()) From 758d87ffcff721b58026607e6fa118ae028fe3fe Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 22:45:55 -0400 Subject: [PATCH 0086/1060] Remove the unused get_value() method, which is just DICT.get() anyways. --- riak/transports/http.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 57fba0cc..fa752fc8 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -366,13 +366,6 @@ def post_request(self, uri=None, body=None, params=None, content_type="applicati # Utility functions used by Riak library. - @classmethod - def get_value(cls, key, array, defaultValue) : - if key in array: - return array[key] - else: - return defaultValue - def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : """ Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, From d68e46b16830d80e4ad8bba6bbeefbbf7e3f5f36 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 22:48:49 -0400 Subject: [PATCH 0087/1060] Remove pycurl: it does not provide anything beyond the builtin httplib. --- riak/transports/http.py | 61 ++--------------------------------------- 1 file changed, 2 insertions(+), 59 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index fa752fc8..1783bd7f 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -19,13 +19,7 @@ """ import urllib, re from cStringIO import StringIO -# Use pycurl as first choice, httplib as second choice. -try: - import pycurl - HAS_PYCURL = True -except ImportError: - import httplib - HAS_PYCURL = False +import httplib try: import json except ImportError: @@ -403,11 +397,7 @@ def http_request(cls, method, host, port, url, headers = None, obj = '') : """ if not headers: headers = {} - if HAS_PYCURL: - return cls.pycurl_request(method, host, port, url, headers, obj) - else: - return cls.httplib_request(method, host, port, url, headers, obj) - + return cls.httplib_request(method, host, port, url, headers, obj) @classmethod def httplib_request(cls, method, host, port, uri, headers = None, body=''): @@ -436,53 +426,6 @@ def httplib_request(cls, method, host, port, uri, headers = None, body=''): if response is not None: response.close() raise - - @classmethod - def pycurl_request(cls, method, host, port, uri, headers, body=''): - if not headers: - headers = {} - url = "http://" + host + ":" + str(port) + uri - # Set up Curl... - client = pycurl.Curl() - client.setopt(pycurl.URL, url) - client.setopt(pycurl.HTTPHEADER, cls.build_headers(headers)) - if method == 'GET': - client.setopt(pycurl.HTTPGET, 1) - elif method == 'POST': - client.setopt(pycurl.POST, 1) - client.setopt(pycurl.POSTFIELDS, body) - elif method == 'PUT': - client.setopt(pycurl.CUSTOMREQUEST, method) - client.setopt(pycurl.POSTFIELDS, body) - elif method == 'DELETE': - client.setopt(pycurl.CUSTOMREQUEST, method) - - # Capture the response headers... - response_headers_io = StringIO() - client.setopt(pycurl.HEADERFUNCTION, response_headers_io.write) - - # Capture the response body... - response_body_io = StringIO() - client.setopt(pycurl.WRITEFUNCTION, response_body_io.write) - - try: - # Run the request. - client.perform() - http_code = client.getinfo(pycurl.HTTP_CODE) - client.close() - - # Get the headers... - response_headers = cls.parse_http_headers(response_headers_io.getvalue()) - response_headers['http_code'] = http_code - - # Get the body... - response_body = response_body_io.getvalue() - - return response_headers, response_body - except: - if client is not None: client.close() - raise - @classmethod def build_headers(cls, headers): return ['%s: %s' % (header, value) for header, value in headers.iteritems()] From 0f6f01a5c29b998cc2e8f514de5dcb4d595bdc3b Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 23:04:06 -0400 Subject: [PATCH 0088/1060] Remove host/port parameters from various method signatures, and rely on self._host and self._port for the values. * riak/transports/http.py: (RiakHttpTransport.ping): remove host/port from .http_request() call (RiakHttpTransport.get, RiakHttpRequest.delete, RiakHttpRequest.get_keys, RiakHttpRequest.get_buckets, RiakHttpRequest.get_bucket_props, RiakHttpRequest.set_bucket_props, RiakHttpRequest.get_request, RiakHttpRequest.get_file, RiakHttpRequest.delete_file, RiakHttpRequest.post_request): remove host/port return values from build_rest_path() and avoid passing to .http_request() (RiakHttpTransport.put, RiakHttpTransport.store_file): remove host/port return values from build_rest_path(), and remove the host/port from from .do_put() (RiakHttpTransport.do_put): remove host/port from the signature, and avoid passing them to .http_request() (RiakHttpTransport.mapred): remove host/port localvars, and avoid passing to .http_request() (RiakHttpTransport.build_rest_path): only return the constructed path (RiakHttpTransport.http_request): move to an instance method, and use self._host and self._port rather than params (which are now eliminated from the func signature). (RiakHttpTransport.httplib_request, RiakHttpReuseTransport.httplib_request): remove host/port from the signature and use self._host and self._port instead. Switch to an instance method to support this. (RiakHttpPoolTransport.httplib_request): maintain existing functionality by storing the pool in the class, but this seems wrong. otherwise, in the main: eliminate the host/port parameters. --- riak/transports/http.py | 95 ++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 49 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 1783bd7f..2e9ec583 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -73,7 +73,7 @@ def ping(self) : """ Check server is alive over HTTP """ - response = self.http_request('GET', self._host, self._port, '/ping') + response = self.http_request('GET', '/ping') return(response is not None) and (response[1] == 'OK') @@ -84,9 +84,9 @@ def get(self, robj, r, vtag = None) : params = {'r' : r} if vtag is not None: params['vtag'] = vtag - host, port, url = self.build_rest_path(robj.get_bucket(), robj.get_key(), - params=params) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(robj.get_bucket(), robj.get_key(), + params=params) + response = self.http_request('GET', url) return self.parse_body(response, [200, 300, 404]) def put(self, robj, w = None, dw = None, return_body = True): @@ -95,8 +95,8 @@ def put(self, robj, w = None, dw = None, return_body = True): """ # Construct the URL... params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw} - host, port, url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), - params=params) + url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), + params=params) # Construct the headers... headers = MultiDict({'Accept' : 'text/plain, */*; q=0.5', @@ -114,13 +114,13 @@ def put(self, robj, w = None, dw = None, return_body = True): headers['X-Riak-Meta-%s' % key] = value content = robj.get_encoded_data() - return self.do_put(host, port, url, headers, content, return_body, key=robj.get_key()) + return self.do_put(url, headers, content, return_body, key=robj.get_key()) - def do_put(self, host, port, url, headers, content, return_body=False, key=None): + def do_put(self, url, headers, content, return_body=False, key=None): if key is None: - response = self.http_request('POST', host, port, url, headers, content) + response = self.http_request('POST', url, headers, content) else: - response = self.http_request('PUT', host, port, url, headers, content) + response = self.http_request('PUT', url, headers, content) if return_body: return self.parse_body(response, [200, 201, 300]) @@ -131,18 +131,18 @@ def do_put(self, host, port, url, headers, content, return_body=False, key=None) def delete(self, robj, rw): # Construct the URL... params = {'rw' : rw} - host, port, url = self.build_rest_path(robj.get_bucket(), robj.get_key(), - params=params) + url = self.build_rest_path(robj.get_bucket(), robj.get_key(), + params=params) # Run the operation.. - response = self.http_request('DELETE', host, port, url) + response = self.http_request('DELETE', url) self.check_http_code(response, [204, 404]) return self def get_keys(self, bucket): params = {'props' : 'True', 'keys' : 'true'} - host, port, url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(bucket, params=params) + response = self.http_request('GET', url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: @@ -153,8 +153,8 @@ def get_keys(self, bucket): def get_buckets(self): params = {'buckets': 'true'} - host, port, url = self.build_rest_path(None, params=params) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(None, params=params) + response = self.http_request('GET', url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: @@ -166,8 +166,8 @@ def get_buckets(self): def get_bucket_props(self, bucket): # Run the request... params = {'props' : 'True', 'keys' : 'False'} - host, port, url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(bucket, params=params) + response = self.http_request('GET', url) headers = response[0] encoded_props = response[1] @@ -182,12 +182,12 @@ def set_bucket_props(self, bucket, props): """ Set the properties on the bucket object given """ - host, port, url = self.build_rest_path(bucket) + url = self.build_rest_path(bucket) headers = {'Content-Type' : 'application/json'} content = json.dumps({'props' : props}) # Run the request... - response = self.http_request('PUT', host, port, url, headers, content) + response = self.http_request('PUT', url, headers, content) # Handle the response... if response is None: @@ -208,10 +208,8 @@ def mapred(self, inputs, query, timeout=None): content = json.dumps(job) # Do the request... - host = self._host - port = self._port url = "/" + self._mapred_prefix - response = self.http_request('POST', host, port, url, {}, content) + response = self.http_request('POST', url, {}, content) result = json.loads(response[1]) return result @@ -330,19 +328,19 @@ def add_links_for_riak_object(self, robject, headers): return headers def get_request(self, uri=None, params=None): - host, port, url = self.build_rest_path(bucket=None, params=params, prefix=uri) - return self.http_request('GET', host, port, url) + url = self.build_rest_path(bucket=None, params=params, prefix=uri) + return self.http_request('GET', url) def store_file(self, key, content_type="application/octet-stream", content=None): - host, port, url = self.build_rest_path(prefix='luwak', key=key) + url = self.build_rest_path(prefix='luwak', key=key) headers = {'Content-Type' : content_type, 'X-Riak-ClientId' : self._client_id} - return self.do_put(host, port, url, headers, content, key=key) + return self.do_put(url, headers, content, key=key) def get_file(self, key): - host, port, url = self.build_rest_path(prefix='luwak', key=key) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(prefix='luwak', key=key) + response = self.http_request('GET', url) result = self.parse_body(response, [200, 300, 404]) if result is not None: (vclock, data) = result @@ -350,13 +348,13 @@ def get_file(self, key): return body def delete_file(self, key): - host, port, url = self.build_rest_path(prefix='luwak', key=key) - response = self.http_request('DELETE', host, port, url) + url = self.build_rest_path(prefix='luwak', key=key) + response = self.http_request('DELETE', url) self.parse_body(response, [204, 404]) def post_request(self, uri=None, body=None, params=None, content_type="application/json"): - host, port, uri = self.build_rest_path(prefix=uri, params=params) - return self.http_request('POST', self._host, self._port, uri, {'Content-Type': content_type}, body) + uri = self.build_rest_path(prefix=uri, params=params) + return self.http_request('POST', uri, {'Content-Type': content_type}, body) # Utility functions used by Riak library. @@ -386,10 +384,9 @@ def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : path += '?' + s # Return. - return self._host, self._port, path + return path - @classmethod - def http_request(cls, method, host, port, url, headers = None, obj = '') : + def http_request(self, method, url, headers = None, obj = '') : """ Given a Method, URL, Headers, and Body, perform and HTTP request, and return an array of arity 2 containing an associative array of @@ -397,17 +394,16 @@ def http_request(cls, method, host, port, url, headers = None, obj = '') : """ if not headers: headers = {} - return cls.httplib_request(method, host, port, url, headers, obj) + return self.httplib_request(method, url, headers, obj) - @classmethod - def httplib_request(cls, method, host, port, uri, headers = None, body=''): + def httplib_request(self, method, uri, headers = None, body=''): if not headers: headers = {} # Run the request... client = None response = None try: - client = httplib.HTTPConnection(host, port) + client = httplib.HTTPConnection(self._host, self._port) client.request(method, uri, body, headers) response = client.getresponse() @@ -473,13 +469,12 @@ def __copy__(self): return RiakHttpReuseTransport(self._host, self._port, self._prefix, self._mapred_prefix) - @classmethod - def httplib_request(cls, method, host, port, uri, headers, body=''): + def httplib_request(self, method, uri, headers, body=''): # Run the request... client = None response = None try: - client = httplib.HTTPConnection(host, port) + client = httplib.HTTPConnection(self._host, self._port) #handle the connection myself, try to reuse sockets client.auto_open = 0 @@ -538,13 +533,15 @@ def __copy__(self): return RiakHttpPoolTransport(self._host, self._port, self._prefix, self._mapred_prefix) - @classmethod - def httplib_request(cls, method, host, port, uri, headers, body=''): + def httplib_request(self, method, uri, headers, body=''): try: - if cls.http_pool is None: - cls.http_pool = urllib3.connection_from_url('http://%s:%d' % (host, port), maxsize=10) + ### it seems wrong to put the pool into a *class* variable, + ### but this code is supporting backwards-compat where the + ### use of a class variable was the design. + if self.__class__.http_pool is None: + self.__class__.http_pool = urllib3.connection_from_url('http://%s:%d' % (self._host, self._port), maxsize=10) - response = cls.http_pool.urlopen(method, uri, body, headers) + response = self.http_pool.urlopen(method, uri, body, headers) response_headers = {'http_code': response.status} for key, value in response.getheaders().iteritems(): From cc9001dd6dc836accb00c2cc3ef3ab3ce1e05636 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 23:28:11 -0400 Subject: [PATCH 0089/1060] Remove unused populate_links() method. --- riak/riak_object.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 01fdae8c..d7010928 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -395,20 +395,6 @@ def populate(self, Result) : else: raise RiakError("do not know how to handle type " + str(type(Result))) - def populate_links(self, linkHeaders) : - """ - Private. - - :rtype: self - """ - for linkHeader in linkHeaders.strip().split(','): - linkHeader = linkHeader.strip() - matches = re.match("\<\/([^\/]+)\/([^\/]+)\/([^\/]+)\>; ?riaktag=\"([^\']+)\"", linkHeader) - if (matches is not None): - link = RiakLink(matches.group(2), matches.group(3), matches.group(4)) - self._links.append(link) - return self - def has_siblings(self): """ Return True if this object has siblings. From e21caf0ece03639480a6c15316332844ca62c805 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 23:32:00 -0400 Subject: [PATCH 0090/1060] Turn a seemingly random base-10 value into an understandable hex value. --- riak/transports/transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index b8245f0e..96069bfb 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -35,7 +35,7 @@ def make_random_client_id(self): Returns a random client identifier """ return 'py_%s' % base64.b64encode( - str(random.randint(1, 1073741824))) + str(random.randint(1, 0x40000000))) @classmethod def make_fixed_client_id(self): From 1a9c8e5a6bf4cc185fd5739bbaf8f946353c24ff Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 23:34:01 -0400 Subject: [PATCH 0091/1060] Fix docstring for RiakTransport.delete_file(). --- riak/transports/transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 96069bfb..7d0be45f 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -134,9 +134,9 @@ def get_file(self, key): """ raise RiakError("luwak not supported by this transport.") + def delete_file(self, key): """ Delete an object in luwak. key = the object's key """ - def delete_file(self, key): raise RiakError("luwak not supported by this transport.") From 3e196c5aa0dd1ed9cac4361ba1249df809ccf9fd Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 5 Sep 2011 23:44:52 -0400 Subject: [PATCH 0092/1060] Fold .httplib_request() directly into .http_request(). * riak/transports/http.py: (RiakHttpTransport.http_request): adjust param names to URI and BODY to match the expections of the .httplib_request() method body. fold that method into http_request. alter header examinatino to "is None" so we don't (unnecessarily) fire on an empty dictionary. (RiakHttpTransport.httplib_request): folded into .http_request() (RiakHttpReuseTransport.httplib_request): renamed to .http_request() (RiakHttpPoolTransport.httplib_request): renamed to .http_request() --- riak/transports/http.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 2e9ec583..3b5d241e 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -386,18 +386,13 @@ def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : # Return. return path - def http_request(self, method, url, headers = None, obj = '') : + def http_request(self, method, uri, headers=None, body='') : """ Given a Method, URL, Headers, and Body, perform and HTTP request, - and return an array of arity 2 containing an associative array of - response headers and the response body. + and return a 2-tuple containing a dictionary of response headers + and the response body. """ - if not headers: - headers = {} - return self.httplib_request(method, url, headers, obj) - - def httplib_request(self, method, uri, headers = None, body=''): - if not headers: + if headers is None: headers = {} # Run the request... client = None @@ -469,7 +464,7 @@ def __copy__(self): return RiakHttpReuseTransport(self._host, self._port, self._prefix, self._mapred_prefix) - def httplib_request(self, method, uri, headers, body=''): + def http_request(self, method, uri, headers, body=''): # Run the request... client = None response = None @@ -533,7 +528,7 @@ def __copy__(self): return RiakHttpPoolTransport(self._host, self._port, self._prefix, self._mapred_prefix) - def httplib_request(self, method, uri, headers, body=''): + def http_request(self, method, uri, headers, body=''): try: ### it seems wrong to put the pool into a *class* variable, ### but this code is supporting backwards-compat where the From 34befab6f89c6baa089daaf575e4c195302016aa Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 7 Sep 2011 19:52:34 -0400 Subject: [PATCH 0093/1060] Build a connection management class to handle multiple connections to multiple hosts, passing them out for use. Both HTTP and bare sockets are supported. * riak/transports/connection.py: (ConnectionManager): new base class for managing connections (HTTPConnectionManager): manage HTTP connections (Socket): new connection type, for bare sockets (SocketConnectionManager): manage bare socket connections (NoHostsDefined): simple exception when we cannot open a conn * riak/transports/http.py: (RiakHttpTransport.__init__): use a connection manager (RiakHttpTransport.__copy__): disable for now (RiakHttpTransport.parse_body): can't print host/port right now (RiakHttpTransport.http_request): rebuild to use the connection manager. --- riak/transports/connection.py | 133 ++++++++++++++++++++++++++++++++++ riak/transports/http.py | 41 ++++++----- 2 files changed, 154 insertions(+), 20 deletions(-) create mode 100644 riak/transports/connection.py diff --git a/riak/transports/connection.py b/riak/transports/connection.py new file mode 100644 index 00000000..754cfd98 --- /dev/null +++ b/riak/transports/connection.py @@ -0,0 +1,133 @@ +# +# ### docco +# + +import httplib +import socket +import contextlib + + +class ConnectionManager(object): + + # Must be constructable with: connection_class(host, port) + # Must have two attribute: host and port + # Must have a close() method + connection_class = None + + def __init__(self, hostports=[]): + self.hostports = hostports[:] + self.conns = [ ] + + def add_hostport(self, host, port): + self.hostports.append((host, port)) + + def remove_host(self, host, port=None): + if port is None: + self.hostports = [(h, p) for h, p in self.hostports + if h != host] + else: + self.hostports.remove((host, port)) + + # just in case somebody wants a host/port combo and typos... + remove_hostport = remove_host + + def take(self): + if len(self.conns) == 0: + # RACE: in a multi-threaded environment, a conn might arrive in + # self.conns, but... no biggy. If we're bouncing up against + # needing a new connection, then we'll just create one. + return self._new_connection() + + # RACE: self.conns might empty out right now, so we need to protect + # our access to it. + try: + # round-robin: take from the front, we'll append when it comes back + return self.conns.pop(0) + except IndexError: + return self._new_connection() + + def giveback(self, conn): + # Connections using a host/port pair that is NOT in self.hostports + # should be ignored. Likely, remove_host() was called while this + # connection was borrowed for some work. + if (conn.host, conn.port) in self.hostports: + self.conns.append(conn) + else: + # Proactively close the connection. The caller won't know whether + # we put it into our list, or left the connection for the caller + # to deal with (and close). + conn.close() + + @contextlib.contextmanager + def withconn(self): + conn = self.take() + try: + yield conn + finally: + self.giveback(conn) + + def _new_connection(self): + if len(self.hostports) == 0: + raise NoHostsDefined() + + # Grab the first host/port combo. We'll put this at the end, so that + # we do a round-robin on the host/port pairs. + host, port = self.hostports[0] + conn = self.connection_class(host, port) + + if len(self.hostports) == 1: + # No rotation needed. + return conn + + # Be careful about rotating. We want to append before removing, so that + # we never hit a len==0 race condition. + self.hostports.append((host, port)) + + # RACE: another thread may have appended the same host/port pair. We + # will add another pair. Each thread will remove one, resulting in + # a correct state of a single pair in the list. + # RACE: another thread may get the host/port pair from hostports[0] + # before we have a chance to remove it. We don't need precision + # round-robin behavior; just something close. + # RACE: another thread may have removed hostports[0], but it will have + # placed another copy at the end. We have added a host/port pair, and + # will remove one, leaving the list in a correct state. + self.hostports.remove((host, port)) + + return conn + + +class HTTPConnectionManager(ConnectionManager): + connection_class = httplib.HTTPConnection + + +class Socket(object): + + def __init__(self, host, port): + self.host = host + self.port = port + + self.sock = None + + def maybe_connect(self): + if self.sock is None: + self.sock = s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + try: + s.connect((self.host, self.port)) + except: + self.close() + raise + + def close(self): + if self.sock is not None: + self.sock.close() + self.sock = None + + +class SocketConnectionManager(ConnectionManager): + connection_class = Socket + + +class NoHostsDefined(Exception): + pass diff --git a/riak/transports/http.py b/riak/transports/http.py index 3b5d241e..b5306d53 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -17,6 +17,8 @@ specific language governing permissions and limitations under the License. """ +from __future__ import with_statement + import urllib, re from cStringIO import StringIO import httplib @@ -30,6 +32,7 @@ from riak.mapreduce import RiakLink from riak import RiakError from riak.multidict import MultiDict +from connection import HTTPConnectionManager MAX_LINK_HEADER_SIZE = 8192 - 8 # substract length of "Link: " header string and newline @@ -51,8 +54,7 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', @param string client_id - client id to use for vector clocks """ super(RiakHttpTransport, self).__init__() - self._host = host - self._port = port + self._conns = HTTPConnectionManager([(host, port)]) self._prefix = prefix self._mapred_prefix = mapred_prefix self._client_id = client_id @@ -60,6 +62,9 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', self._client_id = self.make_random_client_id() def __copy__(self): + ### not implemented right now + raise Exception('not implemented') + ### we don't have _host and _port. will fix after some refactoring... return RiakHttpTransport(self._host, self._port, self._prefix, self._mapred_prefix) @@ -240,7 +245,8 @@ def parse_body(self, response, expected_statuses): # Check if the server is down(status==0) if not status: - m = 'Could not contact Riak Server: http://' + self._host + ':' + str(self._port) + '!' + ### we need the host/port that was used. + m = 'Could not contact Riak Server: http://$HOST:$PORT !' raise RiakError(m) # Verify that we got one of the expected statuses. Otherwise, raise an exception. @@ -395,27 +401,22 @@ def http_request(self, method, uri, headers=None, body='') : if headers is None: headers = {} # Run the request... - client = None - response = None - try: - client = httplib.HTTPConnection(self._host, self._port) - client.request(method, uri, body, headers) - response = client.getresponse() + with self._conns.withconn() as conn: + conn.request(method, uri, body, headers) + response = conn.getresponse() - # Get the response headers... - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value + try: + # Get the response headers... + response_headers = {'http_code': response.status} + for (key, value) in response.getheaders(): + response_headers[key.lower()] = value - # Get the body... - response_body = response.read() - response.close() + # Get the body... + response_body = response.read() + finally: + response.close() return response_headers, response_body - except: - if client is not None: client.close() - if response is not None: response.close() - raise @classmethod def build_headers(cls, headers): From 136448ad16bb9e0c212ea7f2f96564509534a033 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 7 Sep 2011 23:45:05 -0400 Subject: [PATCH 0094/1060] Add/expand some commentary in the ConnectionManager, and add code to open a connection to the Riak server(s) once its host/port is defined. When a host (and port) is removed from the hostport set, then remove connections. --- riak/transports/connection.py | 50 ++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/riak/transports/connection.py b/riak/transports/connection.py index 754cfd98..85b44c2b 100644 --- a/riak/transports/connection.py +++ b/riak/transports/connection.py @@ -15,12 +15,25 @@ class ConnectionManager(object): connection_class = None def __init__(self, hostports=[]): + # We want a private copy of this list: either to detach the argument + # default, or to detach from the caller's list. self.hostports = hostports[:] - self.conns = [ ] + + # Open a connection to each specified host/port. On single-threaded + # systems, this will create a round-robin across all specified servers. + # When multi-threaded, this will give us an initial set for all the + # threads to work with (and more will be created, according to demand). + self.conns = [self.connection_class(host, port) + for host, port in hostports] def add_hostport(self, host, port): self.hostports.append((host, port)) + # Open an initial connection. For single-threaded, this adds to the + # round-robin pool. On multi-threaded, it simply gives us an extra + # connectiong for the load-balancing across the servers. + self.conn.append(self.connection_class(host, port)) + def remove_host(self, host, port=None): if port is None: self.hostports = [(h, p) for h, p in self.hostports @@ -28,7 +41,27 @@ def remove_host(self, host, port=None): else: self.hostports.remove((host, port)) - # just in case somebody wants a host/port combo and typos... + # Now that the host/port pair has been removed from self.hostports, + # no connections on this pair will be added in .giveback(). Thus, the + # existing connections are all that may exist at this time. We'll + # snapshot the list, and look for offending connections, then try and + # remove them, being wary that race conditions may remove them before + # we can remove it. + for conn in self.conns[:]: + if conn.host == host: + try: + if port is None or conn.port == port: + self.conns.remove(conn) + + # If the connection was still present (no ValueError), then we + # should go ahead and close it down. + conn.close() + except ValueError: + # Another thread removed the connection. It won't be coming back, + # so we have nothing to do here. + pass + + # Just in case somebody uses a host/port combo and typos... remove_hostport = remove_host def take(self): @@ -80,17 +113,20 @@ def _new_connection(self): return conn # Be careful about rotating. We want to append before removing, so that - # we never hit a len==0 race condition. + # we never hit a len==0 race condition (which could prevent the creation + # of needed connections). self.hostports.append((host, port)) # RACE: another thread may have appended the same host/port pair. We - # will add another pair. Each thread will remove one, resulting in - # a correct state of a single pair in the list. + # will add another pair. Each thread will remove one (either [0], or + # one that had been appened), resulting in a correct state of a single + # pair in the list. # RACE: another thread may get the host/port pair from hostports[0] # before we have a chance to remove it. We don't need precision # round-robin behavior; just something close. - # RACE: another thread may have removed hostports[0], but it will have - # placed another copy at the end. We have added a host/port pair, and + # RACE: another thread may have removed hostports[0] (which we are + # also trying to remove), but it will have placed another copy at + # the end before doing so. We have also added a host/port pair, and # will remove one, leaving the list in a correct state. self.hostports.remove((host, port)) From bf39809b9d5cf5d43c0a4e9380be1e219fd3f24a Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Thu, 8 Sep 2011 04:06:10 -0400 Subject: [PATCH 0095/1060] Enable the test suite to run under earlier versions of Python, and when some dependencies (protobuf and urllib3) are missing. Adjust the existing SKIP_* tests to use the unittest "skip" mechanisms. --- riak/test_server.py | 11 +++++-- riak/tests/test_all.py | 66 ++++++++++++++++++++++-------------------- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/riak/test_server.py b/riak/test_server.py index 0262aaa4..feed739a 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -1,3 +1,5 @@ +from __future__ import with_statement + import os.path import threading import string @@ -8,6 +10,11 @@ from subprocess import Popen, PIPE from riak.util import deep_merge +try: + bytes +except NameError: + bytes = str + def erlang_config(hash, depth=1): def printable(item): k, v = item @@ -48,7 +55,7 @@ class TestServer: "ring_creation_size": 64 }, "riak_kv": { - "storage_backend": bytearray("riak_kv_test_backend"), + "storage_backend": bytes("riak_kv_test_backend"), "pb_ip": "127.0.0.1", "pb_port": 9002, "js_vm_count": 8, @@ -60,7 +67,7 @@ class TestServer: }, "riak_search": { "enabled": True, - "search_backend": bytearray("riak_search_test_backend") + "search_backend": bytes("riak_search_test_backend") }, "luwak": { "enabled": True diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index d2fa9913..73940fe9 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import with_statement import copy import cPickle @@ -24,12 +25,24 @@ from riak.mapreduce import RiakLink from riak.test_server import TestServer +try: + import riak.transports.riakclient_pb2 + HAVE_PROTO = True +except ImportError: + HAVE_PROTO = False +try: + import urllib3 + HAVE_HTTP_POOL = True +except ImportError: + HAVE_HTTP_POOL = False + HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) PB_HOST = os.environ.get('RIAK_TEST_PB_HOST', HOST) HTTP_PORT = int(os.environ.get('RIAK_TEST_HTTP_PORT', '8098')) PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '0')) +SKIP_LUWAK = int(os.environ.get('SKIP_LUWAK', '0')) USE_TEST_SERVER = int(os.environ.get('USE_TEST_SERVER', '0')) if USE_TEST_SERVER: @@ -40,7 +53,6 @@ test_server.prepare() test_server.start() -SKIP_LUWAK = int(os.environ.get('SKIP_LUWAK', '0')) class NotJsonSerializable(object): @@ -443,10 +455,8 @@ def test_store_of_missing_object(self): self.assertEqual(o.get_content_type(), "application/octet-stream") o.delete() - + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_search_integration(self): - if SKIP_SEARCH: - return True # Create some objects to search across... bucket = self.client.bucket("searchbucket") bucket.new("one", {"foo":"one", "bar":"red"}).store() @@ -698,6 +708,8 @@ class RiakPbcTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): + if not HAVE_PROTO: + self.skipTest('protobuf is unavailable') self.host = PB_HOST self.port = PB_PORT self.transport_class = RiakPbcTransport @@ -712,9 +724,12 @@ def test_uses_client_id_if_given(self): client_id = zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) # + class RiakPbcCachedTransportCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): + if not HAVE_PROTO: + self.skipTest('protobuf is unavailable') self.host = PB_HOST self.port = PB_PORT self.transport_class = RiakPbcCachedTransport @@ -779,10 +794,8 @@ def test_disable_search_commit_hook(self): bucket.disable_search() self.assertFalse(self.client.bucket("no_search_bucket").search_enabled()) + @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_store_file_with_luwak(self): - if SKIP_LUWAK: - return True - file = os.path.dirname(__file__) + "/test_all.py" with open(file, "r") as input_file: data = input_file.read() @@ -790,10 +803,8 @@ def test_store_file_with_luwak(self): key = uuid.uuid1().hex self.client.store_file(key, data) + @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_store_get_file_with_luwak(self): - if SKIP_LUWAK: - return True - file = os.path.dirname(__file__) + "/test_all.py" with open(file, "r") as input_file: data = input_file.read() @@ -804,10 +815,8 @@ def test_store_get_file_with_luwak(self): file = self.client.get_file(key) self.assertEquals(data, file) + @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_delete_file_with_luwak(self): - if SKIP_LUWAK: - return True - file = os.path.dirname(__file__) + "/test_all.py" with open(file, "r") as input_file: data = input_file.read() @@ -820,17 +829,15 @@ def test_delete_file_with_luwak(self): file = self.client.get_file(key) self.assertIsNone(file) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_from_bucket(self): - if SKIP_SEARCH: - return True bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage") self.assertEquals(1, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params_from_bucket(self): - if SKIP_SEARCH: - return True bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage", wt="xml") @@ -838,9 +845,8 @@ def test_solr_search_with_params_from_bucket(self): if not hasattr(result, "iter"): setattr(result, "iter", result.getiterator) self.assertEquals(1, len(list(result.iter("doc")))) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params(self): - if SKIP_SEARCH: - return True bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = self.client.solr().search("searchbucket", "username:roidrage", wt="xml") @@ -848,56 +854,52 @@ def test_solr_search_with_params(self): if not hasattr(result, "iter"): setattr(result, "iter", result.getiterator) self.assertEquals(1, len(list(result.iter("doc")))) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search(self): - if SKIP_SEARCH: - return True bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = self.client.solr().search("searchbucket", "username:roidrage") self.assertEquals(1, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_document_to_index(self): - if SKIP_SEARCH: - return True - self.client.solr().add("searchbucket", {"id": "doc", "username": "tony"}) results = self.client.solr().search("searchbucket", "username:tony") self.assertEquals("tony", results["response"]["docs"][0]["fields"]["username"]) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_multiple_documents_to_index(self): - if SKIP_SEARCH: - return True self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(2, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_id(self): - if SKIP_SEARCH: - return True self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", docs=["dizzy"]) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(1, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): - if SKIP_SEARCH: - return True self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", queries=["username:dizzy", "username:russell"]) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query_and_id(self): - if SKIP_SEARCH: - return True self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", docs=["dizzy"], queries=["username:russell"]) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results["response"]["docs"])) + class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): + if not HAVE_HTTP_POOL: + self.skipTest('urllib3 is unavailable') self.host = HTTP_HOST self.port = HTTP_PORT self.transport_class = RiakHttpPoolTransport From 303bad95a2b3f105d428733e7c45b5345ad21c6b Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 3 Sep 2011 20:40:13 +0800 Subject: [PATCH 0096/1060] change to python 2.5 string --- riak/transports/pbc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index ce1a27b9..365ce54f 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -367,8 +367,8 @@ def send_msg(self, msg_code, msg): pkt = self.encode_msg(msg_code, msg) sent_len = self._sock.send(pkt) if sent_len != len(pkt): - raise RiakError("PB socket returned short write {0} - expected {1}". - format(sent_len, len(pkt))) + raise RiakError("PB socket returned short write %d - expected %d"%\ + (sent_len, len(pkt)) def recv_msg(self): self.recv_pkt() From ca53521551788363c1d162520c9aae1e3d363515 Mon Sep 17 00:00:00 2001 From: Socrates Lee Date: Sat, 3 Sep 2011 22:40:19 +0800 Subject: [PATCH 0097/1060] change string format to 2.5 (cherry picked from commit 80931c36a3aba05ae161baf61971fab2039dc08d) --- riak/transports/pbc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 365ce54f..7d33975d 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -407,7 +407,7 @@ def recv_msg(self): msg = riakclient_pb2.RpbMapRedResp() msg.ParseFromString(self._inbuf[1:]) else: - raise Exception("unknown msg code {0}".format(msg_code)) + raise Exception("unknown msg code %s"%msg_code) return msg_code, msg @@ -415,8 +415,8 @@ def recv_pkt(self): nmsglen = self._sock.recv(4) if len(nmsglen) != 4: self._sock = None - raise RiakError("Socket returned short packet length {0} - expected 4". - format(nmsglen)) + raise RiakError("Socket returned short packet length %d - expected 4"%\ + nmsglen) msglen, = struct.unpack('!i', nmsglen) self._inbuf_len = msglen self._inbuf = '' @@ -426,8 +426,8 @@ def recv_pkt(self): if not recv_buf: break self._inbuf += recv_buf if len(self._inbuf) != self._inbuf_len: - raise RiakError("Socket returned short packet {0} - expected {1}". - format(len(self._inbuf), self._inbuf_len)) + raise RiakError("Socket returned short packet %d - expected %d"%\ + (len(self._inbuf), self._inbuf_len)) def decode_contents(self, rpb_contents): contents = [] From c18c22d89bbb5a3dab0d414bfeb0e8f68cbfab9d Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Fri, 9 Sep 2011 13:01:33 +0100 Subject: [PATCH 0098/1060] Fix syntax error from 303bad9 --- riak/transports/pbc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 7d33975d..7beb5267 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -368,7 +368,7 @@ def send_msg(self, msg_code, msg): sent_len = self._sock.send(pkt) if sent_len != len(pkt): raise RiakError("PB socket returned short write %d - expected %d"%\ - (sent_len, len(pkt)) + (sent_len, len(pkt))) def recv_msg(self): self.recv_pkt() From efa672c3cf5c1cf8f3dbafa572bff460ae7e5fa3 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 9 Sep 2011 21:23:07 -0400 Subject: [PATCH 0099/1060] Align the .http_request() method of the RiakHttpTransport subclasses with the superclass' signature. * riak/transports/http.py: (RiakHttpReuseTransport.http_request, RiakHttpPoolTransport): default the HEADERS param to None, and then set it to an empty dict when the caller doesn't set it --- riak/transports/http.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 3b5d241e..466cf229 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -464,7 +464,9 @@ def __copy__(self): return RiakHttpReuseTransport(self._host, self._port, self._prefix, self._mapred_prefix) - def http_request(self, method, uri, headers, body=''): + def http_request(self, method, uri, headers=None, body=''): + if headers is None: + headers = {} # Run the request... client = None response = None @@ -528,7 +530,9 @@ def __copy__(self): return RiakHttpPoolTransport(self._host, self._port, self._prefix, self._mapred_prefix) - def http_request(self, method, uri, headers, body=''): + def http_request(self, method, uri, headers={}, body=''): + if headers is None: + headers = {} try: ### it seems wrong to put the pool into a *class* variable, ### but this code is supporting backwards-compat where the From 54eae5aa23a24d923cdc8eff1dba28dedd24e3be Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 9 Sep 2011 21:52:39 -0400 Subject: [PATCH 0100/1060] Skip RiakHttpReuseTransport tests because that transport is broken (it is redundant, given the new RiakHttpTransport capabilities) --- riak/tests/test_all.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 73940fe9..f064bb0e 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -926,6 +926,8 @@ def test_set_client_id(self): class RiakHttpReuseTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): + ### RiakHttpReuseTransport does not work on this branch. + self.skipTest('RiakHttpReuseTransport is broken right now.') self.host = HTTP_HOST self.port = HTTP_PORT self.transport_class = RiakHttpReuseTransport From f2c347edf6f221e1d7ff5c297dc2f326cb84d19b Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Sat, 10 Sep 2011 23:18:32 -0400 Subject: [PATCH 0101/1060] Fix the test suite when invoked as "python test_all.py". In this case, __file__ has no directory component, so .dirname() returns '' and adding it to '/test_all.py' (or similar) will result in a reference to a file at th root of the filesystem (oops). Using os.path.join() properly handles this situation, and can increase portability. * riak/tests/test_all.py: (BaseTestCase.test_store_binary_object_from_file, BaseTestCase.test_store_file_with_luwak, BaseTestCase.test_store_get_file_with_luwak, BaseTestCase.test_delete_file_with_luwak): use os.path.join() to properly deal with variant invocations that affect the __file__ value. (BaseTestCase.test_store_binary_object_from_file_should_use_default_mimetype): use os.path.abspath(), os.pardir, and os.path.join() to correctly find the THANKS file. --- riak/tests/test_all.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 73940fe9..50914fa3 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -478,7 +478,8 @@ def test_search_integration(self): def test_store_binary_object_from_file(self): bucket = self.client.bucket('bucket') rand = str(self.randint()) - obj = bucket.new_binary_from_file('foo_from_file', os.path.dirname(__file__) + "/test_all.py") + filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') + obj = bucket.new_binary_from_file('foo_from_file', filepath) obj.store() obj = bucket.get_binary('foo_from_file') self.assertNotEqual(obj.get_data(), None) @@ -487,7 +488,9 @@ def test_store_binary_object_from_file(self): def test_store_binary_object_from_file_should_use_default_mimetype(self): bucket = self.client.bucket('bucket') rand = str(self.randint()) - obj = bucket.new_binary_from_file('foo_from_file', os.path.dirname(__file__) + '/../../THANKS') + filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), + os.pardir, os.pardir, 'THANKS') + obj = bucket.new_binary_from_file('foo_from_file', filepath) obj.store() obj = bucket.get_binary('foo_from_file') self.assertEqual(obj.get_content_type(), 'application/octet-stream') @@ -796,7 +799,7 @@ def test_disable_search_commit_hook(self): @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_store_file_with_luwak(self): - file = os.path.dirname(__file__) + "/test_all.py" + file = os.path.join(os.path.dirname(__file__), "test_all.py") with open(file, "r") as input_file: data = input_file.read() @@ -805,7 +808,7 @@ def test_store_file_with_luwak(self): @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_store_get_file_with_luwak(self): - file = os.path.dirname(__file__) + "/test_all.py" + file = os.path.join(os.path.dirname(__file__), "test_all.py") with open(file, "r") as input_file: data = input_file.read() @@ -817,7 +820,7 @@ def test_store_get_file_with_luwak(self): @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_delete_file_with_luwak(self): - file = os.path.dirname(__file__) + "/test_all.py" + file = os.path.join(os.path.dirname(__file__), "test_all.py") with open(file, "r") as input_file: data = input_file.read() From 18951435310a7a056cccc7e053a68bc09251a303 Mon Sep 17 00:00:00 2001 From: Reid Draper Date: Sun, 11 Sep 2011 12:48:17 +0100 Subject: [PATCH 0102/1060] Fix TypeError in exception text by using len(nmsglen) instead of nmsglen directly --- riak/transports/pbc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 7beb5267..07bf6f85 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -416,7 +416,7 @@ def recv_pkt(self): if len(nmsglen) != 4: self._sock = None raise RiakError("Socket returned short packet length %d - expected 4"%\ - nmsglen) + len(nmsglen)) msglen, = struct.unpack('!i', nmsglen) self._inbuf_len = msglen self._inbuf = '' From 1b4b74d172db61225786dd9acdea3d343a5b63da Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Sun, 11 Sep 2011 19:48:20 -0400 Subject: [PATCH 0103/1060] Fixes via @reiddraper. * riak/transports/connection.py: (ConnectionManager.add_hostport): fix typo: s/conn/conns/ (Connectionmanager.remove_host): tweak control flow to move conn.close() outside of the try/except. --- riak/transports/connection.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/riak/transports/connection.py b/riak/transports/connection.py index 85b44c2b..75669c6e 100644 --- a/riak/transports/connection.py +++ b/riak/transports/connection.py @@ -32,7 +32,7 @@ def add_hostport(self, host, port): # Open an initial connection. For single-threaded, this adds to the # round-robin pool. On multi-threaded, it simply gives us an extra # connectiong for the load-balancing across the servers. - self.conn.append(self.connection_class(host, port)) + self.conns.append(self.connection_class(host, port)) def remove_host(self, host, port=None): if port is None: @@ -48,18 +48,17 @@ def remove_host(self, host, port=None): # remove them, being wary that race conditions may remove them before # we can remove it. for conn in self.conns[:]: - if conn.host == host: + if conn.host == host and (port is None or conn.port == port): try: - if port is None or conn.port == port: - self.conns.remove(conn) - - # If the connection was still present (no ValueError), then we - # should go ahead and close it down. - conn.close() + self.conns.remove(conn) except ValueError: # Another thread removed the connection. It won't be coming back, # so we have nothing to do here. pass + else: + # If the connection was still present (no ValueError), then we + # should go ahead and close it down. + conn.close() # Just in case somebody uses a host/port combo and typos... remove_hostport = remove_host From dcb8678b8f460ce1b5d5d86e14d567a3bcbaa0d1 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 12 Sep 2011 21:08:52 -0400 Subject: [PATCH 0104/1060] Adjust for compatibility with Python 2.5 --- riak/util.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/riak/util.py b/riak/util.py index c2e154a0..39f1a210 100644 --- a/riak/util.py +++ b/riak/util.py @@ -1,8 +1,12 @@ -import collections +try: + from collections import Mapping +except ImportError: + # compatibility with Python 2.5 + Mapping = dict def quacks_like_dict(object): """Check if object is dict-like""" - return isinstance(object, collections.Mapping) + return isinstance(object, Mapping) def deep_merge(a, b): """Merge two deep dicts non-destructively From 6235c0b3c8e6fc49077e840fbe8db164e425d892 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 12 Sep 2011 21:45:00 -0400 Subject: [PATCH 0105/1060] Pass a ConnectionManager to the transport constructor. Create an appropriate CM and pass that to the transport. We also adjust the constructor signature and allow for arbitrary options to be passed to the transport (eg. Http takes a prefix and mapred_prefix). Some backwards compat code has been introduced in all transports (other than the basic RiakHttpTransport) to extract a host/port pair from the CM. This commit also re-enables testing of RiakHttpReuseTransport, even though it is busted and should go away "soon". --- riak/client.py | 25 ++++++++++++++------- riak/tests/test_all.py | 2 -- riak/transports/http.py | 48 ++++++++++++++++++++++------------------- riak/transports/pbc.py | 18 ++++++++++++++-- 4 files changed, 59 insertions(+), 34 deletions(-) diff --git a/riak/client.py b/riak/client.py index 867cd2e4..ced72420 100644 --- a/riak/client.py +++ b/riak/client.py @@ -27,6 +27,8 @@ from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduce from riak.search import RiakSearch +import riak.transports.connection + class RiakClient(object): """ @@ -53,14 +55,21 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', :param solr_transport_class: HTTP-based transport class for Solr interface queries :type transport_class: :class:`RiakHttpTransport` """ - if not transport_class: - self._transport = RiakHttpTransport(host, - port, - prefix, - mapred_prefix, - client_id) - else: - self._transport = transport_class(host, port, client_id=client_id) + if transport_class is None: + transport_class = RiakHttpTransport + + hostports = [ (host, port), ] + self._cm = transport_class.default_cm(hostports) + + ### we need to allow additional transport options. make this an + ### argument to __init__ ? + transport_options = { } + + self._transport = transport_class(self._cm, + prefix=prefix, + mapred_prefix=mapred_prefix, + client_id=client_id, + **transport_options) self._r = "default" self._w = "default" diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 37e69df5..50914fa3 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -929,8 +929,6 @@ def test_set_client_id(self): class RiakHttpReuseTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): - ### RiakHttpReuseTransport does not work on this branch. - self.skipTest('RiakHttpReuseTransport is broken right now.') self.host = HTTP_HOST self.port = HTTP_PORT self.transport_class = RiakHttpReuseTransport diff --git a/riak/transports/http.py b/riak/transports/http.py index 3bb71925..30ca5d58 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -42,9 +42,13 @@ class RiakHttpTransport(RiakTransport) : Riak. The Riak API uses HTTP, so there is no persistent connection, and the RiakClient object is extremely lightweight. """ - def __init__(self, host='127.0.0.1', port=8098, prefix='riak', - mapred_prefix='mapred', - client_id = None): + + # The ConnectionManager class that this transport prefers. + default_cm = HTTPConnectionManager + + def __init__(self, cm, + prefix='riak', mapred_prefix='mapred', client_id=None, + **unused_options): """ Construct a new RiakClient object. @param string host - Hostname or IP address (default '127.0.0.1') @@ -54,7 +58,7 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', @param string client_id - client id to use for vector clocks """ super(RiakHttpTransport, self).__init__() - self._conns = HTTPConnectionManager([(host, port)]) + self._conns = cm self._prefix = prefix self._mapred_prefix = mapred_prefix self._client_id = client_id @@ -451,18 +455,18 @@ class RiakHttpReuseTransport(RiakHttpTransport): Reuse sockets """ - def __init__(self, host='127.0.0.1', port=8098, prefix='riak', - mapred_prefix='mapred', - client_id=None): - super(RiakHttpReuseTransport, self).__init__(host=host, - port=port, - prefix=prefix, - mapred_prefix= + def __init__(self, cm, + prefix='riak', mapred_prefix='mapred', client_id=None, + **unused_options): + super(RiakHttpReuseTransport, self).__init__(cm, + prefix, mapred_prefix, - client_id=client_id) + client_id) + ### for backwards compat + self._host, self._port = cm.hostports[0] def __copy__(self): - return RiakHttpReuseTransport(self._host, self._port, self._prefix, + return RiakHttpReuseTransport(self._conns, self._prefix, self._mapred_prefix) def http_request(self, method, uri, headers=None, body=''): @@ -514,21 +518,21 @@ class RiakHttpPoolTransport(RiakHttpTransport): http_pool = None - def __init__(self, host='127.0.0.1', port=8098, prefix='riak', - mapred_prefix='mapred', - client_id=None): + def __init__(self, cm, + prefix='riak', mapred_prefix='mapred', client_id=None, + **unused_options): if urllib3 is None: raise RiakError("this transport is not available (no urllib3)") - super(RiakHttpPoolTransport, self).__init__(host=host, - port=port, - prefix=prefix, - mapred_prefix= + super(RiakHttpPoolTransport, self).__init__(cm, + prefix, mapred_prefix, - client_id=client_id) + client_id) + ### for backwards compat + self._host, self._port = cm.hostports[0] def __copy__(self): - return RiakHttpPoolTransport(self._host, self._port, self._prefix, + return RiakHttpPoolTransport(self._conns, self._prefix, self._mapred_prefix) def http_request(self, method, uri, headers={}, body=''): diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 07bf6f85..08d08a39 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -30,6 +30,7 @@ from riak.metadata import * from riak.mapreduce import RiakMapReduce, RiakLink from riak import RiakError +from connection import SocketConnectionManager try: import riakclient_pb2 @@ -82,7 +83,11 @@ class RiakPbcTransport(RiakTransport): 'quorum' : RIAKC_RW_QUORUM, 'one' : RIAKC_RW_ONE } - def __init__(self, host='127.0.0.1', port=8087, client_id=None): + + # The ConnectionManager class that this transport prefers. + default_cm = SocketConnectionManager + + def __init__(self, cm, client_id=None, **unused_options): """ Construct a new RiakPbcTransport object. @param string host - Hostname or IP address (default '127.0.0.1') @@ -92,6 +97,10 @@ def __init__(self, host='127.0.0.1', port=8087, client_id=None): raise RiakError("this transport is not available (no protobuf)") super(RiakPbcTransport, self).__init__() + + ### backwards compat. we don't use the ConnectionManager (yet). + host, port = cm.hostports[0] + self._host = host self._port = port self._client_id = client_id @@ -500,10 +509,15 @@ def pbify_content(self, metadata, data, rpb_content) : import contextlib class RiakPbcCachedTransport(RiakTransport): """Threadsafe pool of PBC connections, based on urllib3's pool [aka Queue]""" - def __init__(self, host='127.0.0.1', port=8087, client_id=None, maxsize=0, block=False, timeout=None): + def __init__(self, cm, + client_id=None, maxsize=0, block=False, timeout=None, + **unused_options): if riakclient_pb2 is None: raise RiakError("this transport is not available (no protobuf)") + ### backwards compat. we don't use the ConnectionManager (yet). + host, port = cm.hostports[0] + self.host = host self.port = port self.client_id = client_id From 9d90b0cbb008b1d77426f673f2330312d7a3360e Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Tue, 13 Sep 2011 03:36:27 -0400 Subject: [PATCH 0106/1060] Fix the transport creation in the search object. --- riak/search.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/riak/search.py b/riak/search.py index c01f9d3d..5e006c25 100644 --- a/riak/search.py +++ b/riak/search.py @@ -5,12 +5,12 @@ class RiakSearch: def __init__(self, client, transport_class=None, host="127.0.0.1", port=8098): - if not transport_class: - self._transport = RiakHttpTransport(host, - port, - "/solr") - else: - self._transport = transport_class(host, port, client_id=client_id) + if transport_class is None: + transport_class = RiakHttpTransport + + hostports = [ (host, port), ] + self._cm = transport_class.default_cm(hostports) + self._transport = transport_class(self._cm, prefix="/solr") self._client = client self._decoders = {"text/xml": ElementTree.fromstring} From d0de18b1100f03eb315c3c3eb2a3938c9aebe4cf Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Tue, 13 Sep 2011 03:50:39 -0400 Subject: [PATCH 0107/1060] First draft of a background thread to monitor connections. --- riak/transports/monitor.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 riak/transports/monitor.py diff --git a/riak/transports/monitor.py b/riak/transports/monitor.py new file mode 100644 index 00000000..6380eee1 --- /dev/null +++ b/riak/transports/monitor.py @@ -0,0 +1,29 @@ +import threading +import time + + +class Monitor(object): + + def __init__(self, cm, transport): + self._cm = cm + self._transport = transport + + self._stop_loop = False + self._thread = threading.Thread(target=self._run) + + self._periodic = 0.050 # 50 msec + + def start(self): + self._thread.start() + + def terminate(self): + self._stop_loop = True + self._thread.join() + + def _run(self): + while not self._stop_loop: + ### look for changes in the ring servers + + ### see if some offline servers came back + + time.sleep(self._periodic) From 7d4cdc4ff4f0ed2f2d121bd8905a34343e8b5c56 Mon Sep 17 00:00:00 2001 From: Rusty Klophaus Date: Fri, 9 Sep 2011 08:10:41 -0400 Subject: [PATCH 0108/1060] Add 2i query support. --- docs/tutorial.rst | 28 ++++ riak/client.py | 10 ++ riak/mapreduce.py | 33 ++++- riak/metadata.py | 1 + riak/riak_object.py | 20 ++- riak/tests/test_all.py | 62 ++++++++- riak/transports/http.py | 7 +- riak/transports/pbc.py | 10 ++ riak/transports/riakclient.proto | 23 +++- riak/transports/riakclient_pb2.py | 206 ++++++++++++++++++++++++------ 10 files changed, 352 insertions(+), 48 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 825e9ac3..f7a3cefa 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -387,3 +387,31 @@ tutorial, but usage of this feature looks like:: .. _`Riak Search`: http://wiki.basho.com/Riak-Search.html .. _Lucene: http://lucene.apache.org/ + +Using Secondary Indexes +======================= + +Secondary Indexes is a new feature available as of Riak 1.0. It +allows you to tag an object with index metadata, and then later find +the object by querying the metadata, returning a list of matching keys. + +Usage of this feature looks like:: + + import riak + + client = riak.RiakClient() + bucket = client.bucket('mybucket') + + # Store the object... + obj = bucket.new('mykey1', 'mydata') + obj.set_indexes({ + 'field1_bin': 'val1', + 'field2_int': 1001 + }) + obj.store() + + # Query the indexes. The return value is a list of ``RiakLink`` objects. + results = client.index('mybucket', 'field1_bin', 'val1').run() + + # Query the indexes using a range... + results = client.index('mybucket', 'field1_bin', 'val1', 'val5').run() diff --git a/riak/client.py b/riak/client.py index 867cd2e4..3ea9c7aa 100644 --- a/riak/client.py +++ b/riak/client.py @@ -265,6 +265,16 @@ def search(self, *args): mr = RiakMapReduce(self) return apply(mr.search, args) + def index(self, *args): + """ + Start assembling a Map/Reduce operation based on secondary + index query results. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.index, args) + def link(self, *args): """ Start assembling a Map/Reduce operation. A shortcut for :func:`RiakMapReduce.link`. diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 01020c97..c7181d15 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -63,8 +63,8 @@ def add_object(self, obj): def add_bucket_key_data(self, bucket, key, data) : if self._input_mode == 'bucket': raise Exception('Already added a bucket, can\'t add an object.') - elif self._input_mode == 'search': - raise Exception('Already added a search query, can\'t add an object.') + elif self._input_mode == 'query': + raise Exception('Already added a query, can\'t add an object.') else: self._inputs.append([bucket, key, data]) return self @@ -75,15 +75,15 @@ def add_bucket(self, bucket) : return self def add_key_filters(self, key_filters) : - if self._input_mode == 'search': - raise Exception('Key filters are not supported in search query.') + if self._input_mode == 'query': + raise Exception('Key filters are not supported in a query.') self._key_filters.extend(key_filters) return self def add_key_filter(self, *args) : - if self._input_mode == 'search': - raise Exception('Key filters are not supported in search query.') + if self._input_mode == 'query': + raise Exception('Key filters are not supported in a query.') self._key_filters.append(args) return self @@ -95,12 +95,31 @@ def search(self, bucket, query): @param bucket - The bucket over which to perform the search. @param query - The search query. """ - self._input_mode = 'search' + self._input_mode = 'query' self._inputs = {'module':'riak_search', 'function':'mapred_search', 'arg':[bucket, query]} return self + def index(self, bucket, index, startkey, endkey = None): + """ + Begin a map/reduce operation using a Secondary Index + query. + @param bucket - The bucket over which to perform the search. + @param query - The search query. + """ + self._input_mode = 'query' + + if endkey == None: + self._inputs = {'bucket': bucket, + 'index':index, + 'key':startkey } + else: + self._inputs = {'bucket':bucket, + 'index':index, + 'start':startkey, + 'end':endkey } + return self def link(self, bucket='_', tag='_', keep=False): """ diff --git a/riak/metadata.py b/riak/metadata.py index 5efad289..37c81b2a 100644 --- a/riak/metadata.py +++ b/riak/metadata.py @@ -25,3 +25,4 @@ MD_LASTMOD = "lastmod" MD_LASTMOD_USECS = "lastmod-usecs" MD_USERMETA = "usermeta" +MD_INDEX = "index" diff --git a/riak/riak_object.py b/riak/riak_object.py index d7010928..fe20dbdd 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -47,7 +47,7 @@ def __init__(self, client, bucket, key=None): self._encode_data = True self._vclock = None self._data = None - self._metadata = {MD_USERMETA: {}} + self._metadata = {MD_USERMETA: {}, MD_INDEX: {}} self._links = [] self._siblings = [] self._exists = False @@ -177,6 +177,24 @@ def set_usermeta(self, usermeta): self._metadata[MD_USERMETA] = usermeta return self + def get_indexes(self): + if MD_INDEX in self._metadata: + return self._metadata[MD_INDEX] + else: + return {} + + def set_indexes(self, indexes): + """ + Sets the field/value indexes under which this object will be + indexed. + + :param indexes: The field/value index data. + :type indexes: dict + :rtype: data + """ + self._metadata[MD_INDEX] = indexes + return self + def exists(self): """ Return True if the object exists, False otherwise. Allows you to diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 50914fa3..6825d24e 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -43,6 +43,7 @@ PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '0')) SKIP_LUWAK = int(os.environ.get('SKIP_LUWAK', '0')) +SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '0')) USE_TEST_SERVER = int(os.environ.get('USE_TEST_SERVER', '0')) if USE_TEST_SERVER: @@ -517,6 +518,66 @@ def test_list_buckets(self): buckets = self.client.get_buckets() self.assertTrue("list_bucket" in buckets) + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_secondary_index_store(self): + # Create a new object with indexes... + bucket = self.client.bucket('indexbucket') + rand = self.randint() + obj = bucket.new('mykey1', rand) + obj.set_indexes({ + 'field1_bin': 'val1', + 'field2_int': 1001 + }) + obj.store() + + # Retrieve the object, check that the correct indexes exist... + obj = bucket.get('mykey1') + self.assertEqual('val1', obj.get_indexes()['field1_bin']) + self.assertEqual(1001, int(obj.get_indexes()['field2_int'])) + + # Clean up... + bucket.get('mykey1').delete() + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_secondary_index_query(self): + bucket = self.client.bucket('indexbucket') + bucket.new('mykey1', 'data1').set_indexes({'field1_bin':'val1', 'field2_int':1001}).store() + bucket.new('mykey2', 'data2').set_indexes({'field1_bin':'val2', 'field2_int':1002}).store() + bucket.new('mykey3', 'data3').set_indexes({'field1_bin':'val3', 'field2_int':1003}).store() + bucket.new('mykey4', 'data4').set_indexes({'field1_bin':'val4', 'field2_int':1004}).store() + + # Test an equality query... + results = self.client.index('indexbucket', 'field1_bin', 'val2').run() + self.assertEquals(1, len(results)) + self.assertEquals('mykey2', results[0].get_key()) + + # Test a range query... + results = self.client.index('indexbucket', 'field1_bin', 'val2', 'val4').run() + vals = set() + for i in results: + vals.add(i.get_key()) + self.assertEquals(3, len(results)) + self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) + + # Test an equality query... + results = self.client.index('indexbucket', 'field2_int', 1002).run() + self.assertEquals(1, len(results)) + self.assertEquals('mykey2', results[0].get_key()) + + # Test a range query... + results = self.client.index('indexbucket', 'field2_int', 1002, 1004).run() + vals = set() + for i in results: + vals.add(i.get_key()) + self.assertEquals(3, len(results)) + self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) + + # Clean up... + bucket.get('mykey1').delete() + bucket.get('mykey2').delete() + bucket.get('mykey3').delete() + bucket.get('mykey4').delete() + class MapReduceAliasTestMixIn(object): """This tests the map reduce aliases""" @@ -897,7 +958,6 @@ def test_delete_documents_from_search_by_query_and_id(self): results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results["response"]["docs"])) - class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): diff --git a/riak/transports/http.py b/riak/transports/http.py index 466cf229..2b2a7681 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -113,6 +113,9 @@ def put(self, robj, w = None, dw = None, return_body = True): for key, value in robj.get_usermeta().iteritems(): headers['X-Riak-Meta-%s' % key] = value + for key, value in robj.get_indexes().iteritems(): + headers['X-Riak-Index-%s' % key] = value + content = robj.get_encoded_data() return self.do_put(url, headers, content, return_body, key=robj.get_key()) @@ -261,7 +264,7 @@ def parse_body(self, response, expected_statuses): # Parse the headers... vclock = None - metadata = {MD_USERMETA: {}} + metadata = {MD_USERMETA: {}, MD_INDEX: {}} links = [] for header, value in headers.iteritems(): if header == 'content-type': @@ -278,6 +281,8 @@ def parse_body(self, response, expected_statuses): metadata[MD_LASTMOD] = value elif header.startswith('x-riak-meta-'): metadata[MD_USERMETA][header.replace('x-riak-meta-', '')] = value + elif header.startswith('x-riak-index-'): + metadata[MD_INDEX][header.replace('x-riak-index-', '')] = value elif header == 'x-riak-vclock': vclock = value if links: diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 07bf6f85..9c94d769 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -471,6 +471,11 @@ def decode_content(self, rpb_content): usermeta[usermd.key] = usermd.value if len(usermeta) > 0: metadata[MD_USERMETA] = usermeta + indexes = {} + for index in rpb_content.indexes: + indexes[index.key] = index.value + if len(indexes) > 0: + metadata[MD_INDEX] = indexes return metadata, rpb_content.value def pbify_content(self, metadata, data, rpb_content) : @@ -488,6 +493,11 @@ def pbify_content(self, metadata, data, rpb_content) : pair = rpb_content.usermeta.add() pair.key = uk pair.value = uv + elif k == MD_INDEX: + for uk, uv in v.iteritems(): + pair = rpb_content.indexes.add() + pair.key = uk + pair.value = str(uv) elif k == MD_LINKS: for link in v: pb_link = rpb_content.links.add() diff --git a/riak/transports/riakclient.proto b/riak/transports/riakclient.proto index 6acd0842..ac82cb10 100644 --- a/riak/transports/riakclient.proto +++ b/riak/transports/riakclient.proto @@ -120,12 +120,19 @@ message RpbGetReq { required bytes bucket = 1; required bytes key = 2; optional uint32 r = 3; + optional uint32 pr = 4; + optional bool basic_quorum = 5; + optional bool notfound_ok = 6; + optional bytes if_modified = 7; // fail if the supplied vclock does not match + optional bool head = 8; // return everything but the value + optional bool deletedvclock = 9; // return the tombstone's vclock, if applicable } // Get Response - if the record was not found there will be no content/vclock message RpbGetResp { repeated RpbContent content = 1; optional bytes vclock = 2; // the opaque vector clock for the object + optional bool unchanged = 3; } @@ -133,18 +140,23 @@ message RpbGetResp { // the key will be returned. message RpbPutReq { required bytes bucket = 1; - required bytes key = 2; + optional bytes key = 2; optional bytes vclock = 3; required RpbContent content = 4; optional uint32 w = 5; optional uint32 dw = 6; optional bool return_body = 7; + optional uint32 pw = 8; + optional bool if_not_modified = 9; + optional bool if_none_match = 10; + optional bool return_head = 11; } -// Put response - same as get response +// Put response - same as get response with optional key if one was generated message RpbPutResp { repeated RpbContent content = 1; optional bytes vclock = 2; // the opaque vector clock for the object + optional bytes key = 3; // the key generated, if any } @@ -153,6 +165,12 @@ message RpbDelReq { required bytes bucket = 1; required bytes key = 2; optional uint32 rw = 3; + optional bytes vclock = 4; + optional uint32 r = 5; + optional uint32 w = 6; + optional uint32 pr = 7; + optional uint32 pw = 8; + optional uint32 dw = 9; } // Delete response - not defined, will return a RpbDelResp on success or RpbErrorResp on failure @@ -224,6 +242,7 @@ message RpbContent { optional uint32 last_mod = 7; optional uint32 last_mod_usecs = 8; repeated RpbPair usermeta = 9; // user metadata stored with the object + repeated RpbPair indexes = 10; // user metadata stored with the object } // Key/value pair - used for user metadata diff --git a/riak/transports/riakclient_pb2.py b/riak/transports/riakclient_pb2.py index 68f9b2e0..2715abff 100644 --- a/riak/transports/riakclient_pb2.py +++ b/riak/transports/riakclient_pb2.py @@ -10,7 +10,7 @@ DESCRIPTOR = descriptor.FileDescriptor( name='riakclient.proto', package='', - serialized_pb='\n\x10riakclient.proto\"/\n\x0cRpbErrorResp\x12\x0e\n\x06\x65rrmsg\x18\x01 \x02(\x0c\x12\x0f\n\x07\x65rrcode\x18\x02 \x02(\r\"\'\n\x12RpbGetClientIdResp\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"&\n\x11RpbSetClientIdReq\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"<\n\x14RpbGetServerInfoResp\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12\x16\n\x0eserver_version\x18\x02 \x01(\x0c\"3\n\tRpbGetReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\t\n\x01r\x18\x03 \x01(\r\":\n\nRpbGetResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\"\x82\x01\n\tRpbPutReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\x0e\n\x06vclock\x18\x03 \x01(\x0c\x12\x1c\n\x07\x63ontent\x18\x04 \x02(\x0b\x32\x0b.RpbContent\x12\t\n\x01w\x18\x05 \x01(\r\x12\n\n\x02\x64w\x18\x06 \x01(\r\x12\x13\n\x0breturn_body\x18\x07 \x01(\x08\":\n\nRpbPutResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\"4\n\tRpbDelReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\n\n\x02rw\x18\x03 \x01(\r\"%\n\x12RpbListBucketsResp\x12\x0f\n\x07\x62uckets\x18\x01 \x03(\x0c\" \n\x0eRpbListKeysReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"-\n\x0fRpbListKeysResp\x12\x0c\n\x04keys\x18\x01 \x03(\x0c\x12\x0c\n\x04\x64one\x18\x02 \x01(\x08\"!\n\x0fRpbGetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"2\n\x10RpbGetBucketResp\x12\x1e\n\x05props\x18\x01 \x02(\x0b\x32\x0f.RpbBucketProps\"A\n\x0fRpbSetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x1e\n\x05props\x18\x02 \x02(\x0b\x32\x0f.RpbBucketProps\"5\n\x0cRpbMapRedReq\x12\x0f\n\x07request\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x02(\x0c\">\n\rRpbMapRedResp\x12\r\n\x05phase\x18\x01 \x01(\r\x12\x10\n\x08response\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\"\xc9\x01\n\nRpbContent\x12\r\n\x05value\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x01(\x0c\x12\x0f\n\x07\x63harset\x18\x03 \x01(\x0c\x12\x18\n\x10\x63ontent_encoding\x18\x04 \x01(\x0c\x12\x0c\n\x04vtag\x18\x05 \x01(\x0c\x12\x17\n\x05links\x18\x06 \x03(\x0b\x32\x08.RpbLink\x12\x10\n\x08last_mod\x18\x07 \x01(\r\x12\x16\n\x0elast_mod_usecs\x18\x08 \x01(\r\x12\x1a\n\x08usermeta\x18\t \x03(\x0b\x32\x08.RpbPair\"%\n\x07RpbPair\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c\"3\n\x07RpbLink\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0b\n\x03tag\x18\x03 \x01(\x0c\"3\n\x0eRpbBucketProps\x12\r\n\x05n_val\x18\x01 \x01(\r\x12\x12\n\nallow_mult\x18\x02 \x01(\x08') + serialized_pb='\n\x10riakclient.proto\"/\n\x0cRpbErrorResp\x12\x0e\n\x06\x65rrmsg\x18\x01 \x02(\x0c\x12\x0f\n\x07\x65rrcode\x18\x02 \x02(\r\"\'\n\x12RpbGetClientIdResp\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"&\n\x11RpbSetClientIdReq\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"<\n\x14RpbGetServerInfoResp\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12\x16\n\x0eserver_version\x18\x02 \x01(\x0c\"\xa4\x01\n\tRpbGetReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\t\n\x01r\x18\x03 \x01(\r\x12\n\n\x02pr\x18\x04 \x01(\r\x12\x14\n\x0c\x62\x61sic_quorum\x18\x05 \x01(\x08\x12\x13\n\x0bnotfound_ok\x18\x06 \x01(\x08\x12\x13\n\x0bif_modified\x18\x07 \x01(\x0c\x12\x0c\n\x04head\x18\x08 \x01(\x08\x12\x15\n\rdeletedvclock\x18\t \x01(\x08\"M\n\nRpbGetResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\x12\x11\n\tunchanged\x18\x03 \x01(\x08\"\xd3\x01\n\tRpbPutReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0e\n\x06vclock\x18\x03 \x01(\x0c\x12\x1c\n\x07\x63ontent\x18\x04 \x02(\x0b\x32\x0b.RpbContent\x12\t\n\x01w\x18\x05 \x01(\r\x12\n\n\x02\x64w\x18\x06 \x01(\r\x12\x13\n\x0breturn_body\x18\x07 \x01(\x08\x12\n\n\x02pw\x18\x08 \x01(\r\x12\x17\n\x0fif_not_modified\x18\t \x01(\x08\x12\x15\n\rif_none_match\x18\n \x01(\x08\x12\x13\n\x0breturn_head\x18\x0b \x01(\x08\"G\n\nRpbPutResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\x12\x0b\n\x03key\x18\x03 \x01(\x0c\"~\n\tRpbDelReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\n\n\x02rw\x18\x03 \x01(\r\x12\x0e\n\x06vclock\x18\x04 \x01(\x0c\x12\t\n\x01r\x18\x05 \x01(\r\x12\t\n\x01w\x18\x06 \x01(\r\x12\n\n\x02pr\x18\x07 \x01(\r\x12\n\n\x02pw\x18\x08 \x01(\r\x12\n\n\x02\x64w\x18\t \x01(\r\"%\n\x12RpbListBucketsResp\x12\x0f\n\x07\x62uckets\x18\x01 \x03(\x0c\" \n\x0eRpbListKeysReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"-\n\x0fRpbListKeysResp\x12\x0c\n\x04keys\x18\x01 \x03(\x0c\x12\x0c\n\x04\x64one\x18\x02 \x01(\x08\"!\n\x0fRpbGetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"2\n\x10RpbGetBucketResp\x12\x1e\n\x05props\x18\x01 \x02(\x0b\x32\x0f.RpbBucketProps\"A\n\x0fRpbSetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x1e\n\x05props\x18\x02 \x02(\x0b\x32\x0f.RpbBucketProps\"5\n\x0cRpbMapRedReq\x12\x0f\n\x07request\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x02(\x0c\">\n\rRpbMapRedResp\x12\r\n\x05phase\x18\x01 \x01(\r\x12\x10\n\x08response\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\"\xe4\x01\n\nRpbContent\x12\r\n\x05value\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x01(\x0c\x12\x0f\n\x07\x63harset\x18\x03 \x01(\x0c\x12\x18\n\x10\x63ontent_encoding\x18\x04 \x01(\x0c\x12\x0c\n\x04vtag\x18\x05 \x01(\x0c\x12\x17\n\x05links\x18\x06 \x03(\x0b\x32\x08.RpbLink\x12\x10\n\x08last_mod\x18\x07 \x01(\r\x12\x16\n\x0elast_mod_usecs\x18\x08 \x01(\r\x12\x1a\n\x08usermeta\x18\t \x03(\x0b\x32\x08.RpbPair\x12\x19\n\x07indexes\x18\n \x03(\x0b\x32\x08.RpbPair\"%\n\x07RpbPair\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c\"3\n\x07RpbLink\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0b\n\x03tag\x18\x03 \x01(\x0c\"3\n\x0eRpbBucketProps\x12\r\n\x05n_val\x18\x01 \x01(\r\x12\x12\n\nallow_mult\x18\x02 \x01(\x08') @@ -169,6 +169,48 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='pr', full_name='RpbGetReq.pr', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='basic_quorum', full_name='RpbGetReq.basic_quorum', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='notfound_ok', full_name='RpbGetReq.notfound_ok', index=5, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='if_modified', full_name='RpbGetReq.if_modified', index=6, + number=7, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='head', full_name='RpbGetReq.head', index=7, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='deletedvclock', full_name='RpbGetReq.deletedvclock', index=8, + number=9, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -178,8 +220,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=212, - serialized_end=263, + serialized_start=213, + serialized_end=377, ) @@ -204,6 +246,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='unchanged', full_name='RpbGetResp.unchanged', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -213,8 +262,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=265, - serialized_end=323, + serialized_start=379, + serialized_end=456, ) @@ -234,7 +283,7 @@ options=None), descriptor.FieldDescriptor( name='key', full_name='RpbPutReq.key', index=1, - number=2, type=12, cpp_type=9, label=2, + number=2, type=12, cpp_type=9, label=1, has_default_value=False, default_value="", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -274,6 +323,34 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='pw', full_name='RpbPutReq.pw', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='if_not_modified', full_name='RpbPutReq.if_not_modified', index=8, + number=9, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='if_none_match', full_name='RpbPutReq.if_none_match', index=9, + number=10, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='return_head', full_name='RpbPutReq.return_head', index=10, + number=11, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -283,8 +360,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=326, - serialized_end=456, + serialized_start=459, + serialized_end=670, ) @@ -309,6 +386,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='key', full_name='RpbPutResp.key', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -318,8 +402,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=458, - serialized_end=516, + serialized_start=672, + serialized_end=743, ) @@ -351,6 +435,48 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='vclock', full_name='RpbDelReq.vclock', index=3, + number=4, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='r', full_name='RpbDelReq.r', index=4, + number=5, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='w', full_name='RpbDelReq.w', index=5, + number=6, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='pr', full_name='RpbDelReq.pr', index=6, + number=7, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='pw', full_name='RpbDelReq.pw', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='dw', full_name='RpbDelReq.dw', index=8, + number=9, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -360,8 +486,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=518, - serialized_end=570, + serialized_start=745, + serialized_end=871, ) @@ -388,8 +514,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=572, - serialized_end=609, + serialized_start=873, + serialized_end=910, ) @@ -416,8 +542,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=611, - serialized_end=643, + serialized_start=912, + serialized_end=944, ) @@ -451,8 +577,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=645, - serialized_end=690, + serialized_start=946, + serialized_end=991, ) @@ -479,8 +605,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=692, - serialized_end=725, + serialized_start=993, + serialized_end=1026, ) @@ -507,8 +633,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=727, - serialized_end=777, + serialized_start=1028, + serialized_end=1078, ) @@ -542,8 +668,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=779, - serialized_end=844, + serialized_start=1080, + serialized_end=1145, ) @@ -577,8 +703,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=846, - serialized_end=899, + serialized_start=1147, + serialized_end=1200, ) @@ -619,8 +745,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=901, - serialized_end=963, + serialized_start=1202, + serialized_end=1264, ) @@ -694,6 +820,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='indexes', full_name='RpbContent.indexes', index=9, + number=10, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -703,8 +836,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=966, - serialized_end=1167, + serialized_start=1267, + serialized_end=1495, ) @@ -738,8 +871,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=1169, - serialized_end=1206, + serialized_start=1497, + serialized_end=1534, ) @@ -780,8 +913,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=1208, - serialized_end=1259, + serialized_start=1536, + serialized_end=1587, ) @@ -815,8 +948,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=1261, - serialized_end=1312, + serialized_start=1589, + serialized_end=1640, ) @@ -827,6 +960,7 @@ _RPBSETBUCKETREQ.fields_by_name['props'].message_type = _RPBBUCKETPROPS _RPBCONTENT.fields_by_name['links'].message_type = _RPBLINK _RPBCONTENT.fields_by_name['usermeta'].message_type = _RPBPAIR +_RPBCONTENT.fields_by_name['indexes'].message_type = _RPBPAIR class RpbErrorResp(message.Message): __metaclass__ = reflection.GeneratedProtocolMessageType From 4a18c00646604fe793e2db9977ee43339e9147e1 Mon Sep 17 00:00:00 2001 From: Rusty Klophaus Date: Thu, 15 Sep 2011 09:21:28 -0400 Subject: [PATCH 0109/1060] Detect and return correct error message when MapReduce job fails. Also, correctly handle when there are no MapReduce results. AZ679 --- riak/mapreduce.py | 4 ++++ riak/transports/http.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index c7181d15..d2b956d1 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -234,6 +234,10 @@ def run(self, timeout=None): if not link_results_flag: return result + # If there are no results, then return an empty list. + if result == None: + return [] + # Otherwise, if the last phase IS a link phase, then convert the # results to RiakLink objects. a = [] diff --git a/riak/transports/http.py b/riak/transports/http.py index 2b2a7681..7c06d8c8 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -213,6 +213,12 @@ def mapred(self, inputs, query, timeout=None): # Do the request... url = "/" + self._mapred_prefix response = self.http_request('POST', url, {}, content) + + # Make sure the expected status code came back... + status = response[0]['http_code'] + if status != 200: + raise Exception('Error running MapReduce operation. Status: ' + str(status) + ' : ' + response[1]) + result = json.loads(response[1]) return result From 85e9d5460787d2ad6b66663e07b106f8cd71e05d Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Thu, 15 Sep 2011 16:40:57 -0400 Subject: [PATCH 0110/1060] Add a utility function for marking APIs/usage as deprecated. --- riak/util.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/riak/util.py b/riak/util.py index c2e154a0..0a0cf4cc 100644 --- a/riak/util.py +++ b/riak/util.py @@ -1,4 +1,5 @@ import collections +import warnings def quacks_like_dict(object): """Check if object is dict-like""" @@ -31,3 +32,6 @@ def deep_merge(a, b): current_dst[key] = current_src[key] return dst + +def deprecated(message, stacklevel=3): + warnings.warn(message, DeprecationWarning, stacklevel=stacklevel) From 9099cf72d196760dfc2a8c86192fed5d619f0487 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Thu, 15 Sep 2011 17:50:58 -0400 Subject: [PATCH 0111/1060] Add an API level to the transports, with deprecation warnings. * riak/transports/transport.py: (RiakTransport): document the 'api' classvar, but not enter a default. all subclasses must provide the default explicitly. * riak/transports/http.py: (RiakHttpTransport): notate the class is now API v2. * riak/transports/pbc.py: (RiakPbcTransport): notate the class is now API v2. * riak/client.py: (RiakClient.__init__): handle api < 2 transports * riak/search.py: (RiakSearch.__init__): if api < 2, then raise a deprecation warning. the old code would simply fail, so we are "allowed" to bail out. --- riak/client.py | 31 +++++++++++++++++++------------ riak/search.py | 14 +++++++++++--- riak/transports/http.py | 3 +++ riak/transports/pbc.py | 4 ++++ riak/transports/transport.py | 6 ++++++ 5 files changed, 43 insertions(+), 15 deletions(-) diff --git a/riak/client.py b/riak/client.py index ced72420..9b35b039 100644 --- a/riak/client.py +++ b/riak/client.py @@ -27,6 +27,7 @@ from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduce from riak.search import RiakSearch +from riak.util import deprecated import riak.transports.connection @@ -58,18 +59,24 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', if transport_class is None: transport_class = RiakHttpTransport - hostports = [ (host, port), ] - self._cm = transport_class.default_cm(hostports) - - ### we need to allow additional transport options. make this an - ### argument to __init__ ? - transport_options = { } - - self._transport = transport_class(self._cm, - prefix=prefix, - mapred_prefix=mapred_prefix, - client_id=client_id, - **transport_options) + api = getattr(transport_class, 'api', 1) + if api >= 2: + hostports = [ (host, port), ] + self._cm = transport_class.default_cm(hostports) + + ### we need to allow additional transport options. make this an + ### argument to __init__ ? + transport_options = { } + + self._transport = transport_class(self._cm, + prefix=prefix, + mapred_prefix=mapred_prefix, + client_id=client_id, + **transport_options) + else: + deprecated('please upgrade the transport to the new API') + self._cm = None + self._transport = transport_class(host, port, client_id=client_id) self._r = "default" self._w = "default" diff --git a/riak/search.py b/riak/search.py index 5e006c25..7fe2ab73 100644 --- a/riak/search.py +++ b/riak/search.py @@ -8,9 +8,17 @@ def __init__(self, client, transport_class=None, if transport_class is None: transport_class = RiakHttpTransport - hostports = [ (host, port), ] - self._cm = transport_class.default_cm(hostports) - self._transport = transport_class(self._cm, prefix="/solr") + api = getattr(transport_class, 'api', 1) + if api >= 2: + hostports = [ (host, port), ] + self._cm = transport_class.default_cm(hostports) + self._transport = transport_class(self._cm, prefix="/solr") + else: + # The old code which attempted to use api==1 would actually + # throw a NameError, so it was obviously never used. We will + # simply raise an error here, intead of a gentle warning. + raise DeprecationWarning('please upgrade the transport to the ' + 'new API') self._client = client self._decoders = {"text/xml": ElementTree.fromstring} diff --git a/riak/transports/http.py b/riak/transports/http.py index 30ca5d58..cf286a20 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -43,6 +43,9 @@ class RiakHttpTransport(RiakTransport) : connection, and the RiakClient object is extremely lightweight. """ + # We're using the new RiakTransport API + api = 2 + # The ConnectionManager class that this transport prefers. default_cm = HTTPConnectionManager diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 08d08a39..c28c2041 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -77,6 +77,10 @@ class RiakPbcTransport(RiakTransport): The RiakPbcTransport object holds a connection to the protocol buffers interface on the riak server. """ + + # We're using the new RiakTransport API + api = 2 + rw_names = { 'default' : RIAKC_RW_DEFAULT, 'all' : RIAKC_RW_ALL, diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 7d0be45f..5fb57be6 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -29,6 +29,12 @@ class RiakTransport(object): Class to encapsulate transport details """ + # Subclasses should specify their API level. + # * missing or 1: the API used up and through 1.3.x. + # * 2: the API introduced with 1.4.x + # + # api = 2 + @classmethod def make_random_client_id(self): """ From 2cb424989f6d3fac62affdc13cc9da1e7544514e Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Thu, 15 Sep 2011 21:44:54 -0400 Subject: [PATCH 0112/1060] First draft at fixing issue #53. Introduce transport.put_new() to deal with returning the key generated by the server (the signature for .put() has no place to return it). * riak/riak_object.py: (RiakObject.store): if the object does not (yet) have a key, then use the .put_new() method to save the object and get the new key. * riak/transports/transport.py: (RiakTransport.put_new): new abstract method for storing an object and allowing the server to generate the key * riak/transports/http.py: (RiakHttpTransport.put): factor the header creation out into ... (RiakHttpTransport.build_put_headers): ... this. (RiakHttpTransport.put_new): new method to store the new object and return the server-generated key * riak/transport/pbc.py: (RiakHttpTransport.put_new): add a draft stub for implementing put_new and returning the server-generated key (doesn't work). --- riak/riak_object.py | 16 +++++++--- riak/transports/http.py | 59 ++++++++++++++++++++++++------------ riak/transports/pbc.py | 10 ++++++ riak/transports/transport.py | 10 ++++++ 4 files changed, 72 insertions(+), 23 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index fe20dbdd..6bee76c5 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -309,11 +309,19 @@ def store(self, w=None, dw=None, return_body=True): w = self._bucket.get_w(w) dw = self._bucket.get_dw(w) - # Issue the get over our transport + # Issue the put over our transport t = self._client.get_transport() - Result = t.put(self, w, dw, return_body) - if Result is not None: - self.populate(Result) + + if self._key is None: + key, vclock, metadata = t.put_new(self, w, dw, return_body) + self._exists = True + self._key = key + self._vclock = vclock + self.set_metadata(metadata) + else: + Result = t.put(self, w, dw, return_body) + if Result is not None: + self.populate(Result) return self diff --git a/riak/transports/http.py b/riak/transports/http.py index 27a41aac..a28209c3 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -109,25 +109,7 @@ def put(self, robj, w = None, dw = None, return_body = True): params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw} url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), params=params) - - # Construct the headers... - headers = MultiDict({'Accept' : 'text/plain, */*; q=0.5', - 'Content-Type' : robj.get_content_type(), - 'X-Riak-ClientId' : self._client_id}) - - # Add the vclock if it exists... - if robj.vclock() is not None: - headers['X-Riak-Vclock'] = robj.vclock() - - # Create the header from metadata - links = self.add_links_for_riak_object(robj, headers) - - for key, value in robj.get_usermeta().iteritems(): - headers['X-Riak-Meta-%s' % key] = value - - for key, value in robj.get_indexes().iteritems(): - headers['X-Riak-Index-%s' % key] = value - + headers = self.build_put_headers(robj) content = robj.get_encoded_data() return self.do_put(url, headers, content, return_body, key=robj.get_key()) @@ -143,6 +125,22 @@ def do_put(self, url, headers, content, return_body=False, key=None): self.check_http_code(response, [204]) return None + def put_new(self, robj, w=None, dw=None, return_meta=True): + """Put a new object into the Riak store, returning its (new) key.""" + # Construct the URL... + params = {'returnbody' : str(return_meta).lower(), 'w' : w, 'dw' : dw} + url = self.build_rest_path(bucket=robj.get_bucket(), params=params) + headers = self.build_put_headers(robj) + content = robj.get_encoded_data() + response = self.http_request('POST', url, headers, content) + key = response[0]['location'] + if return_meta: + vclock, [(metadata, data)] = self.parse_body(response, [201]) + return key, vclock, metadata + else: + self.check_http_code(response, [201]) + return key, None, None + def delete(self, robj, rw): # Construct the URL... params = {'rw' : rw} @@ -410,6 +408,29 @@ def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : # Return. return path + def build_put_headers(self, robj): + """Build the headers for a POST/PUT request.""" + + # Construct the headers... + headers = MultiDict({'Accept' : 'text/plain, */*; q=0.5', + 'Content-Type' : robj.get_content_type(), + 'X-Riak-ClientId' : self._client_id}) + + # Add the vclock if it exists... + if robj.vclock() is not None: + headers['X-Riak-Vclock'] = robj.vclock() + + # Create the header from metadata + links = self.add_links_for_riak_object(robj, headers) + + for key, value in robj.get_usermeta().iteritems(): + headers['X-Riak-Meta-%s' % key] = value + + for key, value in robj.get_indexes().iteritems(): + headers['X-Riak-Index-%s' % key] = value + + return headers + def http_request(self, method, uri, headers=None, body='') : """ Given a Method, URL, Headers, and Body, perform and HTTP request, diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 0b0ffa1c..b1c3b97f 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -216,6 +216,16 @@ def put(self, robj, w = None, dw = None, return_body = True): contents.append(self.decode_content(c)) return resp.vclock, contents + def put_new(self, robj, w=None, dw=None, return_meta=True): + ### not sure about all this. just use put() for now. we need the + ### resp.key value from self.put(). maybe refactor. + response = self.put(robj, w, dw, return_meta) + if response is None: + return None, None, None + assert len(response[1]) == 1 + return None, response[0], response[1][0][0] + + def delete(self, robj, rw = None): """ Serialize get request and deserialize response diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 5fb57be6..4454f48d 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -75,6 +75,16 @@ def put(self, robj, w = None, dw = None, return_body = True): """ raise RiakError("not implemented") + def put_new(self, robj, w=None, dw=None, return_meta=True): + """Put a new object into the Riak store, returning its (new) key. + + If return_meta is False, then the vlock and metadata return values + will be None. + + @return (key, vclock, metadata) + """ + raise RiakError("not implemented") + def delete(self, robj, rw = None): """ Serialize delete request and deserialize response From b264f823d804101cdd32bb403932546ebcaac64f Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 16 Sep 2011 11:40:49 -0400 Subject: [PATCH 0113/1060] Test the retrieval of a new object's key, and its format. --- riak/tests/test_all.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6825d24e..ac94fd67 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -828,7 +828,11 @@ def test_generate_key(self): bucket = self.client.bucket('random_key_bucket') for key in bucket.get_keys(): bucket.get(str(key)).delete() - bucket.new(None, data={}).store() + o = bucket.new(None, data={}) + self.assertIsNone(o.get_key()) + o.store() + self.assertIsNotNone(o.get_key()) + self.assertNotIn('/', o.get_key()) self.assertEqual(len(bucket.get_keys()), 1) def test_too_many_link_headers_shouldnt_break_http(self): From 6cfd296a86c1b475101c179a45a7453b76dcbfd5 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Mon, 12 Sep 2011 21:08:52 -0400 Subject: [PATCH 0114/1060] Adjust for compatibility with Python 2.5 --- riak/util.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/riak/util.py b/riak/util.py index c2e154a0..39f1a210 100644 --- a/riak/util.py +++ b/riak/util.py @@ -1,8 +1,12 @@ -import collections +try: + from collections import Mapping +except ImportError: + # compatibility with Python 2.5 + Mapping = dict def quacks_like_dict(object): """Check if object is dict-like""" - return isinstance(object, collections.Mapping) + return isinstance(object, Mapping) def deep_merge(a, b): """Merge two deep dicts non-destructively From 225335a68f79a6e89118d16346d1c4e7ebcb8a63 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 16 Sep 2011 11:43:02 -0400 Subject: [PATCH 0115/1060] Remove any leftover objects from a failed test_store_of_missing_object. --- riak/tests/test_all.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 50914fa3..45a76bfc 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -92,6 +92,13 @@ def create_client(self, host=None, port=None, transport_class=None): def setUp(self): self.client = self.create_client() + # make sure these are not left over from a previous, failed run + bucket = self.client.bucket('bucket') + o = bucket.get('nonexistent_key_json') + o.delete() + o = bucket.get('nonexistent_key_binary') + o.delete() + def test_is_alive(self): self.assertTrue(self.client.is_alive()) From 127492c44ce63db9e21f3e136c45479c588ea789 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 16 Sep 2011 12:09:16 -0400 Subject: [PATCH 0116/1060] Fix extraction of the key from the Location: header. --- riak/transports/http.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index a28209c3..9e61d545 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -133,7 +133,9 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): headers = self.build_put_headers(robj) content = robj.get_encoded_data() response = self.http_request('POST', url, headers, content) - key = response[0]['location'] + location = response[0]['location'] + idx = location.rindex('/') + key = location[idx+1:] if return_meta: vclock, [(metadata, data)] = self.parse_body(response, [201]) return key, vclock, metadata From db03c077469d0d6f4f7fb80153806d56a72837bb Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 16 Sep 2011 12:23:03 -0400 Subject: [PATCH 0117/1060] Add RiakPbcCachedTransport.put_new() and some missing docstrings. --- riak/transports/pbc.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index b1c3b97f..cf7f0cdb 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -217,6 +217,13 @@ def put(self, robj, w = None, dw = None, return_body = True): return resp.vclock, contents def put_new(self, robj, w=None, dw=None, return_meta=True): + """Put a new object into the Riak store, returning its (new) key. + + If return_meta is False, then the vlock and metadata return values + will be None. + + @return (key, vclock, metadata) + """ ### not sure about all this. just use put() for now. we need the ### resp.key value from self.put(). maybe refactor. response = self.put(robj, w, dw, return_meta) @@ -604,6 +611,17 @@ def put(self, robj, w = None, dw = None, return_body = True): with self._get_connection_from_pool() as connection: return connection.put(robj, w, dw, return_body) + def put_new(self, robj, w=None, dw=None, return_meta=True): + """Put a new object into the Riak store, returning its (new) key. + + If return_meta is False, then the vlock and metadata return values + will be None. + + @return (key, vclock, metadata) + """ + with self._get_connection_from_pool() as connection: + return connection.put_new(robj, w, dw, return_meta) + def delete(self, robj, rw = None): """ Serialize delete request and deserialize response From 3f28c3087a867cc187dece98521d252995f54920 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Fri, 16 Sep 2011 12:30:03 -0400 Subject: [PATCH 0118/1060] Properly implement RiakPbcTransport.put_new() --- riak/transports/pbc.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index cf7f0cdb..ccd3d8d7 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -224,14 +224,30 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): @return (key, vclock, metadata) """ - ### not sure about all this. just use put() for now. we need the - ### resp.key value from self.put(). maybe refactor. - response = self.put(robj, w, dw, return_meta) - if response is None: - return None, None, None - assert len(response[1]) == 1 - return None, response[0], response[1][0][0] - + bucket = robj.get_bucket() + + req = riakclient_pb2.RpbPutReq() + req.w = self.translate_rw_val(w) + req.dw = self.translate_rw_val(dw) + if return_meta: + req.return_body = 1 + + req.bucket = bucket.get_name() + + self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) + + self.maybe_connect() + self.send_msg(MSG_CODE_PUT_REQ, req) + msg_code, resp = self.recv_msg() + if msg_code != MSG_CODE_PUT_RESP: + raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + if not resp: + raise RiakError("missing response object") + if len(resp.content) != 1: + raise RiakError("siblings were returned from object creation") + + metadata, content = self.decode_content(resp.content[0]) + return resp.key, resp.vclock, metadata def delete(self, robj, rw = None): """ From 75d8853289ad22749f035206497028a6e4616383 Mon Sep 17 00:00:00 2001 From: David Koblas Date: Fri, 16 Sep 2011 11:58:30 -0700 Subject: [PATCH 0119/1060] Fix the typo where w was used rather than dw as the default value for dw --- riak/riak_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index fe20dbdd..26d342af 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -307,7 +307,7 @@ def store(self, w=None, dw=None, return_body=True): """ # Use defaults if not specified... w = self._bucket.get_w(w) - dw = self._bucket.get_dw(w) + dw = self._bucket.get_dw(dw) # Issue the get over our transport t = self._client.get_transport() From aacedcc9580c4a0c5c8d84aea141310b30a3ad3b Mon Sep 17 00:00:00 2001 From: Jeffrey Massung Date: Mon, 26 Sep 2011 15:26:55 -0600 Subject: [PATCH 0120/1060] Perform a dummy 2i query to see if an exception detailing indexes_not_supported and, if so, don't perform the request of the test. --- riak/tests/test_all.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6825d24e..af397c3d 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -546,6 +546,13 @@ def test_secondary_index_query(self): bucket.new('mykey3', 'data3').set_indexes({'field1_bin':'val3', 'field2_int':1003}).store() bucket.new('mykey4', 'data4').set_indexes({'field1_bin':'val4', 'field2_int':1004}).store() + # Immediate test to see if 2i is even supported w/ the backend + try: + self.client.index('foo','bar_bin','baz').run() + except Exception as e: + if "indexes_not_supported" in str(e): + return True + # Test an equality query... results = self.client.index('indexbucket', 'field1_bin', 'val2').run() self.assertEquals(1, len(results)) From f62b7ef46fb45a58328a29e867fa5af2a908a169 Mon Sep 17 00:00:00 2001 From: Rusty Klophaus Date: Fri, 30 Sep 2011 10:12:00 -0400 Subject: [PATCH 0121/1060] Update 2i support to handle multi-valued index entries. AZ789 --- docs/tutorial.rst | 18 +++++--- riak/riak_index_entry.py | 62 ++++++++++++++++++++++++++++ riak/riak_object.py | 53 ++++++++++++++++++------ riak/tests/test_all.py | 89 +++++++++++++++++++++++++++++++++++----- riak/transports/http.py | 21 +++++++--- riak/transports/pbc.py | 12 +++--- 6 files changed, 216 insertions(+), 39 deletions(-) create mode 100644 riak/riak_index_entry.py diff --git a/docs/tutorial.rst b/docs/tutorial.rst index f7a3cefa..ab780da7 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -395,19 +395,20 @@ Secondary Indexes is a new feature available as of Riak 1.0. It allows you to tag an object with index metadata, and then later find the object by querying the metadata, returning a list of matching keys. -Usage of this feature looks like:: +Your Riak cluster must have Secondary Indexes enabled. See the Riak +documentation for details. + +Usage of this feature looks like: import riak client = riak.RiakClient() bucket = client.bucket('mybucket') - # Store the object... + # Create and store the object with indexes... obj = bucket.new('mykey1', 'mydata') - obj.set_indexes({ - 'field1_bin': 'val1', - 'field2_int': 1001 - }) + obj.add_index('field1_bin', 'val1') + obj.add_index('field2_int', 1001) obj.store() # Query the indexes. The return value is a list of ``RiakLink`` objects. @@ -415,3 +416,8 @@ Usage of this feature looks like:: # Query the indexes using a range... results = client.index('mybucket', 'field1_bin', 'val1', 'val5').run() + + # Remove an index entry... + obj = bucket.get('mykey1') + obj.remove_index('field1_bin', 'val1') + obj.store() diff --git a/riak/riak_index_entry.py b/riak/riak_index_entry.py new file mode 100644 index 00000000..0a26bb13 --- /dev/null +++ b/riak/riak_index_entry.py @@ -0,0 +1,62 @@ +""" +Copyright 2010 Rusty Klophaus +Copyright 2010 Justin Sheehy +Copyright 2009 Jay Baird + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +class RiakIndexEntry: + def __init__(self, field, value): + self._field = field + self._value = str(value) + + def get_field(self): + return self._field + + def get_value(self): + return self._value + + def __str__(self): + return "RiakIndexEntry(field = '%s', value='%s')" % (self._field, self._value) + + def __eq__(self, other): + if not isinstance(other, RiakIndexEntry): + return False + + return \ + self.get_field() == other.get_field() and \ + self.get_value() == other.get_value() + + def __cmp__(self, other): + if other == None: + raise TypeError("RiakIndexEntry cannot be compared to None") + + if not isinstance(other, RiakIndexEntry): + raise TypeError("RiakIndexEntry cannot be compared to %s" % other.__class__.__name__) + + if self.get_field() < other.get_field(): + return -1 + + if self.get_field() > other.get_field(): + return 1 + + if self.get_value() < other.get_value(): + return -1 + + if self.get_value() > other.get_value(): + return 1 + + return 0 diff --git a/riak/riak_object.py b/riak/riak_object.py index 26d342af..7ad05032 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -20,6 +20,7 @@ import types, copy, re from metadata import * from riak import RiakError +from riak.riak_index_entry import RiakIndexEntry class RiakObject(object): """ @@ -47,7 +48,7 @@ def __init__(self, client, bucket, key=None): self._encode_data = True self._vclock = None self._data = None - self._metadata = {MD_USERMETA: {}, MD_INDEX: {}} + self._metadata = {MD_USERMETA: {}, MD_INDEX: []} self._links = [] self._siblings = [] self._exists = False @@ -177,24 +178,50 @@ def set_usermeta(self, usermeta): self._metadata[MD_USERMETA] = usermeta return self - def get_indexes(self): - if MD_INDEX in self._metadata: - return self._metadata[MD_INDEX] - else: - return {} + def add_index(self, field, value): + """ + Tag this object with the specified field/value pair for indexing. - def set_indexes(self, indexes): + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: self """ - Sets the field/value indexes under which this object will be - indexed. + rie = RiakIndexEntry(field, value) + if not rie in self._metadata[MD_INDEX]: + self._metadata[MD_INDEX].append(rie) - :param indexes: The field/value index data. - :type indexes: dict - :rtype: data + return self + + def remove_index(self, field, value): + """ + Remove the specified field/value pair as an index on this object. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: self """ - self._metadata[MD_INDEX] = indexes + rie = RiakIndexEntry(field, value) + if rie in self._metadata[MD_INDEX]: + self._metadata[MD_INDEX].remove(rie) return self + def get_indexes(self, field = None): + """ + Get a list of the index entries for this object. If a field is provided, returns a list + + :param field: The index field. + :type field: string or None + :rtype: (array of RiakIndexEntry) or (array of string or integer) + """ + if field == None: + return self._metadata[MD_INDEX] + else: + return [x.get_value() for x in self._metadata[MD_INDEX] if x.get_field() == field] + def exists(self): """ Return True if the object exists, False otherwise. Allows you to diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index af397c3d..ba0d9ccb 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -22,6 +22,7 @@ from riak import RiakPbcTransport, RiakPbcCachedTransport from riak import RiakHttpTransport, RiakHttpPoolTransport, RiakHttpReuseTransport from riak import RiakKeyFilter, key_filter +from riak.riak_index_entry import RiakIndexEntry from riak.mapreduce import RiakLink from riak.test_server import TestServer @@ -524,16 +525,67 @@ def test_secondary_index_store(self): bucket = self.client.bucket('indexbucket') rand = self.randint() obj = bucket.new('mykey1', rand) - obj.set_indexes({ - 'field1_bin': 'val1', - 'field2_int': 1001 - }) + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_int', 1011) obj.store() # Retrieve the object, check that the correct indexes exist... obj = bucket.get('mykey1') - self.assertEqual('val1', obj.get_indexes()['field1_bin']) - self.assertEqual(1001, int(obj.get_indexes()['field2_int'])) + self.assertEqual(['val1a'], sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1011'], sorted(obj.get_indexes('field1_int'))) + + # Add more indexes and save... + obj.add_index('field1_bin', 'val1b') + obj.add_index('field1_int', 1012) + obj.store() + + # Retrieve the object, check that the correct indexes exist... + obj = bucket.get('mykey1') + self.assertEqual(['val1a', 'val1b'], sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1011', '1012'], sorted(obj.get_indexes('field1_int'))) + + # Check the get_indexes() function... + self.assertEqual([ + RiakIndexEntry('field1_bin', 'val1a'), + RiakIndexEntry('field1_bin', 'val1b'), + RiakIndexEntry('field1_int', 1011), + RiakIndexEntry('field1_int', 1012) + ], sorted(obj.get_indexes())) + + # Delete an index... + obj.remove_index('field1_bin', 'val1a') + obj.remove_index('field1_int', 1011) + obj.store() + + # Retrieve the object, check that the correct indexes exist... + obj = bucket.get('mykey1') + self.assertEqual(['val1b'], sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1012'], sorted(obj.get_indexes('field1_int'))) + + # Check duplicate entries... + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_int', 1011) + obj.add_index('field1_int', 1011) + obj.add_index('field1_int', 1011) + + self.assertEqual([ + RiakIndexEntry('field1_bin', 'val1a'), + RiakIndexEntry('field1_bin', 'val1b'), + RiakIndexEntry('field1_int', 1011), + RiakIndexEntry('field1_int', 1012) + ], sorted(obj.get_indexes())) + + obj.store() + obj = bucket.get('mykey1') + + self.assertEqual([ + RiakIndexEntry('field1_bin', 'val1a'), + RiakIndexEntry('field1_bin', 'val1b'), + RiakIndexEntry('field1_int', 1011), + RiakIndexEntry('field1_int', 1012) + ], sorted(obj.get_indexes())) # Clean up... bucket.get('mykey1').delete() @@ -541,10 +593,27 @@ def test_secondary_index_store(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_query(self): bucket = self.client.bucket('indexbucket') - bucket.new('mykey1', 'data1').set_indexes({'field1_bin':'val1', 'field2_int':1001}).store() - bucket.new('mykey2', 'data2').set_indexes({'field1_bin':'val2', 'field2_int':1002}).store() - bucket.new('mykey3', 'data3').set_indexes({'field1_bin':'val3', 'field2_int':1003}).store() - bucket.new('mykey4', 'data4').set_indexes({'field1_bin':'val4', 'field2_int':1004}).store() + + bucket.\ + new('mykey1', 'data1').\ + add_index('field1_bin', 'val1').\ + add_index('field2_int', 1001).\ + store() + bucket.\ + new('mykey2', 'data1').\ + add_index('field1_bin', 'val2').\ + add_index('field2_int', 1002).\ + store() + bucket.\ + new('mykey3', 'data1').\ + add_index('field1_bin', 'val3').\ + add_index('field2_int', 1003).\ + store() + bucket.\ + new('mykey4', 'data1').\ + add_index('field1_bin', 'val4').\ + add_index('field2_int', 1004).\ + store() # Immediate test to see if 2i is even supported w/ the backend try: diff --git a/riak/transports/http.py b/riak/transports/http.py index 7c06d8c8..456b91b5 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -17,7 +17,7 @@ specific language governing permissions and limitations under the License. """ -import urllib, re +import urllib, re, csv from cStringIO import StringIO import httplib try: @@ -29,6 +29,7 @@ from riak.metadata import * from riak.mapreduce import RiakLink from riak import RiakError +from riak.riak_index_entry import RiakIndexEntry from riak.multidict import MultiDict MAX_LINK_HEADER_SIZE = 8192 - 8 # substract length of "Link: " header string and newline @@ -113,8 +114,12 @@ def put(self, robj, w = None, dw = None, return_body = True): for key, value in robj.get_usermeta().iteritems(): headers['X-Riak-Meta-%s' % key] = value - for key, value in robj.get_indexes().iteritems(): - headers['X-Riak-Index-%s' % key] = value + for rie in robj.get_indexes(): + key = 'X-Riak-Index-%s' % rie.get_field() + if key in headers: + headers[key] += ", " + rie.get_value() + else: + headers[key] = rie.get_value() content = robj.get_encoded_data() return self.do_put(url, headers, content, return_body, key=robj.get_key()) @@ -270,7 +275,7 @@ def parse_body(self, response, expected_statuses): # Parse the headers... vclock = None - metadata = {MD_USERMETA: {}, MD_INDEX: {}} + metadata = {MD_USERMETA: {}, MD_INDEX: []} links = [] for header, value in headers.iteritems(): if header == 'content-type': @@ -288,7 +293,13 @@ def parse_body(self, response, expected_statuses): elif header.startswith('x-riak-meta-'): metadata[MD_USERMETA][header.replace('x-riak-meta-', '')] = value elif header.startswith('x-riak-index-'): - metadata[MD_INDEX][header.replace('x-riak-index-', '')] = value + field = header.replace('x-riak-index-', '') + reader = csv.reader([value], skipinitialspace=True) + for line in reader: + for token in line: + rie = RiakIndexEntry(field, token) + metadata[MD_INDEX].append(rie) + elif header == 'x-riak-vclock': vclock = value if links: diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 9c94d769..9b84076d 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -30,6 +30,7 @@ from riak.metadata import * from riak.mapreduce import RiakMapReduce, RiakLink from riak import RiakError +from riak.riak_index_entry import RiakIndexEntry try: import riakclient_pb2 @@ -471,9 +472,10 @@ def decode_content(self, rpb_content): usermeta[usermd.key] = usermd.value if len(usermeta) > 0: metadata[MD_USERMETA] = usermeta - indexes = {} + indexes = [] for index in rpb_content.indexes: - indexes[index.key] = index.value + rie = RiakIndexEntry(index.key, index.value) + indexes.append(rie) if len(indexes) > 0: metadata[MD_INDEX] = indexes return metadata, rpb_content.value @@ -494,10 +496,10 @@ def pbify_content(self, metadata, data, rpb_content) : pair.key = uk pair.value = uv elif k == MD_INDEX: - for uk, uv in v.iteritems(): + for rie in v: pair = rpb_content.indexes.add() - pair.key = uk - pair.value = str(uv) + pair.key = rie.get_field() + pair.value = rie.get_value() elif k == MD_LINKS: for link in v: pb_link = rpb_content.links.add() From 71fdc33f5f2bf6e41e87e8b2e3c255fbae91c701 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Tue, 4 Oct 2011 14:05:56 -0400 Subject: [PATCH 0122/1060] Add retry logic to the HTTP request method. This also closes connections whenever an error occurs, for safety. This ensures the HTTPConnection object returns to a usable state. --- riak/transports/http.py | 55 +++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 27a41aac..22578d62 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -22,6 +22,8 @@ import urllib, re from cStringIO import StringIO import httplib +import socket +import errno try: import json except ImportError: @@ -49,6 +51,9 @@ class RiakHttpTransport(RiakTransport) : # The ConnectionManager class that this transport prefers. default_cm = HTTPConnectionManager + # How many times to retry a request + RETRY_COUNT = 3 + def __init__(self, cm, prefix='riak', mapred_prefix='mapred', client_id=None, **unused_options): @@ -419,22 +424,40 @@ def http_request(self, method, uri, headers=None, body='') : if headers is None: headers = {} # Run the request... - with self._conns.withconn() as conn: - conn.request(method, uri, body, headers) - response = conn.getresponse() - - try: - # Get the response headers... - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value - - # Get the body... - response_body = response.read() - finally: - response.close() - - return response_headers, response_body + for retry in range(self.RETRY_COUNT): + with self._conns.withconn() as conn: + ### should probably build this try/except into a custom + ### contextmanager for the connection. + try: + conn.request(method, uri, body, headers) + response = conn.getresponse() + + try: + # Get the response headers... + response_headers = {'http_code': response.status} + for (key, value) in response.getheaders(): + response_headers[key.lower()] = value + + # Get the body... + response_body = response.read() + finally: + response.close() + + return response_headers, response_body + except socket.error, e: + conn.close() + if e[0] == errno.ECONNRESET: + # Grab another connection and try again. + continue + # Don't know how to handle this. + raise + except httplib.HTTPException: + # Just close the connection and try again. + conn.close() + continue + + # No luck, even with retrying. + raise RiakError("could not get a response") @classmethod def build_headers(cls, headers): From b36ecb34708f59cf755a48998ab18ca2e846aa5d Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 18 Oct 2011 16:10:58 +0200 Subject: [PATCH 0123/1060] Copy erl_src/* from Ruby client --- erl_src/riak_kv_test_backend.beam | Bin 7160 -> 5984 bytes erl_src/riak_kv_test_backend.erl | 602 ++++++++++++++++++++------ erl_src/riak_search_test_backend.beam | Bin 8484 -> 4440 bytes erl_src/riak_search_test_backend.erl | 6 +- 4 files changed, 478 insertions(+), 130 deletions(-) diff --git a/erl_src/riak_kv_test_backend.beam b/erl_src/riak_kv_test_backend.beam index f5a61ce9315fc57dd32fe28e7056f968f25d1a86..485d6b75110592ff546372210b389188cc490b74 100644 GIT binary patch literal 5984 zcmbVQTWlNGnV!)ZiX%!V9&sqi@g*L67|Ss&YZTHKN|AlUk;OQ^gchAf&5fkUkxYvs zY0gN#)GmzJ1r+Q)IJFnshdv0&1`90qA;`n-zHK3F7uaH36lnTTbRU8Z7TAZP2-*fH ziazxFheM0b1!zm)*E#1u|M}1NU(SDyX71(r2}v4R$efwEFyl2=C5iouBuNXgR<*QJ zSXnQ4j_VZ`OXU@(UJ11v*YSq^sdA&`6k65#a$&VpFD*N*{$LJ%FwLIo|wa9AeMq$-iZM1Iot<@{eQnl_>dYx9SR9}`4 z@2J*22b&bUMxj=9y#e||gAy<^$8%ZTxlYPFM{&JIv$x{Z9M56E)LUybR@Rz5o*P;! z)m%sMT5C?oX|)=yzIwweEH$tlYp!{$-ncF&+v{}u+Ug)TliEvu!SI2{cZ3C@>Kt7qp}9Cki&KmkgjoTY}>N& zGHkOSGn}v<$NgiV}OgGbfNS_2586C@Y%?2Y0M$*V6rFVEqB9(;28xf;1 zt$Em5&e#TQw(ezY@i1fOVCqO7;oAHVxt5HH_u);bnicG_cvunlJmKCK2!J26D(q7Lh|XatL{3lru@;Cb(h4eZ{+!(0*d3 zM1EEzFDLkc@|JLQ;j(ESm(>HBbyv7CQTDSog{;lDv-Vj(Yg7DLWUh(KH7Rp7h5q5XHlfogK)x80H82gwa|JW*M2HsioC)2=uo zVzvSP)Zz&`H3^)WUHlME&U3PQh_;TSGO=YT2u$Q)J2EH<>T_Y~lYkuq*B9Kdq_Fi_ znjeOP4l@JORDwDM3Kdm7OlT7n!6ShD93YRS`4K=q(hd0tK^8?MHU-YqMdUD5T9HqM z4OK(xv>j@2J1D%qA+fLlLo71H@{j*f7&9)bNBxryU$YHC_`aNFKIVf?0(44NliiTM5_GCt;%h)3BhbeI`lu$W zC(`^l0(1Pyiy^^})2(0#s@%wjq!QND!6W>nvgLc3MzVTRvrtp0gvO!p(&2 zomRHAoL?Ct$Fl5<-oyOrJl#$Np%da5-HbRsk>kDuk`4v%SXMb>qLNT8KZSiySh9Lr zv#hoyCb+oXBPW4oW|?F%7^MwawG(_+-&!U^XXz*-;Y_wnS$_r+laQE8@GK+*SlMlf z9ELAKVha156=sqGon*^JuwMfE1!T@St`)_cVdG)|Mm=k1O&9N>6i_uM@=o^}br6bx215YSTE3r|jRQ>-^?JXxXDvfHRQl~A1ilY)k zWOp*9?cvK%IR_J8dD22NYs0*YFi%#$0g){5zIul*g(853tFCq9l zC6eTCdY|CTvpBBx*XZ(4m`!CSmc*5T5*fms0#)CH$`@hCYmZ-LIxZJHVbrG+{B;(2 zNg(kkbgxqw_fr|5jInSX`p+&J2Spm?4q?E1<&B{7zlqq;)69RE#WxJNhH80;LL;kh zr1+aC)HE9cM7aM=I&NTaF+jfv}3JYc~n*%`t5MS@la zvJSHDm&iF_8Y*u1WiVfcIaeO1@I~u0S*?PdLwv8Mc@>6LeZy?L zp7UVXVAyzD7xA1a<2eKLD`gZm z8rD!>Ye{I#;bv#_IwVbHXmZJP*AA4`p<^DX8fjjKj(Qsv20&CNM7hobE5Vz2lM)Lt zE$injA4{`?^mol`dAg zNrN6Hn9(uNjRBFv>oG=Wul6{sCuxa;p1+ilMacX${W4PJzDxh_3?T%>m1zazmv)zI}m{IGrp! zh1(bSE!c6(x1(LOZ&BI#7L+V#vU>X@zrzMJZH6&a-Y@;9LKjB+klcpwofQ8T@)7@l z<-OKrG}77CCR+JEx(0}*6cGJ-LHOTHl0h8w6sRZY??Bt()BkqV9|T1~Bt!inP=`Mf z@JY`zAo@{%AE=|#=<4rBpRjcFkW3u(EQtD}Ali$5By$Kv@(IupPzS>}`W+0<2YlK$ z*(F2#j?8qx7iecdq|*k?f;u?Q1by1~EU2^g9Qq_r`X>TD^)G&*-@*S1_?`YYsDI`L z#X|ZI5c%{1h;%OneKb-9bOJ>CoCc*pB=Z8u3i^|1j|P2`9Rn#Kvh%s1PrL&`pL|KN zBANZ5y`T_iCukT%dl8-(i1z9S^??XG<%bHQAFUxC{d(rs>SW75gGlC85b3!Iq8|(W z?E?9lWQgy9NVhEC`UT14LBwx?NT)3SBK2Fe^B}_60MUGpGWy#8puH4~SJBeE+;c_W zLYoW5SJC?T*zd-Fh5pMRT7LyZdCq!w_r89<>Jgp=(Cat8UoIAlOU2@YD?cf|RQYfD zqn-Jk{}%n5#Baa*>Cb-S&VO|GzM1Fh?lm(G5XwNwi`HONf10osP^Aw0MkZ$S|pYoXUjN~cTr$8h_ zXMl2!WJq5+=-X&df(XORjOS4-FWjr)Z_kyUip194It}EyCjH~bpZ;NL_TN?S?ahDq z&v$o_0sUSF?5%fgMOhcCJA_D1FF zl@;mv@mKJ*&>eT3Qhj{ZS$ArUCVf&JZ&jPknuDgaGFGltiG{Dc1-DfuZM}^(uTpLO zRqK5#!oip26)BWDdg5q`lq&R@@k;L^&c-iGeWhq=s5Dv{^0#vvYpt@gBAxro;rb?g S@e}z`*g}4qS#&)#(w_j{AzNes literal 7160 zcmbVQ2UL^Uwodp136URu2)zbEl@>@u0t%r91Qh{$NkRx12ni$s#I9f&%ZOvaf?`3` z5fx-q22l}lY$&!-Y$%}EuwfrZefvc4a_^n>)_N~%eV%>x*=P524!_YO6a7den^)mc zVZ*~T$}AE|C!9ngg`28m;!I&?o=_uIYlJCcNv2ehs;kzBRT_G_R3TJLRe4esUCR*4 zWr|E)l~gU&=t)&_u_B$WlBUbl8mUT}sw)RVt(&Hlr^<1aR3eq-NrhTj9c89os#Gr3 zNGURfOk*TdXr$>6lL(Oe9kQPfg-8m&(;Ce^4hg)|?zq>&DoDpsV* zr9z2VtugtoQz(|ph02r+sYIiuDwHZ|w!A=Bs!}Oc{}PtVv2>|sfYwS%E`+FoC_s*a zyv4F}H8h!8D9y^&6i`xC%4`Y}q^LDOsb$lV*J(;@XwXoK1Z>gNsyv}~O{2<{Qqc@_ z>&cbM%-n446!huI6p%O#9j$(MQ@ai6)y)!XBvXyhN6et-IWh$fsQj#tmkflt`g~T8UBk)JqZ*J!OBWrTGFULDASK-a} zW_W9h56c*f zPr-~SR5lZM6ap8*RF{wvfFMAEJlA3e##@60t-+Y#7>N;% z5iHL&j6^d&g|V0*@mUy^$%g$TaT%yA!B{2u&y$pMe%&s-NBKC!a;3GUp<F}a;6F;4|A$o6KT28jc@&N@lYV%`r`zL} zbO+puOtUm#iue&2WS*a7nojU=2TBlt9ZrGbQ&9387?3v=CD1%T_dZ6(w5{=PiL?FRRls)Q6~QM(v!Tp(&`jDw)9efgc5g)p zeCR(qZr@+T`5#3b{?#DNUXmcw9HCcOFgo$+EXD{X4M-NS++lR);jUN^fc@i~;jWNs z0}}EEJoUs~FuDgEP7@O3&LsvT(_E3}5kwW4<_219pc&n@ZORhRZ3K{?UK1X$I1m;G z@^Ei3;jK0C{fOY+EHDAbk)u6+Oqy{Z?x{VmJlqS8jTeIn$A$yPCW4E5LG`^j6oQ@b z{qRL-+tU*kFus8Oy;;2oHh~Tj-3m@pZv%V(aMK4oabS~az675g&E^Q`4g!XN#pm=o z3Rf203zq$OI0qc%XdTraAWxjbp_}0(WSY~v2ila%q?vf){{8aekMhF#VUVk1(g=|c z>P<^yzpMm+F&|hP%)@zLjMqOaJd_m@$_iM3t_$pEArM*M{jd-OYMh{c!F;+?Zx};> zCC=EqQ!8d*&9tPE5aRTvBRv0TH ziHnCO5zK_B2nGY3(}R6vS{N`!}Gyx%ktCR&*M0zj*bSoEL05UBGOv7yxtG#Wo@3&1XocF$m zrhl$tQLf+uhBOa@xR@};@#)Oo3d94;5m>|dbjM!S2w>T>$+VGut!J-oBl;k1`>|I{B4j{a@6zzk?(LQK!U2+&u87u8uj|)Lm#Kote zl#5|&Cxec0=m3z6A%NUV8^PGeM}PP9@r{8- zKO}DgKr#$~X+PQm=stO?UK+{t*+sMqpijSRFOB%fAG8A z`6w>`Ub;^Y8j;VzfIi<4jr60nkX{vzLy8(H4O8L;#!sNCw5w zmxDn-I{=VQUw{n&$-4ts03?t0Sp$#^s-HOk$)MWf03?I-=>pIX^$=RuB05)rV(0eG zLwm;*7{C9va!?(H0FeI40HlwsYp~^gyi9}kW&@T@E0u^uA^?$fQ8Y{}8ty&toG*;N zXPbv#@bxASqjA$YuU40vmo&M#xTf#y+Ce(ibYcl_l}XZ)#8va2&a++`XA+kCCkp1OJNXadQ0Ro#Db4xp;{$D+VqsUnpBzU|Fu7rL{F8b6!_`k+x@zg{{&P`9F_X(xW*W@pWja=F!psNWRm-_wDDc+gCUYeh_e$%M|1t%gKTh}hDZlMiuEDt{Wi%UpE$jFfBy7Y=8 zdw!h}-$O}Cn*45$aP-)3H^)EuKo>^7V4R-(qG{#C<(uBMr)^F6*g5R1XQ{d~u&3mR zpZars`0lR3H-~-8Xg{y~X7cI9uQhcqnkpH==M}kYblVPlWz@||y;Cw}>mB-0#j1h@ z_gbTkzucR?V5{^{$G*>E=VB?V``qz}b7g;yzpokh+eN2sA7&WNe|Jl4y7bb;{jb(h zgQ{&#DUN?Ue(k4|0fLL)yswXU^*VH>SK}_=qwl6AHLVIUjI2Dl+{~Ktri|!Pr8F7M)K5Ky*So5V z_FnsWt;T<9Vh8mgdrj%m@z;8=u^nIPt*6hV+}Oi89AiB5lyPL`x!r4uPQ{C!2tB4R zc=N<&XBWr5b!>RDs7}L#q6BNyF0xBor z>X-Sm_HW;Azi^b%Gg58WT=z~x^PhCGejT^e*4UwGo?%5>uwJRS)SP_(^tyvvwtW1M zVU}LF!sap|U^W>qPj4zZ6ZJ5!DP#7?-@V@~d`QjBPd%93)*PAP@N=l|*PM<^#uZ5k zCeLE@LUeB%ovbhCCPkEd49&SFb@y5MKJL~_m-3L0sSBc-Uy4df%H!N)X@zaaPZl2Q zOgg=C^E|f828&gdmFKOSrFfUWZ};Lq3qINw--vOYIj837l*_lqiiYibymhRq>)aad zUOyi_AMc(F(-th}@ZAR$tLIynma`Ad92yv^~vUTs2KgaB! zzCif8qUrD4F6U!@E5<$)1$pehoTDfk<$5D=;xFGanX$Xoit^A&QD?u{fBrV*(^z#D zTm9_opX+9C&&dAcmAiOavY(gTvI_a2WqJjZSJWvU(RR)MtD*Dp>ZAL{2qxLyh`q8e zvu)K`_G7Bwo;S(gvexHs|8^;G4m0O#!@VkZI?LI5*%$KxM{-V{=^B2p*t)dMIq9#8 z#%Wh?omE!EujLEulOq18?S7GU_!PH5V(m0%M##n;Y$o10wkV}VM@2_sih|+T_A}HPZhG|ai z6USVO?2K>88gk9+RcxIVcf+6M^NTNYVjr*PPN$Zv{$o4gH8wnb454?(#ltbFxY#2r zcZo@(?(q@9MJG@5+Es<4B}cC|aN|8%lCIRmTODY%82n(!hOUuw5>8gs5d1q=onOe* zqc=n=FPy6z9)G9ns2h!SU7>JqIXq}I8_&;nxnsOEc;@`KjfT?})z(XL^cSw`S+{j3 zRs7txX;8$Wdd^MbSgS|#$%4e)&b+DK*Yy%Bmvmaa&D+J;8{Fvd!s^z73!Y7(dxoSD zY5L-tCt)LRwzurCStZ|2c5!aJ`8O>+wx}kpsOItWEsLuzo~ei}2*gTUI%aS?W>Bsi zeaqj|edoMaL|Ao4fK3A5vbLn9!8T`bx<7kSo2x-K|E~teA&7(@yrjJTDt)xrK z!!x#Q%X2%Id}_;)l;VW5Zd=1+cCB+xzI3~VqJPLP@znhjh8Ije3?ciK-4*W2c@rJ; z=9A<2%FD%Fmv5)O*zoL2Jum;!^N%mD*57q2II==1UOm*u_g+jS?#`-iYgv&yuc=P- zN8Q}E*^ZJ6TkhNXSn?)SF(*~kyqYU2AiY?7t(6+BG3wbq(b?Q$wC3i7v%04)XKoK_ zjjCPZyKUt(GhuwdwSx&+oL6;6cis2y+JAd#&-Iv%-5)m^lu^y^&X)~fd?+sZbSX~R zeyw(qp3KNk_GQC7mop8&1k^=ZjSF|PJKrXaEtq)jk;vojk;kP&olXo%4 z;pX_9$K|TU($jHD_ce}=mM`@zwla^Fl61n$B{zw*(@Reed*b))Jnq)g@$-|{ZZFr_ zUtRvxjdEq?J^U^ACVz$5+rWm1zx>!2Ptp{Pn9Hkgd6{qMMxuR**}u#6KG<29Z`*UV z>q@xh!qWQYa;M25YJ;>*DW<&Kr&od|yZ-r`!8304p~{r`CZE5qF6aqwbEhsGYqoFv z=H=JZ$6YwXEL=wJ_IhN!REp(HIEYnx+_$RD6g|o~G38`y=GuL$)`U+E-ZSHgbh}}t zA$D>@t?ASBcja}Bi7hR;lOG1nni8w5j(P0u`?S=9+W&Wq5)=#D zGY)U6=~;MAxvs1|VPAcspYFO5GGd~QF#FI2@92$Hw*q{-YNxu>Iw!Q^L+T^hoN#wmxIuiQHK?A*lSGVkRno;z0`s@(CL)N_Zo|8fn@d>prJ z=i|VvpFVj+zq;nWWv6@UIQDJptA|FhZx<7lKMhFR__u9}Y4f4^2Q~@Ww|7lep3Iz= z+%CMXYJL46-9XPTxc2<>2g+abM7x4-JsdIOMSkV)_g*&-UsIG`{?4j4u}U;&VRh*F zd&$oS?wIVeakI%-r-4G+Jz#)~{=?+ah;*`Y4HX>q`*xr{oc+=Taj!GX_>5us0#{C<0FQo7^R^sgpgJ*me` zsV)zxXX7{qlO`N}>vhkBTjS`#@ZB5l9`}12Zc(&z?AtvK+i5~^$^K@=U&HSGeTnv7 z^yK-I?Dwj!YnyXNJiqzZ6Z_qJj+Sn$89Mjp=x?{`yAGH(XD;g+)V$uRC4P_3f1F3**n}Z)wYh z>=bmbvacTba_D=!c5bk=@WV>IfTvuMTfuhWvFeLugQMLF3O}4tR)3u+89ys72-{-JFvk8PapKO`cX zT5D|IT5ZSI^*n?vc}}`IT@?G0Jp3g&M%nc#VSFGr{j6i<iTa3_A)u^U+1!~s1=M-{(ZUXfdLtcm;AJL3_zMtxoQh~}>;-)i@e`-qjP4mlv7 z#*DgQ^+&(*p|iqh|gync|sPwxd|=FDKf8)q0Q zfyc7f?>=rbZq+yFjyxH3R==Ap27_u8_zU%w>*9$EZkA% z7rk$CUiRRJ+kU}hag?i@f?8MY2%KHL84J18G9Nc^$R^J`(;X4kri^!piTEP= +%%
  • `ttl' - The time in seconds that an object should live before being expired.
  • +%%
  • `max_memory' - The amount of memory in megabytes to limit the backend to.
  • +%% +%% -module(riak_kv_test_backend). -behavior(riak_kv_backend). --behavior(gen_server). + +%% KV Backend API +-export([api_version/0, + start/2, + stop/1, + get/3, + put/5, + delete/4, + drop/1, + fold_buckets/4, + fold_keys/4, + fold_objects/4, + is_empty/1, + status/1, + callback/3, + reset/0]). + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -endif. --export([start/2,stop/1,get/2,put/3,list/1,list_bucket/2,delete/2, - is_empty/1, drop/1, fold/3, callback/3, reset/0]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). +-define(API_VERSION, 1). +-define(CAPABILITIES, [async_fold]). + +-record(state, {data_ref :: integer() | atom(), + time_ref :: integer() | atom(), + max_memory :: undefined | integer(), + used_memory=0 :: integer(), + ttl :: integer()}). +-type state() :: #state{}. +-type config() :: []. -% @type state() = term(). --record(state, {t, p}). +%% =================================================================== +%% Public API +%% =================================================================== -% @spec start(Partition :: integer(), Config :: proplist()) -> -% {ok, state()} | {{error, Reason :: term()}, state()} -start(Partition, _Config) -> - gen_server:start_link(?MODULE, [Partition], []). +%% TestServer reset -% @spec reset() -> ok | {error, timeout} +-spec reset() -> ok | {error, timeout}. reset() -> - Pids = lists:foldl(fun(Item, Acc) -> - case lists:prefix("test_backend", atom_to_list(Item)) of - true -> [whereis(Item)|Acc]; - _ -> Acc - end - end, [], registered()), - [gen_server:cast(Pid,{reset, self()})|| Pid <- Pids], - receive_reset(Pids). - -receive_reset([]) -> ok; -receive_reset(Pids) -> - receive - {reset, Pid} -> - receive_reset(lists:delete(Pid, Pids)) - after 1000 -> - {error, timeout} - end. + {ok, Ring} = riak_core_ring_manager:get_my_ring(), + [ ets:delete_all_objects(list_to_atom("kv" ++ integer_to_list(P))) || + P <- riak_core_ring:my_indices(Ring) ], + ok. -%% @private -init([Partition]) -> - PName = list_to_atom("test_backend" ++ integer_to_list(Partition)), - P = list_to_atom(integer_to_list(Partition)), - register(PName, self()), - {ok, #state{t=ets:new(P,[]), p=P}}. +%% KV Backend API -%% @private -handle_cast({reset,From}, State) -> - ets:delete_all_objects(State#state.t), - From ! {reset, self()}, - {noreply, State}; -handle_cast(_, State) -> {noreply, State}. +%% @doc Return the major version of the +%% current API and a capabilities list. +-spec api_version() -> {integer(), [atom()]}. +api_version() -> + {?API_VERSION, ?CAPABILITIES}. -%% @private -handle_call(stop,_From,State) -> {reply, srv_stop(State), State}; -handle_call({get,BKey},_From,State) -> {reply, srv_get(State,BKey), State}; -handle_call({put,BKey,Val},_From,State) -> - {reply, srv_put(State,BKey,Val),State}; -handle_call({delete,BKey},_From,State) -> {reply, srv_delete(State,BKey),State}; -handle_call(list,_From,State) -> {reply, srv_list(State), State}; -handle_call({list_bucket,Bucket},_From,State) -> - {reply, srv_list_bucket(State, Bucket), State}; -handle_call(is_empty, _From, State) -> - {reply, ets:info(State#state.t, size) =:= 0, State}; -handle_call(drop, _From, State) -> - ets:delete(State#state.t), - {reply, ok, State}; -handle_call({fold, Fun0, Acc}, _From, State) -> - Fun = fun({{B,K}, V}, AccIn) -> Fun0({B,K}, V, AccIn) end, - Reply = ets:foldl(Fun, Acc, State#state.t), - {reply, Reply, State}. - -% @spec stop(state()) -> ok | {error, Reason :: term()} -stop(SrvRef) -> gen_server:call(SrvRef,stop). -srv_stop(State) -> - true = ets:delete(State#state.t), +%% @doc Start the memory backend +-spec start(integer(), config()) -> {ok, state()}. +start(Partition, Config) -> + TTL = config_value(ttl, Config), + MemoryMB = config_value(max_memory, Config), + case MemoryMB of + undefined -> + MaxMemory = undefined, + TimeRef = undefined; + _ -> + MaxMemory = MemoryMB * 1024 * 1024, + TimeRef = ets:new(list_to_atom(integer_to_list(Partition)), [ordered_set]) + end, + DataRef = ets:new(list_to_atom("kv" ++ integer_to_list(Partition)), [named_table, public]), + {ok, #state{data_ref=DataRef, + max_memory=MaxMemory, + time_ref=TimeRef, + ttl=TTL}}. + +%% @doc Stop the memory backend +-spec stop(state()) -> ok. +stop(#state{data_ref=DataRef, + max_memory=MaxMemory, + time_ref=TimeRef}) -> + catch ets:delete(DataRef), + case MaxMemory of + undefined -> + ok; + _ -> + catch ets:delete(TimeRef) + end, ok. -% get(state(), riak_object:bkey()) -> -% {ok, Val :: binary()} | {error, Reason :: term()} -% key must be 160b -get(SrvRef, BKey) -> gen_server:call(SrvRef,{get,BKey}). -srv_get(State, BKey) -> - case ets:lookup(State#state.t,BKey) of - [] -> {error, notfound}; - [{BKey,Val}] -> {ok, Val}; - Err -> {error, Err} +%% @doc Retrieve an object from the memory backend +-spec get(riak_object:bucket(), riak_object:key(), state()) -> + {ok, any(), state()} | + {ok, not_found, state()} | + {error, term(), state()}. +get(Bucket, Key, State=#state{data_ref=DataRef, + ttl=TTL}) -> + case ets:lookup(DataRef, {Bucket, Key}) of + [] -> {error, not_found, State}; + [{{Bucket, Key}, {{ts, Timestamp}, Val}}] -> + case exceeds_ttl(Timestamp, TTL) of + true -> + delete(Bucket, Key, undefined, State), + {error, not_found, State}; + false -> + {ok, Val, State} + end; + [{{Bucket, Key}, Val}] -> + {ok, Val, State}; + Error -> + {error, Error, State} end. -% put(state(), riak_object:bkey(), Val :: binary()) -> -% ok | {error, Reason :: term()} -% key must be 160b -put(SrvRef, BKey, Val) -> gen_server:call(SrvRef,{put,BKey,Val}). -srv_put(State,BKey,Val) -> - true = ets:insert(State#state.t, {BKey,Val}), - ok. +%% @doc Insert an object into the memory backend. +%% NOTE: The memory backend does not currently +%% support secondary indexing and the _IndexSpecs +%% parameter is ignored. +-type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. +-spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> + {ok, state()} | + {error, term(), state()}. +put(Bucket, PrimaryKey, _IndexSpecs, Val, State=#state{data_ref=DataRef, + max_memory=MaxMemory, + time_ref=TimeRef, + ttl=TTL, + used_memory=UsedMemory}) -> + Now = now(), + case TTL of + undefined -> + Val1 = Val; + _ -> + Val1 = {{ts, Now}, Val} + end, + case do_put(Bucket, PrimaryKey, Val1, DataRef) of + {ok, Size} -> + %% If the memory is capped update timestamp table + %% and check if the memory usage is over the cap. + case MaxMemory of + undefined -> + UsedMemory1 = UsedMemory; + _ -> + time_entry(Bucket, PrimaryKey, Now, TimeRef), + Freed = trim_data_table(MaxMemory, + UsedMemory + Size, + DataRef, + TimeRef, + 0), + UsedMemory1 = UsedMemory + Size - Freed + end, + {ok, State#state{used_memory=UsedMemory1}}; + {error, Reason} -> + {error, Reason, State} + end. -% delete(state(), riak_object:bkey()) -> -% ok | {error, Reason :: term()} -% key must be 160b -delete(SrvRef, BKey) -> gen_server:call(SrvRef,{delete,BKey}). -srv_delete(State, BKey) -> - true = ets:delete(State#state.t, BKey), - ok. +%% @doc Delete an object from the memory backend +%% NOTE: The memory backend does not currently +%% support secondary indexing and the _IndexSpecs +%% parameter is ignored. +-spec delete(riak_object:bucket(), riak_object:key(), [index_spec()], state()) -> + {ok, state()}. +delete(Bucket, Key, _IndexSpecs, State=#state{data_ref=DataRef, + time_ref=TimeRef, + used_memory=UsedMemory}) -> + case TimeRef of + undefined -> + UsedMemory1 = UsedMemory; + _ -> + %% Lookup the object so we can delete its + %% entry from the time table and account + %% for the memory used. + [Object] = ets:lookup(DataRef, {Bucket, Key}), + case Object of + {_, {{ts, Timestamp}, _}} -> + ets:delete(TimeRef, Timestamp), + UsedMemory1 = UsedMemory - object_size(Object); + _ -> + UsedMemory1 = UsedMemory + end + end, + ets:delete(DataRef, {Bucket, Key}), + {ok, State#state{used_memory=UsedMemory1}}. + +%% @doc Fold over all the buckets. +-spec fold_buckets(riak_kv_backend:fold_buckets_fun(), + any(), + [], + state()) -> {ok, any()}. +fold_buckets(FoldBucketsFun, Acc, Opts, #state{data_ref=DataRef}) -> + FoldFun = fold_buckets_fun(FoldBucketsFun), + case lists:member(async_fold, Opts) of + true -> + BucketFolder = + fun() -> + {Acc0, _} = ets:foldl(FoldFun, {Acc, sets:new()}, DataRef), + Acc0 + end, + {async, BucketFolder}; + false -> + {Acc0, _} = ets:foldl(FoldFun, {Acc, sets:new()}, DataRef), + {ok, Acc0} + end. + +%% @doc Fold over all the keys for one or all buckets. +-spec fold_keys(riak_kv_backend:fold_keys_fun(), + any(), + [{atom(), term()}], + state()) -> {ok, term()} | {async, fun()}. +fold_keys(FoldKeysFun, Acc, Opts, #state{data_ref=DataRef}) -> + Bucket = proplists:get_value(bucket, Opts), + FoldFun = fold_keys_fun(FoldKeysFun, Bucket), + case lists:member(async_fold, Opts) of + true -> + {async, get_folder(FoldFun, Acc, DataRef)}; + false -> + Acc0 = ets:foldl(FoldFun, Acc, DataRef), + {ok, Acc0} + end. + +%% @doc Fold over all the objects for one or all buckets. +-spec fold_objects(riak_kv_backend:fold_objects_fun(), + any(), + [{atom(), term()}], + state()) -> {ok, any()} | {async, fun()}. +fold_objects(FoldObjectsFun, Acc, Opts, #state{data_ref=DataRef}) -> + Bucket = proplists:get_value(bucket, Opts), + FoldFun = fold_objects_fun(FoldObjectsFun, Bucket), + case lists:member(async_fold, Opts) of + true -> + {async, get_folder(FoldFun, Acc, DataRef)}; + false -> + Acc0 = ets:foldl(FoldFun, Acc, DataRef), + {ok, Acc0} + end. -% list(state()) -> [riak_object:bkey()] -list(SrvRef) -> gen_server:call(SrvRef,list). -srv_list(State) -> - MList = ets:match(State#state.t,{'$1','_'}), - list(MList,[]). -list([],Acc) -> Acc; -list([[K]|Rest],Acc) -> list(Rest,[K|Acc]). - -% list_bucket(term(), Bucket :: riak_object:bucket()) -> [Key :: binary()] -list_bucket(SrvRef, Bucket) -> - gen_server:call(SrvRef,{list_bucket, Bucket}). -srv_list_bucket(State, {filter, Bucket, Fun}) -> - MList = lists:filter(Fun, ets:match(State#state.t,{{Bucket,'$1'},'_'})), - list(MList,[]); -srv_list_bucket(State, Bucket) -> - case Bucket of - '_' -> MatchSpec = {{'$1','_'},'_'}; - _ -> MatchSpec = {{Bucket,'$1'},'_'} +%% @doc Delete all objects from this memory backend +-spec drop(state()) -> {ok, state()}. +drop(State=#state{data_ref=DataRef, + time_ref=TimeRef}) -> + ets:delete_all_objects(DataRef), + case TimeRef of + undefined -> + ok; + _ -> + ets:delete_all_objects(TimeRef) end, - MList = ets:match(State#state.t,MatchSpec), - list(MList,[]). + {ok, State}. -is_empty(SrvRef) -> gen_server:call(SrvRef, is_empty). +%% @doc Returns true if this memory backend contains any +%% non-tombstone values; otherwise returns false. +-spec is_empty(state()) -> boolean(). +is_empty(#state{data_ref=DataRef}) -> + ets:info(DataRef, size) =:= 0. -drop(SrvRef) -> gen_server:call(SrvRef, drop). +%% @doc Get the status information for this memory backend +-spec status(state()) -> [{atom(), term()}]. +status(#state{data_ref=DataRef, + time_ref=TimeRef}) -> + DataStatus = ets:info(DataRef), + case TimeRef of + undefined -> + [{data_table_status, DataStatus}]; + _ -> + TimeStatus = ets:info(TimeRef), + [{data_table_status, DataStatus}, + {time_table_status, TimeStatus}] + end. -fold(SrvRef, Fun, Acc0) -> gen_server:call(SrvRef, {fold, Fun, Acc0}, infinity). +%% @doc Register an asynchronous callback +-spec callback(reference(), any(), state()) -> {ok, state()}. +callback(_Ref, _Msg, State) -> + {ok, State}. -%% Ignore callbacks for other backends so multi backend works -callback(_State, _Ref, _Msg) -> - ok. +%% =================================================================== +%% Internal functions +%% =================================================================== + +%% @TODO Some of these implementations may be suboptimal. +%% Need to do some measuring and testing to refine the +%% implementations. %% @private -handle_info(_Msg, State) -> {noreply, State}. +%% Return a function to fold over the buckets on this backend +fold_buckets_fun(FoldBucketsFun) -> + fun({{Bucket, _}, _}, {Acc, BucketSet}) -> + case sets:is_element(Bucket, BucketSet) of + true -> + {Acc, BucketSet}; + false -> + {FoldBucketsFun(Bucket, Acc), + sets:add_element(Bucket, BucketSet)} + end + end. %% @private -terminate(_Reason, _State) -> ok. +%% Return a function to fold over keys on this backend +fold_keys_fun(FoldKeysFun, undefined) -> + fun({{Bucket, Key}, _}, Acc) -> + FoldKeysFun(Bucket, Key, Acc) + end; +fold_keys_fun(FoldKeysFun, Bucket) -> + fun({{B, Key}, _}, Acc) -> + case B =:= Bucket of + true -> + FoldKeysFun(Bucket, Key, Acc); + false -> + Acc + end + end. %% @private -code_change(_OldVsn, State, _Extra) -> {ok, State}. +%% Return a function to fold over keys on this backend +fold_objects_fun(FoldObjectsFun, undefined) -> + fun({{Bucket, Key}, Value}, Acc) -> + FoldObjectsFun(Bucket, Key, Value, Acc) + end; +fold_objects_fun(FoldObjectsFun, Bucket) -> + fun({{B, Key}, Value}, Acc) -> + case B =:= Bucket of + true -> + FoldObjectsFun(Bucket, Key, Value, Acc); + false -> + Acc + end + end. + +%% @private +get_folder(FoldFun, Acc, DataRef) -> + fun() -> + ets:foldl(FoldFun, Acc, DataRef) + end. + +%% @private +do_put(Bucket, Key, Val, Ref) -> + Object = {{Bucket, Key}, Val}, + true = ets:insert(Ref, Object), + {ok, object_size(Object)}. + +%% @private +config_value(Key, Config) -> + config_value(Key, Config, undefined). + +%% @private +config_value(Key, Config, Default) -> + case proplists:get_value(Key, Config) of + undefined -> + app_helper:get_env(memory_backend, Key, Default); + Value -> + Value + end. + +%% Check if this timestamp is past the ttl setting. +exceeds_ttl(Timestamp, TTL) -> + Diff = (timer:now_diff(now(), Timestamp) / 1000 / 1000), + Diff > TTL. + +%% @private +time_entry(Bucket, Key, Now, TimeRef) -> + ets:insert(TimeRef, {Now, {Bucket, Key}}). + +%% @private +%% @doc Dump some entries if the max memory size has +%% been breached. +trim_data_table(MaxMemory, UsedMemory, _, _, Freed) when + (UsedMemory - Freed) =< MaxMemory -> + Freed; +trim_data_table(MaxMemory, UsedMemory, DataRef, TimeRef, Freed) -> + %% Delete the oldest object + OldestSize = delete_oldest(DataRef, TimeRef), + trim_data_table(MaxMemory, + UsedMemory, + DataRef, + TimeRef, + Freed + OldestSize). + +%% @private +delete_oldest(DataRef, TimeRef) -> + OldestTime = ets:first(TimeRef), + case OldestTime of + '$end_of_table' -> + 0; + _ -> + OldestKey = ets:lookup_element(TimeRef, OldestTime, 2), + ets:delete(TimeRef, OldestTime), + case ets:lookup(DataRef, OldestKey) of + [] -> + delete_oldest(DataRef, TimeRef); + [Object] -> + ets:delete(DataRef, OldestKey), + object_size(Object) + end + end. + +%% @private +object_size(Object) -> + case Object of + {{Bucket, Key}, {{ts, _}, Val}} -> + ok; + {{Bucket, Key}, Val} -> + ok + end, + size(Bucket) + size(Key) + size(Val). + +%% =================================================================== +%% EUnit tests +%% =================================================================== -%% -%% Test -%% -ifdef(TEST). -% @private -simple_test() -> +simple_test_() -> riak_kv_backend:standard_test(?MODULE, []). --ifdef(EQC). +ttl_test_() -> + Config = [{ttl, 15}], + {ok, State} = start(42, Config), + + Bucket = <<"Bucket">>, + Key = <<"Key">>, + Value = <<"Value">>, + + [ + %% Put an object + ?_assertEqual({ok, State}, put(Bucket, Key, [], Value, State)), + %% Wait 1 second to access it + ?_assertEqual(ok, timer:sleep(1000)), + ?_assertEqual({ok, Value, State}, get(Bucket, Key, State)), + %% Wait 3 seconds and access it again + ?_assertEqual(ok, timer:sleep(3000)), + ?_assertEqual({ok, Value, State}, get(Bucket, Key, State)), + %% Wait 15 seconds and it should expire + {timeout, 30000, ?_assertEqual(ok, timer:sleep(15000))}, + %% This time it should be gone + ?_assertEqual({error, not_found, State}, get(Bucket, Key, State)) + ]. + %% @private -eqc_test() -> - ?assertEqual(true, backend_eqc:test(?MODULE, true)). +max_memory_test_() -> + %% Set max size to 1.5kb + Config = [{max_memory, 1.5 * (1 / 1024)}], + {ok, State} = start(42, Config), + + Bucket = <<"Bucket">>, + Key1 = <<"Key1">>, + Value1 = list_to_binary(string:copies("1", 1024)), + Key2 = <<"Key2">>, + Value2 = list_to_binary(string:copies("2", 1024)), + + %% Write Key1 to the datastore + {ok, State1} = put(Bucket, Key1, [], Value1, State), + timer:sleep(timer:seconds(1)), + %% Write Key2 to the datastore + {ok, State2} = put(Bucket, Key2, [], Value2, State1), + + [ + %% Key1 should be kicked out + ?_assertEqual({error, not_found, State2}, get(Bucket, Key1, State2)), + %% Key2 should still be present + ?_assertEqual({ok, Value2, State2}, get(Bucket, Key2, State2)) + ]. + +-ifdef(EQC). + +eqc_test_() -> + {spawn, + [{inorder, + [{setup, + fun setup/0, + fun cleanup/1, + [ + {timeout, 60000, + [?_assertEqual(true, + backend_eqc:test(?MODULE, true))]} + ]}]}]}. + +setup() -> + application:load(sasl), + application:set_env(sasl, sasl_error_logger, {file, "riak_kv_memory_backend_eqc_sasl.log"}), + error_logger:tty(false), + error_logger:logfile({open, "riak_kv_memory_backend_eqc.log"}), + ok. + +cleanup(_) -> + ok. -endif. % EQC + -endif. % TEST diff --git a/erl_src/riak_search_test_backend.beam b/erl_src/riak_search_test_backend.beam index ddef8d248ea4bc6c7e069a9c7442e48dcf4a9273..b20a76c9de3a21a0177d94e14ef95b5192d14fa3 100644 GIT binary patch delta 282 zcmZ4DbVG^5%|FPHfk7}}BgcIvMxV(qm|}Sa90XiBi&UYPI)KKb!2S{+3DpOUozU<2O}SY|dd| zU|`8$N?~BnFDS{(&npHAFo6V^lgm@G8H^^M7LZaaD9B0GFUm~J)=kdIOwB9NPc6!c zFD^<3sbkA8ElJ5Nf-053rF3$ifSNdGqEMn}qI#kzSYvU1X;E@&HiJLN4pFQcC;t>s O=45b8DlP#!iva*qB~XX} delta 4383 zcmV+)5#a9FBBVkHMo&^P001Ezu?XJ+0Wy>50$sCG1HA~791Iu&006Tz46*|dsxi*| zQ6wsIczJpvYpDsgllBiae+GkW00004XaZya2XAn6X>V?GYybcN2516g00wV$bYy9A zYXAggVrgz<019MfVs&RMk3WB?ClVQFk%Z*E_AVRCM1Zf9r$WB>taYXApnZewh9 zWMyOk4RU34b#iWBWpZ+Fa&u$=4{~L6b#iWBcVTjFX>Mn8YG?vvlf)1r7Y1PuVGdyv zVH0QqWB>+pZ*_8GWorOVlTHvC6k%&GWMy!3FLG&NYhQC^VRB<=FOw`0BY#0+b94Xz z5C4Nu002Npc$}46`)?dY5?*`#%=>|kLxA!WArLvv1EPdDTyVVDv6VQnF_GI*;zkO%n^m?tXN8{$V5RWgayLnIEmb!c-1pIT~*yPYy6Y9d%mi!cXjuSXIrh@ z-oRWi%;{Pfn8nbTKjY?dxqoA`ZC0+^48y>gu7|eSV&&ROj%&}h&gBl*g5v4GG3Sh$ zZ3e|zBeZLwF>My-Y|rYk1J`U(s5`A(N5!}5u5IQ1o<*t9bXmEsX?xb3cl^N09i&w0 z1Uq06d$H;Vp-^DwENIpq*fl#ebJ6cwXa=F#Dt@=sLca=9ifPBQ?0?00y2G+v8#u)S zu!n6{%#N6m>CJ#l5wYWye6v0JS@K=W%!{90PR+0@)o>|h+hzq~g|Z^!>rBQ7xhG!7 z08#61STpk~4K@Aij#qT+mQBsyGn-=ZU9eDK7r}I5ja?1HfeAuetlTBwyJCP*a%|9I z_iP>(GDE*&?grwj8GqPb$mY81V5*Yi*_KYd2a$Ucqq3=H#Cs9B7l@-tHbsc@D)Bx< z?z3_`0B3?74Zo_dvmc@RfvH!oKmZ1nnwai%ywIMp19Lxwi$-|>DGz|J1*PQ|IpKjc z!h=Y75TyIMZU;+-=LZ!=dN7UjH6(ovtWt%5stIT2J;ya)OMi~+DfV(9pu zQg-}m6fTgmNq>;V)Ml&djCZ&*Jl~44F0YI8E{gN6;tc+XI74Id9wOgsEQf4T9;UE3 zg{V_t3mR&tRP9bP?b>VbD^jmjXWNTxdsBVXo93f4NOeXL=8P)Lc_vIZWXYmaal)kx zdw+neKTzz=A2qZ0hlu=8k?%tteF4!IsD#W87p8N2KY#gHRqhf~ZfDeljGFU}Z3*?- z^|e02wLVhRb0tu}zj0mW(q&8=2IWFgaP zpCahfEPtzwFspS{{Nh~FLsBwy1UUj>XJhXovtUH!enatzsH%>&D4J;8r8Jpu;0ibB zhJvGRBI>5}9EXe|WHc)HFo}|m>IAot;Fi{RZ|V5kh`(*+b~~O6t}>K$ZtHYoNH+#_ zrf0fKkl9AbHD}nP_LAw=G>3i0OuCmgDhN5c&l$jkZBz7o;lnB~xlYRVrc>@9Z`9Un2W2MUOKpwr=k` z2)Y9?Yr2kEv(ZC$bhdYq;;z_JEmc#J>mGvc!LzDqS&lbjq-Ww+2>yzIi?e2+3v3|R zfPa*PO;OzmY>Umtv7ysWA!ovIQmFD1MjgfPV)= z-w+c`k$W3kEddrpF*vab09r=Nv0)U9(s1K1vJHuh`> zm?ok@n$!w?5AFUtTuayMm34aMY0dwOb(owk+RJju<=m|A2#Dr%rNahJS1vHBU zj%*$YC#aH%D}1k}oX2*GCSw^PYNTU2s`Y6Xs%I7ps0cO`bnsy)+t9mZlFBEQR6ZG} za+@qlVultat!kcR#Jng)>Yiji-GV+YRg^Y%DI|^(RqtELC7Tvh?{j+tvVVO{Xh4!L z_8LEv?y8te)3y=U2hQRYvQWh6!3#c3jvgM|0NpWE7@rs&9)Xx3A>2dl-9zo2vN7&K znR_&Q^+KWP=lX%McqC*l3)MCV$vyl#PbVdSy5x5#o$LY%dW(&dB~2+k#&Rv`lmNi>aaV zcF=`wp~ts{o{;4mOneq8&r*M$MK)*EVh0l)uJ6x*C0Kx-l?9Z{2*Wi@OFVmyWRj>m z=ZJ03#Wy1fk8>chkv-Zsun4gRO$pZNci~W2mX+#UlCXJkk|>$yHGeVA#{e04txyln z!wOgA3g=T+pbtgl$VKFhi|Ar>oeOjw3A`Y->_J_xD;KFJE+WB2)e|tWr(U%nI}NBh z726BB?JjB_x!4G0Y57aU@|UDP<+aZx6a9NB+JbC@G|xzf(KDBldYl;OvV7YRJ-@7( zq>q|}-{EkR^hI0mntvU?z(C>qs3{sk_i=0WVVWmnQYQKuWkN`v%>9FvWZD?7O4hMh zSEKE6)m>6Ctqju{^#YsR?_6ciJTt!J=HYDZx_643_!&4TO5=DJ$&u4i=P>ssxu@gLFiM=LK z-pTaZ*^qORmy$^)=OmMJlF2#Avoe^zVYy~M`l?NCHM*bYVHqcTSsWc*cft{_7! zS_ayJ-+4!+0%2wPQwldl-N^5~Q@ID;ya=$b{ES&_|EbHP*glQ91}4K%17da%uEUwUCwIZ_5ECFU?@%FNsn zFn@>A&e2?$!`0{1k~SwkaEnS>tc+I7u;b5V7qdCNn7JfFT(x$kPmp5fB5zmskegD> zwx(zlGYV@xenuIeRRQNXUxOUXD@1{c5(^cez0$|W$MLGG5{39Sc=cUw%4=X8%SQ#d zdW5T|-tyrNA!h4E`YYpeWn}D;p8_VYAb)(*!bd)_U^TmX1?0<0t$YQ1PZen8E6~bU zpl%R?)S49NH7UrvdL?;M%gFiKU&;AelbpwkfG{QJ>+)$mJ?H09+IgDu^SJuFn)CBp z<@~%{bv`@i=k=UlAknp;rg-`U$@vB3y`X3CLQ2kWO`&dJwxm!uFbh$xsS{^88-KDP zSww3r5?vS7t_6hjb$IKzl?5MrlGxzDlY4HRm<{YPJUIj-1Mqy zS@q8{S7Di*ftJ;m#EFyyfQI=ioPX$qw3U8?@GU>Geybf>|EZt9zvD6f9ay2UfTq9$ zYUT&X^}$vHqS{!akU@l(er>xLUQyW@_neqDUZpAI@gDKos5|h)%Xg$I6h>> zv4!$g8;-Kka9LfZl|JvfS%;u&=&zwN$ztiUA1mUPnS@!fV}Dgdos5Z?^0>f8VJFyVl#R$G(_^kF4Tx1$ z{8cp|R(U|IvVd4+;;*uRSmgn+$|PJ>17eliWi{Q}Ix5N{detlDk~du`ugjlE;1esi z^M<`3SCNH-L6R)x#($Es@Dp5^&Pg?>wokbpPZd$pfSHsmyiy()*nfzH*He`;&#)70 zG|EQgHR)3&rq)!M*VLF=<1w|yVrq@ayvAZ`jmOj)6KhS4sWonlH6^CvYQIK53{NQo z&p4B3>4Eu7)mbNFeoA>ghnc?6RzuAo^>!vqGd0g zg%g2ix@5PpPP^mJf(6xXC+J7#=MZQ7g^QlMGDUsDb?r&^(xQVAC-;p zk31{*4(Q(sA5``y<#_)SeMKD6{-oqjGMVwp_hA7){0%jTHySTxGTvNlNJ%!bLvKSK{25#Ep}VI3FIMs&u$t<> zaMgcNRhP>BitGH^^z*-8GrjPeLYE@WZ=Ba}>EZxz6D8ZEX5U0HH}zxbruffiyE2^| zOEuGP%B456O@F_sf5y>72JU(>0R(0JRQbD-JHK1>-7L8edr8-Ni3BgzwO+D!vn{%| zFQv9Gv$cJRHBQd`M?c{Hp@{#7mHY>QdOH4Z@ {ok, Ring} = riak_core_ring_manager:get_my_ring(), - [ ets:delete_all_objects(list_to_atom(integer_to_list(P))) || + [ ets:delete_all_objects(list_to_atom("rs" ++ integer_to_list(P))) || P <- riak_core_ring:my_indices(Ring) ], riak_search_config:clear(), ok. start(Partition, _Config) -> - Table = ets:new(list_to_atom(integer_to_list(Partition)), + Table = ets:new(list_to_atom("rs" ++ integer_to_list(Partition)), [named_table, public, ordered_set]), {ok, #state{partition=Partition, table=Table}}. From 2887c93177d818bf4c54baf787071d9beb6eaed1 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 18 Oct 2011 16:14:11 +0200 Subject: [PATCH 0124/1060] Make TestServer work again Pass search_backend and storage_backend as atoms Set platform_data_dir to point to the temp dir --- riak/test_server.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/riak/test_server.py b/riak/test_server.py index b15d343e..57025387 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -15,6 +15,16 @@ except NameError: bytes = str +class Atom(object): + def __init__(self, s): + self.str = s + + def __str__(self): + return str(self.str) + + def __repr__(self): + return repr(self.str) + def erlang_config(hash, depth=1): def printable(item): k, v = item @@ -55,7 +65,7 @@ class TestServer: "ring_creation_size": 64 }, "riak_kv": { - "storage_backend": bytes("riak_kv_test_backend"), + "storage_backend": Atom("riak_kv_test_backend"), "pb_ip": "127.0.0.1", "pb_port": 9002, "js_vm_count": 8, @@ -67,7 +77,7 @@ class TestServer: }, "riak_search": { "enabled": True, - "search_backend": bytes("riak_search_test_backend") + "search_backend": Atom("riak_search_test_backend") }, "luwak": { "enabled": True @@ -92,6 +102,7 @@ def __init__(self, tmp_dir="/tmp/riak/test_server", self.app_config[key] = deep_merge(self.app_config[key], value) self.app_config["riak_core"]["ring_state_dir"] = os.path.join(self.temp_dir, "data", "ring") + self.app_config["riak_core"]["platform_data_dir"] = self.temp_dir def prepare(self): if not self._prepared: From 17b85b93334e8038ff0920f56344b78362f701bb Mon Sep 17 00:00:00 2001 From: Ian Plosker Date: Tue, 18 Oct 2011 12:16:01 -0400 Subject: [PATCH 0125/1060] Fix issue #77 incorrectly formatted long headers. https://github.com/basho/riak-python-client/issues/77 --- riak/transports/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 456b91b5..f4159648 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -339,7 +339,7 @@ def add_links_for_riak_object(self, robject, headers): for link in links: header = self.to_link_header(link) if len(current_header + header) > MAX_LINK_HEADER_SIZE: - headers.setdefault('Link', []).append(current_header) + headers.add('Link', current_header) current_header = '' if current_header != '': header = ', ' + header From f0c6fc9bd8d151311276d795071a875daa42380b Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 19 Oct 2011 07:55:15 +0200 Subject: [PATCH 0126/1060] Fix TestServer.recycle() Adding the Atom class broke recycle(), because it depends on being able to compare the {storage,search}_backend settings to a couple of strings. Fix this by adding __cmp__ and __eq__ methods to the Atom class so these comparisons work again. --- riak/test_server.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/riak/test_server.py b/riak/test_server.py index 57025387..c60bf9ab 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -25,6 +25,12 @@ def __str__(self): def __repr__(self): return repr(self.str) + def __eq__(self, other): + return self.str == other + + def __cmp__(self, other): + return cmp(self.str, other) + def erlang_config(hash, depth=1): def printable(item): k, v = item From adaf3bdd7003701010c5a1398e654590a986d833 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 19 Oct 2011 13:50:51 +0200 Subject: [PATCH 0127/1060] Add a bunch of missing stuff to the tarball --- MANIFEST.in | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..e691aa45 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include docs/* +include erl_src/* +include THANKS +include README.rst +include LICENSE +include RELEASE_NOTES.md From bb25b0c0d27dca8082b78a8f3e60acd66b96d0b5 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sun, 4 Sep 2011 20:42:03 +0200 Subject: [PATCH 0128/1060] Move erl_src under riak/'s namespace. This gets erl_src installed by setup.py install which makes TestServer work outside of a SCM checkout. --- {erl_src => riak/erl_src}/riak_kv_test_backend.beam | Bin {erl_src => riak/erl_src}/riak_kv_test_backend.erl | 0 .../erl_src}/riak_search_test_backend.beam | Bin .../erl_src}/riak_search_test_backend.erl | 0 riak/test_server.py | 2 +- setup.py | 4 +++- 6 files changed, 4 insertions(+), 2 deletions(-) rename {erl_src => riak/erl_src}/riak_kv_test_backend.beam (100%) rename {erl_src => riak/erl_src}/riak_kv_test_backend.erl (100%) rename {erl_src => riak/erl_src}/riak_search_test_backend.beam (100%) rename {erl_src => riak/erl_src}/riak_search_test_backend.erl (100%) diff --git a/erl_src/riak_kv_test_backend.beam b/riak/erl_src/riak_kv_test_backend.beam similarity index 100% rename from erl_src/riak_kv_test_backend.beam rename to riak/erl_src/riak_kv_test_backend.beam diff --git a/erl_src/riak_kv_test_backend.erl b/riak/erl_src/riak_kv_test_backend.erl similarity index 100% rename from erl_src/riak_kv_test_backend.erl rename to riak/erl_src/riak_kv_test_backend.erl diff --git a/erl_src/riak_search_test_backend.beam b/riak/erl_src/riak_search_test_backend.beam similarity index 100% rename from erl_src/riak_search_test_backend.beam rename to riak/erl_src/riak_search_test_backend.beam diff --git a/erl_src/riak_search_test_backend.erl b/riak/erl_src/riak_search_test_backend.erl similarity index 100% rename from erl_src/riak_search_test_backend.erl rename to riak/erl_src/riak_search_test_backend.erl diff --git a/riak/test_server.py b/riak/test_server.py index b15d343e..403360fb 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -44,7 +44,7 @@ class TestServer: "-smp": "enable", "-env ERL_MAX_PORTS": 4096, "-env ERL_FULLSWEEP_AFTER": 10, - "-pa": os.path.abspath(os.path.join(os.path.dirname(__file__), "../erl_src")) + "-pa": os.path.abspath(os.path.join(os.path.dirname(__file__), "erl_src")) } APP_CONFIG_DEFAULTS = { diff --git a/setup.py b/setup.py index 7814cde2..ae14401b 100755 --- a/setup.py +++ b/setup.py @@ -23,10 +23,12 @@ def make_pb(): install_requires = ['protobuf>=2.3.0', 'urllib3>=0.4.0'], dependency_links = ["http://downloads.basho.com/support"], package_data = { - '' : ['*.proto'] + '' : ['*.proto'], + 'riak' : ['erl_src/*'] }, description='Python client for Riak', zip_safe=True, + include_package_data=True, license='Apache 2', platforms='Platform Independent', author='Basho Technologies', From 4031652267c3f1c6677a3cf5e41364ea3adb991e Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 20 Oct 2011 10:35:46 +0200 Subject: [PATCH 0129/1060] Temporary, simple fix for issue #32 https://issues.basho.com/show_bug.cgi?id=649 has more details. Eventually, we want to allow unicode keys and bucket names, but we have yet to decide how to do that. Until we figure that out, this patch will at least let us use the search interface as long as we only use ascii-encodable keys and bucket names. --- riak/bucket.py | 8 ++++++-- riak/mapreduce.py | 4 +++- riak/riak_object.py | 4 +++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index cef43271..69c044ee 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -38,7 +38,9 @@ def __init__(self, client, name): :param name: The bucket name :type name: string """ - if isinstance(name, unicode): + try: + name.encode('ascii') + except UnicodeEncodeError: raise TypeError('Unicode bucket names are not supported.') self._client = client @@ -210,7 +212,9 @@ def new(self, key, data=None, content_type='application/json'): :type data: object :rtype: :class:`RiakObject ` """ - if isinstance(data, unicode): + try: + data.encode('ascii') + except: raise TypeError('Unicode data values are not supported.') obj = RiakObject(self._client, self, key) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index d2b956d1..0a3ddf30 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -320,7 +320,9 @@ def __init__(self, type, function, language, keep, arg): @param mixed arg - Additional value to pass into the map or reduce function. """ - if isinstance(function, unicode): + try: + function.encode('ascii') + except UnicodeEncodeError: raise TypeError('Unicode encoded functions are not supported.') self._type = type diff --git a/riak/riak_object.py b/riak/riak_object.py index 7ad05032..7e99174b 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -39,7 +39,9 @@ def __init__(self, client, bucket, key=None): is generated by the server when :func:`store` is called. :type key: string """ - if isinstance(key, unicode): + try: + key.encode('ascii') + except UnicodeEncodeError: raise TypeError('Unicode keys are not supported.') self._client = client From a7ab12423a5671d338077bb8e72219beaea08829 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 20 Oct 2011 10:56:19 +0200 Subject: [PATCH 0130/1060] Only attempt encoding to ascii if it is a type of string. --- riak/bucket.py | 6 ++++-- riak/mapreduce.py | 3 ++- riak/riak_object.py | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 69c044ee..b1af10b7 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -39,7 +39,8 @@ def __init__(self, client, name): :type name: string """ try: - name.encode('ascii') + if isinstance(name, basestring): + name.encode('ascii') except UnicodeEncodeError: raise TypeError('Unicode bucket names are not supported.') @@ -213,7 +214,8 @@ def new(self, key, data=None, content_type='application/json'): :rtype: :class:`RiakObject ` """ try: - data.encode('ascii') + if isinstance(data, basestring): + data.encode('ascii') except: raise TypeError('Unicode data values are not supported.') diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 0a3ddf30..3b7ce54d 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -321,7 +321,8 @@ def __init__(self, type, function, language, keep, arg): reduce function. """ try: - function.encode('ascii') + if isinstance(function, basestring): + function.encode('ascii') except UnicodeEncodeError: raise TypeError('Unicode encoded functions are not supported.') diff --git a/riak/riak_object.py b/riak/riak_object.py index 7e99174b..6b03d217 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -40,7 +40,8 @@ def __init__(self, client, bucket, key=None): :type key: string """ try: - key.encode('ascii') + if isinstance(key, basestring): + key.encode('ascii') except UnicodeEncodeError: raise TypeError('Unicode keys are not supported.') From 09c1951d5a454b659a2e316dc1e93196ec55e71a Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 26 Oct 2011 16:14:17 -0400 Subject: [PATCH 0131/1060] Fix RiakPbcCachedTransport to use the new API. --- riak/transports/pbc.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 63239904..a7a56a43 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -558,6 +558,13 @@ def pbify_content(self, metadata, data, rpb_content) : import contextlib class RiakPbcCachedTransport(RiakTransport): """Threadsafe pool of PBC connections, based on urllib3's pool [aka Queue]""" + + # We're using the new RiakTransport API + api = 2 + + # The ConnectionManager class that this transport prefers. + default_cm = SocketConnectionManager + def __init__(self, cm, client_id=None, maxsize=0, block=False, timeout=None, **unused_options): @@ -566,6 +573,7 @@ def __init__(self, cm, ### backwards compat. we don't use the ConnectionManager (yet). host, port = cm.hostports[0] + self._cm = cm self.host = host self.port = port @@ -579,7 +587,7 @@ def __init__(self, cm, def _new_connection(self): """New PBC connection""" - return RiakPbcTransport(self.host, self.port, self.client_id) + return RiakPbcTransport(self._cm, self.client_id) def _get_connection(self): connection = None From 14d3dbc1bcffa44377f426121faaf4fbf13cdb3b Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Sun, 6 Nov 2011 18:05:55 -0500 Subject: [PATCH 0132/1060] Collapse the usage of .maybe_connect() into the send methods. --- riak/transports/pbc.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index a7a56a43..22444d29 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -125,7 +125,6 @@ def ping(self): Ping the remote server @return boolean """ - self.maybe_connect() self.send_msg_code(MSG_CODE_PING_REQ) msg_code, msg = self.recv_msg() if msg_code == MSG_CODE_PING_RESP: @@ -137,7 +136,6 @@ def get_client_id(self): """ Get the client id used by this connection """ - self.maybe_connect() self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ) msg_code, resp = self.recv_msg() if msg_code == MSG_CODE_GET_CLIENT_ID_RESP: @@ -152,7 +150,6 @@ def set_client_id(self, client_id): req = riakclient_pb2.RpbSetClientIdReq() req.client_id = client_id - self.maybe_connect() self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req) msg_code, resp = self.recv_msg() if msg_code == MSG_CODE_SET_CLIENT_ID_RESP: @@ -175,7 +172,6 @@ def get(self, robj, r = None, vtag = None): req.bucket = bucket.get_name() req.key = robj.get_key() - self.maybe_connect() self.send_msg(MSG_CODE_GET_REQ, req) msg_code, resp = self.recv_msg() if msg_code == MSG_CODE_GET_RESP: @@ -206,7 +202,6 @@ def put(self, robj, w = None, dw = None, return_body = True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - self.maybe_connect() self.send_msg(MSG_CODE_PUT_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_PUT_RESP: @@ -237,7 +232,6 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - self.maybe_connect() self.send_msg(MSG_CODE_PUT_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_PUT_RESP: @@ -262,7 +256,6 @@ def delete(self, robj, rw = None): req.bucket = bucket.get_name() req.key = robj.get_key() - self.maybe_connect() self.send_msg(MSG_CODE_DEL_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_DEL_RESP: @@ -276,7 +269,6 @@ def get_keys(self, bucket): req = riakclient_pb2.RpbListKeysReq() req.bucket = bucket.get_name() - self.maybe_connect() self.send_msg(MSG_CODE_LIST_KEYS_REQ, req) keys = [] while True: @@ -296,7 +288,6 @@ def get_buckets(self): """ Serialize bucket listing request and deserialize response """ - self.maybe_connect() self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_LIST_BUCKETS_RESP: @@ -310,7 +301,6 @@ def get_bucket_props(self, bucket): req = riakclient_pb2.RpbGetBucketReq() req.bucket = bucket.get_name() - self.maybe_connect() self.send_msg(MSG_CODE_GET_BUCKET_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_GET_BUCKET_RESP: @@ -337,7 +327,6 @@ def set_bucket_props(self, bucket, props): if 'allow_mult' in props: req.props.allow_mult = props['allow_mult'] - self.maybe_connect() self.send_msg(MSG_CODE_SET_BUCKET_REQ, req) msg_code, resp = self.recv_msg() if msg_code != MSG_CODE_SET_BUCKET_RESP: @@ -357,7 +346,6 @@ def mapred(self, inputs, query, timeout=None): req.request = content req.content_type = "application/json" - self.maybe_connect() self.send_msg(MSG_CODE_MAPRED_REQ, req) # dictionary of phase results - each content should be an encoded array @@ -401,6 +389,7 @@ def maybe_connect(self): self.set_client_id(self._client_id) def send_msg_code(self, msg_code): + self.maybe_connect() pkt = struct.pack("!iB", 1, msg_code) self._sock.send(pkt) @@ -411,6 +400,7 @@ def encode_msg(self, msg_code, msg): return hdr + str def send_msg(self, msg_code, msg): + self.maybe_connect() pkt = self.encode_msg(msg_code, msg) sent_len = self._sock.send(pkt) if sent_len != len(pkt): From 2b03743aaf9766e63034bf2047e0460abeba3db6 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Sun, 6 Nov 2011 18:16:29 -0500 Subject: [PATCH 0133/1060] Fold recv_msg() into the send_msg_code() method. --- riak/transports/pbc.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 22444d29..f833f7b4 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -125,8 +125,7 @@ def ping(self): Ping the remote server @return boolean """ - self.send_msg_code(MSG_CODE_PING_REQ) - msg_code, msg = self.recv_msg() + msg_code, msg = self.send_msg_code(MSG_CODE_PING_REQ) if msg_code == MSG_CODE_PING_RESP: return 1 else: @@ -136,8 +135,7 @@ def get_client_id(self): """ Get the client id used by this connection """ - self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ) if msg_code == MSG_CODE_GET_CLIENT_ID_RESP: return resp.client_id else: @@ -288,8 +286,7 @@ def get_buckets(self): """ Serialize bucket listing request and deserialize response """ - self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ) if msg_code != MSG_CODE_LIST_BUCKETS_RESP: raise RiakError("unexpected protocol buffer message code: %d"%msg_code) return resp.buckets @@ -392,6 +389,7 @@ def send_msg_code(self, msg_code): self.maybe_connect() pkt = struct.pack("!iB", 1, msg_code) self._sock.send(pkt) + return self.recv_msg() def encode_msg(self, msg_code, msg): str = msg.SerializeToString() From 65b5dafae02c960c605028e5f2d6bf001ca36d33 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Sun, 6 Nov 2011 18:27:39 -0500 Subject: [PATCH 0134/1060] Shuffle recv_msg() into the return value for send_msg(). Additionally, to support the methods where multiple recv_msg() calls are made (multiple responses), a new method is introduced which does the upfront request in preparation for the following receives. --- riak/transports/pbc.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index f833f7b4..9d36ce9a 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -148,8 +148,7 @@ def set_client_id(self, client_id): req = riakclient_pb2.RpbSetClientIdReq() req.client_id = client_id - self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req) if msg_code == MSG_CODE_SET_CLIENT_ID_RESP: return True else: @@ -170,8 +169,7 @@ def get(self, robj, r = None, vtag = None): req.bucket = bucket.get_name() req.key = robj.get_key() - self.send_msg(MSG_CODE_GET_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_GET_REQ, req) if msg_code == MSG_CODE_GET_RESP: contents = [] for c in resp.content: @@ -200,8 +198,7 @@ def put(self, robj, w = None, dw = None, return_body = True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - self.send_msg(MSG_CODE_PUT_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req) if msg_code != MSG_CODE_PUT_RESP: raise RiakError("unexpected protocol buffer message code: %d"%msg_code) if resp is not None: @@ -230,8 +227,7 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - self.send_msg(MSG_CODE_PUT_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req) if msg_code != MSG_CODE_PUT_RESP: raise RiakError("unexpected protocol buffer message code: %d"%msg_code) if not resp: @@ -254,8 +250,7 @@ def delete(self, robj, rw = None): req.bucket = bucket.get_name() req.key = robj.get_key() - self.send_msg(MSG_CODE_DEL_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_DEL_REQ, req) if msg_code != MSG_CODE_DEL_RESP: raise RiakError("unexpected protocol buffer message code: %d"%msg_code) return self @@ -267,7 +262,7 @@ def get_keys(self, bucket): req = riakclient_pb2.RpbListKeysReq() req.bucket = bucket.get_name() - self.send_msg(MSG_CODE_LIST_KEYS_REQ, req) + self.send_msg_multi(MSG_CODE_LIST_KEYS_REQ, req) keys = [] while True: msg_code, resp = self.recv_msg() @@ -298,8 +293,7 @@ def get_bucket_props(self, bucket): req = riakclient_pb2.RpbGetBucketReq() req.bucket = bucket.get_name() - self.send_msg(MSG_CODE_GET_BUCKET_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_GET_BUCKET_REQ, req) if msg_code != MSG_CODE_GET_BUCKET_RESP: raise RiakError("unexpected protocol buffer message code: %d"%msg_code) props = {} @@ -324,8 +318,7 @@ def set_bucket_props(self, bucket, props): if 'allow_mult' in props: req.props.allow_mult = props['allow_mult'] - self.send_msg(MSG_CODE_SET_BUCKET_REQ, req) - msg_code, resp = self.recv_msg() + msg_code, resp = self.send_msg(MSG_CODE_SET_BUCKET_REQ, req) if msg_code != MSG_CODE_SET_BUCKET_RESP: raise RiakError("unexpected protocol buffer message code: %d"%msg_code) @@ -343,7 +336,7 @@ def mapred(self, inputs, query, timeout=None): req.request = content req.content_type = "application/json" - self.send_msg(MSG_CODE_MAPRED_REQ, req) + self.send_msg_multi(MSG_CODE_MAPRED_REQ, req) # dictionary of phase results - each content should be an encoded array # which is appended to the result for that phase. @@ -398,6 +391,10 @@ def encode_msg(self, msg_code, msg): return hdr + str def send_msg(self, msg_code, msg): + self.send_msg_multi(msg_code, msg) + return self.recv_msg() + + def send_msg_multi(self, msg_code, msg): self.maybe_connect() pkt = self.encode_msg(msg_code, msg) sent_len = self._sock.send(pkt) From 7a07e0328d88c1916d0a6217cf01c99877cf9275 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Sun, 6 Nov 2011 18:33:01 -0500 Subject: [PATCH 0135/1060] Fix the __copy__ method. Pass the ConnectionManager to the copy. --- riak/transports/pbc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 9d36ce9a..0c726da8 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -106,6 +106,7 @@ def __init__(self, cm, client_id=None, **unused_options): ### backwards compat. we don't use the ConnectionManager (yet). host, port = cm.hostports[0] + self._cm = cm self._host = host self._port = port self._client_id = client_id @@ -118,7 +119,7 @@ def translate_rw_val(self, rw): return val def __copy__(self): - return RiakPbcTransport(self._host, self._port) + return RiakPbcTransport(self._cm, self._client_id) def ping(self): """ From 9ef7b2aae153aa1f5eeea13f2ee4a36c0e74508b Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 9 Nov 2011 14:28:27 -0500 Subject: [PATCH 0136/1060] Consolidate checking of the response's message code. --- riak/transports/pbc.py | 74 ++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 42 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 0c726da8..e050135f 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -126,7 +126,8 @@ def ping(self): Ping the remote server @return boolean """ - msg_code, msg = self.send_msg_code(MSG_CODE_PING_REQ) + # An expected response code of None implies "any response is valid". + msg_code, msg = self.send_msg_code(MSG_CODE_PING_REQ, None) if msg_code == MSG_CODE_PING_RESP: return 1 else: @@ -136,11 +137,9 @@ def get_client_id(self): """ Get the client id used by this connection """ - msg_code, resp = self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ) - if msg_code == MSG_CODE_GET_CLIENT_ID_RESP: - return resp.client_id - else: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ, + MSG_CODE_GET_CLIENT_ID_RESP) + return resp.client_id def set_client_id(self, client_id): """ @@ -149,11 +148,9 @@ def set_client_id(self, client_id): req = riakclient_pb2.RpbSetClientIdReq() req.client_id = client_id - msg_code, resp = self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req) - if msg_code == MSG_CODE_SET_CLIENT_ID_RESP: - return True - else: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req, + MSG_CODE_SET_CLIENT_ID_RESP) + return True def get(self, robj, r = None, vtag = None): """ @@ -170,7 +167,8 @@ def get(self, robj, r = None, vtag = None): req.bucket = bucket.get_name() req.key = robj.get_key() - msg_code, resp = self.send_msg(MSG_CODE_GET_REQ, req) + # An expected response code of None implies "any response is valid". + msg_code, resp = self.send_msg(MSG_CODE_GET_REQ, req, None) if msg_code == MSG_CODE_GET_RESP: contents = [] for c in resp.content: @@ -199,9 +197,8 @@ def put(self, robj, w = None, dw = None, return_body = True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req) - if msg_code != MSG_CODE_PUT_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req, + MSG_CODE_PUT_RESP) if resp is not None: contents = [] for c in resp.content: @@ -228,9 +225,8 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req) - if msg_code != MSG_CODE_PUT_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req, + MSG_CODE_PUT_RESP) if not resp: raise RiakError("missing response object") if len(resp.content) != 1: @@ -251,9 +247,8 @@ def delete(self, robj, rw = None): req.bucket = bucket.get_name() req.key = robj.get_key() - msg_code, resp = self.send_msg(MSG_CODE_DEL_REQ, req) - if msg_code != MSG_CODE_DEL_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_DEL_REQ, req, + MSG_CODE_DEL_RESP) return self def get_keys(self, bucket): @@ -266,9 +261,7 @@ def get_keys(self, bucket): self.send_msg_multi(MSG_CODE_LIST_KEYS_REQ, req) keys = [] while True: - msg_code, resp = self.recv_msg() - if msg_code != MSG_CODE_LIST_KEYS_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.recv_msg(MSG_CODE_LIST_KEYS_RESP) for key in resp.keys: keys.append(key) @@ -282,9 +275,8 @@ def get_buckets(self): """ Serialize bucket listing request and deserialize response """ - msg_code, resp = self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ) - if msg_code != MSG_CODE_LIST_BUCKETS_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ, + MSG_CODE_LIST_BUCKETS_RESP) return resp.buckets def get_bucket_props(self, bucket): @@ -294,9 +286,8 @@ def get_bucket_props(self, bucket): req = riakclient_pb2.RpbGetBucketReq() req.bucket = bucket.get_name() - msg_code, resp = self.send_msg(MSG_CODE_GET_BUCKET_REQ, req) - if msg_code != MSG_CODE_GET_BUCKET_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_GET_BUCKET_REQ, req, + MSG_CODE_GET_BUCKET_RESP) props = {} if resp.props.HasField('n_val'): props['n_val'] = resp.props.n_val @@ -319,10 +310,8 @@ def set_bucket_props(self, bucket, props): if 'allow_mult' in props: req.props.allow_mult = props['allow_mult'] - msg_code, resp = self.send_msg(MSG_CODE_SET_BUCKET_REQ, req) - if msg_code != MSG_CODE_SET_BUCKET_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) - + msg_code, resp = self.send_msg(MSG_CODE_SET_BUCKET_REQ, req, + MSG_CODE_SET_BUCKET_RESP) return self def mapred(self, inputs, query, timeout=None): @@ -343,9 +332,7 @@ def mapred(self, inputs, query, timeout=None): # which is appended to the result for that phase. result = {} while True: - msg_code, resp = self.recv_msg() - if msg_code != MSG_CODE_MAPRED_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.recv_msg(MSG_CODE_MAPRED_RESP) if resp.HasField("phase") and resp.HasField("response"): content = json.loads(resp.response) if resp.phase in result: @@ -379,11 +366,11 @@ def maybe_connect(self): if self._client_id: self.set_client_id(self._client_id) - def send_msg_code(self, msg_code): + def send_msg_code(self, msg_code, expect): self.maybe_connect() pkt = struct.pack("!iB", 1, msg_code) self._sock.send(pkt) - return self.recv_msg() + return self.recv_msg(expect) def encode_msg(self, msg_code, msg): str = msg.SerializeToString() @@ -391,9 +378,9 @@ def encode_msg(self, msg_code, msg): hdr = struct.pack("!iB", 1 + slen, msg_code) return hdr + str - def send_msg(self, msg_code, msg): + def send_msg(self, msg_code, msg, expect): self.send_msg_multi(msg_code, msg) - return self.recv_msg() + return self.recv_msg(expect) def send_msg_multi(self, msg_code, msg): self.maybe_connect() @@ -403,7 +390,7 @@ def send_msg_multi(self, msg_code, msg): raise RiakError("PB socket returned short write %d - expected %d"%\ (sent_len, len(pkt))) - def recv_msg(self): + def recv_msg(self, expect): self.recv_pkt() msg_code, = struct.unpack("B", self._inbuf[:1]) if msg_code == MSG_CODE_ERROR_RESP: @@ -441,6 +428,9 @@ def recv_msg(self): msg.ParseFromString(self._inbuf[1:]) else: raise Exception("unknown msg code %s"%msg_code) + if expect and msg_code != expect: + raise RiakError("unexpected protocol buffer message code: %d" + % msg_code) return msg_code, msg From 240476c2c755c7e444c58d6c72ff7a3886939459 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 9 Nov 2011 14:45:50 -0500 Subject: [PATCH 0137/1060] Move reading responses into send_msg_multi. This also required rejiggering the three send functions a bit to rely on a new .send_pkt() which also consolidates connection creation and writing to the socket. --- riak/transports/pbc.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index e050135f..916df890 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -258,16 +258,12 @@ def get_keys(self, bucket): req = riakclient_pb2.RpbListKeysReq() req.bucket = bucket.get_name() - self.send_msg_multi(MSG_CODE_LIST_KEYS_REQ, req) keys = [] - while True: - msg_code, resp = self.recv_msg(MSG_CODE_LIST_KEYS_RESP) - + def _handle_response(resp): for key in resp.keys: keys.append(key) - - if resp.HasField("done") and resp.done: - break + self.send_msg_multi(MSG_CODE_LIST_KEYS_REQ, req, + MSG_CODE_LIST_KEYS_RESP, _handle_response) return keys @@ -326,22 +322,18 @@ def mapred(self, inputs, query, timeout=None): req.request = content req.content_type = "application/json" - self.send_msg_multi(MSG_CODE_MAPRED_REQ, req) - # dictionary of phase results - each content should be an encoded array # which is appended to the result for that phase. result = {} - while True: - msg_code, resp = self.recv_msg(MSG_CODE_MAPRED_RESP) + def _handle_response(resp): if resp.HasField("phase") and resp.HasField("response"): content = json.loads(resp.response) if resp.phase in result: result[resp.phase] += content else: result[resp.phase] = content - - if resp.HasField("done") and resp.done: - break; + self.send_msg_multi(MSG_CODE_MAPRED_REQ, req, MSG_CODE_MAPRED_RESP, + _handle_response) # If a single result - return the same as the HTTP interface does # otherwise return all the phase information @@ -367,9 +359,7 @@ def maybe_connect(self): self.set_client_id(self._client_id) def send_msg_code(self, msg_code, expect): - self.maybe_connect() - pkt = struct.pack("!iB", 1, msg_code) - self._sock.send(pkt) + self.send_pkt(struct.pack("!iB", 1, msg_code)) return self.recv_msg(expect) def encode_msg(self, msg_code, msg): @@ -379,12 +369,19 @@ def encode_msg(self, msg_code, msg): return hdr + str def send_msg(self, msg_code, msg, expect): - self.send_msg_multi(msg_code, msg) + self.send_pkt(self.encode_msg(msg_code, msg)) return self.recv_msg(expect) - def send_msg_multi(self, msg_code, msg): + def send_msg_multi(self, msg_code, msg, expect, handler): + self.send_pkt(self.encode_msg(msg_code, msg)) + while True: + msg_code, resp = self.recv_msg(expect) + handler(resp) + if resp.HasField("done") and resp.done: + break + + def send_pkt(self, pkt): self.maybe_connect() - pkt = self.encode_msg(msg_code, msg) sent_len = self._sock.send(pkt) if sent_len != len(pkt): raise RiakError("PB socket returned short write %d - expected %d"%\ From a7c05b01d013d17aeaf104b671aa2ed7ba021c97 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 9 Nov 2011 16:41:53 -0500 Subject: [PATCH 0138/1060] Create a new FactoryConnectionManager to simplify CM setup. This change makes it possible to avoid a CM subclass just to specify the connection class. --- riak/transports/connection.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/riak/transports/connection.py b/riak/transports/connection.py index 75669c6e..44dfad53 100644 --- a/riak/transports/connection.py +++ b/riak/transports/connection.py @@ -5,6 +5,7 @@ import httplib import socket import contextlib +import functools class ConnectionManager(object): @@ -132,10 +133,6 @@ def _new_connection(self): return conn -class HTTPConnectionManager(ConnectionManager): - connection_class = httplib.HTTPConnection - - class Socket(object): def __init__(self, host, port): @@ -160,8 +157,18 @@ def close(self): self.sock = None -class SocketConnectionManager(ConnectionManager): - connection_class = Socket +class FactoryConnectionManager(ConnectionManager): + + def __init__(self, connection_class, hostports=[]): + self.connection_class = connection_class + ConnectionManager.__init__(self, hostports) + + +def cm_using(connection_class): + return functools.partial(FactoryConnectionManager, connection_class) + +HTTPConnectionManager = cm_using(httplib.HTTPConnection) +SocketConnectionManager = cm_using(Socket) class NoHostsDefined(Exception): From 07226cd050a7d93d9cb97e1660d9d0cd6ac88f84 Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 9 Nov 2011 17:43:18 -0500 Subject: [PATCH 0139/1060] Make the protobuf transport use the ConnectionManager This uses a subclass of the generic connection.Socket since we need to record the connection-stateful value for client_id. There are certainly issues around using variant client_id values in a pooled environment (as noted by the old CachedTransport, and documented within the set_client_id method here). The send/recv sections now use cm.withconn() to acquire and use a connection, which is then passed down into the methods which do the actual send/recv of the bits. This also switches over to sock.sendall() to try and avoid short writes. The host/port has been stripped out, since that is now managed within the connection manager. --- riak/transports/pbc.py | 122 +++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 53 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 916df890..e70aa9c2 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -26,12 +26,12 @@ except ImportError: import simplejson as json -from transport import RiakTransport +from riak.transports.transport import RiakTransport from riak.metadata import * from riak.mapreduce import RiakMapReduce, RiakLink from riak import RiakError from riak.riak_index_entry import RiakIndexEntry -from connection import SocketConnectionManager +from riak.transports import connection try: import riakclient_pb2 @@ -71,6 +71,24 @@ RIAKC_RW_DEFAULT = 4294967291 +class SocketWithId(connection.Socket): + def __init__(self, host, port): + connection.Socket.__init__(self, host, port) + self.last_client_id = None + + def maybe_connect(self): + # If we're going to establish a new connection, then reset the last + # client_id used on this connection. + if self.sock is None: + self.last_client_id = None + + connection.Socket.maybe_connect(self) + + def send(self, pkt): + self.sock.sendall(pkt) + + def recv(self, want_len): + return self.sock.recv(want_len) class RiakPbcTransport(RiakTransport): @@ -90,27 +108,19 @@ class RiakPbcTransport(RiakTransport): } # The ConnectionManager class that this transport prefers. - default_cm = SocketConnectionManager + default_cm = connection.cm_using(SocketWithId) def __init__(self, cm, client_id=None, **unused_options): """ Construct a new RiakPbcTransport object. - @param string host - Hostname or IP address (default '127.0.0.1') - @param int port - Port number (default 8087) """ if riakclient_pb2 is None: raise RiakError("this transport is not available (no protobuf)") super(RiakPbcTransport, self).__init__() - ### backwards compat. we don't use the ConnectionManager (yet). - host, port = cm.hostports[0] - self._cm = cm - self._host = host - self._port = port self._client_id = client_id - self._sock = None def translate_rw_val(self, rw): val = self.rw_names.get(rw) @@ -150,6 +160,18 @@ def set_client_id(self, client_id): msg_code, resp = self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req, MSG_CODE_SET_CLIENT_ID_RESP) + + # Using different client_id values across connections is a bad idea + # since you never know which connection you might use for a given + # API call. Setting the client_id manually (rather than as part of + # the transport construction) can be error-prone since the connection + # could drop and be reinstated using self._client_id. + # + # To minimize the potential impact of variant client_id values across + # connections, we'll store this new client_id and use it for all + # future connections. + self._client_id = client_id + return True def get(self, robj, r = None, vtag = None): @@ -344,23 +366,10 @@ def _handle_response(resp): else: return result - - def maybe_connect(self): - if self._sock is None: - self._sock = s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - try: - s.connect((self._host, self._port)) - except: - self._sock = None - raise - - if self._client_id: - self.set_client_id(self._client_id) - def send_msg_code(self, msg_code, expect): - self.send_pkt(struct.pack("!iB", 1, msg_code)) - return self.recv_msg(expect) + with self._cm.withconn() as conn: + self.send_pkt(conn, struct.pack("!iB", 1, msg_code)) + return self.recv_msg(conn, expect) def encode_msg(self, msg_code, msg): str = msg.SerializeToString() @@ -369,26 +378,37 @@ def encode_msg(self, msg_code, msg): return hdr + str def send_msg(self, msg_code, msg, expect): - self.send_pkt(self.encode_msg(msg_code, msg)) - return self.recv_msg(expect) + with self._cm.withconn() as conn: + self.send_pkt(conn, self.encode_msg(msg_code, msg)) + if msg_code == MSG_CODE_SET_CLIENT_ID_REQ: + conn.last_client_id = self._client_id + return self.recv_msg(conn, expect) def send_msg_multi(self, msg_code, msg, expect, handler): - self.send_pkt(self.encode_msg(msg_code, msg)) - while True: - msg_code, resp = self.recv_msg(expect) - handler(resp) - if resp.HasField("done") and resp.done: - break - - def send_pkt(self, pkt): - self.maybe_connect() - sent_len = self._sock.send(pkt) - if sent_len != len(pkt): - raise RiakError("PB socket returned short write %d - expected %d"%\ - (sent_len, len(pkt))) - - def recv_msg(self, expect): - self.recv_pkt() + with self._cm.withconn() as conn: + self.send_pkt(conn, self.encode_msg(msg_code, msg)) + while True: + msg_code, resp = self.recv_msg(conn, expect) + handler(resp) + if resp.HasField("done") and resp.done: + break + + def send_pkt(self, conn, pkt): + conn.maybe_connect() + + # If the last client_id used on this connection is different than our + # client_id, then set a new ID on the connection. + if conn.last_client_id != self._client_id: + req = riakclient_pb2.RpbSetClientIdReq() + req.client_id = self._client_id + conn.send(self.encode_msg(MSG_CODE_SET_CLIENT_ID_REQ, req)) + conn.last_client_id = self._client_id + self.recv_msg(conn, MSG_CODE_SET_CLIENT_ID_RESP) + + conn.send(pkt) + + def recv_msg(self, conn, expect): + self.recv_pkt(conn) msg_code, = struct.unpack("B", self._inbuf[:1]) if msg_code == MSG_CODE_ERROR_RESP: msg = riakclient_pb2.RpbErrorResp() @@ -431,10 +451,9 @@ def recv_msg(self, expect): return msg_code, msg - def recv_pkt(self): - nmsglen = self._sock.recv(4) + def recv_pkt(self, conn): + nmsglen = conn.recv(4) if len(nmsglen) != 4: - self._sock = None raise RiakError("Socket returned short packet length %d - expected 4"%\ len(nmsglen)) msglen, = struct.unpack('!i', nmsglen) @@ -442,7 +461,7 @@ def recv_pkt(self): self._inbuf = '' while len(self._inbuf) < msglen: want_len = min(8192, msglen - len(self._inbuf)) - recv_buf = self._sock.recv(want_len) + recv_buf = conn.recv(want_len) if not recv_buf: break self._inbuf += recv_buf if len(self._inbuf) != self._inbuf_len: @@ -536,7 +555,7 @@ class RiakPbcCachedTransport(RiakTransport): api = 2 # The ConnectionManager class that this transport prefers. - default_cm = SocketConnectionManager + default_cm = connection.cm_using(SocketWithId) def __init__(self, cm, client_id=None, maxsize=0, block=False, timeout=None, @@ -545,11 +564,8 @@ def __init__(self, cm, raise RiakError("this transport is not available (no protobuf)") ### backwards compat. we don't use the ConnectionManager (yet). - host, port = cm.hostports[0] self._cm = cm - self.host = host - self.port = port self.client_id = client_id self.block = block self.timeout = timeout From 215f3c67e916883dcbeec791cf63347af90ad86b Mon Sep 17 00:00:00 2001 From: Greg Stein Date: Wed, 9 Nov 2011 18:14:07 -0500 Subject: [PATCH 0140/1060] Add license headers. --- riak/transports/connection.py | 20 +++++++++++++++++--- riak/transports/monitor.py | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/riak/transports/connection.py b/riak/transports/connection.py index 75669c6e..6aa14029 100644 --- a/riak/transports/connection.py +++ b/riak/transports/connection.py @@ -1,6 +1,20 @@ -# -# ### docco -# +""" +Copyright 2011 Greg Stein + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" import httplib import socket diff --git a/riak/transports/monitor.py b/riak/transports/monitor.py index 6380eee1..c6223518 100644 --- a/riak/transports/monitor.py +++ b/riak/transports/monitor.py @@ -1,3 +1,21 @@ +""" +Copyright 2011 Greg Stein + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + import threading import time From c2252b2209c3bf4fbf7e2702ac082e916601d438 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 17 Nov 2011 16:10:50 +0100 Subject: [PATCH 0141/1060] Only catch UnicodeEncodeError. --- riak/bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/bucket.py b/riak/bucket.py index b1af10b7..b556c9db 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -216,7 +216,7 @@ def new(self, key, data=None, content_type='application/json'): try: if isinstance(data, basestring): data.encode('ascii') - except: + except UnicodeEncodeError: raise TypeError('Unicode data values are not supported.') obj = RiakObject(self._client, self, key) From 69c1cb0b5f40a31a6d2fc8b584c68a60578d5765 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 18 Nov 2011 16:38:24 +0100 Subject: [PATCH 0142/1060] Unit tests Adjust unit tests so that they verify the patch, and then adjust the patch so that the unit tests pass :) --- riak/bucket.py | 8 ++++---- riak/mapreduce.py | 4 ++-- riak/riak_object.py | 4 ++-- riak/tests/test_all.py | 27 +++++++++++++++++++++------ 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index b556c9db..83818fa6 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -40,8 +40,8 @@ def __init__(self, client, name): """ try: if isinstance(name, basestring): - name.encode('ascii') - except UnicodeEncodeError: + name = name.encode('ascii') + except UnicodeError: raise TypeError('Unicode bucket names are not supported.') self._client = client @@ -215,8 +215,8 @@ def new(self, key, data=None, content_type='application/json'): """ try: if isinstance(data, basestring): - data.encode('ascii') - except UnicodeEncodeError: + data = data.encode('ascii') + except UnicodeError: raise TypeError('Unicode data values are not supported.') obj = RiakObject(self._client, self, key) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 3b7ce54d..fa7d4f2b 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -322,8 +322,8 @@ def __init__(self, type, function, language, keep, arg): """ try: if isinstance(function, basestring): - function.encode('ascii') - except UnicodeEncodeError: + function = function.encode('ascii') + except UnicodeError: raise TypeError('Unicode encoded functions are not supported.') self._type = type diff --git a/riak/riak_object.py b/riak/riak_object.py index 6b03d217..f91fb318 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -41,8 +41,8 @@ def __init__(self, client, bucket, key=None): """ try: if isinstance(key, basestring): - key.encode('ascii') - except UnicodeEncodeError: + key = key.encode('ascii') + except UnicodeError: raise TypeError('Unicode keys are not supported.') self._client = client diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6377b685..1e3e7864 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -115,12 +115,20 @@ def test_store_and_get(self): self.assertEqual(obj.get_key(), 'foo') self.assertEqual(obj.get_data(), rand) - #unicode input should raise a TypeError, - #to avoid issues further down the line - self.assertRaises(TypeError, self.client.bucket, u'bucket') + # unicode objects are fine, as long as they don't + # contain any non-ASCII chars + self.client.bucket(u'bucket') + self.assertRaises(TypeError, self.client.bucket, u'búcket') + self.assertRaises(TypeError, self.client.bucket, 'búcket') + + bucket.get(u'foo') + self.assertRaises(TypeError, bucket.get, u'føø') + self.assertRaises(TypeError, bucket.get, 'føø') + self.assertRaises(TypeError, bucket.new, u'foo', 'éå') + self.assertRaises(TypeError, bucket.new, u'foo', 'éå') + self.assertRaises(TypeError, bucket.new, 'foo', u'éå') self.assertRaises(TypeError, bucket.new, 'foo', u'éå') - self.assertRaises(TypeError, bucket.get, u'foo') def test_binary_store_and_get(self): bucket = self.client.bucket('bucket') @@ -275,9 +283,16 @@ def test_javascript_source_map(self): "function (v) { return [JSON.parse(v.values[0].data)]; }").run() self.assertEqual(result, [2]) - #test unicode function + # test ASCII-encodable unicode is accepted + mr.map(u"function (v) { return [JSON.parse(v.values[0].data)]; }") + + # test non-ASCII-encodable unicode is rejected + self.assertRaises(TypeError, mr.map, + u"function (v) { /* æ */ return [JSON.parse(v.values[0].data)]; }") + + # test non-ASCII-encodable string is rejected self.assertRaises(TypeError, mr.map, - u"function (v) { return [JSON.parse(v.values[0].data)]; }") + "function (v) { /* æ */ return [JSON.parse(v.values[0].data)]; }") def test_javascript_named_map(self): # Create the object... From bd74b800c2717601396322acd58b83cbadeed2ad Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Mon, 21 Nov 2011 10:36:13 +0000 Subject: [PATCH 0143/1060] Merge pull request #89 from gstein/deprecate Deprecate old transports These transports are obsoleted by the new ConnectionManager. --- riak/tests/test_all.py | 79 +--------------------- riak/transports/http.py | 108 +++--------------------------- riak/transports/pbc.py | 144 ++-------------------------------------- setup.py | 4 +- 4 files changed, 23 insertions(+), 312 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 8d8bee3c..439c5b29 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -19,8 +19,8 @@ import time from riak import RiakClient -from riak import RiakPbcTransport, RiakPbcCachedTransport -from riak import RiakHttpTransport, RiakHttpPoolTransport, RiakHttpReuseTransport +from riak import RiakPbcTransport +from riak import RiakHttpTransport from riak import RiakKeyFilter, key_filter from riak.riak_index_entry import RiakIndexEntry from riak.mapreduce import RiakLink @@ -31,11 +31,6 @@ HAVE_PROTO = True except ImportError: HAVE_PROTO = False -try: - import urllib3 - HAVE_HTTP_POOL = True -except ImportError: - HAVE_HTTP_POOL = False HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) @@ -872,26 +867,6 @@ def test_uses_client_id_if_given(self): self.assertEqual(zero_client_id, c.get_client_id()) # -class RiakPbcCachedTransportCase(BaseTestCase, MapReduceAliasTestMixIn, - unittest.TestCase): - def setUp(self): - if not HAVE_PROTO: - self.skipTest('protobuf is unavailable') - self.host = PB_HOST - self.port = PB_PORT - self.transport_class = RiakPbcCachedTransport - super(RiakPbcCachedTransportCase, self).setUp() - - def test_uses_client_id_if_given(self): - self.host = PB_HOST - self.port = PB_PORT - zero_client_id = "\0\0\0\0" - c = RiakClient(PB_HOST, PB_PORT, - transport_class = RiakPbcCachedTransport, - client_id = zero_client_id) - self.assertEqual(zero_client_id, c.get_client_id()) # - - class RiakHttpTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): @@ -1045,56 +1020,6 @@ def test_delete_documents_from_search_by_query_and_id(self): results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results["response"]["docs"])) -class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): - - def setUp(self): - if not HAVE_HTTP_POOL: - self.skipTest('urllib3 is unavailable') - self.host = HTTP_HOST - self.port = HTTP_PORT - self.transport_class = RiakHttpPoolTransport - super(RiakHttpPoolTransportTestCase, self).setUp() - - def test_no_returnbody(self): - bucket = self.client.bucket("bucket") - o = bucket.new("foo", "bar").store(return_body=False) - self.assertEqual(o.vclock(), None) - - def test_generate_key(self): - # Ensure that Riak generates a random key when - # the key passed to bucket.new() is None. - bucket = self.client.bucket('random_key_bucket') - for key in bucket.get_keys(): - bucket.get(str(key)).delete() - bucket.new(None, data={}).store() - self.assertEqual(len(bucket.get_keys()), 1) - - def test_set_client_id(self): - self.client.set_client_id("Client") - self.assertEqual(self.client.get_transport().get_client_id(), "Client") - -class RiakHttpReuseTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): - - def setUp(self): - self.host = HTTP_HOST - self.port = HTTP_PORT - self.transport_class = RiakHttpReuseTransport - super(RiakHttpReuseTransportTestCase, self).setUp() - - def test_no_returnbody(self): - bucket = self.client.bucket("bucket") - o = bucket.new("foo", "bar").store(return_body=False) - self.assertEqual(o.vclock(), None) - - def test_generate_key(self): - # Ensure that Riak generates a random key when - # the key passed to bucket.new() is None. - bucket = self.client.bucket('random_key_bucket') - for key in bucket.get_keys(): - bucket.get(str(key)).delete() - bucket.new(None, data={}).store() - self.assertEqual(len(bucket.get_keys()), 1) - class RiakTestFilter(unittest.TestCase): def test_simple(self): diff --git a/riak/transports/http.py b/riak/transports/http.py index 493eb27d..e90a8b6a 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -36,6 +36,7 @@ from riak.riak_index_entry import RiakIndexEntry from riak.multidict import MultiDict from connection import HTTPConnectionManager +import riak.util MAX_LINK_HEADER_SIZE = 8192 - 8 # substract length of "Link: " header string and newline @@ -519,111 +520,24 @@ def parse_http_headers(cls, headers) : retVal[key] = value return retVal -import socket class RiakHttpReuseTransport(RiakHttpTransport): - """ - Reuse sockets - """ - + "Deprecated transport." def __init__(self, cm, prefix='riak', mapred_prefix='mapred', client_id=None, **unused_options): - super(RiakHttpReuseTransport, self).__init__(cm, - prefix, - mapred_prefix, - client_id) - ### for backwards compat - self._host, self._port = cm.hostports[0] + RiakHttpTransport.__init__(self, cm, prefix, mapred_prefix, + client_id, **unused_options) + riak.util.deprecated('please use RiakHttpTransport instead', + stacklevel=4) - def __copy__(self): - return RiakHttpReuseTransport(self._conns, self._prefix, - self._mapred_prefix) - - def http_request(self, method, uri, headers=None, body=''): - if headers is None: - headers = {} - # Run the request... - client = None - response = None - try: - client = httplib.HTTPConnection(self._host, self._port) - - #handle the connection myself, try to reuse sockets - client.auto_open = 0 - client.connect() - client.sock.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, - client.sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) | 1) - - client.request(method, uri, body, headers) - response = client.getresponse() - - # Get the response headers... - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value - - # Get the body... - response_body = response.read() - response.close() - - #close, this does not make any difference - client.close() - - return response_headers, response_body - except: - if client is not None: client.close() - if response is not None: response.close() - raise - -try: - import urllib3 -except ImportError: - urllib3 = None class RiakHttpPoolTransport(RiakHttpTransport): - """ - Use HTTP pool - """ - - http_pool = None - + "Deprecated transport." def __init__(self, cm, prefix='riak', mapred_prefix='mapred', client_id=None, **unused_options): - if urllib3 is None: - raise RiakError("this transport is not available (no urllib3)") - - super(RiakHttpPoolTransport, self).__init__(cm, - prefix, - mapred_prefix, - client_id) - ### for backwards compat - self._host, self._port = cm.hostports[0] - - def __copy__(self): - return RiakHttpPoolTransport(self._conns, self._prefix, - self._mapred_prefix) - - def http_request(self, method, uri, headers={}, body=''): - if headers is None: - headers = {} - try: - ### it seems wrong to put the pool into a *class* variable, - ### but this code is supporting backwards-compat where the - ### use of a class variable was the design. - if self.__class__.http_pool is None: - self.__class__.http_pool = urllib3.connection_from_url('http://%s:%d' % (self._host, self._port), maxsize=10) - - response = self.http_pool.urlopen(method, uri, body, headers) - - response_headers = {'http_code': response.status} - for key, value in response.getheaders().iteritems(): - response_headers[key.lower()] = value - - response_body = response.data - - return response_headers, response_body - except: - raise + RiakHttpTransport.__init__(self, cm, prefix, mapred_prefix, + client_id, **unused_options) + riak.util.deprecated('please use RiakHttpTransport instead', + stacklevel=4) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index e70aa9c2..3ca3d210 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -32,6 +32,8 @@ from riak import RiakError from riak.riak_index_entry import RiakIndexEntry from riak.transports import connection +from connection import SocketConnectionManager +import riak.util try: import riakclient_pb2 @@ -546,143 +548,11 @@ def pbify_content(self, metadata, data, rpb_content) : pb_link.tag = link.get_tag() rpb_content.value = data -from Queue import Empty, Full, Queue -import contextlib -class RiakPbcCachedTransport(RiakTransport): - """Threadsafe pool of PBC connections, based on urllib3's pool [aka Queue]""" - - # We're using the new RiakTransport API - api = 2 - - # The ConnectionManager class that this transport prefers. - default_cm = connection.cm_using(SocketWithId) - +class RiakPbcCachedTransport(RiakPbcTransport): + "Deprecated transport." def __init__(self, cm, client_id=None, maxsize=0, block=False, timeout=None, **unused_options): - if riakclient_pb2 is None: - raise RiakError("this transport is not available (no protobuf)") - - ### backwards compat. we don't use the ConnectionManager (yet). - self._cm = cm - - self.client_id = client_id - self.block = block - self.timeout = timeout - - self.pool = Queue(maxsize) - # Fill the queue up so that doing get() on it will block properly (check Queue#get) - [self.pool.put(None) for _ in xrange(maxsize)] - - def _new_connection(self): - """New PBC connection""" - return RiakPbcTransport(self._cm, self.client_id) - - def _get_connection(self): - connection = None - try: - connection = self.pool.get(block=self.block, timeout=self.timeout) - except Empty: - pass - return connection or self._new_connection() - - def _put_connection(self, connection): - try: - self.pool.put(connection, block=False) - except Full: - pass - - @contextlib.contextmanager - def _get_connection_from_pool(self): - """checkout conn, try operation, put conn back in pool""" - connection = self._get_connection() - try: - yield connection - finally: - self._put_connection(connection) - - def ping(self): - """ - Ping the remote server - @return boolean - """ - with self._get_connection_from_pool() as connection: - return connection.ping() - - def get(self, robj, r = None, vtag = None): - """ - Serialize get request and deserialize response - @return (vclock=None, [(metadata, value)]=None) - """ - with self._get_connection_from_pool() as connection: - return connection.get(robj, r, vtag) - - def put(self, robj, w = None, dw = None, return_body = True): - """ - Serialize put request and deserialize response - if 'content' - is true, retrieve the updated metadata/content - @return (vclock=None, [(metadata, value)]=None) - """ - with self._get_connection_from_pool() as connection: - return connection.put(robj, w, dw, return_body) - - def put_new(self, robj, w=None, dw=None, return_meta=True): - """Put a new object into the Riak store, returning its (new) key. - - If return_meta is False, then the vlock and metadata return values - will be None. - - @return (key, vclock, metadata) - """ - with self._get_connection_from_pool() as connection: - return connection.put_new(robj, w, dw, return_meta) - - def delete(self, robj, rw = None): - """ - Serialize delete request and deserialize response - @return true - """ - with self._get_connection_from_pool() as connection: - return connection.delete(robj, rw) - - def get_buckets(self): - """ - Serialize bucket listing request and deserialize response - """ - with self._get_connection_from_pool() as connection: - return connection.get_buckets() - - def get_bucket_props(self, bucket) : - """ - Serialize get bucket property request and deserialize response - @return dict() - """ - with self._get_connection_from_pool() as connection: - return connection.get_bucket_props(bucket) - - def set_bucket_props(self, bucket, props) : - """ - Serialize set bucket property request and deserialize response - bucket = bucket object - props = dictionary of properties - @return boolean - """ - with self._get_connection_from_pool() as connection: - return connection.set_bucket_props(bucket, props) - - def mapred(self, inputs, query, timeout = None) : - """ - Serialize map/reduce request - """ - with self._get_connection_from_pool() as connection: - return connection.mapred(inputs, query, timeout) - - def set_client_id(self, client_id): - """Mmm, this can turn ugly if you use different id for different objects in the pool""" - with self._get_connection_from_pool() as connection: - return connection.set_client_id(client_id) - - def get_client_id(self): - """see set_client_id notes, you can do wrong with this""" - with self._get_connection_from_pool() as connection: - return connection.get_client_id() + RiakPbcTransport.__init__(self, cm, client_id, **unused_options) + riak.util.deprecated('please use RiakPbcTransport instead', + stacklevel=4) diff --git a/setup.py b/setup.py index ae14401b..0af5d88a 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,9 @@ def make_pb(): name='riak', version='1.3.0', packages = find_packages(), - install_requires = ['protobuf>=2.3.0', 'urllib3>=0.4.0'], + extras_require = { + 'protobuf': ['protobuf>=2.3.0'], + }, dependency_links = ["http://downloads.basho.com/support"], package_data = { '' : ['*.proto'], From 54e9577c92e8926822dbfa199f1f2af7c75ff111 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Mon, 16 Jan 2012 21:18:30 -0500 Subject: [PATCH 0144/1060] Tutorial rst syntax fix. --- docs/tutorial.rst | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index ab780da7..f1fe1994 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -398,26 +398,26 @@ the object by querying the metadata, returning a list of matching keys. Your Riak cluster must have Secondary Indexes enabled. See the Riak documentation for details. -Usage of this feature looks like: +Usage of this feature looks like:: - import riak + import riak - client = riak.RiakClient() - bucket = client.bucket('mybucket') + client = riak.RiakClient() + bucket = client.bucket('mybucket') - # Create and store the object with indexes... - obj = bucket.new('mykey1', 'mydata') - obj.add_index('field1_bin', 'val1') - obj.add_index('field2_int', 1001) - obj.store() + # Create and store the object with indexes... + obj = bucket.new('mykey1', 'mydata') + obj.add_index('field1_bin', 'val1') + obj.add_index('field2_int', 1001) + obj.store() - # Query the indexes. The return value is a list of ``RiakLink`` objects. - results = client.index('mybucket', 'field1_bin', 'val1').run() + # Query the indexes. The return value is a list of ``RiakLink`` objects. + results = client.index('mybucket', 'field1_bin', 'val1').run() - # Query the indexes using a range... - results = client.index('mybucket', 'field1_bin', 'val1', 'val5').run() + # Query the indexes using a range... + results = client.index('mybucket', 'field1_bin', 'val1', 'val5').run() - # Remove an index entry... - obj = bucket.get('mykey1') - obj.remove_index('field1_bin', 'val1') - obj.store() + # Remove an index entry... + obj = bucket.get('mykey1') + obj.remove_index('field1_bin', 'val1') + obj.store() From be908fcf5aaa578f1375359b7b31b8d08e0ae7a4 Mon Sep 17 00:00:00 2001 From: Jeffrey Massung Date: Wed, 25 Jan 2012 15:45:57 -0700 Subject: [PATCH 0145/1060] Pull the check for 2i outside the tests and call it at the start of each 2i test. --- riak/tests/test_all.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 8c14b84b..edd56e0d 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -536,8 +536,21 @@ def test_list_buckets(self): buckets = self.client.get_buckets() self.assertTrue("list_bucket" in buckets) + def is_2i_supported(self): + # Immediate test to see if 2i is even supported w/ the backend + try: + self.client.index('foo','bar_bin','baz').run() + return True + except Exception as e: + if "indexes_not_supported" in str(e): + return False + raise e # re-raise to fail the test + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_store(self): + if not self.is_2i_supported(): + return True + # Create a new object with indexes... bucket = self.client.bucket('indexbucket') rand = self.randint() @@ -609,6 +622,9 @@ def test_secondary_index_store(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_query(self): + if not self.is_2i_supported(): + return True + bucket = self.client.bucket('indexbucket') bucket.\ @@ -632,13 +648,6 @@ def test_secondary_index_query(self): add_index('field2_int', 1004).\ store() - # Immediate test to see if 2i is even supported w/ the backend - try: - self.client.index('foo','bar_bin','baz').run() - except Exception as e: - if "indexes_not_supported" in str(e): - return True - # Test an equality query... results = self.client.index('indexbucket', 'field1_bin', 'val2').run() self.assertEquals(1, len(results)) From d77fd766f1b10bfe95d796a9c55080f67928cd91 Mon Sep 17 00:00:00 2001 From: Jeffrey Massung Date: Wed, 25 Jan 2012 16:14:18 -0700 Subject: [PATCH 0146/1060] Indexes are supported if the server error message isn't that they are not. --- riak/tests/test_all.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index edd56e0d..cd00fe8c 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -540,11 +540,10 @@ def is_2i_supported(self): # Immediate test to see if 2i is even supported w/ the backend try: self.client.index('foo','bar_bin','baz').run() - return True except Exception as e: if "indexes_not_supported" in str(e): return False - raise e # re-raise to fail the test + raise True # re-raise to fail the test @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_store(self): From 3d71cef488d620d0dea432864e106a55d093278b Mon Sep 17 00:00:00 2001 From: Jeffrey Massung Date: Wed, 25 Jan 2012 16:17:13 -0700 Subject: [PATCH 0147/1060] Handling extra exit condition - success. --- riak/tests/test_all.py | 1 + 1 file changed, 1 insertion(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index cd00fe8c..acb35022 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -540,6 +540,7 @@ def is_2i_supported(self): # Immediate test to see if 2i is even supported w/ the backend try: self.client.index('foo','bar_bin','baz').run() + return True except Exception as e: if "indexes_not_supported" in str(e): return False From 3a74100f78ca011a733be97efa7f7855e8cff8a3 Mon Sep 17 00:00:00 2001 From: Jeffrey Massung Date: Wed, 25 Jan 2012 17:31:18 -0700 Subject: [PATCH 0148/1060] Fixed a typo.. raise -> return. --- riak/tests/test_all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index acb35022..6de3f2d4 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -544,7 +544,7 @@ def is_2i_supported(self): except Exception as e: if "indexes_not_supported" in str(e): return False - raise True # re-raise to fail the test + return True # it failed, but is supported! @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_store(self): From d00d4f58077c58c22526b466a80b642f9af644bf Mon Sep 17 00:00:00 2001 From: Jeffrey Massung Date: Mon, 30 Jan 2012 15:35:38 -0700 Subject: [PATCH 0149/1060] Removed hack check for 2i enabled. --- riak/tests/test_all.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 8c14b84b..61bb6b16 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -632,13 +632,6 @@ def test_secondary_index_query(self): add_index('field2_int', 1004).\ store() - # Immediate test to see if 2i is even supported w/ the backend - try: - self.client.index('foo','bar_bin','baz').run() - except Exception as e: - if "indexes_not_supported" in str(e): - return True - # Test an equality query... results = self.client.index('indexbucket', 'field1_bin', 'val2').run() self.assertEquals(1, len(results)) From 24027b50cf6d2ebd5b3bfe13af37ec8c871cbfc2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 7 Feb 2012 16:28:01 -0500 Subject: [PATCH 0150/1060] Require unittest2 on Python 2.6. --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0af5d88a..7bcb9360 100755 --- a/setup.py +++ b/setup.py @@ -2,9 +2,9 @@ import glob import os import subprocess +import platform from setuptools import setup, find_packages - def make_docs(): if not os.path.exists('docs'): os.mkdir('docs') @@ -16,10 +16,16 @@ def make_pb(): subprocess.call(['protoc', '--python_out=.', './riak/transports/riakclient.proto']) if __name__ == "__main__": + if platform.python_version() < '2.7': + test_require = ["unittest2"] + else: + test_require = [] + setup( name='riak', version='1.3.0', packages = find_packages(), + requires = test_require, extras_require = { 'protobuf': ['protobuf>=2.3.0'], }, From bc0decf25ba621c610aa4a341636c1a60a179fbc Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 7 Feb 2012 16:33:34 -0500 Subject: [PATCH 0151/1060] Remove deprecated transports. --- riak/__init__.py | 4 ++-- riak/transports/__init__.py | 4 ++-- riak/transports/http.py | 22 ---------------------- riak/transports/pbc.py | 9 --------- 4 files changed, 4 insertions(+), 35 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index d8577d1d..8a334910 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -41,8 +41,8 @@ def __str__(self): from client import RiakClient from mapreduce import RiakMapReduce, RiakMapReducePhase, RiakLinkPhase,\ RiakKeyFilter -from transports.pbc import RiakPbcTransport, RiakPbcCachedTransport -from transports.http import RiakHttpTransport, RiakHttpReuseTransport, RiakHttpPoolTransport +from transports.pbc import RiakPbcTransport +from transports.http import RiakHttpTransport ONE = "one" ALL = "all" diff --git a/riak/transports/__init__.py b/riak/transports/__init__.py index f970fed9..55b7da3f 100644 --- a/riak/transports/__init__.py +++ b/riak/transports/__init__.py @@ -1,4 +1,4 @@ -from http import RiakHttpTransport, RiakHttpReuseTransport, RiakHttpPoolTransport -from pbc import RiakPbcTransport, RiakPbcCachedTransport +from http import RiakHttpTransport +from pbc import RiakPbcTransport diff --git a/riak/transports/http.py b/riak/transports/http.py index e90a8b6a..92d534b4 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -519,25 +519,3 @@ def parse_http_headers(cls, headers) : else: retVal[key] = value return retVal - - -class RiakHttpReuseTransport(RiakHttpTransport): - "Deprecated transport." - def __init__(self, cm, - prefix='riak', mapred_prefix='mapred', client_id=None, - **unused_options): - RiakHttpTransport.__init__(self, cm, prefix, mapred_prefix, - client_id, **unused_options) - riak.util.deprecated('please use RiakHttpTransport instead', - stacklevel=4) - - -class RiakHttpPoolTransport(RiakHttpTransport): - "Deprecated transport." - def __init__(self, cm, - prefix='riak', mapred_prefix='mapred', client_id=None, - **unused_options): - RiakHttpTransport.__init__(self, cm, prefix, mapred_prefix, - client_id, **unused_options) - riak.util.deprecated('please use RiakHttpTransport instead', - stacklevel=4) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 3ca3d210..150bf58d 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -547,12 +547,3 @@ def pbify_content(self, metadata, data, rpb_content) : pb_link.key = link.get_key() pb_link.tag = link.get_tag() rpb_content.value = data - -class RiakPbcCachedTransport(RiakPbcTransport): - "Deprecated transport." - def __init__(self, cm, - client_id=None, maxsize=0, block=False, timeout=None, - **unused_options): - RiakPbcTransport.__init__(self, cm, client_id, **unused_options) - riak.util.deprecated('please use RiakPbcTransport instead', - stacklevel=4) From c7bb6ab16666f4802c6384d564bb74e1079e6d31 Mon Sep 17 00:00:00 2001 From: Gregory Burd Date: Sat, 15 Oct 2011 17:39:35 -0400 Subject: [PATCH 0152/1060] Unquote items stored in the link header. Without this links with characters that must be encoded are never decoded. The unit test: python -m unittest riak.tests.test_all.RiakHttpPoolTransportTestCase did not check the content of the links, only that the right number of links are returned so the "foo3" bucket with a link containing special characters "tag2!@#%^&*)" results in the following when retrieved "tag2%21%40%23%25%5E%26%2A%29" --- riak/tests/test_all.py | 11 +++++++++++ riak/transports/http.py | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6de3f2d4..e708298d 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -440,6 +440,17 @@ def test_store_and_get_links(self): obj = bucket.get("foo") links = obj.get_links() self.assertEqual(len(links), 3) + for l in links: + if (l.get_key() == "foo1"): + self.assertEqual(l.get_tag(), "") + next + if (l.get_key() == "foo2"): + self.assertEqual(l.get_tag(), "tag") + next + if (l.get_key() == "foo3"): + self.assertEqual(l.get_tag(), "tag2!@#%^&*)") + next + self.assertEqual("unknown key", l.get_key()) def test_link_walking(self): # Create the object... diff --git a/riak/transports/http.py b/riak/transports/http.py index e90a8b6a..c0e38bfb 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -343,7 +343,9 @@ def parse_links(self, links, linkHeaders): linkHeader = linkHeader.strip() matches = re.match("; ?riaktag=\"([^\']+)\"", linkHeader) if matches is not None: - link = RiakLink(matches.group(2), matches.group(3), matches.group(4)) + link = RiakLink(urllib.unquote_plus(matches.group(2)), + urllib.unquote_plus(matches.group(3)), + urllib.unquote_plus(matches.group(4))) links.append(link) return self From f152ac1483cd5bb2330c00b1185dfbfacb09fdc0 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 11:56:37 -0800 Subject: [PATCH 0153/1060] Allow transport specific options to be passed through RiakClient. Added a new optional parameter transport_options, which is optionally provided and passed through to the transport class constructor. Allows custom settings to be passed through. --- riak/client.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/riak/client.py b/riak/client.py index bf1e2fcc..722f953d 100644 --- a/riak/client.py +++ b/riak/client.py @@ -39,7 +39,8 @@ class RiakClient(object): """ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', mapred_prefix='mapred', transport_class=None, - client_id=None, solr_transport_class=None): + client_id=None, solr_transport_class=None, + transport_options=None): """ Construct a new ``RiakClient`` object. @@ -55,6 +56,8 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', :type transport_class: :class:`RiakTransport` :param solr_transport_class: HTTP-based transport class for Solr interface queries :type transport_class: :class:`RiakHttpTransport` + :param transport_options: Optional key-value args to pass to the transport constuctor + :type transport_options: dict """ if transport_class is None: transport_class = RiakHttpTransport @@ -64,9 +67,10 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', hostports = [ (host, port), ] self._cm = transport_class.default_cm(hostports) - ### we need to allow additional transport options. make this an - ### argument to __init__ ? - transport_options = { } + # If no transport options are provided, then default to the + # empty dict, otherwise just pass through what we are provided. + if transport_options is None: + transport_options = {} self._transport = transport_class(self._cm, prefix=prefix, From e90f5c3ddc241475a57faf386d71ccbde3c42a91 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 11:42:04 -0800 Subject: [PATCH 0154/1060] Fixing PEP8 compliance, styling, and imports. Changed some code to be more PEP8 compliant in terms of style guides. Reworked the imports to be nicer, and avoid the splat-style import. This allows pyflakes to pickup any missing imports. --- riak/transports/pbc.py | 60 +++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 150bf58d..7a238ea1 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -19,20 +19,31 @@ """ from __future__ import with_statement -import socket, struct +import errno +import socket +import struct try: import json except ImportError: import simplejson as json -from riak.transports.transport import RiakTransport -from riak.metadata import * -from riak.mapreduce import RiakMapReduce, RiakLink +from riak.metadata import ( + MD_CHARSET, + MD_CTYPE, + MD_ENCODING, + MD_INDEX, + MD_LASTMOD, + MD_LASTMOD_USECS, + MD_LINKS, + MD_USERMETA, + MD_VTAG, + ) from riak import RiakError +from riak.mapreduce import RiakLink from riak.riak_index_entry import RiakIndexEntry from riak.transports import connection -from connection import SocketConnectionManager +from riak.transports.transport import RiakTransport import riak.util try: @@ -103,10 +114,10 @@ class RiakPbcTransport(RiakTransport): api = 2 rw_names = { - 'default' : RIAKC_RW_DEFAULT, - 'all' : RIAKC_RW_ALL, - 'quorum' : RIAKC_RW_QUORUM, - 'one' : RIAKC_RW_ONE + 'default': RIAKC_RW_DEFAULT, + 'all': RIAKC_RW_ALL, + 'quorum': RIAKC_RW_QUORUM, + 'one': RIAKC_RW_ONE } # The ConnectionManager class that this transport prefers. @@ -176,7 +187,7 @@ def set_client_id(self, client_id): return True - def get(self, robj, r = None, vtag = None): + def get(self, robj, r=None, vtag=None): """ Serialize get request and deserialize response """ @@ -201,7 +212,7 @@ def get(self, robj, r = None, vtag = None): else: return 0 - def put(self, robj, w = None, dw = None, return_body = True): + def put(self, robj, w=None, dw=None, return_body=True): """ Serialize get request and deserialize response """ @@ -259,7 +270,7 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): metadata, content = self.decode_content(resp.content[0]) return resp.key, resp.vclock, metadata - def delete(self, robj, rw = None): + def delete(self, robj, rw=None): """ Serialize get request and deserialize response """ @@ -316,14 +327,14 @@ def get_bucket_props(self, bucket): return props - def set_bucket_props(self, bucket, props): """ Serialize set bucket property request and deserialize response """ req = riakclient_pb2.RpbSetBucketReq() req.bucket = bucket.get_name() - if not 'n_val' in props and not 'allow_mult' in props: return self + if not 'n_val' in props and not 'allow_mult' in props: + return self if 'n_val' in props: req.props.n_val = props['n_val'] @@ -336,7 +347,7 @@ def set_bucket_props(self, bucket, props): def mapred(self, inputs, query, timeout=None): # Construct the job, optionally set the timeout... - job = {'inputs':inputs, 'query':query} + job = {'inputs': inputs, 'query': query} if timeout is not None: job['timeout'] = timeout @@ -446,29 +457,29 @@ def recv_msg(self, conn, expect): msg = riakclient_pb2.RpbMapRedResp() msg.ParseFromString(self._inbuf[1:]) else: - raise Exception("unknown msg code %s"%msg_code) + raise Exception("unknown msg code %s" % msg_code) if expect and msg_code != expect: raise RiakError("unexpected protocol buffer message code: %d" % msg_code) return msg_code, msg - def recv_pkt(self, conn): nmsglen = conn.recv(4) if len(nmsglen) != 4: - raise RiakError("Socket returned short packet length %d - expected 4"%\ - len(nmsglen)) + raise RiakError("Socket returned short packet length %d - expected 4" + % len(nmsglen)) msglen, = struct.unpack('!i', nmsglen) self._inbuf_len = msglen self._inbuf = '' while len(self._inbuf) < msglen: want_len = min(8192, msglen - len(self._inbuf)) recv_buf = conn.recv(want_len) - if not recv_buf: break + if not recv_buf: + break self._inbuf += recv_buf if len(self._inbuf) != self._inbuf_len: - raise RiakError("Socket returned short packet %d - expected %d"%\ - (len(self._inbuf), self._inbuf_len)) + raise RiakError("Socket returned short packet %d - expected %d" + % (len(self._inbuf), self._inbuf_len)) def decode_contents(self, rpb_contents): contents = [] @@ -520,10 +531,10 @@ def decode_content(self, rpb_content): metadata[MD_INDEX] = indexes return metadata, rpb_content.value - def pbify_content(self, metadata, data, rpb_content) : + def pbify_content(self, metadata, data, rpb_content): # Convert the broken out fields, building up # pbmetadata for any unknown ones - for k,v in metadata.iteritems(): + for k, v in metadata.iteritems(): if k == MD_CTYPE: rpb_content.content_type = v elif k == MD_CHARSET: @@ -547,3 +558,4 @@ def pbify_content(self, metadata, data, rpb_content) : pb_link.key = link.get_key() pb_link.tag = link.get_tag() rpb_content.value = data + From 5fb87576f21053c277b3d1f851177b534a1ac2c1 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 11:47:39 -0800 Subject: [PATCH 0155/1060] Alphabetize the imports for pbc transport --- riak/transports/pbc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 7a238ea1..327bcb90 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -28,6 +28,8 @@ except ImportError: import simplejson as json +from riak import RiakError +from riak.mapreduce import RiakLink from riak.metadata import ( MD_CHARSET, MD_CTYPE, @@ -39,8 +41,6 @@ MD_USERMETA, MD_VTAG, ) -from riak import RiakError -from riak.mapreduce import RiakLink from riak.riak_index_entry import RiakIndexEntry from riak.transports import connection from riak.transports.transport import RiakTransport From c4e68488eff40ea4f012638de75e73215ccad7f9 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 11:55:53 -0800 Subject: [PATCH 0156/1060] PEP8 fixes and import re-ordering for riak client. Fix some minor styling to be PEP8 compliant. Re-order the imports. --- riak/client.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/riak/client.py b/riak/client.py index 722f953d..5e78fe27 100644 --- a/riak/client.py +++ b/riak/client.py @@ -23,12 +23,11 @@ except ImportError: import simplejson as json -from riak.transports import RiakHttpTransport from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduce from riak.search import RiakSearch +from riak.transports import RiakHttpTransport from riak.util import deprecated -import riak.transports.connection class RiakClient(object): @@ -86,10 +85,10 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', self._w = "default" self._dw = "default" self._rw = "default" - self._encoders = {'application/json':json.dumps, - 'text/json':json.dumps} - self._decoders = {'application/json':json.loads, - 'text/json':json.loads} + self._encoders = {'application/json': json.dumps, + 'text/json': json.dumps} + self._decoders = {'application/json': json.loads, + 'text/json': json.loads} self._solr = None self._host = host self._port = port @@ -288,7 +287,7 @@ def search(self, *args): def index(self, *args): """ Start assembling a Map/Reduce operation based on secondary - index query results. + index query results. :rtype: :class:`RiakMapReduce` """ From 602b7f205d762706764b8619c6daa68831e6daf7 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 13:01:52 -0800 Subject: [PATCH 0157/1060] Remove the import, moved to appropriate branch --- riak/transports/pbc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 327bcb90..21a47b72 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -19,7 +19,6 @@ """ from __future__ import with_statement -import errno import socket import struct From 6bb7d8dfd57e27862e749d705b402d09b362bde7 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 11:44:04 -0800 Subject: [PATCH 0158/1060] Improvements to SocketWithId SocketWithId will now uses super() to invoke methods from the parent class, since it is a new style class. send() and recv() now both catch any errors that indicate the socket is closed, and close the internal socket to prevent it from being re-used. --- riak/transports/pbc.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 21a47b72..e2ec1a35 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -82,10 +82,22 @@ RIAKC_RW_ALL = 4294967292 RIAKC_RW_DEFAULT = 4294967291 +# These are a specific set of socket errors +# that could be raised on send/recv that indicate +# that the socket is closed or reset, and is not +# usable. On seeing any of these errors, the socket +# should be closed, and the connection re-established. +CONN_CLOSED_ERRORS = ( + errno.EHOSTUNREACH, + errno.ECONNRESET, + errno.EBADF, + errno.EPIPE + ) + class SocketWithId(connection.Socket): def __init__(self, host, port): - connection.Socket.__init__(self, host, port) + super(SocketWithId, self).__init__(host, port) self.last_client_id = None def maybe_connect(self): @@ -93,14 +105,27 @@ def maybe_connect(self): # client_id used on this connection. if self.sock is None: self.last_client_id = None - - connection.Socket.maybe_connect(self) + super(SocketWithId, self).maybe_connect() def send(self, pkt): - self.sock.sendall(pkt) + try: + self.sock.sendall(pkt) + except socket.error, e: + # If the socket is in a bad state, close it and allow it + # to re-connect on the next try + if e[0] in CONN_CLOSED_ERRORS: + self.close() + raise def recv(self, want_len): - return self.sock.recv(want_len) + try: + return self.sock.recv(want_len) + except socket.error, e: + # If the socket is in a bad state, close it and allow it + # to re-connect on the next try + if e[0] in CONN_CLOSED_ERRORS: + self.close() + raise class RiakPbcTransport(RiakTransport): From e8a798dbaa69ace40efc08f30ea4b71ff2f779b4 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 11:58:52 -0800 Subject: [PATCH 0159/1060] Make RiakPbcTransport more robust to connection errors. Modified RiakPbcTransport to take an optional retries parameter, which is defaulted to 1 for backwards compatibility. send_pkt will make up to 'retries' attempts to send in the face of any error which indicates the connection has been closed/reset (CONN_CLOSED_ERRORS). --- riak/transports/pbc.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index e2ec1a35..810cd958 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -147,7 +147,7 @@ class RiakPbcTransport(RiakTransport): # The ConnectionManager class that this transport prefers. default_cm = connection.cm_using(SocketWithId) - def __init__(self, cm, client_id=None, **unused_options): + def __init__(self, cm, client_id=None, retries=1, **unused_options): """ Construct a new RiakPbcTransport object. """ @@ -158,6 +158,7 @@ def __init__(self, cm, client_id=None, **unused_options): self._cm = cm self._client_id = client_id + self._retries = retries def translate_rw_val(self, rw): val = self.rw_names.get(rw) @@ -431,18 +432,30 @@ def send_msg_multi(self, msg_code, msg, expect, handler): break def send_pkt(self, conn, pkt): - conn.maybe_connect() - - # If the last client_id used on this connection is different than our - # client_id, then set a new ID on the connection. - if conn.last_client_id != self._client_id: - req = riakclient_pb2.RpbSetClientIdReq() - req.client_id = self._client_id - conn.send(self.encode_msg(MSG_CODE_SET_CLIENT_ID_REQ, req)) - conn.last_client_id = self._client_id - self.recv_msg(conn, MSG_CODE_SET_CLIENT_ID_RESP) - - conn.send(pkt) + attempt, e = 0, None + for attempt in xrange(self._retries): + try: + conn.maybe_connect() + + # If the last client_id used on this connection is different than our + # client_id, then set a new ID on the connection. + if conn.last_client_id != self._client_id: + req = riakclient_pb2.RpbSetClientIdReq() + req.client_id = self._client_id + conn.send(self.encode_msg(MSG_CODE_SET_CLIENT_ID_REQ, req)) + conn.last_client_id = self._client_id + self.recv_msg(conn, MSG_CODE_SET_CLIENT_ID_RESP) + + conn.send(pkt) + except socket.error, e: + # If this is some unknown socket error bail out + # instead of retrying + if e[0] not in CONN_CLOSED_ERRORS: + raise + + # Max attempts reached, raise whatever exception we are getting + if attempt + 1 == self.retries and e is not None: + raise e def recv_msg(self, conn, expect): self.recv_pkt(conn) From 30068063852d554cdb73abdaa0ee5eab704e32bf Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 12:10:59 -0800 Subject: [PATCH 0160/1060] Rename the 'retries' argument to RiakPbcTransport to 'max_attempts' It is much more clear as max_attempts, since previously if retries was 3, there would be a total of 3 attempts, not 1 attempt and 3 retries as the name would indicate. --- riak/transports/pbc.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 810cd958..afd7f1fe 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -147,7 +147,7 @@ class RiakPbcTransport(RiakTransport): # The ConnectionManager class that this transport prefers. default_cm = connection.cm_using(SocketWithId) - def __init__(self, cm, client_id=None, retries=1, **unused_options): + def __init__(self, cm, client_id=None, max_attempts=1, **unused_options): """ Construct a new RiakPbcTransport object. """ @@ -158,7 +158,7 @@ def __init__(self, cm, client_id=None, retries=1, **unused_options): self._cm = cm self._client_id = client_id - self._retries = retries + self._max_attempts = max_attempts def translate_rw_val(self, rw): val = self.rw_names.get(rw) @@ -432,8 +432,10 @@ def send_msg_multi(self, msg_code, msg, expect, handler): break def send_pkt(self, conn, pkt): - attempt, e = 0, None - for attempt in xrange(self._retries): + attempt = 0 + e = None + for attempt in xrange(self._max_attempts): + e = None try: conn.maybe_connect() @@ -454,7 +456,7 @@ def send_pkt(self, conn, pkt): raise # Max attempts reached, raise whatever exception we are getting - if attempt + 1 == self.retries and e is not None: + if attempt + 1 == self._max_attempts and e is not None: raise e def recv_msg(self, conn, expect): From 0b5bc9d12388da640aa0e674f8912f625d42fdd0 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 12:32:04 -0800 Subject: [PATCH 0161/1060] Handle implicit socket close on SocketWithId If we do a blocking read with non-zero request size, and we get a 0 byte response, assume the socket is closed since this indicates EOF. --- riak/transports/pbc.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index afd7f1fe..c19de917 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -119,7 +119,15 @@ def send(self, pkt): def recv(self, want_len): try: - return self.sock.recv(want_len) + res = self.sock.recv(want_len) + + # Assume the socket is closed if no data is + # returned on a blocking read. + if len(res) == 0 and want_len > 0: + self.close() + + return res + except socket.error, e: # If the socket is in a bad state, close it and allow it # to re-connect on the next try From 99262c1904ca7404f75d870541b7ef4e84cfd143 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 13:02:42 -0800 Subject: [PATCH 0162/1060] Import errno --- riak/transports/pbc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index c19de917..77c5ddcb 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -19,6 +19,7 @@ """ from __future__ import with_statement +import errno import socket import struct From b0029413f4fde156a1185270654a3f014ccaf253 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 15:08:57 -0800 Subject: [PATCH 0163/1060] Adding test to verify behavior on underlying socket being closed Try to verify the retry logic works properly by closing the underlying socket. --- riak/tests/test_all.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6de3f2d4..814e784e 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -890,6 +890,33 @@ def test_uses_client_id_if_given(self): client_id = zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) # + def test_close_underlying_socket(self): + c = RiakClient(PB_HOST, PB_PORT, transport_class = RiakPbcTransport) + + bucket = self.client.bucket('bucket_test_close') + rand = self.randint() + obj = bucket.new('foo', rand) + obj.store() + obj = bucket.get('foo') + self.assertTrue(obj.exists()) + self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') + self.assertEqual(obj.get_key(), 'foo') + self.assertEqual(obj.get_data(), rand) + + # Close the underlying socket. This gets a bit sketchy, + # since we are reaching into the internals, but there is + # no other way to get at the socket + conns = c._cm.conns + for conn in conns: + if conn.sock is not None: + conn.sock.close() + + obj = bucket.get('foo') + self.assertTrue(obj.exists()) + self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') + self.assertEqual(obj.get_key(), 'foo') + self.assertEqual(obj.get_data(), rand) + class RiakHttpTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): From e701404150df88525fd63ce87085d586ce856567 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 16:21:13 -0800 Subject: [PATCH 0164/1060] Added critical 'break' statement Need to ensure we break after the first successful send over the PBC socket. --- riak/transports/pbc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 77c5ddcb..06496faf 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -458,6 +458,7 @@ def send_pkt(self, conn, pkt): self.recv_msg(conn, MSG_CODE_SET_CLIENT_ID_RESP) conn.send(pkt) + break except socket.error, e: # If this is some unknown socket error bail out # instead of retrying From 427d40ed2c758cf41cf3b490a35264ea79a5c734 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 16:21:50 -0800 Subject: [PATCH 0165/1060] Added 2 new unit tests for PBC changes Added unit tests to test the behavior when the max_attempts are 1 and 2. In the first case we expect the operation to fail, while with retries it should succeed. --- riak/tests/test_all.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 814e784e..b6ee3b17 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -9,6 +9,7 @@ import simplejson as json import os import random +import socket import platform if platform.python_version() < '2.7': @@ -890,10 +891,10 @@ def test_uses_client_id_if_given(self): client_id = zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) # - def test_close_underlying_socket(self): + def test_close_underlying_socket_fails(self): c = RiakClient(PB_HOST, PB_PORT, transport_class = RiakPbcTransport) - bucket = self.client.bucket('bucket_test_close') + bucket = c.bucket('bucket_test_close') rand = self.randint() obj = bucket.new('foo', rand) obj.store() @@ -907,14 +908,36 @@ def test_close_underlying_socket(self): # since we are reaching into the internals, but there is # no other way to get at the socket conns = c._cm.conns - for conn in conns: - if conn.sock is not None: - conn.sock.close() + conns[0].sock.close() - obj = bucket.get('foo') + # This shoud fail with a socket error now + self.assertRaises(socket.error, bucket.get, 'foo') + + def test_close_underlying_socket_retry(self): + c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport, + transport_options={"max_attempts": 2}) + + bucket = c.bucket('bucket_test_close') + rand = self.randint() + obj = bucket.new('barbaz', rand) + obj.store() + obj = bucket.get('barbaz') self.assertTrue(obj.exists()) self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') - self.assertEqual(obj.get_key(), 'foo') + self.assertEqual(obj.get_key(), 'barbaz') + self.assertEqual(obj.get_data(), rand) + + # Close the underlying socket. This gets a bit sketchy, + # since we are reaching into the internals, but there is + # no other way to get at the socket + conns = c._cm.conns + conns[0].sock.close() + + # This should work, since we have a retry + obj = bucket.get('barbaz') + self.assertTrue(obj.exists()) + self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') + self.assertEqual(obj.get_key(), 'barbaz') self.assertEqual(obj.get_data(), rand) From 735eb0bf127b2eeb14582cd116c5bca1a9d724bb Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 14 Feb 2012 16:42:50 -0800 Subject: [PATCH 0166/1060] Fix RiakObject.get_content_type() when content type not yet set. This method currently raises a KeyError instead of appropriately returning the default content types. --- riak/riak_object.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index c2e48f0e..f6e12bbe 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -214,7 +214,7 @@ def remove_index(self, field, value): def get_indexes(self, field = None): """ - Get a list of the index entries for this object. If a field is provided, returns a list + Get a list of the index entries for this object. If a field is provided, returns a list :param field: The index field. :type field: string or None @@ -244,7 +244,13 @@ def get_content_type(self): :rtype: string """ - return self._metadata[MD_CTYPE] + try: + return self._metadata[MD_CTYPE] + except KeyError: + if self._encode_data: + return "application/json" + else: + return "application/octet-stream" def set_content_type(self, content_type): """ From 8ff2f9eadb287c3516425d1819f0cd374c596d04 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Wed, 15 Feb 2012 09:55:15 -0500 Subject: [PATCH 0167/1060] Addressed basho/riak-python-client#83 Provided a set_links method. --- riak/riak_object.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index c2e48f0e..1c40382a 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -214,7 +214,7 @@ def remove_index(self, field, value): def get_indexes(self, field = None): """ - Get a list of the index entries for this object. If a field is provided, returns a list + Get a list of the index entries for this object. If a field is provided, returns a list :param field: The index field. :type field: string or None @@ -257,6 +257,27 @@ def set_content_type(self, content_type): self._metadata[MD_CTYPE] = content_type return self + def set_links(self, links): + """ + Replaces all links to a RiakObject + + :param links: An iterable of 2-item tuples, consisting of (RiakObject, tag). This could also be an iterable of + just a RiakObject, instead of the tuple, then a tag of None would be used. Lastly, it could also be an + iterable of RiakLink. They have tags built-in. + """ + new_links = [] + for item in links: + if isinstance(item, RiakLink): + link = item + elif isinstance(item, RiakObject): + link = RiakLink(item._bucket._name, item._key, None) + else: + link = RiakLink(item[0]._bucket._name, item._key, item[1]) + new_links.append(link) + + self._metadata[MD_LINKS] = new_links + return self + def add_link(self, obj, tag=None): """ Add a link to a RiakObject. From 8a05a2203bafb651157735229f223e7958e701a9 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Wed, 15 Feb 2012 10:24:27 -0500 Subject: [PATCH 0168/1060] Added unittest and fixed a small bug Last commit had a small bug with it, also added unittest for set_links. Signed-off-by: Shuhao Wu --- riak/riak_object.py | 2 +- riak/tests/test_all.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 1c40382a..11cc29a9 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -272,7 +272,7 @@ def set_links(self, links): elif isinstance(item, RiakObject): link = RiakLink(item._bucket._name, item._key, None) else: - link = RiakLink(item[0]._bucket._name, item._key, item[1]) + link = RiakLink(item[0]._bucket._name, item[0]._key, item[1]) new_links.append(link) self._metadata[MD_LINKS] = new_links diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 6de3f2d4..4a367a15 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -441,6 +441,21 @@ def test_store_and_get_links(self): links = obj.get_links() self.assertEqual(len(links), 3) + def test_set_links(self): + # Create the object + bucket = self.client.bucket("bucket") + bucket.new("foo", 2).set_links([bucket.new("foo1"), + (bucket.new("foo2"), "tag"), + RiakLink("bucket", "foo2", "tag2")]).store() + obj = bucket.get("foo") + links = sorted(obj.get_links(), key=lambda x: x.get_key()) + self.assertEqual(len(links), 3) + self.assertEqual(links[0].get_key(), "foo1") + self.assertEqual(links[1].get_key(), "foo2") + self.assertEqual(links[1].get_tag(), "tag") + self.assertEqual(links[2].get_key(), "foo2") + self.assertEqual(links[2].get_tag(), "tag2") + def test_link_walking(self): # Create the object... bucket = self.client.bucket("bucket") From a0c48a57510ff60a40f039a1edc6d8292158be8b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 15 Feb 2012 13:30:19 -0500 Subject: [PATCH 0169/1060] Adjust link-matching spec to accommodate 1.0+ URLs and fix the unit test. --- riak/tests/test_all.py | 16 +++++++--------- riak/transports/http.py | 9 +++++---- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index e708298d..540486c4 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -432,25 +432,23 @@ def test_map_reduce_from_object(self): def test_store_and_get_links(self): # Create the object... bucket = self.client.bucket("bucket") - bucket.new("foo", 2) \ + bucket.new_binary("test_store_and_get_links", '2') \ .add_link(bucket.new("foo1")) \ .add_link(bucket.new("foo2"), "tag") \ .add_link(bucket.new("foo3"), "tag2!@#%^&*)") \ .store() - obj = bucket.get("foo") + obj = bucket.get("test_store_and_get_links") links = obj.get_links() self.assertEqual(len(links), 3) for l in links: if (l.get_key() == "foo1"): - self.assertEqual(l.get_tag(), "") - next - if (l.get_key() == "foo2"): + self.assertEqual(l.get_tag(), "bucket") + elif (l.get_key() == "foo2"): self.assertEqual(l.get_tag(), "tag") - next - if (l.get_key() == "foo3"): + elif (l.get_key() == "foo3"): self.assertEqual(l.get_tag(), "tag2!@#%^&*)") - next - self.assertEqual("unknown key", l.get_key()) + else: + self.assertEqual("unknown key", l.get_key()) def test_link_walking(self): # Create the object... diff --git a/riak/transports/http.py b/riak/transports/http.py index c0e38bfb..81ad330f 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -131,7 +131,7 @@ def do_put(self, url, headers, content, return_body=False, key=None): else: self.check_http_code(response, [204]) return None - + def put_new(self, robj, w=None, dw=None, return_meta=True): """Put a new object into the Riak store, returning its (new) key.""" # Construct the URL... @@ -341,7 +341,8 @@ def parse_links(self, links, linkHeaders): """ for linkHeader in linkHeaders.strip().split(','): linkHeader = linkHeader.strip() - matches = re.match("; ?riaktag=\"([^\']+)\"", linkHeader) + matches = re.match("; ?riaktag=\"([^\']+)\"", linkHeader) or \ + re.match("; ?riaktag=\"([^\']+)\"", linkHeader) if matches is not None: link = RiakLink(urllib.unquote_plus(matches.group(2)), urllib.unquote_plus(matches.group(3)), @@ -393,7 +394,7 @@ def delete_file(self, key): def post_request(self, uri=None, body=None, params=None, content_type="application/json"): uri = self.build_rest_path(prefix=uri, params=params) - return self.http_request('POST', uri, {'Content-Type': content_type}, body) + return self.http_request('POST', uri, {'Content-Type': content_type}, body) # Utility functions used by Riak library. @@ -449,7 +450,7 @@ def build_put_headers(self, robj): headers[key] += ", " + rie.get_value() else: headers[key] = rie.get_value() - + return headers def http_request(self, method, uri, headers=None, body='') : From 0045932d87827ee6dd204a6c99790f6565e58a17 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Mon, 27 Feb 2012 16:21:17 -0500 Subject: [PATCH 0170/1060] Added a way to efficiently set_links Before, there's no actual way to set_links without having the function iterate through the entire iterable. Now with an all_link=True or False, we could now set links directly to an list. Signed-off-by: Shuhao Wu --- riak/riak_object.py | 8 +++++++- riak/tests/test_all.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 11cc29a9..fbbedee8 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -257,14 +257,20 @@ def set_content_type(self, content_type): self._metadata[MD_CTYPE] = content_type return self - def set_links(self, links): + def set_links(self, links, all_link=False): """ Replaces all links to a RiakObject :param links: An iterable of 2-item tuples, consisting of (RiakObject, tag). This could also be an iterable of just a RiakObject, instead of the tuple, then a tag of None would be used. Lastly, it could also be an iterable of RiakLink. They have tags built-in. + :param all_link: A boolean indicates if links is all RiakLink object + This speeds up the operation. """ + if all_link: + self._metadata[MD_LINKS] = links + return self + new_links = [] for item in links: if isinstance(item, RiakLink): diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 4a367a15..e766cd45 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -456,6 +456,16 @@ def test_set_links(self): self.assertEqual(links[2].get_key(), "foo2") self.assertEqual(links[2].get_tag(), "tag2") + def test_set_links_all_links(self): + bucket = self.client.bucket("bucket") + foo1 = bucket.new("foo", 1) + foo2 = bucket.new("foo2", 2).store() + links = [RiakLink("bucket", "foo2")] + foo1.set_links(links, True) + links = foo1.get_links() + self.assertEqual(len(links), 1) + self.assertEqual(links[0].get_key(), "foo2") + def test_link_walking(self): # Create the object... bucket = self.client.bucket("bucket") From 4d3016b146a25453bcb65567ee004d12beda2031 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Mon, 27 Feb 2012 19:08:05 -0500 Subject: [PATCH 0171/1060] Added a set_indexes function like set_links Realized that I needed that for my code, I implemented this with this as well. set_links takes an iterable of 2 item tuples that's (field, value). It will set the indexes similarly to set_links Signed-off-by: Shuhao Wu --- riak/riak_object.py | 16 ++++++++++++++++ riak/tests/test_all.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/riak/riak_object.py b/riak/riak_object.py index fbbedee8..1a34fa45 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -212,6 +212,22 @@ def remove_index(self, field, value): self._metadata[MD_INDEX].remove(rie) return self + def set_indexes(self, indexes): + """ + Sets indexes once and for all. Currenly supports an iterable of 2 item tuples, + (field, value) + + :param indexes: iterable of 2 item tuples consisting the field and value. + :rtype: self + """ + new_indexes = [] + for field, value in indexes: + rie = RiakIndexEntry(field, value) + new_indexes.append(rie) + self._metadata[MD_INDEX] = new_indexes + + return self + def get_indexes(self, field = None): """ Get a list of the index entries for this object. If a field is provided, returns a list diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index e766cd45..f6f1da64 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -645,6 +645,22 @@ def test_secondary_index_store(self): # Clean up... bucket.get('mykey1').delete() + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_set_indexes(self): + if not self.is_2i_supported(): + return True + + bucket = self.client.bucket('indexbucket') + foo = bucket.new('foo', 1) + foo.set_indexes((('field1_bin', 'test'), ('field2_int', 1337))).store() + result = self.client.index('indexbucket', 'field2_int', 1337).run() + self.assertEqual(1, len(result)) + self.assertEqual('foo', result[0].get_key()) + + result = self.client.index('indexbucket', 'field1_bin', 'test').run() + self.assertEqual(1, len(result)) + self.assertEqual('foo', result[0].get_key()) + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_query(self): if not self.is_2i_supported(): From 1c3ae3c06868de01c903d308eb5705e77e8124a8 Mon Sep 17 00:00:00 2001 From: William Kral Date: Mon, 5 Mar 2012 16:35:38 -0800 Subject: [PATCH 0172/1060] Fixed a path conflict with PLATFORM_DATA_DIR introduce in riak 1.x --- riak/test_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/riak/test_server.py b/riak/test_server.py index f4645e3a..3922125d 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -200,6 +200,7 @@ def write_riak_script(self): line = re.sub("(RUNNER_USER=)(.*)", r'\1', line) line = re.sub("(RUNNER_LOG_DIR=)(.*)", r'\1%s' % self._temp_log, line) line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % self._temp_pipe, line) + line = re.sub("(PLATFORM_DATA_DIR=)(.*)", r'\1%s' % self.temp_dir) if string.strip(line) == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}": line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join(self.bin_dir, "..")) From 88ba5a924144a8c307ae4a4feb56b269bd1e81d3 Mon Sep 17 00:00:00 2001 From: William Kral Date: Tue, 6 Mar 2012 01:01:48 -0800 Subject: [PATCH 0173/1060] Fixed it properly in my live version but missed the line argument here --- riak/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/test_server.py b/riak/test_server.py index 3922125d..56715fbe 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -200,7 +200,7 @@ def write_riak_script(self): line = re.sub("(RUNNER_USER=)(.*)", r'\1', line) line = re.sub("(RUNNER_LOG_DIR=)(.*)", r'\1%s' % self._temp_log, line) line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % self._temp_pipe, line) - line = re.sub("(PLATFORM_DATA_DIR=)(.*)", r'\1%s' % self.temp_dir) + line = re.sub("(PLATFORM_DATA_DIR=)(.*)", r'\1%s' % self.temp_dir, line) if string.strip(line) == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}": line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join(self.bin_dir, "..")) From 2f890af19335023f9ae76a4b2eb189d7581e1112 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 6 Mar 2012 16:29:16 -0800 Subject: [PATCH 0174/1060] Adding support for if_none_match --- riak/riak_object.py | 9 ++++++--- riak/transports/http.py | 12 +++++++----- riak/transports/pbc.py | 8 ++++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index f6e12bbe..3f6acaea 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -324,7 +324,7 @@ def get_links(self): else: return [] - def store(self, w=None, dw=None, return_body=True): + def store(self, w=None, dw=None, return_body=True, if_none_match=False): """ Store the object in Riak. When this operation completes, the object could contain new metadata and possibly new data if Riak @@ -339,6 +339,9 @@ def store(self, w=None, dw=None, return_body=True): :type dw: integer :param return_body: if the newly stored object should be retrieved :type return_body: bool + :param if_none_match: Should the object be stored only if there is no + key previously defined + :type if_none_match: bool :rtype: self """ # Use defaults if not specified... @@ -349,13 +352,13 @@ def store(self, w=None, dw=None, return_body=True): t = self._client.get_transport() if self._key is None: - key, vclock, metadata = t.put_new(self, w, dw, return_body) + key, vclock, metadata = t.put_new(self, w, dw, return_body, if_none_match) self._exists = True self._key = key self._vclock = vclock self.set_metadata(metadata) else: - Result = t.put(self, w, dw, return_body) + Result = t.put(self, w, dw, return_body, if_none_match) if Result is not None: self.populate(Result) diff --git a/riak/transports/http.py b/riak/transports/http.py index 92d534b4..c00af22a 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -108,7 +108,7 @@ def get(self, robj, r, vtag = None) : response = self.http_request('GET', url) return self.parse_body(response, [200, 300, 404]) - def put(self, robj, w = None, dw = None, return_body = True): + def put(self, robj, w = None, dw = None, return_body = True, if_none_match=False): """ Serialize put request and deserialize response """ @@ -117,6 +117,8 @@ def put(self, robj, w = None, dw = None, return_body = True): url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), params=params) headers = self.build_put_headers(robj) + if if_none_match: + headers["If-None-Match"] = "1" content = robj.get_encoded_data() return self.do_put(url, headers, content, return_body, key=robj.get_key()) @@ -131,8 +133,8 @@ def do_put(self, url, headers, content, return_body=False, key=None): else: self.check_http_code(response, [204]) return None - - def put_new(self, robj, w=None, dw=None, return_meta=True): + + def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): """Put a new object into the Riak store, returning its (new) key.""" # Construct the URL... params = {'returnbody' : str(return_meta).lower(), 'w' : w, 'dw' : dw} @@ -391,7 +393,7 @@ def delete_file(self, key): def post_request(self, uri=None, body=None, params=None, content_type="application/json"): uri = self.build_rest_path(prefix=uri, params=params) - return self.http_request('POST', uri, {'Content-Type': content_type}, body) + return self.http_request('POST', uri, {'Content-Type': content_type}, body) # Utility functions used by Riak library. @@ -447,7 +449,7 @@ def build_put_headers(self, robj): headers[key] += ", " + rie.get_value() else: headers[key] = rie.get_value() - + return headers def http_request(self, method, uri, headers=None, body='') : diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 06496faf..60499a4e 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -246,7 +246,7 @@ def get(self, robj, r=None, vtag=None): else: return 0 - def put(self, robj, w=None, dw=None, return_body=True): + def put(self, robj, w=None, dw=None, return_body=True, if_none_match=False): """ Serialize get request and deserialize response """ @@ -257,6 +257,8 @@ def put(self, robj, w=None, dw=None, return_body=True): req.dw = self.translate_rw_val(dw) if return_body: req.return_body = 1 + if if_none_match: + req.if_none_match = 1 req.bucket = bucket.get_name() req.key = robj.get_key() @@ -274,7 +276,7 @@ def put(self, robj, w=None, dw=None, return_body=True): contents.append(self.decode_content(c)) return resp.vclock, contents - def put_new(self, robj, w=None, dw=None, return_meta=True): + def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): """Put a new object into the Riak store, returning its (new) key. If return_meta is False, then the vlock and metadata return values @@ -289,6 +291,8 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): req.dw = self.translate_rw_val(dw) if return_meta: req.return_body = 1 + if if_none_match: + req.if_none_match = 1 req.bucket = bucket.get_name() From a586849036f5acd5ffc300e9adf2abc2abb5c04f Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 6 Mar 2012 16:53:39 -0800 Subject: [PATCH 0175/1060] Tweaking the Http transport --- riak/transports/http.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index c00af22a..e207df72 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -118,7 +118,7 @@ def put(self, robj, w = None, dw = None, return_body = True, if_none_match=False params=params) headers = self.build_put_headers(robj) if if_none_match: - headers["If-None-Match"] = "1" + headers["If-None-Match"] = "*" content = robj.get_encoded_data() return self.do_put(url, headers, content, return_body, key=robj.get_key()) @@ -140,6 +140,8 @@ def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): params = {'returnbody' : str(return_meta).lower(), 'w' : w, 'dw' : dw} url = self.build_rest_path(bucket=robj.get_bucket(), params=params) headers = self.build_put_headers(robj) + if if_none_match: + headers["If-None-Match"] = "*" content = robj.get_encoded_data() response = self.http_request('POST', url, headers, content) location = response[0]['location'] From 02738ab2625f8cca0bb6aa8b4bf94b4acd3056b1 Mon Sep 17 00:00:00 2001 From: Brian Roach Date: Fri, 16 Mar 2012 14:52:35 -0600 Subject: [PATCH 0176/1060] Tiny doc fix --- riak/riak_object.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index cd9c8af7..e43aee30 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -214,7 +214,7 @@ def remove_index(self, field, value): def set_indexes(self, indexes): """ - Sets indexes once and for all. Currenly supports an iterable of 2 item tuples, + Replaces all indexes on a Riak object. Currenly supports an iterable of 2 item tuples, (field, value) :param indexes: iterable of 2 item tuples consisting the field and value. @@ -286,7 +286,7 @@ def set_links(self, links, all_link=False): :param links: An iterable of 2-item tuples, consisting of (RiakObject, tag). This could also be an iterable of just a RiakObject, instead of the tuple, then a tag of None would be used. Lastly, it could also be an iterable of RiakLink. They have tags built-in. - :param all_link: A boolean indicates if links is all RiakLink object + :param all_link: A boolean indicates if links are all RiakLink objects This speeds up the operation. """ if all_link: From f622f01a40309b284d9f2bf39de9f5d7c2371cb4 Mon Sep 17 00:00:00 2001 From: Sreejith Kesavan Date: Wed, 28 Mar 2012 14:53:11 +0530 Subject: [PATCH 0177/1060] Never fail without a reason --- riak/transports/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 301c7465..2d0d84f4 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -244,7 +244,7 @@ def mapred(self, inputs, query, timeout=None): def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] if not status in expected_statuses: - m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) + m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) + ':' + response[1] raise Exception(m) def parse_body(self, response, expected_statuses): From 7f1129776bb7250f0328d887c6edb31300a46dd1 Mon Sep 17 00:00:00 2001 From: Sreejith Kesavan Date: Wed, 28 Mar 2012 14:57:06 +0530 Subject: [PATCH 0178/1060] Removing ambiguous code. Already checked in check_http_code. --- riak/transports/http.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 2d0d84f4..e25be682 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -272,11 +272,6 @@ def parse_body(self, response, expected_statuses): m = 'Could not contact Riak Server: http://$HOST:$PORT !' raise RiakError(m) - # Verify that we got one of the expected statuses. Otherwise, raise an exception. - if not status in expected_statuses: - m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) - raise RiakError(m) - # If 404(Not Found), then clear the object. if status == 404: return None From 871d1495370c8940765bf9fecdabab6925e0d983 Mon Sep 17 00:00:00 2001 From: Sreejith Kesavan Date: Wed, 28 Mar 2012 15:01:44 +0530 Subject: [PATCH 0179/1060] Fixing string format --- riak/transports/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index e25be682..1fb7147e 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -244,7 +244,7 @@ def mapred(self, inputs, query, timeout=None): def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] if not status in expected_statuses: - m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) + ':' + response[1] + m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) + ' : ' + response[1] raise Exception(m) def parse_body(self, response, expected_statuses): From 13de673ec98a41666f90cd3f3c00fce69920b265 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 28 Mar 2012 17:44:40 -0400 Subject: [PATCH 0180/1060] Added test for if_none_match functionality. --- riak/tests/test_all.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index b6ee3b17..4e743890 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -222,6 +222,20 @@ def test_rw_settings(self): bucket.set_rw("one") self.assertEqual(bucket.get_rw(), "one") + def test_if_none_match(self): + bucket = self.client.bucket('if_none_match_test') + obj = bucket.get('obj') + obj.delete() + + obj.reload() + self.assertFalse(obj.exists()) + obj.set_data(["first store"]) + obj.store() + + obj.set_data(["second store"]) + with self.assertRaises(Exception): + obj.store(if_none_match=True) + def test_siblings(self): # Set up the bucket, clear any existing object... bucket = self.client.bucket('multiBucket') From 8f012cb2e6d39f2f91fcdf6e1ea33e2179bc7668 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 28 Mar 2012 18:33:34 -0400 Subject: [PATCH 0181/1060] PR/PW step 1: add accessors to bucket and client. --- riak/bucket.py | 55 +++++++++++++++++++++++++++++++++++++++++- riak/client.py | 42 ++++++++++++++++++++++++++++++++ riak/tests/test_all.py | 11 +++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/riak/bucket.py b/riak/bucket.py index 83818fa6..3cf1e3af 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -50,6 +50,8 @@ def __init__(self, client, name): self._w = None self._dw = None self._rw = None + self._pr = None + self._pw = None self._encoders = {} self._decoders = {} @@ -159,6 +161,57 @@ def set_rw(self, rw): self._rw = rw return self + def get_pr(self, pr=None): + """ + Get the PR-value for this bucket, if it is set, otherwise return + the PR-value for the client. + + :rtype: integer + """ + if (pr is not None): + return pr + if (self._pr is not None): + return self._pr + return self._client.get_pr() + + def set_pr(self, pr): + """ + Set the PR-value for this bucket. See :func:`set_r` for more + information. + + :param pr: The new PR-value + :type pr: integer + :rtype: self + """ + self._pr = pr + return self + + + def get_pw(self, pw=None): + """ + Get the PW-value for this bucket, if it is set, otherwise return + the PW-value for the client. + + :rtype: integer + """ + if (pw is not None): + return pw + if (self._pw is not None): + return self._pw + return self._client.get_pw() + + def set_pw(self, pw): + """ + Set the PW-value for this bucket. See :func:`set_r` for more + information. + + :param pw: The new PR-value + :type pw: integer + :rtype: self + """ + self._pw = pw + return self + def get_encoder(self, content_type): """ Get the encoding function for the provided content type for this bucket. @@ -428,7 +481,7 @@ def new_binary_from_file(self, key, filename): if not mimetype: mimetype = 'application/octet-stream' return self.new_binary(key, binary_data, mimetype) - + def search_enabled(self): """ Returns True if the search precommit hook is enabled for this bucket. diff --git a/riak/client.py b/riak/client.py index 5e78fe27..e53f2347 100644 --- a/riak/client.py +++ b/riak/client.py @@ -85,6 +85,8 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', self._w = "default" self._dw = "default" self._rw = "default" + self._pr = "default" + self._pw = "default" self._encoders = {'application/json': json.dumps, 'text/json': json.dumps} self._decoders = {'application/json': json.loads, @@ -182,6 +184,46 @@ def set_rw(self, rw): self._rw = rw return self + def get_pr(self): + """ + Get the PR-value setting for this ``RiakClient``. (default 0) + + :rtype: integer + """ + return self._pr + + def set_pr(self, pr): + """ + Set the PR-value for this ``RiakClient`` instance. See :func:`set_r` for a + description of how these values are used. + + :param pr: The PR value. + :type pr: integer + :rtype: self + """ + self._pr = pr + return self + + def get_pw(self): + """ + Get the PW-value setting for this ``RiakClient``. (default 0) + + :rtype: integer + """ + return self._pr + + def set_pw(self, pw): + """ + Set the PW-value for this ``RiakClient`` instance. See :func:`set_r` for a + description of how these values are used. + + :param pw: The W value. + :type pw: integer + :rtype: self + """ + self._pr = pr + return self + def get_client_id(self): """ Get the ``client_id`` for this ``RiakClient`` instance. diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 70281e5b..17610406 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -222,6 +222,17 @@ def test_rw_settings(self): bucket.set_rw("one") self.assertEqual(bucket.get_rw(), "one") + def test_primary_quora(self): + bucket = self.client.bucket('primary_quora') + self.assertEqual(bucket.get_pr(), "default") + self.assertEqual(bucket.get_pw(), "default") + + bucket.set_pr(1) + self.assertEqual(bucket.get_pr(), 1) + + bucket.set_pw("quorum") + self.assertEqual(bucket.get_pw(), "quorum") + def test_if_none_match(self): bucket = self.client.bucket('if_none_match_test') obj = bucket.get('obj') From 7a9bec2b8783b2ae8b0c0c029886c028c5e31695 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 28 Mar 2012 19:36:05 -0400 Subject: [PATCH 0182/1060] PR/PW step 1.5: Exclude query params that are 'None'. --- riak/tests/test_all.py | 2 ++ riak/transports/http.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 17610406..0a434d1c 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -1169,6 +1169,8 @@ def test_delete_documents_from_search_by_query_and_id(self): results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results["response"]["docs"])) + def test_build_rest_path_excludes_empty_query_params(self): + self.assertEquals(self.client.get_transport().build_rest_path(bucket=self.client.bucket("foo"), key="bar", params={'r': None}), "/riak/foo/bar?") class RiakTestFilter(unittest.TestCase): def test_simple(self): diff --git a/riak/transports/http.py b/riak/transports/http.py index 14468e34..e1e85184 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -418,8 +418,9 @@ def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : if params is not None: s = '' for key in params.keys(): - if s != '': s += '&' - s += urllib.quote_plus(key) + '=' + urllib.quote_plus(str(params[key])) + if params[key] is not None: + if s != '': s += '&' + s += urllib.quote_plus(key) + '=' + urllib.quote_plus(str(params[key])) path += '?' + s # Return. From d3baaebbdef4f8a846f4921ba22c5768be6e4f2d Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 28 Mar 2012 19:47:03 -0400 Subject: [PATCH 0183/1060] PR/PW step 2: Add support for primary quora. * return_meta/return_body are now consistent across Transport.put()/put_new(). * riak_object.store() supports pw. * riak_bucket.get() and riak_bucket.get_binary() support pr. * riak_object.delete() supports r, pr, w, dw, pw. * Some internal method calls were changed to use named arguments where necessary. --- riak/bucket.py | 14 ++++++++--- riak/riak_object.py | 41 +++++++++++++++++++++++++------ riak/transports/http.py | 19 +++++++------- riak/transports/pbc.py | 22 +++++++++++++---- riak/transports/riakclient_pb2.py | 23 ++++++++++++++++- setup.py | 11 +++------ 6 files changed, 95 insertions(+), 35 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 3cf1e3af..64c492c0 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -297,7 +297,7 @@ def new_binary(self, key, data, content_type='application/octet-stream'): obj._encode_data = False return obj - def get(self, key, r=None): + def get(self, key, r=None, pr=None): """ Retrieve a JSON-encoded object from Riak. @@ -305,14 +305,17 @@ def get(self, key, r=None): :type key: string :param r: R-Value of the request (defaults to bucket's R) :type r: integer + :param pr: PR-Value of the request (defaults to bucket's PR) + :type pr: integer :rtype: :class:`RiakObject ` """ obj = RiakObject(self._client, self, key) obj._encode_data = True r = self.get_r(r) - return obj.reload(r) + pr = self.get_pr(pr) + return obj.reload(r=r, pr=pr) - def get_binary(self, key, r=None): + def get_binary(self, key, r=None, pr=None): """ Retrieve a binary/string object from Riak. @@ -320,12 +323,15 @@ def get_binary(self, key, r=None): :type key: string :param r: R-Value of the request (defaults to bucket's R) :type r: integer + :param pr: PR-Value of the request (defaults to bucket's PR) + :type pr: integer :rtype: :class:`RiakObject ` """ obj = RiakObject(self._client, self, key) obj._encode_data = False r = self.get_r(r) - return obj.reload(r) + pr = self.get_pr(pr) + return obj.reload(r=r, pr=pr) def set_n_val(self, nval): """ diff --git a/riak/riak_object.py b/riak/riak_object.py index b0d8a784..3a3351a6 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -367,7 +367,7 @@ def get_links(self): else: return [] - def store(self, w=None, dw=None, return_body=True, if_none_match=False): + def store(self, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """ Store the object in Riak. When this operation completes, the object could contain new metadata and possibly new data if Riak @@ -380,6 +380,9 @@ def store(self, w=None, dw=None, return_body=True, if_none_match=False): :param dw: DW-value, wait for this many partitions to confirm the write before returning to client. :type dw: integer + :param pw: PW-value, require this many primary partitions to be available + before performing the put + :type pw: integer :param return_body: if the newly stored object should be retrieved :type return_body: bool :param if_none_match: Should the object be stored only if there is no @@ -390,25 +393,26 @@ def store(self, w=None, dw=None, return_body=True, if_none_match=False): # Use defaults if not specified... w = self._bucket.get_w(w) dw = self._bucket.get_dw(dw) + pw = self._bucket.get_pw(pw) # Issue the put over our transport t = self._client.get_transport() if self._key is None: - key, vclock, metadata = t.put_new(self, w, dw, return_body, if_none_match) + key, vclock, metadata = t.put_new(self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) self._exists = True self._key = key self._vclock = vclock self.set_metadata(metadata) else: - Result = t.put(self, w, dw, return_body, if_none_match) + Result = t.put(self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) if Result is not None: self.populate(Result) return self - def reload(self, r=None, vtag=None): + def reload(self, r=None, pr=None, vtag=None): """ Reload the object from Riak. When this operation completes, the object could contain new metadata and a new value, if the object @@ -421,8 +425,9 @@ def reload(self, r=None, vtag=None): """ # Do the request... r = self._bucket.get_r(r) + pr = self._bucket.get_pr(pr) t = self._client.get_transport() - Result = t.get(self, r, vtag) + Result = t.get(self, r=r, pr=pr, vtag=vtag) self.clear() if Result is not None: @@ -431,19 +436,39 @@ def reload(self, r=None, vtag=None): return self - def delete(self, rw=None): + def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Delete this object from Riak. :param rw: RW-value. Wait until this many partitions have - deleted the object before responding. + deleted the object before responding. (deprecated in Riak 1.0+, use R/W/DW) :type rw: integer + :param r: R-value, wait for this many partitions to read object + before performing the put + :type r: integer + :param w: W-value, wait for this many partitions to respond + before returning to client. + :type w: integer + :param dw: DW-value, wait for this many partitions to + confirm the write before returning to client. + :type dw: integer + :param pr: PR-value, require this many primary partitions to be available + before performing the read that precedes the put + :type pr: integer + :param pr: PW-value, require this many primary partitions to be available + before performing the put + :type pw: integer :rtype: self """ # Use defaults if not specified... rw = self._bucket.get_rw(rw) + r = self._bucket.get_r(r) + w = self._bucket.get_w(w) + dw = self._bucket.get_dw(dw) + pr = self._bucket.get_pr(pr) + pw = self._bucket.get_pw(pw) t = self._client.get_transport() - Result = t.delete(self, rw) + Result = t.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) self.clear() return self diff --git a/riak/transports/http.py b/riak/transports/http.py index e1e85184..c77955b6 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -96,11 +96,11 @@ def ping(self) : return(response is not None) and (response[1] == 'OK') - def get(self, robj, r, vtag = None) : + def get(self, robj, r = None, pr = None, vtag = None) : """ Get a bucket/key from the server """ - params = {'r' : r} + params = {'r' : r, 'pr': pr} if vtag is not None: params['vtag'] = vtag url = self.build_rest_path(robj.get_bucket(), robj.get_key(), @@ -108,12 +108,12 @@ def get(self, robj, r, vtag = None) : response = self.http_request('GET', url) return self.parse_body(response, [200, 300, 404]) - def put(self, robj, w = None, dw = None, return_body = True, if_none_match=False): + def put(self, robj, w = None, dw = None, pw = None, return_body = True, if_none_match=False): """ Serialize put request and deserialize response """ # Construct the URL... - params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw} + params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw, 'pw' : pw } url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), params=params) headers = self.build_put_headers(robj) @@ -134,10 +134,10 @@ def do_put(self, url, headers, content, return_body=False, key=None): self.check_http_code(response, [204]) return None - def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): + def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """Put a new object into the Riak store, returning its (new) key.""" # Construct the URL... - params = {'returnbody' : str(return_meta).lower(), 'w' : w, 'dw' : dw} + params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw, 'pw' : pw} url = self.build_rest_path(bucket=robj.get_bucket(), params=params) headers = self.build_put_headers(robj) if if_none_match: @@ -147,18 +147,19 @@ def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): location = response[0]['location'] idx = location.rindex('/') key = location[idx+1:] - if return_meta: + if return_body: vclock, [(metadata, data)] = self.parse_body(response, [201]) return key, vclock, metadata else: self.check_http_code(response, [201]) return key, None, None - def delete(self, robj, rw): + def delete(self, robj, rw=None, r = None, w = None, dw = None, pr = None, pw = None): # Construct the URL... - params = {'rw' : rw} + params = {'rw' : rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} url = self.build_rest_path(robj.get_bucket(), robj.get_key(), params=params) + # TODO: Send vclock of robj if it exists # Run the operation.. response = self.http_request('DELETE', url) self.check_http_code(response, [204, 404]) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 60499a4e..886558a9 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -221,7 +221,7 @@ def set_client_id(self, client_id): return True - def get(self, robj, r=None, vtag=None): + def get(self, robj, r=None, pr=None, vtag=None): """ Serialize get request and deserialize response """ @@ -232,6 +232,7 @@ def get(self, robj, r=None, vtag=None): req = riakclient_pb2.RpbGetReq() req.r = self.translate_rw_val(r) + req.pr = self.translate_rw_val(pr) req.bucket = bucket.get_name() req.key = robj.get_key() @@ -246,7 +247,7 @@ def get(self, robj, r=None, vtag=None): else: return 0 - def put(self, robj, w=None, dw=None, return_body=True, if_none_match=False): + def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """ Serialize get request and deserialize response """ @@ -255,6 +256,8 @@ def put(self, robj, w=None, dw=None, return_body=True, if_none_match=False): req = riakclient_pb2.RpbPutReq() req.w = self.translate_rw_val(w) req.dw = self.translate_rw_val(dw) + req.pw = self.translate_rw_val(pw) + if return_body: req.return_body = 1 if if_none_match: @@ -276,7 +279,7 @@ def put(self, robj, w=None, dw=None, return_body=True, if_none_match=False): contents.append(self.decode_content(c)) return resp.vclock, contents - def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): + def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """Put a new object into the Riak store, returning its (new) key. If return_meta is False, then the vlock and metadata return values @@ -289,7 +292,9 @@ def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): req = riakclient_pb2.RpbPutReq() req.w = self.translate_rw_val(w) req.dw = self.translate_rw_val(dw) - if return_meta: + req.pw = self.translate_rw_val(pw) + + if return_body: req.return_body = 1 if if_none_match: req.if_none_match = 1 @@ -308,7 +313,7 @@ def put_new(self, robj, w=None, dw=None, return_meta=True, if_none_match=False): metadata, content = self.decode_content(resp.content[0]) return resp.key, resp.vclock, metadata - def delete(self, robj, rw=None): + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Serialize get request and deserialize response """ @@ -316,6 +321,13 @@ def delete(self, robj, rw=None): req = riakclient_pb2.RpbDelReq() req.rw = self.translate_rw_val(rw) + req.r = self.translate_rw_val(r) + req.w = self.translate_rw_val(w) + req.dw = self.translate_rw_val(dw) + req.pr = self.translate_rw_val(pr) + req.pw = self.translate_rw_val(pw) + + # TODO: Set the vclock if present req.bucket = bucket.get_name() req.key = robj.get_key() diff --git a/riak/transports/riakclient_pb2.py b/riak/transports/riakclient_pb2.py index 2715abff..e607fe72 100644 --- a/riak/transports/riakclient_pb2.py +++ b/riak/transports/riakclient_pb2.py @@ -7,6 +7,7 @@ # @@protoc_insertion_point(imports) + DESCRIPTOR = descriptor.FileDescriptor( name='riakclient.proto', package='', @@ -952,7 +953,6 @@ serialized_end=1640, ) - _RPBGETRESP.fields_by_name['content'].message_type = _RPBCONTENT _RPBPUTREQ.fields_by_name['content'].message_type = _RPBCONTENT _RPBPUTRESP.fields_by_name['content'].message_type = _RPBCONTENT @@ -961,6 +961,27 @@ _RPBCONTENT.fields_by_name['links'].message_type = _RPBLINK _RPBCONTENT.fields_by_name['usermeta'].message_type = _RPBPAIR _RPBCONTENT.fields_by_name['indexes'].message_type = _RPBPAIR +DESCRIPTOR.message_types_by_name['RpbErrorResp'] = _RPBERRORRESP +DESCRIPTOR.message_types_by_name['RpbGetClientIdResp'] = _RPBGETCLIENTIDRESP +DESCRIPTOR.message_types_by_name['RpbSetClientIdReq'] = _RPBSETCLIENTIDREQ +DESCRIPTOR.message_types_by_name['RpbGetServerInfoResp'] = _RPBGETSERVERINFORESP +DESCRIPTOR.message_types_by_name['RpbGetReq'] = _RPBGETREQ +DESCRIPTOR.message_types_by_name['RpbGetResp'] = _RPBGETRESP +DESCRIPTOR.message_types_by_name['RpbPutReq'] = _RPBPUTREQ +DESCRIPTOR.message_types_by_name['RpbPutResp'] = _RPBPUTRESP +DESCRIPTOR.message_types_by_name['RpbDelReq'] = _RPBDELREQ +DESCRIPTOR.message_types_by_name['RpbListBucketsResp'] = _RPBLISTBUCKETSRESP +DESCRIPTOR.message_types_by_name['RpbListKeysReq'] = _RPBLISTKEYSREQ +DESCRIPTOR.message_types_by_name['RpbListKeysResp'] = _RPBLISTKEYSRESP +DESCRIPTOR.message_types_by_name['RpbGetBucketReq'] = _RPBGETBUCKETREQ +DESCRIPTOR.message_types_by_name['RpbGetBucketResp'] = _RPBGETBUCKETRESP +DESCRIPTOR.message_types_by_name['RpbSetBucketReq'] = _RPBSETBUCKETREQ +DESCRIPTOR.message_types_by_name['RpbMapRedReq'] = _RPBMAPREDREQ +DESCRIPTOR.message_types_by_name['RpbMapRedResp'] = _RPBMAPREDRESP +DESCRIPTOR.message_types_by_name['RpbContent'] = _RPBCONTENT +DESCRIPTOR.message_types_by_name['RpbPair'] = _RPBPAIR +DESCRIPTOR.message_types_by_name['RpbLink'] = _RPBLINK +DESCRIPTOR.message_types_by_name['RpbBucketProps'] = _RPBBUCKETPROPS class RpbErrorResp(message.Message): __metaclass__ = reflection.GeneratedProtocolMessageType diff --git a/setup.py b/setup.py index 7bcb9360..c97fb055 100755 --- a/setup.py +++ b/setup.py @@ -16,20 +16,15 @@ def make_pb(): subprocess.call(['protoc', '--python_out=.', './riak/transports/riakclient.proto']) if __name__ == "__main__": + requires = ['protobuf(==2.4.1)'] if platform.python_version() < '2.7': - test_require = ["unittest2"] - else: - test_require = [] + requires.append("unittest2") setup( name='riak', version='1.3.0', packages = find_packages(), - requires = test_require, - extras_require = { - 'protobuf': ['protobuf>=2.3.0'], - }, - dependency_links = ["http://downloads.basho.com/support"], + requires = requires, package_data = { '' : ['*.proto'], 'riak' : ['erl_src/*'] From 6f0cb113ee94ca20eabb5430eea1cc70923474e8 Mon Sep 17 00:00:00 2001 From: Sreejith K Date: Thu, 29 Mar 2012 12:50:03 +0530 Subject: [PATCH 0184/1060] Remove all indexes for a field --- riak/riak_object.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index b0d8a784..d86b95f2 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -197,7 +197,7 @@ def add_index(self, field, value): return self - def remove_index(self, field, value): + def remove_index(self, field=None, value=None): """ Remove the specified field/value pair as an index on this object. @@ -207,11 +207,22 @@ def remove_index(self, field, value): :type value: string or integer :rtype: self """ - rie = RiakIndexEntry(field, value) - if rie in self._metadata[MD_INDEX]: - self._metadata[MD_INDEX].remove(rie) + if not field and not value: + ries = self._metadata[MD_INDEX] + elif field and not value: + ries = [x for x in self._metadata[MD_INDEX] if x.get_field() == field] + elif field and value: + ries = [RiakIndexEntry(field, value)] + else: + raise Exception("Cannot pass value without a field name while removing index") + + for rie in ries: + if rie in self._metadata[MD_INDEX]: + self._metadata[MD_INDEX].remove(rie) return self + remove_indexes = remove_index + def set_indexes(self, indexes): """ Replaces all indexes on a Riak object. Currenly supports an iterable of 2 item tuples, From 434dd805ca1b3c2e7ff7f13e92ec749c0197e7ff Mon Sep 17 00:00:00 2001 From: Sreejith K Date: Thu, 29 Mar 2012 12:53:42 +0530 Subject: [PATCH 0185/1060] Using a copy of Index metadata --- riak/riak_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index d86b95f2..59a18c82 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -208,7 +208,7 @@ def remove_index(self, field=None, value=None): :rtype: self """ if not field and not value: - ries = self._metadata[MD_INDEX] + ries = self._metadata[MD_INDEX][:] elif field and not value: ries = [x for x in self._metadata[MD_INDEX] if x.get_field() == field] elif field and value: From 7fe0524065b41092dcb67472e2161292a04091b8 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 10:04:50 -0400 Subject: [PATCH 0186/1060] Move test_generate_key to base test case (supported by both transports) and resolve key counts problem. Fixes #96. --- riak/tests/test_all.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 70281e5b..d3dbdb21 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -126,6 +126,19 @@ def test_store_and_get(self): self.assertRaises(TypeError, bucket.new, 'foo', u'éå') self.assertRaises(TypeError, bucket.new, 'foo', u'éå') + def test_generate_key(self): + # Ensure that Riak generates a random key when + # the key passed to bucket.new() is None. + bucket = self.client.bucket('random_key_bucket') + existing_keys = bucket.get_keys() + o = bucket.new(None, data={}) + self.assertIsNone(o.get_key()) + o.store() + self.assertIsNotNone(o.get_key()) + self.assertNotIn('/', o.get_key()) + self.assertNotIn(o.get_key(), existing_keys) + self.assertEqual(len(bucket.get_keys()), len(existing_keys) + 1) + def test_binary_store_and_get(self): bucket = self.client.bucket('bucket') # Store as binary, retrieve as binary, then compare... @@ -1018,19 +1031,6 @@ def test_no_returnbody(self): o = bucket.new("foo", "bar").store(return_body=False) self.assertEqual(o.vclock(), None) - def test_generate_key(self): - # Ensure that Riak generates a random key when - # the key passed to bucket.new() is None. - bucket = self.client.bucket('random_key_bucket') - for key in bucket.get_keys(): - bucket.get(str(key)).delete() - o = bucket.new(None, data={}) - self.assertIsNone(o.get_key()) - o.store() - self.assertIsNotNone(o.get_key()) - self.assertNotIn('/', o.get_key()) - self.assertEqual(len(bucket.get_keys()), 1) - def test_too_many_link_headers_shouldnt_break_http(self): bucket = self.client.bucket("bucket") o = bucket.new("lots_of_links", "My god, it's full of links!") From ea666fbcd29b37be39f342a7cd9dfee58b73ebbf Mon Sep 17 00:00:00 2001 From: Sreejith K Date: Thu, 29 Mar 2012 20:06:21 +0530 Subject: [PATCH 0187/1060] Tests for remove_indexes --- riak/tests/test_all.py | 50 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 70281e5b..5c50d647 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -685,6 +685,56 @@ def test_set_indexes(self): self.assertEqual(1, len(result)) self.assertEqual('foo', result[0].get_key()) + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_remove_indexes(self): + if not self.is_2i_supported(): + return True + + bucket = self.client.bucket('indexbucket') + bar = bucket.new('bar', 1).add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + result = self.client.index('indexbucket', 'bar_int', 1).run() + self.assertEqual(1, len(result)) + self.assertEqual(3, len(bar.get_indexes())) + self.assertEqual(2, len(bar.get_indexes('bar_int'))) + + # remove all indexes + bar = bar.remove_indexes().store() + result = self.client.index('indexbucket', 'bar_int', 1).run() + self.assertEqual(0, len(result)) + result = self.client.index('indexbucket', 'baz_bin', 'baz').run() + self.assertEqual(0, len(result)) + self.assertEqual(0, len(bar.get_indexes())) + self.assertEqual(0, len(bar.get_indexes('bar_int'))) + self.assertEqual(0, len(bar.get_indexes('baz_bin'))) + + # add index again + bar = bar.add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + # remove all index with field='bar_int' + bar = bar.remove_index(field='bar_int').store() + result = self.client.index('indexbucket', 'bar_int', 1).run() + self.assertEqual(0, len(result)) + result = self.client.index('indexbucket', 'bar_int', 2).run() + self.assertEqual(0, len(result)) + result = self.client.index('indexbucket', 'baz_bin', 'baz').run() + self.assertEqual(1, len(result)) + self.assertEqual(1, len(bar.get_indexes())) + self.assertEqual(0, len(bar.get_indexes('bar_int'))) + self.assertEqual(1, len(bar.get_indexes('baz_bin'))) + + # add index again + bar = bar.add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + # remove an index field value pair + bar = bar.remove_index(field='bar_int', value=2).store() + result = self.client.index('indexbucket', 'bar_int', 1).run() + self.assertEqual(1, len(result)) + result = self.client.index('indexbucket', 'bar_int', 2).run() + self.assertEqual(0, len(result)) + result = self.client.index('indexbucket', 'baz_bin', 'baz').run() + self.assertEqual(1, len(result)) + self.assertEqual(2, len(bar.get_indexes())) + self.assertEqual(1, len(bar.get_indexes('bar_int'))) + self.assertEqual(1, len(bar.get_indexes('baz_bin'))) + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_query(self): if not self.is_2i_supported(): From 16d5ca34bb14ab3be4548bcdbb59609e8e0fc721 Mon Sep 17 00:00:00 2001 From: Sreejith Kesavan Date: Thu, 29 Mar 2012 23:42:37 +0530 Subject: [PATCH 0188/1060] Result in RiakObject.populate doesn't have MD_INDEX field when using RiakPbcTransport. --- riak/riak_object.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/riak/riak_object.py b/riak/riak_object.py index 59a18c82..190d0c81 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -500,6 +500,8 @@ def populate(self, Result) : if len(contents) > 0: (metadata, data) = contents.pop(0) self._exists = True + if not metadata.has_key(MD_INDEX): + metadata[MD_INDEX] = [] self.set_metadata(metadata) self.set_encoded_data(data) # Create objects for all siblings From e68c188fd50f420576003f8896d9c7cb3ce1e945 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 14:44:40 -0400 Subject: [PATCH 0189/1060] Add Travis-CI support. --- .travis.yml | 10 ++++++++++ README.rst | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..bce57023 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: python +python: + - "2.6" + - "2.7" +install: python setup.py develop +script: python setup.py test +# TODO: get search enabled on Travis' Riak install, or fix the test server +env: "SKIP_LUWAK=1 SKIP_SEARCH=1" +notifications: + email: clients@basho.com diff --git a/README.rst b/README.rst index 17565f96..75a0bc81 100644 --- a/README.rst +++ b/README.rst @@ -2,6 +2,7 @@ Python Client for Riak ======================== +.. image:: https://secure.travis-ci.org/basho/riak-python-client.png?branch=master Documentation ============== @@ -11,7 +12,7 @@ The documentation source is found in `docs/ subdirectory `_ and can be built with `Sphinx `_. -Documentation for Riak is available at http://wiki.basho.com/How-Things-Work.html +Documentation for Riak is available at http://wiki.basho.com/Riak.html Install ======= From bcda5d70f3d56f85cf9ed895283d36f76a7c0296 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 15:06:03 -0400 Subject: [PATCH 0190/1060] Manually install protobuf using setup.py. --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bce57023..40b19756 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,9 @@ language: python python: - "2.6" - "2.7" -install: python setup.py develop +install: + - python setup.py easy_install protobuf=2.4.1 + - python setup.py develop script: python setup.py test # TODO: get search enabled on Travis' Riak install, or fix the test server env: "SKIP_LUWAK=1 SKIP_SEARCH=1" From 7a6931987bd4698546b87a8cb2338c5bf955c20b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 15:31:30 -0400 Subject: [PATCH 0191/1060] Fixing dependencies and build script again. --- .travis.yml | 6 ++---- setup.py | 10 ++++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 40b19756..d4c4ec76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,8 @@ language: python python: - "2.6" - "2.7" -install: - - python setup.py easy_install protobuf=2.4.1 - - python setup.py develop -script: python setup.py test +install: ./setup.py develop +script: ./setup.py test # TODO: get search enabled on Travis' Riak install, or fix the test server env: "SKIP_LUWAK=1 SKIP_SEARCH=1" notifications: diff --git a/setup.py b/setup.py index c97fb055..3de069a2 100755 --- a/setup.py +++ b/setup.py @@ -16,15 +16,17 @@ def make_pb(): subprocess.call(['protoc', '--python_out=.', './riak/transports/riakclient.proto']) if __name__ == "__main__": - requires = ['protobuf(==2.4.1)'] + install_requires = {'protobuf': ['>= 2.4.0', '< 2.5.0'] } + tests_require = [] if platform.python_version() < '2.7': - requires.append("unittest2") + tests_require.append("unittest2") setup( name='riak', version='1.3.0', packages = find_packages(), - requires = requires, + install_requires = install_requires, + tests_require = tests_require, package_data = { '' : ['*.proto'], 'riak' : ['erl_src/*'] @@ -35,7 +37,7 @@ def make_pb(): license='Apache 2', platforms='Platform Independent', author='Basho Technologies', - author_email='riak@basho.com', + author_email='clients@basho.com', test_suite='riak.tests.suite', url='https://github.com/basho/riak-python-client' ) From c935fc2b1706eed683ecdfaeb257b436103f0d9f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 15:53:51 -0400 Subject: [PATCH 0192/1060] Update build status image to link to the build status page. [ci skip] --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 75a0bc81..62cf297e 100644 --- a/README.rst +++ b/README.rst @@ -3,6 +3,7 @@ Python Client for Riak ======================== .. image:: https://secure.travis-ci.org/basho/riak-python-client.png?branch=master + :target: http://travis-ci.org/basho/riak-python-client Documentation ============== From 28b83f2b45b04500591d908af9232775a464d4f5 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 17:12:16 -0400 Subject: [PATCH 0193/1060] Missed this spot where arguments were positional. --- riak/riak_object.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 1eefd6c8..6a36a905 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -557,7 +557,7 @@ def get_sibling_count(self): """ return len(self._siblings) - def get_sibling(self, i, r=None): + def get_sibling(self, i, r=None, pr=None): """ Retrieve a sibling by sibling number. @@ -573,11 +573,12 @@ def get_sibling(self, i, r=None): else: # Use defaults if not specified. r = self._bucket.get_r(r) + pr = self._bucket.get_pr(pr) # Run the request... vtag = self._siblings[i] obj = RiakObject(self._client, self._bucket, self._key) - obj.reload(r, vtag) + obj.reload(r=r, pr=pr, vtag=vtag) # And make sure it knows who it's siblings are self._siblings[i] = obj From a5902a46ad04220e9309ddae46fd28f85a3b17f7 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 17:25:11 -0400 Subject: [PATCH 0194/1060] Now HTTP and PBC both fail in the same place in test_siblings. --- riak/tests/test_all.py | 6 +++--- riak/transports/pbc.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index efe9f718..9bfe99a4 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -264,7 +264,7 @@ def test_siblings(self): # Set up the bucket, clear any existing object... bucket = self.client.bucket('multiBucket') bucket.set_allow_multiples(True) - obj = bucket.get('foo') + obj = bucket.get_binary('foo') obj.delete() obj.reload() @@ -281,9 +281,9 @@ def test_siblings(self): if randval not in vals: break - other_obj = other_bucket.new('foo', randval) + other_obj = other_bucket.new_binary('foo', str(randval)) other_obj.store() - vals.add(randval) + vals.add(str(randval)) # Make sure the object has itself plus four siblings... obj.reload() diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 886558a9..0538c4f0 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -245,7 +245,7 @@ def get(self, robj, r=None, pr=None, vtag=None): contents.append(self.decode_content(c)) return resp.vclock, contents else: - return 0 + return None def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """ From 0f794c94503f3d543e6f4f5e5393636bb215553c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 29 Mar 2012 17:35:15 -0400 Subject: [PATCH 0195/1060] Use proper behavior for sibling generation in test_siblings. --- riak/tests/test_all.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 9bfe99a4..3ae08a4e 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -265,11 +265,10 @@ def test_siblings(self): bucket = self.client.bucket('multiBucket') bucket.set_allow_multiples(True) obj = bucket.get_binary('foo') - obj.delete() - - obj.reload() - self.assertFalse(obj.exists()) - self.assertEqual(obj.get_data(), None) + # Even if it previously existed, let's store a base resolved version + # from which we can diverge by sending a stale vclock. + obj.set_data('start') + obj.store() # Store the same object five times... vals = set() @@ -282,6 +281,7 @@ def test_siblings(self): break other_obj = other_bucket.new_binary('foo', str(randval)) + other_obj._vclock = obj._vclock other_obj.store() vals.add(str(randval)) @@ -304,9 +304,6 @@ def test_siblings(self): self.assertEqual(obj.get_sibling_count(), 0) self.assertEqual(obj.get_data(), obj3.get_data()) - # Clean up for next test... - obj.delete() - def test_javascript_source_map(self): # Create the object... bucket = self.client.bucket("bucket") From 9f3013839c02a3303465fd5c9466e123b310b4e2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 30 Mar 2012 15:18:08 -0400 Subject: [PATCH 0196/1060] Release 1.4.0. --- RELEASE_NOTES.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ THANKS | 44 ++++++++++++++++++++++++++++++++++++-------- setup.py | 8 ++++++-- 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 3dc10669..fc181517 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,50 @@ # Riak Python Client Release Notes +## 1.4.0 Feature Release - 2012-03-30 + +Release 1.4.0 is a feature release comprising over 117 individual +commits. + +Noteworthy features: + +* Python 2.6 and 2.7 are supported. On 2.6, the unittest2 package is + required to run the test suite. +* Google's official protobuf package (2.4.1 or later) is now a + dependency. The package from downloads.basho.com/support is no + longer necessary. +* Travis-CI is enabled on the client. Go to + http://travis-ci.org/basho/riak-python-client for build status. +* Riak 1.0+ features, namely secondary indexes and primary quora + (PR/PW), are supported. +* `if_none_match` is a valid request option when storing objects, and + will prevent the write when set to `True` if the key already exists. +* Links can be set wholesale using the `set_links()` method. +* Transport-specific options can be passed through when creating a + `Client` object. +* A connection manager was added that will (when manipulated manually) + allow connections to multiple Riak nodes. This will be fully + integrated in a future release. + +Noteworthy bugfixes: + +* Links now use the proper URL-encoding in HTTP headers, preventing + problems with explosion from multiple encoding passes. +* Many fixes were applied to make the Protocol Buffers transport more + stable. +* `RiakObject.get_content_type()` will behave properly when content + type is not set. +* Deprecated transport classes were removed since their functionality + had folded into the primary transports. +* A temporary fix was made for unicode bucket/key names which raises + an error when they are used and cannot be coerced to ASCII. +* The Erlang sources/beams for the TestServer are now included in the + package. +* MapReduce failures will now produce a more useful error message and + be handled properly when no results are returned. + +There are lots of other great fixes from our wonderful +community. [Check them out!](https://github.com/basho/riak-python-client/compare/1.3.0...1.4.0) + ## 1.3.0 Feature Release - 2011-08-04 Release 1.3.0 is a feature release bringing a slew of updates. diff --git a/THANKS b/THANKS index c8c1371a..16927ec4 100644 --- a/THANKS +++ b/THANKS @@ -1,16 +1,44 @@ The following people have contributed to the Riak Python client: +Andrew Thompson Andy Gross -Justin Sheehy -Rusty Klophaus +Armon Dadgar +Brett Hoerner +Brian Roach +Bryan Fink +Daniel Lindsley +Daniel Néri +Daniel Reverri +David Koblas +Dmitry Rozhkov +Eric Florenzano +Eric Moritz +Filip de Waard +Gilles Devaux +Greg Nelson +Greg Stein +Gregory Burd +Ian Plosker Jayson Baird +Jeffrey Massung Jon Meredith -Eric Florenzano -Silas Sewell -Matt Heitzenroder +Josip Lisec +Justin Sheehy +Kevin Smith Mark Erdmann -Greg Nelson +Mark Phillips +Mathias Meyer +Matt Heitzenroder Mikhail Sobolev -Eric Moritz -Brett Hoerner +Reid Draper +Russell Brown +Rusty Klophaus Scott Lystig Fritchie +Sean Cribbs +Shuhao Wu +Silas Sewell +Socrates Lee +Soren Hansen +Sreejith Kesavan +Timothée Peignier +William Kral diff --git a/setup.py b/setup.py index 3de069a2..50980edc 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def make_pb(): setup( name='riak', - version='1.3.0', + version='1.4.0', packages = find_packages(), install_requires = install_requires, tests_require = tests_require, @@ -39,5 +39,9 @@ def make_pb(): author='Basho Technologies', author_email='clients@basho.com', test_suite='riak.tests.suite', - url='https://github.com/basho/riak-python-client' + url='https://github.com/basho/riak-python-client', + classifiers = ['License :: OSI Approved :: Apache Software License', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Topic :: Database'] ) From bcf4dfb12912157656ba36e9f5dc9f21026abfbb Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 30 Mar 2012 16:06:21 -0400 Subject: [PATCH 0197/1060] Update some documentation for 1.4.0. --- docs/conf.py | 34 +++++++++++++++++----------------- docs/index.rst | 9 ++++----- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 50cd55bf..4d66b707 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,17 +40,17 @@ master_doc = 'index' # General information about the project. -project = u'Riak (Python binding)' -copyright = u'2010, Daniel Lindsley' +project = u'Riak Python Client' +copyright = u'2010-2012, Basho Technologies' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '1.1.1' +version = '1.4.0' # The full version, including alpha/beta/rc tags. -release = '1.1.1' +release = '1.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -120,7 +120,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. @@ -177,10 +177,10 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'RiakPythonbinding.tex', u'Riak (Python binding) Documentation', - u'Daniel Lindsley', 'manual'), -] +# latex_documents = [ +# ('index', 'RiakPythonbinding.tex', u'Riak (Python binding) Documentation', +# u'Daniel Lindsley', 'manual'), +# ] # The name of an image file (relative to this directory) to place at the top of # the title page. @@ -210,19 +210,19 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'riakpythonbinding', u'Riak (Python binding) Documentation', - [u'Daniel Lindsley'], 1) -] +# man_pages = [ +# ('index', 'riakpythonbinding', u'Riak (Python binding) Documentation', +# [u'Daniel Lindsley'], 1) +# ] # -- Options for Epub output --------------------------------------------------- # Bibliographic Dublin Core info. -epub_title = u'Riak (Python binding)' -epub_author = u'Daniel Lindsley' -epub_publisher = u'Daniel Lindsley' -epub_copyright = u'2010, Daniel Lindsley' +# epub_title = u'Riak (Python binding)' +# epub_author = u'Daniel Lindsley' +# epub_publisher = u'Daniel Lindsley' +# epub_copyright = u'2010, Daniel Lindsley' # The language of the text. It defaults to the language option # or en if the language is not set. diff --git a/docs/index.rst b/docs/index.rst index 40449224..f8261594 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,21 +1,20 @@ -Riak (Python binding) +Riak Python Client ===================== Installation ------------ #. Ensure Riak installed & running. (``riak ping``) -#. Install the Python binding: +#. Install the Python client: #. If you use Pip_, ``pip install riak``. - #. If you use easy_install_, you should consider using Pip_, but can run - ``easy_install riak``. + #. If you use easy_install_, run ``easy_install riak``. #. You can download the package off PyPI_, extract it and run ``python setup.py install``. .. _Pip: http://pip.openplans.org/ .. _easy_install: http://pypi.python.org/pypi/setuptools -.. _PyPI: http://pypi.python.org/pypi/riak/1.1.1 +.. _PyPI: http://pypi.python.org/pypi/riak/1.4.0 Contents: From d3debb687a6326a29e8ade372a95121b7c08b921 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Sat, 7 Apr 2012 14:48:45 -0400 Subject: [PATCH 0198/1060] Fixed indentation according to PEP8 --- riak/transports/connection.py | 281 +++++++++++++++++----------------- 1 file changed, 140 insertions(+), 141 deletions(-) diff --git a/riak/transports/connection.py b/riak/transports/connection.py index 67b16b46..12172df4 100644 --- a/riak/transports/connection.py +++ b/riak/transports/connection.py @@ -21,169 +21,168 @@ import contextlib import functools - class ConnectionManager(object): - # Must be constructable with: connection_class(host, port) - # Must have two attribute: host and port - # Must have a close() method - connection_class = None - - def __init__(self, hostports=[]): - # We want a private copy of this list: either to detach the argument - # default, or to detach from the caller's list. - self.hostports = hostports[:] - - # Open a connection to each specified host/port. On single-threaded - # systems, this will create a round-robin across all specified servers. - # When multi-threaded, this will give us an initial set for all the - # threads to work with (and more will be created, according to demand). - self.conns = [self.connection_class(host, port) - for host, port in hostports] - - def add_hostport(self, host, port): - self.hostports.append((host, port)) - - # Open an initial connection. For single-threaded, this adds to the - # round-robin pool. On multi-threaded, it simply gives us an extra - # connectiong for the load-balancing across the servers. - self.conns.append(self.connection_class(host, port)) - - def remove_host(self, host, port=None): - if port is None: - self.hostports = [(h, p) for h, p in self.hostports - if h != host] - else: - self.hostports.remove((host, port)) - - # Now that the host/port pair has been removed from self.hostports, - # no connections on this pair will be added in .giveback(). Thus, the - # existing connections are all that may exist at this time. We'll - # snapshot the list, and look for offending connections, then try and - # remove them, being wary that race conditions may remove them before - # we can remove it. - for conn in self.conns[:]: - if conn.host == host and (port is None or conn.port == port): + # Must be constructable with: connection_class(host, port) + # Must have two attribute: host and port + # Must have a close() method + connection_class = None + + def __init__(self, hostports=[]): + # We want a private copy of this list: either to detach the argument + # default, or to detach from the caller's list. + self.hostports = hostports[:] + + # Open a connection to each specified host/port. On single-threaded + # systems, this will create a round-robin across all specified servers. + # When multi-threaded, this will give us an initial set for all the + # threads to work with (and more will be created, according to demand). + self.conns = [self.connection_class(host, port) + for host, port in hostports] + + def add_hostport(self, host, port): + self.hostports.append((host, port)) + + # Open an initial connection. For single-threaded, this adds to the + # round-robin pool. On multi-threaded, it simply gives us an extra + # connectiong for the load-balancing across the servers. + self.conns.append(self.connection_class(host, port)) + + def remove_host(self, host, port=None): + if port is None: + self.hostports = [(h, p) for h, p in self.hostports + if h != host] + else: + self.hostports.remove((host, port)) + + # Now that the host/port pair has been removed from self.hostports, + # no connections on this pair will be added in .giveback(). Thus, the + # existing connections are all that may exist at this time. We'll + # snapshot the list, and look for offending connections, then try and + # remove them, being wary that race conditions may remove them before + # we can remove it. + for conn in self.conns[:]: + if conn.host == host and (port is None or conn.port == port): + try: + self.conns.remove(conn) + except ValueError: + # Another thread removed the connection. It won't be coming back, + # so we have nothing to do here. + pass + else: + # If the connection was still present (no ValueError), then we + # should go ahead and close it down. + conn.close() + + # Just in case somebody uses a host/port combo and typos... + remove_hostport = remove_host + + def take(self): + if len(self.conns) == 0: + # RACE: in a multi-threaded environment, a conn might arrive in + # self.conns, but... no biggy. If we're bouncing up against + # needing a new connection, then we'll just create one. + return self._new_connection() + + # RACE: self.conns might empty out right now, so we need to protect + # our access to it. try: - self.conns.remove(conn) - except ValueError: - # Another thread removed the connection. It won't be coming back, - # so we have nothing to do here. - pass + # round-robin: take from the front, we'll append when it comes back + return self.conns.pop(0) + except IndexError: + return self._new_connection() + + def giveback(self, conn): + # Connections using a host/port pair that is NOT in self.hostports + # should be ignored. Likely, remove_host() was called while this + # connection was borrowed for some work. + if (conn.host, conn.port) in self.hostports: + self.conns.append(conn) else: - # If the connection was still present (no ValueError), then we - # should go ahead and close it down. - conn.close() - - # Just in case somebody uses a host/port combo and typos... - remove_hostport = remove_host - - def take(self): - if len(self.conns) == 0: - # RACE: in a multi-threaded environment, a conn might arrive in - # self.conns, but... no biggy. If we're bouncing up against - # needing a new connection, then we'll just create one. - return self._new_connection() - - # RACE: self.conns might empty out right now, so we need to protect - # our access to it. - try: - # round-robin: take from the front, we'll append when it comes back - return self.conns.pop(0) - except IndexError: - return self._new_connection() - - def giveback(self, conn): - # Connections using a host/port pair that is NOT in self.hostports - # should be ignored. Likely, remove_host() was called while this - # connection was borrowed for some work. - if (conn.host, conn.port) in self.hostports: - self.conns.append(conn) - else: - # Proactively close the connection. The caller won't know whether - # we put it into our list, or left the connection for the caller - # to deal with (and close). - conn.close() - - @contextlib.contextmanager - def withconn(self): - conn = self.take() - try: - yield conn - finally: - self.giveback(conn) - - def _new_connection(self): - if len(self.hostports) == 0: - raise NoHostsDefined() - - # Grab the first host/port combo. We'll put this at the end, so that - # we do a round-robin on the host/port pairs. - host, port = self.hostports[0] - conn = self.connection_class(host, port) - - if len(self.hostports) == 1: - # No rotation needed. - return conn - - # Be careful about rotating. We want to append before removing, so that - # we never hit a len==0 race condition (which could prevent the creation - # of needed connections). - self.hostports.append((host, port)) - - # RACE: another thread may have appended the same host/port pair. We - # will add another pair. Each thread will remove one (either [0], or - # one that had been appened), resulting in a correct state of a single - # pair in the list. - # RACE: another thread may get the host/port pair from hostports[0] - # before we have a chance to remove it. We don't need precision - # round-robin behavior; just something close. - # RACE: another thread may have removed hostports[0] (which we are - # also trying to remove), but it will have placed another copy at - # the end before doing so. We have also added a host/port pair, and - # will remove one, leaving the list in a correct state. - self.hostports.remove((host, port)) - - return conn + # Proactively close the connection. The caller won't know whether + # we put it into our list, or left the connection for the caller + # to deal with (and close). + conn.close() + + @contextlib.contextmanager + def withconn(self): + conn = self.take() + try: + yield conn + finally: + self.giveback(conn) + + def _new_connection(self): + if len(self.hostports) == 0: + raise NoHostsDefined() + + # Grab the first host/port combo. We'll put this at the end, so that + # we do a round-robin on the host/port pairs. + host, port = self.hostports[0] + conn = self.connection_class(host, port) + + if len(self.hostports) == 1: + # No rotation needed. + return conn + + # Be careful about rotating. We want to append before removing, so that + # we never hit a len==0 race condition (which could prevent the creation + # of needed connections). + self.hostports.append((host, port)) + + # RACE: another thread may have appended the same host/port pair. We + # will add another pair. Each thread will remove one (either [0], or + # one that had been appened), resulting in a correct state of a single + # pair in the list. + # RACE: another thread may get the host/port pair from hostports[0] + # before we have a chance to remove it. We don't need precision + # round-robin behavior; just something close. + # RACE: another thread may have removed hostports[0] (which we are + # also trying to remove), but it will have placed another copy at + # the end before doing so. We have also added a host/port pair, and + # will remove one, leaving the list in a correct state. + self.hostports.remove((host, port)) + + return conn class Socket(object): - def __init__(self, host, port): - self.host = host - self.port = port + def __init__(self, host, port): + self.host = host + self.port = port - self.sock = None + self.sock = None - def maybe_connect(self): - if self.sock is None: - self.sock = s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + def maybe_connect(self): + if self.sock is None: + self.sock = s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - s.connect((self.host, self.port)) - except: - self.close() - raise + try: + s.connect((self.host, self.port)) + except: + self.close() + raise - def close(self): - if self.sock is not None: - self.sock.close() - self.sock = None + def close(self): + if self.sock is not None: + self.sock.close() + self.sock = None class FactoryConnectionManager(ConnectionManager): - def __init__(self, connection_class, hostports=[]): - self.connection_class = connection_class - ConnectionManager.__init__(self, hostports) + def __init__(self, connection_class, hostports=[]): + self.connection_class = connection_class + ConnectionManager.__init__(self, hostports) def cm_using(connection_class): - return functools.partial(FactoryConnectionManager, connection_class) + return functools.partial(FactoryConnectionManager, connection_class) HTTPConnectionManager = cm_using(httplib.HTTPConnection) SocketConnectionManager = cm_using(Socket) class NoHostsDefined(Exception): - pass + pass From 89ea88fb8152b6cab577687ad1c12469072a5473 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Sat, 2 Jun 2012 21:30:05 -0400 Subject: [PATCH 0199/1060] bucket.new's key argument should be None This is done as Riak should automatically generate a key for the object upon save. This is already done by RiakObject. Many people who are new to the riak-python client do not know this could be done. Adding this will help. --- riak/bucket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 64c492c0..348d26ad 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -255,12 +255,12 @@ def set_decoder(self, content_type, decoder): self._decoders[content_type] = decoder return self - def new(self, key, data=None, content_type='application/json'): + def new(self, key=None, data=None, content_type='application/json'): """ Create a new :class:`RiakObject ` that will be stored as JSON. A shortcut for manually instantiating a :class:`RiakObject `. - :param key: Name of the key. + :param key: Name of the key. Leaving this to be None (default) will make Riak generate the key on store. :type key: string :param data: The data to store. :type data: object From 861b694629df8b421e53ee771765a5b4667756e6 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Fri, 15 Jun 2012 22:45:25 -0400 Subject: [PATCH 0200/1060] Fixed issue #127 Honestly I think #121 should be adopted instead of continuing with this. --- riak/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/client.py b/riak/client.py index e53f2347..00be8b89 100644 --- a/riak/client.py +++ b/riak/client.py @@ -210,7 +210,7 @@ def get_pw(self): :rtype: integer """ - return self._pr + return self._pw def set_pw(self, pw): """ @@ -221,7 +221,7 @@ def set_pw(self, pw): :type pw: integer :rtype: self """ - self._pr = pr + self._pw = pw return self def get_client_id(self): From 332e0390116b63bd2a3e3fab7d4c9519d7e4d2ec Mon Sep 17 00:00:00 2001 From: Reid Draper Date: Tue, 19 Jun 2012 17:49:55 -0400 Subject: [PATCH 0201/1060] Add a proper content-type to map reduce requests --- riak/transports/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index c77955b6..9e740de9 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -236,7 +236,8 @@ def mapred(self, inputs, query, timeout=None): # Do the request... url = "/" + self._mapred_prefix - response = self.http_request('POST', url, {}, content) + headers = {'Content-Type': 'application/json'} + response = self.http_request('POST', url, headers, content) # Make sure the expected status code came back... status = response[0]['http_code'] From b437d6bda8b372a2ad8c14ebd1b5e4248f94fdde Mon Sep 17 00:00:00 2001 From: Reid Draper Date: Tue, 19 Jun 2012 18:39:24 -0400 Subject: [PATCH 0202/1060] Version bump and release notes for 1.4.1 --- RELEASE_NOTES.md | 10 ++++++++++ docs/conf.py | 4 ++-- setup.py | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index fc181517..0095f2c7 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,15 @@ # Riak Python Client Release Notes +## 1.4.1 Patch Release - 2012-06-19 + +Noteworthy features: + +* New Riak objects support Riak-created random keys + +Noteworthy bugfixes: + +* Map Reduce queries now use "application/json" as the Content-Type + ## 1.4.0 Feature Release - 2012-03-30 Release 1.4.0 is a feature release comprising over 117 individual diff --git a/docs/conf.py b/docs/conf.py index 4d66b707..d8e2e07d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,9 +48,9 @@ # built documents. # # The short X.Y version. -version = '1.4.0' +version = '1.4.1' # The full version, including alpha/beta/rc tags. -release = '1.4.0' +release = '1.4.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 50980edc..0c8f5b07 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def make_pb(): setup( name='riak', - version='1.4.0', + version='1.4.1', packages = find_packages(), install_requires = install_requires, tests_require = tests_require, From 24bad05f3471a3fd8c1cd790445cccb2b65de888 Mon Sep 17 00:00:00 2001 From: Peter Teichman Date: Fri, 22 Jun 2012 09:31:13 -0400 Subject: [PATCH 0203/1060] Correct the MANIFEST.in path to erl_src (now riak/erl_src/*) Without a proper path to erl_src, the TestServer memory backends aren't included and TestServer cannot start. --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index e691aa45..6f864d48 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include docs/* -include erl_src/* +include riak/erl_src/* include THANKS include README.rst include LICENSE From 212ab82f7e36bae8b99dc18f3770668f7e927594 Mon Sep 17 00:00:00 2001 From: Peter Teichman Date: Fri, 22 Jun 2012 10:28:25 -0400 Subject: [PATCH 0204/1060] Import socket in TestServer This fixes a few NameError exceptions when attempting to use TestServer, which attempts to create socket connections and handle socket.error without importing it. --- riak/test_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/riak/test_server.py b/riak/test_server.py index 56715fbe..0d89daa6 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -6,6 +6,7 @@ import re import random import shutil +import socket import time from subprocess import Popen, PIPE from riak.util import deep_merge From 6a3d927b5b10014efbdcc491f2a75220bac4c40d Mon Sep 17 00:00:00 2001 From: Max Countryman Date: Fri, 22 Jun 2012 17:31:55 -0400 Subject: [PATCH 0205/1060] updating TestServer to use new-style classes --- riak/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/test_server.py b/riak/test_server.py index 0d89daa6..58507b6d 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -52,7 +52,7 @@ def printable(item): return "[\n%s%s\n%s]" % (padding, values, parent_padding) -class TestServer: +class TestServer(object): VM_ARGS_DEFAULTS = { "-name": "riaktest%d@127.0.0.1" % random.randint(0, 100000), "-setcookie": "%d_%d" % (random.randint(0, 100000), random.randint(0, 100000)), From 904d4514fd423664a59ed2e69320ee1b897b6ba5 Mon Sep 17 00:00:00 2001 From: Max Countryman Date: Fri, 22 Jun 2012 17:33:11 -0400 Subject: [PATCH 0206/1060] updating TestServer to use new-style classes --- riak/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/test_server.py b/riak/test_server.py index 0d89daa6..58507b6d 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -52,7 +52,7 @@ def printable(item): return "[\n%s%s\n%s]" % (padding, values, parent_padding) -class TestServer: +class TestServer(object): VM_ARGS_DEFAULTS = { "-name": "riaktest%d@127.0.0.1" % random.randint(0, 100000), "-setcookie": "%d_%d" % (random.randint(0, 100000), random.randint(0, 100000)), From bd78399995c10c16dd583f7a5c6391f1b8853ef0 Mon Sep 17 00:00:00 2001 From: Max Countryman Date: Fri, 22 Jun 2012 17:39:46 -0400 Subject: [PATCH 0207/1060] removing unused import; updating to new-style classes --- riak/riak_index_entry.py | 2 +- riak/riak_object.py | 2 +- riak/search.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/riak/riak_index_entry.py b/riak/riak_index_entry.py index 0a26bb13..c642512c 100644 --- a/riak/riak_index_entry.py +++ b/riak/riak_index_entry.py @@ -18,7 +18,7 @@ under the License. """ -class RiakIndexEntry: +class RiakIndexEntry(object): def __init__(self, field, value): self._field = field self._value = str(value) diff --git a/riak/riak_object.py b/riak/riak_object.py index 6a36a905..4b995c88 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -17,7 +17,7 @@ specific language governing permissions and limitations under the License. """ -import types, copy, re +import types, copy from metadata import * from riak import RiakError from riak.riak_index_entry import RiakIndexEntry diff --git a/riak/search.py b/riak/search.py index 7fe2ab73..3b36dd8e 100644 --- a/riak/search.py +++ b/riak/search.py @@ -2,7 +2,7 @@ from xml.etree import ElementTree from xml.dom.minidom import Document -class RiakSearch: +class RiakSearch(object): def __init__(self, client, transport_class=None, host="127.0.0.1", port=8098): if transport_class is None: From 543bf6b2be906055b3ac21f1d857d93da88d88fd Mon Sep 17 00:00:00 2001 From: Jeremy Thurgood Date: Sat, 30 Jun 2012 13:56:51 +0200 Subject: [PATCH 0208/1060] Throw an exception if the TestServer fails to start. --- riak/test_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/riak/test_server.py b/riak/test_server.py index 0d89daa6..a3d26d7b 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -191,6 +191,8 @@ def wait_for_erlang_prompt(self): buffer += line if re.search(r"\(%s\)\d+>" % self.vm_args["-name"], buffer): prompted = True + if re.search(r'"Kernel pid terminated".*\n', buffer): + raise Exception("Riak test server failed to start.") def write_riak_script(self): with open(self._riak_script, "wb") as temp_bin_file: From 10a01d8e142d54af37ef1f5edfb67af94e6bf6e8 Mon Sep 17 00:00:00 2001 From: Jeremy Thurgood Date: Sat, 30 Jun 2012 15:05:01 +0200 Subject: [PATCH 0209/1060] Add crash_log to TestServer app_config. --- riak/test_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/riak/test_server.py b/riak/test_server.py index 0d89daa6..ffec8a14 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -110,6 +110,7 @@ def __init__(self, tmp_dir="/tmp/riak/test_server", self.app_config["riak_core"]["ring_state_dir"] = os.path.join(self.temp_dir, "data", "ring") self.app_config["riak_core"]["platform_data_dir"] = self.temp_dir + self.app_config["lager"] = {"crash_log": os.path.join(self.temp_dir, "log", "crash.log")} def prepare(self): if not self._prepared: From 9890cd0293b8b76deef3dc4e8fcc97c25230fb99 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 18 Jul 2012 17:33:42 -0400 Subject: [PATCH 0210/1060] Switch to using the riak_pb external library. --- riak/transports/pbc.py | 52 +- riak/transports/riakclient.proto | 265 ------- riak/transports/riakclient_pb2.py | 1112 ----------------------------- setup.py | 59 +- 4 files changed, 58 insertions(+), 1430 deletions(-) delete mode 100644 riak/transports/riakclient.proto delete mode 100644 riak/transports/riakclient_pb2.py diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 0538c4f0..06105b4b 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -47,9 +47,9 @@ import riak.util try: - import riakclient_pb2 + import riak_pb except ImportError: - riakclient_pb2 = None + riak_pb = None ## Protocol codes MSG_CODE_ERROR_RESP = 0 @@ -77,6 +77,10 @@ MSG_CODE_SET_BUCKET_RESP = 22 MSG_CODE_MAPRED_REQ = 23 MSG_CODE_MAPRED_RESP = 24 +MSG_CODE_INDEX_QUERY_REQ = 25 +MSG_CODE_INDEX_QUERY_RES = 26 +MSG_CODE_SEARCH_QUERY_REQ = 27 +MSG_CODE_SEARCH_QUERY_RESP = 28 RIAKC_RW_ONE = 4294967294 RIAKC_RW_QUORUM = 4294967293 @@ -160,7 +164,7 @@ def __init__(self, cm, client_id=None, max_attempts=1, **unused_options): """ Construct a new RiakPbcTransport object. """ - if riakclient_pb2 is None: + if riak_pb is None: raise RiakError("this transport is not available (no protobuf)") super(RiakPbcTransport, self).__init__() @@ -202,7 +206,7 @@ def set_client_id(self, client_id): """ Set the client id used by this connection """ - req = riakclient_pb2.RpbSetClientIdReq() + req = riak_pb.RpbSetClientIdReq() req.client_id = client_id msg_code, resp = self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req, @@ -230,7 +234,7 @@ def get(self, robj, r=None, pr=None, vtag=None): bucket = robj.get_bucket() - req = riakclient_pb2.RpbGetReq() + req = riak_pb.RpbGetReq() req.r = self.translate_rw_val(r) req.pr = self.translate_rw_val(pr) @@ -253,7 +257,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=Fa """ bucket = robj.get_bucket() - req = riakclient_pb2.RpbPutReq() + req = riak_pb.RpbPutReq() req.w = self.translate_rw_val(w) req.dw = self.translate_rw_val(dw) req.pw = self.translate_rw_val(pw) @@ -289,7 +293,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_matc """ bucket = robj.get_bucket() - req = riakclient_pb2.RpbPutReq() + req = riak_pb.RpbPutReq() req.w = self.translate_rw_val(w) req.dw = self.translate_rw_val(dw) req.pw = self.translate_rw_val(pw) @@ -319,7 +323,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ bucket = robj.get_bucket() - req = riakclient_pb2.RpbDelReq() + req = riak_pb.RpbDelReq() req.rw = self.translate_rw_val(rw) req.r = self.translate_rw_val(r) req.w = self.translate_rw_val(w) @@ -340,7 +344,7 @@ def get_keys(self, bucket): """ Lists all keys within a bucket. """ - req = riakclient_pb2.RpbListKeysReq() + req = riak_pb.RpbListKeysReq() req.bucket = bucket.get_name() keys = [] @@ -364,7 +368,7 @@ def get_bucket_props(self, bucket): """ Serialize bucket property request and deserialize response """ - req = riakclient_pb2.RpbGetBucketReq() + req = riak_pb.RpbGetBucketReq() req.bucket = bucket.get_name() msg_code, resp = self.send_msg(MSG_CODE_GET_BUCKET_REQ, req, @@ -381,7 +385,7 @@ def set_bucket_props(self, bucket, props): """ Serialize set bucket property request and deserialize response """ - req = riakclient_pb2.RpbSetBucketReq() + req = riak_pb.RpbSetBucketReq() req.bucket = bucket.get_name() if not 'n_val' in props and not 'allow_mult' in props: return self @@ -403,7 +407,7 @@ def mapred(self, inputs, query, timeout=None): content = json.dumps(job) - req = riakclient_pb2.RpbMapRedReq() + req = riak_pb.RpbMapRedReq() req.request = content req.content_type = "application/json" @@ -467,7 +471,7 @@ def send_pkt(self, conn, pkt): # If the last client_id used on this connection is different than our # client_id, then set a new ID on the connection. if conn.last_client_id != self._client_id: - req = riakclient_pb2.RpbSetClientIdReq() + req = riak_pb.RpbSetClientIdReq() req.client_id = self._client_id conn.send(self.encode_msg(MSG_CODE_SET_CLIENT_ID_REQ, req)) conn.last_client_id = self._client_id @@ -489,37 +493,43 @@ def recv_msg(self, conn, expect): self.recv_pkt(conn) msg_code, = struct.unpack("B", self._inbuf[:1]) if msg_code == MSG_CODE_ERROR_RESP: - msg = riakclient_pb2.RpbErrorResp() + msg = riak_pb.RpbErrorResp() msg.ParseFromString(self._inbuf[1:]) raise Exception(msg.errmsg) elif msg_code == MSG_CODE_PING_RESP: msg = None elif msg_code == MSG_CODE_GET_CLIENT_ID_RESP: - msg = riakclient_pb2.RpbGetClientIdResp() + msg = riak_pb.RpbGetClientIdResp() msg.ParseFromString(self._inbuf[1:]) elif msg_code == MSG_CODE_SET_CLIENT_ID_RESP: msg = None elif msg_code == MSG_CODE_GET_RESP: - msg = riakclient_pb2.RpbGetResp() + msg = riak_pb.RpbGetResp() msg.ParseFromString(self._inbuf[1:]) elif msg_code == MSG_CODE_PUT_RESP: - msg = riakclient_pb2.RpbPutResp() + msg = riak_pb.RpbPutResp() msg.ParseFromString(self._inbuf[1:]) elif msg_code == MSG_CODE_DEL_RESP: msg = None elif msg_code == MSG_CODE_LIST_KEYS_RESP: - msg = riakclient_pb2.RpbListKeysResp() + msg = riak_pb.RpbListKeysResp() msg.ParseFromString(self._inbuf[1:]) elif msg_code == MSG_CODE_LIST_BUCKETS_RESP: - msg = riakclient_pb2.RpbListBucketsResp() + msg = riak_pb.RpbListBucketsResp() msg.ParseFromString(self._inbuf[1:]) elif msg_code == MSG_CODE_GET_BUCKET_RESP: - msg = riakclient_pb2.RpbGetBucketResp() + msg = riak_pb.RpbGetBucketResp() msg.ParseFromString(self._inbuf[1:]) elif msg_code == MSG_CODE_SET_BUCKET_RESP: msg = None elif msg_code == MSG_CODE_MAPRED_RESP: - msg = riakclient_pb2.RpbMapRedResp() + msg = riak_pb.RpbMapRedResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_INDEX_QUERY_RESP: + msg = riak_pb.RpbIndexQueryResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_SEARCH_QUERY_RESP: + msg = riak_pb.RpbSearchQueryResp() msg.ParseFromString(self._inbuf[1:]) else: raise Exception("unknown msg code %s" % msg_code) diff --git a/riak/transports/riakclient.proto b/riak/transports/riakclient.proto deleted file mode 100644 index ac82cb10..00000000 --- a/riak/transports/riakclient.proto +++ /dev/null @@ -1,265 +0,0 @@ -/* ------------------------------------------------------------------- -** -** riakclient.proto: Protocol buffers for riak -** -** Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. -** -** This file is provided to you under the Apache License, -** Version 2.0 (the "License"); you may not use this file -** except in compliance with the License. You may obtain -** a copy of the License at -** -** http://www.apache.org/licenses/LICENSE-2.0 -** -** Unless required by applicable law or agreed to in writing, -** software distributed under the License is distributed on an -** "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -** KIND, either express or implied. See the License for the -** specific language governing permissions and limitations -** under the License. -** -** ------------------------------------------------------------------- -*/ -/* -** Revision: 1.1 -** -** Lowest Common Denominator Protocol Buffers Client -** - no ENUM (protobuffs_erlang does not support) -** -** Protocol -** -** The protocol encodes requests and responses as protocol buffer messages. -** Each request message results in one or more response messages. -** As message type and length are not encoded by PB they are sent -** on the wire as -** -** -** -** length is the length of msg_code (1 byte) plus the message length -** in bytes encoded in network order (big endian) -** -** msg_code indicates what is encoded as pbmsg -** -** pbmsg is the encoded protocol buffer message -** -** On connect, the client can make requests and will receive responses. -** For each request message there is a corresponding response message, -** or the server will respond with an error message if something has -** gone wrong. -** -** The client should be prepared to handle messages without any pbmsg -** (i.e. length==1) for requests like ping or a put without return_body set. -** -** RpbGetClientIdReq -> RpbGetClientIdResp -** RpbSetClientIdReq -> RpbSetClientIdResp -** RpbGetServerInfoReq -> RpbGetServerInfoResp -** RpbPingReq -> RpbPingResp -** RpbGetReq -> RpbErrorResp | RbpGetResp -** RpbPutReq -> RpbErrorResp | RpbPutResp -** RpbDelReq -> RpbErrorResp | RpbDelResp -** RpbListBucketsReq -> RpbErrorResp | RpbListBucketsResp -** RpbListKeysReq -> RpbErrorResp | RpbListKeysResp{1,} -** RpbGetBucketReq -> RpbErrorResp | RpbGetBucketResp -** -** -** Message Codes -** 0 - RpbErrorResp -** 1 - RpbPingReq - 0 length -** 2 - RpbPingResp (pong) - 0 length -** 3 - RpbGetClientIdReq -** 4 - RpbGetClientIdResp -** 5 - RpbSetClientIdReq -** 6 - RpbSetClientIdResp -** 7 - RpbGetServerInfoReq -** 8 - RpbGetServerInfoResp -** 9 - RpbGetReq -** 10 - RpbGetResp -** 11 - RpbPutReq -** 12 - RpbPutResp - 0 length -** 13 - RpbDelReq -** 14 - RpbDelResp -** 15 - RpbListBucketsReq -** 16 - RpbListBucketsResp -** 17 - RpbListKeysReq -** 18 - RpbListKeysResp{1,} -** 19 - RpbGetBucketReq -** 20 - RpbGetBucketResp -** 21 - RpbSetBucketReq -** 22 - RpbSetBucketResp -** 23 - RpbMapRedReq -** 24 - RpbMapRedResp{1,} -** -*/ - -// Error response - may be generated for any Req -message RpbErrorResp { - required bytes errmsg = 1; - required uint32 errcode = 2; -} - -// Get ClientId Request - no message defined, just send RpbGetClientIdReq message code -message RpbGetClientIdResp { - required bytes client_id = 1; // Client id in use for this connection -} - -message RpbSetClientIdReq { - required bytes client_id = 1; // Client id to use for this connection -} -// Set ClientId Request - no message defined, just send RpbSetClientIdReq message code - -// Get server info request - no message defined, just send RpbGetServerInfoReq message code - -message RpbGetServerInfoResp { - optional bytes node = 1; - optional bytes server_version = 2; -} - - -// Get Request - retrieve bucket/key -message RpbGetReq { - required bytes bucket = 1; - required bytes key = 2; - optional uint32 r = 3; - optional uint32 pr = 4; - optional bool basic_quorum = 5; - optional bool notfound_ok = 6; - optional bytes if_modified = 7; // fail if the supplied vclock does not match - optional bool head = 8; // return everything but the value - optional bool deletedvclock = 9; // return the tombstone's vclock, if applicable -} - -// Get Response - if the record was not found there will be no content/vclock -message RpbGetResp { - repeated RpbContent content = 1; - optional bytes vclock = 2; // the opaque vector clock for the object - optional bool unchanged = 3; -} - - -// Put request - if options.return_body is set then the updated metadata/data for -// the key will be returned. -message RpbPutReq { - required bytes bucket = 1; - optional bytes key = 2; - optional bytes vclock = 3; - required RpbContent content = 4; - optional uint32 w = 5; - optional uint32 dw = 6; - optional bool return_body = 7; - optional uint32 pw = 8; - optional bool if_not_modified = 9; - optional bool if_none_match = 10; - optional bool return_head = 11; -} - -// Put response - same as get response with optional key if one was generated -message RpbPutResp { - repeated RpbContent content = 1; - optional bytes vclock = 2; // the opaque vector clock for the object - optional bytes key = 3; // the key generated, if any -} - - -// Delete request -message RpbDelReq { - required bytes bucket = 1; - required bytes key = 2; - optional uint32 rw = 3; - optional bytes vclock = 4; - optional uint32 r = 5; - optional uint32 w = 6; - optional uint32 pr = 7; - optional uint32 pw = 8; - optional uint32 dw = 9; -} - -// Delete response - not defined, will return a RpbDelResp on success or RpbErrorResp on failure - -// List buckets request - no message defined, just send RpbListBucketsReq - -// List buckets response -message RpbListBucketsResp { - repeated bytes buckets = 1; -} - - -// List keys in bucket request -message RpbListKeysReq { - required bytes bucket = 1; -} - -// List keys in bucket response - one or more of these packets will be sent -// the last one will have done set true (and may not have any keys in it) -message RpbListKeysResp { - repeated bytes keys = 1; - optional bool done = 2; -} - -// Get bucket properties request -message RpbGetBucketReq { - required bytes bucket = 1; -} - -// Get bucket properties response -message RpbGetBucketResp { - required RpbBucketProps props = 1; -} - -// Set bucket properties request -message RpbSetBucketReq { - required bytes bucket = 1; - required RpbBucketProps props = 2; -} - - -// Set bucket properties response - no message defined, just send RpbSetBucketResp - - -// Map/Reduce request -message RpbMapRedReq { - required bytes request = 1; - required bytes content_type = 2; -} - -// Map/Reduce response -// one or more of these packets will be sent the last one will have done set -// true (and may not have phase/data in it) -message RpbMapRedResp { - optional uint32 phase = 1; - optional bytes response = 2; - optional bool done = 3; -} - -// Content message included in get/put responses -// Holds the value and associated metadata -message RpbContent { - required bytes value = 1; - optional bytes content_type = 2; // the media type/format - optional bytes charset = 3; - optional bytes content_encoding = 4; - optional bytes vtag = 5; - repeated RpbLink links = 6; // links to other resources - optional uint32 last_mod = 7; - optional uint32 last_mod_usecs = 8; - repeated RpbPair usermeta = 9; // user metadata stored with the object - repeated RpbPair indexes = 10; // user metadata stored with the object -} - -// Key/value pair - used for user metadata -message RpbPair { - required bytes key = 1; - optional bytes value = 2; -} - -// Link metadata -message RpbLink { - optional bytes bucket = 1; - optional bytes key = 2; - optional bytes tag = 3; -} - -// Bucket properties -message RpbBucketProps { - optional uint32 n_val = 1; - optional bool allow_mult = 2; -} diff --git a/riak/transports/riakclient_pb2.py b/riak/transports/riakclient_pb2.py deleted file mode 100644 index e607fe72..00000000 --- a/riak/transports/riakclient_pb2.py +++ /dev/null @@ -1,1112 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! - -from google.protobuf import descriptor -from google.protobuf import message -from google.protobuf import reflection -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - - - -DESCRIPTOR = descriptor.FileDescriptor( - name='riakclient.proto', - package='', - serialized_pb='\n\x10riakclient.proto\"/\n\x0cRpbErrorResp\x12\x0e\n\x06\x65rrmsg\x18\x01 \x02(\x0c\x12\x0f\n\x07\x65rrcode\x18\x02 \x02(\r\"\'\n\x12RpbGetClientIdResp\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"&\n\x11RpbSetClientIdReq\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"<\n\x14RpbGetServerInfoResp\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12\x16\n\x0eserver_version\x18\x02 \x01(\x0c\"\xa4\x01\n\tRpbGetReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\t\n\x01r\x18\x03 \x01(\r\x12\n\n\x02pr\x18\x04 \x01(\r\x12\x14\n\x0c\x62\x61sic_quorum\x18\x05 \x01(\x08\x12\x13\n\x0bnotfound_ok\x18\x06 \x01(\x08\x12\x13\n\x0bif_modified\x18\x07 \x01(\x0c\x12\x0c\n\x04head\x18\x08 \x01(\x08\x12\x15\n\rdeletedvclock\x18\t \x01(\x08\"M\n\nRpbGetResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\x12\x11\n\tunchanged\x18\x03 \x01(\x08\"\xd3\x01\n\tRpbPutReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0e\n\x06vclock\x18\x03 \x01(\x0c\x12\x1c\n\x07\x63ontent\x18\x04 \x02(\x0b\x32\x0b.RpbContent\x12\t\n\x01w\x18\x05 \x01(\r\x12\n\n\x02\x64w\x18\x06 \x01(\r\x12\x13\n\x0breturn_body\x18\x07 \x01(\x08\x12\n\n\x02pw\x18\x08 \x01(\r\x12\x17\n\x0fif_not_modified\x18\t \x01(\x08\x12\x15\n\rif_none_match\x18\n \x01(\x08\x12\x13\n\x0breturn_head\x18\x0b \x01(\x08\"G\n\nRpbPutResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\x12\x0b\n\x03key\x18\x03 \x01(\x0c\"~\n\tRpbDelReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\n\n\x02rw\x18\x03 \x01(\r\x12\x0e\n\x06vclock\x18\x04 \x01(\x0c\x12\t\n\x01r\x18\x05 \x01(\r\x12\t\n\x01w\x18\x06 \x01(\r\x12\n\n\x02pr\x18\x07 \x01(\r\x12\n\n\x02pw\x18\x08 \x01(\r\x12\n\n\x02\x64w\x18\t \x01(\r\"%\n\x12RpbListBucketsResp\x12\x0f\n\x07\x62uckets\x18\x01 \x03(\x0c\" \n\x0eRpbListKeysReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"-\n\x0fRpbListKeysResp\x12\x0c\n\x04keys\x18\x01 \x03(\x0c\x12\x0c\n\x04\x64one\x18\x02 \x01(\x08\"!\n\x0fRpbGetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"2\n\x10RpbGetBucketResp\x12\x1e\n\x05props\x18\x01 \x02(\x0b\x32\x0f.RpbBucketProps\"A\n\x0fRpbSetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x1e\n\x05props\x18\x02 \x02(\x0b\x32\x0f.RpbBucketProps\"5\n\x0cRpbMapRedReq\x12\x0f\n\x07request\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x02(\x0c\">\n\rRpbMapRedResp\x12\r\n\x05phase\x18\x01 \x01(\r\x12\x10\n\x08response\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\"\xe4\x01\n\nRpbContent\x12\r\n\x05value\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x01(\x0c\x12\x0f\n\x07\x63harset\x18\x03 \x01(\x0c\x12\x18\n\x10\x63ontent_encoding\x18\x04 \x01(\x0c\x12\x0c\n\x04vtag\x18\x05 \x01(\x0c\x12\x17\n\x05links\x18\x06 \x03(\x0b\x32\x08.RpbLink\x12\x10\n\x08last_mod\x18\x07 \x01(\r\x12\x16\n\x0elast_mod_usecs\x18\x08 \x01(\r\x12\x1a\n\x08usermeta\x18\t \x03(\x0b\x32\x08.RpbPair\x12\x19\n\x07indexes\x18\n \x03(\x0b\x32\x08.RpbPair\"%\n\x07RpbPair\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c\"3\n\x07RpbLink\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0b\n\x03tag\x18\x03 \x01(\x0c\"3\n\x0eRpbBucketProps\x12\r\n\x05n_val\x18\x01 \x01(\r\x12\x12\n\nallow_mult\x18\x02 \x01(\x08') - - - - -_RPBERRORRESP = descriptor.Descriptor( - name='RpbErrorResp', - full_name='RpbErrorResp', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='errmsg', full_name='RpbErrorResp.errmsg', index=0, - number=1, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='errcode', full_name='RpbErrorResp.errcode', index=1, - number=2, type=13, cpp_type=3, label=2, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=20, - serialized_end=67, -) - - -_RPBGETCLIENTIDRESP = descriptor.Descriptor( - name='RpbGetClientIdResp', - full_name='RpbGetClientIdResp', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='client_id', full_name='RpbGetClientIdResp.client_id', index=0, - number=1, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=69, - serialized_end=108, -) - - -_RPBSETCLIENTIDREQ = descriptor.Descriptor( - name='RpbSetClientIdReq', - full_name='RpbSetClientIdReq', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='client_id', full_name='RpbSetClientIdReq.client_id', index=0, - number=1, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=110, - serialized_end=148, -) - - -_RPBGETSERVERINFORESP = descriptor.Descriptor( - name='RpbGetServerInfoResp', - full_name='RpbGetServerInfoResp', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='node', full_name='RpbGetServerInfoResp.node', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='server_version', full_name='RpbGetServerInfoResp.server_version', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=150, - serialized_end=210, -) - - -_RPBGETREQ = descriptor.Descriptor( - name='RpbGetReq', - full_name='RpbGetReq', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='bucket', full_name='RpbGetReq.bucket', index=0, - number=1, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='key', full_name='RpbGetReq.key', index=1, - number=2, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='r', full_name='RpbGetReq.r', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='pr', full_name='RpbGetReq.pr', index=3, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='basic_quorum', full_name='RpbGetReq.basic_quorum', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='notfound_ok', full_name='RpbGetReq.notfound_ok', index=5, - number=6, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='if_modified', full_name='RpbGetReq.if_modified', index=6, - number=7, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='head', full_name='RpbGetReq.head', index=7, - number=8, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='deletedvclock', full_name='RpbGetReq.deletedvclock', index=8, - number=9, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=213, - serialized_end=377, -) - - -_RPBGETRESP = descriptor.Descriptor( - name='RpbGetResp', - full_name='RpbGetResp', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='content', full_name='RpbGetResp.content', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='vclock', full_name='RpbGetResp.vclock', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='unchanged', full_name='RpbGetResp.unchanged', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=379, - serialized_end=456, -) - - -_RPBPUTREQ = descriptor.Descriptor( - name='RpbPutReq', - full_name='RpbPutReq', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='bucket', full_name='RpbPutReq.bucket', index=0, - number=1, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='key', full_name='RpbPutReq.key', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='vclock', full_name='RpbPutReq.vclock', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='content', full_name='RpbPutReq.content', index=3, - number=4, type=11, cpp_type=10, label=2, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='w', full_name='RpbPutReq.w', index=4, - number=5, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='dw', full_name='RpbPutReq.dw', index=5, - number=6, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='return_body', full_name='RpbPutReq.return_body', index=6, - number=7, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='pw', full_name='RpbPutReq.pw', index=7, - number=8, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='if_not_modified', full_name='RpbPutReq.if_not_modified', index=8, - number=9, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='if_none_match', full_name='RpbPutReq.if_none_match', index=9, - number=10, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='return_head', full_name='RpbPutReq.return_head', index=10, - number=11, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=459, - serialized_end=670, -) - - -_RPBPUTRESP = descriptor.Descriptor( - name='RpbPutResp', - full_name='RpbPutResp', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='content', full_name='RpbPutResp.content', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='vclock', full_name='RpbPutResp.vclock', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='key', full_name='RpbPutResp.key', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=672, - serialized_end=743, -) - - -_RPBDELREQ = descriptor.Descriptor( - name='RpbDelReq', - full_name='RpbDelReq', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='bucket', full_name='RpbDelReq.bucket', index=0, - number=1, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='key', full_name='RpbDelReq.key', index=1, - number=2, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='rw', full_name='RpbDelReq.rw', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='vclock', full_name='RpbDelReq.vclock', index=3, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='r', full_name='RpbDelReq.r', index=4, - number=5, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='w', full_name='RpbDelReq.w', index=5, - number=6, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='pr', full_name='RpbDelReq.pr', index=6, - number=7, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='pw', full_name='RpbDelReq.pw', index=7, - number=8, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='dw', full_name='RpbDelReq.dw', index=8, - number=9, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=745, - serialized_end=871, -) - - -_RPBLISTBUCKETSRESP = descriptor.Descriptor( - name='RpbListBucketsResp', - full_name='RpbListBucketsResp', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='buckets', full_name='RpbListBucketsResp.buckets', index=0, - number=1, type=12, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=873, - serialized_end=910, -) - - -_RPBLISTKEYSREQ = descriptor.Descriptor( - name='RpbListKeysReq', - full_name='RpbListKeysReq', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='bucket', full_name='RpbListKeysReq.bucket', index=0, - number=1, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=912, - serialized_end=944, -) - - -_RPBLISTKEYSRESP = descriptor.Descriptor( - name='RpbListKeysResp', - full_name='RpbListKeysResp', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='keys', full_name='RpbListKeysResp.keys', index=0, - number=1, type=12, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='done', full_name='RpbListKeysResp.done', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=946, - serialized_end=991, -) - - -_RPBGETBUCKETREQ = descriptor.Descriptor( - name='RpbGetBucketReq', - full_name='RpbGetBucketReq', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='bucket', full_name='RpbGetBucketReq.bucket', index=0, - number=1, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=993, - serialized_end=1026, -) - - -_RPBGETBUCKETRESP = descriptor.Descriptor( - name='RpbGetBucketResp', - full_name='RpbGetBucketResp', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='props', full_name='RpbGetBucketResp.props', index=0, - number=1, type=11, cpp_type=10, label=2, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=1028, - serialized_end=1078, -) - - -_RPBSETBUCKETREQ = descriptor.Descriptor( - name='RpbSetBucketReq', - full_name='RpbSetBucketReq', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='bucket', full_name='RpbSetBucketReq.bucket', index=0, - number=1, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='props', full_name='RpbSetBucketReq.props', index=1, - number=2, type=11, cpp_type=10, label=2, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=1080, - serialized_end=1145, -) - - -_RPBMAPREDREQ = descriptor.Descriptor( - name='RpbMapRedReq', - full_name='RpbMapRedReq', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='request', full_name='RpbMapRedReq.request', index=0, - number=1, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='content_type', full_name='RpbMapRedReq.content_type', index=1, - number=2, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=1147, - serialized_end=1200, -) - - -_RPBMAPREDRESP = descriptor.Descriptor( - name='RpbMapRedResp', - full_name='RpbMapRedResp', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='phase', full_name='RpbMapRedResp.phase', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='response', full_name='RpbMapRedResp.response', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='done', full_name='RpbMapRedResp.done', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=1202, - serialized_end=1264, -) - - -_RPBCONTENT = descriptor.Descriptor( - name='RpbContent', - full_name='RpbContent', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='value', full_name='RpbContent.value', index=0, - number=1, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='content_type', full_name='RpbContent.content_type', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='charset', full_name='RpbContent.charset', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='content_encoding', full_name='RpbContent.content_encoding', index=3, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='vtag', full_name='RpbContent.vtag', index=4, - number=5, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='links', full_name='RpbContent.links', index=5, - number=6, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='last_mod', full_name='RpbContent.last_mod', index=6, - number=7, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='last_mod_usecs', full_name='RpbContent.last_mod_usecs', index=7, - number=8, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='usermeta', full_name='RpbContent.usermeta', index=8, - number=9, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='indexes', full_name='RpbContent.indexes', index=9, - number=10, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=1267, - serialized_end=1495, -) - - -_RPBPAIR = descriptor.Descriptor( - name='RpbPair', - full_name='RpbPair', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='key', full_name='RpbPair.key', index=0, - number=1, type=12, cpp_type=9, label=2, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='value', full_name='RpbPair.value', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=1497, - serialized_end=1534, -) - - -_RPBLINK = descriptor.Descriptor( - name='RpbLink', - full_name='RpbLink', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='bucket', full_name='RpbLink.bucket', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='key', full_name='RpbLink.key', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='tag', full_name='RpbLink.tag', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=1536, - serialized_end=1587, -) - - -_RPBBUCKETPROPS = descriptor.Descriptor( - name='RpbBucketProps', - full_name='RpbBucketProps', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='n_val', full_name='RpbBucketProps.n_val', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='allow_mult', full_name='RpbBucketProps.allow_mult', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=1589, - serialized_end=1640, -) - -_RPBGETRESP.fields_by_name['content'].message_type = _RPBCONTENT -_RPBPUTREQ.fields_by_name['content'].message_type = _RPBCONTENT -_RPBPUTRESP.fields_by_name['content'].message_type = _RPBCONTENT -_RPBGETBUCKETRESP.fields_by_name['props'].message_type = _RPBBUCKETPROPS -_RPBSETBUCKETREQ.fields_by_name['props'].message_type = _RPBBUCKETPROPS -_RPBCONTENT.fields_by_name['links'].message_type = _RPBLINK -_RPBCONTENT.fields_by_name['usermeta'].message_type = _RPBPAIR -_RPBCONTENT.fields_by_name['indexes'].message_type = _RPBPAIR -DESCRIPTOR.message_types_by_name['RpbErrorResp'] = _RPBERRORRESP -DESCRIPTOR.message_types_by_name['RpbGetClientIdResp'] = _RPBGETCLIENTIDRESP -DESCRIPTOR.message_types_by_name['RpbSetClientIdReq'] = _RPBSETCLIENTIDREQ -DESCRIPTOR.message_types_by_name['RpbGetServerInfoResp'] = _RPBGETSERVERINFORESP -DESCRIPTOR.message_types_by_name['RpbGetReq'] = _RPBGETREQ -DESCRIPTOR.message_types_by_name['RpbGetResp'] = _RPBGETRESP -DESCRIPTOR.message_types_by_name['RpbPutReq'] = _RPBPUTREQ -DESCRIPTOR.message_types_by_name['RpbPutResp'] = _RPBPUTRESP -DESCRIPTOR.message_types_by_name['RpbDelReq'] = _RPBDELREQ -DESCRIPTOR.message_types_by_name['RpbListBucketsResp'] = _RPBLISTBUCKETSRESP -DESCRIPTOR.message_types_by_name['RpbListKeysReq'] = _RPBLISTKEYSREQ -DESCRIPTOR.message_types_by_name['RpbListKeysResp'] = _RPBLISTKEYSRESP -DESCRIPTOR.message_types_by_name['RpbGetBucketReq'] = _RPBGETBUCKETREQ -DESCRIPTOR.message_types_by_name['RpbGetBucketResp'] = _RPBGETBUCKETRESP -DESCRIPTOR.message_types_by_name['RpbSetBucketReq'] = _RPBSETBUCKETREQ -DESCRIPTOR.message_types_by_name['RpbMapRedReq'] = _RPBMAPREDREQ -DESCRIPTOR.message_types_by_name['RpbMapRedResp'] = _RPBMAPREDRESP -DESCRIPTOR.message_types_by_name['RpbContent'] = _RPBCONTENT -DESCRIPTOR.message_types_by_name['RpbPair'] = _RPBPAIR -DESCRIPTOR.message_types_by_name['RpbLink'] = _RPBLINK -DESCRIPTOR.message_types_by_name['RpbBucketProps'] = _RPBBUCKETPROPS - -class RpbErrorResp(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBERRORRESP - - # @@protoc_insertion_point(class_scope:RpbErrorResp) - -class RpbGetClientIdResp(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBGETCLIENTIDRESP - - # @@protoc_insertion_point(class_scope:RpbGetClientIdResp) - -class RpbSetClientIdReq(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBSETCLIENTIDREQ - - # @@protoc_insertion_point(class_scope:RpbSetClientIdReq) - -class RpbGetServerInfoResp(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBGETSERVERINFORESP - - # @@protoc_insertion_point(class_scope:RpbGetServerInfoResp) - -class RpbGetReq(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBGETREQ - - # @@protoc_insertion_point(class_scope:RpbGetReq) - -class RpbGetResp(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBGETRESP - - # @@protoc_insertion_point(class_scope:RpbGetResp) - -class RpbPutReq(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBPUTREQ - - # @@protoc_insertion_point(class_scope:RpbPutReq) - -class RpbPutResp(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBPUTRESP - - # @@protoc_insertion_point(class_scope:RpbPutResp) - -class RpbDelReq(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBDELREQ - - # @@protoc_insertion_point(class_scope:RpbDelReq) - -class RpbListBucketsResp(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBLISTBUCKETSRESP - - # @@protoc_insertion_point(class_scope:RpbListBucketsResp) - -class RpbListKeysReq(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBLISTKEYSREQ - - # @@protoc_insertion_point(class_scope:RpbListKeysReq) - -class RpbListKeysResp(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBLISTKEYSRESP - - # @@protoc_insertion_point(class_scope:RpbListKeysResp) - -class RpbGetBucketReq(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBGETBUCKETREQ - - # @@protoc_insertion_point(class_scope:RpbGetBucketReq) - -class RpbGetBucketResp(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBGETBUCKETRESP - - # @@protoc_insertion_point(class_scope:RpbGetBucketResp) - -class RpbSetBucketReq(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBSETBUCKETREQ - - # @@protoc_insertion_point(class_scope:RpbSetBucketReq) - -class RpbMapRedReq(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBMAPREDREQ - - # @@protoc_insertion_point(class_scope:RpbMapRedReq) - -class RpbMapRedResp(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBMAPREDRESP - - # @@protoc_insertion_point(class_scope:RpbMapRedResp) - -class RpbContent(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBCONTENT - - # @@protoc_insertion_point(class_scope:RpbContent) - -class RpbPair(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBPAIR - - # @@protoc_insertion_point(class_scope:RpbPair) - -class RpbLink(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBLINK - - # @@protoc_insertion_point(class_scope:RpbLink) - -class RpbBucketProps(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _RPBBUCKETPROPS - - # @@protoc_insertion_point(class_scope:RpbBucketProps) - -# @@protoc_insertion_point(module_scope) diff --git a/setup.py b/setup.py index 0c8f5b07..d6a1d020 100755 --- a/setup.py +++ b/setup.py @@ -12,36 +12,31 @@ def make_docs(): for name in glob.glob('*.html'): os.rename(name, 'docs/%s' % name) -def make_pb(): - subprocess.call(['protoc', '--python_out=.', './riak/transports/riakclient.proto']) +install_requires = ["riak_pb >=1.2.0, < 1.3.0"] +requires = ["riak_pb(>=1.2.0,<1.3.0)"] +tests_require = [] +if platform.python_version() < '2.7': + tests_require.append("unittest2") -if __name__ == "__main__": - install_requires = {'protobuf': ['>= 2.4.0', '< 2.5.0'] } - tests_require = [] - if platform.python_version() < '2.7': - tests_require.append("unittest2") - - setup( - name='riak', - version='1.4.1', - packages = find_packages(), - install_requires = install_requires, - tests_require = tests_require, - package_data = { - '' : ['*.proto'], - 'riak' : ['erl_src/*'] - }, - description='Python client for Riak', - zip_safe=True, - include_package_data=True, - license='Apache 2', - platforms='Platform Independent', - author='Basho Technologies', - author_email='clients@basho.com', - test_suite='riak.tests.suite', - url='https://github.com/basho/riak-python-client', - classifiers = ['License :: OSI Approved :: Apache Software License', - 'Intended Audience :: Developers', - 'Operating System :: OS Independent', - 'Topic :: Database'] - ) +setup( + name='riak', + version='1.5.0', + packages = find_packages(), + requires = requires, + install_requires = install_requires, + tests_require = tests_require, + package_data = {'riak' : ['erl_src/*']}, + description='Python client for Riak', + zip_safe=True, + include_package_data=True, + license='Apache 2', + platforms='Platform Independent', + author='Basho Technologies', + author_email='clients@basho.com', + test_suite='riak.tests.suite', + url='https://github.com/basho/riak-python-client', + classifiers = ['License :: OSI Approved :: Apache Software License', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Topic :: Database'] + ) From e9fad39b1f84a44b09a3da5a0ce69bd7847ffa59 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 19 Jul 2012 11:23:00 -0400 Subject: [PATCH 0211/1060] Detect PB properly in the test. --- riak/tests/test_all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 3ae08a4e..54aaee16 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -28,7 +28,7 @@ from riak.test_server import TestServer try: - import riak.transports.riakclient_pb2 + import riak_pb HAVE_PROTO = True except ImportError: HAVE_PROTO = False From 9c15349195336c4fcb8f3a0a0e292607c9d636e9 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 19 Jul 2012 14:38:01 -0400 Subject: [PATCH 0212/1060] Add feature detection based on version reported from Riak. --- riak/tests/test_feature_detection.py | 86 +++++++++++++++++++++ riak/transports/__init__.py | 2 - riak/transports/feature_detect.py | 109 +++++++++++++++++++++++++++ riak/transports/http.py | 34 +++++++++ riak/transports/pbc.py | 15 ++++ riak/transports/transport.py | 3 +- 6 files changed, 246 insertions(+), 3 deletions(-) create mode 100644 riak/tests/test_feature_detection.py create mode 100644 riak/transports/feature_detect.py diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py new file mode 100644 index 00000000..dac59697 --- /dev/null +++ b/riak/tests/test_feature_detection.py @@ -0,0 +1,86 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import platform + +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest + +from riak.transports.feature_detect import FeatureDetection + +class IncompleteTransport(FeatureDetection): + pass + +class DummyTransport(FeatureDetection): + def __init__(self, version): + self._version = version + + def _server_version(self): + return self._version + +class FeatureDetectionTest(unittest.TestCase): + def test_implements_server_version(self): + t = IncompleteTransport() + def get_server_version(): + t.server_version + self.assertRaises(NotImplementedError, get_server_version) + + def test_pre_10(self): + t = DummyTransport("0.14.2") + self.assertFalse(t.phaseless_mapred()) + self.assertFalse(t.pb_indexes()) + self.assertFalse(t.pb_search()) + self.assertFalse(t.pb_conditionals()) + self.assertFalse(t.quorum_controls()) + self.assertFalse(t.tombstone_vclocks()) + self.assertFalse(t.pb_head()) + + def test_10(self): + t = DummyTransport("1.0.3") + self.assertFalse(t.phaseless_mapred()) + self.assertFalse(t.pb_indexes()) + self.assertFalse(t.pb_search()) + self.assertTrue(t.pb_conditionals()) + self.assertTrue(t.quorum_controls()) + self.assertTrue(t.tombstone_vclocks()) + self.assertTrue(t.pb_head()) + + def test_11(self): + t = DummyTransport("1.1.4") + self.assertTrue(t.phaseless_mapred()) + self.assertFalse(t.pb_indexes()) + self.assertFalse(t.pb_search()) + self.assertTrue(t.pb_conditionals()) + self.assertTrue(t.quorum_controls()) + self.assertTrue(t.tombstone_vclocks()) + self.assertTrue(t.pb_head()) + + def test_12(self): + t = DummyTransport("1.2.0") + self.assertTrue(t.phaseless_mapred()) + self.assertTrue(t.pb_indexes()) + self.assertTrue(t.pb_search()) + self.assertTrue(t.pb_conditionals()) + self.assertTrue(t.quorum_controls()) + self.assertTrue(t.tombstone_vclocks()) + self.assertTrue(t.pb_head()) + +if __name__ == '__main__': + unittest.main() diff --git a/riak/transports/__init__.py b/riak/transports/__init__.py index 55b7da3f..9c59adb6 100644 --- a/riak/transports/__init__.py +++ b/riak/transports/__init__.py @@ -1,4 +1,2 @@ from http import RiakHttpTransport from pbc import RiakPbcTransport - - diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py new file mode 100644 index 00000000..d56ff334 --- /dev/null +++ b/riak/transports/feature_detect.py @@ -0,0 +1,109 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from distutils.version import StrictVersion + +versions = { + 1: StrictVersion("1.0.0"), + 1.1: StrictVersion("1.1.0"), + 1.2: StrictVersion("1.2.0") + } + +class lazy_property(object): + ''' + meant to be used for lazy evaluation of an object attribute. + property should represent non-mutable data, as it replaces itself. + ''' + + def __init__(self,fget): + self.fget = fget + self.func_name = fget.__name__ + + def __get__(self,obj,cls): + if obj is None: + return None + value = self.fget(obj) + setattr(obj,self.func_name,value) + return value + +class FeatureDetection(object): + def _server_version(self): + """ + Gets the server version from the server. To be implemented by + the individual transport class. + :rtype string + """ + raise NotImplementedError + + def phaseless_mapred(self): + """ + Whether MapReduce requests can be submitted without phases. + :rtype bool + """ + return self.server_version >= versions[1.1] + + def pb_indexes(self): + """ + Whether secondary index queries are supported over Protocol + Buffers + + :rtype bool + """ + return self.server_version >= versions[1.2] + + def pb_search(self): + """ + Whether search queries are supported over Protocol Buffers + :rtype bool + """ + return self.server_version >= versions[1.2] + + def pb_conditionals(self): + """ + Whether conditional fetch/store semantics are supported over + Protocol Buffers + :rtype bool + """ + return self.server_version >= versions[1] + + def quorum_controls(self): + """ + Whether additional quorums and FSM controls are available, + e.g. primary quorums, basic_quorum, notfound_ok + :rtype bool + """ + return self.server_version >= versions[1] + + def tombstone_vclocks(self): + """ + Whether 'not found' responses might include vclocks + :rtype bool + """ + return self.server_version >= versions[1] + + def pb_head(self): + """ + Whether partial-fetches (vclock and metadata only) are + supported over Protocol Buffers + :rtype bool + """ + return self.server_version >= versions[1] + + @lazy_property + def server_version(self): + return StrictVersion(self._server_version()) diff --git a/riak/transports/http.py b/riak/transports/http.py index 9e740de9..9a83fc0b 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -95,6 +95,40 @@ def ping(self) : response = self.http_request('GET', '/ping') return(response is not None) and (response[1] == 'OK') + def stats(self): + """ + Gets performance statistics and server information + """ + # TODO: use resource detection + response = self.http_request('GET', '/stats', {'Accept':'application/json'}) + if response[0]['http_status'] is 200: + return json.loads(response[1]) + else: + return None + + # FeatureDetection API - private + def _server_version(self): + stats = self.stats() + if stats is not None: + return stats['riak_kv_version'] + # If stats is disabled, we can't assume the Riak version + # is >= 1.1. However, we can assume the new URL scheme is + # at least version 1.0 + elif 'riak_kv_wm_buckets' in self.get_resources(): + return "1.0.0" + else: + return "0.14.0" + + def get_resources(self): + """ + Gets a JSON mapping of server-side resource names to paths + :rtype dict + """ + response = self.http_request('GET', '/', {'Accept':'application/json'}) + if response[0]['http_status'] is 200: + return json.loads(response[1]) + else: + return dict() def get(self, robj, r = None, pr = None, vtag = None) : """ diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 06105b4b..898d7568 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -173,6 +173,10 @@ def __init__(self, cm, client_id=None, max_attempts=1, **unused_options): self._client_id = client_id self._max_attempts = max_attempts + # FeatureDetection API + def _server_version(self): + return self.get_server_info()['server_version'] + def translate_rw_val(self, rw): val = self.rw_names.get(rw) if val is None: @@ -194,6 +198,14 @@ def ping(self): else: return 0 + def get_server_info(self): + """ + Get information about the server + """ + msg_code, resp = self.send_msg_code(MSG_CODE_GET_SERVER_INFO_REQ, + MSG_CODE_GET_SERVER_INFO_RESP) + return {'node':resp.node, 'server_version':resp.server_version} + def get_client_id(self): """ Get the client id used by this connection @@ -498,6 +510,9 @@ def recv_msg(self, conn, expect): raise Exception(msg.errmsg) elif msg_code == MSG_CODE_PING_RESP: msg = None + elif msg_code == MSG_CODE_GET_SERVER_INFO_RESP: + msg = riak_pb.RpbGetServerInfoResp() + msg.ParseFromString(self._inbuf[1:]) elif msg_code == MSG_CODE_GET_CLIENT_ID_RESP: msg = riak_pb.RpbGetClientIdResp() msg.ParseFromString(self._inbuf[1:]) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 4454f48d..03489a5e 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -23,8 +23,9 @@ import threading import platform import os +from feature_detect import FeatureDetection -class RiakTransport(object): +class RiakTransport(FeatureDetection): """ Class to encapsulate transport details """ From 55ee9c80d8ce516da0710052a9a2e06314159b2a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 24 Jul 2012 10:31:40 -0400 Subject: [PATCH 0213/1060] Move lazy_property to riak.util. --- riak/transports/feature_detect.py | 18 +----------------- riak/util.py | 25 +++++++++++++++++++++---- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index d56ff334..952cc8a8 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -17,6 +17,7 @@ """ from distutils.version import StrictVersion +from riak.util import lazy_property versions = { 1: StrictVersion("1.0.0"), @@ -24,23 +25,6 @@ 1.2: StrictVersion("1.2.0") } -class lazy_property(object): - ''' - meant to be used for lazy evaluation of an object attribute. - property should represent non-mutable data, as it replaces itself. - ''' - - def __init__(self,fget): - self.fget = fget - self.func_name = fget.__name__ - - def __get__(self,obj,cls): - if obj is None: - return None - value = self.fget(obj) - setattr(obj,self.func_name,value) - return value - class FeatureDetection(object): def _server_version(self): """ diff --git a/riak/util.py b/riak/util.py index f0f8958d..0ca12bc7 100644 --- a/riak/util.py +++ b/riak/util.py @@ -9,12 +9,12 @@ def quacks_like_dict(object): """Check if object is dict-like""" return isinstance(object, Mapping) - + def deep_merge(a, b): """Merge two deep dicts non-destructively - + Uses a stack to avoid maximum recursion depth exceptions - + >>> a = {'a': 1, 'b': {1: 1, 2: 2}, 'd': 6} >>> b = {'c': 3, 'b': {2: 7}, 'd': {'z': [1, 2, 3]}} >>> c = merge(a, b) @@ -23,7 +23,7 @@ def deep_merge(a, b): """ assert quacks_like_dict(a), quacks_like_dict(b) dst = a.copy() - + stack = [(dst, b)] while stack: current_dst, current_src = stack.pop() @@ -40,3 +40,20 @@ def deep_merge(a, b): def deprecated(message, stacklevel=3): warnings.warn(message, DeprecationWarning, stacklevel=stacklevel) + +class lazy_property(object): + ''' + meant to be used for lazy evaluation of an object attribute. + property should represent non-mutable data, as it replaces itself. + ''' + + def __init__(self,fget): + self.fget = fget + self.func_name = fget.__name__ + + def __get__(self,obj,cls): + if obj is None: + return None + value = self.fget(obj) + setattr(obj,self.func_name,value) + return value From 9f23c9fbaa67d6e15837c6c1e584f7c62e576c2b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 24 Jul 2012 12:03:40 -0400 Subject: [PATCH 0214/1060] Add feature detection switches to HTTP. --- riak/transports/http.py | 48 ++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 9a83fc0b..37ae5d96 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -101,7 +101,7 @@ def stats(self): """ # TODO: use resource detection response = self.http_request('GET', '/stats', {'Accept':'application/json'}) - if response[0]['http_status'] is 200: + if response[0]['http_code'] is 200: return json.loads(response[1]) else: return None @@ -128,12 +128,14 @@ def get_resources(self): if response[0]['http_status'] is 200: return json.loads(response[1]) else: - return dict() + return {} def get(self, robj, r = None, pr = None, vtag = None) : """ Get a bucket/key from the server """ + # We could detect quorum_controls here but HTTP ignores + # unknown flags/params. params = {'r' : r, 'pr': pr} if vtag is not None: params['vtag'] = vtag @@ -146,11 +148,14 @@ def put(self, robj, w = None, dw = None, pw = None, return_body = True, if_none_ """ Serialize put request and deserialize response """ - # Construct the URL... + # We could detect quorum_controls here but HTTP ignores + # unknown flags/params. params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw, 'pw' : pw } url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), params=params) headers = self.build_put_headers(robj) + # TODO: use a more general 'prevent_stale_writes' semantics, + # which is a superset of the if_none_match semantics. if if_none_match: headers["If-None-Match"] = "*" content = robj.get_encoded_data() @@ -170,10 +175,13 @@ def do_put(self, url, headers, content, return_body=False, key=None): def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """Put a new object into the Riak store, returning its (new) key.""" - # Construct the URL... + # We could detect quorum_controls here but HTTP ignores + # unknown flags/params. params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw, 'pw' : pw} url = self.build_rest_path(bucket=robj.get_bucket(), params=params) headers = self.build_put_headers(robj) + # TODO: use a more general 'prevent_stale_writes' semantics, + # which is a superset of the if_none_match semantics. if if_none_match: headers["If-None-Match"] = "*" content = robj.get_encoded_data() @@ -189,18 +197,25 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_matc return key, None, None def delete(self, robj, rw=None, r = None, w = None, dw = None, pr = None, pw = None): - # Construct the URL... + """ + Delete an object. + """ + # We could detect quorum_controls here but HTTP ignores + # unknown flags/params. params = {'rw' : rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} + headers = {} url = self.build_rest_path(robj.get_bucket(), robj.get_key(), params=params) - # TODO: Send vclock of robj if it exists - # Run the operation.. - response = self.http_request('DELETE', url) + if self.tombstone_vclocks() and robj.vclock() is not None: + headers['X-Riak-Vclock'] = robj.vclock() + response = self.http_request('DELETE', url, headers) self.check_http_code(response, [204, 404]) return self - def get_keys(self, bucket): + """ + Fetch a list of keys for the bucket + """ params = {'props' : 'True', 'keys' : 'true'} url = self.build_rest_path(bucket, params=params) response = self.http_request('GET', url) @@ -213,6 +228,9 @@ def get_keys(self, bucket): raise Exception('Error getting bucket properties.') def get_buckets(self): + """ + Fetch a list of all buckets + """ params = {'buckets': 'true'} url = self.build_rest_path(None, params=params) response = self.http_request('GET', url) @@ -225,6 +243,9 @@ def get_buckets(self): raise Exception('Error getting buckets.') def get_bucket_props(self, bucket): + """ + Get properties for a bucket + """ # Run the request... params = {'props' : 'True', 'keys' : 'False'} url = self.build_rest_path(bucket, params=params) @@ -261,6 +282,12 @@ def set_bucket_props(self, bucket, props): return True def mapred(self, inputs, query, timeout=None): + """ + Run a MapReduce query. + """ + if not self.phaseless_mapred() and (query is None or len(query) is 0): + raise Exception('Phase-less MapReduce is supported by this Riak node') + # Construct the job, optionally set the timeout... job = {'inputs':inputs, 'query':query} if timeout is not None: @@ -276,7 +303,8 @@ def mapred(self, inputs, query, timeout=None): # Make sure the expected status code came back... status = response[0]['http_code'] if status != 200: - raise Exception('Error running MapReduce operation. Status: ' + str(status) + ' : ' + response[1]) + raise Exception('Error running MapReduce operation. Headers: %s Body: %s' % + (repr(response[0]),repr(response[1]))) result = json.loads(response[1]) return result From 928dba539c4cceccfe35e5ac339c4f029f7b81aa Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 27 Jul 2012 15:20:48 -0500 Subject: [PATCH 0215/1060] Completing basic Riak 1.2 features. * Implement search and 2I on PBC. * Normalize return format of searches so it is consistent across PBC and HTTP. * Add get_index method on RiakBucket and RiakClient, passes through to transport. * Make RiakSearch.search() pass through to the client transport. (In the future, this class will be removed.) * Modify tests to cover new PBC features. * Add "deleted" metadata. * Use feature detection to prevent sending unrecognized messages on PBC. --- riak/bucket.py | 9 +++ riak/client.py | 3 + riak/metadata.py | 1 + riak/search.py | 9 +-- riak/tests/test_all.py | 122 ++++++++++++++++------------------- riak/transports/http.py | 122 ++++++++++++++++++++++++++++++++++- riak/transports/pbc.py | 93 ++++++++++++++++++++++---- riak/transports/transport.py | 62 ++++++++++++++++++ 8 files changed, 336 insertions(+), 85 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 348d26ad..586f01ba 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -517,4 +517,13 @@ def disable_search(self): return True def search(self, query, **params): + """ + Queries a search index over objects in this bucket/index. + """ return self._client.solr().search(self._name, query, **params) + + def get_index(self, index, startkey, endkey=None): + """ + Queries a secondary index over objects in this bucket, returning keys. + """ + return self._client._transport.get_index(self._name, index, startkey, endkey) diff --git a/riak/client.py b/riak/client.py index 00be8b89..7bb5e8dc 100644 --- a/riak/client.py +++ b/riak/client.py @@ -375,6 +375,9 @@ def get_file(self, filename): def delete_file(self, filename): self._transport.delete_file(filename) + def get_index(self, bucket, index, startkey, endkey=None): + return self._transport.get_index(bucket, index, startkey, endkey) + def solr(self): if self._solr is None: self._solr = RiakSearch(self, host=self._host, port=self._port) diff --git a/riak/metadata.py b/riak/metadata.py index 37c81b2a..095bd0f3 100644 --- a/riak/metadata.py +++ b/riak/metadata.py @@ -26,3 +26,4 @@ MD_LASTMOD_USECS = "lastmod-usecs" MD_USERMETA = "usermeta" MD_INDEX = "index" +MD_DELETED = "deleted" diff --git a/riak/search.py b/riak/search.py index 3b36dd8e..a2066fbe 100644 --- a/riak/search.py +++ b/riak/search.py @@ -78,11 +78,6 @@ def delete(self, index, docs=None, queries=None): remove = delete def search(self, index, query, **params): - options = {'q': query, 'wt': 'json'} - options.update(params) - uri = "/solr/%s/select" % index - headers, results = self._transport.get_request(uri, options) - decoder = self.get_decoder(headers['content-type']) - return decoder(results) - + return self._client._transport.search(index, query, **params) + select = search diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 54aaee16..7ae5dafc 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -544,6 +544,34 @@ def test_store_of_missing_object(self): self.assertEqual(o.get_content_type(), "application/octet-stream") o.delete() + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_solr_search_from_bucket(self): + bucket = self.client.bucket('searchbucket') + bucket.new("user", {"username": "roidrage"}).store() + results = bucket.search("username:roidrage") + self.assertEquals(1, len(results['docs'])) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_solr_search_with_params_from_bucket(self): + bucket = self.client.bucket('searchbucket') + bucket.new("user", {"username": "roidrage"}).store() + results = bucket.search("username:roidrage", wt="xml") + self.assertEquals(1, len(results['docs'])) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_solr_search_with_params(self): + bucket = self.client.bucket('searchbucket') + bucket.new("user", {"username": "roidrage"}).store() + results = self.client.solr().search("searchbucket", "username:roidrage", wt="xml") + self.assertEquals(1, len(results['docs'])) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_solr_search(self): + bucket = self.client.bucket('searchbucket') + bucket.new("user", {"username": "roidrage"}).store() + results = self.client.solr().search("searchbucket", "username:roidrage") + self.assertEquals(1, len(results["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_search_integration(self): # Create some objects to search across... @@ -555,14 +583,14 @@ def test_search_integration(self): bucket.new("five", {"foo":"five", "bar":"yellow"}).store() # Run some operations... - results = self.client.search("searchbucket", "foo:one OR foo:two").run() + results = self.client.solr().search("searchbucket", "foo:one OR foo:two") if (len(results) == 0): print "\n\nNot running test \"testSearchIntegration()\".\n" print "Please ensure that you have installed the Riak Search hook on bucket \"searchbucket\" by running \"bin/search-cmd install searchbucket\".\n\n" return - self.assertEqual(len(results), 2) - results = self.client.search("searchbucket", "(foo:one OR foo:two OR foo:three OR foo:four) AND (NOT bar:green)").run() - self.assertEqual(len(results), 3) + self.assertEqual(len(results['docs']), 2) + results = self.client.solr().search("searchbucket", "(foo:one OR foo:two OR foo:three OR foo:four) AND (NOT bar:green)") + self.assertEqual(len(results['docs']), 3) def test_store_binary_object_from_file(self): bucket = self.client.bucket('bucket') @@ -702,9 +730,9 @@ def test_set_indexes(self): self.assertEqual(1, len(result)) self.assertEqual('foo', result[0].get_key()) - result = self.client.index('indexbucket', 'field1_bin', 'test').run() + result = bucket.get_index('field1_bin', 'test') self.assertEqual(1, len(result)) - self.assertEqual('foo', result[0].get_key()) + self.assertEqual('foo', str(result[0])) @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_remove_indexes(self): @@ -713,16 +741,16 @@ def test_remove_indexes(self): bucket = self.client.bucket('indexbucket') bar = bucket.new('bar', 1).add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() - result = self.client.index('indexbucket', 'bar_int', 1).run() + result = bucket.get_index('bar_int', 1) self.assertEqual(1, len(result)) self.assertEqual(3, len(bar.get_indexes())) self.assertEqual(2, len(bar.get_indexes('bar_int'))) - + # remove all indexes bar = bar.remove_indexes().store() - result = self.client.index('indexbucket', 'bar_int', 1).run() + result = bucket.get_index('bar_int', 1) self.assertEqual(0, len(result)) - result = self.client.index('indexbucket', 'baz_bin', 'baz').run() + result = bucket.get_index('baz_bin', 'baz') self.assertEqual(0, len(result)) self.assertEqual(0, len(bar.get_indexes())) self.assertEqual(0, len(bar.get_indexes('bar_int'))) @@ -732,11 +760,11 @@ def test_remove_indexes(self): bar = bar.add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() # remove all index with field='bar_int' bar = bar.remove_index(field='bar_int').store() - result = self.client.index('indexbucket', 'bar_int', 1).run() + result = bucket.get_index('bar_int', 1) self.assertEqual(0, len(result)) - result = self.client.index('indexbucket', 'bar_int', 2).run() + result = bucket.get_index('bar_int', 2) self.assertEqual(0, len(result)) - result = self.client.index('indexbucket', 'baz_bin', 'baz').run() + result = bucket.get_index('baz_bin', 'baz') self.assertEqual(1, len(result)) self.assertEqual(1, len(bar.get_indexes())) self.assertEqual(0, len(bar.get_indexes('bar_int'))) @@ -746,11 +774,11 @@ def test_remove_indexes(self): bar = bar.add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() # remove an index field value pair bar = bar.remove_index(field='bar_int', value=2).store() - result = self.client.index('indexbucket', 'bar_int', 1).run() + result = bucket.get_index('bar_int', 1) self.assertEqual(1, len(result)) - result = self.client.index('indexbucket', 'bar_int', 2).run() + result = bucket.get_index('bar_int', 2) self.assertEqual(0, len(result)) - result = self.client.index('indexbucket', 'baz_bin', 'baz').run() + result = bucket.get_index('baz_bin', 'baz') self.assertEqual(1, len(result)) self.assertEqual(2, len(bar.get_indexes())) self.assertEqual(1, len(bar.get_indexes('bar_int'))) @@ -785,28 +813,24 @@ def test_secondary_index_query(self): store() # Test an equality query... - results = self.client.index('indexbucket', 'field1_bin', 'val2').run() + results = bucket.get_index('field1_bin', 'val2') self.assertEquals(1, len(results)) - self.assertEquals('mykey2', results[0].get_key()) + self.assertEquals('mykey2', str(results[0])) # Test a range query... - results = self.client.index('indexbucket', 'field1_bin', 'val2', 'val4').run() - vals = set() - for i in results: - vals.add(i.get_key()) + results = bucket.get_index('field1_bin', 'val2', 'val4') + vals = set([ str(key) for key in results ]) self.assertEquals(3, len(results)) self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) # Test an equality query... - results = self.client.index('indexbucket', 'field2_int', 1002).run() + results = bucket.get_index('field2_int', 1002) self.assertEquals(1, len(results)) - self.assertEquals('mykey2', results[0].get_key()) + self.assertEquals('mykey2', str(results[0])) # Test a range query... - results = self.client.index('indexbucket', 'field2_int', 1002, 1004).run() - vals = set() - for i in results: - vals.add(i.get_key()) + results = bucket.get_index('field2_int', 1002, 1004) + vals = set([str(key) for key in results ]) self.assertEquals(3, len(results)) self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) @@ -1151,70 +1175,38 @@ def test_delete_file_with_luwak(self): file = self.client.get_file(key) self.assertIsNone(file) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_solr_search_from_bucket(self): - bucket = self.client.bucket('searchbucket') - bucket.new("user", {"username": "roidrage"}).store() - results = bucket.search("username:roidrage") - self.assertEquals(1, len(results["response"]["docs"])) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_solr_search_with_params_from_bucket(self): - bucket = self.client.bucket('searchbucket') - bucket.new("user", {"username": "roidrage"}).store() - results = bucket.search("username:roidrage", wt="xml") - result = results.find("result") - if not hasattr(result, "iter"): setattr(result, "iter", result.getiterator) - self.assertEquals(1, len(list(result.iter("doc")))) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_solr_search_with_params(self): - bucket = self.client.bucket('searchbucket') - bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr().search("searchbucket", "username:roidrage", wt="xml") - result = results.find("result") - if not hasattr(result, "iter"): setattr(result, "iter", result.getiterator) - self.assertEquals(1, len(list(result.iter("doc")))) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_solr_search(self): - bucket = self.client.bucket('searchbucket') - bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr().search("searchbucket", "username:roidrage") - self.assertEquals(1, len(results["response"]["docs"])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_document_to_index(self): self.client.solr().add("searchbucket", {"id": "doc", "username": "tony"}) results = self.client.solr().search("searchbucket", "username:tony") - self.assertEquals("tony", results["response"]["docs"][0]["fields"]["username"]) + self.assertEquals("tony", results['docs'][0]['username']) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_multiple_documents_to_index(self): self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") - self.assertEquals(2, len(results["response"]["docs"])) + self.assertEquals(2, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_id(self): self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", docs=["dizzy"]) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") - self.assertEquals(1, len(results["response"]["docs"])) + self.assertEquals(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", queries=["username:dizzy", "username:russell"]) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") - self.assertEquals(0, len(results["response"]["docs"])) + self.assertEquals(0, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query_and_id(self): self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", docs=["dizzy"], queries=["username:russell"]) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") - self.assertEquals(0, len(results["response"]["docs"])) + self.assertEquals(0, len(results['docs'])) def test_build_rest_path_excludes_empty_query_params(self): self.assertEquals(self.client.get_transport().build_rest_path(bucket=self.client.bucket("foo"), key="bar", params={'r': None}), "/riak/foo/bar?") diff --git a/riak/transports/http.py b/riak/transports/http.py index 37ae5d96..ce75f1c1 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -37,6 +37,7 @@ from riak.multidict import MultiDict from connection import HTTPConnectionManager import riak.util +from xml.etree import ElementTree MAX_LINK_HEADER_SIZE = 8192 - 8 # substract length of "Link: " header string and newline @@ -259,7 +260,6 @@ def get_bucket_props(self, bucket): else: raise Exception('Error getting bucket properties.') - def set_bucket_props(self, bucket, props): """ Set the properties on the bucket object given @@ -309,6 +309,45 @@ def mapred(self, inputs, query, timeout=None): result = json.loads(response[1]) return result + def get_index(self, bucket, index, startkey, endkey=None): + """ + Performs a secondary index query. + """ + # TODO: use resource detection + segments = ["buckets", bucket, "index", index, str(startkey)] + if endkey: + segments.append(str(endkey)) + uri = '/%s' % ('/'.join(segments)) + headers, data = response = self.get_request(uri) + self.check_http_code(response, [200]) + jsonData = json.loads(data) + return jsonData[u'keys'][:] + + def search(self, index, query, **params): + """ + Performs a search query. + """ + if index is None: + index = 'search' + + options = {'q':query, 'wt':'json'} + if 'op' in params: + op = params.pop('op') + options['q.op'] = op + + options.update(params) + # TODO: use resource detection + uri = "/solr/%s/select" % index + headers, data = response = self.get_request(uri, options) + self.check_http_code(response, [200]) + if 'json' in headers['content-type']: + results = json.loads(data) + return self._normalize_json_search_response(results) + elif 'xml' in headers['content-type']: + return self._normalize_xml_search_response(data) + else: + raise ValueError("Could not decode search response") + def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] if not status in expected_statuses: @@ -377,9 +416,10 @@ def parse_body(self, response, expected_statuses): for token in line: rie = RiakIndexEntry(field, token) metadata[MD_INDEX].append(rie) - elif header == 'x-riak-vclock': vclock = value + elif header == 'x-riak-deleted': + metadata[MD_DELETED] = True if links: metadata[MD_LINKS] = links @@ -561,6 +601,35 @@ def http_request(self, method, uri, headers=None, body='') : # No luck, even with retrying. raise RiakError("could not get a response") + def _normalize_json_search_response(self, json): + """ + Normalizes a JSON search response so that PB and HTTP have the + same return value + """ + result = {} + if u'response' in json: + result['num_found'] = json[u'response'][u'numFound'] + result['max_score'] = float(json[u'response'][u'maxScore']) + docs = [] + for doc in json[u'response'][u'docs']: + resdoc = {u'id': doc[u'id']} + if u'fields' in doc: + for k, v in doc[u'fields'].iteritems(): + resdoc[k] = v + docs.append(resdoc) + result['docs'] = docs + return result + + def _normalize_xml_search_response(self, xml): + """ + Normalizes an XML search response so that PB and HTTP have the + same return value + """ + target = XMLSearchResult() + parser = ElementTree.XMLParser(target = target) + parser.feed(xml) + return parser.close() + @classmethod def build_headers(cls, headers): return ['%s: %s' % (header, value) for header, value in headers.iteritems()] @@ -586,3 +655,52 @@ def parse_http_headers(cls, headers) : else: retVal[key] = value return retVal + +class XMLSearchResult(object): + # Match tags that are document fields + fieldtags = ['str', 'int', 'date'] + + def __init__(self): + # Results + self.num_found = 0 + self.max_score = 0.0 + self.docs = [] + + # Parser state + self.currdoc = None + self.currfield = None + self.currvalue = None + + def start(self, tag, attrib): + if tag == 'result': + self.num_found = int(attrib['numFound']) + self.max_score = float(attrib['maxScore']) + elif tag == 'doc': + self.currdoc = {} + elif tag in self.fieldtags and self.currdoc is not None: + self.currfield = attrib['name'] + + def end(self, tag): + if tag == 'doc' and self.currdoc is not None: + self.docs.append(self.currdoc) + self.currdoc = None + elif tag in self.fieldtags and self.currdoc is not None: + if tag == 'int': + self.currvalue = int(self.currvalue) + self.currdoc[self.currfield] = self.currvalue + self.currfield = None + self.currvalue = None + + def data(self, data): + if self.currfield: + # riak_solr_output adds NL + 6 spaces + data = data.rstrip() + if self.currvalue: + self.currvalue += data + else: + self.currvalue = data + + def close(self): + return {'num_found':self.num_found, + 'max_score':self.max_score, + 'docs': self.docs } diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 898d7568..bb08b462 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -77,8 +77,8 @@ MSG_CODE_SET_BUCKET_RESP = 22 MSG_CODE_MAPRED_REQ = 23 MSG_CODE_MAPRED_RESP = 24 -MSG_CODE_INDEX_QUERY_REQ = 25 -MSG_CODE_INDEX_QUERY_RES = 26 +MSG_CODE_INDEX_REQ = 25 +MSG_CODE_INDEX_RESP = 26 MSG_CODE_SEARCH_QUERY_REQ = 27 MSG_CODE_SEARCH_QUERY_RESP = 28 @@ -248,7 +248,11 @@ def get(self, robj, r=None, pr=None, vtag=None): req = riak_pb.RpbGetReq() req.r = self.translate_rw_val(r) - req.pr = self.translate_rw_val(pr) + if self.quorum_controls(): + req.pr = self.translate_rw_val(pr) + + if self.tombstone_vclocks(): + req.deletedvclock = 1 req.bucket = bucket.get_name() req.key = robj.get_key() @@ -272,7 +276,8 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=Fa req = riak_pb.RpbPutReq() req.w = self.translate_rw_val(w) req.dw = self.translate_rw_val(dw) - req.pw = self.translate_rw_val(pw) + if self.quorum_controls(): + req.pw = self.translate_rw_val(pw) if return_body: req.return_body = 1 @@ -282,7 +287,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=Fa req.bucket = bucket.get_name() req.key = robj.get_key() vclock = robj.vclock() - if vclock is not None: + if vclock: req.vclock = vclock self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) @@ -303,6 +308,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_matc @return (key, vclock, metadata) """ + # Note that this won't work on 0.14 nodes. bucket = robj.get_bucket() req = riak_pb.RpbPutReq() @@ -340,10 +346,13 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): req.r = self.translate_rw_val(r) req.w = self.translate_rw_val(w) req.dw = self.translate_rw_val(dw) - req.pr = self.translate_rw_val(pr) - req.pw = self.translate_rw_val(pw) - # TODO: Set the vclock if present + if self.quorum_controls(): + req.pr = self.translate_rw_val(pr) + req.pw = self.translate_rw_val(pw) + + if self.tombstone_vclocks() and robj.vclock(): + req.vclock = robj.vclock() req.bucket = bucket.get_name() req.key = robj.get_key() @@ -445,6 +454,67 @@ def _handle_response(resp): else: return result + def get_index(self, bucket, index, startkey, endkey=None): + if not self.pb_indexes(): + return self._get_index_mapred_emu(bucket, index, startkey, endkey) + + req = riak_pb.RpbIndexReq(bucket=bucket, index=index) + if endkey: + req.qtype = riak_pb.RpbIndexReq.range + req.range_min = str(startkey) + req.range_max = str(endkey) + else: + req.qtype = riak_pb.RpbIndexReq.eq + req.key = str(startkey) + + msg_code, resp = self.send_msg(MSG_CODE_INDEX_REQ, req, + MSG_CODE_INDEX_RESP) + return resp.keys + + def search(self, index, query, **params): + if not self.pb_search(): + return self._search_mapred_emu(index, query) + + req = riak_pb.RpbSearchQueryReq(index=index, q=query) + if 'rows' in params: + req.rows = params['rows'] + if 'start' in params: + req.start = params['start'] + if 'sort' in params: + req.sort = params['sort'] + if 'filter' in params: + req.filter = params['filter'] + if 'df' in params: + req.df = params['df'] + if 'op' in params: + req.op = params['op'] + if 'q.op' in params: + req.op = params['q.op'] + if 'fl' in params: + if isinstance(params['fl'], list): + req.fl.extend(params['fl']) + else: + req.fl.append(params['fl']) + if 'presort' in params: + req.presort = params['presort'] + + msg_code, resp = self.send_msg(MSG_CODE_SEARCH_QUERY_REQ, req, + MSG_CODE_SEARCH_QUERY_RESP) + + result = {} + if resp.HasField('max_score'): + result['max_score'] = resp.max_score + if resp.HasField('num_found'): + result['num_found'] = resp.num_found + docs = [] + for doc in resp.docs: + resultdoc = {} + for pair in doc.fields: + resultdoc[pair.key] = pair.value + docs.append(resultdoc) + result['docs'] = docs + return result + def send_msg_code(self, msg_code, expect): with self._cm.withconn() as conn: self.send_pkt(conn, struct.pack("!iB", 1, msg_code)) @@ -540,8 +610,8 @@ def recv_msg(self, conn, expect): elif msg_code == MSG_CODE_MAPRED_RESP: msg = riak_pb.RpbMapRedResp() msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_INDEX_QUERY_RESP: - msg = riak_pb.RpbIndexQueryResp() + elif msg_code == MSG_CODE_INDEX_RESP: + msg = riak_pb.RpbIndexResp() msg.ParseFromString(self._inbuf[1:]) elif msg_code == MSG_CODE_SEARCH_QUERY_RESP: msg = riak_pb.RpbSearchQueryResp() @@ -579,6 +649,8 @@ def decode_contents(self, rpb_contents): def decode_content(self, rpb_content): metadata = {} + if rpb_content.HasField("deleted"): + metadata[MD_DELETED] = True if rpb_content.HasField("content_type"): metadata[MD_CTYPE] = rpb_content.content_type if rpb_content.HasField("charset"): @@ -648,4 +720,3 @@ def pbify_content(self, metadata, data, rpb_content): pb_link.key = link.get_key() pb_link.tag = link.get_tag() rpb_content.value = data - diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 03489a5e..4ac7786c 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -135,6 +135,68 @@ def get_client_id(self): """ raise RiakError("not implemented") + def search(self, index, query, **params): + """ + Performs a search query. + """ + raise RiakError("not implemented") + + def get_index(self, bucket, index, startkey, endkey=None): + """ + Performs a secondary index query. + """ + raise RiakError("not implemented") + + def _search_mapred_emu(self, index, query): + """ + Emulates a search request via MapReduce. Used in the case + where the transport supports MapReduce but has no native + search capability. + """ + phases = [] + if not self.phaseless_mapred(): + phases.append({'language':'erlang', + 'module':'riak_kv_mapreduce', + 'function':'reduce_identity', + 'keep':True}) + mr_result = self.mapred({'module':'riak_search', + 'function':'mapred_search', + 'arg':[index, query]}, + phases) + result = {'num_found': len(mr_result), + 'max_score': 0.0, + 'docs': []} + for bucket, key, data in mr_result: + if u'score' in data and data[u'score'][0] > result['max_score']: + result['max_score'] = data[u'score'][0] + result['docs'].append({u'id': key}) + return result + + def _get_index_mapred_emu(self, bucket, index, startkey, endkey=None): + """ + Emulates a secondary index request via MapReduce. Used in the + case where the transport supports MapReduce but has no native + secondary index query capability. + """ + phases = [] + if not self.phaseless_mapred(): + phases.append({'language':'erlang', + 'module':'riak_kv_mapreduce', + 'function':'reduce_identity', + 'keep':True}) + if endkey: + result = self.mapred({'bucket':bucket, + 'index':index, + 'start':startkey, + 'end':endkey}, + phases) + else: + result = self.mapred({'bucket':bucket, + 'index':index, + 'key':startkey}, + phases) + return [ key for bucket, key in result ] + def store_file(self, key, content_type="application/octet-stream", content=None): """ Store a large piece of data in luwak. From 28a739e5731d6b607e92734797c8b36253272f7c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sat, 28 Jul 2012 12:02:46 -0500 Subject: [PATCH 0216/1060] Add search support on Travis. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4c4ec76..51398d3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ python: - "2.7" install: ./setup.py develop script: ./setup.py test -# TODO: get search enabled on Travis' Riak install, or fix the test server -env: "SKIP_LUWAK=1 SKIP_SEARCH=1" +before_script: sudo search-cmd install searchbucket +env: "SKIP_LUWAK=1" notifications: email: clients@basho.com From 7595749ee24d03fde4719ce3f2e9958211ee423f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 2 Aug 2012 12:48:32 -0400 Subject: [PATCH 0217/1060] Fix typo. --- riak/transports/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index ce75f1c1..d5150706 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -286,7 +286,7 @@ def mapred(self, inputs, query, timeout=None): Run a MapReduce query. """ if not self.phaseless_mapred() and (query is None or len(query) is 0): - raise Exception('Phase-less MapReduce is supported by this Riak node') + raise Exception('Phase-less MapReduce is not supported by this Riak node') # Construct the job, optionally set the timeout... job = {'inputs':inputs, 'query':query} From 1d88cac9f31a25ce4e603d2c1aa40e16c698f0bc Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 1 Aug 2012 18:46:17 -0400 Subject: [PATCH 0218/1060] PEP8 style-compliant. --- riak/__init__.py | 14 +-- riak/bucket.py | 58 ++++++---- riak/client.py | 74 +++++++----- riak/mapreduce.py | 108 +++++++++--------- riak/metadata.py | 14 +-- riak/multidict.py | 14 +-- riak/riak_index_entry.py | 7 +- riak/riak_object.py | 129 ++++++++++++--------- riak/search.py | 8 +- riak/test_server.py | 74 ++++++++---- riak/tests/suite.py | 1 + riak/tests/test_all.py | 161 +++++++++++++++++---------- riak/tests/test_feature_detection.py | 4 + riak/tests/test_server_test.py | 32 ++++-- riak/transports/connection.py | 40 ++++--- riak/transports/feature_detect.py | 2 + riak/transports/http.py | 146 ++++++++++++++---------- riak/transports/pbc.py | 90 ++++++++------- riak/transports/transport.py | 61 +++++----- riak/util.py | 12 +- 20 files changed, 621 insertions(+), 428 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index 8a334910..1e116161 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -30,11 +30,13 @@ @author Jay Baird (@skatterbean) (jay@mochimedia.com) """ -class RiakError(Exception) : - def __init__(self, value): - self.value = value - def __str__(self): - return repr(self.value) + +class RiakError(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) from riak_object import RiakObject from bucket import RiakBucket @@ -49,5 +51,3 @@ def __str__(self): QUORUM = "quorum" key_filter = RiakKeyFilter() - - diff --git a/riak/bucket.py b/riak/bucket.py index 586f01ba..3b1460aa 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -20,6 +20,7 @@ from riak_object import RiakObject import mimetypes + class RiakBucket(object): """ The ``RiakBucket`` object allows you to access and change information @@ -186,7 +187,6 @@ def set_pr(self, pr): self._pr = pr return self - def get_pw(self, pw=None): """ Get the PW-value for this bucket, if it is set, otherwise return @@ -214,7 +214,8 @@ def set_pw(self, pw): def get_encoder(self, content_type): """ - Get the encoding function for the provided content type for this bucket. + Get the encoding function for the provided content type for + this bucket. :param content_type: Content type requested """ @@ -225,18 +226,20 @@ def get_encoder(self, content_type): def set_encoder(self, content_type, encoder): """ - Set the encoding function for the provided content type for this bucket. + Set the encoding function for the provided content type for + this bucket. :param content_type: Content type for encoder - :param encoder: Function to encode with - will be called with data as single - argument. + :param encoder: Function to encode with - will be called with + data as single argument. """ self._encoders[content_type] = encoder return self def get_decoder(self, content_type): """ - Get the decoding function for the provided content type for this bucket. + Get the decoding function for the provided content type for + this bucket. :param content_type: Content type for decoder """ @@ -247,20 +250,25 @@ def get_decoder(self, content_type): def set_decoder(self, content_type, decoder): """ - Set the decoding function for the provided content type for this bucket. + Set the decoding function for the provided content type for + this bucket. :param content_type: Content type for decoder - :param decoder: Function to decode with - will be called with string + :param decoder: Function to decode with - will be called with + string """ self._decoders[content_type] = decoder return self def new(self, key=None, data=None, content_type='application/json'): """ - Create a new :class:`RiakObject ` that will be stored as JSON. A shortcut for - manually instantiating a :class:`RiakObject `. + Create a new :class:`RiakObject ` + that will be stored as JSON. A shortcut for manually + instantiating a :class:`RiakObject + `. - :param key: Name of the key. Leaving this to be None (default) will make Riak generate the key on store. + :param key: Name of the key. Leaving this to be None (default) + will make Riak generate the key on store. :type key: string :param data: The data to store. :type data: object @@ -280,8 +288,10 @@ def new(self, key=None, data=None, content_type='application/json'): def new_binary(self, key, data, content_type='application/octet-stream'): """ - Create a new :class:`RiakObject ` that will be stored as plain text/binary. - A shortcut for manually instantiating a :class:`RiakObject `. + Create a new :class:`RiakObject ` + that will be stored as plain text/binary. A shortcut for + manually instantiating a :class:`RiakObject + `. :param key: Name of the key. :type key: string @@ -387,11 +397,6 @@ def set_allow_multiples(self, bool): and returned to the client. This situation can be detected by calling has_siblings() and get_siblings(). - .. warning:: - - This should only be used if you know what you are doing, as it can lead to - unexpected results. - :param bool: True to store and return conflicting writes. :type bool: boolean """ @@ -418,11 +423,12 @@ def set_property(self, key, value): :param value: Property value. :type value: mixed """ - return self.set_properties({key : value}) + return self.set_properties({key: value}) def get_bool_property(self, key): """ - Get a boolean bucket property. Converts to a ``True`` or ``False`` value. + Get a boolean bucket property. Converts to a ``True`` or + ``False`` value. :param key: Property to set. :type key: string @@ -480,7 +486,8 @@ def get_keys(self): def new_binary_from_file(self, key, filename): """ - Create a new Riak object in the bucket, using the content of the specified file. + Create a new Riak object in the bucket, using the content of + the specified file. """ binary_data = open(filename, "rb").read() mimetype, encoding = mimetypes.guess_type(filename) @@ -490,9 +497,11 @@ def new_binary_from_file(self, key, filename): def search_enabled(self): """ - Returns True if the search precommit hook is enabled for this bucket. + Returns True if the search precommit hook is enabled for this + bucket. """ - return self.SEARCH_PRECOMMIT_HOOK in (self.get_property("precommit") or []) + return self.SEARCH_PRECOMMIT_HOOK in (self.get_property("precommit") or + []) def enable_search(self): """ @@ -526,4 +535,5 @@ def get_index(self, index, startkey, endkey=None): """ Queries a secondary index over objects in this bucket, returning keys. """ - return self._client._transport.get_index(self._name, index, startkey, endkey) + return self._client._transport.get_index(self._name, index, startkey, + endkey) diff --git a/riak/client.py b/riak/client.py index 7bb5e8dc..4b809b75 100644 --- a/riak/client.py +++ b/riak/client.py @@ -53,9 +53,12 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', :type mapred_prefix: string :param transport_class: transport class to use :type transport_class: :class:`RiakTransport` - :param solr_transport_class: HTTP-based transport class for Solr interface queries + + :param solr_transport_class: HTTP-based transport class for + Solr interface queries :type transport_class: :class:`RiakHttpTransport` - :param transport_options: Optional key-value args to pass to the transport constuctor + :param transport_options: Optional key-value args to pass to + the transport constuctor :type transport_options: dict """ if transport_class is None: @@ -63,7 +66,7 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', api = getattr(transport_class, 'api', 1) if api >= 2: - hostports = [ (host, port), ] + hostports = [(host, port), ] self._cm = transport_class.default_cm(hostports) # If no transport options are provided, then default to the @@ -111,11 +114,12 @@ def get_r(self): def set_r(self, r): """ - Set the R-value for this ``RiakClient``. This value will be used - for any calls to :func:`RiakBucket.get ` - or :func:`RiakBucket.get_binary ` - where 1) no R-value is specified in the method call and 2) no R-value has - been set in the :class:`RiakBucket `. + Set the R-value for this ``RiakClient``. This value will be + used for any calls to :func:`RiakBucket.get + ` or :func:`RiakBucket.get_binary + ` where 1) no R-value is + specified in the method call and 2) no R-value has been set in + the :class:`RiakBucket `. :param r: The R value. :type r: integer @@ -126,7 +130,8 @@ def set_r(self, r): def get_w(self): """ - Get the W-value setting for this ``RiakClient``. (default "quorum") + Get the W-value setting for this ``RiakClient``. (default + "quorum") :rtype: integer """ @@ -134,8 +139,8 @@ def get_w(self): def set_w(self, w): """ - Set the W-value for this ``RiakClient`` instance. See :func:`set_r` for a - description of how these values are used. + Set the W-value for this ``RiakClient`` instance. See + :func:`set_r` for a description of how these values are used. :param w: The W value. :type w: integer @@ -146,7 +151,8 @@ def set_w(self, w): def get_dw(self): """ - Get the DW-value for this ``RiakClient`` instance. (default "quorum") + Get the DW-value for this ``RiakClient`` instance. (default + "quorum") :rtype: integer """ @@ -154,8 +160,8 @@ def get_dw(self): def set_dw(self, dw): """ - Set the DW-value for this ``RiakClient`` instance. See :func:`set_r` for a - description of how these values are used. + Set the DW-value for this ``RiakClient`` instance. See + :func:`set_r` for a description of how these values are used. :param dw: The DW value. :type dw: integer @@ -166,7 +172,8 @@ def set_dw(self, dw): def get_rw(self): """ - Get the RW-value for this ``RiakClient`` instance. (default "quorum") + Get the RW-value for this ``RiakClient`` instance. (default + "quorum") :rtype: integer """ @@ -174,8 +181,8 @@ def get_rw(self): def set_rw(self, rw): """ - Set the RW-value for this ``RiakClient`` instance. See :func:`set_r` for a - description of how these values are used. + Set the RW-value for this ``RiakClient`` instance. See + :func:`set_r` for a description of how these values are used. :param rw: The RW value. :type rw: integer @@ -194,8 +201,8 @@ def get_pr(self): def set_pr(self, pr): """ - Set the PR-value for this ``RiakClient`` instance. See :func:`set_r` for a - description of how these values are used. + Set the PR-value for this ``RiakClient`` instance. See + :func:`set_r` for a description of how these values are used. :param pr: The PR value. :type pr: integer @@ -214,8 +221,8 @@ def get_pw(self): def set_pw(self, pw): """ - Set the PW-value for this ``RiakClient`` instance. See :func:`set_r` for a - description of how these values are used. + Set the PW-value for this ``RiakClient`` instance. See + :func:`set_r` for a description of how these values are used. :param pw: The W value. :type pw: integer @@ -238,7 +245,9 @@ def set_client_id(self, client_id): .. warning:: - You should not call this method unless you know what you are doing. + Refer to + http://wiki.basho.com/Client-Implementation-Guide.html#Client-IDs + for information on how to set the client_id. :param client_id: The new client_id. :type client_id: string @@ -308,7 +317,8 @@ def is_alive(self): def add(self, *args): """ - Start assembling a Map/Reduce operation. A shortcut for :func:`RiakMapReduce.add`. + Start assembling a Map/Reduce operation. A shortcut for + :func:`RiakMapReduce.add`. :rtype: :class:`RiakMapReduce` """ @@ -319,7 +329,8 @@ def search(self, *args): """ Start assembling a Map/Reduce operation based on search results. This command will return an error unless executed - against a Riak Search cluster. A shortcut for :func:`RiakMapReduce.search`. + against a Riak Search cluster. A shortcut for + :func:`RiakMapReduce.search`. :rtype: :class:`RiakMapReduce` """ @@ -338,7 +349,8 @@ def index(self, *args): def link(self, *args): """ - Start assembling a Map/Reduce operation. A shortcut for :func:`RiakMapReduce.link`. + Start assembling a Map/Reduce operation. A shortcut for + :func:`RiakMapReduce.link`. :rtype: :class:`RiakMapReduce` """ @@ -347,7 +359,8 @@ def link(self, *args): def map(self, *args): """ - Start assembling a Map/Reduce operation. A shortcut for :func:`RiakMapReduce.map`. + Start assembling a Map/Reduce operation. A shortcut for + :func:`RiakMapReduce.map`. :rtype: :class:`RiakMapReduce` """ @@ -356,18 +369,21 @@ def map(self, *args): def reduce(self, *args): """ - Start assembling a Map/Reduce operation. A shortcut for :func:`RiakMapReduce.reduce`. + Start assembling a Map/Reduce operation. A shortcut for + :func:`RiakMapReduce.reduce`. :rtype: :class:`RiakMapReduce` """ mr = RiakMapReduce(self) return apply(mr.reduce, args) - def store_file(self, filename, data, content_type="application/octet-stream"): + def store_file(self, filename, data, + content_type="application/octet-stream"): """ Store data in luwak using filename as the key """ - self._transport.store_file(filename, content_type=content_type, content=data) + self._transport.store_file(filename, content_type=content_type, + content=data) def get_file(self, filename): return self._transport.get_file(filename) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index fa7d4f2b..26d8b483 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -21,6 +21,7 @@ from riak_object import RiakObject from bucket import RiakBucket + class RiakMapReduce(object): """ The RiakMapReduce object allows you to build up and run a @@ -60,7 +61,7 @@ def add(self, arg1, arg2=None, arg3=None): def add_object(self, obj): return self.add_bucket_key_data(obj._bucket._name, obj._key, None) - def add_bucket_key_data(self, bucket, key, data) : + def add_bucket_key_data(self, bucket, key, data): if self._input_mode == 'bucket': raise Exception('Already added a bucket, can\'t add an object.') elif self._input_mode == 'query': @@ -69,21 +70,21 @@ def add_bucket_key_data(self, bucket, key, data) : self._inputs.append([bucket, key, data]) return self - def add_bucket(self, bucket) : + def add_bucket(self, bucket): self._input_mode = 'bucket' self._inputs = bucket return self - def add_key_filters(self, key_filters) : + def add_key_filters(self, key_filters): if self._input_mode == 'query': - raise Exception('Key filters are not supported in a query.') + raise Exception('Key filters are not supported in a query.') self._key_filters.extend(key_filters) return self - def add_key_filter(self, *args) : + def add_key_filter(self, *args): if self._input_mode == 'query': - raise Exception('Key filters are not supported in a query.') + raise Exception('Key filters are not supported in a query.') self._key_filters.append(args) return self @@ -96,12 +97,12 @@ def search(self, bucket, query): @param query - The search query. """ self._input_mode = 'query' - self._inputs = {'module':'riak_search', - 'function':'mapred_search', - 'arg':[bucket, query]} + self._inputs = {'module': 'riak_search', + 'function': 'mapred_search', + 'arg': [bucket, query]} return self - def index(self, bucket, index, startkey, endkey = None): + def index(self, bucket, index, startkey, endkey=None): """ Begin a map/reduce operation using a Secondary Index query. @@ -112,13 +113,13 @@ def index(self, bucket, index, startkey, endkey = None): if endkey == None: self._inputs = {'bucket': bucket, - 'index':index, - 'key':startkey } + 'index': index, + 'key': startkey} else: - self._inputs = {'bucket':bucket, - 'index':index, - 'start':startkey, - 'end':endkey } + self._inputs = {'bucket': bucket, + 'index': index, + 'start': startkey, + 'end': endkey} return self def link(self, bucket='_', tag='_', keep=False): @@ -151,7 +152,7 @@ def map(self, function, options=None): if isinstance(function, list): language = 'erlang' else: - language='javascript' + language = 'javascript' mr = RiakMapReducePhase('map', function, @@ -176,7 +177,7 @@ def reduce(self, function, options=None): if isinstance(function, list): language = 'erlang' else: - language='javascript' + language = 'javascript' mr = RiakMapReducePhase('reduce', function, @@ -212,26 +213,27 @@ def run(self, timeout=None): phase = self._phases[i] if (i == (num_phases - 1)) and (not keep_flag): phase._keep = True - if phase._keep: keep_flag = True + if phase._keep: + keep_flag = True query.append(phase.to_array()) if (len(self._key_filters) > 0): - bucket_name = None - if (type(self._inputs) == str): - bucket_name = self._inputs - elif (type(self._inputs) == RiakBucket): - bucket_name = self._inputs.get_name() + bucket_name = None + if (type(self._inputs) == str): + bucket_name = self._inputs + elif (type(self._inputs) == RiakBucket): + bucket_name = self._inputs.get_name() - if (bucket_name is not None): - self._inputs = {'bucket': bucket_name, - 'key_filters': self._key_filters} + if (bucket_name is not None): + self._inputs = {'bucket': bucket_name, + 'key_filters': self._key_filters} t = self._client.get_transport() result = t.mapred(self._inputs, query, timeout) # If the last phase is NOT a link phase, then return the result. - link_results_flag = link_results_flag or isinstance(self._phases[-1], RiakLinkPhase) - if not link_results_flag: + if not (link_results_flag + or isinstance(self._phases[-1], RiakLinkPhase)): return result # If there are no results, then return an empty list. @@ -279,7 +281,7 @@ def reduce_sort(self, js_cmp=None, options=None): return self.reduce("Riak.reduceSort", options=options) def reduce_numeric_sort(self, options=None): - return self.reduce("Riak.reduceNumericSort", options=options) + return self.reduce("Riak.reduceNumericSort", options=options) def reduce_limit(self, limit, options=None): if options is None: @@ -287,7 +289,7 @@ def reduce_limit(self, limit, options=None): options['arg'] = limit # reduceLimit is broken in riak_kv - code="""function(value, arg) { + code = """function(value, arg) { return value.slice(0, arg); }""" return self.reduce(code, options=options) @@ -337,24 +339,26 @@ def to_array(self): Convert the RiakMapReducePhase to an associative array. Used internally. """ - stepdef = {'keep':self._keep, - 'language':self._language, - 'arg':self._arg} + stepdef = {'keep': self._keep, + 'language': self._language, + 'arg': self._arg} - if (self._language == 'javascript') and isinstance(self._function, list): - stepdef['bucket'] = self._function[0] - stepdef['key'] = self._function[1] - elif (self._language == 'javascript') and isinstance(self._function, str): - if ("{" in self._function): - stepdef['source'] = self._function - else: - stepdef['name'] = self._function + if self._language == 'javascript': + if isinstance(self._function, list): + stepdef['bucket'] = self._function[0] + stepdef['key'] = self._function[1] + elif isinstance(self._function, str): + if ("{" in self._function): + stepdef['source'] = self._function + else: + stepdef['name'] = self._function elif (self._language == 'erlang' and isinstance(self._function, list)): stepdef['module'] = self._function[0] stepdef['function'] = self._function[1] - return {self._type : stepdef} + return {self._type: stepdef} + class RiakLinkPhase(object): """ @@ -378,10 +382,11 @@ def to_array(self): Convert the RiakLinkPhase to an associative array. Used internally. """ - stepdef = {'bucket':self._bucket, - 'tag':self._tag, - 'keep':self._keep} - return {'link':stepdef} + stepdef = {'bucket': self._bucket, + 'tag': self._tag, + 'keep': self._keep} + return {'link': stepdef} + class RiakLink(object): """ @@ -486,8 +491,11 @@ def isEqual(self, link): @param RiakLink link - A RiakLink object. @return boolean """ - is_equal = (self._bucket == link._bucket) and (self._key == link._key) and (self.get_tag() == link.get_tag()) - return is_equal + + return ((self._bucket == link._bucket) and + (self._key == link._key) and + (self.get_tag() == link.get_tag())) + class RiakKeyFilter(object): def __init__(self, *args): @@ -511,7 +519,7 @@ def _bool_op(self, op, other): return f # Otherwise just create a new RiakKeyFilter() object with an and return RiakKeyFilter(op, self._filters, other._filters) - + def __and__(self, other): return self._bool_op("and", other) diff --git a/riak/metadata.py b/riak/metadata.py index 095bd0f3..b92353d3 100644 --- a/riak/metadata.py +++ b/riak/metadata.py @@ -17,13 +17,13 @@ specific language governing permissions and limitations under the License. """ -MD_CTYPE = "content-type" -MD_CHARSET = "charset" +MD_CTYPE = "content-type" +MD_CHARSET = "charset" MD_ENCODING = "content-encoding" -MD_VTAG = "vtag" -MD_LINKS = "links" -MD_LASTMOD = "lastmod" +MD_VTAG = "vtag" +MD_LINKS = "links" +MD_LASTMOD = "lastmod" MD_LASTMOD_USECS = "lastmod-usecs" MD_USERMETA = "usermeta" -MD_INDEX = "index" -MD_DELETED = "deleted" +MD_INDEX = "index" +MD_DELETED = "deleted" diff --git a/riak/multidict.py b/riak/multidict.py index 5336803e..14761dde 100644 --- a/riak/multidict.py +++ b/riak/multidict.py @@ -1,7 +1,9 @@ -# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) -# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php +# (c) 2005 Ian Bicking and contributors; written for Paste +# (http://pythonpaste.org) Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license.php from UserDict import DictMixin + class MultiDict(DictMixin): """ @@ -106,7 +108,7 @@ def dict_of_lists(self): def __delitem__(self, key): items = self._items found = False - for i in range(len(items)-1, -1, -1): + for i in range(len(items) - 1, -1, -1): if items[i][0] == key: del items[i] found = True @@ -136,8 +138,8 @@ def setdefault(self, key, default=None): def pop(self, key, *args): if len(args) > 1: - raise TypeError, "pop expected at most 2 arguments, got "\ - + repr(1 + len(args)) + raise TypeError("pop expected at most 2 arguments, got %s" % + (1 + len(args))) for i in range(len(self._items)): if self._items[i][0] == key: v = self._items[i][1] @@ -197,5 +199,3 @@ def values(self): def itervalues(self): for k, v in self._items: yield v - - diff --git a/riak/riak_index_entry.py b/riak/riak_index_entry.py index c642512c..85b51ac8 100644 --- a/riak/riak_index_entry.py +++ b/riak/riak_index_entry.py @@ -18,6 +18,7 @@ under the License. """ + class RiakIndexEntry(object): def __init__(self, field, value): self._field = field @@ -30,7 +31,8 @@ def get_value(self): return self._value def __str__(self): - return "RiakIndexEntry(field = '%s', value='%s')" % (self._field, self._value) + return ("RiakIndexEntry(field = '%s', value='%s')" % + (self._field, self._value)) def __eq__(self, other): if not isinstance(other, RiakIndexEntry): @@ -45,7 +47,8 @@ def __cmp__(self, other): raise TypeError("RiakIndexEntry cannot be compared to None") if not isinstance(other, RiakIndexEntry): - raise TypeError("RiakIndexEntry cannot be compared to %s" % other.__class__.__name__) + raise TypeError("RiakIndexEntry cannot be compared to %s" % + other.__class__.__name__) if self.get_field() < other.get_field(): return -1 diff --git a/riak/riak_object.py b/riak/riak_object.py index 4b995c88..fe522cf6 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -17,11 +17,13 @@ specific language governing permissions and limitations under the License. """ -import types, copy +import types +import copy from metadata import * from riak import RiakError from riak.riak_index_entry import RiakIndexEntry + class RiakObject(object): """ The RiakObject holds meta information about a Riak object, plus the @@ -62,7 +64,7 @@ def get_bucket(self): :rtype: RiakBucket """ - return self._bucket; + return self._bucket def get_key(self): """ @@ -72,7 +74,6 @@ def get_key(self): """ return self._key - def get_data(self): """ Get the data stored in this object. Will return an associative @@ -133,7 +134,8 @@ def set_encoded_data(self, data): content_type = self.get_content_type() decoder = self._bucket.get_decoder(content_type) if decoder is None: - # if no decoder, just set as string data for application to handle + # if no decoder, just set as string data for + # application to handle self._data = data else: self._data = decoder(data) @@ -141,7 +143,6 @@ def set_encoded_data(self, data): self._data = data return self - def get_metadata(self): """ Get the metadata stored in this object. Will return an associative @@ -164,15 +165,15 @@ def set_metadata(self, metadata): def get_usermeta(self): if MD_USERMETA in self._metadata: - return self._metadata[MD_USERMETA] + return self._metadata[MD_USERMETA] else: - return {} + return {} def set_usermeta(self, usermeta): """ - Sets the custom user metadata on this object. This doesn't include things - like content type and links, but only user-defined meta attributes stored - with the Riak object. + Sets the custom user metadata on this object. This doesn't + include things like content type and links, but only + user-defined meta attributes stored with the Riak object. :param userdata: The user metadata to store. :type userdata: dict @@ -183,7 +184,8 @@ def set_usermeta(self, usermeta): def add_index(self, field, value): """ - Tag this object with the specified field/value pair for indexing. + Tag this object with the specified field/value pair for + indexing. :param field: The index field. :type field: string @@ -199,7 +201,8 @@ def add_index(self, field, value): def remove_index(self, field=None, value=None): """ - Remove the specified field/value pair as an index on this object. + Remove the specified field/value pair as an index on this + object. :param field: The index field. :type field: string @@ -210,11 +213,13 @@ def remove_index(self, field=None, value=None): if not field and not value: ries = self._metadata[MD_INDEX][:] elif field and not value: - ries = [x for x in self._metadata[MD_INDEX] if x.get_field() == field] + ries = [x for x in self._metadata[MD_INDEX] + if x.get_field() == field] elif field and value: ries = [RiakIndexEntry(field, value)] else: - raise Exception("Cannot pass value without a field name while removing index") + raise Exception( + "Cannot pass value without a field name while removing index") for rie in ries: if rie in self._metadata[MD_INDEX]: @@ -225,10 +230,11 @@ def remove_index(self, field=None, value=None): def set_indexes(self, indexes): """ - Replaces all indexes on a Riak object. Currenly supports an iterable of 2 item tuples, - (field, value) + Replaces all indexes on a Riak object. Currenly supports an + iterable of 2 item tuples, (field, value) - :param indexes: iterable of 2 item tuples consisting the field and value. + :param indexes: iterable of 2 item tuples consisting the field + and value. :rtype: self """ new_indexes = [] @@ -239,9 +245,10 @@ def set_indexes(self, indexes): return self - def get_indexes(self, field = None): + def get_indexes(self, field=None): """ - Get a list of the index entries for this object. If a field is provided, returns a list + Get a list of the index entries for this object. If a field is + provided, returns a list :param field: The index field. :type field: string or None @@ -250,14 +257,16 @@ def get_indexes(self, field = None): if field == None: return self._metadata[MD_INDEX] else: - return [x.get_value() for x in self._metadata[MD_INDEX] if x.get_field() == field] + return [x.get_value() for x in self._metadata[MD_INDEX] + if x.get_field() == field] def exists(self): """ - Return True if the object exists, False otherwise. Allows you to - detect a :func:`RiakBucket.get ` or - :func:`RiakBucket.get_binary ` - operation where the object is missing. + Return True if the object exists, False otherwise. Allows you + to detect a :func:`RiakBucket.get + ` or :func:`RiakBucket.get_binary + ` operation where the + object is missing. :rtype: boolean """ @@ -265,9 +274,10 @@ def exists(self): def get_content_type(self): """ - Get the content type of this object. This is either ``application/json``, or - the provided content type if the object was created via - :func:`RiakBucket.new_binary `. + Get the content type of this object. This is either + ``application/json``, or the provided content type if the + object was created via :func:`RiakBucket.new_binary + `. :rtype: string """ @@ -294,11 +304,14 @@ def set_links(self, links, all_link=False): """ Replaces all links to a RiakObject - :param links: An iterable of 2-item tuples, consisting of (RiakObject, tag). This could also be an iterable of - just a RiakObject, instead of the tuple, then a tag of None would be used. Lastly, it could also be an - iterable of RiakLink. They have tags built-in. - :param all_link: A boolean indicates if links are all RiakLink objects - This speeds up the operation. + :param links: An iterable of 2-item tuples, consisting of + (RiakObject, tag). This could also be an iterable of just + a RiakObject, instead of the tuple, then a tag of None + would be used. Lastly, it could also be an iterable of + RiakLink. They have tags built-in. + + :param all_link: A boolean indicates if links are all RiakLink + objects This speeds up the operation. """ if all_link: self._metadata[MD_LINKS] = links @@ -378,7 +391,8 @@ def get_links(self): else: return [] - def store(self, w=None, dw=None, pw=None, return_body=True, if_none_match=False): + def store(self, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): """ Store the object in Riak. When this operation completes, the object could contain new metadata and possibly new data if Riak @@ -391,13 +405,15 @@ def store(self, w=None, dw=None, pw=None, return_body=True, if_none_match=False) :param dw: DW-value, wait for this many partitions to confirm the write before returning to client. :type dw: integer - :param pw: PW-value, require this many primary partitions to be available - before performing the put + + :param pw: PW-value, require this many primary partitions to + be available before performing the put :type pw: integer - :param return_body: if the newly stored object should be retrieved + :param return_body: if the newly stored object should be + retrieved :type return_body: bool - :param if_none_match: Should the object be stored only if there is no - key previously defined + :param if_none_match: Should the object be stored only if + there is no key previously defined :type if_none_match: bool :rtype: self """ @@ -410,19 +426,21 @@ def store(self, w=None, dw=None, pw=None, return_body=True, if_none_match=False) t = self._client.get_transport() if self._key is None: - key, vclock, metadata = t.put_new(self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) + key, vclock, metadata = t.put_new(self, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) self._exists = True self._key = key self._vclock = vclock self.set_metadata(metadata) else: - Result = t.put(self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) + Result = t.put(self, w=w, dw=dw, pw=pw, return_body=return_body, + if_none_match=if_none_match) if Result is not None: self.populate(Result) return self - def reload(self, r=None, pr=None, vtag=None): """ Reload the object from Riak. When this operation completes, the @@ -434,7 +452,7 @@ def reload(self, r=None, pr=None, vtag=None): :type r: integer :rtype: self """ - # Do the request... + r = self._bucket.get_r(r) pr = self._bucket.get_pr(pr) t = self._client.get_transport() @@ -446,13 +464,13 @@ def reload(self, r=None, pr=None, vtag=None): return self - def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Delete this object from Riak. :param rw: RW-value. Wait until this many partitions have - deleted the object before responding. (deprecated in Riak 1.0+, use R/W/DW) + deleted the object before responding. (deprecated in Riak + 1.0+, use R/W/DW) :type rw: integer :param r: R-value, wait for this many partitions to read object before performing the put @@ -463,11 +481,12 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): :param dw: DW-value, wait for this many partitions to confirm the write before returning to client. :type dw: integer - :param pr: PR-value, require this many primary partitions to be available - before performing the read that precedes the put + :param pr: PR-value, require this many primary partitions to + be available before performing the read that + precedes the put :type pr: integer - :param pr: PW-value, require this many primary partitions to be available - before performing the put + :param pw: PW-value, require this many primary partitions to + be available before performing the put :type pw: integer :rtype: self """ @@ -483,7 +502,7 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): self.clear() return self - def clear(self) : + def clear(self): """ Reset this object. @@ -496,7 +515,7 @@ def clear(self) : self._siblings = [] return self - def vclock(self) : + def vclock(self): """ Get the vclock of this object. @@ -504,7 +523,7 @@ def vclock(self) : """ return self._vclock - def populate(self, Result) : + def populate(self, Result): """ Populate the object based on the return from get. @@ -525,7 +544,7 @@ def populate(self, Result) : if len(contents) > 0: (metadata, data) = contents.pop(0) self._exists = True - if not metadata.has_key(MD_INDEX): + if not MD_INDEX in metadata: metadata[MD_INDEX] = [] self.set_metadata(metadata) self.set_encoded_data(data) @@ -539,7 +558,7 @@ def populate(self, Result) : for sibling in siblings: sibling.set_siblings(siblings) else: - raise RiakError("do not know how to handle type " + str(type(Result))) + raise RiakError("do not know how to handle type %s" % type(Result)) def has_siblings(self): """ @@ -605,8 +624,8 @@ def set_siblings(self, siblings): .. warning:: - Make sure this object is at index 0 so get_siblings(0) always returns - the current object + Make sure this object is at index 0 so get_siblings(0) + always returns the current object """ try: i = siblings.index(self) diff --git a/riak/search.py b/riak/search.py index a2066fbe..8a3e9a91 100644 --- a/riak/search.py +++ b/riak/search.py @@ -2,6 +2,7 @@ from xml.etree import ElementTree from xml.dom.minidom import Document + class RiakSearch(object): def __init__(self, client, transport_class=None, host="127.0.0.1", port=8098): @@ -10,7 +11,7 @@ def __init__(self, client, transport_class=None, api = getattr(transport_class, 'api', 1) if api >= 2: - hostports = [ (host, port), ] + hostports = [(host, port), ] self._cm = transport_class.default_cm(hostports) self._transport = transport_class(self._cm, prefix="/solr") else: @@ -22,9 +23,10 @@ def __init__(self, client, transport_class=None, self._client = client self._decoders = {"text/xml": ElementTree.fromstring} - + def get_decoder(self, content_type): - decoder = self._client.get_decoder(content_type) or self._decoders[content_type] + decoder = (self._client.get_decoder(content_type) + or self._decoders[content_type]) if not decoder: decoder = self.decode diff --git a/riak/test_server.py b/riak/test_server.py index 4c7f076b..a25b01e2 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -12,9 +12,10 @@ from riak.util import deep_merge try: - bytes + bytes except NameError: - bytes = str + bytes = str + class Atom(object): def __init__(self, s): @@ -32,6 +33,7 @@ def __eq__(self, other): def __cmp__(self, other): return cmp(self.str, other) + def erlang_config(hash, depth=1): def printable(item): k, v = item @@ -47,7 +49,7 @@ def printable(item): return "{%s, %s}" % (k, p) padding = ' ' * depth - parent_padding = ' ' * (depth-1) + parent_padding = ' ' * (depth - 1) values = (",\n%s" % padding).join(map(printable, hash.items())) return "[\n%s%s\n%s]" % (padding, values, parent_padding) @@ -55,13 +57,15 @@ def printable(item): class TestServer(object): VM_ARGS_DEFAULTS = { "-name": "riaktest%d@127.0.0.1" % random.randint(0, 100000), - "-setcookie": "%d_%d" % (random.randint(0, 100000), random.randint(0, 100000)), + "-setcookie": "%d_%d" % (random.randint(0, 100000), + random.randint(0, 100000)), "+K": "true", "+A": 64, "-smp": "enable", "-env ERL_MAX_PORTS": 4096, "-env ERL_FULLSWEEP_AFTER": 10, - "-pa": os.path.abspath(os.path.join(os.path.dirname(__file__), "erl_src")) + "-pa": os.path.abspath(os.path.join(os.path.dirname(__file__), + "erl_src")) } APP_CONFIG_DEFAULTS = { @@ -107,10 +111,11 @@ def __init__(self, tmp_dir="/tmp/riak/test_server", for key, value in options.items(): if key in self.app_config: self.app_config[key] = deep_merge(self.app_config[key], value) - - self.app_config["riak_core"]["ring_state_dir"] = os.path.join(self.temp_dir, "data", "ring") + ring_dir = os.path.join(self.temp_dir, "data", "ring") + crash_log = os.path.join(self.temp_dir, "log", "crash.log") + self.app_config["riak_core"]["ring_state_dir"] = ring_dir self.app_config["riak_core"]["platform_data_dir"] = self.temp_dir - self.app_config["lager"] = {"crash_log": os.path.join(self.temp_dir, "log", "crash.log")} + self.app_config["lager"] = {"crash_log": crash_log} def prepare(self): if not self._prepared: @@ -132,7 +137,8 @@ def create_temp_directories(self): def start(self): if self._prepared and not self._started: with self._lock: - self._server = Popen([self._riak_script, "console"], stdin=PIPE, stdout=PIPE, stderr=PIPE) + self._server = Popen([self._riak_script, "console"], + stdin=PIPE, stdout=PIPE, stderr=PIPE) self._server.stdin.write("\n") self._server.stdin.flush() self.wait_for_erlang_prompt() @@ -157,7 +163,7 @@ def recycle(self): if self._started: with self._lock: stdin = self._server.stdin - if self.app_config["riak_kv"]["storage_backend"] == "riak_kv_test_backend": + if self._kv_backend() == "riak_kv_test_backend": stdin.write("riak_kv_test_backend:reset().\n") stdin.flush() self.wait_for_erlang_prompt() @@ -176,13 +182,13 @@ def wait_for_startup(self): listening = False while not listening: try: - s = socket.create_connection((self.app_config["riak_core"]["web_ip"], self.app_config["riak_core"]["web_port"]), 1.0) + socket.create_connection((self._http_ip(), self._http_port()), + 1.0) except socket.error, (value, message): pass else: listening = True - def wait_for_erlang_prompt(self): prompted = False buffer = "" @@ -199,30 +205,54 @@ def write_riak_script(self): with open(self._riak_script, "wb") as temp_bin_file: with open(os.path.join(self.bin_dir, "riak"), "r") as riak_file: for line in riak_file.readlines(): - line = re.sub("(RUNNER_SCRIPT_DIR=)(.*)", r'\1%s' % self._temp_bin, line) - line = re.sub("(RUNNER_ETC_DIR=)(.*)", r'\1%s' % self._temp_etc, line) + line = re.sub("(RUNNER_SCRIPT_DIR=)(.*)", r'\1%s' % + self._temp_bin, + line) + line = re.sub("(RUNNER_ETC_DIR=)(.*)", r'\1%s' % + self._temp_etc, line) line = re.sub("(RUNNER_USER=)(.*)", r'\1', line) - line = re.sub("(RUNNER_LOG_DIR=)(.*)", r'\1%s' % self._temp_log, line) - line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % self._temp_pipe, line) - line = re.sub("(PLATFORM_DATA_DIR=)(.*)", r'\1%s' % self.temp_dir, line) - - if string.strip(line) == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}": - line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join(self.bin_dir, "..")) + line = re.sub("(RUNNER_LOG_DIR=)(.*)", r'\1%s' % + self._temp_log, line) + line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % + self._temp_pipe, line) + line = re.sub("(PLATFORM_DATA_DIR=)(.*)", r'\1%s' % + self.temp_dir, line) + + if (string.strip(line) == + "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}"): + line = ("RUNNER_BASE_DIR=%s\n" % + os.path.normpath(os.path.join(self.bin_dir, + ".."))) temp_bin_file.write(line) os.fchmod(temp_bin_file.fileno(), 0755) def write_vm_args(self): - with open(os.path.join(self._temp_etc, "vm.args"), 'wb') as vm_args: + with open(self._vm_args_path(), 'wb') as vm_args: for arg, value in self.vm_args.items(): vm_args.write("%s %s\n" % (arg, value)) def write_app_config(self): - with open(os.path.join(self._temp_etc, "app.config"), "wb") as app_config: + with open(self._app_config(), "wb") as app_config: app_config.write(erlang_config(self.app_config)) app_config.write(".") + def _kv_backend(self): + return self.app_config["riak_kv"]["storage_backend"] + + def _http_ip(self): + return self.app_config["riak_core"]["web_ip"] + + def _http_port(self): + return self.app_config["riak_core"]["web_port"] + + def _app_config_path(self): + return os.path.join(self._temp_etc, "app.config") + + def _vm_args_path(self): + return os.path.join(self._temp_etc, "vm.args") + if __name__ == "__main__": server = TestServer() diff --git a/riak/tests/suite.py b/riak/tests/suite.py index 2e4735b7..af74197a 100644 --- a/riak/tests/suite.py +++ b/riak/tests/suite.py @@ -7,6 +7,7 @@ else: import unittest + def additional_tests(): top_level = os.path.join(os.path.dirname(__file__), "../../") start_dir = os.path.dirname(__file__) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 7ae5dafc..af15ed84 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -75,6 +75,7 @@ def __eq__(self, other): return False return True + class BaseTestCase(object): @staticmethod @@ -85,7 +86,8 @@ def create_client(self, host=None, port=None, transport_class=None): host = host or self.host port = port or self.port transport_class = transport_class or self.transport_class - return RiakClient(self.host, self.port, transport_class=self.transport_class) + return RiakClient(self.host, self.port, + transport_class=self.transport_class) def setUp(self): self.client = self.create_client() @@ -160,7 +162,7 @@ def test_custom_bucket_encoder_decoder(self): bucket = self.client.bucket("picklin_bucket") bucket.set_encoder('application/x-pickle', cPickle.dumps) bucket.set_decoder('application/x-pickle', cPickle.loads) - data = {'array':[1, 2, 3], 'badforjson': NotJsonSerializable(1,3)} + data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} obj = bucket.new("foo", data, 'application/x-pickle').store() obj.store() obj2 = bucket.get("foo") @@ -171,7 +173,7 @@ def test_custom_client_encoder_decoder(self): bucket = self.client.bucket("picklin_client") self.client.set_encoder('application/x-pickle', cPickle.dumps) self.client.set_decoder('application/x-pickle', cPickle.loads) - data = {'array':[1, 2, 3], 'badforjson':NotJsonSerializable(1,3)} + data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} obj = bucket.new("foo", data, 'application/x-pickle').store() obj.store() obj2 = bucket.get("foo") @@ -212,7 +214,7 @@ def test_set_bucket_properties(self): bucket.set_n_val(3) self.assertEqual(bucket.get_n_val(), 3) # Test setting multiple properties... - bucket.set_properties({"allow_mult":False, "n_val":2}) + bucket.set_properties({"allow_mult": False, "n_val": 2}) self.assertFalse(bucket.get_allow_multiples()) self.assertEqual(bucket.get_n_val(), 2) @@ -419,7 +421,6 @@ def test_key_filters_f_chain(self): bucket.new("google-20110103", 2).store() bucket.new("yahoo-20090613", 3).store() - # compose a chain of key filters using f as the root of # two filters ANDed together to ensure that f can be the root # of multiple chains @@ -434,12 +435,12 @@ def test_key_filters_f_chain(self): self.assertEqual(result, ["yahoo-20090613"]) - def test_key_filters_with_search_query(self): - mapreduce = self.client \ - .search("kftest", "query") - self.assertRaises(Exception, mapreduce.add_key_filters, [["tokenize", "-", 2]]) - self.assertRaises(Exception, mapreduce.add_key_filter, "ends_with", "0613") + mapreduce = self.client.search("kftest", "query") + self.assertRaises(Exception, mapreduce.add_key_filters, + [["tokenize", "-", 2]]) + self.assertRaises(Exception, mapreduce.add_key_filter, + "ends_with", "0613") def test_erlang_map_reduce(self): # Create the object... @@ -530,9 +531,9 @@ def test_store_of_missing_object(self): # for json objects o = bucket.get("nonexistent_key_json") self.assertEqual(o.exists(), False) - o.set_data({"foo" : "bar"}) + o.set_data({"foo": "bar"}) o = o.store() - self.assertEqual(o.get_data(), {"foo" : "bar"}) + self.assertEqual(o.get_data(), {"foo": "bar"}) self.assertEqual(o.get_content_type(), "application/json") o.delete() # for binary objects @@ -562,34 +563,41 @@ def test_solr_search_with_params_from_bucket(self): def test_solr_search_with_params(self): bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr().search("searchbucket", "username:roidrage", wt="xml") + results = self.client.solr().search("searchbucket", + "username:roidrage", wt="xml") self.assertEquals(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search(self): bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr().search("searchbucket", "username:roidrage") + results = self.client.solr().search("searchbucket", + "username:roidrage") self.assertEquals(1, len(results["docs"])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_search_integration(self): # Create some objects to search across... bucket = self.client.bucket("searchbucket") - bucket.new("one", {"foo":"one", "bar":"red"}).store() - bucket.new("two", {"foo":"two", "bar":"green"}).store() - bucket.new("three", {"foo":"three", "bar":"blue"}).store() - bucket.new("four", {"foo":"four", "bar":"orange"}).store() - bucket.new("five", {"foo":"five", "bar":"yellow"}).store() + bucket.new("one", {"foo": "one", "bar": "red"}).store() + bucket.new("two", {"foo": "two", "bar": "green"}).store() + bucket.new("three", {"foo": "three", "bar": "blue"}).store() + bucket.new("four", {"foo": "four", "bar": "orange"}).store() + bucket.new("five", {"foo": "five", "bar": "yellow"}).store() # Run some operations... - results = self.client.solr().search("searchbucket", "foo:one OR foo:two") + results = self.client.solr().search("searchbucket", + "foo:one OR foo:two") if (len(results) == 0): print "\n\nNot running test \"testSearchIntegration()\".\n" - print "Please ensure that you have installed the Riak Search hook on bucket \"searchbucket\" by running \"bin/search-cmd install searchbucket\".\n\n" + print """Please ensure that you have installed the Riak + Search hook on bucket \"searchbucket\" by running + \"bin/search-cmd install searchbucket\".\n\n""" return self.assertEqual(len(results['docs']), 2) - results = self.client.solr().search("searchbucket", "(foo:one OR foo:two OR foo:three OR foo:four) AND (NOT bar:green)") + query = "(foo:one OR foo:two OR foo:three OR foo:four) AND\ + (NOT bar:green)" + results = self.client.solr().search("searchbucket", query) self.assertEqual(len(results['docs']), 3) def test_store_binary_object_from_file(self): @@ -624,25 +632,26 @@ def test_store_metadata(self): def test_store_binary_object_from_file_should_fail_if_file_not_found(self): bucket = self.client.bucket('bucket') rand = str(self.randint()) - self.assertRaises(IOError, bucket.new_binary_from_file, 'not_found_from_file', 'FILE_NOT_FOUND') + self.assertRaises(IOError, bucket.new_binary_from_file, + 'not_found_from_file', 'FILE_NOT_FOUND') obj = bucket.get_binary('not_found_from_file') self.assertEqual(obj.get_data(), None) def test_list_buckets(self): bucket = self.client.bucket("list_bucket") - bucket.new("one", {"foo":"one", "bar":"red"}).store() + bucket.new("one", {"foo": "one", "bar": "red"}).store() buckets = self.client.get_buckets() self.assertTrue("list_bucket" in buckets) def is_2i_supported(self): # Immediate test to see if 2i is even supported w/ the backend try: - self.client.index('foo','bar_bin','baz').run() + self.client.index('foo', 'bar_bin', 'baz').run() return True except Exception as e: if "indexes_not_supported" in str(e): return False - return True # it failed, but is supported! + return True # it failed, but is supported! @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_store(self): @@ -669,8 +678,10 @@ def test_secondary_index_store(self): # Retrieve the object, check that the correct indexes exist... obj = bucket.get('mykey1') - self.assertEqual(['val1a', 'val1b'], sorted(obj.get_indexes('field1_bin'))) - self.assertEqual(['1011', '1012'], sorted(obj.get_indexes('field1_int'))) + self.assertEqual(['val1a', 'val1b'], + sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1011', '1012'], + sorted(obj.get_indexes('field1_int'))) # Check the get_indexes() function... self.assertEqual([ @@ -740,7 +751,8 @@ def test_remove_indexes(self): return True bucket = self.client.bucket('indexbucket') - bar = bucket.new('bar', 1).add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + bar = bucket.new('bar', 1).add_index('bar_int', 1)\ + .add_index('bar_int', 2).add_index('baz_bin', 'baz').store() result = bucket.get_index('bar_int', 1) self.assertEqual(1, len(result)) self.assertEqual(3, len(bar.get_indexes())) @@ -757,7 +769,8 @@ def test_remove_indexes(self): self.assertEqual(0, len(bar.get_indexes('baz_bin'))) # add index again - bar = bar.add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + bar = bar.add_index('bar_int', 1).add_index('bar_int', 2)\ + .add_index('baz_bin', 'baz').store() # remove all index with field='bar_int' bar = bar.remove_index(field='bar_int').store() result = bucket.get_index('bar_int', 1) @@ -771,7 +784,8 @@ def test_remove_indexes(self): self.assertEqual(1, len(bar.get_indexes('baz_bin'))) # add index again - bar = bar.add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + bar = bar.add_index('bar_int', 1).add_index('bar_int', 2)\ + .add_index('baz_bin', 'baz').store() # remove an index field value pair bar = bar.remove_index(field='bar_int', value=2).store() result = bucket.get_index('bar_int', 1) @@ -819,7 +833,7 @@ def test_secondary_index_query(self): # Test a range query... results = bucket.get_index('field1_bin', 'val2', 'val4') - vals = set([ str(key) for key in results ]) + vals = set([str(key) for key in results]) self.assertEquals(3, len(results)) self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) @@ -830,7 +844,7 @@ def test_secondary_index_query(self): # Test a range query... results = bucket.get_index('field2_int', 1002, 1004) - vals = set([str(key) for key in results ]) + vals = set([str(key) for key in results]) self.assertEquals(3, len(results)) self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) @@ -840,6 +854,7 @@ def test_secondary_index_query(self): bucket.get('mykey3').delete() bucket.get('mykey4').delete() + class MapReduceAliasTestMixIn(object): """This tests the map reduce aliases""" @@ -939,7 +954,7 @@ def test_reduce_sort(self): # Use the map_values alias result = mr.map_values_json().reduce_sort().run() - self.assertEqual(result, ["value1","value2"]) + self.assertEqual(result, ["value1", "value2"]) def test_reduce_sort_custom(self): # Add a value to the bucket @@ -957,7 +972,7 @@ def test_reduce_sort_custom(self): return x > y ? -1 : 1; }""").run() - self.assertEqual(result, ["value2","value1"]) + self.assertEqual(result, ["value2", "value1"]) def test_reduce_numeric_sort(self): # Add a value to the bucket @@ -972,7 +987,7 @@ def test_reduce_numeric_sort(self): # Use the map_values alias result = mr.map_values_json().reduce_numeric_sort().run() - self.assertEqual(result, [1,2]) + self.assertEqual(result, [1, 2]) def test_reduce_limit(self): # Add a value to the bucket @@ -1004,7 +1019,7 @@ def test_reduce_slice(self): # Use the map_values alias result = mr.map_values_json()\ .reduce_numeric_sort()\ - .reduce_slice(1,2).run() + .reduce_slice(1, 2).run() self.assertEqual(result, [2]) @@ -1027,7 +1042,7 @@ def test_filter_not_found(self): .filter_not_found()\ .run() - self.assertEqual(sorted(result), [1,2]) + self.assertEqual(sorted(result), [1, 2]) class RiakPbcTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, @@ -1035,7 +1050,7 @@ class RiakPbcTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, def setUp(self): if not HAVE_PROTO: - self.skipTest('protobuf is unavailable') + self.skipTest('protobuf is unavailable') self.host = PB_HOST self.port = PB_PORT self.transport_class = RiakPbcTransport @@ -1046,12 +1061,12 @@ def test_uses_client_id_if_given(self): self.port = PB_PORT zero_client_id = "\0\0\0\0" c = RiakClient(PB_HOST, PB_PORT, - transport_class = RiakPbcTransport, - client_id = zero_client_id) - self.assertEqual(zero_client_id, c.get_client_id()) # + transport_class=RiakPbcTransport, + client_id=zero_client_id) + self.assertEqual(zero_client_id, c.get_client_id()) def test_close_underlying_socket_fails(self): - c = RiakClient(PB_HOST, PB_PORT, transport_class = RiakPbcTransport) + c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport) bucket = c.bucket('bucket_test_close') rand = self.randint() @@ -1100,7 +1115,8 @@ def test_close_underlying_socket_retry(self): self.assertEqual(obj.get_data(), rand) -class RiakHttpTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): +class RiakHttpTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, + unittest.TestCase): def setUp(self): self.host = HTTP_HOST @@ -1136,9 +1152,11 @@ def test_enable_search_commit_hook(self): def test_disable_search_commit_hook(self): bucket = self.client.bucket("no_search_bucket") bucket.enable_search() - self.assertTrue(self.client.bucket("no_search_bucket").search_enabled()) + self.assertTrue(self.client.bucket("no_search_bucket")\ + .search_enabled()) bucket.disable_search() - self.assertFalse(self.client.bucket("no_search_bucket").search_enabled()) + self.assertFalse(self.client.bucket("no_search_bucket")\ + .search_enabled()) @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_store_file_with_luwak(self): @@ -1177,39 +1195,61 @@ def test_delete_file_with_luwak(self): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_document_to_index(self): - self.client.solr().add("searchbucket", {"id": "doc", "username": "tony"}) + self.client.solr().add("searchbucket", + {"id": "doc", "username": "tony"}) results = self.client.solr().search("searchbucket", "username:tony") self.assertEquals("tony", results['docs'][0]['username']) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_multiple_documents_to_index(self): - self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") + self.client.solr().add("searchbucket", + {"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}) + results = self.client.solr()\ + .search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(2, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_id(self): - self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) + self.client.solr().add("searchbucket", + {"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", docs=["dizzy"]) - results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") + results = self.client.solr()\ + .search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): - self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - self.client.solr().delete("searchbucket", queries=["username:dizzy", "username:russell"]) - results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") + self.client.solr().add("searchbucket", + {"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}) + self.client.solr()\ + .delete("searchbucket", + queries=["username:dizzy", "username:russell"]) + results = self.client.solr()\ + .search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query_and_id(self): - self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - self.client.solr().delete("searchbucket", docs=["dizzy"], queries=["username:russell"]) - results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") + self.client.solr().add("searchbucket", + {"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}) + self.client.solr().delete("searchbucket", + docs=["dizzy"], + queries=["username:russell"]) + results = self.client.solr()\ + .search("searchbucket", + "username:russell OR username:dizzy") self.assertEquals(0, len(results['docs'])) def test_build_rest_path_excludes_empty_query_params(self): - self.assertEquals(self.client.get_transport().build_rest_path(bucket=self.client.bucket("foo"), key="bar", params={'r': None}), "/riak/foo/bar?") + self.assertEquals( + self.client.get_transport().build_rest_path( + bucket=self.client.bucket("foo"), + key="bar", params={'r': None}), "/riak/foo/bar?") + class RiakTestFilter(unittest.TestCase): def test_simple(self): @@ -1226,7 +1266,10 @@ def test_and(self): f1 = RiakKeyFilter("starts_with", "2005-") f2 = RiakKeyFilter("ends_with", "-01") f3 = f1 & f2 - self.assertEqual(list(f3), [["and", [["starts_with", "2005-"]], [["ends_with", "-01"]]]]) + self.assertEqual(list(f3), + [["and", + [["starts_with", "2005-"]], + [["ends_with", "-01"]]]]) def test_multi_and(self): f1 = RiakKeyFilter("starts_with", "2005-") diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py index dac59697..a1a7ff29 100644 --- a/riak/tests/test_feature_detection.py +++ b/riak/tests/test_feature_detection.py @@ -25,9 +25,11 @@ from riak.transports.feature_detect import FeatureDetection + class IncompleteTransport(FeatureDetection): pass + class DummyTransport(FeatureDetection): def __init__(self, version): self._version = version @@ -35,9 +37,11 @@ def __init__(self, version): def _server_version(self): return self._version + class FeatureDetectionTest(unittest.TestCase): def test_implements_server_version(self): t = IncompleteTransport() + def get_server_version(): t.server_version self.assertRaises(NotImplementedError, get_server_version) diff --git a/riak/tests/test_server_test.py b/riak/tests/test_server_test.py index 1c1307cb..adca451f 100644 --- a/riak/tests/test_server_test.py +++ b/riak/tests/test_server_test.py @@ -1,6 +1,7 @@ from riak.test_server import TestServer import unittest + class TestServerTestCase(unittest.TestCase): def setUp(self): self.test_server = TestServer() @@ -9,33 +10,41 @@ def tearDown(self): pass def test_options_defaults(self): - self.assertEquals(self.test_server.app_config["riak_core"]["handoff_port"], 9001) - self.assertEquals(self.test_server.app_config["riak_kv"]["pb_ip"], "127.0.0.1") + self.assertEquals( + self.test_server.app_config["riak_core"]["handoff_port"], 9001) + self.assertEquals( + self.test_server.app_config["riak_kv"]["pb_ip"], "127.0.0.1") def test_merge_riak_core_options(self): self.test_server = TestServer(riak_core={"handoff_port": 10000}) - self.assertEquals(self.test_server.app_config["riak_core"]["handoff_port"], 10000) + self.assertEquals( + self.test_server.app_config["riak_core"]["handoff_port"], 10000) def test_merge_luwak_options(self): self.test_server = TestServer(luwak={"enabled": False}) - self.assertEquals(self.test_server.app_config["luwak"]["enabled"], False) + self.assertEquals( + self.test_server.app_config["luwak"]["enabled"], False) def test_merge_riak_search_options(self): - self.test_server = TestServer(riak_search={"search_backend": "riak_search_backend"}) - self.assertEquals(self.test_server.app_config["riak_search"]["search_backend"], - "riak_search_backend") + self.test_server = TestServer( + riak_search={"search_backend": "riak_search_backend"}) + self.assertEquals( + self.test_server.app_config["riak_search"]["search_backend"], + "riak_search_backend") def test_merge_riak_kv_options(self): self.test_server = TestServer(riak_kv={"pb_ip": "192.168.2.1"}) - self.assertEquals(self.test_server.app_config["riak_kv"]["pb_ip"], "192.168.2.1") + self.assertEquals(self.test_server.app_config["riak_kv"]["pb_ip"], + "192.168.2.1") def test_merge_vmargs(self): self.test_server = TestServer(vm_args={"-P": 65000}) self.assertEquals(self.test_server.vm_args["-P"], 65000) def test_set_ring_state_dir(self): - self.assertEquals(self.test_server.app_config["riak_core"]["ring_state_dir"], - "/tmp/riak/test_server/data/ring") + self.assertEquals( + self.test_server.app_config["riak_core"]["ring_state_dir"], + "/tmp/riak/test_server/data/ring") def test_set_default_tmp_dir(self): self.assertEquals(self.test_server.temp_dir, "/tmp/riak/test_server") @@ -45,9 +54,8 @@ def test_set_non_default_tmp_dir(self): server = TestServer(tmp_dir=tmp_dir) self.assertEquals(server.temp_dir, tmp_dir) + def suite(): suite = unittest.TestSuite() suite.addTest(TestServerTestCase()) return suite - - diff --git a/riak/transports/connection.py b/riak/transports/connection.py index 12172df4..d26b85f4 100644 --- a/riak/transports/connection.py +++ b/riak/transports/connection.py @@ -21,6 +21,7 @@ import contextlib import functools + class ConnectionManager(object): # Must be constructable with: connection_class(host, port) @@ -66,12 +67,13 @@ def remove_host(self, host, port=None): try: self.conns.remove(conn) except ValueError: - # Another thread removed the connection. It won't be coming back, - # so we have nothing to do here. + # Another thread removed the connection. It won't + # be coming back, so we have nothing to do here. pass else: - # If the connection was still present (no ValueError), then we - # should go ahead and close it down. + # If the connection was still present (no + # ValueError), then we should go ahead and close + # it down. conn.close() # Just in case somebody uses a host/port combo and typos... @@ -125,22 +127,24 @@ def _new_connection(self): # No rotation needed. return conn - # Be careful about rotating. We want to append before removing, so that - # we never hit a len==0 race condition (which could prevent the creation - # of needed connections). + # Be careful about rotating. We want to append before + # removing, so that we never hit a len==0 race condition + # (which could prevent the creation of needed connections). self.hostports.append((host, port)) - # RACE: another thread may have appended the same host/port pair. We - # will add another pair. Each thread will remove one (either [0], or - # one that had been appened), resulting in a correct state of a single - # pair in the list. - # RACE: another thread may get the host/port pair from hostports[0] - # before we have a chance to remove it. We don't need precision - # round-robin behavior; just something close. - # RACE: another thread may have removed hostports[0] (which we are - # also trying to remove), but it will have placed another copy at - # the end before doing so. We have also added a host/port pair, and - # will remove one, leaving the list in a correct state. + # RACE: another thread may have appended the same host/port + # pair. We will add another pair. Each thread will remove + # one (either [0], or one that had been appened), resulting + # in a correct state of a single pair in the list. + # RACE: another thread may get the host/port pair from + # hostports[0] before we have a chance to remove it. We + # don't need precision round-robin behavior; just something + # close. + # RACE: another thread may have removed hostports[0] (which we + # are also trying to remove), but it will have placed + # another copy at the end before doing so. We have also + # added a host/port pair, and will remove one, leaving the + # list in a correct state. self.hostports.remove((host, port)) return conn diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 952cc8a8..07584472 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -19,12 +19,14 @@ from distutils.version import StrictVersion from riak.util import lazy_property + versions = { 1: StrictVersion("1.0.0"), 1.1: StrictVersion("1.1.0"), 1.2: StrictVersion("1.2.0") } + class FeatureDetection(object): def _server_version(self): """ diff --git a/riak/transports/http.py b/riak/transports/http.py index d5150706..17f5fef7 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -19,7 +19,9 @@ """ from __future__ import with_statement -import urllib, re, csv +import urllib +import re +import csv from cStringIO import StringIO import httplib import socket @@ -39,12 +41,14 @@ import riak.util from xml.etree import ElementTree -MAX_LINK_HEADER_SIZE = 8192 - 8 # substract length of "Link: " header string and newline +# subtract length of "Link: " header string and newline +MAX_LINK_HEADER_SIZE = 8192 - 8 -class RiakHttpTransport(RiakTransport) : + +class RiakHttpTransport(RiakTransport): """ - The RiakHttpTransport object holds information necessary to connect to - Riak. The Riak API uses HTTP, so there is no persistent + The RiakHttpTransport object holds information necessary to + connect to Riak. The Riak API uses HTTP, so there is no persistent connection, and the RiakClient object is extremely lightweight. """ @@ -89,7 +93,7 @@ def set_client_id(self, client_id): def get_client_id(self): return self._client_id - def ping(self) : + def ping(self): """ Check server is alive over HTTP """ @@ -101,7 +105,8 @@ def stats(self): Gets performance statistics and server information """ # TODO: use resource detection - response = self.http_request('GET', '/stats', {'Accept':'application/json'}) + response = self.http_request('GET', '/stats', + {'Accept': 'application/json'}) if response[0]['http_code'] is 200: return json.loads(response[1]) else: @@ -125,19 +130,20 @@ def get_resources(self): Gets a JSON mapping of server-side resource names to paths :rtype dict """ - response = self.http_request('GET', '/', {'Accept':'application/json'}) + response = self.http_request('GET', '/', + {'Accept': 'application/json'}) if response[0]['http_status'] is 200: return json.loads(response[1]) else: return {} - def get(self, robj, r = None, pr = None, vtag = None) : + def get(self, robj, r=None, pr=None, vtag=None): """ Get a bucket/key from the server """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'r' : r, 'pr': pr} + params = {'r': r, 'pr': pr} if vtag is not None: params['vtag'] = vtag url = self.build_rest_path(robj.get_bucket(), robj.get_key(), @@ -145,14 +151,17 @@ def get(self, robj, r = None, pr = None, vtag = None) : response = self.http_request('GET', url) return self.parse_body(response, [200, 300, 404]) - def put(self, robj, w = None, dw = None, pw = None, return_body = True, if_none_match=False): + def put(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): """ Serialize put request and deserialize response """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw, 'pw' : pw } - url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), + params = {'returnbody': str(return_body).lower(), + 'w': w, 'dw': dw, 'pw': pw} + url = self.build_rest_path(bucket=robj.get_bucket(), + key=robj.get_key(), params=params) headers = self.build_put_headers(robj) # TODO: use a more general 'prevent_stale_writes' semantics, @@ -160,25 +169,28 @@ def put(self, robj, w = None, dw = None, pw = None, return_body = True, if_none_ if if_none_match: headers["If-None-Match"] = "*" content = robj.get_encoded_data() - return self.do_put(url, headers, content, return_body, key=robj.get_key()) + return self.do_put(url, headers, content, return_body, + key=robj.get_key()) def do_put(self, url, headers, content, return_body=False, key=None): if key is None: - response = self.http_request('POST', url, headers, content) + response = self.http_request('POST', url, headers, content) else: - response = self.http_request('PUT', url, headers, content) + response = self.http_request('PUT', url, headers, content) if return_body: - return self.parse_body(response, [200, 201, 300]) + return self.parse_body(response, [200, 201, 300]) else: - self.check_http_code(response, [204]) - return None + self.check_http_code(response, [204]) + return None - def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): + def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): """Put a new object into the Riak store, returning its (new) key.""" # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw, 'pw' : pw} + params = {'returnbody': str(return_body).lower(), 'w': w, 'dw': dw, + 'pw': pw} url = self.build_rest_path(bucket=robj.get_bucket(), params=params) headers = self.build_put_headers(robj) # TODO: use a more general 'prevent_stale_writes' semantics, @@ -189,7 +201,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_matc response = self.http_request('POST', url, headers, content) location = response[0]['location'] idx = location.rindex('/') - key = location[idx+1:] + key = location[(idx + 1):] if return_body: vclock, [(metadata, data)] = self.parse_body(response, [201]) return key, vclock, metadata @@ -197,13 +209,13 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_matc self.check_http_code(response, [201]) return key, None, None - def delete(self, robj, rw=None, r = None, w = None, dw = None, pr = None, pw = None): + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Delete an object. """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'rw' : rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} + params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} headers = {} url = self.build_rest_path(robj.get_bucket(), robj.get_key(), params=params) @@ -217,7 +229,7 @@ def get_keys(self, bucket): """ Fetch a list of keys for the bucket """ - params = {'props' : 'True', 'keys' : 'true'} + params = {'props': 'True', 'keys': 'true'} url = self.build_rest_path(bucket, params=params) response = self.http_request('GET', url) @@ -248,7 +260,7 @@ def get_bucket_props(self, bucket): Get properties for a bucket """ # Run the request... - params = {'props' : 'True', 'keys' : 'False'} + params = {'props': 'true', 'keys': 'false'} url = self.build_rest_path(bucket, params=params) response = self.http_request('GET', url) @@ -265,8 +277,8 @@ def set_bucket_props(self, bucket, props): Set the properties on the bucket object given """ url = self.build_rest_path(bucket) - headers = {'Content-Type' : 'application/json'} - content = json.dumps({'props' : props}) + headers = {'Content-Type': 'application/json'} + content = json.dumps({'props': props}) # Run the request... response = self.http_request('PUT', url, headers, content) @@ -286,10 +298,11 @@ def mapred(self, inputs, query, timeout=None): Run a MapReduce query. """ if not self.phaseless_mapred() and (query is None or len(query) is 0): - raise Exception('Phase-less MapReduce is not supported by this Riak node') + raise Exception( + 'Phase-less MapReduce is not supported by Riak node') # Construct the job, optionally set the timeout... - job = {'inputs':inputs, 'query':query} + job = {'inputs': inputs, 'query': query} if timeout is not None: job['timeout'] = timeout @@ -303,8 +316,9 @@ def mapred(self, inputs, query, timeout=None): # Make sure the expected status code came back... status = response[0]['http_code'] if status != 200: - raise Exception('Error running MapReduce operation. Headers: %s Body: %s' % - (repr(response[0]),repr(response[1]))) + raise Exception( + 'Error running MapReduce operation. Headers: %s Body: %s' % + (repr(response[0]), repr(response[1]))) result = json.loads(response[1]) return result @@ -330,7 +344,7 @@ def search(self, index, query, **params): if index is None: index = 'search' - options = {'q':query, 'wt':'json'} + options = {'q': query, 'wt': 'json'} if 'op' in params: op = params.pop('op') options['q.op'] = op @@ -351,8 +365,8 @@ def search(self, index, query, **params): def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] if not status in expected_statuses: - m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) + ' : ' + response[1] - raise Exception(m) + raise Exception('Expected status %s, received %s : %s' % + (expected_statuses, status, response[1])) def parse_body(self, response, expected_statuses): """ @@ -403,12 +417,13 @@ def parse_body(self, response, expected_statuses): metadata[MD_CTYPE] = value elif header == 'etag': metadata[MD_VTAG] = value - elif header =='link': + elif header == 'link': self.parse_links(links, headers['link']) elif header == 'last-modified': metadata[MD_LASTMOD] = value elif header.startswith('x-riak-meta-'): - metadata[MD_USERMETA][header.replace('x-riak-meta-', '')] = value + metakey = header.replace('x-riak-meta-', '') + metadata[MD_USERMETA][metakey] = value elif header.startswith('x-riak-index-'): field = header.replace('x-riak-index-', '') reader = csv.reader([value], skipinitialspace=True) @@ -442,10 +457,12 @@ def parse_links(self, links, linkHeaders): Private. @return self """ + oldform = "; ?riaktag=\"([^\']+)\"" + newform = "; ?riaktag=\"([^\']+)\"" for linkHeader in linkHeaders.strip().split(','): linkHeader = linkHeader.strip() - matches = re.match("; ?riaktag=\"([^\']+)\"", linkHeader) or \ - re.match("; ?riaktag=\"([^\']+)\"", linkHeader) + matches = (re.match(oldform, linkHeader) or + re.match(newform, linkHeader)) if matches is not None: link = RiakLink(urllib.unquote_plus(matches.group(2)), urllib.unquote_plus(matches.group(3)), @@ -463,7 +480,8 @@ def add_links_for_riak_object(self, robject, headers): headers.add('Link', current_header) current_header = '' - if current_header != '': header = ', ' + header + if current_header != '': + header = ', ' + header current_header += header headers.add('Link', current_header) @@ -474,10 +492,11 @@ def get_request(self, uri=None, params=None): url = self.build_rest_path(bucket=None, params=params, prefix=uri) return self.http_request('GET', url) - def store_file(self, key, content_type="application/octet-stream", content=None): + def store_file(self, key, content_type="application/octet-stream", + content=None): url = self.build_rest_path(prefix='luwak', key=key) - headers = {'Content-Type' : content_type, - 'X-Riak-ClientId' : self._client_id} + headers = {'Content-Type': content_type, + 'X-Riak-ClientId': self._client_id} return self.do_put(url, headers, content, key=key) @@ -495,13 +514,15 @@ def delete_file(self, key): response = self.http_request('DELETE', url) self.parse_body(response, [204, 404]) - def post_request(self, uri=None, body=None, params=None, content_type="application/json"): + def post_request(self, uri=None, body=None, params=None, + content_type="application/json"): uri = self.build_rest_path(prefix=uri, params=params) - return self.http_request('POST', uri, {'Content-Type': content_type}, body) + return self.http_request('POST', uri, {'Content-Type': content_type}, + body) # Utility functions used by Riak library. - def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : + def build_rest_path(self, bucket=None, key=None, params=None, prefix=None): """ Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, construct and return a URL. @@ -523,8 +544,10 @@ def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : s = '' for key in params.keys(): if params[key] is not None: - if s != '': s += '&' - s += urllib.quote_plus(key) + '=' + urllib.quote_plus(str(params[key])) + if s != '': + s += '&' + s += (urllib.quote_plus(key) + '=' + + urllib.quote_plus(str(params[key]))) path += '?' + s # Return. @@ -534,9 +557,9 @@ def build_put_headers(self, robj): """Build the headers for a POST/PUT request.""" # Construct the headers... - headers = MultiDict({'Accept' : 'text/plain, */*; q=0.5', - 'Content-Type' : robj.get_content_type(), - 'X-Riak-ClientId' : self._client_id}) + headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', + 'Content-Type': robj.get_content_type(), + 'X-Riak-ClientId': self._client_id}) # Add the vclock if it exists... if robj.vclock() is not None: @@ -557,7 +580,7 @@ def build_put_headers(self, robj): return headers - def http_request(self, method, uri, headers=None, body='') : + def http_request(self, method, uri, headers=None, body=''): """ Given a Method, URL, Headers, and Body, perform and HTTP request, and return a 2-tuple containing a dictionary of response headers @@ -583,7 +606,7 @@ def http_request(self, method, uri, headers=None, body='') : # Get the body... response_body = response.read() finally: - response.close() + response.close() return response_headers, response_body except socket.error, e: @@ -626,16 +649,17 @@ def _normalize_xml_search_response(self, xml): same return value """ target = XMLSearchResult() - parser = ElementTree.XMLParser(target = target) + parser = ElementTree.XMLParser(target=target) parser.feed(xml) return parser.close() @classmethod def build_headers(cls, headers): - return ['%s: %s' % (header, value) for header, value in headers.iteritems()] + return ['%s: %s' % (header, value) + for header, value in headers.iteritems()] @classmethod - def parse_http_headers(cls, headers) : + def parse_http_headers(cls, headers): """ Parse an HTTP Header string into an asssociative array of response headers. @@ -644,7 +668,8 @@ def parse_http_headers(cls, headers) : fields = headers.split("\n") for field in fields: matches = re.match("([^:]+):(.+)", field) - if matches is None: continue + if matches is None: + continue key = matches.group(1).lower() value = matches.group(2).strip() if key in retVal.keys(): @@ -656,6 +681,7 @@ def parse_http_headers(cls, headers) : retVal[key] = value return retVal + class XMLSearchResult(object): # Match tags that are document fields fieldtags = ['str', 'int', 'date'] @@ -701,6 +727,6 @@ def data(self, data): self.currvalue = data def close(self): - return {'num_found':self.num_found, - 'max_score':self.max_score, - 'docs': self.docs } + return {'num_found': self.num_found, + 'max_score': self.max_score, + 'docs': self.docs} diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index bb08b462..19afe85f 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -52,35 +52,35 @@ riak_pb = None ## Protocol codes -MSG_CODE_ERROR_RESP = 0 -MSG_CODE_PING_REQ = 1 -MSG_CODE_PING_RESP = 2 -MSG_CODE_GET_CLIENT_ID_REQ = 3 -MSG_CODE_GET_CLIENT_ID_RESP = 4 -MSG_CODE_SET_CLIENT_ID_REQ = 5 -MSG_CODE_SET_CLIENT_ID_RESP = 6 -MSG_CODE_GET_SERVER_INFO_REQ = 7 -MSG_CODE_GET_SERVER_INFO_RESP = 8 -MSG_CODE_GET_REQ = 9 -MSG_CODE_GET_RESP = 10 -MSG_CODE_PUT_REQ = 11 -MSG_CODE_PUT_RESP = 12 -MSG_CODE_DEL_REQ = 13 -MSG_CODE_DEL_RESP = 14 -MSG_CODE_LIST_BUCKETS_REQ = 15 -MSG_CODE_LIST_BUCKETS_RESP = 16 -MSG_CODE_LIST_KEYS_REQ = 17 -MSG_CODE_LIST_KEYS_RESP = 18 -MSG_CODE_GET_BUCKET_REQ = 19 -MSG_CODE_GET_BUCKET_RESP = 20 -MSG_CODE_SET_BUCKET_REQ = 21 -MSG_CODE_SET_BUCKET_RESP = 22 -MSG_CODE_MAPRED_REQ = 23 -MSG_CODE_MAPRED_RESP = 24 -MSG_CODE_INDEX_REQ = 25 -MSG_CODE_INDEX_RESP = 26 -MSG_CODE_SEARCH_QUERY_REQ = 27 -MSG_CODE_SEARCH_QUERY_RESP = 28 +MSG_CODE_ERROR_RESP = 0 +MSG_CODE_PING_REQ = 1 +MSG_CODE_PING_RESP = 2 +MSG_CODE_GET_CLIENT_ID_REQ = 3 +MSG_CODE_GET_CLIENT_ID_RESP = 4 +MSG_CODE_SET_CLIENT_ID_REQ = 5 +MSG_CODE_SET_CLIENT_ID_RESP = 6 +MSG_CODE_GET_SERVER_INFO_REQ = 7 +MSG_CODE_GET_SERVER_INFO_RESP = 8 +MSG_CODE_GET_REQ = 9 +MSG_CODE_GET_RESP = 10 +MSG_CODE_PUT_REQ = 11 +MSG_CODE_PUT_RESP = 12 +MSG_CODE_DEL_REQ = 13 +MSG_CODE_DEL_RESP = 14 +MSG_CODE_LIST_BUCKETS_REQ = 15 +MSG_CODE_LIST_BUCKETS_RESP = 16 +MSG_CODE_LIST_KEYS_REQ = 17 +MSG_CODE_LIST_KEYS_RESP = 18 +MSG_CODE_GET_BUCKET_REQ = 19 +MSG_CODE_GET_BUCKET_RESP = 20 +MSG_CODE_SET_BUCKET_REQ = 21 +MSG_CODE_SET_BUCKET_RESP = 22 +MSG_CODE_MAPRED_REQ = 23 +MSG_CODE_MAPRED_RESP = 24 +MSG_CODE_INDEX_REQ = 25 +MSG_CODE_INDEX_RESP = 26 +MSG_CODE_SEARCH_QUERY_REQ = 27 +MSG_CODE_SEARCH_QUERY_RESP = 28 RIAKC_RW_ONE = 4294967294 RIAKC_RW_QUORUM = 4294967293 @@ -143,8 +143,8 @@ def recv(self, want_len): class RiakPbcTransport(RiakTransport): """ - The RiakPbcTransport object holds a connection to the protocol buffers interface - on the riak server. + The RiakPbcTransport object holds a connection to the protocol + buffers interface on the riak server. """ # We're using the new RiakTransport API @@ -204,7 +204,7 @@ def get_server_info(self): """ msg_code, resp = self.send_msg_code(MSG_CODE_GET_SERVER_INFO_REQ, MSG_CODE_GET_SERVER_INFO_RESP) - return {'node':resp.node, 'server_version':resp.server_version} + return {'node': resp.node, 'server_version': resp.server_version} def get_client_id(self): """ @@ -267,7 +267,8 @@ def get(self, robj, r=None, pr=None, vtag=None): else: return None - def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): + def put(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): """ Serialize get request and deserialize response """ @@ -290,7 +291,9 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=Fa if vclock: req.vclock = vclock - self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) + self.pbify_content(robj.get_metadata(), + robj.get_encoded_data(), + req.content) msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req, MSG_CODE_PUT_RESP) @@ -300,7 +303,8 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=Fa contents.append(self.decode_content(c)) return resp.vclock, contents - def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): + def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): """Put a new object into the Riak store, returning its (new) key. If return_meta is False, then the vlock and metadata return values @@ -323,7 +327,9 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_matc req.bucket = bucket.get_name() - self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) + self.pbify_content(robj.get_metadata(), + robj.get_encoded_data(), + req.content) msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req, MSG_CODE_PUT_RESP) @@ -369,6 +375,7 @@ def get_keys(self, bucket): req.bucket = bucket.get_name() keys = [] + def _handle_response(resp): for key in resp.keys: keys.append(key) @@ -435,6 +442,7 @@ def mapred(self, inputs, query, timeout=None): # dictionary of phase results - each content should be an encoded array # which is appended to the result for that phase. result = {} + def _handle_response(resp): if resp.HasField("phase") and resp.HasField("response"): content = json.loads(resp.response) @@ -550,8 +558,9 @@ def send_pkt(self, conn, pkt): try: conn.maybe_connect() - # If the last client_id used on this connection is different than our - # client_id, then set a new ID on the connection. + # If the last client_id used on this connection is + # different than our client_id, then set a new ID on + # the connection. if conn.last_client_id != self._client_id: req = riak_pb.RpbSetClientIdReq() req.client_id = self._client_id @@ -626,8 +635,9 @@ def recv_msg(self, conn, expect): def recv_pkt(self, conn): nmsglen = conn.recv(4) if len(nmsglen) != 4: - raise RiakError("Socket returned short packet length %d - expected 4" - % len(nmsglen)) + raise RiakError( + "Socket returned short packet length %d - expected 4" + % len(nmsglen)) msglen, = struct.unpack('!i', nmsglen) self._inbuf_len = msglen self._inbuf = '' diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 4ac7786c..075274e6 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -25,6 +25,7 @@ import os from feature_detect import FeatureDetection + class RiakTransport(FeatureDetection): """ Class to encapsulate transport details @@ -61,14 +62,14 @@ def ping(self): """ raise RiakError("not implemented") - def get(self, robj, r = None, vtag = None): + def get(self, robj, r=None, vtag=None): """ Serialize get request and deserialize response @return (vclock=None, [(metadata, value)]=None) """ raise RiakError("not implemented") - def put(self, robj, w = None, dw = None, return_body = True): + def put(self, robj, w=None, dw=None, return_body=True): """ Serialize put request and deserialize response - if 'content' is true, retrieve the updated metadata/content @@ -86,28 +87,28 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): """ raise RiakError("not implemented") - def delete(self, robj, rw = None): + def delete(self, robj, rw=None): """ Serialize delete request and deserialize response @return true """ raise RiakError("not implemented") - def get_buckets(self) : + def get_buckets(self): """ Serialize get buckets request and deserialize response @return dict() """ raise RiakError("not implemented") - def get_bucket_props(self, bucket) : + def get_bucket_props(self, bucket): """ Serialize get bucket property request and deserialize response @return dict() """ raise RiakError("not implemented") - def set_bucket_props(self, bucket, props) : + def set_bucket_props(self, bucket, props): """ Serialize set bucket property request and deserialize response bucket = bucket object @@ -116,7 +117,7 @@ def set_bucket_props(self, bucket, props) : """ raise RiakError("not implemented") - def mapred(self, inputs, query, timeout = None) : + def mapred(self, inputs, query, timeout=None): """ Serialize map/reduce request """ @@ -124,8 +125,9 @@ def mapred(self, inputs, query, timeout = None) : def set_client_id(self, client_id): """ - Set the client id. This overrides the default, random client id, which is automatically - generated when none is specified in when creating the transport object. + Set the client id. This overrides the default, random client + id, which is automatically generated when none is specified in + when creating the transport object. """ raise RiakError("not implemented") @@ -155,13 +157,13 @@ def _search_mapred_emu(self, index, query): """ phases = [] if not self.phaseless_mapred(): - phases.append({'language':'erlang', - 'module':'riak_kv_mapreduce', - 'function':'reduce_identity', - 'keep':True}) - mr_result = self.mapred({'module':'riak_search', - 'function':'mapred_search', - 'arg':[index, query]}, + phases.append({'language': 'erlang', + 'module': 'riak_kv_mapreduce', + 'function': 'reduce_identity', + 'keep': True}) + mr_result = self.mapred({'module': 'riak_search', + 'function': 'mapred_search', + 'arg': [index, query]}, phases) result = {'num_found': len(mr_result), 'max_score': 0.0, @@ -180,24 +182,25 @@ def _get_index_mapred_emu(self, bucket, index, startkey, endkey=None): """ phases = [] if not self.phaseless_mapred(): - phases.append({'language':'erlang', - 'module':'riak_kv_mapreduce', - 'function':'reduce_identity', - 'keep':True}) + phases.append({'language': 'erlang', + 'module': 'riak_kv_mapreduce', + 'function': 'reduce_identity', + 'keep': True}) if endkey: - result = self.mapred({'bucket':bucket, - 'index':index, - 'start':startkey, - 'end':endkey}, + result = self.mapred({'bucket': bucket, + 'index': index, + 'start': startkey, + 'end': endkey}, phases) else: - result = self.mapred({'bucket':bucket, - 'index':index, - 'key':startkey}, + result = self.mapred({'bucket': bucket, + 'index': index, + 'key': startkey}, phases) - return [ key for bucket, key in result ] + return [key for bucket, key in result] - def store_file(self, key, content_type="application/octet-stream", content=None): + def store_file(self, key, content_type="application/octet-stream", + content=None): """ Store a large piece of data in luwak. key = the key/filename for the object diff --git a/riak/util.py b/riak/util.py index 0ca12bc7..85f49056 100644 --- a/riak/util.py +++ b/riak/util.py @@ -6,10 +6,12 @@ # compatibility with Python 2.5 Mapping = dict + def quacks_like_dict(object): """Check if object is dict-like""" return isinstance(object, Mapping) + def deep_merge(a, b): """Merge two deep dicts non-destructively @@ -31,7 +33,8 @@ def deep_merge(a, b): if key not in current_dst: current_dst[key] = current_src[key] else: - if quacks_like_dict(current_src[key]) and quacks_like_dict(current_dst[key]) : + if (quacks_like_dict(current_src[key]) + and quacks_like_dict(current_dst[key])): stack.append((current_dst[key], current_src[key])) else: current_dst[key] = current_src[key] @@ -41,19 +44,20 @@ def deep_merge(a, b): def deprecated(message, stacklevel=3): warnings.warn(message, DeprecationWarning, stacklevel=stacklevel) + class lazy_property(object): ''' meant to be used for lazy evaluation of an object attribute. property should represent non-mutable data, as it replaces itself. ''' - def __init__(self,fget): + def __init__(self, fget): self.fget = fget self.func_name = fget.__name__ - def __get__(self,obj,cls): + def __get__(self, obj, cls): if obj is None: return None value = self.fget(obj) - setattr(obj,self.func_name,value) + setattr(obj, self.func_name, value) return value From 8046e4d227e9adce166dfaf2a37c57e07734a2df Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Tue, 28 Aug 2012 02:58:24 +0400 Subject: [PATCH 0219/1060] Make .travis.yml future proof MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See http://about.travis-ci.org/blog/august-2012-upcoming-ci-environment-updates/ --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 51398d3b..eff146cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,3 +8,5 @@ before_script: sudo search-cmd install searchbucket env: "SKIP_LUWAK=1" notifications: email: clients@basho.com +services: + - riak From 77809b604a0b76147483f8e5c9aa24955f068886 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 29 Aug 2012 21:30:07 +0300 Subject: [PATCH 0220/1060] Bump officially to 1.5.0. --- RELEASE_NOTES.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 0095f2c7..5641db7d 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,33 @@ # Riak Python Client Release Notes +## 1.5.0 Feature Release - 2012-08-29 + +Release 1.5.0 is a feature release that supports Riak 1.2. + +Noteworthy features: + +* Riak 1.2 features are now supported, including Search and 2I queries + over Protocol Buffers transport. The Protocol Buffers message + definitions now exist as a separate package, available on + [PyPi](http://pypi.python.org/pypi/riak_pb/1.2.0). + + **NOTE:** The return value of search queries over HTTP and MapReduce + were changed to be compatible with the results returned from the + Protocol Buffers interface. +* The client will use a version-based feature detection scheme to + enable or disable various features, including the new Riak 1.2 + features. This enables compatibility with older nodes during a + rolling upgrade, or usage of the newer client with older clusters. + +Noteworthy bugfixes: + +* The code formatting and style was adjusted to fit PEP8 standards. +* All classes in the package are now "new-style". +* The PW accessor methods on RiakClient now get and set the right + instance variable. +* Various fixes were made to the TestServer and it will throw an + exception when it fails to start. + ## 1.4.1 Patch Release - 2012-06-19 Noteworthy features: From 0ae1cc51ec9f8029e1dfcc5e59933a99bbeef6f3 Mon Sep 17 00:00:00 2001 From: Hendrik Volkmer Date: Sun, 9 Sep 2012 14:18:36 +0300 Subject: [PATCH 0221/1060] Fix doc of index method --- riak/mapreduce.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 26d8b483..ce70fd8d 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -107,7 +107,9 @@ def index(self, bucket, index, startkey, endkey=None): Begin a map/reduce operation using a Secondary Index query. @param bucket - The bucket over which to perform the search. - @param query - The search query. + @param index - The index to use for query + @param startkey - The start key of index range + @param endkey - The end key of index range or blank """ self._input_mode = 'query' From 4b3fcd892a8b3a23264eb6ed62d987da5147cb54 Mon Sep 17 00:00:00 2001 From: Joshua Barratt Date: Thu, 13 Sep 2012 16:41:27 -0700 Subject: [PATCH 0222/1060] http_status should be http_code --- riak/transports/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http.py b/riak/transports/http.py index 17f5fef7..8aefabb2 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -132,7 +132,7 @@ def get_resources(self): """ response = self.http_request('GET', '/', {'Accept': 'application/json'}) - if response[0]['http_status'] is 200: + if response[0]['http_code'] is 200: return json.loads(response[1]) else: return {} From c244c872be7147f41cadb38dddc2fedc1062359a Mon Sep 17 00:00:00 2001 From: Joshua Pyle Date: Thu, 4 Oct 2012 00:30:11 -0500 Subject: [PATCH 0223/1060] Fix docstring of RiakClient.__init__ Docstring causes an inspection warning when setting transport_class to RiakPbcTransport. --- riak/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/client.py b/riak/client.py index 4b809b75..092c9eae 100644 --- a/riak/client.py +++ b/riak/client.py @@ -56,7 +56,7 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', :param solr_transport_class: HTTP-based transport class for Solr interface queries - :type transport_class: :class:`RiakHttpTransport` + :type solr_transport_class: :class:`RiakHttpTransport` :param transport_options: Optional key-value args to pass to the transport constuctor :type transport_options: dict From 374789d2737a0744116b7db8683fe76777c37234 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 24 Oct 2012 13:52:12 -0400 Subject: [PATCH 0224/1060] Bump to 1.5.1. --- RELEASE_NOTES.md | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 5641db7d..b229baa0 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,13 @@ # Riak Python Client Release Notes +## 1.5.1 Patch Release - 2012-10-24 + +Release 1.5.1 fixes one bug and some documentation errors. + +* Fix bug where `http_status` is used instead of `http_code`. +* Fix documentation of `RiakMapReduce.index` method. +* Fix documentation of `RiakClient.__init__` method. + ## 1.5.0 Feature Release - 2012-08-29 Release 1.5.0 is a feature release that supports Riak 1.2. diff --git a/setup.py b/setup.py index d6a1d020..a375c388 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ def make_docs(): setup( name='riak', - version='1.5.0', + version='1.5.1', packages = find_packages(), requires = requires, install_requires = install_requires, From ef83401c5b2289cacd2bfe0e6a2a342891b40d8b Mon Sep 17 00:00:00 2001 From: Carl Johan Gustavsson Date: Thu, 25 Oct 2012 10:36:58 +0200 Subject: [PATCH 0225/1060] Fix writing of app config for test server --- riak/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/test_server.py b/riak/test_server.py index a25b01e2..92c46003 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -234,7 +234,7 @@ def write_vm_args(self): vm_args.write("%s %s\n" % (arg, value)) def write_app_config(self): - with open(self._app_config(), "wb") as app_config: + with open(self._app_config_path(), "wb") as app_config: app_config.write(erlang_config(self.app_config)) app_config.write(".") From 250b3f4cdae74bc93027fd7ab241578753203941 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 2 Nov 2012 14:30:28 -0400 Subject: [PATCH 0226/1060] Add basic resource pool, ported from the one used in riak-ruby-client. --- riak/tests/test_pool.py | 149 ++++++++++++++++++++++++++++++++++++++ riak/transports/pool.py | 156 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 305 insertions(+) create mode 100644 riak/tests/test_pool.py create mode 100644 riak/transports/pool.py diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py new file mode 100644 index 00000000..5863b685 --- /dev/null +++ b/riak/tests/test_pool.py @@ -0,0 +1,149 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import platform +from Queue import Queue +from threading import Thread, currentThread +from riak.transports.pool import Pool, BadResource + +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest + + +class SimplePool(Pool): + def __init__(self): + self.count = 0 + Pool.__init__(self) + + def create_resource(self): + self.count += 1 + return [self.count] + + +class EmptyListPool(Pool): + def create_resource(self): + return [] + + +class PoolTest(unittest.TestCase): + def test_yields_new_object_when_empty(self): + pool = SimplePool() + with pool.take() as element: + self.assertEqual([1], element) + + def test_yields_same_object_in_serial_access(self): + pool = SimplePool() + + with pool.take() as element: + self.assertEqual([1], element) + element.append(2) + + with pool.take() as element2: + self.assertEqual(1, len(pool.elements)) + self.assertEqual([1, 2], element) + + self.assertEqual(1, len(pool.elements)) + + def test_reentrance(self): + pool = SimplePool() + with pool.take() as first: + self.assertEqual([1], first) + with pool.take() as second: + self.assertEqual([2], second) + with pool.take() as third: + self.assertEqual([3], third) + + def test_unlocks_when_exception_raised(self): + pool = SimplePool() + try: + with pool.take() as x: + with pool.take() as y: + raise RuntimeError + except: + self.assertEqual(2, len(pool.elements)) + for e in pool.elements: + self.assertFalse(e.claimed) + + def test_removes_bad_resource(self): + pool = SimplePool() + with pool.take() as element: + self.assertEqual([1], element) + element.append(2) + try: + with pool.take() as baddie: + raise BadResource + except BadResource: + self.assertEqual(0, len(pool.elements)) + with pool.take() as goodie: + self.assertEqual([2], goodie) + + def test_filter_skips_unmatching_elements(self): + def filtereven(numlist): + return numlist[0] % 2 == 0 + + pool = SimplePool() + with pool.take() as x: + with pool.take() as y: + pass + + with pool.take(_filter=filtereven) as f: + self.assertEqual([2], f) + + def test_yields_default_when_empty(self): + pool = SimplePool() + with pool.take(default='default') as x: + self.assertEqual('default', x) + + def test_thread_safety(self): + n = 10 + pool = EmptyListPool() + readyq = Queue() + finishq = Queue() + threads = [] + + def _run(): + with pool.take() as resource: + readyq.put(1) + resource.append(currentThread()) + finishq.get(True) + finishq.task_done() + + for i in range(n): + th = Thread(target=_run) + threads.append(th) + th.start() + + for i in range(n): + readyq.get() + readyq.task_done() + + for i in range(n): + finishq.put(1) + + for thr in threads: + thr.join() + + self.assertEqual(n, len(pool.elements)) + for element in pool.elements: + self.assertFalse(element.claimed) + self.assertEqual(1, len(element.object)) + +if __name__ == '__main__': + unittest.main() diff --git a/riak/transports/pool.py b/riak/transports/pool.py new file mode 100644 index 00000000..44b5a2df --- /dev/null +++ b/riak/transports/pool.py @@ -0,0 +1,156 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from contextlib import contextmanager +import threading + + +# This file is a rough port of the Innertube Ruby library +class BadResource(StandardError): + """ + Users of a Pool should raise this error when the pool element + currently in-use is bad and should be removed from the pool. + """ + pass + + +class Element(object): + """ + A member of the Pool, a container for the actual resource being + pooled and a marker for whether the resource is currently claimed. + """ + def __init__(self, obj): + """ + Creates a new Element, wrapping the passed object as the + pooled resource. + :param obj: the resource to wrap + :type obj: object + """ + + """The wrapped pool resource.""" + self.object = obj + """Whether the resource is currently in use.""" + self.claimed = False + + +class Pool(object): + """ + A thread-safe, reentrant resource pool, ported from the + "Innertube" Ruby library. Pool should be subclassed to implement + the create_resource and destroy_resource functions that are + responsible for creating and cleaning up the resources in the + pool, respectively. Claiming a resource of the pool for a block of + code is done using a with statement on the take method. The take + method also allows filtering of the pool and supplying a default + value to be used as the resource if no elements are free. + + Example: + + from riak.Pool import Pool, BadResource + class ListPool(Pool): + def create_resource(self): + return [] + + def destroy_resource(self): + # Lists don't need to be cleaned up + pass + + pool = ListPool() + with pool.take() as resource: + resource.append(1) + with pool.take() as resource2: + print repr(resource2) # should be [1] + """ + + def __init__(self): + """ + Creates a new Pool. This should be called manually if you + override the __init__ method in a subclass. + """ + self.lock = threading.Lock() + self.elements = list() + + @contextmanager + def take(self, _filter=None, default=None): + """ + Claims a resource from the pool for use in a thread-safe, + reentrant manner (as part of a with statement). Resources are + created as needed when all members of the pool are claimed or + the pool is empty. + + :param _filter: a filter that can be used to select a member + of the pool + :type _filter: callable + :param default: a value that will be used instead of calling + create_resource if a new resource needs to be created + """ + element = None + if not callable(_filter): + def _filter(obj): + return True + with self.lock: + for e in self.elements: + if not e.claimed and _filter(e.object): + element = e + break + if element is None: + if default is not None: + element = Element(default) + else: + element = Element(self.create_resource()) + self.elements.append(element) + element.claimed = True + try: + yield element.object + except BadResource: + self.delete_element(element) + raise + finally: + element.claimed = False + + def delete_element(self, element): + """ + Deletes the element from the pool and destroys the associated + resource. Not usually needed by users of the pool, but called + internally when BadResource is raised. + + :param element: the element to remove + :type element: Element + """ + with self.lock: + self.elements.remove(element) + self.destroy_resource(element.object) + del element + + def create_resource(self): + """ + Implemented by subclasses to allocate a new resource for use + in the pool. + """ + raise NotImplemented + + def destroy_resource(self, obj): + """ + Called when removing a resource from the pool so that it can + be cleanly deallocated. Subclasses should implement this + method if additional cleanup is needed beyond normal GC. The + default implementation is a no-op. + + :param obj: the resource being removed + """ + pass From 8a03edccdf3a4918507a45fe13bcddd58a8601aa Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 2 Nov 2012 15:42:20 -0400 Subject: [PATCH 0227/1060] Give full path to search-cmd binary. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index eff146cc..b876f932 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ python: - "2.7" install: ./setup.py develop script: ./setup.py test -before_script: sudo search-cmd install searchbucket +before_script: sudo /usr/sbin/search-cmd install searchbucket env: "SKIP_LUWAK=1" notifications: email: clients@basho.com From 6d59cb4cf9349401b195dc9474e114073eba397c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sat, 3 Nov 2012 12:32:53 -0400 Subject: [PATCH 0228/1060] Be more paranoid on the type of the _filter argument. --- riak/tests/test_pool.py | 8 ++++++++ riak/transports/pool.py | 7 +++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index 5863b685..d2f4f98d 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -106,6 +106,14 @@ def filtereven(numlist): with pool.take(_filter=filtereven) as f: self.assertEqual([2], f) + def test_requires_filter_to_be_callable(self): + badfilter = 'foo' + pool = SimplePool() + + with self.assertRaises(TypeError): + with pool.take(_filter=badfilter) as resource: + pass + def test_yields_default_when_empty(self): pool = SimplePool() with pool.take(default='default') as x: diff --git a/riak/transports/pool.py b/riak/transports/pool.py index 44b5a2df..4944b36a 100644 --- a/riak/transports/pool.py +++ b/riak/transports/pool.py @@ -99,10 +99,13 @@ def take(self, _filter=None, default=None): :param default: a value that will be used instead of calling create_resource if a new resource needs to be created """ - element = None - if not callable(_filter): + if not _filter: def _filter(obj): return True + elif not callable(_filter): + raise TypeError("_filter is not a callable") + + element = None with self.lock: for e in self.elements: if not e.claimed and _filter(e.object): From a8d9fd9a98d5d1a6d1b4da77883fb737bd112e6f Mon Sep 17 00:00:00 2001 From: evan Date: Sun, 4 Nov 2012 18:07:26 -0800 Subject: [PATCH 0229/1060] reorganize the test into multiple files and functional units. --- .gitignore | 3 + riak/tests/test_2i.py | 222 +++++++ riak/tests/test_all.py | 1102 ++-------------------------------- riak/tests/test_kv.py | 304 ++++++++++ riak/tests/test_mapreduce.py | 419 +++++++++++++ riak/tests/test_search.py | 147 +++++ 6 files changed, 1134 insertions(+), 1063 deletions(-) create mode 100644 riak/tests/test_2i.py create mode 100644 riak/tests/test_kv.py create mode 100644 riak/tests/test_mapreduce.py create mode 100644 riak/tests/test_search.py diff --git a/.gitignore b/.gitignore index ef74d171..99bba8c8 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ build/ dist/ riak.egg-info/ *.egg + +#*# +*~ \ No newline at end of file diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py new file mode 100644 index 00000000..4c2e2718 --- /dev/null +++ b/riak/tests/test_2i.py @@ -0,0 +1,222 @@ +# -*- coding: utf-8 -*- +import os, platform +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest + +from riak.riak_index_entry import RiakIndexEntry + +SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '0')) + +class TwoITests(object): + def is_2i_supported(self): + # Immediate test to see if 2i is even supported w/ the backend + try: + self.client.index('foo', 'bar_bin', 'baz').run() + return True + except Exception as e: + if "indexes_not_supported" in str(e): + return False + return True # it failed, but is supported! + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_secondary_index_store(self): + if not self.is_2i_supported(): + return True + + # Create a new object with indexes... + bucket = self.client.bucket('indexbucket') + rand = self.randint() + obj = bucket.new('mykey1', rand) + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_int', 1011) + obj.store() + + # Retrieve the object, check that the correct indexes exist... + obj = bucket.get('mykey1') + self.assertEqual(['val1a'], sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1011'], sorted(obj.get_indexes('field1_int'))) + + # Add more indexes and save... + obj.add_index('field1_bin', 'val1b') + obj.add_index('field1_int', 1012) + obj.store() + + # Retrieve the object, check that the correct indexes exist... + obj = bucket.get('mykey1') + self.assertEqual(['val1a', 'val1b'], + sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1011', '1012'], + sorted(obj.get_indexes('field1_int'))) + + # Check the get_indexes() function... + self.assertEqual([ + RiakIndexEntry('field1_bin', 'val1a'), + RiakIndexEntry('field1_bin', 'val1b'), + RiakIndexEntry('field1_int', 1011), + RiakIndexEntry('field1_int', 1012) + ], sorted(obj.get_indexes())) + + # Delete an index... + obj.remove_index('field1_bin', 'val1a') + obj.remove_index('field1_int', 1011) + obj.store() + + # Retrieve the object, check that the correct indexes exist... + obj = bucket.get('mykey1') + self.assertEqual(['val1b'], sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1012'], sorted(obj.get_indexes('field1_int'))) + + # Check duplicate entries... + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_int', 1011) + obj.add_index('field1_int', 1011) + obj.add_index('field1_int', 1011) + + self.assertEqual([ + RiakIndexEntry('field1_bin', 'val1a'), + RiakIndexEntry('field1_bin', 'val1b'), + RiakIndexEntry('field1_int', 1011), + RiakIndexEntry('field1_int', 1012) + ], sorted(obj.get_indexes())) + + obj.store() + obj = bucket.get('mykey1') + + self.assertEqual([ + RiakIndexEntry('field1_bin', 'val1a'), + RiakIndexEntry('field1_bin', 'val1b'), + RiakIndexEntry('field1_int', 1011), + RiakIndexEntry('field1_int', 1012) + ], sorted(obj.get_indexes())) + + # Clean up... + bucket.get('mykey1').delete() + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_set_indexes(self): + if not self.is_2i_supported(): + return True + + bucket = self.client.bucket('indexbucket') + foo = bucket.new('foo', 1) + foo.set_indexes((('field1_bin', 'test'), ('field2_int', 1337))).store() + result = self.client.index('indexbucket', 'field2_int', 1337).run() + self.assertEqual(1, len(result)) + self.assertEqual('foo', result[0].get_key()) + + result = bucket.get_index('field1_bin', 'test') + self.assertEqual(1, len(result)) + self.assertEqual('foo', str(result[0])) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_remove_indexes(self): + if not self.is_2i_supported(): + return True + + bucket = self.client.bucket('indexbucket') + bar = bucket.new('bar', 1).add_index('bar_int', 1)\ + .add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + result = bucket.get_index('bar_int', 1) + self.assertEqual(1, len(result)) + self.assertEqual(3, len(bar.get_indexes())) + self.assertEqual(2, len(bar.get_indexes('bar_int'))) + + # remove all indexes + bar = bar.remove_indexes().store() + result = bucket.get_index('bar_int', 1) + self.assertEqual(0, len(result)) + result = bucket.get_index('baz_bin', 'baz') + self.assertEqual(0, len(result)) + self.assertEqual(0, len(bar.get_indexes())) + self.assertEqual(0, len(bar.get_indexes('bar_int'))) + self.assertEqual(0, len(bar.get_indexes('baz_bin'))) + + # add index again + bar = bar.add_index('bar_int', 1).add_index('bar_int', 2)\ + .add_index('baz_bin', 'baz').store() + # remove all index with field='bar_int' + bar = bar.remove_index(field='bar_int').store() + result = bucket.get_index('bar_int', 1) + self.assertEqual(0, len(result)) + result = bucket.get_index('bar_int', 2) + self.assertEqual(0, len(result)) + result = bucket.get_index('baz_bin', 'baz') + self.assertEqual(1, len(result)) + self.assertEqual(1, len(bar.get_indexes())) + self.assertEqual(0, len(bar.get_indexes('bar_int'))) + self.assertEqual(1, len(bar.get_indexes('baz_bin'))) + + # add index again + bar = bar.add_index('bar_int', 1).add_index('bar_int', 2)\ + .add_index('baz_bin', 'baz').store() + # remove an index field value pair + bar = bar.remove_index(field='bar_int', value=2).store() + result = bucket.get_index('bar_int', 1) + self.assertEqual(1, len(result)) + result = bucket.get_index('bar_int', 2) + self.assertEqual(0, len(result)) + result = bucket.get_index('baz_bin', 'baz') + self.assertEqual(1, len(result)) + self.assertEqual(2, len(bar.get_indexes())) + self.assertEqual(1, len(bar.get_indexes('bar_int'))) + self.assertEqual(1, len(bar.get_indexes('baz_bin'))) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_secondary_index_query(self): + if not self.is_2i_supported(): + return True + + bucket = self.client.bucket('indexbucket') + + bucket.\ + new('mykey1', 'data1').\ + add_index('field1_bin', 'val1').\ + add_index('field2_int', 1001).\ + store() + bucket.\ + new('mykey2', 'data1').\ + add_index('field1_bin', 'val2').\ + add_index('field2_int', 1002).\ + store() + bucket.\ + new('mykey3', 'data1').\ + add_index('field1_bin', 'val3').\ + add_index('field2_int', 1003).\ + store() + bucket.\ + new('mykey4', 'data1').\ + add_index('field1_bin', 'val4').\ + add_index('field2_int', 1004).\ + store() + + # Test an equality query... + results = bucket.get_index('field1_bin', 'val2') + self.assertEquals(1, len(results)) + self.assertEquals('mykey2', str(results[0])) + + # Test a range query... + results = bucket.get_index('field1_bin', 'val2', 'val4') + vals = set([str(key) for key in results]) + self.assertEquals(3, len(results)) + self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) + + # Test an equality query... + results = bucket.get_index('field2_int', 1002) + self.assertEquals(1, len(results)) + self.assertEquals('mykey2', str(results[0])) + + # Test a range query... + results = bucket.get_index('field2_int', 1002, 1004) + vals = set([str(key) for key in results]) + self.assertEquals(3, len(results)) + self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) + + # Clean up... + bucket.get('mykey1').delete() + bucket.get('mykey2').delete() + bucket.get('mykey3').delete() + bucket.get('mykey4').delete() diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index af15ed84..29497b29 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -1,12 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import with_statement -import copy -import cPickle -try: - import json -except ImportError: - import simplejson as json import os import random import socket @@ -22,11 +16,18 @@ from riak import RiakClient from riak import RiakPbcTransport from riak import RiakHttpTransport -from riak import RiakKeyFilter, key_filter -from riak.riak_index_entry import RiakIndexEntry from riak.mapreduce import RiakLink +from riak import RiakKeyFilter, key_filter + from riak.test_server import TestServer +from riak.tests.test_search import SearchTests, \ + EnableSearchTests, SolrSearchTests +from riak.tests.test_mapreduce import MapReduceAliasTests, \ + ErlangMapReduceTests, JSMapReduceTests, LinkTests +from riak.tests.test_kv import BasicKVTests, KVFileTests +from riak.tests.test_2i import TwoITests + try: import riak_pb HAVE_PROTO = True @@ -34,13 +35,15 @@ HAVE_PROTO = False HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') -HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) + PB_HOST = os.environ.get('RIAK_TEST_PB_HOST', HOST) -HTTP_PORT = int(os.environ.get('RIAK_TEST_HTTP_PORT', '8098')) PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) -SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '0')) + +HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) +HTTP_PORT = int(os.environ.get('RIAK_TEST_HTTP_PORT', '8098')) + SKIP_LUWAK = int(os.environ.get('SKIP_LUWAK', '0')) -SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '0')) + USE_TEST_SERVER = int(os.environ.get('USE_TEST_SERVER', '0')) if USE_TEST_SERVER: @@ -51,31 +54,6 @@ test_server.prepare() test_server.start() - -class NotJsonSerializable(object): - - def __init__(self, *args, **kwargs): - self.args = list(args) - self.kwargs = kwargs - - def __eq__(self, other): - if len(self.args) != len(other.args): - return False - if len(self.kwargs) != len(other.kwargs): - return False - for name, value in self.kwargs.items(): - if other.kwargs[name] != value: - return False - value1_args = copy.copy(self.args) - value2_args = copy.copy(other.args) - value1_args.sort() - value2_args.sort() - for i in xrange(len(value1_args)): - if value1_args[i] != value2_args[i]: - return False - return True - - class BaseTestCase(object): @staticmethod @@ -99,953 +77,15 @@ def setUp(self): o = bucket.get('nonexistent_key_binary') o.delete() - def test_is_alive(self): - self.assertTrue(self.client.is_alive()) - - def test_store_and_get(self): - bucket = self.client.bucket('bucket') - rand = self.randint() - obj = bucket.new('foo', rand) - obj.store() - obj = bucket.get('foo') - self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().get_name(), 'bucket') - self.assertEqual(obj.get_key(), 'foo') - self.assertEqual(obj.get_data(), rand) - - # unicode objects are fine, as long as they don't - # contain any non-ASCII chars - self.client.bucket(u'bucket') - self.assertRaises(TypeError, self.client.bucket, u'búcket') - self.assertRaises(TypeError, self.client.bucket, 'búcket') - - bucket.get(u'foo') - self.assertRaises(TypeError, bucket.get, u'føø') - self.assertRaises(TypeError, bucket.get, 'føø') - - self.assertRaises(TypeError, bucket.new, u'foo', 'éå') - self.assertRaises(TypeError, bucket.new, u'foo', 'éå') - self.assertRaises(TypeError, bucket.new, 'foo', u'éå') - self.assertRaises(TypeError, bucket.new, 'foo', u'éå') - - def test_generate_key(self): - # Ensure that Riak generates a random key when - # the key passed to bucket.new() is None. - bucket = self.client.bucket('random_key_bucket') - existing_keys = bucket.get_keys() - o = bucket.new(None, data={}) - self.assertIsNone(o.get_key()) - o.store() - self.assertIsNotNone(o.get_key()) - self.assertNotIn('/', o.get_key()) - self.assertNotIn(o.get_key(), existing_keys) - self.assertEqual(len(bucket.get_keys()), len(existing_keys) + 1) - - def test_binary_store_and_get(self): - bucket = self.client.bucket('bucket') - # Store as binary, retrieve as binary, then compare... - rand = str(self.randint()) - obj = bucket.new_binary('foo1', rand) - obj.store() - obj = bucket.get_binary('foo1') - self.assertTrue(obj.exists()) - self.assertEqual(obj.get_data(), rand) - # Store as JSON, retrieve as binary, JSON-decode, then compare... - data = [self.randint(), self.randint(), self.randint()] - obj = bucket.new('foo2', data) - obj.store() - obj = bucket.get_binary('foo2') - self.assertEqual(data, json.loads(obj.get_data())) - - def test_custom_bucket_encoder_decoder(self): - # Teach the bucket how to pickle - bucket = self.client.bucket("picklin_bucket") - bucket.set_encoder('application/x-pickle', cPickle.dumps) - bucket.set_decoder('application/x-pickle', cPickle.loads) - data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} - obj = bucket.new("foo", data, 'application/x-pickle').store() - obj.store() - obj2 = bucket.get("foo") - self.assertEqual(data, obj2.get_data()) - - def test_custom_client_encoder_decoder(self): - # Teach the bucket how to pickle - bucket = self.client.bucket("picklin_client") - self.client.set_encoder('application/x-pickle', cPickle.dumps) - self.client.set_decoder('application/x-pickle', cPickle.loads) - data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} - obj = bucket.new("foo", data, 'application/x-pickle').store() - obj.store() - obj2 = bucket.get("foo") - self.assertEqual(data, obj2.get_data()) - - def test_unknown_content_type_encoder_decoder(self): - # Teach the bucket how to pickle - bucket = self.client.bucket("unknown_contenttype") - data = "some funny data" - obj = bucket.new("foo", data, 'application/x-frobnicator').store() - obj.store() - obj2 = bucket.get("foo") - self.assertEqual(data, obj2.get_data()) - - def test_missing_object(self): - bucket = self.client.bucket('bucket') - obj = bucket.get("missing") - self.assertFalse(obj.exists()) - self.assertEqual(obj.get_data(), None) - - def test_delete(self): - bucket = self.client.bucket('bucket') - rand = self.randint() - obj = bucket.new('foo', rand) - obj.store() - obj = bucket.get('foo') - self.assertTrue(obj.exists()) - obj.delete() - obj.reload() - self.assertFalse(obj.exists()) - - def test_set_bucket_properties(self): - bucket = self.client.bucket('bucket') - # Test setting allow mult... - bucket.set_allow_multiples(True) - self.assertTrue(bucket.get_allow_multiples()) - # Test setting nval... - bucket.set_n_val(3) - self.assertEqual(bucket.get_n_val(), 3) - # Test setting multiple properties... - bucket.set_properties({"allow_mult": False, "n_val": 2}) - self.assertFalse(bucket.get_allow_multiples()) - self.assertEqual(bucket.get_n_val(), 2) - - def test_rw_settings(self): - bucket = self.client.bucket('rwsettings') - self.assertEqual(bucket.get_r(), "default") - self.assertEqual(bucket.get_w(), "default") - self.assertEqual(bucket.get_dw(), "default") - self.assertEqual(bucket.get_rw(), "default") - - bucket.set_w(1) - self.assertEqual(bucket.get_w(), 1) - - bucket.set_r("quorum") - self.assertEqual(bucket.get_r(), "quorum") - - bucket.set_dw("all") - self.assertEqual(bucket.get_dw(), "all") - - bucket.set_rw("one") - self.assertEqual(bucket.get_rw(), "one") - - def test_primary_quora(self): - bucket = self.client.bucket('primary_quora') - self.assertEqual(bucket.get_pr(), "default") - self.assertEqual(bucket.get_pw(), "default") - - bucket.set_pr(1) - self.assertEqual(bucket.get_pr(), 1) - - bucket.set_pw("quorum") - self.assertEqual(bucket.get_pw(), "quorum") - - def test_if_none_match(self): - bucket = self.client.bucket('if_none_match_test') - obj = bucket.get('obj') - obj.delete() - - obj.reload() - self.assertFalse(obj.exists()) - obj.set_data(["first store"]) - obj.store() - - obj.set_data(["second store"]) - with self.assertRaises(Exception): - obj.store(if_none_match=True) - - def test_siblings(self): - # Set up the bucket, clear any existing object... - bucket = self.client.bucket('multiBucket') - bucket.set_allow_multiples(True) - obj = bucket.get_binary('foo') - # Even if it previously existed, let's store a base resolved version - # from which we can diverge by sending a stale vclock. - obj.set_data('start') - obj.store() - - # Store the same object five times... - vals = set() - for i in range(5): - other_client = self.create_client() - other_bucket = other_client.bucket('multiBucket') - while True: - randval = self.randint() - if randval not in vals: - break - - other_obj = other_bucket.new_binary('foo', str(randval)) - other_obj._vclock = obj._vclock - other_obj.store() - vals.add(str(randval)) - - # Make sure the object has itself plus four siblings... - obj.reload() - self.assertTrue(obj.has_siblings()) - self.assertEqual(obj.get_sibling_count(), 5) - - # Get each of the values - make sure they match what was assigned - vals2 = set() - for i in range(5): - vals2.add(obj.get_sibling(i).get_data()) - self.assertEqual(vals, vals2) - - # Resolve the conflict, and then do a get... - obj3 = obj.get_sibling(3) - obj3.store() - - obj.reload() - self.assertEqual(obj.get_sibling_count(), 0) - self.assertEqual(obj.get_data(), obj3.get_data()) - - def test_javascript_source_map(self): - # Create the object... - bucket = self.client.bucket("bucket") - bucket.new("foo", 2).store() - # Run the map... - mr = self.client.add("bucket", "foo") - result = mr.map( - "function (v) { return [JSON.parse(v.values[0].data)]; }").run() - self.assertEqual(result, [2]) - - # test ASCII-encodable unicode is accepted - mr.map(u"function (v) { return [JSON.parse(v.values[0].data)]; }") - - # test non-ASCII-encodable unicode is rejected - self.assertRaises(TypeError, mr.map, - u"function (v) { /* æ */ return [JSON.parse(v.values[0].data)]; }") - - # test non-ASCII-encodable string is rejected - self.assertRaises(TypeError, mr.map, - "function (v) { /* æ */ return [JSON.parse(v.values[0].data)]; }") - - def test_javascript_named_map(self): - # Create the object... - bucket = self.client.bucket("bucket") - bucket.new("foo", 2).store() - # Run the map... - result = self.client \ - .add("bucket", "foo") \ - .map("Riak.mapValuesJson") \ - .run() - self.assertEqual(result, [2]) - - def test_javascript_source_map_reduce(self): - # Create the object... - bucket = self.client.bucket("bucket") - bucket.new("foo", 2).store() - bucket.new("bar", 3).store() - bucket.new("baz", 4).store() - # Run the map... - result = self.client \ - .add("bucket", "foo") \ - .add("bucket", "bar") \ - .add("bucket", "baz") \ - .map("function (v) { return [1]; }") \ - .reduce("Riak.reduceSum") \ - .run() - self.assertEqual(result, [3]) - - def test_javascript_named_map_reduce(self): - # Create the object... - bucket = self.client.bucket("bucket") - bucket.new("foo", 2).store() - bucket.new("bar", 3).store() - bucket.new("baz", 4).store() - # Run the map... - result = self.client \ - .add("bucket", "foo") \ - .add("bucket", "bar") \ - .add("bucket", "baz") \ - .map("Riak.mapValuesJson") \ - .reduce("Riak.reduceSum") \ - .run() - self.assertEqual(result, [9]) - - def test_javascript_bucket_map_reduce(self): - # Create the object... - bucket = self.client.bucket("bucket_%s" % self.randint()) - bucket.new("foo", 2).store() - bucket.new("bar", 3).store() - bucket.new("baz", 4).store() - # Run the map... - result = self.client \ - .add(bucket.get_name()) \ - .map("Riak.mapValuesJson") \ - .reduce("Riak.reduceSum") \ - .run() - self.assertEqual(result, [9]) - - def test_javascript_arg_map_reduce(self): - # Create the object... - bucket = self.client.bucket("bucket") - bucket.new("foo", 2).store() - # Run the map... - result = self.client \ - .add("bucket", "foo", 5) \ - .add("bucket", "foo", 10) \ - .add("bucket", "foo", 15) \ - .add("bucket", "foo", -15) \ - .add("bucket", "foo", -5) \ - .map("function(v, arg) { return [arg]; }") \ - .reduce("Riak.reduceSum") \ - .run() - self.assertEqual(result, [10]) - - def test_key_filters(self): - bucket = self.client.bucket("kftest") - bucket.new("basho-20101215", 1).store() - bucket.new("google-20110103", 2).store() - bucket.new("yahoo-20090613", 3).store() - - result = self.client \ - .add("kftest") \ - .add_key_filters([["tokenize", "-", 2]]) \ - .add_key_filter("ends_with", "0613") \ - .map("function (v, keydata) { return [v.key]; }") \ - .run() - - self.assertEqual(result, ["yahoo-20090613"]) - - def test_key_filters_f_chain(self): - bucket = self.client.bucket("kftest") - bucket.new("basho-20101215", 1).store() - bucket.new("google-20110103", 2).store() - bucket.new("yahoo-20090613", 3).store() - - # compose a chain of key filters using f as the root of - # two filters ANDed together to ensure that f can be the root - # of multiple chains - filters = key_filter.tokenize("-", 1).eq("yahoo") \ - & key_filter.tokenize("-", 2).ends_with("0613") - - result = self.client \ - .add("kftest") \ - .add_key_filters(filters) \ - .map("function (v, keydata) { return [v.key]; }") \ - .run() - - self.assertEqual(result, ["yahoo-20090613"]) - - def test_key_filters_with_search_query(self): - mapreduce = self.client.search("kftest", "query") - self.assertRaises(Exception, mapreduce.add_key_filters, - [["tokenize", "-", 2]]) - self.assertRaises(Exception, mapreduce.add_key_filter, - "ends_with", "0613") - - def test_erlang_map_reduce(self): - # Create the object... - bucket = self.client.bucket("bucket") - bucket.new("foo", 2).store() - bucket.new("bar", 2).store() - bucket.new("baz", 4).store() - # Run the map... - result = self.client \ - .add("bucket", "foo") \ - .add("bucket", "bar") \ - .add("bucket", "baz") \ - .map(["riak_kv_mapreduce", "map_object_value"]) \ - .reduce(["riak_kv_mapreduce", "reduce_set_union"]) \ - .run() - self.assertEqual(len(result), 2) - - def test_map_reduce_from_object(self): - # Create the object... - bucket = self.client.bucket("bucket") - bucket.new("foo", 2).store() - obj = bucket.get("foo") - result = obj.map("Riak.mapValuesJson").run() - self.assertEqual(result, [2]) - - def test_store_and_get_links(self): - # Create the object... - bucket = self.client.bucket("bucket") - bucket.new_binary("test_store_and_get_links", '2') \ - .add_link(bucket.new("foo1")) \ - .add_link(bucket.new("foo2"), "tag") \ - .add_link(bucket.new("foo3"), "tag2!@#%^&*)") \ - .store() - obj = bucket.get("test_store_and_get_links") - links = obj.get_links() - self.assertEqual(len(links), 3) - for l in links: - if (l.get_key() == "foo1"): - self.assertEqual(l.get_tag(), "bucket") - elif (l.get_key() == "foo2"): - self.assertEqual(l.get_tag(), "tag") - elif (l.get_key() == "foo3"): - self.assertEqual(l.get_tag(), "tag2!@#%^&*)") - else: - self.assertEqual("unknown key", l.get_key()) - - def test_set_links(self): - # Create the object - bucket = self.client.bucket("bucket") - bucket.new("foo", 2).set_links([bucket.new("foo1"), - (bucket.new("foo2"), "tag"), - RiakLink("bucket", "foo2", "tag2")]).store() - obj = bucket.get("foo") - links = sorted(obj.get_links(), key=lambda x: x.get_key()) - self.assertEqual(len(links), 3) - self.assertEqual(links[0].get_key(), "foo1") - self.assertEqual(links[1].get_key(), "foo2") - self.assertEqual(links[1].get_tag(), "tag") - self.assertEqual(links[2].get_key(), "foo2") - self.assertEqual(links[2].get_tag(), "tag2") - - def test_set_links_all_links(self): - bucket = self.client.bucket("bucket") - foo1 = bucket.new("foo", 1) - foo2 = bucket.new("foo2", 2).store() - links = [RiakLink("bucket", "foo2")] - foo1.set_links(links, True) - links = foo1.get_links() - self.assertEqual(len(links), 1) - self.assertEqual(links[0].get_key(), "foo2") - - def test_link_walking(self): - # Create the object... - bucket = self.client.bucket("bucket") - bucket.new("foo", 2) \ - .add_link(bucket.new("foo1", "test1").store()) \ - .add_link(bucket.new("foo2", "test2").store(), "tag") \ - .add_link(bucket.new("foo3", "test3").store(), "tag2!@#%^&*)") \ - .store() - obj = bucket.get("foo") - results = obj.link("bucket").run() - self.assertEqual(len(results), 3) - results = obj.link("bucket", "tag").run() - self.assertEqual(len(results), 1) - - def test_store_of_missing_object(self): - bucket = self.client.bucket("bucket") - # for json objects - o = bucket.get("nonexistent_key_json") - self.assertEqual(o.exists(), False) - o.set_data({"foo": "bar"}) - o = o.store() - self.assertEqual(o.get_data(), {"foo": "bar"}) - self.assertEqual(o.get_content_type(), "application/json") - o.delete() - # for binary objects - o = bucket.get_binary("nonexistent_key_binary") - self.assertEqual(o.exists(), False) - o.set_data("1234567890") - o = o.store() - self.assertEqual(o.get_data(), "1234567890") - self.assertEqual(o.get_content_type(), "application/octet-stream") - o.delete() - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_solr_search_from_bucket(self): - bucket = self.client.bucket('searchbucket') - bucket.new("user", {"username": "roidrage"}).store() - results = bucket.search("username:roidrage") - self.assertEquals(1, len(results['docs'])) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_solr_search_with_params_from_bucket(self): - bucket = self.client.bucket('searchbucket') - bucket.new("user", {"username": "roidrage"}).store() - results = bucket.search("username:roidrage", wt="xml") - self.assertEquals(1, len(results['docs'])) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_solr_search_with_params(self): - bucket = self.client.bucket('searchbucket') - bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr().search("searchbucket", - "username:roidrage", wt="xml") - self.assertEquals(1, len(results['docs'])) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_solr_search(self): - bucket = self.client.bucket('searchbucket') - bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr().search("searchbucket", - "username:roidrage") - self.assertEquals(1, len(results["docs"])) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_search_integration(self): - # Create some objects to search across... - bucket = self.client.bucket("searchbucket") - bucket.new("one", {"foo": "one", "bar": "red"}).store() - bucket.new("two", {"foo": "two", "bar": "green"}).store() - bucket.new("three", {"foo": "three", "bar": "blue"}).store() - bucket.new("four", {"foo": "four", "bar": "orange"}).store() - bucket.new("five", {"foo": "five", "bar": "yellow"}).store() - - # Run some operations... - results = self.client.solr().search("searchbucket", - "foo:one OR foo:two") - if (len(results) == 0): - print "\n\nNot running test \"testSearchIntegration()\".\n" - print """Please ensure that you have installed the Riak - Search hook on bucket \"searchbucket\" by running - \"bin/search-cmd install searchbucket\".\n\n""" - return - self.assertEqual(len(results['docs']), 2) - query = "(foo:one OR foo:two OR foo:three OR foo:four) AND\ - (NOT bar:green)" - results = self.client.solr().search("searchbucket", query) - self.assertEqual(len(results['docs']), 3) - - def test_store_binary_object_from_file(self): - bucket = self.client.bucket('bucket') - rand = str(self.randint()) - filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') - obj = bucket.new_binary_from_file('foo_from_file', filepath) - obj.store() - obj = bucket.get_binary('foo_from_file') - self.assertNotEqual(obj.get_data(), None) - self.assertEqual(obj.get_content_type(), "text/x-python") - - def test_store_binary_object_from_file_should_use_default_mimetype(self): - bucket = self.client.bucket('bucket') - rand = str(self.randint()) - filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), - os.pardir, os.pardir, 'THANKS') - obj = bucket.new_binary_from_file('foo_from_file', filepath) - obj.store() - obj = bucket.get_binary('foo_from_file') - self.assertEqual(obj.get_content_type(), 'application/octet-stream') - - def test_store_metadata(self): - bucket = self.client.bucket('bucket') - rand = self.randint() - obj = bucket.new('fooster', rand) - obj.set_usermeta({'custom': 'some metadata'}) - obj.store() - obj = bucket.get('fooster') - self.assertEqual('some metadata', obj.get_usermeta()['custom']) - - def test_store_binary_object_from_file_should_fail_if_file_not_found(self): - bucket = self.client.bucket('bucket') - rand = str(self.randint()) - self.assertRaises(IOError, bucket.new_binary_from_file, - 'not_found_from_file', 'FILE_NOT_FOUND') - obj = bucket.get_binary('not_found_from_file') - self.assertEqual(obj.get_data(), None) - - def test_list_buckets(self): - bucket = self.client.bucket("list_bucket") - bucket.new("one", {"foo": "one", "bar": "red"}).store() - buckets = self.client.get_buckets() - self.assertTrue("list_bucket" in buckets) - - def is_2i_supported(self): - # Immediate test to see if 2i is even supported w/ the backend - try: - self.client.index('foo', 'bar_bin', 'baz').run() - return True - except Exception as e: - if "indexes_not_supported" in str(e): - return False - return True # it failed, but is supported! - - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') - def test_secondary_index_store(self): - if not self.is_2i_supported(): - return True - - # Create a new object with indexes... - bucket = self.client.bucket('indexbucket') - rand = self.randint() - obj = bucket.new('mykey1', rand) - obj.add_index('field1_bin', 'val1a') - obj.add_index('field1_int', 1011) - obj.store() - - # Retrieve the object, check that the correct indexes exist... - obj = bucket.get('mykey1') - self.assertEqual(['val1a'], sorted(obj.get_indexes('field1_bin'))) - self.assertEqual(['1011'], sorted(obj.get_indexes('field1_int'))) - - # Add more indexes and save... - obj.add_index('field1_bin', 'val1b') - obj.add_index('field1_int', 1012) - obj.store() - - # Retrieve the object, check that the correct indexes exist... - obj = bucket.get('mykey1') - self.assertEqual(['val1a', 'val1b'], - sorted(obj.get_indexes('field1_bin'))) - self.assertEqual(['1011', '1012'], - sorted(obj.get_indexes('field1_int'))) - - # Check the get_indexes() function... - self.assertEqual([ - RiakIndexEntry('field1_bin', 'val1a'), - RiakIndexEntry('field1_bin', 'val1b'), - RiakIndexEntry('field1_int', 1011), - RiakIndexEntry('field1_int', 1012) - ], sorted(obj.get_indexes())) - - # Delete an index... - obj.remove_index('field1_bin', 'val1a') - obj.remove_index('field1_int', 1011) - obj.store() - - # Retrieve the object, check that the correct indexes exist... - obj = bucket.get('mykey1') - self.assertEqual(['val1b'], sorted(obj.get_indexes('field1_bin'))) - self.assertEqual(['1012'], sorted(obj.get_indexes('field1_int'))) - - # Check duplicate entries... - obj.add_index('field1_bin', 'val1a') - obj.add_index('field1_bin', 'val1a') - obj.add_index('field1_bin', 'val1a') - obj.add_index('field1_int', 1011) - obj.add_index('field1_int', 1011) - obj.add_index('field1_int', 1011) - - self.assertEqual([ - RiakIndexEntry('field1_bin', 'val1a'), - RiakIndexEntry('field1_bin', 'val1b'), - RiakIndexEntry('field1_int', 1011), - RiakIndexEntry('field1_int', 1012) - ], sorted(obj.get_indexes())) - - obj.store() - obj = bucket.get('mykey1') - - self.assertEqual([ - RiakIndexEntry('field1_bin', 'val1a'), - RiakIndexEntry('field1_bin', 'val1b'), - RiakIndexEntry('field1_int', 1011), - RiakIndexEntry('field1_int', 1012) - ], sorted(obj.get_indexes())) - - # Clean up... - bucket.get('mykey1').delete() - - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') - def test_set_indexes(self): - if not self.is_2i_supported(): - return True - - bucket = self.client.bucket('indexbucket') - foo = bucket.new('foo', 1) - foo.set_indexes((('field1_bin', 'test'), ('field2_int', 1337))).store() - result = self.client.index('indexbucket', 'field2_int', 1337).run() - self.assertEqual(1, len(result)) - self.assertEqual('foo', result[0].get_key()) - - result = bucket.get_index('field1_bin', 'test') - self.assertEqual(1, len(result)) - self.assertEqual('foo', str(result[0])) - - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') - def test_remove_indexes(self): - if not self.is_2i_supported(): - return True - - bucket = self.client.bucket('indexbucket') - bar = bucket.new('bar', 1).add_index('bar_int', 1)\ - .add_index('bar_int', 2).add_index('baz_bin', 'baz').store() - result = bucket.get_index('bar_int', 1) - self.assertEqual(1, len(result)) - self.assertEqual(3, len(bar.get_indexes())) - self.assertEqual(2, len(bar.get_indexes('bar_int'))) - - # remove all indexes - bar = bar.remove_indexes().store() - result = bucket.get_index('bar_int', 1) - self.assertEqual(0, len(result)) - result = bucket.get_index('baz_bin', 'baz') - self.assertEqual(0, len(result)) - self.assertEqual(0, len(bar.get_indexes())) - self.assertEqual(0, len(bar.get_indexes('bar_int'))) - self.assertEqual(0, len(bar.get_indexes('baz_bin'))) - - # add index again - bar = bar.add_index('bar_int', 1).add_index('bar_int', 2)\ - .add_index('baz_bin', 'baz').store() - # remove all index with field='bar_int' - bar = bar.remove_index(field='bar_int').store() - result = bucket.get_index('bar_int', 1) - self.assertEqual(0, len(result)) - result = bucket.get_index('bar_int', 2) - self.assertEqual(0, len(result)) - result = bucket.get_index('baz_bin', 'baz') - self.assertEqual(1, len(result)) - self.assertEqual(1, len(bar.get_indexes())) - self.assertEqual(0, len(bar.get_indexes('bar_int'))) - self.assertEqual(1, len(bar.get_indexes('baz_bin'))) - - # add index again - bar = bar.add_index('bar_int', 1).add_index('bar_int', 2)\ - .add_index('baz_bin', 'baz').store() - # remove an index field value pair - bar = bar.remove_index(field='bar_int', value=2).store() - result = bucket.get_index('bar_int', 1) - self.assertEqual(1, len(result)) - result = bucket.get_index('bar_int', 2) - self.assertEqual(0, len(result)) - result = bucket.get_index('baz_bin', 'baz') - self.assertEqual(1, len(result)) - self.assertEqual(2, len(bar.get_indexes())) - self.assertEqual(1, len(bar.get_indexes('bar_int'))) - self.assertEqual(1, len(bar.get_indexes('baz_bin'))) - - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') - def test_secondary_index_query(self): - if not self.is_2i_supported(): - return True - - bucket = self.client.bucket('indexbucket') - - bucket.\ - new('mykey1', 'data1').\ - add_index('field1_bin', 'val1').\ - add_index('field2_int', 1001).\ - store() - bucket.\ - new('mykey2', 'data1').\ - add_index('field1_bin', 'val2').\ - add_index('field2_int', 1002).\ - store() - bucket.\ - new('mykey3', 'data1').\ - add_index('field1_bin', 'val3').\ - add_index('field2_int', 1003).\ - store() - bucket.\ - new('mykey4', 'data1').\ - add_index('field1_bin', 'val4').\ - add_index('field2_int', 1004).\ - store() - - # Test an equality query... - results = bucket.get_index('field1_bin', 'val2') - self.assertEquals(1, len(results)) - self.assertEquals('mykey2', str(results[0])) - - # Test a range query... - results = bucket.get_index('field1_bin', 'val2', 'val4') - vals = set([str(key) for key in results]) - self.assertEquals(3, len(results)) - self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) - - # Test an equality query... - results = bucket.get_index('field2_int', 1002) - self.assertEquals(1, len(results)) - self.assertEquals('mykey2', str(results[0])) - - # Test a range query... - results = bucket.get_index('field2_int', 1002, 1004) - vals = set([str(key) for key in results]) - self.assertEquals(3, len(results)) - self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) - - # Clean up... - bucket.get('mykey1').delete() - bucket.get('mykey2').delete() - bucket.get('mykey3').delete() - bucket.get('mykey4').delete() - - -class MapReduceAliasTestMixIn(object): - """This tests the map reduce aliases""" - - def test_map_values(self): - # Add a value to the bucket - bucket = self.client.bucket('bucket') - bucket.new_binary('one', data='value_1').store() - bucket.new_binary('two', data='value_2').store() - - # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') - - # Use the map_values alias - result = mr.map_values().run() - - # Sort the result so that we can have a consistent - # expected value - result.sort() - - self.assertEqual(result, ["value_1", "value_2"]) - - def test_map_values_json(self): - # Add a value to the bucket - bucket = self.client.bucket('bucket') - bucket.new('one', data={'val': 'value_1'}).store() - bucket.new('two', data={'val': 'value_2'}).store() - - # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') - - # Use the map_values alias - result = mr.map_values_json().run() - - # Sort the result so that we can have a consistent - # expected value - result.sort(key=lambda x: x['val']) - - self.assertEqual(result, [{'val': "value_1"}, {'val': "value_2"}]) - - def test_reduce_sum(self): - # Add a value to the bucket - bucket = self.client.bucket('bucket') - bucket.new('one', data=1).store() - bucket.new('two', data=2).store() - - # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') - - # Use the map_values alias - result = mr.map_values_json().reduce_sum().run() - - self.assertEqual(result, [3]) - - def test_reduce_min(self): - # Add a value to the bucket - bucket = self.client.bucket('bucket') - bucket.new('one', data=1).store() - bucket.new('two', data=2).store() - - # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') - - # Use the map_values alias - result = mr.map_values_json().reduce_min().run() - - self.assertEqual(result, [1]) - - def test_reduce_max(self): - # Add a value to the bucket - bucket = self.client.bucket('bucket') - bucket.new('one', data=1).store() - bucket.new('two', data=2).store() - - # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') - - # Use the map_values alias - result = mr.map_values_json().reduce_max().run() - - self.assertEqual(result, [2]) - - def test_reduce_sort(self): - # Add a value to the bucket - bucket = self.client.bucket('bucket') - bucket.new('one', data="value1").store() - bucket.new('two', data="value2").store() - - # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') - - # Use the map_values alias - result = mr.map_values_json().reduce_sort().run() - - self.assertEqual(result, ["value1", "value2"]) - - def test_reduce_sort_custom(self): - # Add a value to the bucket - bucket = self.client.bucket('bucket') - bucket.new('one', data="value1").store() - bucket.new('two', data="value2").store() - - # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') - - # Use the map_values alias - result = mr.map_values_json().reduce_sort("""function(x,y) { - if(x == y) return 0; - return x > y ? -1 : 1; - }""").run() - - self.assertEqual(result, ["value2", "value1"]) - - def test_reduce_numeric_sort(self): - # Add a value to the bucket - bucket = self.client.bucket('bucket') - bucket.new('one', data=1).store() - bucket.new('two', data=2).store() - - # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') - - # Use the map_values alias - result = mr.map_values_json().reduce_numeric_sort().run() - - self.assertEqual(result, [1, 2]) - - def test_reduce_limit(self): - # Add a value to the bucket - bucket = self.client.bucket('bucket') - bucket.new('one', data=1).store() - bucket.new('two', data=2).store() - - # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') - - # Use the map_values alias - result = mr.map_values_json()\ - .reduce_numeric_sort()\ - .reduce_limit(1).run() - - self.assertEqual(result, [1]) - - def test_reduce_slice(self): - # Add a value to the bucket - bucket = self.client.bucket('bucket') - bucket.new('one', data=1).store() - bucket.new('two', data=2).store() - - # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') - - # Use the map_values alias - result = mr.map_values_json()\ - .reduce_numeric_sort()\ - .reduce_slice(1, 2).run() - - self.assertEqual(result, [2]) - - def test_filter_not_found(self): - # Add a value to the bucket - bucket = self.client.bucket('bucket') - bucket.new('one', data=1).store() - bucket.new('two', data=2).store() - - # Make sure "three" does not exist - bucket.get('three').delete() - - # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two')\ - .add('bucket', 'three') - - # Use the map_values alias - result = mr.map_values_json()\ - .filter_not_found()\ - .run() - - self.assertEqual(sorted(result), [1, 2]) - - -class RiakPbcTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, +class RiakPbcTransportTestCase(BasicKVTests, + KVFileTests, + TwoITests, + LinkTests, + ErlangMapReduceTests, + JSMapReduceTests, + MapReduceAliasTests, + SearchTests, + BaseTestCase, unittest.TestCase): def setUp(self): @@ -1112,10 +152,20 @@ def test_close_underlying_socket_retry(self): self.assertTrue(obj.exists()) self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') self.assertEqual(obj.get_key(), 'barbaz') - self.assertEqual(obj.get_data(), rand) - - -class RiakHttpTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, + self.assertEqual(obj.get_data(), rand) + + +class RiakHttpTransportTestCase(BasicKVTests, + KVFileTests, + TwoITests, + LinkTests, + ErlangMapReduceTests, + JSMapReduceTests, + MapReduceAliasTests, + EnableSearchTests, + SolrSearchTests, + SearchTests, + BaseTestCase, unittest.TestCase): def setUp(self): @@ -1140,24 +190,6 @@ def test_too_many_link_headers_shouldnt_break_http(self): stored_object = bucket.get("lots_of_links") self.assertEqual(len(stored_object.get_links()), 400) - def test_bucket_search_enabled(self): - bucket = self.client.bucket("unsearch_bucket") - self.assertFalse(bucket.search_enabled()) - - def test_enable_search_commit_hook(self): - bucket = self.client.bucket("search_bucket") - bucket.enable_search() - self.assertTrue(self.client.bucket("search_bucket").search_enabled()) - - def test_disable_search_commit_hook(self): - bucket = self.client.bucket("no_search_bucket") - bucket.enable_search() - self.assertTrue(self.client.bucket("no_search_bucket")\ - .search_enabled()) - bucket.disable_search() - self.assertFalse(self.client.bucket("no_search_bucket")\ - .search_enabled()) - @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_store_file_with_luwak(self): file = os.path.join(os.path.dirname(__file__), "test_all.py") @@ -1193,65 +225,9 @@ def test_delete_file_with_luwak(self): file = self.client.get_file(key) self.assertIsNone(file) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_add_document_to_index(self): - self.client.solr().add("searchbucket", - {"id": "doc", "username": "tony"}) - results = self.client.solr().search("searchbucket", "username:tony") - self.assertEquals("tony", results['docs'][0]['username']) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_add_multiple_documents_to_index(self): - self.client.solr().add("searchbucket", - {"id": "dizzy", "username": "dizzy"}, - {"id": "russell", "username": "russell"}) - results = self.client.solr()\ - .search("searchbucket", "username:russell OR username:dizzy") - self.assertEquals(2, len(results['docs'])) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_delete_documents_from_search_by_id(self): - self.client.solr().add("searchbucket", - {"id": "dizzy", "username": "dizzy"}, - {"id": "russell", "username": "russell"}) - self.client.solr().delete("searchbucket", docs=["dizzy"]) - results = self.client.solr()\ - .search("searchbucket", "username:russell OR username:dizzy") - self.assertEquals(1, len(results['docs'])) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_delete_documents_from_search_by_query(self): - self.client.solr().add("searchbucket", - {"id": "dizzy", "username": "dizzy"}, - {"id": "russell", "username": "russell"}) - self.client.solr()\ - .delete("searchbucket", - queries=["username:dizzy", "username:russell"]) - results = self.client.solr()\ - .search("searchbucket", "username:russell OR username:dizzy") - self.assertEquals(0, len(results['docs'])) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_delete_documents_from_search_by_query_and_id(self): - self.client.solr().add("searchbucket", - {"id": "dizzy", "username": "dizzy"}, - {"id": "russell", "username": "russell"}) - self.client.solr().delete("searchbucket", - docs=["dizzy"], - queries=["username:russell"]) - results = self.client.solr()\ - .search("searchbucket", - "username:russell OR username:dizzy") - self.assertEquals(0, len(results['docs'])) - - def test_build_rest_path_excludes_empty_query_params(self): - self.assertEquals( - self.client.get_transport().build_rest_path( - bucket=self.client.bucket("foo"), - key="bar", params={'r': None}), "/riak/foo/bar?") -class RiakTestFilter(unittest.TestCase): +class FilterTests(unittest.TestCase): def test_simple(self): f1 = RiakKeyFilter("tokenize", "-", 1) self.assertEqual(f1._filters, [["tokenize", "-", 1]]) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py new file mode 100644 index 00000000..fdf83364 --- /dev/null +++ b/riak/tests/test_kv.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- +import os +import cPickle +import copy +try: + import json +except ImportError: + import simplejson as json + +class NotJsonSerializable(object): + + def __init__(self, *args, **kwargs): + self.args = list(args) + self.kwargs = kwargs + + def __eq__(self, other): + if len(self.args) != len(other.args): + return False + if len(self.kwargs) != len(other.kwargs): + return False + for name, value in self.kwargs.items(): + if other.kwargs[name] != value: + return False + value1_args = copy.copy(self.args) + value2_args = copy.copy(other.args) + value1_args.sort() + value2_args.sort() + for i in xrange(len(value1_args)): + if value1_args[i] != value2_args[i]: + return False + return True + + +class BasicKVTests(object): + def test_is_alive(self): + self.assertTrue(self.client.is_alive()) + + def test_store_and_get(self): + bucket = self.client.bucket('bucket') + rand = self.randint() + obj = bucket.new('foo', rand) + obj.store() + obj = bucket.get('foo') + self.assertTrue(obj.exists()) + self.assertEqual(obj.get_bucket().get_name(), 'bucket') + self.assertEqual(obj.get_key(), 'foo') + self.assertEqual(obj.get_data(), rand) + + # unicode objects are fine, as long as they don't + # contain any non-ASCII chars + self.client.bucket(u'bucket') + self.assertRaises(TypeError, self.client.bucket, u'búcket') + self.assertRaises(TypeError, self.client.bucket, 'búcket') + + bucket.get(u'foo') + self.assertRaises(TypeError, bucket.get, u'føø') + self.assertRaises(TypeError, bucket.get, 'føø') + + self.assertRaises(TypeError, bucket.new, u'foo', 'éå') + self.assertRaises(TypeError, bucket.new, u'foo', 'éå') + self.assertRaises(TypeError, bucket.new, 'foo', u'éå') + self.assertRaises(TypeError, bucket.new, 'foo', u'éå') + + def test_generate_key(self): + # Ensure that Riak generates a random key when + # the key passed to bucket.new() is None. + bucket = self.client.bucket('random_key_bucket') + existing_keys = bucket.get_keys() + o = bucket.new(None, data={}) + self.assertIsNone(o.get_key()) + o.store() + self.assertIsNotNone(o.get_key()) + self.assertNotIn('/', o.get_key()) + self.assertNotIn(o.get_key(), existing_keys) + self.assertEqual(len(bucket.get_keys()), len(existing_keys) + 1) + + def test_binary_store_and_get(self): + bucket = self.client.bucket('bucket') + # Store as binary, retrieve as binary, then compare... + rand = str(self.randint()) + obj = bucket.new_binary('foo1', rand) + obj.store() + obj = bucket.get_binary('foo1') + self.assertTrue(obj.exists()) + self.assertEqual(obj.get_data(), rand) + # Store as JSON, retrieve as binary, JSON-decode, then compare... + data = [self.randint(), self.randint(), self.randint()] + obj = bucket.new('foo2', data) + obj.store() + obj = bucket.get_binary('foo2') + self.assertEqual(data, json.loads(obj.get_data())) + + def test_custom_bucket_encoder_decoder(self): + # Teach the bucket how to pickle + bucket = self.client.bucket("picklin_bucket") + bucket.set_encoder('application/x-pickle', cPickle.dumps) + bucket.set_decoder('application/x-pickle', cPickle.loads) + data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} + obj = bucket.new("foo", data, 'application/x-pickle').store() + obj.store() + obj2 = bucket.get("foo") + self.assertEqual(data, obj2.get_data()) + + def test_custom_client_encoder_decoder(self): + # Teach the bucket how to pickle + bucket = self.client.bucket("picklin_client") + self.client.set_encoder('application/x-pickle', cPickle.dumps) + self.client.set_decoder('application/x-pickle', cPickle.loads) + data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} + obj = bucket.new("foo", data, 'application/x-pickle').store() + obj.store() + obj2 = bucket.get("foo") + self.assertEqual(data, obj2.get_data()) + + def test_unknown_content_type_encoder_decoder(self): + # Teach the bucket how to pickle + bucket = self.client.bucket("unknown_contenttype") + data = "some funny data" + obj = bucket.new("foo", data, 'application/x-frobnicator').store() + obj.store() + obj2 = bucket.get("foo") + self.assertEqual(data, obj2.get_data()) + + def test_missing_object(self): + bucket = self.client.bucket('bucket') + obj = bucket.get("missing") + self.assertFalse(obj.exists()) + self.assertEqual(obj.get_data(), None) + + def test_delete(self): + bucket = self.client.bucket('bucket') + rand = self.randint() + obj = bucket.new('foo', rand) + obj.store() + obj = bucket.get('foo') + self.assertTrue(obj.exists()) + obj.delete() + obj.reload() + self.assertFalse(obj.exists()) + + def test_set_bucket_properties(self): + bucket = self.client.bucket('bucket') + # Test setting allow mult... + bucket.set_allow_multiples(True) + self.assertTrue(bucket.get_allow_multiples()) + # Test setting nval... + bucket.set_n_val(3) + self.assertEqual(bucket.get_n_val(), 3) + # Test setting multiple properties... + bucket.set_properties({"allow_mult": False, "n_val": 2}) + self.assertFalse(bucket.get_allow_multiples()) + self.assertEqual(bucket.get_n_val(), 2) + + def test_rw_settings(self): + bucket = self.client.bucket('rwsettings') + self.assertEqual(bucket.get_r(), "default") + self.assertEqual(bucket.get_w(), "default") + self.assertEqual(bucket.get_dw(), "default") + self.assertEqual(bucket.get_rw(), "default") + + bucket.set_w(1) + self.assertEqual(bucket.get_w(), 1) + + bucket.set_r("quorum") + self.assertEqual(bucket.get_r(), "quorum") + + bucket.set_dw("all") + self.assertEqual(bucket.get_dw(), "all") + + bucket.set_rw("one") + self.assertEqual(bucket.get_rw(), "one") + + def test_primary_quora(self): + bucket = self.client.bucket('primary_quora') + self.assertEqual(bucket.get_pr(), "default") + self.assertEqual(bucket.get_pw(), "default") + + bucket.set_pr(1) + self.assertEqual(bucket.get_pr(), 1) + + bucket.set_pw("quorum") + self.assertEqual(bucket.get_pw(), "quorum") + + def test_if_none_match(self): + bucket = self.client.bucket('if_none_match_test') + obj = bucket.get('obj') + obj.delete() + + obj.reload() + self.assertFalse(obj.exists()) + obj.set_data(["first store"]) + obj.store() + + obj.set_data(["second store"]) + with self.assertRaises(Exception): + obj.store(if_none_match=True) + + def test_siblings(self): + # Set up the bucket, clear any existing object... + bucket = self.client.bucket('multiBucket') + bucket.set_allow_multiples(True) + obj = bucket.get_binary('foo') + # Even if it previously existed, let's store a base resolved version + # from which we can diverge by sending a stale vclock. + obj.set_data('start') + obj.store() + + # Store the same object five times... + vals = set() + for i in range(5): + other_client = self.create_client() + other_bucket = other_client.bucket('multiBucket') + while True: + randval = self.randint() + if randval not in vals: + break + + other_obj = other_bucket.new_binary('foo', str(randval)) + other_obj._vclock = obj._vclock + other_obj.store() + vals.add(str(randval)) + + # Make sure the object has itself plus four siblings... + obj.reload() + self.assertTrue(obj.has_siblings()) + self.assertEqual(obj.get_sibling_count(), 5) + + # Get each of the values - make sure they match what was assigned + vals2 = set() + for i in range(5): + vals2.add(obj.get_sibling(i).get_data()) + self.assertEqual(vals, vals2) + + # Resolve the conflict, and then do a get... + obj3 = obj.get_sibling(3) + obj3.store() + + obj.reload() + self.assertEqual(obj.get_sibling_count(), 0) + self.assertEqual(obj.get_data(), obj3.get_data()) + + def test_store_of_missing_object(self): + bucket = self.client.bucket("bucket") + # for json objects + o = bucket.get("nonexistent_key_json") + self.assertEqual(o.exists(), False) + o.set_data({"foo": "bar"}) + o = o.store() + self.assertEqual(o.get_data(), {"foo": "bar"}) + self.assertEqual(o.get_content_type(), "application/json") + o.delete() + # for binary objects + o = bucket.get_binary("nonexistent_key_binary") + self.assertEqual(o.exists(), False) + o.set_data("1234567890") + o = o.store() + self.assertEqual(o.get_data(), "1234567890") + self.assertEqual(o.get_content_type(), "application/octet-stream") + o.delete() + + def test_store_metadata(self): + bucket = self.client.bucket('bucket') + rand = self.randint() + obj = bucket.new('fooster', rand) + obj.set_usermeta({'custom': 'some metadata'}) + obj.store() + obj = bucket.get('fooster') + self.assertEqual('some metadata', obj.get_usermeta()['custom']) + + def test_list_buckets(self): + bucket = self.client.bucket("list_bucket") + bucket.new("one", {"foo": "one", "bar": "red"}).store() + buckets = self.client.get_buckets() + self.assertTrue("list_bucket" in buckets) + + +class KVFileTests(object): + def test_store_binary_object_from_file(self): + bucket = self.client.bucket('bucket') + rand = str(self.randint()) + filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') + obj = bucket.new_binary_from_file('foo_from_file', filepath) + obj.store() + obj = bucket.get_binary('foo_from_file') + self.assertNotEqual(obj.get_data(), None) + self.assertEqual(obj.get_content_type(), "text/x-python") + + def test_store_binary_object_from_file_should_use_default_mimetype(self): + bucket = self.client.bucket('bucket') + rand = str(self.randint()) + filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), + os.pardir, os.pardir, 'THANKS') + obj = bucket.new_binary_from_file('foo_from_file', filepath) + obj.store() + obj = bucket.get_binary('foo_from_file') + self.assertEqual(obj.get_content_type(), 'application/octet-stream') + + def test_store_binary_object_from_file_should_fail_if_file_not_found(self): + bucket = self.client.bucket('bucket') + rand = str(self.randint()) + self.assertRaises(IOError, bucket.new_binary_from_file, + 'not_found_from_file', 'FILE_NOT_FOUND') + obj = bucket.get_binary('not_found_from_file') + self.assertEqual(obj.get_data(), None) diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py new file mode 100644 index 00000000..c374287f --- /dev/null +++ b/riak/tests/test_mapreduce.py @@ -0,0 +1,419 @@ +# -*- coding: utf-8 -*- + +from riak.mapreduce import RiakLink +from riak import RiakKeyFilter, key_filter + +class LinkTests(object): + def test_store_and_get_links(self): + # Create the object... + bucket = self.client.bucket("bucket") + bucket.new_binary("test_store_and_get_links", '2') \ + .add_link(bucket.new("foo1")) \ + .add_link(bucket.new("foo2"), "tag") \ + .add_link(bucket.new("foo3"), "tag2!@#%^&*)") \ + .store() + obj = bucket.get("test_store_and_get_links") + links = obj.get_links() + self.assertEqual(len(links), 3) + for l in links: + if (l.get_key() == "foo1"): + self.assertEqual(l.get_tag(), "bucket") + elif (l.get_key() == "foo2"): + self.assertEqual(l.get_tag(), "tag") + elif (l.get_key() == "foo3"): + self.assertEqual(l.get_tag(), "tag2!@#%^&*)") + else: + self.assertEqual("unknown key", l.get_key()) + + def test_set_links(self): + # Create the object + bucket = self.client.bucket("bucket") + bucket.new("foo", 2).set_links([bucket.new("foo1"), + (bucket.new("foo2"), "tag"), + RiakLink("bucket", "foo2", "tag2")]).store() + obj = bucket.get("foo") + links = sorted(obj.get_links(), key=lambda x: x.get_key()) + self.assertEqual(len(links), 3) + self.assertEqual(links[0].get_key(), "foo1") + self.assertEqual(links[1].get_key(), "foo2") + self.assertEqual(links[1].get_tag(), "tag") + self.assertEqual(links[2].get_key(), "foo2") + self.assertEqual(links[2].get_tag(), "tag2") + + def test_set_links_all_links(self): + bucket = self.client.bucket("bucket") + foo1 = bucket.new("foo", 1) + foo2 = bucket.new("foo2", 2).store() + links = [RiakLink("bucket", "foo2")] + foo1.set_links(links, True) + links = foo1.get_links() + self.assertEqual(len(links), 1) + self.assertEqual(links[0].get_key(), "foo2") + + def test_link_walking(self): + # Create the object... + bucket = self.client.bucket("bucket") + bucket.new("foo", 2) \ + .add_link(bucket.new("foo1", "test1").store()) \ + .add_link(bucket.new("foo2", "test2").store(), "tag") \ + .add_link(bucket.new("foo3", "test3").store(), "tag2!@#%^&*)") \ + .store() + obj = bucket.get("foo") + results = obj.link("bucket").run() + self.assertEqual(len(results), 3) + results = obj.link("bucket", "tag").run() + self.assertEqual(len(results), 1) + + +class ErlangMapReduceTests(object): + def test_erlang_map_reduce(self): + # Create the object... + bucket = self.client.bucket("bucket") + bucket.new("foo", 2).store() + bucket.new("bar", 2).store() + bucket.new("baz", 4).store() + # Run the map... + result = self.client \ + .add("bucket", "foo") \ + .add("bucket", "bar") \ + .add("bucket", "baz") \ + .map(["riak_kv_mapreduce", "map_object_value"]) \ + .reduce(["riak_kv_mapreduce", "reduce_set_union"]) \ + .run() + self.assertEqual(len(result), 2) + + +class JSMapReduceTests(object): + def test_javascript_source_map(self): + # Create the object... + bucket = self.client.bucket("bucket") + bucket.new("foo", 2).store() + # Run the map... + mr = self.client.add("bucket", "foo") + result = mr.map( + "function (v) { return [JSON.parse(v.values[0].data)]; }").run() + self.assertEqual(result, [2]) + + # test ASCII-encodable unicode is accepted + mr.map(u"function (v) { return [JSON.parse(v.values[0].data)]; }") + + # test non-ASCII-encodable unicode is rejected + self.assertRaises(TypeError, mr.map, + u"function (v) { /* æ */ return [JSON.parse(v.values[0].data)]; }") + + # test non-ASCII-encodable string is rejected + self.assertRaises(TypeError, mr.map, + "function (v) { /* æ */ return [JSON.parse(v.values[0].data)]; }") + + def test_javascript_named_map(self): + # Create the object... + bucket = self.client.bucket("bucket") + bucket.new("foo", 2).store() + # Run the map... + result = self.client \ + .add("bucket", "foo") \ + .map("Riak.mapValuesJson") \ + .run() + self.assertEqual(result, [2]) + + def test_javascript_source_map_reduce(self): + # Create the object... + bucket = self.client.bucket("bucket") + bucket.new("foo", 2).store() + bucket.new("bar", 3).store() + bucket.new("baz", 4).store() + # Run the map... + result = self.client \ + .add("bucket", "foo") \ + .add("bucket", "bar") \ + .add("bucket", "baz") \ + .map("function (v) { return [1]; }") \ + .reduce("Riak.reduceSum") \ + .run() + self.assertEqual(result, [3]) + + def test_javascript_named_map_reduce(self): + # Create the object... + bucket = self.client.bucket("bucket") + bucket.new("foo", 2).store() + bucket.new("bar", 3).store() + bucket.new("baz", 4).store() + # Run the map... + result = self.client \ + .add("bucket", "foo") \ + .add("bucket", "bar") \ + .add("bucket", "baz") \ + .map("Riak.mapValuesJson") \ + .reduce("Riak.reduceSum") \ + .run() + self.assertEqual(result, [9]) + + def test_javascript_bucket_map_reduce(self): + # Create the object... + bucket = self.client.bucket("bucket_%s" % self.randint()) + bucket.new("foo", 2).store() + bucket.new("bar", 3).store() + bucket.new("baz", 4).store() + # Run the map... + result = self.client \ + .add(bucket.get_name()) \ + .map("Riak.mapValuesJson") \ + .reduce("Riak.reduceSum") \ + .run() + self.assertEqual(result, [9]) + + def test_javascript_arg_map_reduce(self): + # Create the object... + bucket = self.client.bucket("bucket") + bucket.new("foo", 2).store() + # Run the map... + result = self.client \ + .add("bucket", "foo", 5) \ + .add("bucket", "foo", 10) \ + .add("bucket", "foo", 15) \ + .add("bucket", "foo", -15) \ + .add("bucket", "foo", -5) \ + .map("function(v, arg) { return [arg]; }") \ + .reduce("Riak.reduceSum") \ + .run() + self.assertEqual(result, [10]) + + def test_key_filters(self): + bucket = self.client.bucket("kftest") + bucket.new("basho-20101215", 1).store() + bucket.new("google-20110103", 2).store() + bucket.new("yahoo-20090613", 3).store() + + result = self.client \ + .add("kftest") \ + .add_key_filters([["tokenize", "-", 2]]) \ + .add_key_filter("ends_with", "0613") \ + .map("function (v, keydata) { return [v.key]; }") \ + .run() + + self.assertEqual(result, ["yahoo-20090613"]) + + def test_key_filters_f_chain(self): + bucket = self.client.bucket("kftest") + bucket.new("basho-20101215", 1).store() + bucket.new("google-20110103", 2).store() + bucket.new("yahoo-20090613", 3).store() + + # compose a chain of key filters using f as the root of + # two filters ANDed together to ensure that f can be the root + # of multiple chains + filters = key_filter.tokenize("-", 1).eq("yahoo") \ + & key_filter.tokenize("-", 2).ends_with("0613") + + result = self.client \ + .add("kftest") \ + .add_key_filters(filters) \ + .map("function (v, keydata) { return [v.key]; }") \ + .run() + + self.assertEqual(result, ["yahoo-20090613"]) + + def test_key_filters_with_search_query(self): + mapreduce = self.client.search("kftest", "query") + self.assertRaises(Exception, mapreduce.add_key_filters, + [["tokenize", "-", 2]]) + self.assertRaises(Exception, mapreduce.add_key_filter, + "ends_with", "0613") + + def test_map_reduce_from_object(self): + # Create the object... + bucket = self.client.bucket("bucket") + bucket.new("foo", 2).store() + obj = bucket.get("foo") + result = obj.map("Riak.mapValuesJson").run() + self.assertEqual(result, [2]) + + +class MapReduceAliasTests(object): + """This tests the map reduce aliases""" + + def test_map_values(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new_binary('one', data='value_1').store() + bucket.new_binary('two', data='value_2').store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values().run() + + # Sort the result so that we can have a consistent + # expected value + result.sort() + + self.assertEqual(result, ["value_1", "value_2"]) + + def test_map_values_json(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data={'val': 'value_1'}).store() + bucket.new('two', data={'val': 'value_2'}).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().run() + + # Sort the result so that we can have a consistent + # expected value + result.sort(key=lambda x: x['val']) + + self.assertEqual(result, [{'val': "value_1"}, {'val': "value_2"}]) + + def test_reduce_sum(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().reduce_sum().run() + + self.assertEqual(result, [3]) + + def test_reduce_min(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().reduce_min().run() + + self.assertEqual(result, [1]) + + def test_reduce_max(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().reduce_max().run() + + self.assertEqual(result, [2]) + + def test_reduce_sort(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data="value1").store() + bucket.new('two', data="value2").store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().reduce_sort().run() + + self.assertEqual(result, ["value1", "value2"]) + + def test_reduce_sort_custom(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data="value1").store() + bucket.new('two', data="value2").store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().reduce_sort("""function(x,y) { + if(x == y) return 0; + return x > y ? -1 : 1; + }""").run() + + self.assertEqual(result, ["value2", "value1"]) + + def test_reduce_numeric_sort(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json().reduce_numeric_sort().run() + + self.assertEqual(result, [1, 2]) + + def test_reduce_limit(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json()\ + .reduce_numeric_sort()\ + .reduce_limit(1).run() + + self.assertEqual(result, [1]) + + def test_reduce_slice(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two') + + # Use the map_values alias + result = mr.map_values_json()\ + .reduce_numeric_sort()\ + .reduce_slice(1, 2).run() + + self.assertEqual(result, [2]) + + def test_filter_not_found(self): + # Add a value to the bucket + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + # Make sure "three" does not exist + bucket.get('three').delete() + + # Create a map reduce object and use one and two as inputs + mr = self.client.add('bucket', 'one')\ + .add('bucket', 'two')\ + .add('bucket', 'three') + + # Use the map_values alias + result = mr.map_values_json()\ + .filter_not_found()\ + .run() + + self.assertEqual(sorted(result), [1, 2]) diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py new file mode 100644 index 00000000..697e17f1 --- /dev/null +++ b/riak/tests/test_search.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +import os +import platform +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest + +SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '0')) + +class EnableSearchTests(object): + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_bucket_search_enabled(self): + bucket = self.client.bucket("unsearch_bucket") + self.assertFalse(bucket.search_enabled()) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_enable_search_commit_hook(self): + bucket = self.client.bucket("search_bucket") + bucket.enable_search() + self.assertTrue(self.client.bucket("search_bucket").search_enabled()) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_disable_search_commit_hook(self): + bucket = self.client.bucket("no_search_bucket") + bucket.enable_search() + self.assertTrue(self.client.bucket("no_search_bucket")\ + .search_enabled()) + bucket.disable_search() + self.assertFalse(self.client.bucket("no_search_bucket")\ + .search_enabled()) + + +class SolrSearchTests(object): + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_add_document_to_index(self): + self.client.solr().add("searchbucket", + {"id": "doc", "username": "tony"}) + results = self.client.solr().search("searchbucket", "username:tony") + self.assertEquals("tony", results['docs'][0]['username']) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_add_multiple_documents_to_index(self): + self.client.solr().add("searchbucket", + {"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}) + results = self.client.solr()\ + .search("searchbucket", "username:russell OR username:dizzy") + self.assertEquals(2, len(results['docs'])) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_delete_documents_from_search_by_id(self): + self.client.solr().add("searchbucket", + {"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}) + self.client.solr().delete("searchbucket", docs=["dizzy"]) + results = self.client.solr()\ + .search("searchbucket", "username:russell OR username:dizzy") + self.assertEquals(1, len(results['docs'])) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_delete_documents_from_search_by_query(self): + self.client.solr().add("searchbucket", + {"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}) + self.client.solr()\ + .delete("searchbucket", + queries=["username:dizzy", "username:russell"]) + results = self.client.solr()\ + .search("searchbucket", "username:russell OR username:dizzy") + self.assertEquals(0, len(results['docs'])) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_delete_documents_from_search_by_query_and_id(self): + self.client.solr().add("searchbucket", + {"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}) + self.client.solr().delete("searchbucket", + docs=["dizzy"], + queries=["username:russell"]) + results = self.client.solr()\ + .search("searchbucket", + "username:russell OR username:dizzy") + self.assertEquals(0, len(results['docs'])) + + def test_build_rest_path_excludes_empty_query_params(self): + self.assertEquals( + self.client.get_transport().build_rest_path( + bucket=self.client.bucket("foo"), + key="bar", params={'r': None}), "/riak/foo/bar?") + + +class SearchTests(object): + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_solr_search_from_bucket(self): + bucket = self.client.bucket('searchbucket') + bucket.new("user", {"username": "roidrage"}).store() + results = bucket.search("username:roidrage") + self.assertEquals(1, len(results['docs'])) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_solr_search_with_params_from_bucket(self): + bucket = self.client.bucket('searchbucket') + bucket.new("user", {"username": "roidrage"}).store() + results = bucket.search("username:roidrage", wt="xml") + self.assertEquals(1, len(results['docs'])) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_solr_search_with_params(self): + bucket = self.client.bucket('searchbucket') + bucket.new("user", {"username": "roidrage"}).store() + results = self.client.solr().search("searchbucket", + "username:roidrage", wt="xml") + self.assertEquals(1, len(results['docs'])) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_solr_search(self): + bucket = self.client.bucket('searchbucket') + bucket.new("user", {"username": "roidrage"}).store() + results = self.client.solr().search("searchbucket", + "username:roidrage") + self.assertEquals(1, len(results["docs"])) + + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + def test_search_integration(self): + # Create some objects to search across... + bucket = self.client.bucket("searchbucket") + bucket.new("one", {"foo": "one", "bar": "red"}).store() + bucket.new("two", {"foo": "two", "bar": "green"}).store() + bucket.new("three", {"foo": "three", "bar": "blue"}).store() + bucket.new("four", {"foo": "four", "bar": "orange"}).store() + bucket.new("five", {"foo": "five", "bar": "yellow"}).store() + + # Run some operations... + results = self.client.solr().search("searchbucket", + "foo:one OR foo:two") + if (len(results) == 0): + print "\n\nNot running test \"testSearchIntegration()\".\n" + print """Please ensure that you have installed the Riak + Search hook on bucket \"searchbucket\" by running + \"bin/search-cmd install searchbucket\".\n\n""" + return + self.assertEqual(len(results['docs']), 2) + query = "(foo:one OR foo:two OR foo:three OR foo:four) AND\ + (NOT bar:green)" + results = self.client.solr().search("searchbucket", query) + self.assertEqual(len(results['docs']), 3) From 8e73f899be1dd7c4ed779c80968bafc707356007 Mon Sep 17 00:00:00 2001 From: evan Date: Mon, 5 Nov 2012 08:18:05 -0800 Subject: [PATCH 0230/1060] missed newline at end of gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 99bba8c8..34e7a5bb 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ riak.egg-info/ *.egg #*# -*~ \ No newline at end of file +*~ From 500edb37339b0a557a54d4a4b611e01cba461c14 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 5 Nov 2012 14:12:55 -0500 Subject: [PATCH 0231/1060] Add iteration to the pool. --- riak/tests/test_pool.py | 35 ++++++++++++++++++++++++++++ riak/transports/pool.py | 51 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index d2f4f98d..b20008eb 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -20,6 +20,8 @@ from Queue import Queue from threading import Thread, currentThread from riak.transports.pool import Pool, BadResource +from random import SystemRandom +from time import sleep if platform.python_version() < '2.7': unittest = __import__('unittest2') @@ -153,5 +155,38 @@ def _run(): self.assertFalse(element.claimed) self.assertEqual(1, len(element.object)) + def test_iteration(self): + started = Queue() + n = 30 + threads = [] + touched = [] + pool = EmptyListPool() + rand = SystemRandom() + + def _run(): + psleep = rand.uniform(0, 0.75) + with pool.take() as a: + started.put(1) + started.join() + a.append(rand.uniform(0, 1)) + sleep(psleep) + + for i in range(n): + th = Thread(target=_run) + threads.append(th) + th.start() + + for i in range(n): + started.get() + started.task_done() + + for element in pool: + touched.append(element) + + for thr in threads: + thr.join() + + self.assertItemsEqual(pool.elements, touched) + if __name__ == '__main__': unittest.main() diff --git a/riak/transports/pool.py b/riak/transports/pool.py index 4944b36a..c376ad5c 100644 --- a/riak/transports/pool.py +++ b/riak/transports/pool.py @@ -83,6 +83,7 @@ def __init__(self): override the __init__ method in a subclass. """ self.lock = threading.Lock() + self.releaser = threading.Condition() self.elements = list() @contextmanager @@ -124,7 +125,9 @@ def _filter(obj): self.delete_element(element) raise finally: - element.claimed = False + with self.releaser: + element.claimed = False + self.releaser.notify() def delete_element(self, element): """ @@ -140,12 +143,15 @@ def delete_element(self, element): self.destroy_resource(element.object) del element + def __iter__(self): + return PoolIterator(self) + def create_resource(self): """ Implemented by subclasses to allocate a new resource for use in the pool. """ - raise NotImplemented + raise NotImplementedError def destroy_resource(self, obj): """ @@ -157,3 +163,44 @@ def destroy_resource(self, obj): :param obj: the resource being removed """ pass + + +class PoolIterator(object): + """ + Iterates over a snapshot of the pool in a thread-safe manner, + eventually touching all resources that were known when the + iteration started. + """ + + def __init__(self, pool): + self.targets = pool.elements[:] + self.unlocked = [] + self.lock = pool.lock + self.releaser = pool.releaser + + def __iter__(self): + return self + + def next(self): + if len(self.targets) == 0: + raise StopIteration + if len(self.unlocked) == 0: + self.reclaim() + return self.unlocked.pop(0) + + def reclaim(self): + with self.lock: + if self.all_claimed(): + with self.releaser: + self.releaser.wait() + for element in self.targets[:]: + if not element.claimed: + self.targets.remove(element) + self.unlocked.append(element) + element.claimed = True + + def all_claimed(self): + for element in self.targets[:]: + if not element.claimed: + return False + return True From 36a24514207413f5fd4e1da27342993d5e701edc Mon Sep 17 00:00:00 2001 From: evan Date: Mon, 5 Nov 2012 08:42:46 -0800 Subject: [PATCH 0232/1060] change all of the transport base class RiakErrors to NotImplemented exceptions. Add a test that catches when they are thrown. --- riak/tests/test_all.py | 10 +++++++++- riak/transports/transport.py | 38 ++++++++++++++++++------------------ 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 29497b29..1663ab34 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -152,8 +152,16 @@ def test_close_underlying_socket_retry(self): self.assertTrue(obj.exists()) self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') self.assertEqual(obj.get_key(), 'barbaz') - self.assertEqual(obj.get_data(), rand) + self.assertEqual(obj.get_data(), rand) + + def test_bucket_search_enabled(self): + bucket = self.client.bucket("unsearch_bucket") + self.assertRaises(NotImplemented) + def test_enable_search_commit_hook(self): + bucket = self.client.bucket("search_bucket") + bucket.enable_search() + self.assertRaises(NotImplemented) class RiakHttpTransportTestCase(BasicKVTests, KVFileTests, diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 075274e6..32ccbfef 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -60,14 +60,14 @@ def ping(self): Ping the remote server @return boolean """ - raise RiakError("not implemented") + raise NotImplemented def get(self, robj, r=None, vtag=None): """ Serialize get request and deserialize response @return (vclock=None, [(metadata, value)]=None) """ - raise RiakError("not implemented") + raise NotImplemented def put(self, robj, w=None, dw=None, return_body=True): """ @@ -75,7 +75,7 @@ def put(self, robj, w=None, dw=None, return_body=True): is true, retrieve the updated metadata/content @return (vclock=None, [(metadata, value)]=None) """ - raise RiakError("not implemented") + raise NotImplemented def put_new(self, robj, w=None, dw=None, return_meta=True): """Put a new object into the Riak store, returning its (new) key. @@ -85,28 +85,28 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): @return (key, vclock, metadata) """ - raise RiakError("not implemented") + raise NotImplemented def delete(self, robj, rw=None): """ Serialize delete request and deserialize response @return true """ - raise RiakError("not implemented") + raise NotImplemented def get_buckets(self): """ Serialize get buckets request and deserialize response @return dict() """ - raise RiakError("not implemented") + raise NotImplemented def get_bucket_props(self, bucket): """ Serialize get bucket property request and deserialize response @return dict() """ - raise RiakError("not implemented") + raise NotImplemented def set_bucket_props(self, bucket, props): """ @@ -115,13 +115,13 @@ def set_bucket_props(self, bucket, props): props = dictionary of properties @return boolean """ - raise RiakError("not implemented") + raise NotImplemented def mapred(self, inputs, query, timeout=None): """ Serialize map/reduce request """ - raise RiakError("not implemented") + raise NotImplemented def set_client_id(self, client_id): """ @@ -129,25 +129,25 @@ def set_client_id(self, client_id): id, which is automatically generated when none is specified in when creating the transport object. """ - raise RiakError("not implemented") + raise NotImplemented def get_client_id(self): """ Fetch the client id for the transport. """ - raise RiakError("not implemented") + raise NotImplemented def search(self, index, query, **params): """ Performs a search query. """ - raise RiakError("not implemented") + raise NotImplemented def get_index(self, bucket, index, startkey, endkey=None): """ Performs a secondary index query. """ - raise RiakError("not implemented") + raise NotImplemented def _search_mapred_emu(self, index, query): """ @@ -158,9 +158,9 @@ def _search_mapred_emu(self, index, query): phases = [] if not self.phaseless_mapred(): phases.append({'language': 'erlang', - 'module': 'riak_kv_mapreduce', - 'function': 'reduce_identity', - 'keep': True}) + 'module': 'riak_kv_mapreduce', + 'function': 'reduce_identity', + 'keep': True}) mr_result = self.mapred({'module': 'riak_search', 'function': 'mapred_search', 'arg': [index, query]}, @@ -207,18 +207,18 @@ def store_file(self, key, content_type="application/octet-stream", content_type = the object's content type content = the object's data """ - raise RiakError("luwak not supported by this transport.") + raise NotImplemented def get_file(self, key): """ Get an object from luwak. key = the object's key """ - raise RiakError("luwak not supported by this transport.") + raise NotImplemented def delete_file(self, key): """ Delete an object in luwak. key = the object's key """ - raise RiakError("luwak not supported by this transport.") + raise NotImplemented From 93d1524c80d5ad63c2bba1778b83d8aa5cc8f287 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 5 Nov 2012 16:05:58 -0500 Subject: [PATCH 0233/1060] NotImplemented -> NotImplementedError (as recommended by Python docs). --- riak/tests/test_all.py | 4 ++-- riak/transports/transport.py | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 1663ab34..21d67b65 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -156,12 +156,12 @@ def test_close_underlying_socket_retry(self): def test_bucket_search_enabled(self): bucket = self.client.bucket("unsearch_bucket") - self.assertRaises(NotImplemented) + self.assertRaises(NotImplementedError) def test_enable_search_commit_hook(self): bucket = self.client.bucket("search_bucket") bucket.enable_search() - self.assertRaises(NotImplemented) + self.assertRaises(NotImplementedError) class RiakHttpTransportTestCase(BasicKVTests, KVFileTests, diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 32ccbfef..481020d1 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -60,14 +60,14 @@ def ping(self): Ping the remote server @return boolean """ - raise NotImplemented + raise NotImplementedError def get(self, robj, r=None, vtag=None): """ Serialize get request and deserialize response @return (vclock=None, [(metadata, value)]=None) """ - raise NotImplemented + raise NotImplementedError def put(self, robj, w=None, dw=None, return_body=True): """ @@ -75,7 +75,7 @@ def put(self, robj, w=None, dw=None, return_body=True): is true, retrieve the updated metadata/content @return (vclock=None, [(metadata, value)]=None) """ - raise NotImplemented + raise NotImplementedError def put_new(self, robj, w=None, dw=None, return_meta=True): """Put a new object into the Riak store, returning its (new) key. @@ -85,28 +85,28 @@ def put_new(self, robj, w=None, dw=None, return_meta=True): @return (key, vclock, metadata) """ - raise NotImplemented + raise NotImplementedError def delete(self, robj, rw=None): """ Serialize delete request and deserialize response @return true """ - raise NotImplemented + raise NotImplementedError def get_buckets(self): """ Serialize get buckets request and deserialize response @return dict() """ - raise NotImplemented + raise NotImplementedError def get_bucket_props(self, bucket): """ Serialize get bucket property request and deserialize response @return dict() """ - raise NotImplemented + raise NotImplementedError def set_bucket_props(self, bucket, props): """ @@ -115,13 +115,13 @@ def set_bucket_props(self, bucket, props): props = dictionary of properties @return boolean """ - raise NotImplemented + raise NotImplementedError def mapred(self, inputs, query, timeout=None): """ Serialize map/reduce request """ - raise NotImplemented + raise NotImplementedError def set_client_id(self, client_id): """ @@ -129,25 +129,25 @@ def set_client_id(self, client_id): id, which is automatically generated when none is specified in when creating the transport object. """ - raise NotImplemented + raise NotImplementedError def get_client_id(self): """ Fetch the client id for the transport. """ - raise NotImplemented + raise NotImplementedError def search(self, index, query, **params): """ Performs a search query. """ - raise NotImplemented + raise NotImplementedError def get_index(self, bucket, index, startkey, endkey=None): """ Performs a secondary index query. """ - raise NotImplemented + raise NotImplementedError def _search_mapred_emu(self, index, query): """ @@ -207,18 +207,18 @@ def store_file(self, key, content_type="application/octet-stream", content_type = the object's content type content = the object's data """ - raise NotImplemented + raise NotImplementedError def get_file(self, key): """ Get an object from luwak. key = the object's key """ - raise NotImplemented + raise NotImplementedError def delete_file(self, key): """ Delete an object in luwak. key = the object's key """ - raise NotImplemented + raise NotImplementedError From 57bf2382283117488c48ebf75bcb3be3f6952d81 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 5 Nov 2012 19:11:17 -0500 Subject: [PATCH 0234/1060] Add a method to clear the pool. * Document each test case in the pool suite. * Rename reclaim -> claim_elements in the PoolIterator. * Copy the pool elements inside the lock to avoid some races. --- riak/tests/test_pool.py | 96 +++++++++++++++++++++++++++++++++++++++++ riak/transports/pool.py | 18 ++++++-- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index b20008eb..ec7cfa1a 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -38,6 +38,9 @@ def create_resource(self): self.count += 1 return [self.count] + def destroy_resource(self, resource): + del resource[:] + class EmptyListPool(Pool): def create_resource(self): @@ -46,11 +49,18 @@ def create_resource(self): class PoolTest(unittest.TestCase): def test_yields_new_object_when_empty(self): + """ + The pool should create new resources as needed. + """ pool = SimplePool() with pool.take() as element: self.assertEqual([1], element) def test_yields_same_object_in_serial_access(self): + """ + The pool should reuse resources that already exist, when used + serially. + """ pool = SimplePool() with pool.take() as element: @@ -64,6 +74,10 @@ def test_yields_same_object_in_serial_access(self): self.assertEqual(1, len(pool.elements)) def test_reentrance(self): + """ + The pool should be re-entrant, that is, yield new resources + while one is already claimed in the same code path. + """ pool = SimplePool() with pool.take() as first: self.assertEqual([1], first) @@ -73,6 +87,10 @@ def test_reentrance(self): self.assertEqual([3], third) def test_unlocks_when_exception_raised(self): + """ + The pool should unlock all resources that were previously + claimed when an exception occurs. + """ pool = SimplePool() try: with pool.take() as x: @@ -84,6 +102,10 @@ def test_unlocks_when_exception_raised(self): self.assertFalse(e.claimed) def test_removes_bad_resource(self): + """ + The pool should remove resources that are considered bad by + user code throwing a BadResource exception. + """ pool = SimplePool() with pool.take() as element: self.assertEqual([1], element) @@ -97,6 +119,10 @@ def test_removes_bad_resource(self): self.assertEqual([2], goodie) def test_filter_skips_unmatching_elements(self): + """ + The _filter parameter should cause the pool to yield the first + unclaimed resource that passes the filter. + """ def filtereven(numlist): return numlist[0] % 2 == 0 @@ -109,6 +135,10 @@ def filtereven(numlist): self.assertEqual([2], f) def test_requires_filter_to_be_callable(self): + """ + The _filter parameter should be required to be a callable, or + None. + """ badfilter = 'foo' pool = SimplePool() @@ -117,11 +147,18 @@ def test_requires_filter_to_be_callable(self): pass def test_yields_default_when_empty(self): + """ + The pool should yield the given default when no existing + resources are free. + """ pool = SimplePool() with pool.take(default='default') as x: self.assertEqual('default', x) def test_thread_safety(self): + """ + The pool should allocate n objects for n concurrent operations. + """ n = 10 pool = EmptyListPool() readyq = Queue() @@ -154,8 +191,14 @@ def _run(): for element in pool.elements: self.assertFalse(element.claimed) self.assertEqual(1, len(element.object)) + self.assertIn(element.object[0], threads) def test_iteration(self): + """ + Iteration over the pool resources, even when some are claimed, + should eventually touch all resources (excluding ones created + during iteration). + """ started = Queue() n = 30 threads = [] @@ -188,5 +231,58 @@ def _run(): self.assertItemsEqual(pool.elements, touched) + def test_clear(self): + """ + Clearing the pool should remove all resources known at the + time of the call. + """ + n = 10 + startq = Queue() + finishq = Queue() + rand = SystemRandom() + threads = [] + pusher = None + pool = SimplePool() + + def worker_run(): + with pool.take() as a: + startq.put(1) + startq.join() + sleep(rand.uniform(0, 0.5)) + finishq.get() + finishq.task_done() + + def pusher_run(): + for i in range(n): + finishq.put(1) + sleep(rand.uniform(0, 0.1)) + finishq.join() + + # Allocate 10 resources in the pool by spinning up 10 threads + for i in range(n): + th = Thread(target=worker_run) + threads.append(th) + th.start() + + # Pull everything off the queue, allowing the workers to run + for i in range(n): + startq.get() + startq.task_done() + + # Start the pusher that will allow them to proceed and exit + pusher = Thread(target=pusher_run) + threads.append(pusher) + pusher.start() + + # Clear the pool + pool.clear() + + # Wait for all threads to complete + for t in threads: + t.join() + + # Make sure that the pool resources are gone + self.assertEqual(0, len(pool.elements)) + if __name__ == '__main__': unittest.main() diff --git a/riak/transports/pool.py b/riak/transports/pool.py index c376ad5c..090354bd 100644 --- a/riak/transports/pool.py +++ b/riak/transports/pool.py @@ -144,8 +144,19 @@ def delete_element(self, element): del element def __iter__(self): + """ + Iterator callback to iterate over the elements of the pool. + """ return PoolIterator(self) + def clear(self): + """ + Removes all resources from the pool, calling delete_element + with each one so that the resources are cleaned up. + """ + for element in self: + self.delete_element(element) + def create_resource(self): """ Implemented by subclasses to allocate a new resource for use @@ -173,7 +184,8 @@ class PoolIterator(object): """ def __init__(self, pool): - self.targets = pool.elements[:] + with pool.lock: + self.targets = pool.elements[:] self.unlocked = [] self.lock = pool.lock self.releaser = pool.releaser @@ -185,10 +197,10 @@ def next(self): if len(self.targets) == 0: raise StopIteration if len(self.unlocked) == 0: - self.reclaim() + self.claim_elements() return self.unlocked.pop(0) - def reclaim(self): + def claim_elements(self): with self.lock: if self.all_claimed(): with self.releaser: From eaffbfde964d64c533fc3f1e8a4e469590f2bf30 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 5 Nov 2012 19:28:43 -0500 Subject: [PATCH 0235/1060] Add stress test. --- riak/tests/test_pool.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index ec7cfa1a..f57a2cd3 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -284,5 +284,41 @@ def pusher_run(): # Make sure that the pool resources are gone self.assertEqual(0, len(pool.elements)) + def test_stress(self): + """ + Runs a large number of threads doing operations with elements + checked out, ensuring properties of the pool. + """ + rand = SystemRandom() + n = rand.randint(1, 400) + passes = rand.randint(1, 20) + rounds = rand.randint(1, 200) + breaker = rand.uniform(0, 1) + pool = EmptyListPool() + + def _run(): + for i in range(rounds): + with pool.take() as a: + self.assertEqual([], a) + a.append(currentThread()) + self.assertEqual([currentThread()], a) + + for p in range(passes): + self.assertEqual([currentThread()], a) + if rand.uniform(0, 1) > breaker: + break + + a.remove(currentThread()) + + threads = [] + + for i in range(n): + th = Thread(target=_run) + threads.append(th) + th.start() + + for th in threads: + th.join() + if __name__ == '__main__': unittest.main() From b3e5a9b9549871a70039906e9ecc59cbc80de82c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 6 Nov 2012 17:47:03 -0800 Subject: [PATCH 0236/1060] Deprecate all usages of get_* and set_* methods on quorums. Instead, use bucket properties or request-time quorums. --- riak/bucket.py | 166 ++--------------------------------------- riak/client.py | 136 +-------------------------------- riak/riak_object.py | 18 ----- riak/transports/pbc.py | 40 ++++++---- riak/util.py | 46 ++++++++++++ 5 files changed, 80 insertions(+), 326 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 3b1460aa..6199b575 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -19,8 +19,14 @@ """ from riak_object import RiakObject import mimetypes +from riak.util import deprecateQuorumAccessors as deprecateQuorumAccessors +def deprecateBucketQuorumAccessors(klass): + return deprecateQuorumAccessors(klass, parent='_client') + + +@deprecateBucketQuorumAccessors class RiakBucket(object): """ The ``RiakBucket`` object allows you to access and change information @@ -47,12 +53,6 @@ def __init__(self, client, name): self._client = client self._name = name - self._r = None - self._w = None - self._dw = None - self._rw = None - self._pr = None - self._pw = None self._encoders = {} self._decoders = {} @@ -62,156 +62,6 @@ def get_name(self): """ return self._name - def get_r(self, r=None): - """ - Get the R-value for this bucket, if it is set, otherwise return - the R-value for the client. - - :rtype: integer - """ - if (r is not None): - return r - if (self._r is not None): - return self._r - return self._client.get_r() - - def set_r(self, r): - """ - Set the R-value for this bucket. This value is used by :func:`get` - and :func:`get_binary` operations that do not specify an R-value. - - :param r: The new R-value. - :type r: integer - :rtype: self - """ - self._r = r - return self - - def get_w(self, w=None): - """ - Get the W-value for this bucket, if it is set, otherwise return - the W-value for the client. - - :rtype: integer - """ - if (w is not None): - return w - if (self._w is not None): - return self._w - return self._client.get_w() - - def set_w(self, w): - """ - Set the W-value for this bucket. See :func:`set_r` for - more information. - - :param w: The new W-value. - :type w: integer - :rtype: self - """ - self._w = w - return self - - def get_dw(self, dw=None): - """ - Get the DW-value for this bucket, if it is set, otherwise return - the DW-value for the client. - - :rtype: integer - """ - if (dw is not None): - return dw - if (self._dw is not None): - return self._dw - return self._client.get_dw() - - def set_dw(self, dw): - """ - Set the DW-value for this bucket. See :func:`set_r` for more - information. - - :param dw: The new DW-value - :type dw: integer - :rtype: self - """ - self._dw = dw - return self - - def get_rw(self, rw=None): - """ - Get the RW-value for this bucket, if it is set, otherwise return - the RW-value for the client. - - :rtype: integer - """ - if (rw is not None): - return rw - if (self._rw is not None): - return self._rw - return self._client.get_rw() - - def set_rw(self, rw): - """ - Set the RW-value for this bucket. See :func:`set_r` for more - information. - - :param rw: The new RW-value - :type rw: integer - :rtype: self - """ - self._rw = rw - return self - - def get_pr(self, pr=None): - """ - Get the PR-value for this bucket, if it is set, otherwise return - the PR-value for the client. - - :rtype: integer - """ - if (pr is not None): - return pr - if (self._pr is not None): - return self._pr - return self._client.get_pr() - - def set_pr(self, pr): - """ - Set the PR-value for this bucket. See :func:`set_r` for more - information. - - :param pr: The new PR-value - :type pr: integer - :rtype: self - """ - self._pr = pr - return self - - def get_pw(self, pw=None): - """ - Get the PW-value for this bucket, if it is set, otherwise return - the PW-value for the client. - - :rtype: integer - """ - if (pw is not None): - return pw - if (self._pw is not None): - return self._pw - return self._client.get_pw() - - def set_pw(self, pw): - """ - Set the PW-value for this bucket. See :func:`set_r` for more - information. - - :param pw: The new PR-value - :type pw: integer - :rtype: self - """ - self._pw = pw - return self - def get_encoder(self, content_type): """ Get the encoding function for the provided content type for @@ -321,8 +171,6 @@ def get(self, key, r=None, pr=None): """ obj = RiakObject(self._client, self, key) obj._encode_data = True - r = self.get_r(r) - pr = self.get_pr(pr) return obj.reload(r=r, pr=pr) def get_binary(self, key, r=None, pr=None): @@ -339,8 +187,6 @@ def get_binary(self, key, r=None, pr=None): """ obj = RiakObject(self._client, self, key) obj._encode_data = False - r = self.get_r(r) - pr = self.get_pr(pr) return obj.reload(r=r, pr=pr) def set_n_val(self, nval): diff --git a/riak/client.py b/riak/client.py index 092c9eae..96d7272d 100644 --- a/riak/client.py +++ b/riak/client.py @@ -28,8 +28,9 @@ from riak.search import RiakSearch from riak.transports import RiakHttpTransport from riak.util import deprecated +from riak.util import deprecateQuorumAccessors - +@deprecateQuorumAccessors class RiakClient(object): """ The ``RiakClient`` object holds information necessary to connect to @@ -84,12 +85,6 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', self._cm = None self._transport = transport_class(host, port, client_id=client_id) - self._r = "default" - self._w = "default" - self._dw = "default" - self._rw = "default" - self._pr = "default" - self._pw = "default" self._encoders = {'application/json': json.dumps, 'text/json': json.dumps} self._decoders = {'application/json': json.loads, @@ -104,133 +99,6 @@ def get_transport(self): """ return self._transport - def get_r(self): - """ - Get the R-value setting for this ``RiakClient``. (default "quorum") - - :rtype: integer - """ - return self._r - - def set_r(self, r): - """ - Set the R-value for this ``RiakClient``. This value will be - used for any calls to :func:`RiakBucket.get - ` or :func:`RiakBucket.get_binary - ` where 1) no R-value is - specified in the method call and 2) no R-value has been set in - the :class:`RiakBucket `. - - :param r: The R value. - :type r: integer - :rtype: self - """ - self._r = r - return self - - def get_w(self): - """ - Get the W-value setting for this ``RiakClient``. (default - "quorum") - - :rtype: integer - """ - return self._w - - def set_w(self, w): - """ - Set the W-value for this ``RiakClient`` instance. See - :func:`set_r` for a description of how these values are used. - - :param w: The W value. - :type w: integer - :rtype: self - """ - self._w = w - return self - - def get_dw(self): - """ - Get the DW-value for this ``RiakClient`` instance. (default - "quorum") - - :rtype: integer - """ - return self._dw - - def set_dw(self, dw): - """ - Set the DW-value for this ``RiakClient`` instance. See - :func:`set_r` for a description of how these values are used. - - :param dw: The DW value. - :type dw: integer - :rtype: self - """ - self._dw = dw - return self - - def get_rw(self): - """ - Get the RW-value for this ``RiakClient`` instance. (default - "quorum") - - :rtype: integer - """ - return self._rw - - def set_rw(self, rw): - """ - Set the RW-value for this ``RiakClient`` instance. See - :func:`set_r` for a description of how these values are used. - - :param rw: The RW value. - :type rw: integer - :rtype: self - """ - self._rw = rw - return self - - def get_pr(self): - """ - Get the PR-value setting for this ``RiakClient``. (default 0) - - :rtype: integer - """ - return self._pr - - def set_pr(self, pr): - """ - Set the PR-value for this ``RiakClient`` instance. See - :func:`set_r` for a description of how these values are used. - - :param pr: The PR value. - :type pr: integer - :rtype: self - """ - self._pr = pr - return self - - def get_pw(self): - """ - Get the PW-value setting for this ``RiakClient``. (default 0) - - :rtype: integer - """ - return self._pw - - def set_pw(self, pw): - """ - Set the PW-value for this ``RiakClient`` instance. See - :func:`set_r` for a description of how these values are used. - - :param pw: The W value. - :type pw: integer - :rtype: self - """ - self._pw = pw - return self - def get_client_id(self): """ Get the ``client_id`` for this ``RiakClient`` instance. diff --git a/riak/riak_object.py b/riak/riak_object.py index fe522cf6..4f71bb65 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -417,11 +417,6 @@ def store(self, w=None, dw=None, pw=None, return_body=True, :type if_none_match: bool :rtype: self """ - # Use defaults if not specified... - w = self._bucket.get_w(w) - dw = self._bucket.get_dw(dw) - pw = self._bucket.get_pw(pw) - # Issue the put over our transport t = self._client.get_transport() @@ -453,8 +448,6 @@ def reload(self, r=None, pr=None, vtag=None): :rtype: self """ - r = self._bucket.get_r(r) - pr = self._bucket.get_pr(pr) t = self._client.get_transport() Result = t.get(self, r=r, pr=pr, vtag=vtag) @@ -490,13 +483,6 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): :type pw: integer :rtype: self """ - # Use defaults if not specified... - rw = self._bucket.get_rw(rw) - r = self._bucket.get_r(r) - w = self._bucket.get_w(w) - dw = self._bucket.get_dw(dw) - pr = self._bucket.get_pr(pr) - pw = self._bucket.get_pw(pw) t = self._client.get_transport() Result = t.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) self.clear() @@ -590,10 +576,6 @@ def get_sibling(self, i, r=None, pr=None): if isinstance(self._siblings[i], RiakObject): return self._siblings[i] else: - # Use defaults if not specified. - r = self._bucket.get_r(r) - pr = self._bucket.get_pr(pr) - # Run the request... vtag = self._siblings[i] obj = RiakObject(self._client, self._bucket, self._key) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 19afe85f..f57f995d 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -247,8 +247,9 @@ def get(self, robj, r=None, pr=None, vtag=None): bucket = robj.get_bucket() req = riak_pb.RpbGetReq() - req.r = self.translate_rw_val(r) - if self.quorum_controls(): + if r: + req.r = self.translate_rw_val(r) + if self.quorum_controls() and pr: req.pr = self.translate_rw_val(pr) if self.tombstone_vclocks(): @@ -275,9 +276,11 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, bucket = robj.get_bucket() req = riak_pb.RpbPutReq() - req.w = self.translate_rw_val(w) - req.dw = self.translate_rw_val(dw) - if self.quorum_controls(): + if w: + req.w = self.translate_rw_val(w) + if dw: + req.dw = self.translate_rw_val(dw) + if self.quorum_controls() and pw: req.pw = self.translate_rw_val(pw) if return_body: @@ -316,9 +319,12 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, bucket = robj.get_bucket() req = riak_pb.RpbPutReq() - req.w = self.translate_rw_val(w) - req.dw = self.translate_rw_val(dw) - req.pw = self.translate_rw_val(pw) + if w: + req.w = self.translate_rw_val(w) + if dw: + req.dw = self.translate_rw_val(dw) + if self.quorum_controls() and pw: + req.pw = self.translate_rw_val(pw) if return_body: req.return_body = 1 @@ -348,14 +354,20 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): bucket = robj.get_bucket() req = riak_pb.RpbDelReq() - req.rw = self.translate_rw_val(rw) - req.r = self.translate_rw_val(r) - req.w = self.translate_rw_val(w) - req.dw = self.translate_rw_val(dw) + if rw: + req.rw = self.translate_rw_val(rw) + if r: + req.r = self.translate_rw_val(r) + if w: + req.w = self.translate_rw_val(w) + if dw: + req.dw = self.translate_rw_val(dw) if self.quorum_controls(): - req.pr = self.translate_rw_val(pr) - req.pw = self.translate_rw_val(pw) + if pr: + req.pr = self.translate_rw_val(pr) + if pw: + req.pw = self.translate_rw_val(pw) if self.tombstone_vclocks() and robj.vclock(): req.vclock = robj.vclock() diff --git a/riak/util.py b/riak/util.py index 85f49056..6d58dcd8 100644 --- a/riak/util.py +++ b/riak/util.py @@ -44,6 +44,52 @@ def deep_merge(a, b): def deprecated(message, stacklevel=3): warnings.warn(message, DeprecationWarning, stacklevel=stacklevel) +QUORUMS = ['r', 'pr', 'w', 'dw', 'pw', 'rw'] +QDEPMESSAGE = """ +Quorum accessors on type %s are deprecated. Use request-specific +parameters or bucket properties instead. +""" + + +def deprecateQuorumAccessors(klass, parent=None): + """ + Adds deprecation warnings for the quorum get_* and set_* + accessors, informing the user to switch to the appropriate bucket + properties or requests parameters. + """ + for q in QUORUMS: + __deprecateQuorumAccessor(klass, parent, q) + return klass + + +def __deprecateQuorumAccessor(klass, parent, quorum): + propname = "_%s" % quorum + getter_name = "get_%s" % quorum + setter_name = "set_%s" % quorum + if not parent: + def getter(self, val=None): + deprecated(QDEPMESSAGE % klass.__name__) + if val: + return val + return getattr(self, propname, "default") + + else: + def getter(self, val=None): + deprecated(QDEPMESSAGE % klass.__name__) + if val: + return val + parentInstance = getattr(self, parent) + return getattr(self, propname, + getattr(parentInstance, propname, "default")) + + def setter(self, value): + deprecated(QDEPMESSAGE % klass.__name__) + setattr(self, propname, value) + return self + + setattr(klass, getter_name, getter) + setattr(klass, setter_name, setter) + class lazy_property(object): ''' From 91e7f589f193d465b9b2c1b71bc4c66478476775 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 6 Nov 2012 17:52:45 -0800 Subject: [PATCH 0237/1060] Fix pep8 stuffs. [ci skip] --- riak/tests/test_2i.py | 4 +++- riak/tests/test_all.py | 4 +++- riak/tests/test_kv.py | 3 ++- riak/tests/test_mapreduce.py | 1 + riak/tests/test_search.py | 1 + 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 4c2e2718..d813e5d6 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- -import os, platform +import os +import platform if platform.python_version() < '2.7': unittest = __import__('unittest2') else: @@ -9,6 +10,7 @@ SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '0')) + class TwoITests(object): def is_2i_supported(self): # Immediate test to see if 2i is even supported w/ the backend diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 21d67b65..bc0ab70c 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -54,6 +54,7 @@ test_server.prepare() test_server.start() + class BaseTestCase(object): @staticmethod @@ -77,6 +78,7 @@ def setUp(self): o = bucket.get('nonexistent_key_binary') o.delete() + class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, TwoITests, @@ -163,6 +165,7 @@ def test_enable_search_commit_hook(self): bucket.enable_search() self.assertRaises(NotImplementedError) + class RiakHttpTransportTestCase(BasicKVTests, KVFileTests, TwoITests, @@ -234,7 +237,6 @@ def test_delete_file_with_luwak(self): self.assertIsNone(file) - class FilterTests(unittest.TestCase): def test_simple(self): f1 = RiakKeyFilter("tokenize", "-", 1) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index fdf83364..943f6ef0 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -import os +import os import cPickle import copy try: @@ -7,6 +7,7 @@ except ImportError: import simplejson as json + class NotJsonSerializable(object): def __init__(self, *args, **kwargs): diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index c374287f..84ba7d9f 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -3,6 +3,7 @@ from riak.mapreduce import RiakLink from riak import RiakKeyFilter, key_filter + class LinkTests(object): def test_store_and_get_links(self): # Create the object... diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index 697e17f1..f3223da0 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -8,6 +8,7 @@ SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '0')) + class EnableSearchTests(object): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_bucket_search_enabled(self): From d88fd537cd72e2fc10012bafdbaf511642b6c74a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 8 Nov 2012 09:37:55 -0800 Subject: [PATCH 0238/1060] Don't need to alias that import. --- riak/bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/bucket.py b/riak/bucket.py index 6199b575..b664420b 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -19,7 +19,7 @@ """ from riak_object import RiakObject import mimetypes -from riak.util import deprecateQuorumAccessors as deprecateQuorumAccessors +from riak.util import deprecateQuorumAccessors def deprecateBucketQuorumAccessors(klass): From e64a89b1839d68eb1f94f82fa16486854e34bfe4 Mon Sep 17 00:00:00 2001 From: Michael Clemmons Date: Wed, 7 Nov 2012 14:38:09 -0800 Subject: [PATCH 0239/1060] added _ for private methods in riak_object --- riak/riak_object.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index fe522cf6..350c78a7 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -437,7 +437,7 @@ def store(self, w=None, dw=None, pw=None, return_body=True, Result = t.put(self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) if Result is not None: - self.populate(Result) + self._populate(Result) return self @@ -460,7 +460,7 @@ def reload(self, r=None, pr=None, vtag=None): self.clear() if Result is not None: - self.populate(Result) + self._populate(Result) return self @@ -523,7 +523,7 @@ def vclock(self): """ return self._vclock - def populate(self, Result): + def _populate(self, Result): """ Populate the object based on the return from get. @@ -537,7 +537,7 @@ def populate(self, Result): if Result is None: return self elif type(Result) == types.ListType: - self.set_siblings(Result) + self._set_siblings(Result) elif type(Result) == types.TupleType: (vclock, contents) = Result self._vclock = vclock @@ -556,7 +556,7 @@ def populate(self, Result): sibling.set_encoded_data(data) siblings.append(sibling) for sibling in siblings: - sibling.set_siblings(siblings) + sibling._set_siblings(siblings) else: raise RiakError("do not know how to handle type %s" % type(Result)) @@ -601,7 +601,7 @@ def get_sibling(self, i, r=None, pr=None): # And make sure it knows who it's siblings are self._siblings[i] = obj - obj.set_siblings(self._siblings) + obj._set_siblings(self._siblings) return obj def get_siblings(self, r=None): @@ -618,7 +618,7 @@ def get_siblings(self, r=None): a.append(self.get_sibling(i, r)) return a - def set_siblings(self, siblings): + def _set_siblings(self, siblings): """ Set the array of siblings - used internally From 045d406d04f8e048bafa0b5e131b1f0e9b4f14aa Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 9 Nov 2012 15:29:47 -0800 Subject: [PATCH 0240/1060] Make non-public methods uber-private on the iterator and add a disclaimer to the docstring. --- riak/transports/pool.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/riak/transports/pool.py b/riak/transports/pool.py index 090354bd..062e66e0 100644 --- a/riak/transports/pool.py +++ b/riak/transports/pool.py @@ -181,6 +181,13 @@ class PoolIterator(object): Iterates over a snapshot of the pool in a thread-safe manner, eventually touching all resources that were known when the iteration started. + + Note that if claimed resources are not released for long periods, + the iterator may hang, waiting for those last resources to be + released. The iteration and pool functionality is only meant to be + used internally within the client, and resources will be claimed + per client operation, making this an unlikely event (although + still possible). """ def __init__(self, pool): @@ -197,12 +204,12 @@ def next(self): if len(self.targets) == 0: raise StopIteration if len(self.unlocked) == 0: - self.claim_elements() + self.__claim_elements() return self.unlocked.pop(0) - def claim_elements(self): + def __claim_elements(self): with self.lock: - if self.all_claimed(): + if self.__all_claimed(): with self.releaser: self.releaser.wait() for element in self.targets[:]: @@ -211,7 +218,7 @@ def claim_elements(self): self.unlocked.append(element) element.claimed = True - def all_claimed(self): + def __all_claimed(self): for element in self.targets[:]: if not element.claimed: return False From e10428c152e3dddc21d63552ea30cbda60853e15 Mon Sep 17 00:00:00 2001 From: evan Date: Thu, 1 Nov 2012 17:05:52 -0700 Subject: [PATCH 0241/1060] remove all luwak support and tests. --- README.rst | 23 +-------------------- riak/client.py | 14 ------------- riak/test_server.py | 3 --- riak/tests/test_all.py | 37 ---------------------------------- riak/tests/test_server_test.py | 5 ----- riak/transports/http.py | 22 -------------------- riak/transports/transport.py | 24 ---------------------- 7 files changed, 1 insertion(+), 127 deletions(-) diff --git a/README.rst b/README.rst index 62cf297e..51171234 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ To run the unit tests against a Riak server (with default TCP port configuration python setup.py test -If you don't have `Luwak `_ or `Riak Search `_ enabled you can set the ``SKIP_LUWAK`` and ``SKIP_SEARCH`` environment variables to skip those tests. +If you don't have `Riak Search `_ enabled you can set the ``SKIP_SEARCH`` environment variable to skip that tests. If your Riak server isn't running on localhost, use the environment variables ``RIAK_TEST_HOST`` and ``RIAK_TEST_HTTP_PORT`` and ``RIAK_TEST_PB_PORT=8087`` to specify where to find the Riak server. @@ -585,24 +585,3 @@ suites or in subsequent test runs, be sure to call cleanup() before starting or after stopping it. .. _Ripple: https://github.com/seancribbs/ripple - -Luwak for Large File Storage -============================ - -If your Riak installation has Luwak support enabled, you can use the client to -interact with it, storing, fetching and deleting files. Note that Luwak is HTTP -only and will always use the settings provided for the HTTP transport. If you -mix Luwak with normal Riak usage through the Protocol Buffers interface, it's -best to use multiple client objects for each separate use case:: - - client = riak.RiakClient() - - image = open('hulk.jpg', 'rb') - client.store_file('image.jpg', image.read(), content_type="image/jpeg") - - # Returns just the data stored in luwak - client.get_file('image.jpg') - - client.delete_file('image.jpg') - -.. _`Luwak`: http://wiki.basho.com/Luwak.html diff --git a/riak/client.py b/riak/client.py index 96d7272d..3a81836c 100644 --- a/riak/client.py +++ b/riak/client.py @@ -245,20 +245,6 @@ def reduce(self, *args): mr = RiakMapReduce(self) return apply(mr.reduce, args) - def store_file(self, filename, data, - content_type="application/octet-stream"): - """ - Store data in luwak using filename as the key - """ - self._transport.store_file(filename, content_type=content_type, - content=data) - - def get_file(self, filename): - return self._transport.get_file(filename) - - def delete_file(self, filename): - self._transport.delete_file(filename) - def get_index(self, bucket, index, startkey, endkey=None): return self._transport.get_index(bucket, index, startkey, endkey) diff --git a/riak/test_server.py b/riak/test_server.py index 92c46003..7c6348af 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -90,9 +90,6 @@ class TestServer(object): "enabled": True, "search_backend": Atom("riak_search_test_backend") }, - "luwak": { - "enabled": True - } } def __init__(self, tmp_dir="/tmp/riak/test_server", diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index bc0ab70c..97044ead 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -42,8 +42,6 @@ HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) HTTP_PORT = int(os.environ.get('RIAK_TEST_HTTP_PORT', '8098')) -SKIP_LUWAK = int(os.environ.get('SKIP_LUWAK', '0')) - USE_TEST_SERVER = int(os.environ.get('USE_TEST_SERVER', '0')) if USE_TEST_SERVER: @@ -201,41 +199,6 @@ def test_too_many_link_headers_shouldnt_break_http(self): stored_object = bucket.get("lots_of_links") self.assertEqual(len(stored_object.get_links()), 400) - @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') - def test_store_file_with_luwak(self): - file = os.path.join(os.path.dirname(__file__), "test_all.py") - with open(file, "r") as input_file: - data = input_file.read() - - key = uuid.uuid1().hex - self.client.store_file(key, data) - - @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') - def test_store_get_file_with_luwak(self): - file = os.path.join(os.path.dirname(__file__), "test_all.py") - with open(file, "r") as input_file: - data = input_file.read() - - key = uuid.uuid1().hex - self.client.store_file(key, data) - time.sleep(1) - file = self.client.get_file(key) - self.assertEquals(data, file) - - @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') - def test_delete_file_with_luwak(self): - file = os.path.join(os.path.dirname(__file__), "test_all.py") - with open(file, "r") as input_file: - data = input_file.read() - - key = uuid.uuid1().hex - self.client.store_file(key, data) - time.sleep(1) - self.client.delete_file(key) - time.sleep(1) - file = self.client.get_file(key) - self.assertIsNone(file) - class FilterTests(unittest.TestCase): def test_simple(self): diff --git a/riak/tests/test_server_test.py b/riak/tests/test_server_test.py index adca451f..98826d2c 100644 --- a/riak/tests/test_server_test.py +++ b/riak/tests/test_server_test.py @@ -20,11 +20,6 @@ def test_merge_riak_core_options(self): self.assertEquals( self.test_server.app_config["riak_core"]["handoff_port"], 10000) - def test_merge_luwak_options(self): - self.test_server = TestServer(luwak={"enabled": False}) - self.assertEquals( - self.test_server.app_config["luwak"]["enabled"], False) - def test_merge_riak_search_options(self): self.test_server = TestServer( riak_search={"search_backend": "riak_search_backend"}) diff --git a/riak/transports/http.py b/riak/transports/http.py index 8aefabb2..19aebf86 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -492,28 +492,6 @@ def get_request(self, uri=None, params=None): url = self.build_rest_path(bucket=None, params=params, prefix=uri) return self.http_request('GET', url) - def store_file(self, key, content_type="application/octet-stream", - content=None): - url = self.build_rest_path(prefix='luwak', key=key) - headers = {'Content-Type': content_type, - 'X-Riak-ClientId': self._client_id} - - return self.do_put(url, headers, content, key=key) - - def get_file(self, key): - url = self.build_rest_path(prefix='luwak', key=key) - response = self.http_request('GET', url) - result = self.parse_body(response, [200, 300, 404]) - if result is not None: - (vclock, data) = result - (headers, body) = data.pop() - return body - - def delete_file(self, key): - url = self.build_rest_path(prefix='luwak', key=key) - response = self.http_request('DELETE', url) - self.parse_body(response, [204, 404]) - def post_request(self, uri=None, body=None, params=None, content_type="application/json"): uri = self.build_rest_path(prefix=uri, params=params) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 481020d1..0b0ff32c 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -198,27 +198,3 @@ def _get_index_mapred_emu(self, bucket, index, startkey, endkey=None): 'key': startkey}, phases) return [key for bucket, key in result] - - def store_file(self, key, content_type="application/octet-stream", - content=None): - """ - Store a large piece of data in luwak. - key = the key/filename for the object - content_type = the object's content type - content = the object's data - """ - raise NotImplementedError - - def get_file(self, key): - """ - Get an object from luwak. - key = the object's key - """ - raise NotImplementedError - - def delete_file(self, key): - """ - Delete an object in luwak. - key = the object's key - """ - raise NotImplementedError From 901693a3bbad0331d098a2d5236177f6bb608dfb Mon Sep 17 00:00:00 2001 From: evan Date: Mon, 12 Nov 2012 09:35:52 -0800 Subject: [PATCH 0242/1060] remove travis env setting --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b876f932..3e63c8c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ python: install: ./setup.py develop script: ./setup.py test before_script: sudo /usr/sbin/search-cmd install searchbucket -env: "SKIP_LUWAK=1" notifications: email: clients@basho.com services: From 4cb0fb369c129d7015add678583e29c26e9c849a Mon Sep 17 00:00:00 2001 From: evan Date: Tue, 13 Nov 2012 17:38:09 -0800 Subject: [PATCH 0243/1060] - rework bucket props to be properties - change pbc client to complain when used for props, instead of failing silently - test fixes --- riak/bucket.py | 148 +++++++++++++++-------------------- riak/riak_object.py | 16 ++-- riak/tests/test_all.py | 24 +++--- riak/tests/test_kv.py | 110 +++++++++++++++++--------- riak/tests/test_mapreduce.py | 2 +- riak/tests/test_pool.py | 1 + riak/transports/http.py | 2 +- riak/transports/pbc.py | 20 ++--- 8 files changed, 172 insertions(+), 151 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index b664420b..2cab2e9e 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -25,7 +25,6 @@ def deprecateBucketQuorumAccessors(klass): return deprecateQuorumAccessors(klass, parent='_client') - @deprecateBucketQuorumAccessors class RiakBucket(object): """ @@ -52,16 +51,10 @@ def __init__(self, client, name): raise TypeError('Unicode bucket names are not supported.') self._client = client - self._name = name + self.name = name self._encoders = {} self._decoders = {} - def get_name(self): - """ - Get the bucket name as a string. - """ - return self._name - def get_encoder(self, content_type): """ Get the encoding function for the provided content type for @@ -189,72 +182,72 @@ def get_binary(self, key, r=None, pr=None): obj._encode_data = False return obj.reload(r=r, pr=pr) - def set_n_val(self, nval): - """ - Set the N-value for this bucket, which is the number of replicas - that will be written of each object in the bucket. - - .. warning:: - - Set this once before you write any data to the bucket, and never - change it again, otherwise unpredictable things could happen. - This should only be used if you know what you are doing. - - :param nval: The new N-Val. - :type nval: integer - """ + def _set_n_val(self, nval): return self.set_property('n_val', nval) - - def get_n_val(self): - """ - Retrieve the N-value for this bucket. - - :rtype: integer - """ + def _get_n_val(self): return self.get_property('n_val') - - def set_default_r_val(self, rval): - return self.set_property('r', rval) - - def get_default_r_val(self): + n_val = property(_get_n_val, _set_n_val, doc = + """ + N-value for this bucket, which is the number of replicas + that will be written of each object in the bucket. + + .. warning:: + + Set this once before you write any data to the bucket, and never + change it again, otherwise unpredictable things could happen. + This should only be used if you know what you are doing. + + :type nval: integer + """) + + def _set_allow_mult(self, bool): + return self.set_property('allow_mult', bool) + def _get_allow_mult(self): + return self.get_property('allow_mult') + allow_mult = property(_get_allow_mult, _set_allow_mult, doc = + """ + If set to True, then writes with conflicting data will be stored + and returned to the client. This situation can be detected by + calling has_siblings() and get_siblings(). + + :type bool: boolean + """) + + def _set_r(self, val): + return self.set_property('r', val) + def _get_r(self): return self.get_property('r') + r = property(_get_r, _set_r) - def set_default_w_val(self, wval): - return self.set_property('w', wval) - - def get_default_w_val(self): - return self.get_property('w') - - def set_default_dw_val(self, dwval): - return self.set_property('dw', dwval) - - def get_default_dw_val(self): - return self.get_property('dw') - - def set_default_rw_val(self, rwval): - return self.set_property('rw', rwval) + def _set_pr(self, val): + return self.set_property('pr', val) + def _get_pr(self): + return self.get_property('pr') + pr = property(_get_pr, _set_pr) - def get_default_rw_val(self): + def _set_rw(self, val): + return self.set_property('rw', val) + def _get_rw(self): return self.get_property('rw') + rw = property(_get_rw, _set_rw) - def set_allow_multiples(self, bool): - """ - If set to True, then writes with conflicting data will be stored - and returned to the client. This situation can be detected by - calling has_siblings() and get_siblings(). - - :param bool: True to store and return conflicting writes. - :type bool: boolean - """ - return self.set_property('allow_mult', bool) + def _set_w(self, val): + return self.set_property('w', val) + def _get_w(self): + return self.get_property('w') + w = property(_get_w, _set_w) - def get_allow_multiples(self): - """ - Retrieve the 'allow multiples' setting. + def _set_dw(self, val): + return self.set_property('dw', val) + def _get_dw(self): + return self.get_property('dw') + dw = property(_get_dw, _set_dw) - :rtype: Boolean - """ - return self.get_bool_property('allow_mult') + def _set_pw(self, val): + return self.set_property('pw', val) + def _get_pw(self): + return self.get_property('pw') + pw = property(_get_pw, _set_pw) def set_property(self, key, value): """ @@ -271,20 +264,6 @@ def set_property(self, key, value): """ return self.set_properties({key: value}) - def get_bool_property(self, key): - """ - Get a boolean bucket property. Converts to a ``True`` or - ``False`` value. - - :param key: Property to set. - :type key: string - """ - prop = self.get_property(key) - if prop == True or prop > 0: - return True - else: - return False - def get_property(self, key): """ Retrieve a bucket property. @@ -293,9 +272,10 @@ def get_property(self, key): :type key: string :rtype: mixed """ - props = self.get_properties() - if (key in props.keys()): - return props[key] + try: + return self.get_properties()[key] + except KeyError: + raise NotImplementedError def set_properties(self, props): """ @@ -375,11 +355,11 @@ def search(self, query, **params): """ Queries a search index over objects in this bucket/index. """ - return self._client.solr().search(self._name, query, **params) + return self._client.solr().search(self.name, query, **params) def get_index(self, index, startkey, endkey=None): """ Queries a secondary index over objects in this bucket, returning keys. """ - return self._client._transport.get_index(self._name, index, startkey, + return self._client._transport.get_index(self.name, index, startkey, endkey) diff --git a/riak/riak_object.py b/riak/riak_object.py index 44aa5d8a..c1ab33e1 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -322,9 +322,9 @@ def set_links(self, links, all_link=False): if isinstance(item, RiakLink): link = item elif isinstance(item, RiakObject): - link = RiakLink(item._bucket._name, item._key, None) + link = RiakLink(item._bucket.name, item._key, None) else: - link = RiakLink(item[0]._bucket._name, item[0]._key, item[1]) + link = RiakLink(item[0]._bucket.name, item[0]._key, item[1]) new_links.append(link) self._metadata[MD_LINKS] = new_links @@ -344,7 +344,7 @@ def add_link(self, obj, tag=None): if isinstance(obj, RiakLink): newlink = obj else: - newlink = RiakLink(obj._bucket._name, obj._key, tag) + newlink = RiakLink(obj._bucket.name, obj._key, tag) self.remove_link(newlink) links = self._metadata[MD_LINKS] @@ -365,7 +365,7 @@ def remove_link(self, obj, tag=None): if isinstance(obj, RiakLink): oldlink = obj else: - oldlink = RiakLink(obj._bucket._name, obj._key, tag) + oldlink = RiakLink(obj._bucket.name, obj._key, tag) a = [] links = self._metadata.get(MD_LINKS, []) @@ -630,7 +630,7 @@ def add(self, *args): :rtype: RiakMapReduce """ mr = RiakMapReduce(self._client) - mr.add(self._bucket._name, self._key) + mr.add(self._bucket.name, self._key) return apply(mr.add, args) def link(self, *args): @@ -641,7 +641,7 @@ def link(self, *args): :rtype: RiakMapReduce """ mr = RiakMapReduce(self._client) - mr.add(self._bucket._name, self._key) + mr.add(self._bucket.name, self._key) return apply(mr.link, args) def map(self, *args): @@ -652,7 +652,7 @@ def map(self, *args): :rtype: RiakMapReduce """ mr = RiakMapReduce(self._client) - mr.add(self._bucket._name, self._key) + mr.add(self._bucket.name, self._key) return apply(mr.map, args) def reduce(self, params): @@ -663,7 +663,7 @@ def reduce(self, params): :rtype: RiakMapReduce """ mr = RiakMapReduce(self._client) - mr.add(self._bucket._name, self._key) + mr.add(self._bucket.name, self._key) return apply(mr.reduce, params) from mapreduce import * diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 97044ead..b57521f8 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -25,7 +25,8 @@ EnableSearchTests, SolrSearchTests from riak.tests.test_mapreduce import MapReduceAliasTests, \ ErlangMapReduceTests, JSMapReduceTests, LinkTests -from riak.tests.test_kv import BasicKVTests, KVFileTests +from riak.tests.test_kv import BasicKVTests, KVFileTests, \ + HTTPBucketPropsTest, PbcBucketPropsTest from riak.tests.test_2i import TwoITests try: @@ -79,6 +80,7 @@ def setUp(self): class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, + PbcBucketPropsTest, TwoITests, LinkTests, ErlangMapReduceTests, @@ -114,7 +116,7 @@ def test_close_underlying_socket_fails(self): obj.store() obj = bucket.get('foo') self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') + self.assertEqual(obj.get_bucket().name, 'bucket_test_close') self.assertEqual(obj.get_key(), 'foo') self.assertEqual(obj.get_data(), rand) @@ -137,7 +139,7 @@ def test_close_underlying_socket_retry(self): obj.store() obj = bucket.get('barbaz') self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') + self.assertEqual(obj.get_bucket().name, 'bucket_test_close') self.assertEqual(obj.get_key(), 'barbaz') self.assertEqual(obj.get_data(), rand) @@ -150,22 +152,24 @@ def test_close_underlying_socket_retry(self): # This should work, since we have a retry obj = bucket.get('barbaz') self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') + self.assertEqual(obj.get_bucket().name, 'bucket_test_close') self.assertEqual(obj.get_key(), 'barbaz') self.assertEqual(obj.get_data(), rand) def test_bucket_search_enabled(self): - bucket = self.client.bucket("unsearch_bucket") - self.assertRaises(NotImplementedError) - + with self.assertRaises(NotImplementedError): + bucket = self.client.bucket("unsearch_bucket") + test = bucket.search_enabled() + def test_enable_search_commit_hook(self): - bucket = self.client.bucket("search_bucket") - bucket.enable_search() - self.assertRaises(NotImplementedError) + with self.assertRaises(NotImplementedError): + bucket = self.client.bucket("search_bucket") + bucket.enable_search() class RiakHttpTransportTestCase(BasicKVTests, KVFileTests, + HTTPBucketPropsTest, TwoITests, LinkTests, ErlangMapReduceTests, diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 943f6ef0..9b5d6c17 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -43,7 +43,7 @@ def test_store_and_get(self): obj.store() obj = bucket.get('foo') self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().get_name(), 'bucket') + self.assertEqual(obj.get_bucket().name, 'bucket') self.assertEqual(obj.get_key(), 'foo') self.assertEqual(obj.get_data(), rand) @@ -142,45 +142,16 @@ def test_delete(self): def test_set_bucket_properties(self): bucket = self.client.bucket('bucket') # Test setting allow mult... - bucket.set_allow_multiples(True) - self.assertTrue(bucket.get_allow_multiples()) + bucket.allow_mult = True + self.assertTrue(bucket.allow_mult) # Test setting nval... - bucket.set_n_val(3) - self.assertEqual(bucket.get_n_val(), 3) + bucket.n_val = 3 + self.assertEqual(bucket.n_val, 3) # Test setting multiple properties... bucket.set_properties({"allow_mult": False, "n_val": 2}) - self.assertFalse(bucket.get_allow_multiples()) - self.assertEqual(bucket.get_n_val(), 2) + self.assertFalse(bucket.allow_mult) + self.assertEqual(bucket.n_val, 2) - def test_rw_settings(self): - bucket = self.client.bucket('rwsettings') - self.assertEqual(bucket.get_r(), "default") - self.assertEqual(bucket.get_w(), "default") - self.assertEqual(bucket.get_dw(), "default") - self.assertEqual(bucket.get_rw(), "default") - - bucket.set_w(1) - self.assertEqual(bucket.get_w(), 1) - - bucket.set_r("quorum") - self.assertEqual(bucket.get_r(), "quorum") - - bucket.set_dw("all") - self.assertEqual(bucket.get_dw(), "all") - - bucket.set_rw("one") - self.assertEqual(bucket.get_rw(), "one") - - def test_primary_quora(self): - bucket = self.client.bucket('primary_quora') - self.assertEqual(bucket.get_pr(), "default") - self.assertEqual(bucket.get_pw(), "default") - - bucket.set_pr(1) - self.assertEqual(bucket.get_pr(), 1) - - bucket.set_pw("quorum") - self.assertEqual(bucket.get_pw(), "quorum") def test_if_none_match(self): bucket = self.client.bucket('if_none_match_test') @@ -199,7 +170,7 @@ def test_if_none_match(self): def test_siblings(self): # Set up the bucket, clear any existing object... bucket = self.client.bucket('multiBucket') - bucket.set_allow_multiples(True) + bucket.allow_mult = True obj = bucket.get_binary('foo') # Even if it previously existed, let's store a base resolved version # from which we can diverge by sending a stale vclock. @@ -274,6 +245,71 @@ def test_list_buckets(self): buckets = self.client.get_buckets() self.assertTrue("list_bucket" in buckets) +class HTTPBucketPropsTest(object): + def test_rw_settings(self): + bucket = self.client.bucket('rwsettings') + self.assertEqual(bucket.r, "quorum") + self.assertEqual(bucket.w, "quorum") + self.assertEqual(bucket.dw, "quorum") + self.assertEqual(bucket.rw, "quorum") + + bucket.w = 1 + self.assertEqual(bucket.w, 1) + + bucket.r = "quorum" + self.assertEqual(bucket.r, "quorum") + + bucket.dw = "all" + self.assertEqual(bucket.dw, "all") + + bucket.rw = "one" + self.assertEqual(bucket.rw, "one") + + def test_primary_quora(self): + bucket = self.client.bucket('primary_quora') + self.assertEqual(bucket.pr, 0) + self.assertEqual(bucket.pw, 0) + + bucket.pr = 1 + self.assertEqual(bucket.pr, 1) + + bucket.pw = "quorum" + self.assertEqual(bucket.pw, "quorum") + + +class PbcBucketPropsTest(object): + def test_rw_settings(self): + bucket = self.client.bucket('rwsettings') + with self.assertRaises(NotImplementedError): + test = bucket.r + with self.assertRaises(NotImplementedError): + test = bucket.w + with self.assertRaises(NotImplementedError): + test = bucket.dw + with self.assertRaises(NotImplementedError): + test = bucket.rw + + with self.assertRaises(NotImplementedError): + bucket.r = 2 + with self.assertRaises(NotImplementedError): + bucket.w = 2 + with self.assertRaises(NotImplementedError): + bucket.dw = 2 + with self.assertRaises(NotImplementedError): + bucket.rw = 2 + + def test_primary_quora(self): + bucket = self.client.bucket('primary_quora') + with self.assertRaises(NotImplementedError): + test = bucket.pr + with self.assertRaises(NotImplementedError): + test = bucket.pw + + with self.assertRaises(NotImplementedError): + bucket.pr = 2 + with self.assertRaises(NotImplementedError): + bucket.pw = 2 + class KVFileTests(object): def test_store_binary_object_from_file(self): diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 84ba7d9f..ed65e03d 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -157,7 +157,7 @@ def test_javascript_bucket_map_reduce(self): bucket.new("baz", 4).store() # Run the map... result = self.client \ - .add(bucket.get_name()) \ + .add(bucket.name) \ .map("Riak.mapValuesJson") \ .reduce("Riak.reduceSum") \ .run() diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index f57a2cd3..a1c8c66c 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -48,6 +48,7 @@ def create_resource(self): class PoolTest(unittest.TestCase): + maxDiff = None def test_yields_new_object_when_empty(self): """ The pool should create new resources as needed. diff --git a/riak/transports/http.py b/riak/transports/http.py index 19aebf86..8fc2757d 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -511,7 +511,7 @@ def build_rest_path(self, bucket=None, key=None, params=None, prefix=None): # Add '.../bucket' if bucket is not None: - path += '/' + urllib.quote_plus(bucket._name) + path += '/' + urllib.quote_plus(bucket.name) # Add '.../key' if key is not None: diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index f57f995d..1b525505 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -22,7 +22,6 @@ import errno import socket import struct - try: import json except ImportError: @@ -255,7 +254,7 @@ def get(self, robj, r=None, pr=None, vtag=None): if self.tombstone_vclocks(): req.deletedvclock = 1 - req.bucket = bucket.get_name() + req.bucket = bucket.name req.key = robj.get_key() # An expected response code of None implies "any response is valid". @@ -288,7 +287,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, if if_none_match: req.if_none_match = 1 - req.bucket = bucket.get_name() + req.bucket = bucket.name req.key = robj.get_key() vclock = robj.vclock() if vclock: @@ -331,7 +330,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if if_none_match: req.if_none_match = 1 - req.bucket = bucket.get_name() + req.bucket = bucket.name self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), @@ -372,7 +371,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): if self.tombstone_vclocks() and robj.vclock(): req.vclock = robj.vclock() - req.bucket = bucket.get_name() + req.bucket = bucket.name req.key = robj.get_key() msg_code, resp = self.send_msg(MSG_CODE_DEL_REQ, req, @@ -384,7 +383,7 @@ def get_keys(self, bucket): Lists all keys within a bucket. """ req = riak_pb.RpbListKeysReq() - req.bucket = bucket.get_name() + req.bucket = bucket.name keys = [] @@ -409,7 +408,7 @@ def get_bucket_props(self, bucket): Serialize bucket property request and deserialize response """ req = riak_pb.RpbGetBucketReq() - req.bucket = bucket.get_name() + req.bucket = bucket.name msg_code, resp = self.send_msg(MSG_CODE_GET_BUCKET_REQ, req, MSG_CODE_GET_BUCKET_RESP) @@ -426,9 +425,10 @@ def set_bucket_props(self, bucket, props): Serialize set bucket property request and deserialize response """ req = riak_pb.RpbSetBucketReq() - req.bucket = bucket.get_name() - if not 'n_val' in props and not 'allow_mult' in props: - return self + req.bucket = bucket.name + for key in props: + if key not in ['n_val', 'allow_mult']: + raise NotImplementedError if 'n_val' in props: req.props.n_val = props['n_val'] From 72ada1dcb541b626faedb9f998cb8a01d1755128 Mon Sep 17 00:00:00 2001 From: evan Date: Fri, 16 Nov 2012 09:39:59 -0800 Subject: [PATCH 0244/1060] remove testing leftovers --- riak/tests/test_pool.py | 1 - 1 file changed, 1 deletion(-) diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index a1c8c66c..f57a2cd3 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -48,7 +48,6 @@ def create_resource(self): class PoolTest(unittest.TestCase): - maxDiff = None def test_yields_new_object_when_empty(self): """ The pool should create new resources as needed. From 802dd924545b52ce9ffcf752e54656bafa515b69 Mon Sep 17 00:00:00 2001 From: evan Date: Sat, 17 Nov 2012 10:55:51 -0800 Subject: [PATCH 0245/1060] temp commit --- riak/tests/test_all.py | 76 +++++++++++++++++------------- riak/tests/test_kv.py | 97 ++++++++++++++++++++------------------- riak/tests/test_pool.py | 4 +- riak/tests/test_search.py | 66 +++++++++++++------------- 4 files changed, 128 insertions(+), 115 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 97044ead..43a4e343 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -59,24 +59,33 @@ class BaseTestCase(object): def randint(): return random.randint(1, 999999) + @staticmethod + def randname(length = 12): + out = '' + for i in range(length): + out += chr(random.randint(ord('a'), ord('z'))) + return out + def create_client(self, host=None, port=None, transport_class=None): host = host or self.host port = port or self.port transport_class = transport_class or self.transport_class - return RiakClient(self.host, self.port, - transport_class=self.transport_class) + return RiakClient(host, port, + transport_class=transport_class) def setUp(self): - self.client = self.create_client() - - # make sure these are not left over from a previous, failed run - bucket = self.client.bucket('bucket') - o = bucket.get('nonexistent_key_json') - o.delete() - o = bucket.get('nonexistent_key_binary') - o.delete() - + self.bucket_name = self.randname() + self.key_name = self.randname() + if not getattr(self, 'search_bucket', None): + print repr(self), 'creating search bucket' + self.search_bucket = self.randname() + c = self.create_client(HTTP_HOST, HTTP_PORT, + RiakHttpTransport) + b = c.bucket(self.search_bucket) + b.enable_search() + self.client = self.create_client() + class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, TwoITests, @@ -94,6 +103,8 @@ def setUp(self): self.host = PB_HOST self.port = PB_PORT self.transport_class = RiakPbcTransport + self.http_client = self.create_client(HTTP_HOST, HTTP_PORT, + RiakHttpTransport) super(RiakPbcTransportTestCase, self).setUp() def test_uses_client_id_if_given(self): @@ -107,15 +118,14 @@ def test_uses_client_id_if_given(self): def test_close_underlying_socket_fails(self): c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport) - - bucket = c.bucket('bucket_test_close') + bucket = c.bucket(self.bucket_name) rand = self.randint() - obj = bucket.new('foo', rand) + obj = bucket.new(self.key_name, rand) obj.store() - obj = bucket.get('foo') + obj = bucket.get(self.key_name) self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') - self.assertEqual(obj.get_key(), 'foo') + self.assertEqual(obj.get_bucket().get_name(), self.bucket_name) + self.assertEqual(obj.get_key(), self.key_name) self.assertEqual(obj.get_data(), rand) # Close the underlying socket. This gets a bit sketchy, @@ -130,15 +140,15 @@ def test_close_underlying_socket_fails(self): def test_close_underlying_socket_retry(self): c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport, transport_options={"max_attempts": 2}) - - bucket = c.bucket('bucket_test_close') + bucket = c.bucket(self.bucket_name) rand = self.randint() - obj = bucket.new('barbaz', rand) + obj = bucket.new(self.key_name, rand) obj.store() - obj = bucket.get('barbaz') + + obj = bucket.get(self.key_name) self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') - self.assertEqual(obj.get_key(), 'barbaz') + self.assertEqual(obj.get_bucket().get_name(), self.bucket_name) + self.assertEqual(obj.get_key(), self.key_name) self.assertEqual(obj.get_data(), rand) # Close the underlying socket. This gets a bit sketchy, @@ -148,20 +158,20 @@ def test_close_underlying_socket_retry(self): conns[0].sock.close() # This should work, since we have a retry - obj = bucket.get('barbaz') + obj = bucket.get(self.key_name) self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') - self.assertEqual(obj.get_key(), 'barbaz') + self.assertEqual(obj.get_bucket().get_name(), self.bucket_name) + self.assertEqual(obj.get_key(), self.key_name) self.assertEqual(obj.get_data(), rand) def test_bucket_search_enabled(self): - bucket = self.client.bucket("unsearch_bucket") + bucket = self.client.bucket(self.bucket_name) self.assertRaises(NotImplementedError) def test_enable_search_commit_hook(self): - bucket = self.client.bucket("search_bucket") - bucket.enable_search() - self.assertRaises(NotImplementedError) + bucket = self.client.bucket(self.bucket_name) + bucket.enable_search() + self.assertRaises(NotImplementedError) class RiakHttpTransportTestCase(BasicKVTests, @@ -184,12 +194,12 @@ def setUp(self): super(RiakHttpTransportTestCase, self).setUp() def test_no_returnbody(self): - bucket = self.client.bucket("bucket") - o = bucket.new("foo", "bar").store(return_body=False) + bucket = self.client.bucket(self.bucket_name) + o = bucket.new(self.key_name, "bar").store(return_body=False) self.assertEqual(o.vclock(), None) def test_too_many_link_headers_shouldnt_break_http(self): - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) o = bucket.new("lots_of_links", "My god, it's full of links!") for i in range(0, 400): link = RiakLink("other", "key%d" % i, "next") diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 943f6ef0..54eeef08 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -37,19 +37,19 @@ def test_is_alive(self): self.assertTrue(self.client.is_alive()) def test_store_and_get(self): - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) rand = self.randint() obj = bucket.new('foo', rand) obj.store() obj = bucket.get('foo') self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().get_name(), 'bucket') + self.assertEqual(obj.get_bucket().get_name(), self.bucket_name) self.assertEqual(obj.get_key(), 'foo') self.assertEqual(obj.get_data(), rand) # unicode objects are fine, as long as they don't # contain any non-ASCII chars - self.client.bucket(u'bucket') + self.client.bucket(unicode(self.bucket_name)) self.assertRaises(TypeError, self.client.bucket, u'búcket') self.assertRaises(TypeError, self.client.bucket, 'búcket') @@ -65,7 +65,7 @@ def test_store_and_get(self): def test_generate_key(self): # Ensure that Riak generates a random key when # the key passed to bucket.new() is None. - bucket = self.client.bucket('random_key_bucket') + bucket = self.client.bucket(self.bucket_name) existing_keys = bucket.get_keys() o = bucket.new(None, data={}) self.assertIsNone(o.get_key()) @@ -76,71 +76,72 @@ def test_generate_key(self): self.assertEqual(len(bucket.get_keys()), len(existing_keys) + 1) def test_binary_store_and_get(self): - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) # Store as binary, retrieve as binary, then compare... rand = str(self.randint()) - obj = bucket.new_binary('foo1', rand) + obj = bucket.new_binary(self.key_name, rand) obj.store() - obj = bucket.get_binary('foo1') + obj = bucket.get_binary(self.key_name) self.assertTrue(obj.exists()) self.assertEqual(obj.get_data(), rand) # Store as JSON, retrieve as binary, JSON-decode, then compare... data = [self.randint(), self.randint(), self.randint()] - obj = bucket.new('foo2', data) + key2 = self.randname() + obj = bucket.new(key2, data) obj.store() - obj = bucket.get_binary('foo2') + obj = bucket.get_binary(key2) self.assertEqual(data, json.loads(obj.get_data())) def test_custom_bucket_encoder_decoder(self): # Teach the bucket how to pickle - bucket = self.client.bucket("picklin_bucket") + bucket = self.client.bucket(self.bucket_name) bucket.set_encoder('application/x-pickle', cPickle.dumps) bucket.set_decoder('application/x-pickle', cPickle.loads) data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} - obj = bucket.new("foo", data, 'application/x-pickle').store() + obj = bucket.new(self.key_name, data, 'application/x-pickle').store() obj.store() - obj2 = bucket.get("foo") + obj2 = bucket.get(self.key_name) self.assertEqual(data, obj2.get_data()) def test_custom_client_encoder_decoder(self): # Teach the bucket how to pickle - bucket = self.client.bucket("picklin_client") + bucket = self.client.bucket(self.bucket_name) self.client.set_encoder('application/x-pickle', cPickle.dumps) self.client.set_decoder('application/x-pickle', cPickle.loads) data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} - obj = bucket.new("foo", data, 'application/x-pickle').store() + obj = bucket.new(self.key_name, data, 'application/x-pickle').store() obj.store() - obj2 = bucket.get("foo") + obj2 = bucket.get(self.key_name) self.assertEqual(data, obj2.get_data()) def test_unknown_content_type_encoder_decoder(self): # Teach the bucket how to pickle - bucket = self.client.bucket("unknown_contenttype") + bucket = self.client.bucket(self.bucket_name) data = "some funny data" - obj = bucket.new("foo", data, 'application/x-frobnicator').store() + obj = bucket.new(self.key_name, data, 'application/x-frobnicator').store() obj.store() - obj2 = bucket.get("foo") + obj2 = bucket.get(self.key_name) self.assertEqual(data, obj2.get_data()) def test_missing_object(self): - bucket = self.client.bucket('bucket') - obj = bucket.get("missing") + bucket = self.client.bucket(self.bucket_name) + obj = bucket.get(self.key_name) self.assertFalse(obj.exists()) self.assertEqual(obj.get_data(), None) def test_delete(self): - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) rand = self.randint() - obj = bucket.new('foo', rand) + obj = bucket.new(self.key_name, rand) obj.store() - obj = bucket.get('foo') + obj = bucket.get(self.key_name) self.assertTrue(obj.exists()) obj.delete() obj.reload() self.assertFalse(obj.exists()) def test_set_bucket_properties(self): - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) # Test setting allow mult... bucket.set_allow_multiples(True) self.assertTrue(bucket.get_allow_multiples()) @@ -153,7 +154,7 @@ def test_set_bucket_properties(self): self.assertEqual(bucket.get_n_val(), 2) def test_rw_settings(self): - bucket = self.client.bucket('rwsettings') + bucket = self.client.bucket(self.bucket_name) self.assertEqual(bucket.get_r(), "default") self.assertEqual(bucket.get_w(), "default") self.assertEqual(bucket.get_dw(), "default") @@ -172,7 +173,7 @@ def test_rw_settings(self): self.assertEqual(bucket.get_rw(), "one") def test_primary_quora(self): - bucket = self.client.bucket('primary_quora') + bucket = self.client.bucket(self.bucket_name) self.assertEqual(bucket.get_pr(), "default") self.assertEqual(bucket.get_pw(), "default") @@ -183,8 +184,8 @@ def test_primary_quora(self): self.assertEqual(bucket.get_pw(), "quorum") def test_if_none_match(self): - bucket = self.client.bucket('if_none_match_test') - obj = bucket.get('obj') + bucket = self.client.bucket(self.bucket_name) + obj = bucket.get(self.key_name) obj.delete() obj.reload() @@ -198,9 +199,9 @@ def test_if_none_match(self): def test_siblings(self): # Set up the bucket, clear any existing object... - bucket = self.client.bucket('multiBucket') - bucket.set_allow_multiples(True) - obj = bucket.get_binary('foo') + self.create_client().bucket(self.bucket_name).set_allow_multiples(True) + bucket = self.client.bucket(self.bucket_name) + obj = bucket.get_binary(self.key_name) # Even if it previously existed, let's store a base resolved version # from which we can diverge by sending a stale vclock. obj.set_data('start') @@ -210,13 +211,13 @@ def test_siblings(self): vals = set() for i in range(5): other_client = self.create_client() - other_bucket = other_client.bucket('multiBucket') + other_bucket = other_client.bucket(self.bucket_name) while True: randval = self.randint() if randval not in vals: break - other_obj = other_bucket.new_binary('foo', str(randval)) + other_obj = other_bucket.new_binary(self.key_name, str(randval)) other_obj._vclock = obj._vclock other_obj.store() vals.add(str(randval)) @@ -241,9 +242,9 @@ def test_siblings(self): self.assertEqual(obj.get_data(), obj3.get_data()) def test_store_of_missing_object(self): - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) # for json objects - o = bucket.get("nonexistent_key_json") + o = bucket.get(self.key_name) self.assertEqual(o.exists(), False) o.set_data({"foo": "bar"}) o = o.store() @@ -251,7 +252,7 @@ def test_store_of_missing_object(self): self.assertEqual(o.get_content_type(), "application/json") o.delete() # for binary objects - o = bucket.get_binary("nonexistent_key_binary") + o = bucket.get_binary(self.randname()) self.assertEqual(o.exists(), False) o.set_data("1234567890") o = o.store() @@ -260,44 +261,44 @@ def test_store_of_missing_object(self): o.delete() def test_store_metadata(self): - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) rand = self.randint() - obj = bucket.new('fooster', rand) + obj = bucket.new(self.key_name, rand) obj.set_usermeta({'custom': 'some metadata'}) obj.store() - obj = bucket.get('fooster') + obj = bucket.get(self.key_name) self.assertEqual('some metadata', obj.get_usermeta()['custom']) def test_list_buckets(self): - bucket = self.client.bucket("list_bucket") + bucket = self.client.bucket(self.bucket_name) bucket.new("one", {"foo": "one", "bar": "red"}).store() buckets = self.client.get_buckets() - self.assertTrue("list_bucket" in buckets) + self.assertTrue(self.bucket_name in buckets) class KVFileTests(object): def test_store_binary_object_from_file(self): - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) rand = str(self.randint()) filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') - obj = bucket.new_binary_from_file('foo_from_file', filepath) + obj = bucket.new_binary_from_file(self.key_name, filepath) obj.store() - obj = bucket.get_binary('foo_from_file') + obj = bucket.get_binary(self.key_name) self.assertNotEqual(obj.get_data(), None) self.assertEqual(obj.get_content_type(), "text/x-python") def test_store_binary_object_from_file_should_use_default_mimetype(self): - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) rand = str(self.randint()) filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir, 'THANKS') - obj = bucket.new_binary_from_file('foo_from_file', filepath) + obj = bucket.new_binary_from_file(self.key_name, filepath) obj.store() - obj = bucket.get_binary('foo_from_file') + obj = bucket.get_binary(self.key_name) self.assertEqual(obj.get_content_type(), 'application/octet-stream') def test_store_binary_object_from_file_should_fail_if_file_not_found(self): - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) rand = str(self.randint()) self.assertRaises(IOError, bucket.new_binary_from_file, 'not_found_from_file', 'FILE_NOT_FOUND') diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index f57a2cd3..31d949fa 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -27,7 +27,7 @@ unittest = __import__('unittest2') else: import unittest - +import os class SimplePool(Pool): def __init__(self): @@ -47,6 +47,8 @@ def create_resource(self): return [] +@unittest.skipIf(os.environ.get('SKIP_POOL'), + 'Skipping connection pool tests') class PoolTest(unittest.TestCase): def test_yields_new_object_when_empty(self): """ diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index f3223da0..a48ee89b 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -12,120 +12,120 @@ class EnableSearchTests(object): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_bucket_search_enabled(self): - bucket = self.client.bucket("unsearch_bucket") + bucket = self.client.bucket(self.bucket_name) self.assertFalse(bucket.search_enabled()) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_enable_search_commit_hook(self): - bucket = self.client.bucket("search_bucket") + bucket = self.client.bucket(self.bucket_name) bucket.enable_search() - self.assertTrue(self.client.bucket("search_bucket").search_enabled()) + self.assertTrue(self.client.bucket(self.bucket_name).search_enabled()) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_disable_search_commit_hook(self): - bucket = self.client.bucket("no_search_bucket") + bucket = self.client.bucket(self.bucket_name) bucket.enable_search() - self.assertTrue(self.client.bucket("no_search_bucket")\ + self.assertTrue(self.client.bucket(self.bucket_name)\ .search_enabled()) bucket.disable_search() - self.assertFalse(self.client.bucket("no_search_bucket")\ + self.assertFalse(self.client.bucket(self.bucket_name)\ .search_enabled()) class SolrSearchTests(object): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_document_to_index(self): - self.client.solr().add("searchbucket", + self.client.solr().add(self.search_bucket, {"id": "doc", "username": "tony"}) - results = self.client.solr().search("searchbucket", "username:tony") + results = self.client.solr().search(self.search_bucket, "username:tony") self.assertEquals("tony", results['docs'][0]['username']) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_add_multiple_documents_to_index(self): - self.client.solr().add("searchbucket", + def test_add_multiple_documents_to_iindex(self): + self.client.solr().add(self.search_bucket, {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) results = self.client.solr()\ - .search("searchbucket", "username:russell OR username:dizzy") + .search(self.search_bucket, "username:russell OR username:dizzy") self.assertEquals(2, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_id(self): - self.client.solr().add("searchbucket", + self.client.solr().add(self.search_bucket, {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - self.client.solr().delete("searchbucket", docs=["dizzy"]) + self.client.solr().delete(self.search_bucket, docs=["dizzy"]) results = self.client.solr()\ - .search("searchbucket", "username:russell OR username:dizzy") + .search(self.search_bucket, "username:russell OR username:dizzy") self.assertEquals(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): - self.client.solr().add("searchbucket", + self.client.solr().add(self.search_bucket, {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr()\ - .delete("searchbucket", + .delete(self.search_bucket, queries=["username:dizzy", "username:russell"]) results = self.client.solr()\ - .search("searchbucket", "username:russell OR username:dizzy") + .search(self.search_bucket, "username:russell OR username:dizzy") self.assertEquals(0, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query_and_id(self): - self.client.solr().add("searchbucket", + self.client.solr().add(self.search_bucket, {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - self.client.solr().delete("searchbucket", + self.client.solr().delete(self.search_bucket, docs=["dizzy"], queries=["username:russell"]) results = self.client.solr()\ - .search("searchbucket", + .search(self.search_bucket, "username:russell OR username:dizzy") self.assertEquals(0, len(results['docs'])) def test_build_rest_path_excludes_empty_query_params(self): self.assertEquals( self.client.get_transport().build_rest_path( - bucket=self.client.bucket("foo"), - key="bar", params={'r': None}), "/riak/foo/bar?") + bucket=self.client.bucket(self.search_bucket), + key="bar", params={'r': None}), "/riak/"+self.search_bucket+"/bar?") class SearchTests(object): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_from_bucket(self): - bucket = self.client.bucket('searchbucket') + bucket = self.client.bucket(self.search_bucket) bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage") self.assertEquals(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params_from_bucket(self): - bucket = self.client.bucket('searchbucket') + bucket = self.client.bucket(self.search_bucket) bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage", wt="xml") self.assertEquals(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params(self): - bucket = self.client.bucket('searchbucket') + bucket = self.client.bucket(self.search_bucket) bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr().search("searchbucket", + results = self.client.solr().search(self.search_bucket, "username:roidrage", wt="xml") self.assertEquals(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search(self): - bucket = self.client.bucket('searchbucket') + bucket = self.client.bucket(self.search_bucket) bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr().search("searchbucket", + results = self.client.solr().search(self.search_bucket, "username:roidrage") self.assertEquals(1, len(results["docs"])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_search_integration(self): # Create some objects to search across... - bucket = self.client.bucket("searchbucket") + bucket = self.client.bucket(self.search_bucket) bucket.new("one", {"foo": "one", "bar": "red"}).store() bucket.new("two", {"foo": "two", "bar": "green"}).store() bucket.new("three", {"foo": "three", "bar": "blue"}).store() @@ -133,7 +133,7 @@ def test_search_integration(self): bucket.new("five", {"foo": "five", "bar": "yellow"}).store() # Run some operations... - results = self.client.solr().search("searchbucket", + results = self.client.solr().search(self.search_bucket, "foo:one OR foo:two") if (len(results) == 0): print "\n\nNot running test \"testSearchIntegration()\".\n" @@ -142,7 +142,7 @@ def test_search_integration(self): \"bin/search-cmd install searchbucket\".\n\n""" return self.assertEqual(len(results['docs']), 2) - query = "(foo:one OR foo:two OR foo:three OR foo:four) AND\ - (NOT bar:green)" - results = self.client.solr().search("searchbucket", query) + query = "(foo:one OR foo:two OR foo:three OR foo:four) AND" \ + " (NOT bar:green)" + results = self.client.solr().search(self.search_bucket, query) self.assertEqual(len(results['docs']), 3) From cef7733cb1083cef747e4d4c006b0c04f27083d7 Mon Sep 17 00:00:00 2001 From: evan Date: Sat, 17 Nov 2012 14:14:53 -0800 Subject: [PATCH 0246/1060] pep8 fixes --- riak/bucket.py | 34 ++++++++++++++++++++++++++-------- riak/client.py | 1 + riak/tests/test_all.py | 2 +- riak/tests/test_kv.py | 4 ++-- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 2cab2e9e..94029921 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -25,6 +25,7 @@ def deprecateBucketQuorumAccessors(klass): return deprecateQuorumAccessors(klass, parent='_client') + @deprecateBucketQuorumAccessors class RiakBucket(object): """ @@ -184,15 +185,17 @@ def get_binary(self, key, r=None, pr=None): def _set_n_val(self, nval): return self.set_property('n_val', nval) + def _get_n_val(self): return self.get_property('n_val') - n_val = property(_get_n_val, _set_n_val, doc = - """ + + n_val = property(_get_n_val, _set_n_val, doc= + """ N-value for this bucket, which is the number of replicas that will be written of each object in the bucket. - + .. warning:: - + Set this once before you write any data to the bucket, and never change it again, otherwise unpredictable things could happen. This should only be used if you know what you are doing. @@ -202,51 +205,65 @@ def _get_n_val(self): def _set_allow_mult(self, bool): return self.set_property('allow_mult', bool) + def _get_allow_mult(self): return self.get_property('allow_mult') - allow_mult = property(_get_allow_mult, _set_allow_mult, doc = - """ + + allow_mult = property(_get_allow_mult, _set_allow_mult, doc= + """ If set to True, then writes with conflicting data will be stored and returned to the client. This situation can be detected by calling has_siblings() and get_siblings(). - + :type bool: boolean """) def _set_r(self, val): return self.set_property('r', val) + def _get_r(self): return self.get_property('r') + r = property(_get_r, _set_r) def _set_pr(self, val): return self.set_property('pr', val) + def _get_pr(self): return self.get_property('pr') + pr = property(_get_pr, _set_pr) def _set_rw(self, val): return self.set_property('rw', val) + def _get_rw(self): return self.get_property('rw') + rw = property(_get_rw, _set_rw) def _set_w(self, val): return self.set_property('w', val) + def _get_w(self): return self.get_property('w') + w = property(_get_w, _set_w) def _set_dw(self, val): return self.set_property('dw', val) + def _get_dw(self): return self.get_property('dw') + dw = property(_get_dw, _set_dw) def _set_pw(self, val): return self.set_property('pw', val) + def _get_pw(self): return self.get_property('pw') + pw = property(_get_pw, _set_pw) def set_property(self, key, value): @@ -337,7 +354,8 @@ def enable_search(self): precommit_hooks = self.get_property("precommit") or [] if self.SEARCH_PRECOMMIT_HOOK not in precommit_hooks: self.set_properties({"precommit": - precommit_hooks + [self.SEARCH_PRECOMMIT_HOOK]}) + precommit_hooks + + [self.SEARCH_PRECOMMIT_HOOK]}) return True def disable_search(self): diff --git a/riak/client.py b/riak/client.py index 3a81836c..483445a0 100644 --- a/riak/client.py +++ b/riak/client.py @@ -30,6 +30,7 @@ from riak.util import deprecated from riak.util import deprecateQuorumAccessors + @deprecateQuorumAccessors class RiakClient(object): """ diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index b57521f8..c5321f9c 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -160,7 +160,7 @@ def test_bucket_search_enabled(self): with self.assertRaises(NotImplementedError): bucket = self.client.bucket("unsearch_bucket") test = bucket.search_enabled() - + def test_enable_search_commit_hook(self): with self.assertRaises(NotImplementedError): bucket = self.client.bucket("search_bucket") diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 9b5d6c17..5474a009 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -152,7 +152,6 @@ def test_set_bucket_properties(self): self.assertFalse(bucket.allow_mult) self.assertEqual(bucket.n_val, 2) - def test_if_none_match(self): bucket = self.client.bucket('if_none_match_test') obj = bucket.get('obj') @@ -245,6 +244,7 @@ def test_list_buckets(self): buckets = self.client.get_buckets() self.assertTrue("list_bucket" in buckets) + class HTTPBucketPropsTest(object): def test_rw_settings(self): bucket = self.client.bucket('rwsettings') @@ -264,7 +264,7 @@ def test_rw_settings(self): bucket.rw = "one" self.assertEqual(bucket.rw, "one") - + def test_primary_quora(self): bucket = self.client.bucket('primary_quora') self.assertEqual(bucket.pr, 0) From ac2d6bd0af88ae3e86a7bfe96cdb6136da43a810 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 18 Nov 2012 17:01:39 -0500 Subject: [PATCH 0247/1060] Minor pep8 style fix. --- riak/bucket.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 94029921..7e12e51b 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -189,8 +189,7 @@ def _set_n_val(self, nval): def _get_n_val(self): return self.get_property('n_val') - n_val = property(_get_n_val, _set_n_val, doc= - """ + n_val = property(_get_n_val, _set_n_val, doc=""" N-value for this bucket, which is the number of replicas that will be written of each object in the bucket. @@ -209,8 +208,7 @@ def _set_allow_mult(self, bool): def _get_allow_mult(self): return self.get_property('allow_mult') - allow_mult = property(_get_allow_mult, _set_allow_mult, doc= - """ + allow_mult = property(_get_allow_mult, _set_allow_mult, doc=""" If set to True, then writes with conflicting data will be stored and returned to the client. This situation can be detected by calling has_siblings() and get_siblings(). From 2f3fa1e754391bb835516e3a158992c4901e86a4 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 18 Nov 2012 17:18:42 -0500 Subject: [PATCH 0248/1060] Cleanup some bucket properties after setting them in the test. --- riak/tests/test_kv.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 5474a009..b244870d 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -265,6 +265,11 @@ def test_rw_settings(self): bucket.rw = "one" self.assertEqual(bucket.rw, "one") + bucket.set_properties({'w': 'quorum', + 'r': 'quorom', + 'dw': 'quorum', + 'rw': 'quorum'}) + def test_primary_quora(self): bucket = self.client.bucket('primary_quora') self.assertEqual(bucket.pr, 0) @@ -276,6 +281,8 @@ def test_primary_quora(self): bucket.pw = "quorum" self.assertEqual(bucket.pw, "quorum") + bucket.set_properties({'pr': 0, 'pw': 0}) + class PbcBucketPropsTest(object): def test_rw_settings(self): From 466a9cfe2976cd837b1d33f68d9050e98d9e8bf3 Mon Sep 17 00:00:00 2001 From: evan Date: Sun, 18 Nov 2012 21:15:35 -0800 Subject: [PATCH 0249/1060] fix test typo --- riak/tests/test_kv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index b244870d..8af95bbb 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -266,7 +266,7 @@ def test_rw_settings(self): self.assertEqual(bucket.rw, "one") bucket.set_properties({'w': 'quorum', - 'r': 'quorom', + 'r': 'quorum', 'dw': 'quorum', 'rw': 'quorum'}) From 9ca9f12ff2d3a150f4d53f373c71888a55d6adfd Mon Sep 17 00:00:00 2001 From: evan Date: Tue, 20 Nov 2012 10:58:01 -0800 Subject: [PATCH 0250/1060] add the ability to pass in a non-string iterable to specify a list of keys in a bucket to fetch via mapreduce --- riak/mapreduce.py | 14 ++++++--- riak/tests/test_mapreduce.py | 60 ++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index ce70fd8d..6ac38521 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -20,6 +20,7 @@ import urllib from riak_object import RiakObject from bucket import RiakBucket +from collections import Iterable class RiakMapReduce(object): @@ -46,7 +47,7 @@ def add(self, arg1, arg2=None, arg3=None): specify either a RiakObject, a string bucket name, or a bucket, key, and additional arg. @param mixed arg1 - RiakObject or Bucket - @param mixed arg2 - Key or blank + @param mixed arg2 - Key or List or blank @param mixed arg3 - Arg or blank @return RiakMapReduce """ @@ -63,11 +64,16 @@ def add_object(self, obj): def add_bucket_key_data(self, bucket, key, data): if self._input_mode == 'bucket': - raise Exception('Already added a bucket, can\'t add an object.') + raise RiakError('Already added a bucket, can\'t add an object.') elif self._input_mode == 'query': - raise Exception('Already added a query, can\'t add an object.') + raise RiakError('Already added a query, can\'t add an object.') else: - self._inputs.append([bucket, key, data]) + if isinstance(key, Iterable) and \ + not isinstance(key, basestring): + for k in key: + self._inputs.append([bucket, k, data]) + else: + self._inputs.append([bucket, key, data]) return self def add_bucket(self, bucket): diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index ed65e03d..5cbb4a85 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -229,6 +229,66 @@ def test_map_reduce_from_object(self): result = obj.map("Riak.mapValuesJson").run() self.assertEqual(result, [2]) + def test_mr_list_add(self): + bucket = self.client.bucket("abucket") + for x in range(20): + bucket.new('baz' + str(x), + 'bazval' + str(x)).store() + mr = self.client.add('abucket', ['baz' + str(x) + for x in range(2, 5)]) + results = mr.map_values().run() + results.sort() + self.assertEqual(results, + [u'"bazval2"', + u'"bazval3"', + u'"bazval4"']) + + def test_mr_list_add_two_buckets(self): + bucket = self.client.bucket("bucket_a") + for x in range(10): + bucket.new('foo' + str(x), + 'fooval' + str(x)).store() + bucket = self.client.bucket("bucket_b") + for x in range(10): + bucket.new('bar' + str(x), + 'barval' + str(x)).store() + + mr = self.client.add('bucket_a', ['foo' + str(x) + for x in range(2, 4)]) + mr.add('bucket_b', ['bar' + str(x) + for x in range(5, 7)]) + results = mr.map_values().run() + results.sort() + + self.assertEqual(results, + [u'"barval5"', + u'"barval6"', + u'"fooval2"', + u'"fooval3"']) + + def test_mr_list_add_mix(self): + bucket = self.client.bucket("bucket_a") + for x in range(10): + bucket.new('foo' + str(x), + 'fooval' + str(x)).store() + bucket = self.client.bucket("bucket_b") + for x in range(10): + bucket.new('bar' + str(x), + 'barval' + str(x)).store() + + mr = self.client.add('bucket_a', ['foo' + str(x) + for x in range(2, 4)]) + mr.add('bucket_b', 'bar9') + mr.add('bucket_b', 'bar2') + results = mr.map_values().run() + results.sort() + + self.assertEqual(results, + [u'"barval2"', + u'"barval9"', + u'"fooval2"', + u'"fooval3"']) + class MapReduceAliasTests(object): """This tests the map reduce aliases""" From 29b12397d10c346019ea4b3edd7106e03bb0cd70 Mon Sep 17 00:00:00 2001 From: evan Date: Tue, 20 Nov 2012 14:08:33 -0800 Subject: [PATCH 0251/1060] pre-pep8 cleanup --- riak/tests/test_2i.py | 10 +-- riak/tests/test_all.py | 9 ++- riak/tests/test_mapreduce.py | 135 +++++++++++++++++------------------ 3 files changed, 77 insertions(+), 77 deletions(-) diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index d813e5d6..8664f23a 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -28,7 +28,7 @@ def test_secondary_index_store(self): return True # Create a new object with indexes... - bucket = self.client.bucket('indexbucket') + bucket = self.client.bucket(self.bucket_name) rand = self.randint() obj = bucket.new('mykey1', rand) obj.add_index('field1_bin', 'val1a') @@ -103,10 +103,10 @@ def test_set_indexes(self): if not self.is_2i_supported(): return True - bucket = self.client.bucket('indexbucket') + bucket = self.client.bucket(self.bucket_name) foo = bucket.new('foo', 1) foo.set_indexes((('field1_bin', 'test'), ('field2_int', 1337))).store() - result = self.client.index('indexbucket', 'field2_int', 1337).run() + result = self.client.index(self.bucket_name, 'field2_int', 1337).run() self.assertEqual(1, len(result)) self.assertEqual('foo', result[0].get_key()) @@ -119,7 +119,7 @@ def test_remove_indexes(self): if not self.is_2i_supported(): return True - bucket = self.client.bucket('indexbucket') + bucket = self.client.bucket(self.bucket_name) bar = bucket.new('bar', 1).add_index('bar_int', 1)\ .add_index('bar_int', 2).add_index('baz_bin', 'baz').store() result = bucket.get_index('bar_int', 1) @@ -172,7 +172,7 @@ def test_secondary_index_query(self): if not self.is_2i_supported(): return True - bucket = self.client.bucket('indexbucket') + bucket = self.client.bucket(self.bucket_name) bucket.\ new('mykey1', 'data1').\ diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 43a4e343..31d44995 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -52,6 +52,7 @@ test_server.prepare() test_server.start() +testrun_search_bucket = None class BaseTestCase(object): @@ -74,15 +75,17 @@ def create_client(self, host=None, port=None, transport_class=None): transport_class=transport_class) def setUp(self): + global testrun_search_bucket self.bucket_name = self.randname() self.key_name = self.randname() - if not getattr(self, 'search_bucket', None): - print repr(self), 'creating search bucket' - self.search_bucket = self.randname() + if not testrun_search_bucket: + self.search_bucket = testrun_search_bucket = self.randname() c = self.create_client(HTTP_HOST, HTTP_PORT, RiakHttpTransport) b = c.bucket(self.search_bucket) b.enable_search() + else: + self.search_bucket = testrun_search_bucket self.client = self.create_client() diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 84ba7d9f..97448604 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -7,7 +7,7 @@ class LinkTests(object): def test_store_and_get_links(self): # Create the object... - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) bucket.new_binary("test_store_and_get_links", '2') \ .add_link(bucket.new("foo1")) \ .add_link(bucket.new("foo2"), "tag") \ @@ -18,7 +18,7 @@ def test_store_and_get_links(self): self.assertEqual(len(links), 3) for l in links: if (l.get_key() == "foo1"): - self.assertEqual(l.get_tag(), "bucket") + self.assertEqual(l.get_tag(), self.bucket_name) elif (l.get_key() == "foo2"): self.assertEqual(l.get_tag(), "tag") elif (l.get_key() == "foo3"): @@ -28,10 +28,10 @@ def test_store_and_get_links(self): def test_set_links(self): # Create the object - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) bucket.new("foo", 2).set_links([bucket.new("foo1"), (bucket.new("foo2"), "tag"), - RiakLink("bucket", "foo2", "tag2")]).store() + RiakLink(self.bucket_name, "foo2", "tag2")]).store() obj = bucket.get("foo") links = sorted(obj.get_links(), key=lambda x: x.get_key()) self.assertEqual(len(links), 3) @@ -42,10 +42,10 @@ def test_set_links(self): self.assertEqual(links[2].get_tag(), "tag2") def test_set_links_all_links(self): - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) foo1 = bucket.new("foo", 1) foo2 = bucket.new("foo2", 2).store() - links = [RiakLink("bucket", "foo2")] + links = [RiakLink(self.bucket_name, "foo2")] foo1.set_links(links, True) links = foo1.get_links() self.assertEqual(len(links), 1) @@ -53,31 +53,31 @@ def test_set_links_all_links(self): def test_link_walking(self): # Create the object... - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) bucket.new("foo", 2) \ .add_link(bucket.new("foo1", "test1").store()) \ .add_link(bucket.new("foo2", "test2").store(), "tag") \ .add_link(bucket.new("foo3", "test3").store(), "tag2!@#%^&*)") \ .store() obj = bucket.get("foo") - results = obj.link("bucket").run() + results = obj.link(self.bucket_name).run() self.assertEqual(len(results), 3) - results = obj.link("bucket", "tag").run() + results = obj.link(self.bucket_name, "tag").run() self.assertEqual(len(results), 1) class ErlangMapReduceTests(object): def test_erlang_map_reduce(self): # Create the object... - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) bucket.new("foo", 2).store() bucket.new("bar", 2).store() bucket.new("baz", 4).store() # Run the map... result = self.client \ - .add("bucket", "foo") \ - .add("bucket", "bar") \ - .add("bucket", "baz") \ + .add(self.bucket_name, "foo") \ + .add(self.bucket_name, "bar") \ + .add(self.bucket_name, "baz") \ .map(["riak_kv_mapreduce", "map_object_value"]) \ .reduce(["riak_kv_mapreduce", "reduce_set_union"]) \ .run() @@ -87,10 +87,10 @@ def test_erlang_map_reduce(self): class JSMapReduceTests(object): def test_javascript_source_map(self): # Create the object... - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) bucket.new("foo", 2).store() # Run the map... - mr = self.client.add("bucket", "foo") + mr = self.client.add(self.bucket_name, "foo") result = mr.map( "function (v) { return [JSON.parse(v.values[0].data)]; }").run() self.assertEqual(result, [2]) @@ -108,26 +108,26 @@ def test_javascript_source_map(self): def test_javascript_named_map(self): # Create the object... - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) bucket.new("foo", 2).store() # Run the map... result = self.client \ - .add("bucket", "foo") \ + .add(self.bucket_name, "foo") \ .map("Riak.mapValuesJson") \ .run() self.assertEqual(result, [2]) def test_javascript_source_map_reduce(self): # Create the object... - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) bucket.new("foo", 2).store() bucket.new("bar", 3).store() bucket.new("baz", 4).store() # Run the map... result = self.client \ - .add("bucket", "foo") \ - .add("bucket", "bar") \ - .add("bucket", "baz") \ + .add(self.bucket_name, "foo") \ + .add(self.bucket_name, "bar") \ + .add(self.bucket_name, "baz") \ .map("function (v) { return [1]; }") \ .reduce("Riak.reduceSum") \ .run() @@ -135,15 +135,15 @@ def test_javascript_source_map_reduce(self): def test_javascript_named_map_reduce(self): # Create the object... - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) bucket.new("foo", 2).store() bucket.new("bar", 3).store() bucket.new("baz", 4).store() # Run the map... result = self.client \ - .add("bucket", "foo") \ - .add("bucket", "bar") \ - .add("bucket", "baz") \ + .add(self.bucket_name, "foo") \ + .add(self.bucket_name, "bar") \ + .add(self.bucket_name, "baz") \ .map("Riak.mapValuesJson") \ .reduce("Riak.reduceSum") \ .run() @@ -165,15 +165,15 @@ def test_javascript_bucket_map_reduce(self): def test_javascript_arg_map_reduce(self): # Create the object... - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) bucket.new("foo", 2).store() # Run the map... result = self.client \ - .add("bucket", "foo", 5) \ - .add("bucket", "foo", 10) \ - .add("bucket", "foo", 15) \ - .add("bucket", "foo", -15) \ - .add("bucket", "foo", -5) \ + .add(self.bucket_name, "foo", 5) \ + .add(self.bucket_name, "foo", 10) \ + .add(self.bucket_name, "foo", 15) \ + .add(self.bucket_name, "foo", -15) \ + .add(self.bucket_name, "foo", -5) \ .map("function(v, arg) { return [arg]; }") \ .reduce("Riak.reduceSum") \ .run() @@ -223,7 +223,7 @@ def test_key_filters_with_search_query(self): def test_map_reduce_from_object(self): # Create the object... - bucket = self.client.bucket("bucket") + bucket = self.client.bucket(self.bucket_name) bucket.new("foo", 2).store() obj = bucket.get("foo") result = obj.map("Riak.mapValuesJson").run() @@ -235,13 +235,13 @@ class MapReduceAliasTests(object): def test_map_values(self): # Add a value to the bucket - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new_binary('one', data='value_1').store() bucket.new_binary('two', data='value_2').store() # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') + mr = self.client.add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two') # Use the map_values alias result = mr.map_values().run() @@ -254,13 +254,13 @@ def test_map_values(self): def test_map_values_json(self): # Add a value to the bucket - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new('one', data={'val': 'value_1'}).store() bucket.new('two', data={'val': 'value_2'}).store() # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') + mr = self.client.add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two') # Use the map_values alias result = mr.map_values_json().run() @@ -273,13 +273,13 @@ def test_map_values_json(self): def test_reduce_sum(self): # Add a value to the bucket - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() bucket.new('two', data=2).store() # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') + mr = self.client.add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two') # Use the map_values alias result = mr.map_values_json().reduce_sum().run() @@ -288,13 +288,13 @@ def test_reduce_sum(self): def test_reduce_min(self): # Add a value to the bucket - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() bucket.new('two', data=2).store() # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') + mr = self.client.add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two') # Use the map_values alias result = mr.map_values_json().reduce_min().run() @@ -303,13 +303,13 @@ def test_reduce_min(self): def test_reduce_max(self): # Add a value to the bucket - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() bucket.new('two', data=2).store() # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') + mr = self.client.add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two') # Use the map_values alias result = mr.map_values_json().reduce_max().run() @@ -318,13 +318,13 @@ def test_reduce_max(self): def test_reduce_sort(self): # Add a value to the bucket - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new('one', data="value1").store() bucket.new('two', data="value2").store() # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') + mr = self.client.add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two') # Use the map_values alias result = mr.map_values_json().reduce_sort().run() @@ -333,13 +333,13 @@ def test_reduce_sort(self): def test_reduce_sort_custom(self): # Add a value to the bucket - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new('one', data="value1").store() bucket.new('two', data="value2").store() # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') + mr = self.client.add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two') # Use the map_values alias result = mr.map_values_json().reduce_sort("""function(x,y) { @@ -351,13 +351,13 @@ def test_reduce_sort_custom(self): def test_reduce_numeric_sort(self): # Add a value to the bucket - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() bucket.new('two', data=2).store() # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') + mr = self.client.add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two') # Use the map_values alias result = mr.map_values_json().reduce_numeric_sort().run() @@ -366,13 +366,13 @@ def test_reduce_numeric_sort(self): def test_reduce_limit(self): # Add a value to the bucket - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() bucket.new('two', data=2).store() # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') + mr = self.client.add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two') # Use the map_values alias result = mr.map_values_json()\ @@ -383,13 +383,13 @@ def test_reduce_limit(self): def test_reduce_slice(self): # Add a value to the bucket - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() bucket.new('two', data=2).store() # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two') + mr = self.client.add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two') # Use the map_values alias result = mr.map_values_json()\ @@ -400,17 +400,14 @@ def test_reduce_slice(self): def test_filter_not_found(self): # Add a value to the bucket - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() bucket.new('two', data=2).store() - # Make sure "three" does not exist - bucket.get('three').delete() - # Create a map reduce object and use one and two as inputs - mr = self.client.add('bucket', 'one')\ - .add('bucket', 'two')\ - .add('bucket', 'three') + mr = self.client.add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two')\ + .add(self.bucket_name, self.key_name) # Use the map_values alias result = mr.map_values_json()\ From 0e0e8e9e47d435eff393ed1f913652c7a0a20d40 Mon Sep 17 00:00:00 2001 From: evan Date: Tue, 20 Nov 2012 14:45:21 -0800 Subject: [PATCH 0252/1060] pep8 cleanup --- riak/tests/test_all.py | 12 +++++++----- riak/tests/test_pool.py | 3 ++- riak/tests/test_search.py | 8 +++++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 31d44995..5c5ad0fc 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -54,6 +54,7 @@ testrun_search_bucket = None + class BaseTestCase(object): @staticmethod @@ -61,7 +62,7 @@ def randint(): return random.randint(1, 999999) @staticmethod - def randname(length = 12): + def randname(length=12): out = '' for i in range(length): out += chr(random.randint(ord('a'), ord('z'))) @@ -80,7 +81,7 @@ def setUp(self): self.key_name = self.randname() if not testrun_search_bucket: self.search_bucket = testrun_search_bucket = self.randname() - c = self.create_client(HTTP_HOST, HTTP_PORT, + c = self.create_client(HTTP_HOST, HTTP_PORT, RiakHttpTransport) b = c.bucket(self.search_bucket) b.enable_search() @@ -88,7 +89,8 @@ def setUp(self): self.search_bucket = testrun_search_bucket self.client = self.create_client() - + + class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, TwoITests, @@ -173,8 +175,8 @@ def test_bucket_search_enabled(self): def test_enable_search_commit_hook(self): bucket = self.client.bucket(self.bucket_name) - bucket.enable_search() - self.assertRaises(NotImplementedError) + bucket.enable_search() + self.assertRaises(NotImplementedError) class RiakHttpTransportTestCase(BasicKVTests, diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index 31d949fa..5fdb4c7a 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -29,6 +29,7 @@ import unittest import os + class SimplePool(Pool): def __init__(self): self.count = 0 @@ -47,7 +48,7 @@ def create_resource(self): return [] -@unittest.skipIf(os.environ.get('SKIP_POOL'), +@unittest.skipIf(os.environ.get('SKIP_POOL'), 'Skipping connection pool tests') class PoolTest(unittest.TestCase): def test_yields_new_object_when_empty(self): diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index a48ee89b..e8896425 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -37,7 +37,8 @@ class SolrSearchTests(object): def test_add_document_to_index(self): self.client.solr().add(self.search_bucket, {"id": "doc", "username": "tony"}) - results = self.client.solr().search(self.search_bucket, "username:tony") + results = self.client.solr().search(self.search_bucket, + "username:tony") self.assertEquals("tony", results['docs'][0]['username']) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') @@ -61,7 +62,7 @@ def test_delete_documents_from_search_by_id(self): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): - self.client.solr().add(self.search_bucket, + self.client.solr().add(self.search_bucket, {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr()\ @@ -88,7 +89,8 @@ def test_build_rest_path_excludes_empty_query_params(self): self.assertEquals( self.client.get_transport().build_rest_path( bucket=self.client.bucket(self.search_bucket), - key="bar", params={'r': None}), "/riak/"+self.search_bucket+"/bar?") + key="bar", params={'r': None}), + "/riak/" + self.search_bucket + "/bar?") class SearchTests(object): From 6ba788271a5f8c8846e94087b27ce501569b6432 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Fri, 23 Nov 2012 10:27:08 -0500 Subject: [PATCH 0253/1060] Fixed #171 Now checks if the index name ends with `_bin` or `_int` The mechanism for the checking right now uses `[-4:]` as we know it must be 4 characters long at the end as oppose to `endswith` as we would need an `or` as opposte to just 1 statement. --- riak/riak_object.py | 3 +++ riak/tests/test_2i.py | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/riak/riak_object.py b/riak/riak_object.py index c1ab33e1..29e73076 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -193,6 +193,9 @@ def add_index(self, field, value): :type value: string or integer :rtype: self """ + if field[-4:] not in ("_bin", "_int"): + raise RiakError("Riak 2i fields must end with either '_bin' or '_int'.") + rie = RiakIndexEntry(field, value) if not rie in self._metadata[MD_INDEX]: self._metadata[MD_INDEX].append(rie) diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index d813e5d6..f9eef7e4 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -7,6 +7,7 @@ import unittest from riak.riak_index_entry import RiakIndexEntry +from riak import RiakError SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '0')) @@ -222,3 +223,13 @@ def test_secondary_index_query(self): bucket.get('mykey2').delete() bucket.get('mykey3').delete() bucket.get('mykey4').delete() + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_secondary_index_invalid_name(self): + if not self.is_2i_supported(): + return True + + bucket = self.client.bucket('indexbucket') + + testlambda = lambda: bucket.new('k', 'a').add_index('field1', 'value1') + self.assertRaises(RiakError, testlambda) From 2d6c3ef70aabe1273b91588d89aadc382596a5ce Mon Sep 17 00:00:00 2001 From: Shuhao Date: Fri, 23 Nov 2012 14:05:52 -0500 Subject: [PATCH 0254/1060] Changed format for `assertRaises` --- riak/tests/test_2i.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index f9eef7e4..e2d6c5cb 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -231,5 +231,5 @@ def test_secondary_index_invalid_name(self): bucket = self.client.bucket('indexbucket') - testlambda = lambda: bucket.new('k', 'a').add_index('field1', 'value1') - self.assertRaises(RiakError, testlambda) + with self.assertRaises(RiakError): + bucket.new('k', 'a').add_index('field1', 'value1') From 6f363198604e4ac53c1226c3f8e9675be330ac43 Mon Sep 17 00:00:00 2001 From: Daniel Kraft Date: Fri, 7 Dec 2012 16:58:25 +0100 Subject: [PATCH 0255/1060] Allow erlang ad-hoc functions in m/r (allow_strfun) --- riak/mapreduce.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index ce70fd8d..53b2cc19 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -359,6 +359,9 @@ def to_array(self): stepdef['module'] = self._function[0] stepdef['function'] = self._function[1] + elif (self._language == 'erlang' and isinstance(self._function, str)): + stepdef['source'] = self._function + return {self._type: stepdef} From 978a22c0b3841fa08b0b7b6c403d1fc1140bff35 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 9 Dec 2012 12:12:57 -0600 Subject: [PATCH 0256/1060] Loosen the version restriction if Riak is on a prerelease version. --- riak/tests/test_feature_detection.py | 11 +++++++++++ riak/transports/feature_detect.py | 10 +++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py index a1a7ff29..0ab7fde4 100644 --- a/riak/tests/test_feature_detection.py +++ b/riak/tests/test_feature_detection.py @@ -86,5 +86,16 @@ def test_12(self): self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + def test_12_loose(self): + t = DummyTransport("1.2.1p3") + self.assertTrue(t.phaseless_mapred()) + self.assertTrue(t.pb_indexes()) + self.assertTrue(t.pb_search()) + self.assertTrue(t.pb_conditionals()) + self.assertTrue(t.quorum_controls()) + self.assertTrue(t.tombstone_vclocks()) + self.assertTrue(t.pb_head()) + + if __name__ == '__main__': unittest.main() diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 07584472..ae54d193 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -16,14 +16,14 @@ under the License. """ -from distutils.version import StrictVersion +from distutils.version import LooseVersion from riak.util import lazy_property versions = { - 1: StrictVersion("1.0.0"), - 1.1: StrictVersion("1.1.0"), - 1.2: StrictVersion("1.2.0") + 1: LooseVersion("1.0.0"), + 1.1: LooseVersion("1.1.0"), + 1.2: LooseVersion("1.2.0") } @@ -92,4 +92,4 @@ def pb_head(self): @lazy_property def server_version(self): - return StrictVersion(self._server_version()) + return LooseVersion(self._server_version()) From 948fd067bed916c3402b327fa84e9d5b7b7b96d8 Mon Sep 17 00:00:00 2001 From: evan Date: Mon, 17 Dec 2012 11:43:20 -0500 Subject: [PATCH 0257/1060] change exception type --- riak/mapreduce.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 6ac38521..6a13b8af 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -64,9 +64,9 @@ def add_object(self, obj): def add_bucket_key_data(self, bucket, key, data): if self._input_mode == 'bucket': - raise RiakError('Already added a bucket, can\'t add an object.') + raise ValueError('Already added a bucket, can\'t add an object.') elif self._input_mode == 'query': - raise RiakError('Already added a query, can\'t add an object.') + raise ValueError('Already added a query, can\'t add an object.') else: if isinstance(key, Iterable) and \ not isinstance(key, basestring): From e6ed09873027134123d05df4c6e80582cfac8aaf Mon Sep 17 00:00:00 2001 From: evan Date: Tue, 18 Dec 2012 09:53:43 -0500 Subject: [PATCH 0258/1060] riak_object remodel --- riak/bucket.py | 8 +- riak/riak_object.py | 360 +++++++++++++++------------------------- riak/tests/test_all.py | 26 +-- riak/tests/test_kv.py | 91 +++++----- riak/transports/http.py | 24 +-- riak/transports/pbc.py | 24 +-- 6 files changed, 222 insertions(+), 311 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 7e12e51b..0782a2e1 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -125,8 +125,8 @@ def new(self, key=None, data=None, content_type='application/json'): raise TypeError('Unicode data values are not supported.') obj = RiakObject(self._client, self, key) - obj.set_data(data) - obj.set_content_type(content_type) + obj.data = data + obj.content_type = content_type obj._encode_data = True return obj @@ -146,8 +146,8 @@ def new_binary(self, key, data, content_type='application/octet-stream'): :rtype: :class:`RiakObject ` """ obj = RiakObject(self._client, self, key) - obj.set_data(data) - obj.set_content_type(content_type) + obj.data = data + obj.content_type = content_type obj._encode_data = False return obj diff --git a/riak/riak_object.py b/riak/riak_object.py index c1ab33e1..38df4ba9 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -17,7 +17,6 @@ specific language governing permissions and limitations under the License. """ -import types import copy from metadata import * from riak import RiakError @@ -47,75 +46,49 @@ def __init__(self, client, bucket, key=None): except UnicodeError: raise TypeError('Unicode keys are not supported.') - self._client = client - self._bucket = bucket - self._key = key + self.client = client + self.bucket = bucket + self.key = key self._encode_data = True - self._vclock = None self._data = None - self._metadata = {MD_USERMETA: {}, MD_INDEX: []} - self._links = [] - self._siblings = [] - self._exists = False + self.vclock = None + self.metadata = {MD_USERMETA: {}, MD_INDEX: []} + self.links = [] + self.siblings = [] + self.exists = False - def get_bucket(self): - """ - Get the bucket of this object. - - :rtype: RiakBucket - """ - return self._bucket - - def get_key(self): - """ - Get the key of this object. - - :rtype: string - """ - return self._key - - def get_data(self): - """ - Get the data stored in this object. Will return an associative - array, unless the object was constructed with - :func:`RiakBucket.new_binary ` or - :func:`RiakBucket.get_binary `, - in which case this will return a string. - - :rtype: array or string - """ + def _get_data(self): return self._data - def set_data(self, data): - """ - Set the data stored in this object. This data will be - JSON encoded unless the object was constructed with - :func:`RiakBucket.new_binary ` or - :func:`RiakBucket.get_binary `, - in which case it will be stored as a string. - - :param data: The data to store. - :type data: mixed - :rtype: data - """ - self._data = data - if MD_CTYPE not in self._metadata: + def _set_data(self, data, content_type=None): + if MD_CTYPE not in self.metadata: if self._encode_data: - self.set_content_type("application/json") + self.content_type = "application/json" else: - self.set_content_type("application/octet-stream") + self.content_type = "application/octet-stream" + self._data = data return self + data = property(_get_data, _set_data, doc=""" + The data stored in this object. This data will be + JSON encoded on storage unless the object was constructed with + :func:`RiakBucket.new_binary ` or + :func:`RiakBucket.get_binary `, + in which case it will be stored as a string. On return, it shall + either be a dict or a string, depending on its storage type. + + :type mixed """) + def get_encoded_data(self): """ Get the data encoded for storing """ if self._encode_data == True: - content_type = self.get_content_type() - encoder = self._bucket.get_encoder(content_type) + content_type = self.content_type + encoder = self.bucket.get_encoder(content_type) if encoder is None: - if isinstance(self._data, basestring): - return self._data.encode() + if isinstance(self.data, basestring): + return self.data.encode() else: raise RiakError("No encoder for non-string data " "with content type ${0}". @@ -123,7 +96,7 @@ def get_encoded_data(self): else: return encoder(self._data) else: - return self._data + return self.data def set_encoded_data(self, data): """ @@ -131,56 +104,37 @@ def set_encoded_data(self, data): the metadata has been set correctly first. """ if self._encode_data == True: - content_type = self.get_content_type() - decoder = self._bucket.get_decoder(content_type) + content_type = self.content_type + decoder = self.bucket.get_decoder(content_type) if decoder is None: # if no decoder, just set as string data for # application to handle - self._data = data + self.data = data else: - self._data = decoder(data) + self.data = decoder(data) else: - self._data = data - return self - - def get_metadata(self): - """ - Get the metadata stored in this object. Will return an associative - array - - :rtype: dict - """ - return self._metadata - - def set_metadata(self, metadata): - """ - Set the metadata stored in this object. - - :param metadata: The data to store. - :type metadata: dict - :rtype: data - """ - self._metadata = metadata + self.data = data return self - def get_usermeta(self): - if MD_USERMETA in self._metadata: - return self._metadata[MD_USERMETA] + def _get_usermeta(self): + if MD_USERMETA in self.metadata: + return self.metadata[MD_USERMETA] else: return {} - def set_usermeta(self, usermeta): - """ - Sets the custom user metadata on this object. This doesn't + def _set_usermeta(self, usermeta): + self.metadata[MD_USERMETA] = usermeta + return self + + usermeta = property(_get_usermeta, _set_usermeta, + doc=""" + The custom user metadata on this object. This doesn't include things like content type and links, but only user-defined meta attributes stored with the Riak object. :param userdata: The user metadata to store. :type userdata: dict - :rtype: data - """ - self._metadata[MD_USERMETA] = usermeta - return self + """) def add_index(self, field, value): """ @@ -194,8 +148,8 @@ def add_index(self, field, value): :rtype: self """ rie = RiakIndexEntry(field, value) - if not rie in self._metadata[MD_INDEX]: - self._metadata[MD_INDEX].append(rie) + if not rie in self.metadata[MD_INDEX]: + self.metadata[MD_INDEX].append(rie) return self @@ -211,19 +165,19 @@ def remove_index(self, field=None, value=None): :rtype: self """ if not field and not value: - ries = self._metadata[MD_INDEX][:] + ries = self.metadata[MD_INDEX][:] elif field and not value: - ries = [x for x in self._metadata[MD_INDEX] + ries = [x for x in self.metadata[MD_INDEX] if x.get_field() == field] elif field and value: ries = [RiakIndexEntry(field, value)] else: - raise Exception( - "Cannot pass value without a field name while removing index") + raise RiakError("Cannot pass value without a field" + " name while removing index") for rie in ries: - if rie in self._metadata[MD_INDEX]: - self._metadata[MD_INDEX].remove(rie) + if rie in self.metadata[MD_INDEX]: + self.metadata[MD_INDEX].remove(rie) return self remove_indexes = remove_index @@ -241,7 +195,7 @@ def set_indexes(self, indexes): for field, value in indexes: rie = RiakIndexEntry(field, value) new_indexes.append(rie) - self._metadata[MD_INDEX] = new_indexes + self.metadata[MD_INDEX] = new_indexes return self @@ -255,41 +209,21 @@ def get_indexes(self, field=None): :rtype: (array of RiakIndexEntry) or (array of string or integer) """ if field == None: - return self._metadata[MD_INDEX] + return self.metadata[MD_INDEX] else: - return [x.get_value() for x in self._metadata[MD_INDEX] + return [x.get_value() for x in self.metadata[MD_INDEX] if x.get_field() == field] - def exists(self): - """ - Return True if the object exists, False otherwise. Allows you - to detect a :func:`RiakBucket.get - ` or :func:`RiakBucket.get_binary - ` operation where the - object is missing. - - :rtype: boolean - """ - return self._exists - - def get_content_type(self): - """ - Get the content type of this object. This is either - ``application/json``, or the provided content type if the - object was created via :func:`RiakBucket.new_binary - `. - - :rtype: string - """ + def _get_content_type(self): try: - return self._metadata[MD_CTYPE] + return self.metadata[MD_CTYPE] except KeyError: if self._encode_data: return "application/json" else: return "application/octet-stream" - def set_content_type(self, content_type): + def _set_content_type(self, content_type): """ Set the content type of this object. @@ -297,9 +231,18 @@ def set_content_type(self, content_type): :type content_type: string :rtype: self """ - self._metadata[MD_CTYPE] = content_type + self.metadata[MD_CTYPE] = content_type return self + content_type = property(_get_content_type, _set_content_type, + doc=""" + The content type of this object. This is either + ``application/json``, or the provided content type if the + object was created via :func:`RiakBucket.new_binary + `. + + :rtype: string """) + def set_links(self, links, all_link=False): """ Replaces all links to a RiakObject @@ -314,7 +257,7 @@ def set_links(self, links, all_link=False): objects This speeds up the operation. """ if all_link: - self._metadata[MD_LINKS] = links + self.metadata[MD_LINKS] = links return self new_links = [] @@ -322,12 +265,12 @@ def set_links(self, links, all_link=False): if isinstance(item, RiakLink): link = item elif isinstance(item, RiakObject): - link = RiakLink(item._bucket.name, item._key, None) + link = RiakLink(item.bucket.name, item.key, None) else: - link = RiakLink(item[0]._bucket.name, item[0]._key, item[1]) + link = RiakLink(item[0].bucket.name, item[0].key, item[1]) new_links.append(link) - self._metadata[MD_LINKS] = new_links + self.metadata[MD_LINKS] = new_links return self def add_link(self, obj, tag=None): @@ -344,10 +287,10 @@ def add_link(self, obj, tag=None): if isinstance(obj, RiakLink): newlink = obj else: - newlink = RiakLink(obj._bucket.name, obj._key, tag) + newlink = RiakLink(obj.bucket.name, obj.key, tag) self.remove_link(newlink) - links = self._metadata[MD_LINKS] + links = self.metadata[MD_LINKS] links.append(newlink) return self @@ -365,15 +308,15 @@ def remove_link(self, obj, tag=None): if isinstance(obj, RiakLink): oldlink = obj else: - oldlink = RiakLink(obj._bucket.name, obj._key, tag) + oldlink = RiakLink(obj.bucket.name, obj.key, tag) a = [] - links = self._metadata.get(MD_LINKS, []) + links = self.metadata.get(MD_LINKS, []) for link in links: if not link.isEqual(oldlink): a.append(link) - self._metadata[MD_LINKS] = a + self.metadata[MD_LINKS] = a return self def get_links(self): @@ -383,10 +326,10 @@ def get_links(self): :rtype: array() """ # Set the clients before returning... - if MD_LINKS in self._metadata: - links = self._metadata[MD_LINKS] + if MD_LINKS in self.metadata: + links = self.metadata[MD_LINKS] for link in links: - link._client = self._client + link._client = self.client return links else: return [] @@ -415,24 +358,27 @@ def store(self, w=None, dw=None, pw=None, return_body=True, :param if_none_match: Should the object be stored only if there is no key previously defined :type if_none_match: bool - :rtype: self - """ + :rtype: self """ + if self.siblings and not self.data and not self.vclock: + raise RiakError("Attempting to store an invalid object," + "store one of the siblings instead") + # Issue the put over our transport - t = self._client.get_transport() + t = self.client.get_transport() - if self._key is None: + if self.key is None: key, vclock, metadata = t.put_new(self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) - self._exists = True - self._key = key - self._vclock = vclock - self.set_metadata(metadata) + self.exists = True + self.key = key + self.vclock = vclock + self.metadata = metadata else: - Result = t.put(self, w=w, dw=dw, pw=pw, return_body=return_body, + result = t.put(self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) - if Result is not None: - self._populate(Result) + if result is not None and result != ('', []): + self._populate(result) return self @@ -448,12 +394,12 @@ def reload(self, r=None, pr=None, vtag=None): :rtype: self """ - t = self._client.get_transport() - Result = t.get(self, r=r, pr=pr, vtag=vtag) + t = self.client.get_transport() + result = t.get(self, r=r, pr=pr, vtag=vtag) self.clear() - if Result is not None: - self._populate(Result) + if result is not None and result != ('', []): + self._populate(result) return self @@ -483,8 +429,8 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): :type pw: integer :rtype: self """ - t = self._client.get_transport() - Result = t.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) + t = self.client.get_transport() + result = t.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) self.clear() return self @@ -494,22 +440,14 @@ def clear(self): :rtype: self """ - self._headers = [] - self._links = [] - self._data = None - self._exists = False - self._siblings = [] + self.headers = [] + self.links = [] + self.data = None + self.exists = False + self.siblings = [] return self - def vclock(self): - """ - Get the vclock of this object. - - :rtype: string - """ - return self._vclock - - def _populate(self, Result): + def _populate(self, result): """ Populate the object based on the return from get. @@ -520,48 +458,32 @@ def _populate(self, Result): sibling that need to be retrieved with get. """ self.clear() - if Result is None: + if result is None: return self - elif type(Result) == types.ListType: - self._set_siblings(Result) - elif type(Result) == types.TupleType: - (vclock, contents) = Result - self._vclock = vclock + elif type(result) is list: + self._set_siblings(result) + elif type(result) is tuple: + (vclock, contents) = result + self.vclock = vclock if len(contents) > 0: (metadata, data) = contents.pop(0) - self._exists = True + self.exists = True if not MD_INDEX in metadata: metadata[MD_INDEX] = [] - self.set_metadata(metadata) + self.metadata = metadata self.set_encoded_data(data) # Create objects for all siblings siblings = [self] for (metadata, data) in contents: sibling = copy.copy(self) - sibling.set_metadata(metadata) - sibling.set_encoded_data(data) + sibling.metadata = metadata + sibling.data = data siblings.append(sibling) for sibling in siblings: sibling._set_siblings(siblings) else: raise RiakError("do not know how to handle type %s" % type(Result)) - def has_siblings(self): - """ - Return True if this object has siblings. - - :rtype: boolean - """ - return(self.get_sibling_count() > 0) - - def get_sibling_count(self): - """ - Get the number of siblings that this object contains. - - :rtype: integer - """ - return len(self._siblings) - def get_sibling(self, i, r=None, pr=None): """ Retrieve a sibling by sibling number. @@ -573,33 +495,19 @@ def get_sibling(self, i, r=None, pr=None): :type r: integer :rtype: RiakObject. """ - if isinstance(self._siblings[i], RiakObject): - return self._siblings[i] + if isinstance(self.siblings[i], RiakObject): + return self.siblings[i] else: # Run the request... - vtag = self._siblings[i] - obj = RiakObject(self._client, self._bucket, self._key) + vtag = self.siblings[i] + obj = RiakObject(self.client, self.bucket, self.key) obj.reload(r=r, pr=pr, vtag=vtag) - # And make sure it knows who it's siblings are - self._siblings[i] = obj - obj._set_siblings(self._siblings) + # And make sure it knows who its siblings are + self.siblings[i] = obj + obj._set_siblings(self.siblings) return obj - def get_siblings(self, r=None): - """ - Retrieve an array of siblings. - - :param r: R-Value. Wait until this many partitions have - responded before returning to client. - :type r: integer - :rtype: array of RiakObject - """ - a = [] - for i in range(self.get_sibling_count()): - a.append(self.get_sibling(i, r)) - return a - def _set_siblings(self, siblings): """ Set the array of siblings - used internally @@ -618,9 +526,9 @@ def _set_siblings(self, siblings): pass if len(siblings) > 1: - self._siblings = siblings + self.siblings = siblings else: - self._siblings = [] + self.siblings = [] def add(self, *args): """ @@ -629,8 +537,8 @@ def add(self, *args): :rtype: RiakMapReduce """ - mr = RiakMapReduce(self._client) - mr.add(self._bucket.name, self._key) + mr = RiakMapReduce(self.client) + mr.add(self.bucket.name, self.key) return apply(mr.add, args) def link(self, *args): @@ -640,8 +548,8 @@ def link(self, *args): :rtype: RiakMapReduce """ - mr = RiakMapReduce(self._client) - mr.add(self._bucket.name, self._key) + mr = RiakMapReduce(self.client) + mr.add(self.bucket.name, self.key) return apply(mr.link, args) def map(self, *args): @@ -651,8 +559,8 @@ def map(self, *args): :rtype: RiakMapReduce """ - mr = RiakMapReduce(self._client) - mr.add(self._bucket.name, self._key) + mr = RiakMapReduce(self.client) + mr.add(self.bucket.name, self.key) return apply(mr.map, args) def reduce(self, params): @@ -662,8 +570,8 @@ def reduce(self, params): :rtype: RiakMapReduce """ - mr = RiakMapReduce(self._client) - mr.add(self._bucket.name, self._key) + mr = RiakMapReduce(self.client) + mr.add(self.bucket.name, self.key) return apply(mr.reduce, params) from mapreduce import * diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index c5321f9c..5693d797 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -115,10 +115,10 @@ def test_close_underlying_socket_fails(self): obj = bucket.new('foo', rand) obj.store() obj = bucket.get('foo') - self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().name, 'bucket_test_close') - self.assertEqual(obj.get_key(), 'foo') - self.assertEqual(obj.get_data(), rand) + self.assertTrue(obj.exists) + self.assertEqual(obj.bucket.name, 'bucket_test_close') + self.assertEqual(obj.key, 'foo') + self.assertEqual(obj.data, rand) # Close the underlying socket. This gets a bit sketchy, # since we are reaching into the internals, but there is @@ -138,10 +138,10 @@ def test_close_underlying_socket_retry(self): obj = bucket.new('barbaz', rand) obj.store() obj = bucket.get('barbaz') - self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().name, 'bucket_test_close') - self.assertEqual(obj.get_key(), 'barbaz') - self.assertEqual(obj.get_data(), rand) + self.assertTrue(obj.exists) + self.assertEqual(obj.bucket.name, 'bucket_test_close') + self.assertEqual(obj.key, 'barbaz') + self.assertEqual(obj.data, rand) # Close the underlying socket. This gets a bit sketchy, # since we are reaching into the internals, but there is @@ -151,10 +151,10 @@ def test_close_underlying_socket_retry(self): # This should work, since we have a retry obj = bucket.get('barbaz') - self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().name, 'bucket_test_close') - self.assertEqual(obj.get_key(), 'barbaz') - self.assertEqual(obj.get_data(), rand) + self.assertTrue(obj.exists) + self.assertEqual(obj.bucket.name, 'bucket_test_close') + self.assertEqual(obj.key, 'barbaz') + self.assertEqual(obj.data, rand) def test_bucket_search_enabled(self): with self.assertRaises(NotImplementedError): @@ -190,7 +190,7 @@ def setUp(self): def test_no_returnbody(self): bucket = self.client.bucket("bucket") o = bucket.new("foo", "bar").store(return_body=False) - self.assertEqual(o.vclock(), None) + self.assertEqual(o.vclock, None) def test_too_many_link_headers_shouldnt_break_http(self): bucket = self.client.bucket("bucket") diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 8af95bbb..a9e7d24f 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -42,10 +42,10 @@ def test_store_and_get(self): obj = bucket.new('foo', rand) obj.store() obj = bucket.get('foo') - self.assertTrue(obj.exists()) - self.assertEqual(obj.get_bucket().name, 'bucket') - self.assertEqual(obj.get_key(), 'foo') - self.assertEqual(obj.get_data(), rand) + self.assertTrue(obj.exists) + self.assertEqual(obj.bucket.name, 'bucket') + self.assertEqual(obj.key, 'foo') + self.assertEqual(obj.data, rand) # unicode objects are fine, as long as they don't # contain any non-ASCII chars @@ -68,11 +68,11 @@ def test_generate_key(self): bucket = self.client.bucket('random_key_bucket') existing_keys = bucket.get_keys() o = bucket.new(None, data={}) - self.assertIsNone(o.get_key()) + self.assertIsNone(o.key) o.store() - self.assertIsNotNone(o.get_key()) - self.assertNotIn('/', o.get_key()) - self.assertNotIn(o.get_key(), existing_keys) + self.assertIsNotNone(o.key) + self.assertNotIn('/', o.key) + self.assertNotIn(o.key, existing_keys) self.assertEqual(len(bucket.get_keys()), len(existing_keys) + 1) def test_binary_store_and_get(self): @@ -82,14 +82,14 @@ def test_binary_store_and_get(self): obj = bucket.new_binary('foo1', rand) obj.store() obj = bucket.get_binary('foo1') - self.assertTrue(obj.exists()) - self.assertEqual(obj.get_data(), rand) + self.assertTrue(obj.exists) + self.assertEqual(obj.data, rand) # Store as JSON, retrieve as binary, JSON-decode, then compare... data = [self.randint(), self.randint(), self.randint()] obj = bucket.new('foo2', data) obj.store() obj = bucket.get_binary('foo2') - self.assertEqual(data, json.loads(obj.get_data())) + self.assertEqual(data, json.loads(obj.data)) def test_custom_bucket_encoder_decoder(self): # Teach the bucket how to pickle @@ -97,10 +97,10 @@ def test_custom_bucket_encoder_decoder(self): bucket.set_encoder('application/x-pickle', cPickle.dumps) bucket.set_decoder('application/x-pickle', cPickle.loads) data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} - obj = bucket.new("foo", data, 'application/x-pickle').store() + obj = bucket.new("foo", data, 'application/x-pickle') obj.store() obj2 = bucket.get("foo") - self.assertEqual(data, obj2.get_data()) + self.assertEqual(data, obj2.data) def test_custom_client_encoder_decoder(self): # Teach the bucket how to pickle @@ -108,10 +108,10 @@ def test_custom_client_encoder_decoder(self): self.client.set_encoder('application/x-pickle', cPickle.dumps) self.client.set_decoder('application/x-pickle', cPickle.loads) data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} - obj = bucket.new("foo", data, 'application/x-pickle').store() + obj = bucket.new("foo", data, 'application/x-pickle') obj.store() obj2 = bucket.get("foo") - self.assertEqual(data, obj2.get_data()) + self.assertEqual(data, obj2.data) def test_unknown_content_type_encoder_decoder(self): # Teach the bucket how to pickle @@ -120,13 +120,13 @@ def test_unknown_content_type_encoder_decoder(self): obj = bucket.new("foo", data, 'application/x-frobnicator').store() obj.store() obj2 = bucket.get("foo") - self.assertEqual(data, obj2.get_data()) + self.assertEqual(data, obj2.data) def test_missing_object(self): bucket = self.client.bucket('bucket') obj = bucket.get("missing") - self.assertFalse(obj.exists()) - self.assertEqual(obj.get_data(), None) + self.assertFalse(obj.exists) + self.assertEqual(obj.data, None) def test_delete(self): bucket = self.client.bucket('bucket') @@ -134,10 +134,10 @@ def test_delete(self): obj = bucket.new('foo', rand) obj.store() obj = bucket.get('foo') - self.assertTrue(obj.exists()) + self.assertTrue(obj.exists) obj.delete() obj.reload() - self.assertFalse(obj.exists()) + self.assertFalse(obj.exists) def test_set_bucket_properties(self): bucket = self.client.bucket('bucket') @@ -158,11 +158,11 @@ def test_if_none_match(self): obj.delete() obj.reload() - self.assertFalse(obj.exists()) - obj.set_data(["first store"]) + self.assertFalse(obj.exists) + obj.data = ["first store"] obj.store() - obj.set_data(["second store"]) + obj.data = ["second store"] with self.assertRaises(Exception): obj.store(if_none_match=True) @@ -173,9 +173,11 @@ def test_siblings(self): obj = bucket.get_binary('foo') # Even if it previously existed, let's store a base resolved version # from which we can diverge by sending a stale vclock. - obj.set_data('start') + obj.data = 'start' obj.store() + print obj.vclock + # Store the same object five times... vals = set() for i in range(5): @@ -187,19 +189,20 @@ def test_siblings(self): break other_obj = other_bucket.new_binary('foo', str(randval)) - other_obj._vclock = obj._vclock + other_obj.vclock = obj.vclock other_obj.store() vals.add(str(randval)) # Make sure the object has itself plus four siblings... obj.reload() - self.assertTrue(obj.has_siblings()) - self.assertEqual(obj.get_sibling_count(), 5) + print obj.siblings + self.assertTrue(bool(obj.siblings)) + self.assertEqual(len(obj.siblings), 5) # Get each of the values - make sure they match what was assigned vals2 = set() for i in range(5): - vals2.add(obj.get_sibling(i).get_data()) + vals2.add(obj.get_sibling(i).data) self.assertEqual(vals, vals2) # Resolve the conflict, and then do a get... @@ -207,36 +210,36 @@ def test_siblings(self): obj3.store() obj.reload() - self.assertEqual(obj.get_sibling_count(), 0) - self.assertEqual(obj.get_data(), obj3.get_data()) + self.assertEqual(len(obj.siblings), 0) + self.assertEqual(obj.data, obj3.data) def test_store_of_missing_object(self): bucket = self.client.bucket("bucket") # for json objects o = bucket.get("nonexistent_key_json") - self.assertEqual(o.exists(), False) - o.set_data({"foo": "bar"}) + self.assertEqual(o.exists, False) + o.data = {"foo": "bar"} o = o.store() - self.assertEqual(o.get_data(), {"foo": "bar"}) - self.assertEqual(o.get_content_type(), "application/json") + self.assertEqual(o.data, {"foo": "bar"}) + self.assertEqual(o.content_type, "application/json") o.delete() # for binary objects o = bucket.get_binary("nonexistent_key_binary") - self.assertEqual(o.exists(), False) - o.set_data("1234567890") + self.assertEqual(o.exists, False) + o.data = "1234567890" o = o.store() - self.assertEqual(o.get_data(), "1234567890") - self.assertEqual(o.get_content_type(), "application/octet-stream") + self.assertEqual(o.data, "1234567890") + self.assertEqual(o.content_type, "application/octet-stream") o.delete() def test_store_metadata(self): bucket = self.client.bucket('bucket') rand = self.randint() obj = bucket.new('fooster', rand) - obj.set_usermeta({'custom': 'some metadata'}) + obj.usermeta = {'custom': 'some metadata'} obj.store() obj = bucket.get('fooster') - self.assertEqual('some metadata', obj.get_usermeta()['custom']) + self.assertEqual('some metadata', obj.usermeta['custom']) def test_list_buckets(self): bucket = self.client.bucket("list_bucket") @@ -326,8 +329,8 @@ def test_store_binary_object_from_file(self): obj = bucket.new_binary_from_file('foo_from_file', filepath) obj.store() obj = bucket.get_binary('foo_from_file') - self.assertNotEqual(obj.get_data(), None) - self.assertEqual(obj.get_content_type(), "text/x-python") + self.assertNotEqual(obj.data, None) + self.assertEqual(obj.content_type, "text/x-python") def test_store_binary_object_from_file_should_use_default_mimetype(self): bucket = self.client.bucket('bucket') @@ -337,7 +340,7 @@ def test_store_binary_object_from_file_should_use_default_mimetype(self): obj = bucket.new_binary_from_file('foo_from_file', filepath) obj.store() obj = bucket.get_binary('foo_from_file') - self.assertEqual(obj.get_content_type(), 'application/octet-stream') + self.assertEqual(obj.content_type, 'application/octet-stream') def test_store_binary_object_from_file_should_fail_if_file_not_found(self): bucket = self.client.bucket('bucket') @@ -345,4 +348,4 @@ def test_store_binary_object_from_file_should_fail_if_file_not_found(self): self.assertRaises(IOError, bucket.new_binary_from_file, 'not_found_from_file', 'FILE_NOT_FOUND') obj = bucket.get_binary('not_found_from_file') - self.assertEqual(obj.get_data(), None) + self.assertEqual(obj.data, None) diff --git a/riak/transports/http.py b/riak/transports/http.py index 8fc2757d..58ac88b4 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -146,7 +146,7 @@ def get(self, robj, r=None, pr=None, vtag=None): params = {'r': r, 'pr': pr} if vtag is not None: params['vtag'] = vtag - url = self.build_rest_path(robj.get_bucket(), robj.get_key(), + url = self.build_rest_path(robj.bucket, robj.key, params=params) response = self.http_request('GET', url) return self.parse_body(response, [200, 300, 404]) @@ -160,8 +160,8 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, # unknown flags/params. params = {'returnbody': str(return_body).lower(), 'w': w, 'dw': dw, 'pw': pw} - url = self.build_rest_path(bucket=robj.get_bucket(), - key=robj.get_key(), + url = self.build_rest_path(bucket=robj.bucket, + key=robj.key, params=params) headers = self.build_put_headers(robj) # TODO: use a more general 'prevent_stale_writes' semantics, @@ -170,7 +170,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, headers["If-None-Match"] = "*" content = robj.get_encoded_data() return self.do_put(url, headers, content, return_body, - key=robj.get_key()) + key=robj.key) def do_put(self, url, headers, content, return_body=False, key=None): if key is None: @@ -191,7 +191,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, # unknown flags/params. params = {'returnbody': str(return_body).lower(), 'w': w, 'dw': dw, 'pw': pw} - url = self.build_rest_path(bucket=robj.get_bucket(), params=params) + url = self.build_rest_path(bucket=robj.bucket, params=params) headers = self.build_put_headers(robj) # TODO: use a more general 'prevent_stale_writes' semantics, # which is a superset of the if_none_match semantics. @@ -217,10 +217,10 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): # unknown flags/params. params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} headers = {} - url = self.build_rest_path(robj.get_bucket(), robj.get_key(), + url = self.build_rest_path(robj.bucket, robj.key, params=params) - if self.tombstone_vclocks() and robj.vclock() is not None: - headers['X-Riak-Vclock'] = robj.vclock() + if self.tombstone_vclocks() and robj.vclock is not None: + headers['X-Riak-Vclock'] = robj.vclock response = self.http_request('DELETE', url, headers) self.check_http_code(response, [204, 404]) return self @@ -536,17 +536,17 @@ def build_put_headers(self, robj): # Construct the headers... headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', - 'Content-Type': robj.get_content_type(), + 'Content-Type': robj.content_type, 'X-Riak-ClientId': self._client_id}) # Add the vclock if it exists... - if robj.vclock() is not None: - headers['X-Riak-Vclock'] = robj.vclock() + if robj.vclock is not None: + headers['X-Riak-Vclock'] = robj.vclock # Create the header from metadata links = self.add_links_for_riak_object(robj, headers) - for key, value in robj.get_usermeta().iteritems(): + for key, value in robj.usermeta.iteritems(): headers['X-Riak-Meta-%s' % key] = value for rie in robj.get_indexes(): diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index 1b525505..cfef37dd 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -243,7 +243,7 @@ def get(self, robj, r=None, pr=None, vtag=None): if vtag is not None: raise RiakError("PB transport does not support vtags") - bucket = robj.get_bucket() + bucket = robj.bucket req = riak_pb.RpbGetReq() if r: @@ -255,7 +255,7 @@ def get(self, robj, r=None, pr=None, vtag=None): req.deletedvclock = 1 req.bucket = bucket.name - req.key = robj.get_key() + req.key = robj.key # An expected response code of None implies "any response is valid". msg_code, resp = self.send_msg(MSG_CODE_GET_REQ, req, None) @@ -272,7 +272,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, """ Serialize get request and deserialize response """ - bucket = robj.get_bucket() + bucket = robj.bucket req = riak_pb.RpbPutReq() if w: @@ -288,12 +288,12 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, req.if_none_match = 1 req.bucket = bucket.name - req.key = robj.get_key() - vclock = robj.vclock() + req.key = robj.key + vclock = robj.vclock if vclock: req.vclock = vclock - self.pbify_content(robj.get_metadata(), + self.pbify_content(robj.metadata, robj.get_encoded_data(), req.content) @@ -315,7 +315,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, @return (key, vclock, metadata) """ # Note that this won't work on 0.14 nodes. - bucket = robj.get_bucket() + bucket = robj.bucket req = riak_pb.RpbPutReq() if w: @@ -332,7 +332,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, req.bucket = bucket.name - self.pbify_content(robj.get_metadata(), + self.pbify_content(robj.metadata, robj.get_encoded_data(), req.content) @@ -350,7 +350,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Serialize get request and deserialize response """ - bucket = robj.get_bucket() + bucket = robj.bucket req = riak_pb.RpbDelReq() if rw: @@ -368,11 +368,11 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): if pw: req.pw = self.translate_rw_val(pw) - if self.tombstone_vclocks() and robj.vclock(): - req.vclock = robj.vclock() + if self.tombstone_vclocks() and robj.vclock: + req.vclock = robj.vclock req.bucket = bucket.name - req.key = robj.get_key() + req.key = robj.key msg_code, resp = self.send_msg(MSG_CODE_DEL_REQ, req, MSG_CODE_DEL_RESP) From 8cc2bc44b3ca332cd42c3ecd9f98a5db55a82d97 Mon Sep 17 00:00:00 2001 From: evan Date: Tue, 18 Dec 2012 16:10:17 -0500 Subject: [PATCH 0259/1060] add two additional cases and add tests for all execption raising cases --- riak/mapreduce.py | 4 ++-- riak/tests/test_mapreduce.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 6a13b8af..7279b0a0 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -83,14 +83,14 @@ def add_bucket(self, bucket): def add_key_filters(self, key_filters): if self._input_mode == 'query': - raise Exception('Key filters are not supported in a query.') + raise ValueError('Key filters are not supported in a query.') self._key_filters.extend(key_filters) return self def add_key_filter(self, *args): if self._input_mode == 'query': - raise Exception('Key filters are not supported in a query.') + raise ValueError('Key filters are not supported in a query.') self._key_filters.append(args) return self diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 5cbb4a85..23df62e2 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -83,6 +83,27 @@ def test_erlang_map_reduce(self): .run() self.assertEqual(len(result), 2) + def test_client_exceptional_paths(self): + bucket = self.client.bucket('bucket') + bucket.new("foo", 2).store() + bucket.new("bar", 2).store() + bucket.new("baz", 4).store() + + #adding a b-key pair to a bucket input + with self.assertRaises(ValueError): + mr = self.client.add('bucket') + mr.add('bucket', 'bar') + + #adding a b-key pair to a query input + with self.assertRaises(ValueError): + mr = self.client.search('bucket', 'fleh') + mr.add('bucket', 'bar') + + #adding a key filter to a query input + with self.assertRaises(ValueError): + mr = self.client.search('bucket', 'fleh') + mr.add_key_filter("tokenize", "-", 1) + class JSMapReduceTests(object): def test_javascript_source_map(self): From de752f13d69f6a6955c38136fc3cc1b48f643246 Mon Sep 17 00:00:00 2001 From: evan Date: Wed, 19 Dec 2012 12:05:38 -0500 Subject: [PATCH 0260/1060] remove some debugging statements accidentally left in. --- riak/tests/test_kv.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index a9e7d24f..b477f944 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -176,8 +176,6 @@ def test_siblings(self): obj.data = 'start' obj.store() - print obj.vclock - # Store the same object five times... vals = set() for i in range(5): @@ -195,7 +193,6 @@ def test_siblings(self): # Make sure the object has itself plus four siblings... obj.reload() - print obj.siblings self.assertTrue(bool(obj.siblings)) self.assertEqual(len(obj.siblings), 5) From 29d624a1f2217144ff115ad86af8d6a798b4cec0 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 10:40:43 -0500 Subject: [PATCH 0261/1060] WIP transport refactor. Things are broken, do not use. * Connection manager and monitor are gone. * PBC transport is broken up into multiple files for clarity. * Outline of streaming operations (currently only PBC). * All requests go through the RiakClient object, which handles the connection pools. Accessing the transport directly is now forbidden. * The client can connect to multiple nodes, and each node has an associated error rate (to be used later when implementing error recovery). The node selected for new connections depends on the error rate. TODO: * Break up HTTP into smaller files and add resource detection (for URL generation). * Add streaming operations to HTTP. --- riak/bucket.py | 21 +- riak/client.py | 263 +++++------ riak/client/operations.py | 155 +++++++ riak/mapreduce.py | 135 ++++-- riak/node.py | 87 ++++ riak/riak_object.py | 33 +- riak/search.py | 71 +-- riak/tests/test_all.py | 38 +- riak/tests/test_mapreduce.py | 2 +- riak/transports/connection.py | 192 -------- riak/transports/http.py | 121 +++-- riak/transports/monitor.py | 47 -- riak/transports/pbc.py | 744 ------------------------------ riak/transports/pbc/__init__.py | 43 ++ riak/transports/pbc/codec.py | 143 ++++++ riak/transports/pbc/connection.py | 128 +++++ riak/transports/pbc/messages.py | 48 ++ riak/transports/pbc/stream.py | 86 ++++ riak/transports/pbc/transport.py | 401 ++++++++++++++++ riak/transports/transport.py | 25 +- 20 files changed, 1477 insertions(+), 1306 deletions(-) create mode 100644 riak/client/operations.py create mode 100644 riak/node.py delete mode 100644 riak/transports/connection.py delete mode 100644 riak/transports/monitor.py delete mode 100644 riak/transports/pbc.py create mode 100644 riak/transports/pbc/__init__.py create mode 100644 riak/transports/pbc/codec.py create mode 100644 riak/transports/pbc/connection.py create mode 100644 riak/transports/pbc/messages.py create mode 100644 riak/transports/pbc/stream.py create mode 100644 riak/transports/pbc/transport.py diff --git a/riak/bucket.py b/riak/bucket.py index 0782a2e1..eeed19fa 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -303,8 +303,7 @@ def set_properties(self, props): :param props: An associative array of key:value. :type props: array """ - t = self._client.get_transport() - t.set_bucket_props(self, props) + self._client.set_bucket_props(self, props) def get_properties(self): """ @@ -312,8 +311,7 @@ def get_properties(self): :rtype: array """ - t = self._client.get_transport() - return t.get_bucket_props(self) + return self._client.get_bucket_props(self) def get_keys(self): """ @@ -323,7 +321,17 @@ def get_keys(self): At current, this is a very expensive operation. Use with caution. """ - return self._client.get_transport().get_keys(self) + return self._client.get_keys(self) + + def stream_keys(self): + """ + Return all keys within the bucket. + + .. warning:: + + At current, this is a very expensive operation. Use with caution. + """ + return self._client.stream_keys(self) def new_binary_from_file(self, key, filename): """ @@ -377,5 +385,4 @@ def get_index(self, index, startkey, endkey=None): """ Queries a secondary index over objects in this bucket, returning keys. """ - return self._client._transport.get_index(self.name, index, startkey, - endkey) + return self._client.get_index(self.name, index, startkey, endkey) diff --git a/riak/client.py b/riak/client.py index 483445a0..26d2430b 100644 --- a/riak/client.py +++ b/riak/client.py @@ -1,4 +1,5 @@ """ +Copyright 2011 Basho Technologies, Inc. Copyright 2010 Rusty Klophaus Copyright 2010 Justin Sheehy Copyright 2009 Jay Baird @@ -17,120 +18,140 @@ specific language governing permissions and limitations under the License. """ -# Use json as first choice, simplejson as second choice. + try: import json except ImportError: import simplejson as json +from contextlib import contextmanager + +from riak.client.operations import RiakClientOperations +from riak.node import RiakNode from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduce +from riak.mapreduce import RiakMapReduceChain from riak.search import RiakSearch -from riak.transports import RiakHttpTransport +from riak.transports.http import RiakHttpPool +from riak.transports.pbc import RiakPbcPool from riak.util import deprecated from riak.util import deprecateQuorumAccessors +from riak.util import lazy_property @deprecateQuorumAccessors -class RiakClient(object): +class RiakClient(RiakMapReduceChain, RiakClientOperations): """ - The ``RiakClient`` object holds information necessary to connect to - Riak. The Riak API uses HTTP, so there is no persistent - connection, and the ``RiakClient`` object is extremely lightweight. + The ``RiakClient`` object holds information necessary to connect + to Riak. Requests can be made to Riak directly through the client + or by using the methods on related objects. """ - def __init__(self, host='127.0.0.1', port=8098, prefix='riak', - mapred_prefix='mapred', transport_class=None, - client_id=None, solr_transport_class=None, - transport_options=None): + + PROTOCOLS = ['http', 'https', 'pbc'] + + def __init__(self, protocol='http', transport_options={}, + nodes=None, **unused_args): """ Construct a new ``RiakClient`` object. - :param host: Hostname or IP address - :type host: string - :param port: Port number - :type port: integer - :param prefix: Interface prefix - :type prefix: string - :param mapred_prefix: MapReduce prefix - :type mapred_prefix: string - :param transport_class: transport class to use - :type transport_class: :class:`RiakTransport` - - :param solr_transport_class: HTTP-based transport class for - Solr interface queries - :type solr_transport_class: :class:`RiakHttpTransport` + :param protocol: the preferred protocol, defaults to 'http' + :type protocol: string + :param nodes: a list of node configurations, + where each configuration is a dict containing the keys + 'host', 'http_port', and 'pb_port' + :type nodes: list :param transport_options: Optional key-value args to pass to the transport constuctor :type transport_options: dict """ - if transport_class is None: - transport_class = RiakHttpTransport - - api = getattr(transport_class, 'api', 1) - if api >= 2: - hostports = [(host, port), ] - self._cm = transport_class.default_cm(hostports) - - # If no transport options are provided, then default to the - # empty dict, otherwise just pass through what we are provided. - if transport_options is None: - transport_options = {} - - self._transport = transport_class(self._cm, - prefix=prefix, - mapred_prefix=mapred_prefix, - client_id=client_id, - **transport_options) + if 'port' in unused_args: + deprecated("port option is deprecated, use http_port or pb_port," + + " or the nodes option") + + if 'transport_class' in unused_args: + deprecated( + "transport_class is deprecated, use the protocol option") + + if nodes is None: + self.nodes = [self._create_node(unused_args), ] else: - deprecated('please upgrade the transport to the new API') - self._cm = None - self._transport = transport_class(host, port, client_id=client_id) + self.nodes = [self._create_node(n) for n in nodes] + + self.protocol = protocol or 'http' + + self._http_pool = RiakHttpPool(self, **transport_options) + self._pb_pool = RiakPbcPool(self, **transport_options) self._encoders = {'application/json': json.dumps, 'text/json': json.dumps} self._decoders = {'application/json': json.loads, 'text/json': json.loads} - self._solr = None - self._host = host - self._port = port + self._buckets = {} + + @property + def protocol(self): + return self._protocol + + @property.setter + def protocol(self, value): + if protocol and protocol not in self.PROTOCOLS: + raise ValueError("protocol option is invalid, must be one of %s" % + repr(self.PROTOCOLS)) + self._protocol = value def get_transport(self): """ Get the transport instance the client is using for it's connection. """ - return self._transport + deprecated("get_transport is deprecated, use client, " + + "bucket, or object methods instead") + return None def get_client_id(self): """ Get the ``client_id`` for this ``RiakClient`` instance. + DEPRECATED :rtype: string """ - return self._transport.get_client_id() + deprecated( + "``get_client_id`` is deprecated, use the ``client_id`` property") + return self.client_id def set_client_id(self, client_id): """ Set the client_id for this ``RiakClient`` instance. - .. warning:: - - Refer to - http://wiki.basho.com/Client-Implementation-Guide.html#Client-IDs - for information on how to set the client_id. - :param client_id: The new client_id. :type client_id: string - :rtype: self """ - self._transport.set_client_id(client_id) + deprecated( + "``set_client_id`` is deprecated, use the ``client_id`` property") + self.client_id = client_id return self + @property + def client_id(self): + """ + The client ID for this client instance + + :rtype: string + """ + with self.transport() as transport: + return transport.get_client_id() + + @client_id.setter + def client_id(self, client_id): + for http in self._http_pool: + http.client_id = client_id + for pb in self._pb_pool: + pb.client_id = client_id + def get_encoder(self, content_type): """ Get the encoding function for the provided content type. """ - if content_type in self._encoders: - return self._encoders[content_type] + return self._encoders.get(content_type) def set_encoder(self, content_type, encoder): """ @@ -140,14 +161,12 @@ def set_encoder(self, content_type, encoder): :type encoder: function """ self._encoders[content_type] = encoder - return self def get_decoder(self, content_type): """ Get the decoding function for the provided content type. """ - if content_type in self._decoders: - return self._decoders[content_type] + return self._decoders.get(content_type) def set_decoder(self, content_type, decoder): """ @@ -157,15 +176,6 @@ def set_decoder(self, content_type, decoder): :type decoder: function """ self._decoders[content_type] = decoder - return self - - def get_buckets(self): - """ - Get the list of buckets. - NOTE: Do not use this in production, as it requires traversing through - all keys stored in a cluster. - """ - return self._transport.get_buckets() def bucket(self, name): """ @@ -174,83 +184,48 @@ def bucket(self, name): :rtype: :class:`RiakBucket ` """ - return RiakBucket(self, name) - - def is_alive(self): - """ - Check if the Riak server for this ``RiakClient`` instance is alive. - - :rtype: boolean - """ - return self._transport.ping() - - def add(self, *args): - """ - Start assembling a Map/Reduce operation. A shortcut for - :func:`RiakMapReduce.add`. - - :rtype: :class:`RiakMapReduce` - """ - mr = RiakMapReduce(self) - return apply(mr.add, args) - - def search(self, *args): - """ - Start assembling a Map/Reduce operation based on search - results. This command will return an error unless executed - against a Riak Search cluster. A shortcut for - :func:`RiakMapReduce.search`. - - :rtype: :class:`RiakMapReduce` - """ - mr = RiakMapReduce(self) - return apply(mr.search, args) - - def index(self, *args): - """ - Start assembling a Map/Reduce operation based on secondary - index query results. - - :rtype: :class:`RiakMapReduce` - """ - mr = RiakMapReduce(self) - return apply(mr.index, args) - - def link(self, *args): - """ - Start assembling a Map/Reduce operation. A shortcut for - :func:`RiakMapReduce.link`. + if name in self._buckets: + return self._buckets[name] + else: + bucket = RiakBucket(self, name) + self._buckets[name] = bucket + return bucket - :rtype: :class:`RiakMapReduce` + @lazy_property + def solr(self): """ - mr = RiakMapReduce(self) - return apply(mr.link, args) - - def map(self, *args): + Returns a RiakSearch object which can access search indexes. """ - Start assembling a Map/Reduce operation. A shortcut for - :func:`RiakMapReduce.map`. + return RiakSearch(self) - :rtype: :class:`RiakMapReduce` - """ - mr = RiakMapReduce(self) - return apply(mr.map, args) + def _create_node(self, n): + if isinstance(n, RiakNode): + return n + elif isinstance(n, tuple) and len(n) is 3: + host, http_port, pb_port = n + return RiakNode(host=host, + http_port=http_port, + pb_port=pb_port) + elif isinstance(n, dict): + return RiakNode(**n) + else: + raise TypeError("%s is not a valid node configuration" + % repr(n)) - def reduce(self, *args): + def _choose_node(self, nodes=self.nodes): """ - Start assembling a Map/Reduce operation. A shortcut for - :func:`RiakMapReduce.reduce`. - - :rtype: :class:`RiakMapReduce` + Chooses a random node from the list of nodes in the client, + taking into account each node's recent error rate. + :rtype RiakNode """ - mr = RiakMapReduce(self) - return apply(mr.reduce, args) - - def get_index(self, bucket, index, startkey, endkey=None): - return self._transport.get_index(bucket, index, startkey, endkey) + # Prefer nodes which have gone a reasonable time without + # errors + def _error_rate(node): + return node.error_rate.value() + good = [n for n in nodes if _error_rate(n) < 0.1] - def solr(self): - if self._solr is None: - self._solr = RiakSearch(self, host=self._host, port=self._port) - - return self._solr + if len(good) is 0: + # Fall back to a minimally broken node + return min(nodes, key=_error_rate) + else: + return random.choice(good) diff --git a/riak/client/operations.py b/riak/client/operations.py new file mode 100644 index 00000000..f7ede8a6 --- /dev/null +++ b/riak/client/operations.py @@ -0,0 +1,155 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from riak.bucket import RiakBucket + + +class RiakClientOperations(object): + """ + Methods for RiakClient that result in requests sent to the Riak + cluster. + """ + + def get_buckets(self): + """ + Get the list of buckets as RiakBucket instances. + NOTE: Do not use this in production, as it requires traversing through + all keys stored in a cluster. + """ + with self.transport() as transport: + return [RiakBucket(self, name) for name in transport.get_buckets()] + + def ping(self): + """ + Check if the Riak server for this ``RiakClient`` instance is alive. + + :rtype: boolean + """ + with self.transport() as transport: + return transport.ping() + + is_alive = ping + + def get_index(self, bucket, index, startkey, endkey=None): + """ + Queries a secondary index, returning matching keys. + """ + with self.transport() as transport: + return transport.get_index(bucket, index, startkey, endkey) + + def get_bucket_props(self, bucket): + """ + Fetches bucket properties for the given bucket. + """ + with self.transport() as transport: + return transport.get_bucket_props(bucket) + + def set_bucket_props(self, bucket, props): + """ + Sets bucket properties for the given bucket. + """ + with self.transport() as transport: + return transport.set_bucket_props(bucket, props) + + def get_keys(self, bucket): + """ + Lists all keys in a bucket. + """ + with self.transport() as transport: + return transport.get_keys(bucket) + + def stream_keys(self, bucket): + """ + Lists all keys in a bucket via a stream. This is a generator + method which should be iterated over. + """ + with self.transport() as transport: + for keylist in return transport.stream_keys(bucket): + yield keylist + + def put(self, robj, w=None, dw=None, pw=None, return_body=None, + if_none_match=None): + """ + Stores an object in the Riak cluster. + """ + with self.transport() as transport: + return transport.put(robj, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) + + def put_new(self, robj, w=None, dw=None, pw=None, return_body=None, + if_none_match=None): + """ + Stores an object in the Riak cluster with a generated key. + """ + with self.transport() as transport: + return transport.put_new(robj, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) + + def get(self, robj, r=None, pr=None, vtag=None): + """ + Fetches the contents of a Riak object. + """ + with self.transport() as transport: + return transport.get(robj, r=r, pr=pr, vtag=vtag) + + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + """ + Deletes an object from Riak. + """ + with self.transport() as transport: + return transport.delete(robj, rw=rw, r=r, w=w, dw=dw=, pr=pr, + pw=pw) + + def mapred(self, inputs, query, timeout): + """ + Executes a MapReduce query + """ + with self.transport() as transport: + return transport.mapred(inputs, query, timeout) + + def stream_mapred(self, inputs, query, timeout): + """ + Streams a MapReduce query as (phase, data) pairs. This is a + generator method which should be iterated over. + """ + with self.transport() as transport: + for phase, data in transport.stream_mapred(inputs, query, timeout): + yield phase, data + + def fulltext_search(self, index, query, **params): + """ + Performs a full-text search query. + """ + with self.transport() as transport: + return transport.search(index, query, **params) + + def fulltext_add(self, index, docs): + """ + Adds documents to the full-text index. + """ + with self._http_pool.take() as transport: + transport.fulltext_add(self, index, docs) + + def fulltext_delete(self, index, docs=None, queries=None): + """ + Removes documents from the full-text index. + """ + with self._http_pool.take() as transport: + transport.fulltext_delete(index, docs, queries) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 7279b0a0..6db573be 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -202,15 +202,45 @@ def run(self, timeout=None): @param integer timeout - Timeout in milliseconds. @return array() """ + query, link_results_flag = self._normalize_query() + + result = self.client.mapred(self._inputs, query, timeout) + + # If the last phase is NOT a link phase, then return the result. + if not (link_results_flag + or isinstance(self._phases[-1], RiakLinkPhase)): + return result + + # If there are no results, then return an empty list. + if result == None: + return [] + + # Otherwise, if the last phase IS a link phase, then convert the + # results to RiakLink objects. + a = [] + for r in result: + if (len(r) == 2): + link = RiakLink(r[0], r[1]) + elif (len(r) == 3): + link = RiakLink(r[0], r[1], r[2]) + link._client = self._client + a.append(link) + + return a + + def stream(self, timeout=None): + """ + Streams the MapReduce query (returns an iterator). + """ + query, lrf = self._normalize_query() + return self.client.stream_mapred(self._inputs, query, timeout) + + def _normalize_query(self): num_phases = len(self._phases) - # If there are no phases, then just echo the inputs back to the user. - if (num_phases == 0): - self.reduce(["riak_kv_mapreduce", "reduce_identity"]) - num_phases = 1 + # If there are no phases, return the keys as links + if num_phases is 0: link_results_flag = True - else: - link_results_flag = False # Convert all phases to associative arrays. Also, # if none of the phases are accumulating, then set the last one to @@ -236,30 +266,7 @@ def run(self, timeout=None): self._inputs = {'bucket': bucket_name, 'key_filters': self._key_filters} - t = self._client.get_transport() - result = t.mapred(self._inputs, query, timeout) - - # If the last phase is NOT a link phase, then return the result. - if not (link_results_flag - or isinstance(self._phases[-1], RiakLinkPhase)): - return result - - # If there are no results, then return an empty list. - if result == None: - return [] - - # Otherwise, if the last phase IS a link phase, then convert the - # results to RiakLink objects. - a = [] - for r in result: - if (len(r) == 2): - link = RiakLink(r[0], r[1]) - elif (len(r) == 3): - link = RiakLink(r[0], r[1], r[2]) - link._client = self._client - a.append(link) - - return a + return query, link_results_flag ## # Start Shortcuts to built-ins @@ -546,3 +553,71 @@ def function(*args): def __iter__(self): return iter(self._filters) + + +class RiakMapReduceChain(object): + """ + Mixin to add chaining from the client object directly into a + MapReduce operation. + """ + def add(self, *args): + """ + Start assembling a Map/Reduce operation. A shortcut for + :func:`RiakMapReduce.add`. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.add, args) + + def search(self, *args): + """ + Start assembling a Map/Reduce operation based on search + results. This command will return an error unless executed + against a Riak Search cluster. A shortcut for + :func:`RiakMapReduce.search`. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.search, args) + + def index(self, *args): + """ + Start assembling a Map/Reduce operation based on secondary + index query results. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.index, args) + + def link(self, *args): + """ + Start assembling a Map/Reduce operation. A shortcut for + :func:`RiakMapReduce.link`. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.link, args) + + def map(self, *args): + """ + Start assembling a Map/Reduce operation. A shortcut for + :func:`RiakMapReduce.map`. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.map, args) + + def reduce(self, *args): + """ + Start assembling a Map/Reduce operation. A shortcut for + :func:`RiakMapReduce.reduce`. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.reduce, args) diff --git a/riak/node.py b/riak/node.py new file mode 100644 index 00000000..e9307f6a --- /dev/null +++ b/riak/node.py @@ -0,0 +1,87 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" +import math +import time +from threading import RLock +from riak.util import deprecated + + +class Decaying(object): + """ + A float value which decays exponentially toward 0 over time. This + is used internally to select nodes for new connections that have + had the least errors within the recent period. + """ + + def __init__(self, p=0.0, e=math.e, r=None): + """ + Creates a new decaying error counter. + + :param p: the initial value (defaults to 0.0) + :type p: float + :param e: the exponent base (defaults to math.e) + :type e: float + :param r: timescale factor (defaults to decaying 50% over 10 + seconds, i.e. log(0.5) / 10) + :type r: float + """ + self.p = p + self.e = e + self.r = r or (math.log(0.5) / 10) + self.lock = RLock() + self.t0 = time.time() + + def incr(self, d): + """ + Increases the value by the argument. + """ + with self.lock: + self.p = self.value() + d + + def value(self): + """ + Returns the current value (adjusted for the time decay) + """ + with self.lock: + now = time.time() + dt = now - self.t0 + self.t0 = now + self.p = self.p * (math.pow(self.e, self.r * dt)) + return self.p + + +class RiakNode(object): + """ + The internal representation of a Riak node to which the client can + connect. Encapsulates both the configuration for the node and + error tracking used for node-selection. + """ + + def __init__(self, host='127.0.0.1', http_port=8098, pb_port=8087, + **unused_args): + """ + Creates a node. + """ + + if 'port' in unused_args: + deprecated("port option is deprecated, use http_port or pb_port") + + self.host = host + self.http_port = http_port + self.pb_port = pb_port + self.error_rate = Decaying() diff --git a/riak/riak_object.py b/riak/riak_object.py index 4964ae49..78cf21bb 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -18,7 +18,8 @@ under the License. """ import copy -from metadata import * +from riak.metadata import * +from riak.mapreduce import * from riak import RiakError from riak.riak_index_entry import RiakIndexEntry @@ -125,8 +126,8 @@ def _get_usermeta(self): def _set_usermeta(self, usermeta): self.metadata[MD_USERMETA] = usermeta return self - - usermeta = property(_get_usermeta, _set_usermeta, + + usermeta = property(_get_usermeta, _set_usermeta, doc=""" The custom user metadata on this object. This doesn't include things like content type and links, but only @@ -148,7 +149,8 @@ def add_index(self, field, value): :rtype: self """ if field[-4:] not in ("_bin", "_int"): - raise RiakError("Riak 2i fields must end with either '_bin' or '_int'.") + raise RiakError( + "Riak 2i fields must end with either '_bin' or '_int'.") rie = RiakIndexEntry(field, value) if not rie in self.metadata[MD_INDEX]: @@ -367,19 +369,21 @@ def store(self, w=None, dw=None, pw=None, return_body=True, "store one of the siblings instead") # Issue the put over our transport - t = self.client.get_transport() + # t = self.client.get_transport() if self.key is None: - key, vclock, metadata = t.put_new(self, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) + key, vclock, metadata = self.client.put_new( + self, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) self.exists = True self.key = key self.vclock = vclock self.metadata = metadata else: - result = t.put(self, w=w, dw=dw, pw=pw, return_body=return_body, - if_none_match=if_none_match) + result = self.client.put(self, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) if result is not None and result != ('', []): self._populate(result) @@ -397,8 +401,7 @@ def reload(self, r=None, pr=None, vtag=None): :rtype: self """ - t = self.client.get_transport() - result = t.get(self, r=r, pr=pr, vtag=vtag) + result = self.client.get(self, r=r, pr=pr, vtag=vtag) self.clear() if result is not None and result != ('', []): @@ -432,8 +435,8 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): :type pw: integer :rtype: self """ - t = self.client.get_transport() - result = t.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) + + result = self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) self.clear() return self @@ -576,5 +579,3 @@ def reduce(self, params): mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) return apply(mr.reduce, params) - -from mapreduce import * diff --git a/riak/search.py b/riak/search.py index 8a3e9a91..1d8fe914 100644 --- a/riak/search.py +++ b/riak/search.py @@ -1,85 +1,22 @@ from riak.transports import RiakHttpTransport from xml.etree import ElementTree -from xml.dom.minidom import Document class RiakSearch(object): - def __init__(self, client, transport_class=None, - host="127.0.0.1", port=8098): - if transport_class is None: - transport_class = RiakHttpTransport - - api = getattr(transport_class, 'api', 1) - if api >= 2: - hostports = [(host, port), ] - self._cm = transport_class.default_cm(hostports) - self._transport = transport_class(self._cm, prefix="/solr") - else: - # The old code which attempted to use api==1 would actually - # throw a NameError, so it was obviously never used. We will - # simply raise an error here, intead of a gentle warning. - raise DeprecationWarning('please upgrade the transport to the ' - 'new API') - + def __init__(self, client, **unused_args): self._client = client - self._decoders = {"text/xml": ElementTree.fromstring} - - def get_decoder(self, content_type): - decoder = (self._client.get_decoder(content_type) - or self._decoders[content_type]) - if not decoder: - decoder = self.decode - - return decoder - - def decode(self, data): - return data def add(self, index, *docs): - xml = Document() - root = xml.createElement('add') - for doc in docs: - doc_element = xml.createElement('doc') - for key, value in doc.iteritems(): - field = xml.createElement('field') - field.setAttribute("name", key) - text = xml.createTextNode(value) - field.appendChild(text) - doc_element.appendChild(field) - root.appendChild(doc_element) - xml.appendChild(root) - - url = "/solr/%s/update" % index - self._transport.post_request(uri=url, body=xml.toxml(), - content_type="text/xml") + self._client.fulltext_add(index, docs) index = add def delete(self, index, docs=None, queries=None): - xml = Document() - root = xml.createElement('delete') - if docs: - for doc in docs: - doc_element = xml.createElement('id') - text = xml.createTextNode(doc) - doc_element.appendChild(text) - root.appendChild(doc_element) - if queries: - for query in queries: - query_element = xml.createElement('query') - text = xml.createTextNode(query) - query_element.appendChild(text) - root.appendChild(query_element) - - xml.appendChild(root) - - url = "/solr/%s/update" % index - self._transport.post_request(uri=url, body=xml.toxml(), - content_type="text/xml") + self._client.fulltext_delete(index, docs, queries) remove = delete def search(self, index, query, **params): - return self._client._transport.search(index, query, **params) + return self._client.fulltext_search(index, query, **params) select = search diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 5693d797..fe21c6af 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -56,16 +56,24 @@ class BaseTestCase(object): + host = None + pb_port = None + http_port = None + @staticmethod def randint(): return random.randint(1, 999999) - def create_client(self, host=None, port=None, transport_class=None): - host = host or self.host - port = port or self.port - transport_class = transport_class or self.transport_class - return RiakClient(self.host, self.port, - transport_class=self.transport_class) + def create_client(self, host=None, http_port=None, pb_port=None, + protocol=None, **client_args): + host = host or self.host or HOST + http_port = http_port or self.http_port or HTTP_PORT + pb_port = pb_port or self.pb_port or PB_PORT + protocol = protocol or self.protocol + return RiakClient(protocol=protocol, + host=host, + http_port=http_port, + pb_port=pb_port, **client_args) def setUp(self): self.client = self.create_client() @@ -94,21 +102,18 @@ def setUp(self): if not HAVE_PROTO: self.skipTest('protobuf is unavailable') self.host = PB_HOST - self.port = PB_PORT - self.transport_class = RiakPbcTransport + self.pb_port = PB_PORT + self.protocol = 'pbc' super(RiakPbcTransportTestCase, self).setUp() def test_uses_client_id_if_given(self): - self.host = PB_HOST - self.port = PB_PORT zero_client_id = "\0\0\0\0" - c = RiakClient(PB_HOST, PB_PORT, - transport_class=RiakPbcTransport, - client_id=zero_client_id) + c = self.create_client(client_id=zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) def test_close_underlying_socket_fails(self): - c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport) + self.skipTest("TODO: No longer using connection manager, replace") + c = self.create_client() bucket = c.bucket('bucket_test_close') rand = self.randint() @@ -130,6 +135,7 @@ def test_close_underlying_socket_fails(self): self.assertRaises(socket.error, bucket.get, 'foo') def test_close_underlying_socket_retry(self): + self.skipTest("TODO: No longer using bare transport, replace") c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport, transport_options={"max_attempts": 2}) @@ -183,8 +189,8 @@ class RiakHttpTransportTestCase(BasicKVTests, def setUp(self): self.host = HTTP_HOST - self.port = HTTP_PORT - self.transport_class = RiakHttpTransport + self.http_port = HTTP_PORT + self.protocol = 'http' super(RiakHttpTransportTestCase, self).setUp() def test_no_returnbody(self): diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 23df62e2..f8293f8c 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -103,7 +103,7 @@ def test_client_exceptional_paths(self): with self.assertRaises(ValueError): mr = self.client.search('bucket', 'fleh') mr.add_key_filter("tokenize", "-", 1) - + class JSMapReduceTests(object): def test_javascript_source_map(self): diff --git a/riak/transports/connection.py b/riak/transports/connection.py deleted file mode 100644 index d26b85f4..00000000 --- a/riak/transports/connection.py +++ /dev/null @@ -1,192 +0,0 @@ -""" -Copyright 2011 Greg Stein - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -import httplib -import socket -import contextlib -import functools - - -class ConnectionManager(object): - - # Must be constructable with: connection_class(host, port) - # Must have two attribute: host and port - # Must have a close() method - connection_class = None - - def __init__(self, hostports=[]): - # We want a private copy of this list: either to detach the argument - # default, or to detach from the caller's list. - self.hostports = hostports[:] - - # Open a connection to each specified host/port. On single-threaded - # systems, this will create a round-robin across all specified servers. - # When multi-threaded, this will give us an initial set for all the - # threads to work with (and more will be created, according to demand). - self.conns = [self.connection_class(host, port) - for host, port in hostports] - - def add_hostport(self, host, port): - self.hostports.append((host, port)) - - # Open an initial connection. For single-threaded, this adds to the - # round-robin pool. On multi-threaded, it simply gives us an extra - # connectiong for the load-balancing across the servers. - self.conns.append(self.connection_class(host, port)) - - def remove_host(self, host, port=None): - if port is None: - self.hostports = [(h, p) for h, p in self.hostports - if h != host] - else: - self.hostports.remove((host, port)) - - # Now that the host/port pair has been removed from self.hostports, - # no connections on this pair will be added in .giveback(). Thus, the - # existing connections are all that may exist at this time. We'll - # snapshot the list, and look for offending connections, then try and - # remove them, being wary that race conditions may remove them before - # we can remove it. - for conn in self.conns[:]: - if conn.host == host and (port is None or conn.port == port): - try: - self.conns.remove(conn) - except ValueError: - # Another thread removed the connection. It won't - # be coming back, so we have nothing to do here. - pass - else: - # If the connection was still present (no - # ValueError), then we should go ahead and close - # it down. - conn.close() - - # Just in case somebody uses a host/port combo and typos... - remove_hostport = remove_host - - def take(self): - if len(self.conns) == 0: - # RACE: in a multi-threaded environment, a conn might arrive in - # self.conns, but... no biggy. If we're bouncing up against - # needing a new connection, then we'll just create one. - return self._new_connection() - - # RACE: self.conns might empty out right now, so we need to protect - # our access to it. - try: - # round-robin: take from the front, we'll append when it comes back - return self.conns.pop(0) - except IndexError: - return self._new_connection() - - def giveback(self, conn): - # Connections using a host/port pair that is NOT in self.hostports - # should be ignored. Likely, remove_host() was called while this - # connection was borrowed for some work. - if (conn.host, conn.port) in self.hostports: - self.conns.append(conn) - else: - # Proactively close the connection. The caller won't know whether - # we put it into our list, or left the connection for the caller - # to deal with (and close). - conn.close() - - @contextlib.contextmanager - def withconn(self): - conn = self.take() - try: - yield conn - finally: - self.giveback(conn) - - def _new_connection(self): - if len(self.hostports) == 0: - raise NoHostsDefined() - - # Grab the first host/port combo. We'll put this at the end, so that - # we do a round-robin on the host/port pairs. - host, port = self.hostports[0] - conn = self.connection_class(host, port) - - if len(self.hostports) == 1: - # No rotation needed. - return conn - - # Be careful about rotating. We want to append before - # removing, so that we never hit a len==0 race condition - # (which could prevent the creation of needed connections). - self.hostports.append((host, port)) - - # RACE: another thread may have appended the same host/port - # pair. We will add another pair. Each thread will remove - # one (either [0], or one that had been appened), resulting - # in a correct state of a single pair in the list. - # RACE: another thread may get the host/port pair from - # hostports[0] before we have a chance to remove it. We - # don't need precision round-robin behavior; just something - # close. - # RACE: another thread may have removed hostports[0] (which we - # are also trying to remove), but it will have placed - # another copy at the end before doing so. We have also - # added a host/port pair, and will remove one, leaving the - # list in a correct state. - self.hostports.remove((host, port)) - - return conn - - -class Socket(object): - - def __init__(self, host, port): - self.host = host - self.port = port - - self.sock = None - - def maybe_connect(self): - if self.sock is None: - self.sock = s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - try: - s.connect((self.host, self.port)) - except: - self.close() - raise - - def close(self): - if self.sock is not None: - self.sock.close() - self.sock = None - - -class FactoryConnectionManager(ConnectionManager): - - def __init__(self, connection_class, hostports=[]): - self.connection_class = connection_class - ConnectionManager.__init__(self, hostports) - - -def cm_using(connection_class): - return functools.partial(FactoryConnectionManager, connection_class) - -HTTPConnectionManager = cm_using(httplib.HTTPConnection) -SocketConnectionManager = cm_using(Socket) - - -class NoHostsDefined(Exception): - pass diff --git a/riak/transports/http.py b/riak/transports/http.py index 58ac88b4..b8be3d27 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -17,7 +17,6 @@ specific language governing permissions and limitations under the License. """ -from __future__ import with_statement import urllib import re @@ -31,68 +30,71 @@ except ImportError: import simplejson as json -from transport import RiakTransport +from riak.transports.transport import RiakTransport +from riak.transports.pool import Pool, BadResource from riak.metadata import * from riak.mapreduce import RiakLink from riak import RiakError from riak.riak_index_entry import RiakIndexEntry from riak.multidict import MultiDict -from connection import HTTPConnectionManager import riak.util from xml.etree import ElementTree +from xml.dom.minidom import Document + # subtract length of "Link: " header string and newline MAX_LINK_HEADER_SIZE = 8192 - 8 -class RiakHttpTransport(RiakTransport): +class RiakHttpPool(Pool): """ - The RiakHttpTransport object holds information necessary to - connect to Riak. The Riak API uses HTTP, so there is no persistent - connection, and the RiakClient object is extremely lightweight. + A pool of HTTP(S) transport connections. """ + def __init__(self, client, **options): + self.client = client + self.transport_options = options + if client.protocol is 'https': + self.connection_class = httplib.HTTPConnection + else: + self.connection_class = httplib.HTTPSConnection + super(RiakHttpPool, self).__init__() - # We're using the new RiakTransport API - api = 2 + def create_resource(self): + node = self.client.choose_node() + return RiakHttpTransport(node=node, + client=self.client, + connection_class=self.connection_class, + **self.options) - # The ConnectionManager class that this transport prefers. - default_cm = HTTPConnectionManager + def destroy_resource(self, transport): + transport.close() + + +class RiakHttpTransport(RiakTransport): + """ + The RiakHttpTransport object holds information necessary to + connect to Riak via HTTP. + """ - # How many times to retry a request - RETRY_COUNT = 3 + api = 3 - def __init__(self, cm, - prefix='riak', mapred_prefix='mapred', client_id=None, + def __init__(self, node=None, + client=None, + connection_class=httplib.HTTPConnection + client_id=None, **unused_options): """ - Construct a new RiakClient object. - @param string host - Hostname or IP address (default '127.0.0.1') - @param int port - Port number (default 8098) - @param string prefix - Interface prefix (default 'riak') - @param string mapred_prefix - MapReduce prefix (default 'mapred') - @param string client_id - client id to use for vector clocks + Construct a new HTTP connection to Riak. """ super(RiakHttpTransport, self).__init__() - self._conns = cm - self._prefix = prefix - self._mapred_prefix = mapred_prefix + + self._client = client + self._node = node + self._connection_class = connection_class self._client_id = client_id if not self._client_id: self._client_id = self.make_random_client_id() - def __copy__(self): - ### not implemented right now - raise Exception('not implemented') - ### we don't have _host and _port. will fix after some refactoring... - return RiakHttpTransport(self._host, self._port, self._prefix, - self._mapred_prefix) - - def set_client_id(self, client_id): - self._client_id = client_id - - def get_client_id(self): - return self._client_id - def ping(self): """ Check server is alive over HTTP @@ -362,6 +364,51 @@ def search(self, index, query, **params): else: raise ValueError("Could not decode search response") + def fulltext_add(self, index, docs): + """ + Adds documents to the search index. + """ + xml = Document() + root = xml.createElement('add') + for doc in docs: + doc_element = xml.createElement('doc') + for key in doc: + value = doc[key] + field = xml.createElement('field') + field.setAttribute("name", key) + text = xml.createTextNode(value) + field.appendChild(text) + doc_element.appendChild(field) + root.appendChild(doc_element) + xml.appendChild(root) + + url = "/solr/%s/update" % index + self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") + + def fulltext_delete(self, index, docs=None, queries=None): + """ + Removes documents from the full-text index. + """ + xml = Document() + root = xml.createElement('delete') + if docs: + for doc in docs: + doc_element = xml.createElement('id') + text = xml.createTextNode(doc) + doc_element.appendChild(text) + root.appendChild(doc_element) + if queries: + for query in queries: + query_element = xml.createElement('query') + text = xml.createTextNode(query) + query_element.appendChild(text) + root.appendChild(query_element) + + xml.appendChild(root) + + url = "/solr/%s/update" % index + self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") + def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] if not status in expected_statuses: diff --git a/riak/transports/monitor.py b/riak/transports/monitor.py deleted file mode 100644 index c6223518..00000000 --- a/riak/transports/monitor.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Copyright 2011 Greg Stein - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -import threading -import time - - -class Monitor(object): - - def __init__(self, cm, transport): - self._cm = cm - self._transport = transport - - self._stop_loop = False - self._thread = threading.Thread(target=self._run) - - self._periodic = 0.050 # 50 msec - - def start(self): - self._thread.start() - - def terminate(self): - self._stop_loop = True - self._thread.join() - - def _run(self): - while not self._stop_loop: - ### look for changes in the ring servers - - ### see if some offline servers came back - - time.sleep(self._periodic) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py deleted file mode 100644 index cfef37dd..00000000 --- a/riak/transports/pbc.py +++ /dev/null @@ -1,744 +0,0 @@ -""" -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" -from __future__ import with_statement - -import errno -import socket -import struct -try: - import json -except ImportError: - import simplejson as json - -from riak import RiakError -from riak.mapreduce import RiakLink -from riak.metadata import ( - MD_CHARSET, - MD_CTYPE, - MD_ENCODING, - MD_INDEX, - MD_LASTMOD, - MD_LASTMOD_USECS, - MD_LINKS, - MD_USERMETA, - MD_VTAG, - ) -from riak.riak_index_entry import RiakIndexEntry -from riak.transports import connection -from riak.transports.transport import RiakTransport -import riak.util - -try: - import riak_pb -except ImportError: - riak_pb = None - -## Protocol codes -MSG_CODE_ERROR_RESP = 0 -MSG_CODE_PING_REQ = 1 -MSG_CODE_PING_RESP = 2 -MSG_CODE_GET_CLIENT_ID_REQ = 3 -MSG_CODE_GET_CLIENT_ID_RESP = 4 -MSG_CODE_SET_CLIENT_ID_REQ = 5 -MSG_CODE_SET_CLIENT_ID_RESP = 6 -MSG_CODE_GET_SERVER_INFO_REQ = 7 -MSG_CODE_GET_SERVER_INFO_RESP = 8 -MSG_CODE_GET_REQ = 9 -MSG_CODE_GET_RESP = 10 -MSG_CODE_PUT_REQ = 11 -MSG_CODE_PUT_RESP = 12 -MSG_CODE_DEL_REQ = 13 -MSG_CODE_DEL_RESP = 14 -MSG_CODE_LIST_BUCKETS_REQ = 15 -MSG_CODE_LIST_BUCKETS_RESP = 16 -MSG_CODE_LIST_KEYS_REQ = 17 -MSG_CODE_LIST_KEYS_RESP = 18 -MSG_CODE_GET_BUCKET_REQ = 19 -MSG_CODE_GET_BUCKET_RESP = 20 -MSG_CODE_SET_BUCKET_REQ = 21 -MSG_CODE_SET_BUCKET_RESP = 22 -MSG_CODE_MAPRED_REQ = 23 -MSG_CODE_MAPRED_RESP = 24 -MSG_CODE_INDEX_REQ = 25 -MSG_CODE_INDEX_RESP = 26 -MSG_CODE_SEARCH_QUERY_REQ = 27 -MSG_CODE_SEARCH_QUERY_RESP = 28 - -RIAKC_RW_ONE = 4294967294 -RIAKC_RW_QUORUM = 4294967293 -RIAKC_RW_ALL = 4294967292 -RIAKC_RW_DEFAULT = 4294967291 - -# These are a specific set of socket errors -# that could be raised on send/recv that indicate -# that the socket is closed or reset, and is not -# usable. On seeing any of these errors, the socket -# should be closed, and the connection re-established. -CONN_CLOSED_ERRORS = ( - errno.EHOSTUNREACH, - errno.ECONNRESET, - errno.EBADF, - errno.EPIPE - ) - - -class SocketWithId(connection.Socket): - def __init__(self, host, port): - super(SocketWithId, self).__init__(host, port) - self.last_client_id = None - - def maybe_connect(self): - # If we're going to establish a new connection, then reset the last - # client_id used on this connection. - if self.sock is None: - self.last_client_id = None - super(SocketWithId, self).maybe_connect() - - def send(self, pkt): - try: - self.sock.sendall(pkt) - except socket.error, e: - # If the socket is in a bad state, close it and allow it - # to re-connect on the next try - if e[0] in CONN_CLOSED_ERRORS: - self.close() - raise - - def recv(self, want_len): - try: - res = self.sock.recv(want_len) - - # Assume the socket is closed if no data is - # returned on a blocking read. - if len(res) == 0 and want_len > 0: - self.close() - - return res - - except socket.error, e: - # If the socket is in a bad state, close it and allow it - # to re-connect on the next try - if e[0] in CONN_CLOSED_ERRORS: - self.close() - raise - - -class RiakPbcTransport(RiakTransport): - """ - The RiakPbcTransport object holds a connection to the protocol - buffers interface on the riak server. - """ - - # We're using the new RiakTransport API - api = 2 - - rw_names = { - 'default': RIAKC_RW_DEFAULT, - 'all': RIAKC_RW_ALL, - 'quorum': RIAKC_RW_QUORUM, - 'one': RIAKC_RW_ONE - } - - # The ConnectionManager class that this transport prefers. - default_cm = connection.cm_using(SocketWithId) - - def __init__(self, cm, client_id=None, max_attempts=1, **unused_options): - """ - Construct a new RiakPbcTransport object. - """ - if riak_pb is None: - raise RiakError("this transport is not available (no protobuf)") - - super(RiakPbcTransport, self).__init__() - - self._cm = cm - self._client_id = client_id - self._max_attempts = max_attempts - - # FeatureDetection API - def _server_version(self): - return self.get_server_info()['server_version'] - - def translate_rw_val(self, rw): - val = self.rw_names.get(rw) - if val is None: - return rw - return val - - def __copy__(self): - return RiakPbcTransport(self._cm, self._client_id) - - def ping(self): - """ - Ping the remote server - @return boolean - """ - # An expected response code of None implies "any response is valid". - msg_code, msg = self.send_msg_code(MSG_CODE_PING_REQ, None) - if msg_code == MSG_CODE_PING_RESP: - return 1 - else: - return 0 - - def get_server_info(self): - """ - Get information about the server - """ - msg_code, resp = self.send_msg_code(MSG_CODE_GET_SERVER_INFO_REQ, - MSG_CODE_GET_SERVER_INFO_RESP) - return {'node': resp.node, 'server_version': resp.server_version} - - def get_client_id(self): - """ - Get the client id used by this connection - """ - msg_code, resp = self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ, - MSG_CODE_GET_CLIENT_ID_RESP) - return resp.client_id - - def set_client_id(self, client_id): - """ - Set the client id used by this connection - """ - req = riak_pb.RpbSetClientIdReq() - req.client_id = client_id - - msg_code, resp = self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req, - MSG_CODE_SET_CLIENT_ID_RESP) - - # Using different client_id values across connections is a bad idea - # since you never know which connection you might use for a given - # API call. Setting the client_id manually (rather than as part of - # the transport construction) can be error-prone since the connection - # could drop and be reinstated using self._client_id. - # - # To minimize the potential impact of variant client_id values across - # connections, we'll store this new client_id and use it for all - # future connections. - self._client_id = client_id - - return True - - def get(self, robj, r=None, pr=None, vtag=None): - """ - Serialize get request and deserialize response - """ - if vtag is not None: - raise RiakError("PB transport does not support vtags") - - bucket = robj.bucket - - req = riak_pb.RpbGetReq() - if r: - req.r = self.translate_rw_val(r) - if self.quorum_controls() and pr: - req.pr = self.translate_rw_val(pr) - - if self.tombstone_vclocks(): - req.deletedvclock = 1 - - req.bucket = bucket.name - req.key = robj.key - - # An expected response code of None implies "any response is valid". - msg_code, resp = self.send_msg(MSG_CODE_GET_REQ, req, None) - if msg_code == MSG_CODE_GET_RESP: - contents = [] - for c in resp.content: - contents.append(self.decode_content(c)) - return resp.vclock, contents - else: - return None - - def put(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """ - Serialize get request and deserialize response - """ - bucket = robj.bucket - - req = riak_pb.RpbPutReq() - if w: - req.w = self.translate_rw_val(w) - if dw: - req.dw = self.translate_rw_val(dw) - if self.quorum_controls() and pw: - req.pw = self.translate_rw_val(pw) - - if return_body: - req.return_body = 1 - if if_none_match: - req.if_none_match = 1 - - req.bucket = bucket.name - req.key = robj.key - vclock = robj.vclock - if vclock: - req.vclock = vclock - - self.pbify_content(robj.metadata, - robj.get_encoded_data(), - req.content) - - msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req, - MSG_CODE_PUT_RESP) - if resp is not None: - contents = [] - for c in resp.content: - contents.append(self.decode_content(c)) - return resp.vclock, contents - - def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """Put a new object into the Riak store, returning its (new) key. - - If return_meta is False, then the vlock and metadata return values - will be None. - - @return (key, vclock, metadata) - """ - # Note that this won't work on 0.14 nodes. - bucket = robj.bucket - - req = riak_pb.RpbPutReq() - if w: - req.w = self.translate_rw_val(w) - if dw: - req.dw = self.translate_rw_val(dw) - if self.quorum_controls() and pw: - req.pw = self.translate_rw_val(pw) - - if return_body: - req.return_body = 1 - if if_none_match: - req.if_none_match = 1 - - req.bucket = bucket.name - - self.pbify_content(robj.metadata, - robj.get_encoded_data(), - req.content) - - msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req, - MSG_CODE_PUT_RESP) - if not resp: - raise RiakError("missing response object") - if len(resp.content) != 1: - raise RiakError("siblings were returned from object creation") - - metadata, content = self.decode_content(resp.content[0]) - return resp.key, resp.vclock, metadata - - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): - """ - Serialize get request and deserialize response - """ - bucket = robj.bucket - - req = riak_pb.RpbDelReq() - if rw: - req.rw = self.translate_rw_val(rw) - if r: - req.r = self.translate_rw_val(r) - if w: - req.w = self.translate_rw_val(w) - if dw: - req.dw = self.translate_rw_val(dw) - - if self.quorum_controls(): - if pr: - req.pr = self.translate_rw_val(pr) - if pw: - req.pw = self.translate_rw_val(pw) - - if self.tombstone_vclocks() and robj.vclock: - req.vclock = robj.vclock - - req.bucket = bucket.name - req.key = robj.key - - msg_code, resp = self.send_msg(MSG_CODE_DEL_REQ, req, - MSG_CODE_DEL_RESP) - return self - - def get_keys(self, bucket): - """ - Lists all keys within a bucket. - """ - req = riak_pb.RpbListKeysReq() - req.bucket = bucket.name - - keys = [] - - def _handle_response(resp): - for key in resp.keys: - keys.append(key) - self.send_msg_multi(MSG_CODE_LIST_KEYS_REQ, req, - MSG_CODE_LIST_KEYS_RESP, _handle_response) - - return keys - - def get_buckets(self): - """ - Serialize bucket listing request and deserialize response - """ - msg_code, resp = self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ, - MSG_CODE_LIST_BUCKETS_RESP) - return resp.buckets - - def get_bucket_props(self, bucket): - """ - Serialize bucket property request and deserialize response - """ - req = riak_pb.RpbGetBucketReq() - req.bucket = bucket.name - - msg_code, resp = self.send_msg(MSG_CODE_GET_BUCKET_REQ, req, - MSG_CODE_GET_BUCKET_RESP) - props = {} - if resp.props.HasField('n_val'): - props['n_val'] = resp.props.n_val - if resp.props.HasField('allow_mult'): - props['allow_mult'] = resp.props.allow_mult - - return props - - def set_bucket_props(self, bucket, props): - """ - Serialize set bucket property request and deserialize response - """ - req = riak_pb.RpbSetBucketReq() - req.bucket = bucket.name - for key in props: - if key not in ['n_val', 'allow_mult']: - raise NotImplementedError - - if 'n_val' in props: - req.props.n_val = props['n_val'] - if 'allow_mult' in props: - req.props.allow_mult = props['allow_mult'] - - msg_code, resp = self.send_msg(MSG_CODE_SET_BUCKET_REQ, req, - MSG_CODE_SET_BUCKET_RESP) - return self - - def mapred(self, inputs, query, timeout=None): - # Construct the job, optionally set the timeout... - job = {'inputs': inputs, 'query': query} - if timeout is not None: - job['timeout'] = timeout - - content = json.dumps(job) - - req = riak_pb.RpbMapRedReq() - req.request = content - req.content_type = "application/json" - - # dictionary of phase results - each content should be an encoded array - # which is appended to the result for that phase. - result = {} - - def _handle_response(resp): - if resp.HasField("phase") and resp.HasField("response"): - content = json.loads(resp.response) - if resp.phase in result: - result[resp.phase] += content - else: - result[resp.phase] = content - self.send_msg_multi(MSG_CODE_MAPRED_REQ, req, MSG_CODE_MAPRED_RESP, - _handle_response) - - # If a single result - return the same as the HTTP interface does - # otherwise return all the phase information - if not len(result): - return None - elif len(result) == 1: - return result[max(result.keys())] - else: - return result - - def get_index(self, bucket, index, startkey, endkey=None): - if not self.pb_indexes(): - return self._get_index_mapred_emu(bucket, index, startkey, endkey) - - req = riak_pb.RpbIndexReq(bucket=bucket, index=index) - if endkey: - req.qtype = riak_pb.RpbIndexReq.range - req.range_min = str(startkey) - req.range_max = str(endkey) - else: - req.qtype = riak_pb.RpbIndexReq.eq - req.key = str(startkey) - - msg_code, resp = self.send_msg(MSG_CODE_INDEX_REQ, req, - MSG_CODE_INDEX_RESP) - return resp.keys - - def search(self, index, query, **params): - if not self.pb_search(): - return self._search_mapred_emu(index, query) - - req = riak_pb.RpbSearchQueryReq(index=index, q=query) - if 'rows' in params: - req.rows = params['rows'] - if 'start' in params: - req.start = params['start'] - if 'sort' in params: - req.sort = params['sort'] - if 'filter' in params: - req.filter = params['filter'] - if 'df' in params: - req.df = params['df'] - if 'op' in params: - req.op = params['op'] - if 'q.op' in params: - req.op = params['q.op'] - if 'fl' in params: - if isinstance(params['fl'], list): - req.fl.extend(params['fl']) - else: - req.fl.append(params['fl']) - if 'presort' in params: - req.presort = params['presort'] - - msg_code, resp = self.send_msg(MSG_CODE_SEARCH_QUERY_REQ, req, - MSG_CODE_SEARCH_QUERY_RESP) - - result = {} - if resp.HasField('max_score'): - result['max_score'] = resp.max_score - if resp.HasField('num_found'): - result['num_found'] = resp.num_found - docs = [] - for doc in resp.docs: - resultdoc = {} - for pair in doc.fields: - resultdoc[pair.key] = pair.value - docs.append(resultdoc) - result['docs'] = docs - return result - - def send_msg_code(self, msg_code, expect): - with self._cm.withconn() as conn: - self.send_pkt(conn, struct.pack("!iB", 1, msg_code)) - return self.recv_msg(conn, expect) - - def encode_msg(self, msg_code, msg): - str = msg.SerializeToString() - slen = len(str) - hdr = struct.pack("!iB", 1 + slen, msg_code) - return hdr + str - - def send_msg(self, msg_code, msg, expect): - with self._cm.withconn() as conn: - self.send_pkt(conn, self.encode_msg(msg_code, msg)) - if msg_code == MSG_CODE_SET_CLIENT_ID_REQ: - conn.last_client_id = self._client_id - return self.recv_msg(conn, expect) - - def send_msg_multi(self, msg_code, msg, expect, handler): - with self._cm.withconn() as conn: - self.send_pkt(conn, self.encode_msg(msg_code, msg)) - while True: - msg_code, resp = self.recv_msg(conn, expect) - handler(resp) - if resp.HasField("done") and resp.done: - break - - def send_pkt(self, conn, pkt): - attempt = 0 - e = None - for attempt in xrange(self._max_attempts): - e = None - try: - conn.maybe_connect() - - # If the last client_id used on this connection is - # different than our client_id, then set a new ID on - # the connection. - if conn.last_client_id != self._client_id: - req = riak_pb.RpbSetClientIdReq() - req.client_id = self._client_id - conn.send(self.encode_msg(MSG_CODE_SET_CLIENT_ID_REQ, req)) - conn.last_client_id = self._client_id - self.recv_msg(conn, MSG_CODE_SET_CLIENT_ID_RESP) - - conn.send(pkt) - break - except socket.error, e: - # If this is some unknown socket error bail out - # instead of retrying - if e[0] not in CONN_CLOSED_ERRORS: - raise - - # Max attempts reached, raise whatever exception we are getting - if attempt + 1 == self._max_attempts and e is not None: - raise e - - def recv_msg(self, conn, expect): - self.recv_pkt(conn) - msg_code, = struct.unpack("B", self._inbuf[:1]) - if msg_code == MSG_CODE_ERROR_RESP: - msg = riak_pb.RpbErrorResp() - msg.ParseFromString(self._inbuf[1:]) - raise Exception(msg.errmsg) - elif msg_code == MSG_CODE_PING_RESP: - msg = None - elif msg_code == MSG_CODE_GET_SERVER_INFO_RESP: - msg = riak_pb.RpbGetServerInfoResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_GET_CLIENT_ID_RESP: - msg = riak_pb.RpbGetClientIdResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_SET_CLIENT_ID_RESP: - msg = None - elif msg_code == MSG_CODE_GET_RESP: - msg = riak_pb.RpbGetResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_PUT_RESP: - msg = riak_pb.RpbPutResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_DEL_RESP: - msg = None - elif msg_code == MSG_CODE_LIST_KEYS_RESP: - msg = riak_pb.RpbListKeysResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_LIST_BUCKETS_RESP: - msg = riak_pb.RpbListBucketsResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_GET_BUCKET_RESP: - msg = riak_pb.RpbGetBucketResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_SET_BUCKET_RESP: - msg = None - elif msg_code == MSG_CODE_MAPRED_RESP: - msg = riak_pb.RpbMapRedResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_INDEX_RESP: - msg = riak_pb.RpbIndexResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_SEARCH_QUERY_RESP: - msg = riak_pb.RpbSearchQueryResp() - msg.ParseFromString(self._inbuf[1:]) - else: - raise Exception("unknown msg code %s" % msg_code) - if expect and msg_code != expect: - raise RiakError("unexpected protocol buffer message code: %d" - % msg_code) - return msg_code, msg - - def recv_pkt(self, conn): - nmsglen = conn.recv(4) - if len(nmsglen) != 4: - raise RiakError( - "Socket returned short packet length %d - expected 4" - % len(nmsglen)) - msglen, = struct.unpack('!i', nmsglen) - self._inbuf_len = msglen - self._inbuf = '' - while len(self._inbuf) < msglen: - want_len = min(8192, msglen - len(self._inbuf)) - recv_buf = conn.recv(want_len) - if not recv_buf: - break - self._inbuf += recv_buf - if len(self._inbuf) != self._inbuf_len: - raise RiakError("Socket returned short packet %d - expected %d" - % (len(self._inbuf), self._inbuf_len)) - - def decode_contents(self, rpb_contents): - contents = [] - for rpb_c in rpb_contents: - contents.append(self.decode_content(rpb_c)) - return contents - - def decode_content(self, rpb_content): - metadata = {} - if rpb_content.HasField("deleted"): - metadata[MD_DELETED] = True - if rpb_content.HasField("content_type"): - metadata[MD_CTYPE] = rpb_content.content_type - if rpb_content.HasField("charset"): - metadata[MD_CHARSET] = rpb_content.charset - if rpb_content.HasField("content_encoding"): - metadata[MD_ENCODING] = rpb_content.content_encoding - if rpb_content.HasField("vtag"): - metadata[MD_VTAG] = rpb_content.vtag - links = [] - for link in rpb_content.links: - if link.HasField("bucket"): - bucket = link.bucket - else: - bucket = None - if link.HasField("key"): - key = link.key - else: - key = None - if link.HasField("tag"): - tag = link.tag - else: - tag = None - links.append(RiakLink(bucket, key, tag)) - if links: - metadata[MD_LINKS] = links - if rpb_content.HasField("last_mod"): - metadata[MD_LASTMOD] = rpb_content.last_mod - if rpb_content.HasField("last_mod_usecs"): - metadata[MD_LASTMOD_USECS] = rpb_content.last_mod_usecs - usermeta = {} - for usermd in rpb_content.usermeta: - usermeta[usermd.key] = usermd.value - if len(usermeta) > 0: - metadata[MD_USERMETA] = usermeta - indexes = [] - for index in rpb_content.indexes: - rie = RiakIndexEntry(index.key, index.value) - indexes.append(rie) - if len(indexes) > 0: - metadata[MD_INDEX] = indexes - return metadata, rpb_content.value - - def pbify_content(self, metadata, data, rpb_content): - # Convert the broken out fields, building up - # pbmetadata for any unknown ones - for k, v in metadata.iteritems(): - if k == MD_CTYPE: - rpb_content.content_type = v - elif k == MD_CHARSET: - rpb_content.charset = v - elif k == MD_ENCODING: - rpb_content.charset = v - elif k == MD_USERMETA: - for uk, uv in v.iteritems(): - pair = rpb_content.usermeta.add() - pair.key = uk - pair.value = uv - elif k == MD_INDEX: - for rie in v: - pair = rpb_content.indexes.add() - pair.key = rie.get_field() - pair.value = rie.get_value() - elif k == MD_LINKS: - for link in v: - pb_link = rpb_content.links.add() - pb_link.bucket = link.get_bucket() - pb_link.key = link.get_key() - pb_link.tag = link.get_tag() - rpb_content.value = data diff --git a/riak/transports/pbc/__init__.py b/riak/transports/pbc/__init__.py new file mode 100644 index 00000000..d3620738 --- /dev/null +++ b/riak/transports/pbc/__init__.py @@ -0,0 +1,43 @@ +""" +Copyright 2012 Basho Technologies, Inc. +Copyright 2010 Rusty Klophaus +Copyright 2010 Justin Sheehy +Copyright 2009 Jay Baird + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + + +from riak.transports.pool import Pool +from riak.transports.pbc.transport import RiakPbcTransport + + +class RiakPbcPool(Pool): + """ + A resource pool of PBC transports. + """ + def __init__(self, client, **options): + super(RiakPbcPool, self).__init__() + self._client = client + self._options = options + + def create_resource(self): + node = self._client._choose_node() + return RiakPbcTransport(node=node, + client=self._client, + **self._options) + + def destroy_resource(self, pbc): + pbc.close() diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py new file mode 100644 index 00000000..dca7d2f0 --- /dev/null +++ b/riak/transports/pbc/codec.py @@ -0,0 +1,143 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" +from riak.metadata import ( + MD_CHARSET, + MD_CTYPE, + MD_ENCODING, + MD_INDEX, + MD_LASTMOD, + MD_LASTMOD_USECS, + MD_LINKS, + MD_USERMETA, + MD_VTAG, + ) + +try: + import riak_pb +except ImportError: + riak_pb = None + +RIAKC_RW_ONE = 4294967294 +RIAKC_RW_QUORUM = 4294967293 +RIAKC_RW_ALL = 4294967292 +RIAKC_RW_DEFAULT = 4294967291 + + +class RiakPbcCodec(object): + """ + Protobuffs Encoding and decoding methods for RiakPbcTransport. + """ + + rw_names = { + 'default': RIAKC_RW_DEFAULT, + 'all': RIAKC_RW_ALL, + 'quorum': RIAKC_RW_QUORUM, + 'one': RIAKC_RW_ONE + } + + def __init__(self, **unused_args): + if riak_pb is None: + raise NotImplementedError("this transport is not available") + super(RiakPbcCodec, self).__init__(**unused_args) + + def translate_rw_val(self, rw): + val = self.rw_names.get(rw) + if val is None: + return rw + elif type(rw) is int and rw >= 0: + return val + else: + return None + + def decode_contents(self, rpb_contents): + return [self.decode_content(rpb_c) for rpb_c in rpb_contents] + + def decode_content(self, rpb_content): + metadata = {} + if rpb_content.HasField("deleted"): + metadata[MD_DELETED] = True + if rpb_content.HasField("content_type"): + metadata[MD_CTYPE] = rpb_content.content_type + if rpb_content.HasField("charset"): + metadata[MD_CHARSET] = rpb_content.charset + if rpb_content.HasField("content_encoding"): + metadata[MD_ENCODING] = rpb_content.content_encoding + if rpb_content.HasField("vtag"): + metadata[MD_VTAG] = rpb_content.vtag + links = [] + for link in rpb_content.links: + if link.HasField("bucket"): + bucket = link.bucket + else: + bucket = None + if link.HasField("key"): + key = link.key + else: + key = None + if link.HasField("tag"): + tag = link.tag + else: + tag = None + links.append(RiakLink(bucket, key, tag)) + if links: + metadata[MD_LINKS] = links + if rpb_content.HasField("last_mod"): + metadata[MD_LASTMOD] = rpb_content.last_mod + if rpb_content.HasField("last_mod_usecs"): + metadata[MD_LASTMOD_USECS] = rpb_content.last_mod_usecs + usermeta = {} + for usermd in rpb_content.usermeta: + usermeta[usermd.key] = usermd.value + if len(usermeta) > 0: + metadata[MD_USERMETA] = usermeta + indexes = [] + for index in rpb_content.indexes: + rie = RiakIndexEntry(index.key, index.value) + indexes.append(rie) + if len(indexes) > 0: + metadata[MD_INDEX] = indexes + return metadata, rpb_content.value + + def encode_content(self, metadata, data, rpb_content): + # Convert the broken out fields, building up + # pbmetadata for any unknown ones + for k in metadata: + v = metadata[k] + if k == MD_CTYPE: + rpb_content.content_type = v + elif k == MD_CHARSET: + rpb_content.charset = v + elif k == MD_ENCODING: + rpb_content.charset = v + elif k == MD_USERMETA: + for uk in v: + pair = rpb_content.usermeta.add() + pair.key = uk + pair.value = v[uk] + elif k == MD_INDEX: + for rie in v: + pair = rpb_content.indexes.add() + pair.key = rie.get_field() + pair.value = rie.get_value() + elif k == MD_LINKS: + for link in v: + pb_link = rpb_content.links.add() + pb_link.bucket = link.get_bucket() + pb_link.key = link.get_key() + pb_link.tag = link.get_tag() + rpb_content.value = data diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py new file mode 100644 index 00000000..b08b46ba --- /dev/null +++ b/riak/transports/pbc/connection.py @@ -0,0 +1,128 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import errno +import socket +import struct +from contextlib import contextmanager + +# These are a specific set of socket errors +# that could be raised on send/recv that indicate +# that the socket is closed or reset, and is not +# usable. On seeing any of these errors, the socket +# should be closed, and the connection re-established. +CONN_CLOSED_ERRORS = ( + errno.EHOSTUNREACH, + errno.ECONNRESET, + errno.EBADF, + errno.EPIPE + ) + + +class RiakPbcConnection(object): + """ + Connection-related methods for RiakPbcTransport. + """ + def _encode_msg(self, msg_code, msg=None): + if msg is None: + return struct.pack("!iB", 1, msg_code) + msgstr = msg.SerializeToString() + slen = len(msgstr) + hdr = struct.pack("!iB", 1 + slen, msg_code) + return hdr + msgstr + + def _request(self, msg_code, msg=None, expect=None): + self._send_msg(msg_code, msg) + return self._recv_msg(expect) + + def _send_msg(self, msg_code, msg): + self._socket.send(self.encode_msg(msg_code, msg)) + + def _recv_msg(self, expect=None): + self._recv_pkt() + msg_code, = struct.unpack("B", self._inbuf[:1]) + if msg_code == MSG_CODE_ERROR_RESP: + msg = riak_pb.RpbErrorResp() + msg.ParseFromString(self._inbuf[1:]) + raise Exception(msg.errmsg) + elif msg_code == MSG_CODE_PING_RESP: + msg = None + elif msg_code == MSG_CODE_GET_SERVER_INFO_RESP: + msg = riak_pb.RpbGetServerInfoResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_GET_CLIENT_ID_RESP: + msg = riak_pb.RpbGetClientIdResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_SET_CLIENT_ID_RESP: + msg = None + elif msg_code == MSG_CODE_GET_RESP: + msg = riak_pb.RpbGetResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_PUT_RESP: + msg = riak_pb.RpbPutResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_DEL_RESP: + msg = None + elif msg_code == MSG_CODE_LIST_KEYS_RESP: + msg = riak_pb.RpbListKeysResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_LIST_BUCKETS_RESP: + msg = riak_pb.RpbListBucketsResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_GET_BUCKET_RESP: + msg = riak_pb.RpbGetBucketResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_SET_BUCKET_RESP: + msg = None + elif msg_code == MSG_CODE_MAPRED_RESP: + msg = riak_pb.RpbMapRedResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_INDEX_RESP: + msg = riak_pb.RpbIndexResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_SEARCH_QUERY_RESP: + msg = riak_pb.RpbSearchQueryResp() + msg.ParseFromString(self._inbuf[1:]) + else: + raise Exception("unknown msg code %s" % msg_code) + if expect and msg_code != expect: + raise RiakError("unexpected protocol buffer message code: %d" + % msg_code) + return msg_code, msg + + def _recv_pkt(self): + nmsglen = self._socket.recv(4) + if len(nmsglen) != 4: + raise RiakError( + "Socket returned short packet length %d - expected 4" + % len(nmsglen)) + msglen, = struct.unpack('!i', nmsglen) + self._inbuf_len = msglen + self._inbuf = '' + while len(self._inbuf) < msglen: + want_len = min(8192, msglen - len(self._inbuf)) + recv_buf = conn.recv(want_len) + if not recv_buf: + break + self._inbuf += recv_buf + if len(self._inbuf) != self._inbuf_len: + raise RiakError("Socket returned short packet %d - expected %d" + % (len(self._inbuf), self._inbuf_len)) + + def close(self): + self._socket.shutdown(socket.SHUT_RDWR) diff --git a/riak/transports/pbc/messages.py b/riak/transports/pbc/messages.py new file mode 100644 index 00000000..c8175867 --- /dev/null +++ b/riak/transports/pbc/messages.py @@ -0,0 +1,48 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +## Protocol codes +MSG_CODE_ERROR_RESP = 0 +MSG_CODE_PING_REQ = 1 +MSG_CODE_PING_RESP = 2 +MSG_CODE_GET_CLIENT_ID_REQ = 3 +MSG_CODE_GET_CLIENT_ID_RESP = 4 +MSG_CODE_SET_CLIENT_ID_REQ = 5 +MSG_CODE_SET_CLIENT_ID_RESP = 6 +MSG_CODE_GET_SERVER_INFO_REQ = 7 +MSG_CODE_GET_SERVER_INFO_RESP = 8 +MSG_CODE_GET_REQ = 9 +MSG_CODE_GET_RESP = 10 +MSG_CODE_PUT_REQ = 11 +MSG_CODE_PUT_RESP = 12 +MSG_CODE_DEL_REQ = 13 +MSG_CODE_DEL_RESP = 14 +MSG_CODE_LIST_BUCKETS_REQ = 15 +MSG_CODE_LIST_BUCKETS_RESP = 16 +MSG_CODE_LIST_KEYS_REQ = 17 +MSG_CODE_LIST_KEYS_RESP = 18 +MSG_CODE_GET_BUCKET_REQ = 19 +MSG_CODE_GET_BUCKET_RESP = 20 +MSG_CODE_SET_BUCKET_REQ = 21 +MSG_CODE_SET_BUCKET_RESP = 22 +MSG_CODE_MAPRED_REQ = 23 +MSG_CODE_MAPRED_RESP = 24 +MSG_CODE_INDEX_REQ = 25 +MSG_CODE_INDEX_RESP = 26 +MSG_CODE_SEARCH_QUERY_REQ = 27 +MSG_CODE_SEARCH_QUERY_RESP = 28 diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py new file mode 100644 index 00000000..a71f6c20 --- /dev/null +++ b/riak/transports/pbc/stream.py @@ -0,0 +1,86 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +try: + import json +except ImportError: + import simplejson as json + +from riak.transports.pbc.messages import MSG_CODE_LIST_KEYS_RESP +from riak.transports.pbc.messages import MSG_CODE_MAPRED_RESP + + +class RiakPbcStream(class): + """ + Used internally by RiakPbcTransport to implement streaming + operations. Implements the iterator interface. + """ + def __init__(self, transport): + self.transport = transport + + def __iter__(self): + return self + + def next(self): + expect = self._expect + try: + resp = self.transport._recv_msg(expect) + if(self._is_done(resp)): + raise StopIteration + else: + return resp + except StopIteration: + pass + except: + # TODO: which exceptions do we expect to be generated? + # Should we raise BadResource? + raise StopIteration + + def _is_done(self, response): + raise NotImplementedError + + +class RiakPbcKeyStream(RiakPbcStream): + """ + Used internally by RiakPbcTransport to implement key-list streams. + """ + + _expect = MSG_CODE_LIST_KEYS_RESP + + def next(self): + response = super(RiakPbcKeyStream, self).__next__() + return response.keys + + def _is_done(self, response): + return response.done + + +class RiakPbcMapredStream(RiakPbcStream): + """ + Used internally by RiakPbcTransport to implement MapReduce + streams. + """ + + _expect = MSG_CODE_MAPRED_RESP + + def next(self): + response = super(RiakPbcMapredStream, self).next() + return (response.phase, json.loads(response.response)) + + def _is_done(self, response): + return response.done diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py new file mode 100644 index 00000000..90a00b40 --- /dev/null +++ b/riak/transports/pbc/transport.py @@ -0,0 +1,401 @@ +""" +Copyright 2012 Basho Technologies, Inc. +Copyright 2010 Rusty Klophaus +Copyright 2010 Justin Sheehy +Copyright 2009 Jay Baird + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +try: + import json +except ImportError: + import simplejson as json + +from riak import RiakError +from riak.mapreduce import RiakLink +from riak.riak_index_entry import RiakIndexEntry +from riak.transports.transport import RiakTransport +from riak.transports.pbc.connection import RiakPbcConnection +from riak.transports.pbc.stream import RiakPbcKeyStream, RiakPbcMapredStream +from riak.transports.pbc.codec import RiakPbcCodec +from riak.transports.pbc.messages import * +import riak.util + + +class RiakPbcTransport(RiakTransport, RiakPbcConnection, RiakPbcCodec): + """ + The RiakPbcTransport object holds a connection to the protocol + buffers interface on the riak server. + """ + + api = 3 + + def __init__(self, node=None, client=None, connect_timeout=None, + request_timeout=None, **unused_options): + """ + Construct a new RiakPbcTransport object. + """ + super(RiakPbcTransport, self).__init__() + + self._client = client + self._node = node + self._address = (node.host, node.pb_port) + self._timeouts = {'connect': connect_timeout, + 'request': request_timeout} + + # FeatureDetection API + def _server_version(self): + return self.get_server_info()['server_version'] + + def ping(self): + """ + Ping the remote server + @return boolean + """ + + msg_code, msg = self._request(MSG_CODE_PING_REQ) + if msg_code == MSG_CODE_PING_RESP: + return 1 + else: + return 0 + + def get_server_info(self): + """ + Get information about the server + """ + msg_code, resp = self._request(MSG_CODE_GET_SERVER_INFO_REQ, + expect=MSG_CODE_GET_SERVER_INFO_RESP) + return {'node': resp.node, 'server_version': resp.server_version} + + @property + def client_id(self): + """the client ID for this connection""" + msg_code, resp = self._request(MSG_CODE_GET_CLIENT_ID_REQ, + expect=MSG_CODE_GET_CLIENT_ID_RESP) + return resp.client_id + + @client_id.setter + def client_id(self, client_id): + req = riak_pb.RpbSetClientIdReq() + req.client_id = client_id + + msg_code, resp = self._request(MSG_CODE_SET_CLIENT_ID_REQ, req, + MSG_CODE_SET_CLIENT_ID_RESP) + + self._client_id = client_id + + def get(self, robj, r=None, pr=None, vtag=None): + """ + Serialize get request and deserialize response + """ + if vtag is not None: + raise RiakError("PB transport does not support vtags") + + bucket = robj.bucket + + req = riak_pb.RpbGetReq() + if r: + req.r = self.translate_rw_val(r) + if self.quorum_controls() and pr: + req.pr = self.translate_rw_val(pr) + + if self.tombstone_vclocks(): + req.deletedvclock = 1 + + req.bucket = bucket.name + req.key = robj.key + + msg_code, resp = self._request(MSG_CODE_GET_REQ, req) + if msg_code == MSG_CODE_GET_RESP: + contents = [] + for c in resp.content: + contents.append(self.decode_content(c)) + return resp.vclock, contents + else: + return None + + def put(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): + """ + Serialize get request and deserialize response + """ + bucket = robj.bucket + + req = riak_pb.RpbPutReq() + if w: + req.w = self.translate_rw_val(w) + if dw: + req.dw = self.translate_rw_val(dw) + if self.quorum_controls() and pw: + req.pw = self.translate_rw_val(pw) + + if return_body: + req.return_body = 1 + if if_none_match: + req.if_none_match = 1 + + req.bucket = bucket.name + req.key = robj.key + vclock = robj.vclock() + if vclock: + req.vclock = vclock + + self.pbify_content(robj.metadata, + robj.get_encoded_data(), + req.content) + + msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, + MSG_CODE_PUT_RESP) + if resp is not None: + contents = [] + for c in resp.content: + contents.append(self.decode_content(c)) + return resp.vclock, contents + + def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): + """Put a new object into the Riak store, returning its (new) key. + + If return_meta is False, then the vlock and metadata return values + will be None. + + @return (key, vclock, metadata) + """ + # Note that this won't work on 0.14 nodes. + bucket = robj.bucket + + req = riak_pb.RpbPutReq() + if w: + req.w = self.translate_rw_val(w) + if dw: + req.dw = self.translate_rw_val(dw) + if self.quorum_controls() and pw: + req.pw = self.translate_rw_val(pw) + + if return_body: + req.return_body = 1 + if if_none_match: + req.if_none_match = 1 + + req.bucket = bucket.name + + self.pbify_content(robj.metadata, + robj.get_encoded_data(), + req.content) + + msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, + MSG_CODE_PUT_RESP) + if not resp: + raise RiakError("missing response object") + if len(resp.content) != 1: + raise RiakError("siblings were returned from object creation") + + metadata, content = self.decode_content(resp.content[0]) + return resp.key, resp.vclock, metadata + + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + """ + Serialize get request and deserialize response + """ + bucket = robj.bucket + + req = riak_pb.RpbDelReq() + if rw: + req.rw = self.translate_rw_val(rw) + if r: + req.r = self.translate_rw_val(r) + if w: + req.w = self.translate_rw_val(w) + if dw: + req.dw = self.translate_rw_val(dw) + + if self.quorum_controls(): + if pr: + req.pr = self.translate_rw_val(pr) + if pw: + req.pw = self.translate_rw_val(pw) + + if self.tombstone_vclocks() and robj.vclock(): + req.vclock = robj.vclock() + + req.bucket = bucket.name + req.key = robj.key + + msg_code, resp = self._request(MSG_CODE_DEL_REQ, req, + MSG_CODE_DEL_RESP) + return self + + def get_keys(self, bucket): + """ + Lists all keys within a bucket. + """ + keys = [] + for keylist in self.stream_keys(bucket): + keys = keys + keylist + + return keys + + def stream_keys(self, bucket): + """ + Streams keys from a bucket, returning an iterator that yields + lists of keys. + """ + req = riak_pb.RpbListKeysReq() + req.bucket = bucket.name + + self._send_msg(MSG_CODE_LIST_KEYS_REQ, req) + + return RiakPbcKeyStream(self) + + def get_buckets(self): + """ + Serialize bucket listing request and deserialize response + """ + msg_code, resp = self._request(MSG_CODE_LIST_BUCKETS_REQ, + MSG_CODE_LIST_BUCKETS_RESP) + return resp.buckets + + def get_bucket_props(self, bucket): + """ + Serialize bucket property request and deserialize response + """ + req = riak_pb.RpbGetBucketReq() + req.bucket = bucket.name + + msg_code, resp = self._request(MSG_CODE_GET_BUCKET_REQ, req, + MSG_CODE_GET_BUCKET_RESP) + props = {} + if resp.props.HasField('n_val'): + props['n_val'] = resp.props.n_val + if resp.props.HasField('allow_mult'): + props['allow_mult'] = resp.props.allow_mult + + return props + + def set_bucket_props(self, bucket, props): + """ + Serialize set bucket property request and deserialize response + """ + req = riak_pb.RpbSetBucketReq() + req.bucket = bucket.name + for key in props: + if key not in ['n_val', 'allow_mult']: + raise NotImplementedError + + if 'n_val' in props: + req.props.n_val = props['n_val'] + if 'allow_mult' in props: + req.props.allow_mult = props['allow_mult'] + + msg_code, resp = self._request(MSG_CODE_SET_BUCKET_REQ, req, + MSG_CODE_SET_BUCKET_RESP) + return self + + def mapred(self, inputs, query, timeout=None): + # dictionary of phase results - each content should be an encoded array + # which is appended to the result for that phase. + result = {} + for phase, content in self.stream_mapred(inputs, query, timeout): + if phase in result: + result[phase] += content + else: + result[phase] = content + + # If a single result - return the same as the HTTP interface does + # otherwise return all the phase information + if not len(result): + return None + elif len(result) == 1: + return result[max(result.keys())] + else: + return result + + def stream_mapred(self, inputs, query, timeout=None): + # Construct the job, optionally set the timeout... + job = {'inputs': inputs, 'query': query} + if timeout is not None: + job['timeout'] = timeout + + content = json.dumps(job) + + req = riak_pb.RpbMapRedReq() + req.request = content + req.content_type = "application/json" + + self._send_msg(MSG_CODE_MAPRED_REQ, req) + + return RiakPbcMapredStream(self) + + def get_index(self, bucket, index, startkey, endkey=None): + if not self.pb_indexes(): + return self._get_index_mapred_emu(bucket, index, startkey, endkey) + + req = riak_pb.RpbIndexReq(bucket=bucket, index=index) + if endkey: + req.qtype = riak_pb.RpbIndexReq.range + req.range_min = str(startkey) + req.range_max = str(endkey) + else: + req.qtype = riak_pb.RpbIndexReq.eq + req.key = str(startkey) + + msg_code, resp = self._request(MSG_CODE_INDEX_REQ, req, + MSG_CODE_INDEX_RESP) + return resp.keys + + def search(self, index, query, **params): + if not self.pb_search(): + return self._search_mapred_emu(index, query) + + req = riak_pb.RpbSearchQueryReq(index=index, q=query) + if 'rows' in params: + req.rows = params['rows'] + if 'start' in params: + req.start = params['start'] + if 'sort' in params: + req.sort = params['sort'] + if 'filter' in params: + req.filter = params['filter'] + if 'df' in params: + req.df = params['df'] + if 'op' in params: + req.op = params['op'] + if 'q.op' in params: + req.op = params['q.op'] + if 'fl' in params: + if isinstance(params['fl'], list): + req.fl.extend(params['fl']) + else: + req.fl.append(params['fl']) + if 'presort' in params: + req.presort = params['presort'] + + msg_code, resp = self._request(MSG_CODE_SEARCH_QUERY_REQ, req, + MSG_CODE_SEARCH_QUERY_RESP) + + result = {} + if resp.HasField('max_score'): + result['max_score'] = resp.max_score + if resp.HasField('num_found'): + result['num_found'] = resp.num_found + docs = [] + for doc in resp.docs: + resultdoc = {} + for pair in doc.fields: + resultdoc[pair.key] = pair.value + docs.append(resultdoc) + result['docs'] = docs + return result diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 0b0ff32c..975d1dac 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -31,11 +31,14 @@ class RiakTransport(FeatureDetection): Class to encapsulate transport details """ - # Subclasses should specify their API level. - # * missing or 1: the API used up and through 1.3.x. - # * 2: the API introduced with 1.4.x - # - # api = 2 + @property + def client_id(self): + """the client ID for this connection""" + return self._client_id + + @client_id.setter + def client_id(self, value): + self._client_id = value @classmethod def make_random_client_id(self): @@ -149,6 +152,18 @@ def get_index(self, bucket, index, startkey, endkey=None): """ raise NotImplementedError + def fulltext_add(self, index, *docs): + """ + Adds documents to the full-text index. + """ + raise NotImplementedError + + def fulltext_delete(self, index, docs=None, queries=None): + """ + Removes documents from the full-text index. + """ + raise NotImplementedError + def _search_mapred_emu(self, index, query): """ Emulates a search request via MapReduce. Used in the case From 486c5a7056877b6b6102ec104041fc8d0e6b6390 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 12:22:13 -0500 Subject: [PATCH 0262/1060] Actually connect to PBC by connecting the socket. --- riak/transports/pbc/connection.py | 4 ++++ riak/transports/pbc/transport.py | 1 + 2 files changed, 5 insertions(+) diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index b08b46ba..b4fc5853 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -124,5 +124,9 @@ def _recv_pkt(self): raise RiakError("Socket returned short packet %d - expected %d" % (len(self._inbuf), self._inbuf_len)) + def _connect(self): + self._socket = socket.create_connection(self._address, + self._timeouts.connect) + def close(self): self._socket.shutdown(socket.SHUT_RDWR) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 90a00b40..23730cf5 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -55,6 +55,7 @@ def __init__(self, node=None, client=None, connect_timeout=None, self._address = (node.host, node.pb_port) self._timeouts = {'connect': connect_timeout, 'request': request_timeout} + self._connect() # FeatureDetection API def _server_version(self): From 29cd619d00c596e4724c67544e502c453e837d4e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 12:41:05 -0500 Subject: [PATCH 0263/1060] Add transport method to client. --- riak/client.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/riak/client.py b/riak/client.py index 26d2430b..b2f4ff47 100644 --- a/riak/client.py +++ b/riak/client.py @@ -229,3 +229,16 @@ def _error_rate(node): return min(nodes, key=_error_rate) else: return random.choice(good) + + @contextmanager + def transport(self, protocol=self.protocol): + if protocol in ['http', 'https']: + pool = self._http_pool + elif protocol is 'pbc': + pool = self._pb_pool + else: + raise ValueError("invalid protocol %s" % protocol) + + # Replace with recovery logic later + with pool.take() as transport: + yield transport From 32081343794299cb506f49890be30ba3feb0f9b2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 13:06:56 -0500 Subject: [PATCH 0264/1060] Fix some typos and comments from @ultimatebuster. --- riak/client.py | 6 +++--- riak/transports/http.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/riak/client.py b/riak/client.py index b2f4ff47..4aecb1f4 100644 --- a/riak/client.py +++ b/riak/client.py @@ -25,7 +25,7 @@ import simplejson as json from contextlib import contextmanager - +from weakref import WeakValueDictionary from riak.client.operations import RiakClientOperations from riak.node import RiakNode from riak.bucket import RiakBucket @@ -86,7 +86,7 @@ def __init__(self, protocol='http', transport_options={}, 'text/json': json.dumps} self._decoders = {'application/json': json.loads, 'text/json': json.loads} - self._buckets = {} + self._buckets = WeakValueDictionary() @property def protocol(self): @@ -94,7 +94,7 @@ def protocol(self): @property.setter def protocol(self, value): - if protocol and protocol not in self.PROTOCOLS: + if value not in self.PROTOCOLS: raise ValueError("protocol option is invalid, must be one of %s" % repr(self.PROTOCOLS)) self._protocol = value diff --git a/riak/transports/http.py b/riak/transports/http.py index b8be3d27..43f0b7e4 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -52,7 +52,7 @@ class RiakHttpPool(Pool): """ def __init__(self, client, **options): self.client = client - self.transport_options = options + self.options = options if client.protocol is 'https': self.connection_class = httplib.HTTPConnection else: From f60700bc0c341e91edc9afadf68fdaec5286b6c1 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 13:07:46 -0500 Subject: [PATCH 0265/1060] Move client -> client/__init__. --- riak/{client.py => client/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename riak/{client.py => client/__init__.py} (100%) diff --git a/riak/client.py b/riak/client/__init__.py similarity index 100% rename from riak/client.py rename to riak/client/__init__.py From f188f8e7904c631ee1d072fbe846c5a6253e91d5 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 15:08:02 -0500 Subject: [PATCH 0266/1060] Move connection-claiming into a new module, add retry logic. * RiakClient.transport is now RiakClient._transport (via RiakClientTransport mixin). * HTTP and PBC transports define their own version of what are retryable exceptions. --- riak/client/__init__.py | 17 ++------ riak/client/operations.py | 32 +++++++------- riak/client/transport.py | 69 +++++++++++++++++++++++++++++++ riak/transports/http.py | 19 +++++++++ riak/transports/pbc/__init__.py | 30 +++++++++++++- riak/transports/pbc/connection.py | 13 +----- 6 files changed, 137 insertions(+), 43 deletions(-) create mode 100644 riak/client/transport.py diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 4aecb1f4..c57fc9f6 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -27,6 +27,7 @@ from contextlib import contextmanager from weakref import WeakValueDictionary from riak.client.operations import RiakClientOperations +from riak.client.transport import RiakClientTransport from riak.node import RiakNode from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduce @@ -40,7 +41,8 @@ @deprecateQuorumAccessors -class RiakClient(RiakMapReduceChain, RiakClientOperations): +class RiakClient(RiakMapReduceChain, RiakClientOperations, + RiakClientTransport): """ The ``RiakClient`` object holds information necessary to connect to Riak. Requests can be made to Riak directly through the client @@ -229,16 +231,3 @@ def _error_rate(node): return min(nodes, key=_error_rate) else: return random.choice(good) - - @contextmanager - def transport(self, protocol=self.protocol): - if protocol in ['http', 'https']: - pool = self._http_pool - elif protocol is 'pbc': - pool = self._pb_pool - else: - raise ValueError("invalid protocol %s" % protocol) - - # Replace with recovery logic later - with pool.take() as transport: - yield transport diff --git a/riak/client/operations.py b/riak/client/operations.py index f7ede8a6..90928b49 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -31,7 +31,7 @@ def get_buckets(self): NOTE: Do not use this in production, as it requires traversing through all keys stored in a cluster. """ - with self.transport() as transport: + with self._transport() as transport: return [RiakBucket(self, name) for name in transport.get_buckets()] def ping(self): @@ -40,7 +40,7 @@ def ping(self): :rtype: boolean """ - with self.transport() as transport: + with self._transport() as transport: return transport.ping() is_alive = ping @@ -49,28 +49,28 @@ def get_index(self, bucket, index, startkey, endkey=None): """ Queries a secondary index, returning matching keys. """ - with self.transport() as transport: + with self._transport() as transport: return transport.get_index(bucket, index, startkey, endkey) def get_bucket_props(self, bucket): """ Fetches bucket properties for the given bucket. """ - with self.transport() as transport: + with self._transport() as transport: return transport.get_bucket_props(bucket) def set_bucket_props(self, bucket, props): """ Sets bucket properties for the given bucket. """ - with self.transport() as transport: + with self._transport() as transport: return transport.set_bucket_props(bucket, props) def get_keys(self, bucket): """ Lists all keys in a bucket. """ - with self.transport() as transport: + with self._transport() as transport: return transport.get_keys(bucket) def stream_keys(self, bucket): @@ -78,7 +78,7 @@ def stream_keys(self, bucket): Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. """ - with self.transport() as transport: + with self._transport() as transport: for keylist in return transport.stream_keys(bucket): yield keylist @@ -87,7 +87,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=None, """ Stores an object in the Riak cluster. """ - with self.transport() as transport: + with self._transport() as transport: return transport.put(robj, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) @@ -97,7 +97,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=None, """ Stores an object in the Riak cluster with a generated key. """ - with self.transport() as transport: + with self._transport() as transport: return transport.put_new(robj, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) @@ -106,14 +106,14 @@ def get(self, robj, r=None, pr=None, vtag=None): """ Fetches the contents of a Riak object. """ - with self.transport() as transport: + with self._transport() as transport: return transport.get(robj, r=r, pr=pr, vtag=vtag) def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Deletes an object from Riak. """ - with self.transport() as transport: + with self._transport() as transport: return transport.delete(robj, rw=rw, r=r, w=w, dw=dw=, pr=pr, pw=pw) @@ -121,7 +121,7 @@ def mapred(self, inputs, query, timeout): """ Executes a MapReduce query """ - with self.transport() as transport: + with self._transport() as transport: return transport.mapred(inputs, query, timeout) def stream_mapred(self, inputs, query, timeout): @@ -129,7 +129,7 @@ def stream_mapred(self, inputs, query, timeout): Streams a MapReduce query as (phase, data) pairs. This is a generator method which should be iterated over. """ - with self.transport() as transport: + with self._transport() as transport: for phase, data in transport.stream_mapred(inputs, query, timeout): yield phase, data @@ -137,19 +137,19 @@ def fulltext_search(self, index, query, **params): """ Performs a full-text search query. """ - with self.transport() as transport: + with self._transport() as transport: return transport.search(index, query, **params) def fulltext_add(self, index, docs): """ Adds documents to the full-text index. """ - with self._http_pool.take() as transport: + with self._transport(protocol='http') as transport: transport.fulltext_add(self, index, docs) def fulltext_delete(self, index, docs=None, queries=None): """ Removes documents from the full-text index. """ - with self._http_pool.take() as transport: + with self._transport(protocol='http') as transport: transport.fulltext_delete(index, docs, queries) diff --git a/riak/client/transport.py b/riak/client/transport.py new file mode 100644 index 00000000..a1a6cb36 --- /dev/null +++ b/riak/client/transport.py @@ -0,0 +1,69 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from contextlib import contextmanager +from riak.transports.pool import BadResource +from riak.transports.pbc import is_retryable as is_pbc_retryable +from riak.transports.http import is_retryable as is_http_retryable +import httplib + + +class RiakClientTransport(class): + """ + Methods for RiakClient related to transport selection and retries. + """ + + @contextmanager + def _transport(self, protocol=self.protocol): + if protocol in ['http', 'https']: + pool = self._http_pool + elif protocol is 'pbc': + pool = self._pb_pool + else: + raise ValueError("invalid protocol %s" % protocol) + + with self._retryable(pool) as transport: + yield transport + + @contextmanager + def _retryable(self, pool): + skip_nodes = [] + # TODO: Make a property? + retries = 3 + + def _skip_bad_nodes(transport): + return transport.node not in skip_nodes + + while retries > 0: + try: + with pool.take(_filter=_skip_bad_nodes) as transport: + try: + yield transport + except (IOError, httplib.HTTPException) as e: + if is_pbc_retryable(e) or is_http_retryable(e): + retries -= 1 + transport.node.error_rate.incr(1) + skip_nodes.append(transport.node) + raise BadResource(e) + else: + raise e + except BadResource as br: + if retries > 0: + continue + else: + raise br.args[0] diff --git a/riak/transports/http.py b/riak/transports/http.py index 43f0b7e4..aaa8ec1c 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -755,3 +755,22 @@ def close(self): return {'num_found': self.num_found, 'max_score': self.max_score, 'docs': self.docs} + + +CONN_CLOSED_ERRORS = ( + httplib.NotConnected, + httplib.IncompleteRead, + httplib.ImproperConnectionState + ) + + +def is_retryable(err): + """ + Determines if the given exception is something that is + network/socket-related and should thus cause the HTTP connection + to close and the operation retried on another node. + """ + for errtype in CONN_CLOSED_ERRORS: + if isinstance(err, errtype): + return True + return False diff --git a/riak/transports/pbc/__init__.py b/riak/transports/pbc/__init__.py index d3620738..ef772777 100644 --- a/riak/transports/pbc/__init__.py +++ b/riak/transports/pbc/__init__.py @@ -19,7 +19,7 @@ under the License. """ - +import errno from riak.transports.pool import Pool from riak.transports.pbc.transport import RiakPbcTransport @@ -41,3 +41,31 @@ def create_resource(self): def destroy_resource(self, pbc): pbc.close() + +# These are a specific set of socket errors +# that could be raised on send/recv that indicate +# that the socket is closed or reset, and is not +# usable. On seeing any of these errors, the socket +# should be closed, and the connection re-established. +CONN_CLOSED_ERRORS = ( + errno.EHOSTUNREACH, + errno.ECONNRESET, + errno.ECONNREFUSED, + errno.ECONNABORTED, + errno.ETIMEDOUT, + errno.EBADF, + errno.EPIPE + ) + + +def is_retryable(err): + """ + Determines if the given exception is something that is + network/socket-related and should thus cause the PBC connection to + close and the operation retried on another node. + """ + if isinstance(err, socket.error): + code = err.args[0] + return code in CONN_CLOSED_ERRORS + else: + return False diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index b4fc5853..c8ba785f 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -20,18 +20,7 @@ import socket import struct from contextlib import contextmanager - -# These are a specific set of socket errors -# that could be raised on send/recv that indicate -# that the socket is closed or reset, and is not -# usable. On seeing any of these errors, the socket -# should be closed, and the connection re-established. -CONN_CLOSED_ERRORS = ( - errno.EHOSTUNREACH, - errno.ECONNRESET, - errno.EBADF, - errno.EPIPE - ) +from riak import RiakError class RiakPbcConnection(object): From 12f596c0a2edc1d44eb7d2f60fec3c5096d09fd3 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 15:13:07 -0500 Subject: [PATCH 0267/1060] The node is a private attribute on the transport. --- riak/client/transport.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/riak/client/transport.py b/riak/client/transport.py index a1a6cb36..924d00f3 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -47,7 +47,7 @@ def _retryable(self, pool): retries = 3 def _skip_bad_nodes(transport): - return transport.node not in skip_nodes + return transport._node not in skip_nodes while retries > 0: try: @@ -57,8 +57,8 @@ def _skip_bad_nodes(transport): except (IOError, httplib.HTTPException) as e: if is_pbc_retryable(e) or is_http_retryable(e): retries -= 1 - transport.node.error_rate.incr(1) - skip_nodes.append(transport.node) + transport._node.error_rate.incr(1) + skip_nodes.append(transport._node) raise BadResource(e) else: raise e From 618f75a57a83de2dc4d22d689cd4e889ea6c14c6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 15:42:38 -0500 Subject: [PATCH 0268/1060] Move http.py in preparation for refactoring. --- riak/transports/{http.py => http/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename riak/transports/{http.py => http/__init__.py} (100%) diff --git a/riak/transports/http.py b/riak/transports/http/__init__.py similarity index 100% rename from riak/transports/http.py rename to riak/transports/http/__init__.py From f924239c381354c5214631901f864a6ca23830c4 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 17:16:36 -0500 Subject: [PATCH 0269/1060] Start breaking out HTTP into multiple files. --- riak/transports/http/__init__.py | 711 +----------------------------- riak/transports/http/search.py | 48 ++ riak/transports/http/transport.py | 678 ++++++++++++++++++++++++++++ riak/transports/pbc/transport.py | 2 - 4 files changed, 728 insertions(+), 711 deletions(-) create mode 100644 riak/transports/http/search.py create mode 100644 riak/transports/http/transport.py diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index aaa8ec1c..478ebf5c 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -18,28 +18,8 @@ under the License. """ -import urllib -import re -import csv -from cStringIO import StringIO -import httplib -import socket -import errno -try: - import json -except ImportError: - import simplejson as json - -from riak.transports.transport import RiakTransport -from riak.transports.pool import Pool, BadResource -from riak.metadata import * -from riak.mapreduce import RiakLink -from riak import RiakError -from riak.riak_index_entry import RiakIndexEntry -from riak.multidict import MultiDict -import riak.util -from xml.etree import ElementTree -from xml.dom.minidom import Document +from riak.transports.pool import Pool +from riak.transports.http.transport import RiakHttpTransport # subtract length of "Link: " header string and newline @@ -70,693 +50,6 @@ def destroy_resource(self, transport): transport.close() -class RiakHttpTransport(RiakTransport): - """ - The RiakHttpTransport object holds information necessary to - connect to Riak via HTTP. - """ - - api = 3 - - def __init__(self, node=None, - client=None, - connection_class=httplib.HTTPConnection - client_id=None, - **unused_options): - """ - Construct a new HTTP connection to Riak. - """ - super(RiakHttpTransport, self).__init__() - - self._client = client - self._node = node - self._connection_class = connection_class - self._client_id = client_id - if not self._client_id: - self._client_id = self.make_random_client_id() - - def ping(self): - """ - Check server is alive over HTTP - """ - response = self.http_request('GET', '/ping') - return(response is not None) and (response[1] == 'OK') - - def stats(self): - """ - Gets performance statistics and server information - """ - # TODO: use resource detection - response = self.http_request('GET', '/stats', - {'Accept': 'application/json'}) - if response[0]['http_code'] is 200: - return json.loads(response[1]) - else: - return None - - # FeatureDetection API - private - def _server_version(self): - stats = self.stats() - if stats is not None: - return stats['riak_kv_version'] - # If stats is disabled, we can't assume the Riak version - # is >= 1.1. However, we can assume the new URL scheme is - # at least version 1.0 - elif 'riak_kv_wm_buckets' in self.get_resources(): - return "1.0.0" - else: - return "0.14.0" - - def get_resources(self): - """ - Gets a JSON mapping of server-side resource names to paths - :rtype dict - """ - response = self.http_request('GET', '/', - {'Accept': 'application/json'}) - if response[0]['http_code'] is 200: - return json.loads(response[1]) - else: - return {} - - def get(self, robj, r=None, pr=None, vtag=None): - """ - Get a bucket/key from the server - """ - # We could detect quorum_controls here but HTTP ignores - # unknown flags/params. - params = {'r': r, 'pr': pr} - if vtag is not None: - params['vtag'] = vtag - url = self.build_rest_path(robj.bucket, robj.key, - params=params) - response = self.http_request('GET', url) - return self.parse_body(response, [200, 300, 404]) - - def put(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """ - Serialize put request and deserialize response - """ - # We could detect quorum_controls here but HTTP ignores - # unknown flags/params. - params = {'returnbody': str(return_body).lower(), - 'w': w, 'dw': dw, 'pw': pw} - url = self.build_rest_path(bucket=robj.bucket, - key=robj.key, - params=params) - headers = self.build_put_headers(robj) - # TODO: use a more general 'prevent_stale_writes' semantics, - # which is a superset of the if_none_match semantics. - if if_none_match: - headers["If-None-Match"] = "*" - content = robj.get_encoded_data() - return self.do_put(url, headers, content, return_body, - key=robj.key) - - def do_put(self, url, headers, content, return_body=False, key=None): - if key is None: - response = self.http_request('POST', url, headers, content) - else: - response = self.http_request('PUT', url, headers, content) - - if return_body: - return self.parse_body(response, [200, 201, 300]) - else: - self.check_http_code(response, [204]) - return None - - def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """Put a new object into the Riak store, returning its (new) key.""" - # We could detect quorum_controls here but HTTP ignores - # unknown flags/params. - params = {'returnbody': str(return_body).lower(), 'w': w, 'dw': dw, - 'pw': pw} - url = self.build_rest_path(bucket=robj.bucket, params=params) - headers = self.build_put_headers(robj) - # TODO: use a more general 'prevent_stale_writes' semantics, - # which is a superset of the if_none_match semantics. - if if_none_match: - headers["If-None-Match"] = "*" - content = robj.get_encoded_data() - response = self.http_request('POST', url, headers, content) - location = response[0]['location'] - idx = location.rindex('/') - key = location[(idx + 1):] - if return_body: - vclock, [(metadata, data)] = self.parse_body(response, [201]) - return key, vclock, metadata - else: - self.check_http_code(response, [201]) - return key, None, None - - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): - """ - Delete an object. - """ - # We could detect quorum_controls here but HTTP ignores - # unknown flags/params. - params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} - headers = {} - url = self.build_rest_path(robj.bucket, robj.key, - params=params) - if self.tombstone_vclocks() and robj.vclock is not None: - headers['X-Riak-Vclock'] = robj.vclock - response = self.http_request('DELETE', url, headers) - self.check_http_code(response, [204, 404]) - return self - - def get_keys(self, bucket): - """ - Fetch a list of keys for the bucket - """ - params = {'props': 'True', 'keys': 'true'} - url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', url) - - headers, encoded_props = response[0:2] - if headers['http_code'] == 200: - props = json.loads(encoded_props) - return props['keys'] - else: - raise Exception('Error getting bucket properties.') - - def get_buckets(self): - """ - Fetch a list of all buckets - """ - params = {'buckets': 'true'} - url = self.build_rest_path(None, params=params) - response = self.http_request('GET', url) - - headers, encoded_props = response[0:2] - if headers['http_code'] == 200: - props = json.loads(encoded_props) - return props['buckets'] - else: - raise Exception('Error getting buckets.') - - def get_bucket_props(self, bucket): - """ - Get properties for a bucket - """ - # Run the request... - params = {'props': 'true', 'keys': 'false'} - url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', url) - - headers = response[0] - encoded_props = response[1] - if headers['http_code'] == 200: - props = json.loads(encoded_props) - return props['props'] - else: - raise Exception('Error getting bucket properties.') - - def set_bucket_props(self, bucket, props): - """ - Set the properties on the bucket object given - """ - url = self.build_rest_path(bucket) - headers = {'Content-Type': 'application/json'} - content = json.dumps({'props': props}) - - # Run the request... - response = self.http_request('PUT', url, headers, content) - - # Handle the response... - if response is None: - raise Exception('Error setting bucket properties.') - - # Check the response value... - status = response[0]['http_code'] - if status != 204: - raise Exception('Error setting bucket properties.') - return True - - def mapred(self, inputs, query, timeout=None): - """ - Run a MapReduce query. - """ - if not self.phaseless_mapred() and (query is None or len(query) is 0): - raise Exception( - 'Phase-less MapReduce is not supported by Riak node') - - # Construct the job, optionally set the timeout... - job = {'inputs': inputs, 'query': query} - if timeout is not None: - job['timeout'] = timeout - - content = json.dumps(job) - - # Do the request... - url = "/" + self._mapred_prefix - headers = {'Content-Type': 'application/json'} - response = self.http_request('POST', url, headers, content) - - # Make sure the expected status code came back... - status = response[0]['http_code'] - if status != 200: - raise Exception( - 'Error running MapReduce operation. Headers: %s Body: %s' % - (repr(response[0]), repr(response[1]))) - - result = json.loads(response[1]) - return result - - def get_index(self, bucket, index, startkey, endkey=None): - """ - Performs a secondary index query. - """ - # TODO: use resource detection - segments = ["buckets", bucket, "index", index, str(startkey)] - if endkey: - segments.append(str(endkey)) - uri = '/%s' % ('/'.join(segments)) - headers, data = response = self.get_request(uri) - self.check_http_code(response, [200]) - jsonData = json.loads(data) - return jsonData[u'keys'][:] - - def search(self, index, query, **params): - """ - Performs a search query. - """ - if index is None: - index = 'search' - - options = {'q': query, 'wt': 'json'} - if 'op' in params: - op = params.pop('op') - options['q.op'] = op - - options.update(params) - # TODO: use resource detection - uri = "/solr/%s/select" % index - headers, data = response = self.get_request(uri, options) - self.check_http_code(response, [200]) - if 'json' in headers['content-type']: - results = json.loads(data) - return self._normalize_json_search_response(results) - elif 'xml' in headers['content-type']: - return self._normalize_xml_search_response(data) - else: - raise ValueError("Could not decode search response") - - def fulltext_add(self, index, docs): - """ - Adds documents to the search index. - """ - xml = Document() - root = xml.createElement('add') - for doc in docs: - doc_element = xml.createElement('doc') - for key in doc: - value = doc[key] - field = xml.createElement('field') - field.setAttribute("name", key) - text = xml.createTextNode(value) - field.appendChild(text) - doc_element.appendChild(field) - root.appendChild(doc_element) - xml.appendChild(root) - - url = "/solr/%s/update" % index - self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") - - def fulltext_delete(self, index, docs=None, queries=None): - """ - Removes documents from the full-text index. - """ - xml = Document() - root = xml.createElement('delete') - if docs: - for doc in docs: - doc_element = xml.createElement('id') - text = xml.createTextNode(doc) - doc_element.appendChild(text) - root.appendChild(doc_element) - if queries: - for query in queries: - query_element = xml.createElement('query') - text = xml.createTextNode(query) - query_element.appendChild(text) - root.appendChild(query_element) - - xml.appendChild(root) - - url = "/solr/%s/update" % index - self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") - - def check_http_code(self, response, expected_statuses): - status = response[0]['http_code'] - if not status in expected_statuses: - raise Exception('Expected status %s, received %s : %s' % - (expected_statuses, status, response[1])) - - def parse_body(self, response, expected_statuses): - """ - Given the output of RiakUtils.http_request and a list of - statuses, populate the object. Only for use by the Riak client - library. - @return self - """ - # If no response given, then return. - if response is None: - return self - - # Make sure expected code came back - self.check_http_code(response, expected_statuses) - - # Update the object... - headers = response[0] - data = response[1] - status = headers['http_code'] - - # Check if the server is down(status==0) - if not status: - ### we need the host/port that was used. - m = 'Could not contact Riak Server: http://$HOST:$PORT !' - raise RiakError(m) - - # If 404(Not Found), then clear the object. - if status == 404: - return None - - # If 300(Siblings), then return the list of siblings - elif status == 300: - # Parse and get rid of 'Siblings:' string in element 0 - siblings = data.strip().split('\n') - siblings.pop(0) - return siblings - - # Parse the headers... - vclock = None - metadata = {MD_USERMETA: {}, MD_INDEX: []} - links = [] - for header, value in headers.iteritems(): - if header == 'content-type': - metadata[MD_CTYPE] = value - elif header == 'charset': - metadata[MD_CHARSET] = value - elif header == 'content-encoding': - metadata[MD_CTYPE] = value - elif header == 'etag': - metadata[MD_VTAG] = value - elif header == 'link': - self.parse_links(links, headers['link']) - elif header == 'last-modified': - metadata[MD_LASTMOD] = value - elif header.startswith('x-riak-meta-'): - metakey = header.replace('x-riak-meta-', '') - metadata[MD_USERMETA][metakey] = value - elif header.startswith('x-riak-index-'): - field = header.replace('x-riak-index-', '') - reader = csv.reader([value], skipinitialspace=True) - for line in reader: - for token in line: - rie = RiakIndexEntry(field, token) - metadata[MD_INDEX].append(rie) - elif header == 'x-riak-vclock': - vclock = value - elif header == 'x-riak-deleted': - metadata[MD_DELETED] = True - if links: - metadata[MD_LINKS] = links - - return vclock, [(metadata, data)] - - def to_link_header(self, link): - """ - Convert this RiakLink object to a link header string. Used internally. - """ - header = '' - header += '; riaktag="' - header += urllib.quote_plus(link.get_tag()) + '"' - return header - - def parse_links(self, links, linkHeaders): - """ - Private. - @return self - """ - oldform = "; ?riaktag=\"([^\']+)\"" - newform = "; ?riaktag=\"([^\']+)\"" - for linkHeader in linkHeaders.strip().split(','): - linkHeader = linkHeader.strip() - matches = (re.match(oldform, linkHeader) or - re.match(newform, linkHeader)) - if matches is not None: - link = RiakLink(urllib.unquote_plus(matches.group(2)), - urllib.unquote_plus(matches.group(3)), - urllib.unquote_plus(matches.group(4))) - links.append(link) - return self - - def add_links_for_riak_object(self, robject, headers): - links = robject.get_links() - if links: - current_header = '' - for link in links: - header = self.to_link_header(link) - if len(current_header + header) > MAX_LINK_HEADER_SIZE: - headers.add('Link', current_header) - current_header = '' - - if current_header != '': - header = ', ' + header - current_header += header - - headers.add('Link', current_header) - - return headers - - def get_request(self, uri=None, params=None): - url = self.build_rest_path(bucket=None, params=params, prefix=uri) - return self.http_request('GET', url) - - def post_request(self, uri=None, body=None, params=None, - content_type="application/json"): - uri = self.build_rest_path(prefix=uri, params=params) - return self.http_request('POST', uri, {'Content-Type': content_type}, - body) - - # Utility functions used by Riak library. - - def build_rest_path(self, bucket=None, key=None, params=None, prefix=None): - """ - Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, - construct and return a URL. - """ - # Build 'http://hostname:port/prefix/bucket' - path = '' - path += '/' + (prefix or self._prefix) - - # Add '.../bucket' - if bucket is not None: - path += '/' + urllib.quote_plus(bucket.name) - - # Add '.../key' - if key is not None: - path += '/' + urllib.quote_plus(key) - - # Add query parameters. - if params is not None: - s = '' - for key in params.keys(): - if params[key] is not None: - if s != '': - s += '&' - s += (urllib.quote_plus(key) + '=' + - urllib.quote_plus(str(params[key]))) - path += '?' + s - - # Return. - return path - - def build_put_headers(self, robj): - """Build the headers for a POST/PUT request.""" - - # Construct the headers... - headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', - 'Content-Type': robj.content_type, - 'X-Riak-ClientId': self._client_id}) - - # Add the vclock if it exists... - if robj.vclock is not None: - headers['X-Riak-Vclock'] = robj.vclock - - # Create the header from metadata - links = self.add_links_for_riak_object(robj, headers) - - for key, value in robj.usermeta.iteritems(): - headers['X-Riak-Meta-%s' % key] = value - - for rie in robj.get_indexes(): - key = 'X-Riak-Index-%s' % rie.get_field() - if key in headers: - headers[key] += ", " + rie.get_value() - else: - headers[key] = rie.get_value() - - return headers - - def http_request(self, method, uri, headers=None, body=''): - """ - Given a Method, URL, Headers, and Body, perform and HTTP request, - and return a 2-tuple containing a dictionary of response headers - and the response body. - """ - if headers is None: - headers = {} - # Run the request... - for retry in range(self.RETRY_COUNT): - with self._conns.withconn() as conn: - ### should probably build this try/except into a custom - ### contextmanager for the connection. - try: - conn.request(method, uri, body, headers) - response = conn.getresponse() - - try: - # Get the response headers... - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value - - # Get the body... - response_body = response.read() - finally: - response.close() - - return response_headers, response_body - except socket.error, e: - conn.close() - if e[0] == errno.ECONNRESET: - # Grab another connection and try again. - continue - # Don't know how to handle this. - raise - except httplib.HTTPException: - # Just close the connection and try again. - conn.close() - continue - - # No luck, even with retrying. - raise RiakError("could not get a response") - - def _normalize_json_search_response(self, json): - """ - Normalizes a JSON search response so that PB and HTTP have the - same return value - """ - result = {} - if u'response' in json: - result['num_found'] = json[u'response'][u'numFound'] - result['max_score'] = float(json[u'response'][u'maxScore']) - docs = [] - for doc in json[u'response'][u'docs']: - resdoc = {u'id': doc[u'id']} - if u'fields' in doc: - for k, v in doc[u'fields'].iteritems(): - resdoc[k] = v - docs.append(resdoc) - result['docs'] = docs - return result - - def _normalize_xml_search_response(self, xml): - """ - Normalizes an XML search response so that PB and HTTP have the - same return value - """ - target = XMLSearchResult() - parser = ElementTree.XMLParser(target=target) - parser.feed(xml) - return parser.close() - - @classmethod - def build_headers(cls, headers): - return ['%s: %s' % (header, value) - for header, value in headers.iteritems()] - - @classmethod - def parse_http_headers(cls, headers): - """ - Parse an HTTP Header string into an asssociative array of - response headers. - """ - retVal = {} - fields = headers.split("\n") - for field in fields: - matches = re.match("([^:]+):(.+)", field) - if matches is None: - continue - key = matches.group(1).lower() - value = matches.group(2).strip() - if key in retVal.keys(): - if isinstance(retVal[key], list): - retVal[key].append(value) - else: - retVal[key] = [retVal[key]].append(value) - else: - retVal[key] = value - return retVal - - -class XMLSearchResult(object): - # Match tags that are document fields - fieldtags = ['str', 'int', 'date'] - - def __init__(self): - # Results - self.num_found = 0 - self.max_score = 0.0 - self.docs = [] - - # Parser state - self.currdoc = None - self.currfield = None - self.currvalue = None - - def start(self, tag, attrib): - if tag == 'result': - self.num_found = int(attrib['numFound']) - self.max_score = float(attrib['maxScore']) - elif tag == 'doc': - self.currdoc = {} - elif tag in self.fieldtags and self.currdoc is not None: - self.currfield = attrib['name'] - - def end(self, tag): - if tag == 'doc' and self.currdoc is not None: - self.docs.append(self.currdoc) - self.currdoc = None - elif tag in self.fieldtags and self.currdoc is not None: - if tag == 'int': - self.currvalue = int(self.currvalue) - self.currdoc[self.currfield] = self.currvalue - self.currfield = None - self.currvalue = None - - def data(self, data): - if self.currfield: - # riak_solr_output adds NL + 6 spaces - data = data.rstrip() - if self.currvalue: - self.currvalue += data - else: - self.currvalue = data - - def close(self): - return {'num_found': self.num_found, - 'max_score': self.max_score, - 'docs': self.docs} - - CONN_CLOSED_ERRORS = ( httplib.NotConnected, httplib.IncompleteRead, diff --git a/riak/transports/http/search.py b/riak/transports/http/search.py new file mode 100644 index 00000000..4e6c69e6 --- /dev/null +++ b/riak/transports/http/search.py @@ -0,0 +1,48 @@ +class XMLSearchResult(object): + # Match tags that are document fields + fieldtags = ['str', 'int', 'date'] + + def __init__(self): + # Results + self.num_found = 0 + self.max_score = 0.0 + self.docs = [] + + # Parser state + self.currdoc = None + self.currfield = None + self.currvalue = None + + def start(self, tag, attrib): + if tag == 'result': + self.num_found = int(attrib['numFound']) + self.max_score = float(attrib['maxScore']) + elif tag == 'doc': + self.currdoc = {} + elif tag in self.fieldtags and self.currdoc is not None: + self.currfield = attrib['name'] + + def end(self, tag): + if tag == 'doc' and self.currdoc is not None: + self.docs.append(self.currdoc) + self.currdoc = None + elif tag in self.fieldtags and self.currdoc is not None: + if tag == 'int': + self.currvalue = int(self.currvalue) + self.currdoc[self.currfield] = self.currvalue + self.currfield = None + self.currvalue = None + + def data(self, data): + if self.currfield: + # riak_solr_output adds NL + 6 spaces + data = data.rstrip() + if self.currvalue: + self.currvalue += data + else: + self.currvalue = data + + def close(self): + return {'num_found': self.num_found, + 'max_score': self.max_score, + 'docs': self.docs} diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py new file mode 100644 index 00000000..0104e510 --- /dev/null +++ b/riak/transports/http/transport.py @@ -0,0 +1,678 @@ +""" +Copyright 2012 Basho Technologies, Inc. +Copyright 2010 Rusty Klophaus +Copyright 2010 Justin Sheehy +Copyright 2009 Jay Baird + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +try: + import json +except ImportError: + import simplejson as json + +import urllib +import re +import csv +from cStringIO import StringIO +import httplib +import socket +import errno +from riak.transports.transport import RiakTransport +from riak.transports.http.search import XMLSearchResult +from riak.metadata import * +from riak.mapreduce import RiakLink +from riak import RiakError +from riak.riak_index_entry import RiakIndexEntry +from riak.multidict import MultiDict +import riak.util +from xml.etree import ElementTree +from xml.dom.minidom import Document + + +class RiakHttpTransport(RiakTransport): + """ + The RiakHttpTransport object holds information necessary to + connect to Riak via HTTP. + """ + + def __init__(self, node=None, + client=None, + connection_class=httplib.HTTPConnection + client_id=None, + **unused_options): + """ + Construct a new HTTP connection to Riak. + """ + super(RiakHttpTransport, self).__init__() + + self._client = client + self._node = node + self._connection_class = connection_class + self._client_id = client_id + if not self._client_id: + self._client_id = self.make_random_client_id() + + def ping(self): + """ + Check server is alive over HTTP + """ + response = self.http_request('GET', '/ping') + return(response is not None) and (response[1] == 'OK') + + def stats(self): + """ + Gets performance statistics and server information + """ + # TODO: use resource detection + response = self.http_request('GET', '/stats', + {'Accept': 'application/json'}) + if response[0]['http_code'] is 200: + return json.loads(response[1]) + else: + return None + + # FeatureDetection API - private + def _server_version(self): + stats = self.stats() + if stats is not None: + return stats['riak_kv_version'] + # If stats is disabled, we can't assume the Riak version + # is >= 1.1. However, we can assume the new URL scheme is + # at least version 1.0 + elif 'riak_kv_wm_buckets' in self.get_resources(): + return "1.0.0" + else: + return "0.14.0" + + def get_resources(self): + """ + Gets a JSON mapping of server-side resource names to paths + :rtype dict + """ + response = self.http_request('GET', '/', + {'Accept': 'application/json'}) + if response[0]['http_code'] is 200: + return json.loads(response[1]) + else: + return {} + + def get(self, robj, r=None, pr=None, vtag=None): + """ + Get a bucket/key from the server + """ + # We could detect quorum_controls here but HTTP ignores + # unknown flags/params. + params = {'r': r, 'pr': pr} + if vtag is not None: + params['vtag'] = vtag + url = self.build_rest_path(robj.bucket, robj.key, + params=params) + response = self.http_request('GET', url) + return self.parse_body(response, [200, 300, 404]) + + def put(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): + """ + Serialize put request and deserialize response + """ + # We could detect quorum_controls here but HTTP ignores + # unknown flags/params. + params = {'returnbody': str(return_body).lower(), + 'w': w, 'dw': dw, 'pw': pw} + url = self.build_rest_path(bucket=robj.bucket, + key=robj.key, + params=params) + headers = self.build_put_headers(robj) + # TODO: use a more general 'prevent_stale_writes' semantics, + # which is a superset of the if_none_match semantics. + if if_none_match: + headers["If-None-Match"] = "*" + content = robj.get_encoded_data() + return self.do_put(url, headers, content, return_body, + key=robj.key) + + def do_put(self, url, headers, content, return_body=False, key=None): + if key is None: + response = self.http_request('POST', url, headers, content) + else: + response = self.http_request('PUT', url, headers, content) + + if return_body: + return self.parse_body(response, [200, 201, 300]) + else: + self.check_http_code(response, [204]) + return None + + def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): + """Put a new object into the Riak store, returning its (new) key.""" + # We could detect quorum_controls here but HTTP ignores + # unknown flags/params. + params = {'returnbody': str(return_body).lower(), 'w': w, 'dw': dw, + 'pw': pw} + url = self.build_rest_path(bucket=robj.bucket, params=params) + headers = self.build_put_headers(robj) + # TODO: use a more general 'prevent_stale_writes' semantics, + # which is a superset of the if_none_match semantics. + if if_none_match: + headers["If-None-Match"] = "*" + content = robj.get_encoded_data() + response = self.http_request('POST', url, headers, content) + location = response[0]['location'] + idx = location.rindex('/') + key = location[(idx + 1):] + if return_body: + vclock, [(metadata, data)] = self.parse_body(response, [201]) + return key, vclock, metadata + else: + self.check_http_code(response, [201]) + return key, None, None + + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + """ + Delete an object. + """ + # We could detect quorum_controls here but HTTP ignores + # unknown flags/params. + params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} + headers = {} + url = self.build_rest_path(robj.bucket, robj.key, + params=params) + if self.tombstone_vclocks() and robj.vclock is not None: + headers['X-Riak-Vclock'] = robj.vclock + response = self.http_request('DELETE', url, headers) + self.check_http_code(response, [204, 404]) + return self + + def get_keys(self, bucket): + """ + Fetch a list of keys for the bucket + """ + params = {'props': 'True', 'keys': 'true'} + url = self.build_rest_path(bucket, params=params) + response = self.http_request('GET', url) + + headers, encoded_props = response[0:2] + if headers['http_code'] == 200: + props = json.loads(encoded_props) + return props['keys'] + else: + raise Exception('Error getting bucket properties.') + + def get_buckets(self): + """ + Fetch a list of all buckets + """ + params = {'buckets': 'true'} + url = self.build_rest_path(None, params=params) + response = self.http_request('GET', url) + + headers, encoded_props = response[0:2] + if headers['http_code'] == 200: + props = json.loads(encoded_props) + return props['buckets'] + else: + raise Exception('Error getting buckets.') + + def get_bucket_props(self, bucket): + """ + Get properties for a bucket + """ + # Run the request... + params = {'props': 'true', 'keys': 'false'} + url = self.build_rest_path(bucket, params=params) + response = self.http_request('GET', url) + + headers = response[0] + encoded_props = response[1] + if headers['http_code'] == 200: + props = json.loads(encoded_props) + return props['props'] + else: + raise Exception('Error getting bucket properties.') + + def set_bucket_props(self, bucket, props): + """ + Set the properties on the bucket object given + """ + url = self.build_rest_path(bucket) + headers = {'Content-Type': 'application/json'} + content = json.dumps({'props': props}) + + # Run the request... + response = self.http_request('PUT', url, headers, content) + + # Handle the response... + if response is None: + raise Exception('Error setting bucket properties.') + + # Check the response value... + status = response[0]['http_code'] + if status != 204: + raise Exception('Error setting bucket properties.') + return True + + def mapred(self, inputs, query, timeout=None): + """ + Run a MapReduce query. + """ + if not self.phaseless_mapred() and (query is None or len(query) is 0): + raise Exception( + 'Phase-less MapReduce is not supported by Riak node') + + # Construct the job, optionally set the timeout... + job = {'inputs': inputs, 'query': query} + if timeout is not None: + job['timeout'] = timeout + + content = json.dumps(job) + + # Do the request... + url = "/" + self._mapred_prefix + headers = {'Content-Type': 'application/json'} + response = self.http_request('POST', url, headers, content) + + # Make sure the expected status code came back... + status = response[0]['http_code'] + if status != 200: + raise Exception( + 'Error running MapReduce operation. Headers: %s Body: %s' % + (repr(response[0]), repr(response[1]))) + + result = json.loads(response[1]) + return result + + def get_index(self, bucket, index, startkey, endkey=None): + """ + Performs a secondary index query. + """ + # TODO: use resource detection + segments = ["buckets", bucket, "index", index, str(startkey)] + if endkey: + segments.append(str(endkey)) + uri = '/%s' % ('/'.join(segments)) + headers, data = response = self.get_request(uri) + self.check_http_code(response, [200]) + jsonData = json.loads(data) + return jsonData[u'keys'][:] + + def search(self, index, query, **params): + """ + Performs a search query. + """ + if index is None: + index = 'search' + + options = {'q': query, 'wt': 'json'} + if 'op' in params: + op = params.pop('op') + options['q.op'] = op + + options.update(params) + # TODO: use resource detection + uri = "/solr/%s/select" % index + headers, data = response = self.get_request(uri, options) + self.check_http_code(response, [200]) + if 'json' in headers['content-type']: + results = json.loads(data) + return self._normalize_json_search_response(results) + elif 'xml' in headers['content-type']: + return self._normalize_xml_search_response(data) + else: + raise ValueError("Could not decode search response") + + def fulltext_add(self, index, docs): + """ + Adds documents to the search index. + """ + xml = Document() + root = xml.createElement('add') + for doc in docs: + doc_element = xml.createElement('doc') + for key in doc: + value = doc[key] + field = xml.createElement('field') + field.setAttribute("name", key) + text = xml.createTextNode(value) + field.appendChild(text) + doc_element.appendChild(field) + root.appendChild(doc_element) + xml.appendChild(root) + + url = "/solr/%s/update" % index + self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") + + def fulltext_delete(self, index, docs=None, queries=None): + """ + Removes documents from the full-text index. + """ + xml = Document() + root = xml.createElement('delete') + if docs: + for doc in docs: + doc_element = xml.createElement('id') + text = xml.createTextNode(doc) + doc_element.appendChild(text) + root.appendChild(doc_element) + if queries: + for query in queries: + query_element = xml.createElement('query') + text = xml.createTextNode(query) + query_element.appendChild(text) + root.appendChild(query_element) + + xml.appendChild(root) + + url = "/solr/%s/update" % index + self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") + + def check_http_code(self, response, expected_statuses): + status = response[0]['http_code'] + if not status in expected_statuses: + raise Exception('Expected status %s, received %s : %s' % + (expected_statuses, status, response[1])) + + def parse_body(self, response, expected_statuses): + """ + Given the output of RiakUtils.http_request and a list of + statuses, populate the object. Only for use by the Riak client + library. + @return self + """ + # If no response given, then return. + if response is None: + return self + + # Make sure expected code came back + self.check_http_code(response, expected_statuses) + + # Update the object... + headers = response[0] + data = response[1] + status = headers['http_code'] + + # Check if the server is down(status==0) + if not status: + ### we need the host/port that was used. + m = 'Could not contact Riak Server: http://$HOST:$PORT !' + raise RiakError(m) + + # If 404(Not Found), then clear the object. + if status == 404: + return None + + # If 300(Siblings), then return the list of siblings + elif status == 300: + # Parse and get rid of 'Siblings:' string in element 0 + siblings = data.strip().split('\n') + siblings.pop(0) + return siblings + + # Parse the headers... + vclock = None + metadata = {MD_USERMETA: {}, MD_INDEX: []} + links = [] + for header, value in headers.iteritems(): + if header == 'content-type': + metadata[MD_CTYPE] = value + elif header == 'charset': + metadata[MD_CHARSET] = value + elif header == 'content-encoding': + metadata[MD_CTYPE] = value + elif header == 'etag': + metadata[MD_VTAG] = value + elif header == 'link': + self.parse_links(links, headers['link']) + elif header == 'last-modified': + metadata[MD_LASTMOD] = value + elif header.startswith('x-riak-meta-'): + metakey = header.replace('x-riak-meta-', '') + metadata[MD_USERMETA][metakey] = value + elif header.startswith('x-riak-index-'): + field = header.replace('x-riak-index-', '') + reader = csv.reader([value], skipinitialspace=True) + for line in reader: + for token in line: + rie = RiakIndexEntry(field, token) + metadata[MD_INDEX].append(rie) + elif header == 'x-riak-vclock': + vclock = value + elif header == 'x-riak-deleted': + metadata[MD_DELETED] = True + if links: + metadata[MD_LINKS] = links + + return vclock, [(metadata, data)] + + def to_link_header(self, link): + """ + Convert this RiakLink object to a link header string. Used internally. + """ + header = '' + header += '; riaktag="' + header += urllib.quote_plus(link.get_tag()) + '"' + return header + + def parse_links(self, links, linkHeaders): + """ + Private. + @return self + """ + oldform = "; ?riaktag=\"([^\']+)\"" + newform = "; ?riaktag=\"([^\']+)\"" + for linkHeader in linkHeaders.strip().split(','): + linkHeader = linkHeader.strip() + matches = (re.match(oldform, linkHeader) or + re.match(newform, linkHeader)) + if matches is not None: + link = RiakLink(urllib.unquote_plus(matches.group(2)), + urllib.unquote_plus(matches.group(3)), + urllib.unquote_plus(matches.group(4))) + links.append(link) + return self + + def add_links_for_riak_object(self, robject, headers): + links = robject.get_links() + if links: + current_header = '' + for link in links: + header = self.to_link_header(link) + if len(current_header + header) > MAX_LINK_HEADER_SIZE: + headers.add('Link', current_header) + current_header = '' + + if current_header != '': + header = ', ' + header + current_header += header + + headers.add('Link', current_header) + + return headers + + def get_request(self, uri=None, params=None): + url = self.build_rest_path(bucket=None, params=params, prefix=uri) + return self.http_request('GET', url) + + def post_request(self, uri=None, body=None, params=None, + content_type="application/json"): + uri = self.build_rest_path(prefix=uri, params=params) + return self.http_request('POST', uri, {'Content-Type': content_type}, + body) + + # Utility functions used by Riak library. + + def build_rest_path(self, bucket=None, key=None, params=None, prefix=None): + """ + Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, + construct and return a URL. + """ + # Build 'http://hostname:port/prefix/bucket' + path = '' + path += '/' + (prefix or self._prefix) + + # Add '.../bucket' + if bucket is not None: + path += '/' + urllib.quote_plus(bucket.name) + + # Add '.../key' + if key is not None: + path += '/' + urllib.quote_plus(key) + + # Add query parameters. + if params is not None: + s = '' + for key in params.keys(): + if params[key] is not None: + if s != '': + s += '&' + s += (urllib.quote_plus(key) + '=' + + urllib.quote_plus(str(params[key]))) + path += '?' + s + + # Return. + return path + + def build_put_headers(self, robj): + """Build the headers for a POST/PUT request.""" + + # Construct the headers... + headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', + 'Content-Type': robj.content_type, + 'X-Riak-ClientId': self._client_id}) + + # Add the vclock if it exists... + if robj.vclock is not None: + headers['X-Riak-Vclock'] = robj.vclock + + # Create the header from metadata + links = self.add_links_for_riak_object(robj, headers) + + for key, value in robj.usermeta.iteritems(): + headers['X-Riak-Meta-%s' % key] = value + + for rie in robj.get_indexes(): + key = 'X-Riak-Index-%s' % rie.get_field() + if key in headers: + headers[key] += ", " + rie.get_value() + else: + headers[key] = rie.get_value() + + return headers + + def http_request(self, method, uri, headers=None, body=''): + """ + Given a Method, URL, Headers, and Body, perform and HTTP request, + and return a 2-tuple containing a dictionary of response headers + and the response body. + """ + if headers is None: + headers = {} + # Run the request... + for retry in range(self.RETRY_COUNT): + with self._conns.withconn() as conn: + ### should probably build this try/except into a custom + ### contextmanager for the connection. + try: + conn.request(method, uri, body, headers) + response = conn.getresponse() + + try: + # Get the response headers... + response_headers = {'http_code': response.status} + for (key, value) in response.getheaders(): + response_headers[key.lower()] = value + + # Get the body... + response_body = response.read() + finally: + response.close() + + return response_headers, response_body + except socket.error, e: + conn.close() + if e[0] == errno.ECONNRESET: + # Grab another connection and try again. + continue + # Don't know how to handle this. + raise + except httplib.HTTPException: + # Just close the connection and try again. + conn.close() + continue + + # No luck, even with retrying. + raise RiakError("could not get a response") + + def _normalize_json_search_response(self, json): + """ + Normalizes a JSON search response so that PB and HTTP have the + same return value + """ + result = {} + if u'response' in json: + result['num_found'] = json[u'response'][u'numFound'] + result['max_score'] = float(json[u'response'][u'maxScore']) + docs = [] + for doc in json[u'response'][u'docs']: + resdoc = {u'id': doc[u'id']} + if u'fields' in doc: + for k, v in doc[u'fields'].iteritems(): + resdoc[k] = v + docs.append(resdoc) + result['docs'] = docs + return result + + def _normalize_xml_search_response(self, xml): + """ + Normalizes an XML search response so that PB and HTTP have the + same return value + """ + target = XMLSearchResult() + parser = ElementTree.XMLParser(target=target) + parser.feed(xml) + return parser.close() + + @classmethod + def build_headers(cls, headers): + return ['%s: %s' % (header, value) + for header, value in headers.iteritems()] + + @classmethod + def parse_http_headers(cls, headers): + """ + Parse an HTTP Header string into an asssociative array of + response headers. + """ + retVal = {} + fields = headers.split("\n") + for field in fields: + matches = re.match("([^:]+):(.+)", field) + if matches is None: + continue + key = matches.group(1).lower() + value = matches.group(2).strip() + if key in retVal.keys(): + if isinstance(retVal[key], list): + retVal[key].append(value) + else: + retVal[key] = [retVal[key]].append(value) + else: + retVal[key] = value + return retVal diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 23730cf5..6cd96e38 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -41,8 +41,6 @@ class RiakPbcTransport(RiakTransport, RiakPbcConnection, RiakPbcCodec): buffers interface on the riak server. """ - api = 3 - def __init__(self, node=None, client=None, connect_timeout=None, request_timeout=None, **unused_options): """ From 47cc10d1dd2dc5ef2ad8809ea404786b9dbbcdb2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 17:17:21 -0500 Subject: [PATCH 0270/1060] Add resource URL path generator methods, not wired up. --- riak/transports/http/resources.py | 168 ++++++++++++++++++++++++++++++ riak/transports/http/transport.py | 3 +- 2 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 riak/transports/http/resources.py diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py new file mode 100644 index 00000000..ccbfe896 --- /dev/null +++ b/riak/transports/http/resources.py @@ -0,0 +1,168 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import re +from urllib import quote_plus +from riak import RiakError +from riak.util import lazy_property + +class RiakHttpResources(object): + """ + Methods for RiakHttpTransport related to URL generation, i.e. + creating the proper paths. + """ + + def ping_path(self): + return mkpath(self.riak_kv_wm_ping) + + def stats_path(self): + return mkpath(self.riak_kv_wm_stats) + + def mapred_path(self, **options): + return mkpath(self.kv_wm_mapred, **options) + + def bucket_list_path(self, **options): + query = options.copy() + query.update(buckets=True) + if self.riak_kv_wm_buckets: + return mkpath(self.riak_kv_wm_buckets, **query) + else: + return mkpath(self.riak_kv_wm_raw, **query) + + def bucket_properties_path(self, bucket, **options): + if self.riak_kv_wm_buckets: + return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), + "props", **options) + else: + query = options.copy() + query.update(props=True, keys=True) + return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), **query) + + def key_list_path(self, bucket, **options): + query = options.copy() + query.update(keys=True, props=False) + if self.riak_kv_wm_buckets: + return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), "keys", + **query) + else: + return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), **query) + + def object_path(self, bucket, key=None, **options): + if key: + key = quote_plus(key) + if self.riak_kv_wm_buckets: + return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), "keys", + key, **options) + else: + return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), key, + **options) + + # TODO: link_walk_path is undefined here because there is no path + # to it in the client without using MapReduce. + + def index_path(self, bucket, index, start, finish=None, **options): + if not self.riak_kv_wm_buckets: + raise RiakError("Indexes are unsupported by this Riak node") + if finish: + finish = quote_plus(str(finish)) + return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), + "index", quote_plus(str(start)), finish, **options) + + def solr_select_path(self, index, query, **options): + if not self.riak_solr_searcher_wm: + raise RiakError("Riak Search is unsupported by this Riak node") + qs = {'q': query, 'wt': 'json'} + qs.update(options) + if index: + index = quote_plus(index) + return mkpath(self.riak_solr_searcher_wm, index, "select", **qs) + + def solr_update_path(self, index): + if not self.riak_solr_searcher_wm: + raise RiakError("Riak Search is unsupported by this Riak node") + if index: + index = quote_plus(index) + return mkpath(self.riak_solr_indexer_wm, index, "update") + + def luwak_path(self, key=None): + if not self.luwak_wm_file: + raise RiakError("Luwak is unsupported by this Riak node") + if key: + key = quote_plus(key) + return mkpath(self.luwak_wm_file, key) + + @lazy_property + def riak_kv_wm_buckets(self): + return self.resources.get('riak_kv_wm_buckets') + + @lazy_property + def riak_kv_wm_raw(self): + return self.resources.get('riak_kv_wm_raw') or "/riak" + + @lazy_property + def riak_kv_wm_link_walker(self): + return self.resources.get('riak_kv_wm_linkwalker') or "/riak" + + @lazy_property + def riak_kv_wm_mapred(self): + return self.resources.get('riak_kv_wm_mapred') or "/mapred" + + @lazy_property + def riak_kv_wm_ping(self): + return self.resources.get('riak_kv_wm_ping') or "/ping" + + @lazy_property + def riak_kv_wm_stats(self): + return self.resources.get('riak_kv_wm_stats') or "/stats" + + @lazy_property + def riak_solr_searcher_wm(self): + return self.resources.get('riak_solr_searcher_wm') + + @lazy_property + def riak_solr_indexer_wm(self): + return self.resources.get('riak_solr_indexer_wm') + + @lazy_property + def luwak_wm_file(self): + return self.resources.get('luwak_wm_file') + + @lazy_property + def resources(self): + return self.get_resources() + + +def mkpath(*segments, **query): + """ + Constructs the path & query portion of a URI from path segments + and a dict. + """ + # Remove empty segments (e.g. no key specified) + segments = [s for s in segments if s is not None] + # Join the segments into a path + pathstring = '/'.join(segments) + # Remove extra slashes + pathstring = re.sub('/+', '/', pathstring) + # Add the query string if it exists + if len(query) > 0: + pathstring += "?" + urllib.urlencode(query).lower() + + if not pathstring.startswith('/'): + pathstring = '/' + pathstring + + return pathstring diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 0104e510..8af2e940 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -32,6 +32,7 @@ import socket import errno from riak.transports.transport import RiakTransport +from riak.transports.http.resources import RiakHttpResources from riak.transports.http.search import XMLSearchResult from riak.metadata import * from riak.mapreduce import RiakLink @@ -43,7 +44,7 @@ from xml.dom.minidom import Document -class RiakHttpTransport(RiakTransport): +class RiakHttpTransport(RiakHttpResources, RiakTransport): """ The RiakHttpTransport object holds information necessary to connect to Riak via HTTP. From 34078d26c15912301f356db571bfed37e5a6187f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 28 Dec 2012 12:27:28 -0500 Subject: [PATCH 0271/1060] Move HTTP request methods into a new file and wire up URL generators. --- riak/transports/http/connection.py | 71 +++++++++++ riak/transports/http/resources.py | 4 + riak/transports/http/transport.py | 192 +++++++---------------------- 3 files changed, 120 insertions(+), 147 deletions(-) create mode 100644 riak/transports/http/connection.py diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py new file mode 100644 index 00000000..68b922fb --- /dev/null +++ b/riak/transports/http/connection.py @@ -0,0 +1,71 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import httplib + + +class RiakHttpConnection(object): + """ + Connection and low-level request methods for RiakHttpTransport. + """ + + def GET(self, uri, headers={}): + return self._request('GET', uri, headers, '') + + def POST(self, uri, headers={}, body=''): + return self._request('POST', uri, headers, body) + + def PUT(self, uri, headers={}, body=''): + return self._request('PUT', uri, headers, body) + + def DELETE(self, uri, headers={}, body=''): + return self.request('DELETE', uri, headers, body) + + def HEAD(self, uri, headers={}): + return self.request('HEAD', uri, headers, '') + + def _request(self, method, uri, headers={}, body=''): + """ + Given a Method, URL, Headers, and Body, perform and HTTP request, + and return a 2-tuple containing a dictionary of response headers + and the response body. + """ + try: + self._connection.request(method, uri, body, headers) + response = self._connection.getresponse() + + response_headers = {'http_code': response.status} + for (key, value) in response.getheaders(): + response_headers[key.lower()] = value + + # TODO: Support streaming responses + response_body = response.read() + finally: + response.close() + + return response_headers, response_body + + def _connect(self): + self._connection = self._connection_class(self._node.host, + self._node.http_port) + + def close(self): + try: + self._connection.close() + except httplib.NotConnected: + pass diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index ccbfe896..b0a60f22 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -21,6 +21,7 @@ from riak import RiakError from riak.util import lazy_property + class RiakHttpResources(object): """ Methods for RiakHttpTransport related to URL generation, i.e. @@ -160,6 +161,9 @@ def mkpath(*segments, **query): pathstring = re.sub('/+', '/', pathstring) # Add the query string if it exists if len(query) > 0: + for key in query.keys(): + if query[key] is None: + query.pop(key) pathstring += "?" + urllib.urlencode(query).lower() if not pathstring.startswith('/'): diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 8af2e940..e7b4a725 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -33,6 +33,7 @@ import errno from riak.transports.transport import RiakTransport from riak.transports.http.resources import RiakHttpResources +from riak.transports.http.connection import RiakHttpConnection from riak.transports.http.search import XMLSearchResult from riak.metadata import * from riak.mapreduce import RiakLink @@ -44,7 +45,7 @@ from xml.dom.minidom import Document -class RiakHttpTransport(RiakHttpResources, RiakTransport): +class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakTransport): """ The RiakHttpTransport object holds information necessary to connect to Riak via HTTP. @@ -66,21 +67,20 @@ def __init__(self, node=None, self._client_id = client_id if not self._client_id: self._client_id = self.make_random_client_id() + self._connect() def ping(self): """ Check server is alive over HTTP """ - response = self.http_request('GET', '/ping') + response = self.GET(self.ping_path()) return(response is not None) and (response[1] == 'OK') def stats(self): """ Gets performance statistics and server information """ - # TODO: use resource detection - response = self.http_request('GET', '/stats', - {'Accept': 'application/json'}) + response = self.GET(self.stats_path(), {'Accept': 'application/json'}) if response[0]['http_code'] is 200: return json.loads(response[1]) else: @@ -94,7 +94,7 @@ def _server_version(self): # If stats is disabled, we can't assume the Riak version # is >= 1.1. However, we can assume the new URL scheme is # at least version 1.0 - elif 'riak_kv_wm_buckets' in self.get_resources(): + elif self.riak_kv_wm_buckets: return "1.0.0" else: return "0.14.0" @@ -104,8 +104,7 @@ def get_resources(self): Gets a JSON mapping of server-side resource names to paths :rtype dict """ - response = self.http_request('GET', '/', - {'Accept': 'application/json'}) + response = self.GET('/', {'Accept': 'application/json'}) if response[0]['http_code'] is 200: return json.loads(response[1]) else: @@ -117,12 +116,9 @@ def get(self, robj, r=None, pr=None, vtag=None): """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'r': r, 'pr': pr} - if vtag is not None: - params['vtag'] = vtag - url = self.build_rest_path(robj.bucket, robj.key, - params=params) - response = self.http_request('GET', url) + params = {'r': r, 'pr': pr, 'vtag': vtag} + url = self.object_path(robj.bucket.name, robj.key, **params) + response = self.GET(url) return self.parse_body(response, [200, 300, 404]) def put(self, robj, w=None, dw=None, pw=None, return_body=True, @@ -132,12 +128,10 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'returnbody': str(return_body).lower(), - 'w': w, 'dw': dw, 'pw': pw} - url = self.build_rest_path(bucket=robj.bucket, - key=robj.key, - params=params) + params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} + url = self.object_path(robj.bucket.name, robj.key, **params) headers = self.build_put_headers(robj) + # TODO: use a more general 'prevent_stale_writes' semantics, # which is a superset of the if_none_match semantics. if if_none_match: @@ -148,9 +142,9 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, def do_put(self, url, headers, content, return_body=False, key=None): if key is None: - response = self.http_request('POST', url, headers, content) + response = self.POST(url, headers, content) else: - response = self.http_request('PUT', url, headers, content) + response = self.PUT(url, headers, content) if return_body: return self.parse_body(response, [200, 201, 300]) @@ -163,16 +157,15 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, """Put a new object into the Riak store, returning its (new) key.""" # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'returnbody': str(return_body).lower(), 'w': w, 'dw': dw, - 'pw': pw} - url = self.build_rest_path(bucket=robj.bucket, params=params) + params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} + url = self.object_path(robj.bucket.name, **params) headers = self.build_put_headers(robj) # TODO: use a more general 'prevent_stale_writes' semantics, # which is a superset of the if_none_match semantics. if if_none_match: headers["If-None-Match"] = "*" content = robj.get_encoded_data() - response = self.http_request('POST', url, headers, content) + response = self.POST(url, headers, content) location = response[0]['location'] idx = location.rindex('/') key = location[(idx + 1):] @@ -191,11 +184,10 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): # unknown flags/params. params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} headers = {} - url = self.build_rest_path(robj.bucket, robj.key, - params=params) + url = self.object_path(robj.bucket.name, robj.key, **params) if self.tombstone_vclocks() and robj.vclock is not None: headers['X-Riak-Vclock'] = robj.vclock - response = self.http_request('DELETE', url, headers) + response = self.DELETE(url, headers) self.check_http_code(response, [204, 404]) return self @@ -203,24 +195,22 @@ def get_keys(self, bucket): """ Fetch a list of keys for the bucket """ - params = {'props': 'True', 'keys': 'true'} - url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', url) + url = self.key_list_path(bucket.name) + response = self.GET(url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: props = json.loads(encoded_props) return props['keys'] else: - raise Exception('Error getting bucket properties.') + raise Exception('Error listing keys.') def get_buckets(self): """ Fetch a list of all buckets """ - params = {'buckets': 'true'} - url = self.build_rest_path(None, params=params) - response = self.http_request('GET', url) + url = self.bucket_list_path() + response = self.GET(url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: @@ -234,9 +224,8 @@ def get_bucket_props(self, bucket): Get properties for a bucket """ # Run the request... - params = {'props': 'true', 'keys': 'false'} - url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', url) + url = self.bucket_properties_path(bucket.name) + response = self.GET(url) headers = response[0] encoded_props = response[1] @@ -250,12 +239,12 @@ def set_bucket_props(self, bucket, props): """ Set the properties on the bucket object given """ - url = self.build_rest_path(bucket) + url = self.bucket_properties_path(bucket.name) headers = {'Content-Type': 'application/json'} content = json.dumps({'props': props}) # Run the request... - response = self.http_request('PUT', url, headers, content) + response = self.PUT(url, headers, content) # Handle the response... if response is None: @@ -283,9 +272,9 @@ def mapred(self, inputs, query, timeout=None): content = json.dumps(job) # Do the request... - url = "/" + self._mapred_prefix + url = self.mapred_path() headers = {'Content-Type': 'application/json'} - response = self.http_request('POST', url, headers, content) + response = self.POST(url, headers, content) # Make sure the expected status code came back... status = response[0]['http_code'] @@ -301,12 +290,8 @@ def get_index(self, bucket, index, startkey, endkey=None): """ Performs a secondary index query. """ - # TODO: use resource detection - segments = ["buckets", bucket, "index", index, str(startkey)] - if endkey: - segments.append(str(endkey)) - uri = '/%s' % ('/'.join(segments)) - headers, data = response = self.get_request(uri) + response = self.GET(self.index_path(bucket, index, startkey, endkey)) + headers, data = response self.check_http_code(response, [200]) jsonData = json.loads(data) return jsonData[u'keys'][:] @@ -318,15 +303,14 @@ def search(self, index, query, **params): if index is None: index = 'search' - options = {'q': query, 'wt': 'json'} + options = {} if 'op' in params: op = params.pop('op') options['q.op'] = op options.update(params) - # TODO: use resource detection - uri = "/solr/%s/select" % index - headers, data = response = self.get_request(uri, options) + response = self.GET(self.solr_select_path(index, query, **options)) + headers, data = response self.check_http_code(response, [200]) if 'json' in headers['content-type']: results = json.loads(data) @@ -354,8 +338,9 @@ def fulltext_add(self, index, docs): root.appendChild(doc_element) xml.appendChild(root) - url = "/solr/%s/update" % index - self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") + self.POST(self.solr_update_path(index), + {'Content-Type': 'text/xml'}, + xml.toxml()) def fulltext_delete(self, index, docs=None, queries=None): """ @@ -378,8 +363,9 @@ def fulltext_delete(self, index, docs=None, queries=None): xml.appendChild(root) - url = "/solr/%s/update" % index - self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") + self.POST(self.solr_update_path(index), + {'Content-Type': 'text/xml'}, + xml.toxml()) def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] @@ -389,10 +375,7 @@ def check_http_code(self, response, expected_statuses): def parse_body(self, response, expected_statuses): """ - Given the output of RiakUtils.http_request and a list of - statuses, populate the object. Only for use by the Riak client - library. - @return self + Parse the body of an object response and populate the object. """ # If no response given, then return. if response is None: @@ -476,8 +459,8 @@ def parse_links(self, links, linkHeaders): Private. @return self """ - oldform = "; ?riaktag=\"([^\']+)\"" - newform = "; ?riaktag=\"([^\']+)\"" + oldform = "; ?riaktag=\"([^\"]+)\"" + newform = "; ?riaktag=\"([^\"]+)\"" for linkHeader in linkHeaders.strip().split(','): linkHeader = linkHeader.strip() matches = (re.match(oldform, linkHeader) or @@ -507,49 +490,8 @@ def add_links_for_riak_object(self, robject, headers): return headers - def get_request(self, uri=None, params=None): - url = self.build_rest_path(bucket=None, params=params, prefix=uri) - return self.http_request('GET', url) - - def post_request(self, uri=None, body=None, params=None, - content_type="application/json"): - uri = self.build_rest_path(prefix=uri, params=params) - return self.http_request('POST', uri, {'Content-Type': content_type}, - body) - # Utility functions used by Riak library. - def build_rest_path(self, bucket=None, key=None, params=None, prefix=None): - """ - Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, - construct and return a URL. - """ - # Build 'http://hostname:port/prefix/bucket' - path = '' - path += '/' + (prefix or self._prefix) - - # Add '.../bucket' - if bucket is not None: - path += '/' + urllib.quote_plus(bucket.name) - - # Add '.../key' - if key is not None: - path += '/' + urllib.quote_plus(key) - - # Add query parameters. - if params is not None: - s = '' - for key in params.keys(): - if params[key] is not None: - if s != '': - s += '&' - s += (urllib.quote_plus(key) + '=' + - urllib.quote_plus(str(params[key]))) - path += '?' + s - - # Return. - return path - def build_put_headers(self, robj): """Build the headers for a POST/PUT request.""" @@ -577,50 +519,6 @@ def build_put_headers(self, robj): return headers - def http_request(self, method, uri, headers=None, body=''): - """ - Given a Method, URL, Headers, and Body, perform and HTTP request, - and return a 2-tuple containing a dictionary of response headers - and the response body. - """ - if headers is None: - headers = {} - # Run the request... - for retry in range(self.RETRY_COUNT): - with self._conns.withconn() as conn: - ### should probably build this try/except into a custom - ### contextmanager for the connection. - try: - conn.request(method, uri, body, headers) - response = conn.getresponse() - - try: - # Get the response headers... - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value - - # Get the body... - response_body = response.read() - finally: - response.close() - - return response_headers, response_body - except socket.error, e: - conn.close() - if e[0] == errno.ECONNRESET: - # Grab another connection and try again. - continue - # Don't know how to handle this. - raise - except httplib.HTTPException: - # Just close the connection and try again. - conn.close() - continue - - # No luck, even with retrying. - raise RiakError("could not get a response") - def _normalize_json_search_response(self, json): """ Normalizes a JSON search response so that PB and HTTP have the From 21c1c387d66f4973f165759a181ee97be414c941 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 28 Dec 2012 12:43:28 -0500 Subject: [PATCH 0272/1060] Make RETRY_COUNT a property and use a for loop over a range instead of while. --- riak/client/transport.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/riak/client/transport.py b/riak/client/transport.py index 924d00f3..f7a4fd3b 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -28,6 +28,8 @@ class RiakClientTransport(class): Methods for RiakClient related to transport selection and retries. """ + RETRY_COUNT = 3 + @contextmanager def _transport(self, protocol=self.protocol): if protocol in ['http', 'https']: @@ -43,27 +45,24 @@ def _transport(self, protocol=self.protocol): @contextmanager def _retryable(self, pool): skip_nodes = [] - # TODO: Make a property? - retries = 3 def _skip_bad_nodes(transport): return transport._node not in skip_nodes - while retries > 0: + for retry in range(self.RETRY_COUNT): try: with pool.take(_filter=_skip_bad_nodes) as transport: try: yield transport except (IOError, httplib.HTTPException) as e: if is_pbc_retryable(e) or is_http_retryable(e): - retries -= 1 transport._node.error_rate.incr(1) skip_nodes.append(transport._node) raise BadResource(e) else: raise e except BadResource as br: - if retries > 0: + if retry < (self.RETRY_COUNT - 1): continue else: raise br.args[0] From 6db4661993789525b9640b3fed628e143295c258 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 28 Dec 2012 16:32:41 -0500 Subject: [PATCH 0273/1060] Fix gobs of pyflakes, pylint, pep8 problems. --- riak/__init__.py | 8 +-- riak/client/__init__.py | 44 +++++------- riak/client/operations.py | 7 +- riak/client/transport.py | 11 ++- riak/mapreduce.py | 6 +- riak/riak_object.py | 24 +++++-- riak/search.py | 19 ++++- riak/test_server.py | 5 ++ riak/tests/suite.py | 1 - riak/tests/test_all.py | 109 ++++++++++++++--------------- riak/tests/test_kv.py | 20 ++---- riak/tests/test_mapreduce.py | 4 +- riak/tests/test_pool.py | 16 ++--- riak/transports/__init__.py | 2 - riak/transports/http/__init__.py | 5 +- riak/transports/http/connection.py | 8 ++- riak/transports/http/resources.py | 6 +- riak/transports/http/transport.py | 39 ++++++----- riak/transports/pbc/__init__.py | 1 + riak/transports/pbc/codec.py | 8 +-- riak/transports/pbc/connection.py | 50 ++++++++++--- riak/transports/pbc/stream.py | 12 ++-- riak/transports/pbc/transport.py | 73 +++++++++++++------ riak/transports/transport.py | 11 ++- riak/util.py | 25 +++++-- 25 files changed, 304 insertions(+), 210 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index 1e116161..0d8ab38a 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -38,13 +38,7 @@ def __init__(self, value): def __str__(self): return repr(self.value) -from riak_object import RiakObject -from bucket import RiakBucket -from client import RiakClient -from mapreduce import RiakMapReduce, RiakMapReducePhase, RiakLinkPhase,\ - RiakKeyFilter -from transports.pbc import RiakPbcTransport -from transports.http import RiakHttpTransport +from mapreduce import RiakKeyFilter ONE = "one" ALL = "all" diff --git a/riak/client/__init__.py b/riak/client/__init__.py index c57fc9f6..c5619532 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -19,18 +19,12 @@ under the License. """ -try: - import json -except ImportError: - import simplejson as json - -from contextlib import contextmanager +import json +import random from weakref import WeakValueDictionary from riak.client.operations import RiakClientOperations -from riak.client.transport import RiakClientTransport from riak.node import RiakNode from riak.bucket import RiakBucket -from riak.mapreduce import RiakMapReduce from riak.mapreduce import RiakMapReduceChain from riak.search import RiakSearch from riak.transports.http import RiakHttpPool @@ -41,8 +35,7 @@ @deprecateQuorumAccessors -class RiakClient(RiakMapReduceChain, RiakClientOperations, - RiakClientTransport): +class RiakClient(RiakMapReduceChain, RiakClientOperations): """ The ``RiakClient`` object holds information necessary to connect to Riak. Requests can be made to Riak directly through the client @@ -90,17 +83,18 @@ def __init__(self, protocol='http', transport_options={}, 'text/json': json.loads} self._buckets = WeakValueDictionary() - @property - def protocol(self): + def _get_protocol(self): return self._protocol - @property.setter - def protocol(self, value): + def _set_protocol(self, value): if value not in self.PROTOCOLS: raise ValueError("protocol option is invalid, must be one of %s" % repr(self.PROTOCOLS)) self._protocol = value + protocol = property(_get_protocol, _set_protocol, + doc="""which protocol to prefer""") + def get_transport(self): """ Get the transport instance the client is using for it's connection. @@ -132,23 +126,19 @@ def set_client_id(self, client_id): self.client_id = client_id return self - @property - def client_id(self): - """ - The client ID for this client instance - - :rtype: string - """ - with self.transport() as transport: + def _get_client_id(self): + with self._transport() as transport: return transport.get_client_id() - @client_id.setter - def client_id(self, client_id): + def _set_client_id(self, client_id): for http in self._http_pool: http.client_id = client_id for pb in self._pb_pool: pb.client_id = client_id + client_id = property(_get_client_id, _set_client_id, + doc="""The client ID for this client instance""") + def get_encoder(self, content_type): """ Get the encoding function for the provided content type. @@ -214,16 +204,20 @@ def _create_node(self, n): raise TypeError("%s is not a valid node configuration" % repr(n)) - def _choose_node(self, nodes=self.nodes): + def _choose_node(self, nodes=None): """ Chooses a random node from the list of nodes in the client, taking into account each node's recent error rate. :rtype RiakNode """ + if not nodes: + nodes = self.nodes + # Prefer nodes which have gone a reasonable time without # errors def _error_rate(node): return node.error_rate.value() + good = [n for n in nodes if _error_rate(n) < 0.1] if len(good) is 0: diff --git a/riak/client/operations.py b/riak/client/operations.py index 90928b49..49fa25e4 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -17,9 +17,10 @@ """ from riak.bucket import RiakBucket +from transport import RiakClientTransport -class RiakClientOperations(object): +class RiakClientOperations(RiakClientTransport): """ Methods for RiakClient that result in requests sent to the Riak cluster. @@ -79,7 +80,7 @@ def stream_keys(self, bucket): method which should be iterated over. """ with self._transport() as transport: - for keylist in return transport.stream_keys(bucket): + for keylist in transport.stream_keys(bucket): yield keylist def put(self, robj, w=None, dw=None, pw=None, return_body=None, @@ -114,7 +115,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): Deletes an object from Riak. """ with self._transport() as transport: - return transport.delete(robj, rw=rw, r=r, w=w, dw=dw=, pr=pr, + return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) def mapred(self, inputs, query, timeout): diff --git a/riak/client/transport.py b/riak/client/transport.py index f7a4fd3b..4f449994 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -23,7 +23,7 @@ import httplib -class RiakClientTransport(class): +class RiakClientTransport(object): """ Methods for RiakClient related to transport selection and retries. """ @@ -31,7 +31,9 @@ class RiakClientTransport(class): RETRY_COUNT = 3 @contextmanager - def _transport(self, protocol=self.protocol): + def _transport(self, protocol=None): + if not protocol: + protocol = self.protocol if protocol in ['http', 'https']: pool = self._http_pool elif protocol is 'pbc': @@ -66,3 +68,8 @@ def _skip_bad_nodes(transport): continue else: raise br.args[0] + + # These will be set or redefined by the RiakClient initializer + protocol = 'http' + _http_pool = None + _pb_pool = None diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 6db573be..d64108cc 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -204,7 +204,7 @@ def run(self, timeout=None): """ query, link_results_flag = self._normalize_query() - result = self.client.mapred(self._inputs, query, timeout) + result = self._client.mapred(self._inputs, query, timeout) # If the last phase is NOT a link phase, then return the result. if not (link_results_flag @@ -233,7 +233,7 @@ def stream(self, timeout=None): Streams the MapReduce query (returns an iterator). """ query, lrf = self._normalize_query() - return self.client.stream_mapred(self._inputs, query, timeout) + return self._client.stream_mapred(self._inputs, query, timeout) def _normalize_query(self): num_phases = len(self._phases) @@ -260,7 +260,7 @@ def _normalize_query(self): if (type(self._inputs) == str): bucket_name = self._inputs elif (type(self._inputs) == RiakBucket): - bucket_name = self._inputs.get_name() + bucket_name = self._inputs.name if (bucket_name is not None): self._inputs = {'bucket': bucket_name, diff --git a/riak/riak_object.py b/riak/riak_object.py index 78cf21bb..b37e4ebd 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -18,10 +18,24 @@ under the License. """ import copy -from riak.metadata import * -from riak.mapreduce import * +from riak.metadata import ( + # MD_CHARSET, + MD_CTYPE, + # MD_ENCODING, + MD_INDEX, + # MD_LASTMOD, + # MD_LASTMOD_USECS, + MD_LINKS, + MD_USERMETA + # MD_VTAG, + # MD_DELETED + ) +from mapreduce import ( + RiakMapReduce, + RiakLink + ) from riak import RiakError -from riak.riak_index_entry import RiakIndexEntry +from riak_index_entry import RiakIndexEntry class RiakObject(object): @@ -436,7 +450,7 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): :rtype: self """ - result = self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) + self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) self.clear() return self @@ -488,7 +502,7 @@ def _populate(self, result): for sibling in siblings: sibling._set_siblings(siblings) else: - raise RiakError("do not know how to handle type %s" % type(Result)) + raise RiakError("do not know how to handle type %s" % type(result)) def get_sibling(self, i, r=None, pr=None): """ diff --git a/riak/search.py b/riak/search.py index 1d8fe914..6a25c9eb 100644 --- a/riak/search.py +++ b/riak/search.py @@ -1,5 +1,20 @@ -from riak.transports import RiakHttpTransport -from xml.etree import ElementTree +""" +Copyright 2010 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" class RiakSearch(object): diff --git a/riak/test_server.py b/riak/test_server.py index 7c6348af..4010fb76 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -92,6 +92,11 @@ class TestServer(object): }, } + _temp_bin = None + _temp_etc = None + _temp_log = None + _temp_pipe = None + def __init__(self, tmp_dir="/tmp/riak/test_server", bin_dir=os.path.expanduser("~/.riak/install/riak-0.14.2/bin"), vm_args=None, **options): diff --git a/riak/tests/suite.py b/riak/tests/suite.py index af74197a..6f96c84d 100644 --- a/riak/tests/suite.py +++ b/riak/tests/suite.py @@ -1,4 +1,3 @@ -import riak.tests.test_server_test import os.path import platform diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index fe21c6af..7d9611a8 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -3,19 +3,14 @@ import os import random -import socket import platform if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest -import uuid -import time from riak import RiakClient -from riak import RiakPbcTransport -from riak import RiakHttpTransport from riak.mapreduce import RiakLink from riak import RiakKeyFilter, key_filter @@ -30,7 +25,7 @@ from riak.tests.test_2i import TwoITests try: - import riak_pb + __import__('riak_pb') HAVE_PROTO = True except ImportError: HAVE_PROTO = False @@ -111,61 +106,61 @@ def test_uses_client_id_if_given(self): c = self.create_client(client_id=zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) - def test_close_underlying_socket_fails(self): - self.skipTest("TODO: No longer using connection manager, replace") - c = self.create_client() - - bucket = c.bucket('bucket_test_close') - rand = self.randint() - obj = bucket.new('foo', rand) - obj.store() - obj = bucket.get('foo') - self.assertTrue(obj.exists) - self.assertEqual(obj.bucket.name, 'bucket_test_close') - self.assertEqual(obj.key, 'foo') - self.assertEqual(obj.data, rand) - - # Close the underlying socket. This gets a bit sketchy, - # since we are reaching into the internals, but there is - # no other way to get at the socket - conns = c._cm.conns - conns[0].sock.close() - - # This shoud fail with a socket error now - self.assertRaises(socket.error, bucket.get, 'foo') - - def test_close_underlying_socket_retry(self): - self.skipTest("TODO: No longer using bare transport, replace") - c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport, - transport_options={"max_attempts": 2}) - - bucket = c.bucket('bucket_test_close') - rand = self.randint() - obj = bucket.new('barbaz', rand) - obj.store() - obj = bucket.get('barbaz') - self.assertTrue(obj.exists) - self.assertEqual(obj.bucket.name, 'bucket_test_close') - self.assertEqual(obj.key, 'barbaz') - self.assertEqual(obj.data, rand) - - # Close the underlying socket. This gets a bit sketchy, - # since we are reaching into the internals, but there is - # no other way to get at the socket - conns = c._cm.conns - conns[0].sock.close() - - # This should work, since we have a retry - obj = bucket.get('barbaz') - self.assertTrue(obj.exists) - self.assertEqual(obj.bucket.name, 'bucket_test_close') - self.assertEqual(obj.key, 'barbaz') - self.assertEqual(obj.data, rand) + # def test_close_underlying_socket_fails(self): + # self.skipTest("TODO: No longer using connection manager, replace") + # c = self.create_client() + + # bucket = c.bucket('bucket_test_close') + # rand = self.randint() + # obj = bucket.new('foo', rand) + # obj.store() + # obj = bucket.get('foo') + # self.assertTrue(obj.exists) + # self.assertEqual(obj.bucket.name, 'bucket_test_close') + # self.assertEqual(obj.key, 'foo') + # self.assertEqual(obj.data, rand) + + # # Close the underlying socket. This gets a bit sketchy, + # # since we are reaching into the internals, but there is + # # no other way to get at the socket + # conns = c._cm.conns + # conns[0].sock.close() + + # # This shoud fail with a socket error now + # self.assertRaises(socket.error, bucket.get, 'foo') + + # def test_close_underlying_socket_retry(self): + # self.skipTest("TODO: No longer using bare transport, replace") + # c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport, + # transport_options={"max_attempts": 2}) + + # bucket = c.bucket('bucket_test_close') + # rand = self.randint() + # obj = bucket.new('barbaz', rand) + # obj.store() + # obj = bucket.get('barbaz') + # self.assertTrue(obj.exists) + # self.assertEqual(obj.bucket.name, 'bucket_test_close') + # self.assertEqual(obj.key, 'barbaz') + # self.assertEqual(obj.data, rand) + + # # Close the underlying socket. This gets a bit sketchy, + # # since we are reaching into the internals, but there is + # # no other way to get at the socket + # conns = c._cm.conns + # conns[0].sock.close() + + # # This should work, since we have a retry + # obj = bucket.get('barbaz') + # self.assertTrue(obj.exists) + # self.assertEqual(obj.bucket.name, 'bucket_test_close') + # self.assertEqual(obj.key, 'barbaz') + # self.assertEqual(obj.data, rand) def test_bucket_search_enabled(self): with self.assertRaises(NotImplementedError): bucket = self.client.bucket("unsearch_bucket") - test = bucket.search_enabled() + bucket.search_enabled() def test_enable_search_commit_hook(self): with self.assertRaises(NotImplementedError): diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index b477f944..08a16c04 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -2,10 +2,7 @@ import os import cPickle import copy -try: - import json -except ImportError: - import simplejson as json +import json class NotJsonSerializable(object): @@ -288,13 +285,13 @@ class PbcBucketPropsTest(object): def test_rw_settings(self): bucket = self.client.bucket('rwsettings') with self.assertRaises(NotImplementedError): - test = bucket.r + bucket.r with self.assertRaises(NotImplementedError): - test = bucket.w + bucket.w with self.assertRaises(NotImplementedError): - test = bucket.dw + bucket.dw with self.assertRaises(NotImplementedError): - test = bucket.rw + bucket.rw with self.assertRaises(NotImplementedError): bucket.r = 2 @@ -308,9 +305,9 @@ def test_rw_settings(self): def test_primary_quora(self): bucket = self.client.bucket('primary_quora') with self.assertRaises(NotImplementedError): - test = bucket.pr + bucket.pr with self.assertRaises(NotImplementedError): - test = bucket.pw + bucket.pw with self.assertRaises(NotImplementedError): bucket.pr = 2 @@ -321,7 +318,6 @@ def test_primary_quora(self): class KVFileTests(object): def test_store_binary_object_from_file(self): bucket = self.client.bucket('bucket') - rand = str(self.randint()) filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') obj = bucket.new_binary_from_file('foo_from_file', filepath) obj.store() @@ -331,7 +327,6 @@ def test_store_binary_object_from_file(self): def test_store_binary_object_from_file_should_use_default_mimetype(self): bucket = self.client.bucket('bucket') - rand = str(self.randint()) filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir, 'THANKS') obj = bucket.new_binary_from_file('foo_from_file', filepath) @@ -341,7 +336,6 @@ def test_store_binary_object_from_file_should_use_default_mimetype(self): def test_store_binary_object_from_file_should_fail_if_file_not_found(self): bucket = self.client.bucket('bucket') - rand = str(self.randint()) self.assertRaises(IOError, bucket.new_binary_from_file, 'not_found_from_file', 'FILE_NOT_FOUND') obj = bucket.get_binary('not_found_from_file') diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index f8293f8c..249d7590 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from riak.mapreduce import RiakLink -from riak import RiakKeyFilter, key_filter +from riak import key_filter class LinkTests(object): @@ -44,7 +44,7 @@ def test_set_links(self): def test_set_links_all_links(self): bucket = self.client.bucket("bucket") foo1 = bucket.new("foo", 1) - foo2 = bucket.new("foo2", 2).store() + bucket.new("foo2", 2).store() links = [RiakLink("bucket", "foo2")] foo1.set_links(links, True) links = foo1.get_links() diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index f57a2cd3..d552bcaa 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -69,7 +69,7 @@ def test_yields_same_object_in_serial_access(self): with pool.take() as element2: self.assertEqual(1, len(pool.elements)) - self.assertEqual([1, 2], element) + self.assertEqual([1, 2], element2) self.assertEqual(1, len(pool.elements)) @@ -93,8 +93,8 @@ def test_unlocks_when_exception_raised(self): """ pool = SimplePool() try: - with pool.take() as x: - with pool.take() as y: + with pool.take(): + with pool.take(): raise RuntimeError except: self.assertEqual(2, len(pool.elements)) @@ -111,7 +111,7 @@ def test_removes_bad_resource(self): self.assertEqual([1], element) element.append(2) try: - with pool.take() as baddie: + with pool.take(): raise BadResource except BadResource: self.assertEqual(0, len(pool.elements)) @@ -127,8 +127,8 @@ def filtereven(numlist): return numlist[0] % 2 == 0 pool = SimplePool() - with pool.take() as x: - with pool.take() as y: + with pool.take(): + with pool.take(): pass with pool.take(_filter=filtereven) as f: @@ -143,7 +143,7 @@ def test_requires_filter_to_be_callable(self): pool = SimplePool() with self.assertRaises(TypeError): - with pool.take(_filter=badfilter) as resource: + with pool.take(_filter=badfilter): pass def test_yields_default_when_empty(self): @@ -245,7 +245,7 @@ def test_clear(self): pool = SimplePool() def worker_run(): - with pool.take() as a: + with pool.take(): startq.put(1) startq.join() sleep(rand.uniform(0, 0.5)) diff --git a/riak/transports/__init__.py b/riak/transports/__init__.py index 9c59adb6..e69de29b 100644 --- a/riak/transports/__init__.py +++ b/riak/transports/__init__.py @@ -1,2 +0,0 @@ -from http import RiakHttpTransport -from pbc import RiakPbcTransport diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index 478ebf5c..f3461e2e 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -18,14 +18,11 @@ under the License. """ +import httplib from riak.transports.pool import Pool from riak.transports.http.transport import RiakHttpTransport -# subtract length of "Link: " header string and newline -MAX_LINK_HEADER_SIZE = 8192 - 8 - - class RiakHttpPool(Pool): """ A pool of HTTP(S) transport connections. diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py index 68b922fb..0eeb5425 100644 --- a/riak/transports/http/connection.py +++ b/riak/transports/http/connection.py @@ -34,10 +34,10 @@ def PUT(self, uri, headers={}, body=''): return self._request('PUT', uri, headers, body) def DELETE(self, uri, headers={}, body=''): - return self.request('DELETE', uri, headers, body) + return self._request('DELETE', uri, headers, body) def HEAD(self, uri, headers={}): - return self.request('HEAD', uri, headers, '') + return self._request('HEAD', uri, headers, '') def _request(self, method, uri, headers={}, body=''): """ @@ -69,3 +69,7 @@ def close(self): self._connection.close() except httplib.NotConnected: pass + + # These are set by the RiakHttpTransport initializer + _connection_class = httplib.HTTPConnection + _node = None diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index b0a60f22..44d1b7ff 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -17,7 +17,7 @@ """ import re -from urllib import quote_plus +from urllib import quote_plus, urlencode from riak import RiakError from riak.util import lazy_property @@ -35,7 +35,7 @@ def stats_path(self): return mkpath(self.riak_kv_wm_stats) def mapred_path(self, **options): - return mkpath(self.kv_wm_mapred, **options) + return mkpath(self.riak_kv_wm_mapred, **options) def bucket_list_path(self, **options): query = options.copy() @@ -164,7 +164,7 @@ def mkpath(*segments, **query): for key in query.keys(): if query[key] is None: query.pop(key) - pathstring += "?" + urllib.urlencode(query).lower() + pathstring += "?" + urlencode(query).lower() if not pathstring.startswith('/'): pathstring = '/' + pathstring diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index e7b4a725..342188bd 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -19,32 +19,39 @@ under the License. """ -try: - import json -except ImportError: - import simplejson as json - +import json import urllib import re import csv -from cStringIO import StringIO import httplib -import socket -import errno from riak.transports.transport import RiakTransport from riak.transports.http.resources import RiakHttpResources from riak.transports.http.connection import RiakHttpConnection from riak.transports.http.search import XMLSearchResult -from riak.metadata import * +from riak.metadata import ( + MD_CHARSET, + MD_CTYPE, + # MD_ENCODING, + MD_INDEX, + MD_LASTMOD, + # MD_LASTMOD_USECS, + MD_LINKS, + MD_USERMETA, + MD_VTAG, + MD_DELETED + ) from riak.mapreduce import RiakLink from riak import RiakError from riak.riak_index_entry import RiakIndexEntry from riak.multidict import MultiDict -import riak.util from xml.etree import ElementTree from xml.dom.minidom import Document +# subtract length of "Link: " header string and newline +MAX_LINK_HEADER_SIZE = 8192 - 8 + + class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakTransport): """ The RiakHttpTransport object holds information necessary to @@ -53,7 +60,7 @@ class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakTransport): def __init__(self, node=None, client=None, - connection_class=httplib.HTTPConnection + connection_class=httplib.HTTPConnection, client_id=None, **unused_options): """ @@ -446,12 +453,8 @@ def to_link_header(self, link): """ Convert this RiakLink object to a link header string. Used internally. """ - header = '' - header += '; riaktag="' - header += urllib.quote_plus(link.get_tag()) + '"' + url = self.object_path(link.get_bucket(), link.get_key()) + header = '<%s>; riaktag="%s"' % (url, link.get_tag()) return header def parse_links(self, links, linkHeaders): @@ -505,7 +508,7 @@ def build_put_headers(self, robj): headers['X-Riak-Vclock'] = robj.vclock # Create the header from metadata - links = self.add_links_for_riak_object(robj, headers) + self.add_links_for_riak_object(robj, headers) for key, value in robj.usermeta.iteritems(): headers['X-Riak-Meta-%s' % key] = value diff --git a/riak/transports/pbc/__init__.py b/riak/transports/pbc/__init__.py index ef772777..14705673 100644 --- a/riak/transports/pbc/__init__.py +++ b/riak/transports/pbc/__init__.py @@ -20,6 +20,7 @@ """ import errno +import socket from riak.transports.pool import Pool from riak.transports.pbc.transport import RiakPbcTransport diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index dca7d2f0..ae9a67b9 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -25,12 +25,12 @@ MD_LINKS, MD_USERMETA, MD_VTAG, + MD_DELETED ) -try: - import riak_pb -except ImportError: - riak_pb = None +import riak_pb +from riak.riak_index_entry import RiakIndexEntry +from riak.mapreduce import RiakLink RIAKC_RW_ONE = 4294967294 RIAKC_RW_QUORUM = 4294967293 diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index c8ba785f..1b55b843 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -16,11 +16,41 @@ under the License. """ -import errno import socket import struct -from contextlib import contextmanager +import riak_pb from riak import RiakError +from messages import ( + MSG_CODE_ERROR_RESP, + # MSG_CODE_PING_REQ, + MSG_CODE_PING_RESP, + # MSG_CODE_GET_CLIENT_ID_REQ, + MSG_CODE_GET_CLIENT_ID_RESP, + # MSG_CODE_SET_CLIENT_ID_REQ, + MSG_CODE_SET_CLIENT_ID_RESP, + # MSG_CODE_GET_SERVER_INFO_REQ, + MSG_CODE_GET_SERVER_INFO_RESP, + # MSG_CODE_GET_REQ, + MSG_CODE_GET_RESP, + # MSG_CODE_PUT_REQ, + MSG_CODE_PUT_RESP, + # MSG_CODE_DEL_REQ, + MSG_CODE_DEL_RESP, + # MSG_CODE_LIST_BUCKETS_REQ, + MSG_CODE_LIST_BUCKETS_RESP, + # MSG_CODE_LIST_KEYS_REQ, + MSG_CODE_LIST_KEYS_RESP, + # MSG_CODE_GET_BUCKET_REQ, + MSG_CODE_GET_BUCKET_RESP, + # MSG_CODE_SET_BUCKET_REQ, + MSG_CODE_SET_BUCKET_RESP, + # MSG_CODE_MAPRED_REQ, + MSG_CODE_MAPRED_RESP, + # MSG_CODE_INDEX_REQ, + MSG_CODE_INDEX_RESP, + # MSG_CODE_SEARCH_QUERY_REQ, + MSG_CODE_SEARCH_QUERY_RESP + ) class RiakPbcConnection(object): @@ -40,15 +70,15 @@ def _request(self, msg_code, msg=None, expect=None): return self._recv_msg(expect) def _send_msg(self, msg_code, msg): - self._socket.send(self.encode_msg(msg_code, msg)) + self._socket.send(self._encode_msg(msg_code, msg)) def _recv_msg(self, expect=None): self._recv_pkt() msg_code, = struct.unpack("B", self._inbuf[:1]) if msg_code == MSG_CODE_ERROR_RESP: - msg = riak_pb.RpbErrorResp() - msg.ParseFromString(self._inbuf[1:]) - raise Exception(msg.errmsg) + err = riak_pb.RpbErrorResp() + err.ParseFromString(self._inbuf[1:]) + raise RiakError(err.errmsg) elif msg_code == MSG_CODE_PING_RESP: msg = None elif msg_code == MSG_CODE_GET_SERVER_INFO_RESP: @@ -105,7 +135,7 @@ def _recv_pkt(self): self._inbuf = '' while len(self._inbuf) < msglen: want_len = min(8192, msglen - len(self._inbuf)) - recv_buf = conn.recv(want_len) + recv_buf = self._socket.recv(want_len) if not recv_buf: break self._inbuf += recv_buf @@ -115,7 +145,11 @@ def _recv_pkt(self): def _connect(self): self._socket = socket.create_connection(self._address, - self._timeouts.connect) + self._timeouts['connect']) def close(self): self._socket.shutdown(socket.SHUT_RDWR) + + # These are set in the RiakPbcTransport initializer + _address = None + _timeouts = {} diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index a71f6c20..f63fb645 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -16,20 +16,20 @@ under the License. """ -try: - import json -except ImportError: - import simplejson as json +import json from riak.transports.pbc.messages import MSG_CODE_LIST_KEYS_RESP from riak.transports.pbc.messages import MSG_CODE_MAPRED_RESP -class RiakPbcStream(class): +class RiakPbcStream(object): """ Used internally by RiakPbcTransport to implement streaming operations. Implements the iterator interface. """ + + _expect = None + def __init__(self, transport): self.transport = transport @@ -80,7 +80,7 @@ class RiakPbcMapredStream(RiakPbcStream): def next(self): response = super(RiakPbcMapredStream, self).next() - return (response.phase, json.loads(response.response)) + return response.phase, json.loads(response.response) def _is_done(self, response): return response.done diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 6cd96e38..ebe37685 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -19,20 +19,47 @@ under the License. """ -try: - import json -except ImportError: - import simplejson as json - +import json +import riak_pb from riak import RiakError -from riak.mapreduce import RiakLink -from riak.riak_index_entry import RiakIndexEntry from riak.transports.transport import RiakTransport -from riak.transports.pbc.connection import RiakPbcConnection -from riak.transports.pbc.stream import RiakPbcKeyStream, RiakPbcMapredStream -from riak.transports.pbc.codec import RiakPbcCodec -from riak.transports.pbc.messages import * -import riak.util +from connection import RiakPbcConnection +from stream import RiakPbcKeyStream, RiakPbcMapredStream +from codec import RiakPbcCodec +from messages import ( + # MSG_CODE_ERROR_RESP, + MSG_CODE_PING_REQ, + MSG_CODE_PING_RESP, + MSG_CODE_GET_CLIENT_ID_REQ, + MSG_CODE_GET_CLIENT_ID_RESP, + MSG_CODE_SET_CLIENT_ID_REQ, + MSG_CODE_SET_CLIENT_ID_RESP, + MSG_CODE_GET_SERVER_INFO_REQ, + MSG_CODE_GET_SERVER_INFO_RESP, + MSG_CODE_GET_REQ, + MSG_CODE_GET_RESP, + MSG_CODE_PUT_REQ, + MSG_CODE_PUT_RESP, + MSG_CODE_DEL_REQ, + MSG_CODE_DEL_RESP, + MSG_CODE_LIST_BUCKETS_REQ, + MSG_CODE_LIST_BUCKETS_RESP, + MSG_CODE_LIST_KEYS_REQ, + # MSG_CODE_LIST_KEYS_RESP, + MSG_CODE_GET_BUCKET_REQ, + MSG_CODE_GET_BUCKET_RESP, + MSG_CODE_SET_BUCKET_REQ, + MSG_CODE_SET_BUCKET_RESP, + MSG_CODE_MAPRED_REQ, + # MSG_CODE_MAPRED_RESP, + MSG_CODE_INDEX_REQ, + MSG_CODE_INDEX_RESP, + MSG_CODE_SEARCH_QUERY_REQ, + MSG_CODE_SEARCH_QUERY_RESP + ) + + +# from messages import * class RiakPbcTransport(RiakTransport, RiakPbcConnection, RiakPbcCodec): @@ -79,15 +106,12 @@ def get_server_info(self): expect=MSG_CODE_GET_SERVER_INFO_RESP) return {'node': resp.node, 'server_version': resp.server_version} - @property - def client_id(self): - """the client ID for this connection""" + def _get_client_id(self): msg_code, resp = self._request(MSG_CODE_GET_CLIENT_ID_REQ, expect=MSG_CODE_GET_CLIENT_ID_RESP) return resp.client_id - @client_id.setter - def client_id(self, client_id): + def _set_client_id(self, client_id): req = riak_pb.RpbSetClientIdReq() req.client_id = client_id @@ -96,6 +120,9 @@ def client_id(self, client_id): self._client_id = client_id + client_id = property(_get_client_id, _set_client_id, + doc="""the client ID for this connection""") + def get(self, robj, r=None, pr=None, vtag=None): """ Serialize get request and deserialize response @@ -152,9 +179,9 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, if vclock: req.vclock = vclock - self.pbify_content(robj.metadata, - robj.get_encoded_data(), - req.content) + self.encode_content(robj.metadata, + robj.get_encoded_data(), + req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, MSG_CODE_PUT_RESP) @@ -191,9 +218,9 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, req.bucket = bucket.name - self.pbify_content(robj.metadata, - robj.get_encoded_data(), - req.content) + self.encode_content(robj.metadata, + robj.get_encoded_data(), + req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, MSG_CODE_PUT_RESP) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 975d1dac..25e2cba1 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -17,7 +17,6 @@ specific language governing permissions and limitations under the License. """ -from riak import RiakError import base64 import random import threading @@ -31,15 +30,15 @@ class RiakTransport(FeatureDetection): Class to encapsulate transport details """ - @property - def client_id(self): - """the client ID for this connection""" + def _get_client_id(self): return self._client_id - @client_id.setter - def client_id(self, value): + def _set_client_id(self, value): self._client_id = value + client_id = property(_get_client_id, _set_client_id, + doc="""the client ID for this connection""") + @classmethod def make_random_client_id(self): """ diff --git a/riak/util.py b/riak/util.py index 6d58dcd8..c590db53 100644 --- a/riak/util.py +++ b/riak/util.py @@ -1,10 +1,23 @@ -import warnings +""" +Copyright 2010 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 -try: - from collections import Mapping -except ImportError: - # compatibility with Python 2.5 - Mapping = dict +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import warnings +from collections import Mapping def quacks_like_dict(object): From 991df2b368cd53c19948bd765a08e1fcc0cc0fec Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 28 Dec 2012 18:21:03 -0500 Subject: [PATCH 0274/1060] Fixed a bunch of test failures and stupid typos/bugs. --- riak/bucket.py | 2 +- riak/client/__init__.py | 2 +- riak/client/operations.py | 2 +- riak/client/transport.py | 13 +++++---- riak/mapreduce.py | 7 +++-- riak/riak_object.py | 9 +++--- riak/search.py | 4 +-- riak/tests/test_all.py | 8 +++--- riak/tests/test_search.py | 44 +++++++++++++----------------- riak/transports/http/__init__.py | 6 ++-- riak/transports/http/connection.py | 4 ++- riak/transports/http/resources.py | 15 ++++++---- riak/transports/pbc/connection.py | 4 +-- riak/transports/pbc/stream.py | 2 +- riak/transports/pbc/transport.py | 10 +++---- 15 files changed, 67 insertions(+), 65 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index eeed19fa..9560026f 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -379,7 +379,7 @@ def search(self, query, **params): """ Queries a search index over objects in this bucket/index. """ - return self._client.solr().search(self.name, query, **params) + return self._client.solr.search(self.name, query, **params) def get_index(self, index, startkey, endkey=None): """ diff --git a/riak/client/__init__.py b/riak/client/__init__.py index c5619532..4b816551 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -128,7 +128,7 @@ def set_client_id(self, client_id): def _get_client_id(self): with self._transport() as transport: - return transport.get_client_id() + return transport.client_id def _set_client_id(self, client_id): for http in self._http_pool: diff --git a/riak/client/operations.py b/riak/client/operations.py index 49fa25e4..5a5614ab 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -146,7 +146,7 @@ def fulltext_add(self, index, docs): Adds documents to the full-text index. """ with self._transport(protocol='http') as transport: - transport.fulltext_add(self, index, docs) + transport.fulltext_add(index, docs) def fulltext_delete(self, index, docs=None, queries=None): """ diff --git a/riak/client/transport.py b/riak/client/transport.py index 4f449994..c6db8823 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -41,7 +41,7 @@ def _transport(self, protocol=None): else: raise ValueError("invalid protocol %s" % protocol) - with self._retryable(pool) as transport: + with pool.take() as transport: yield transport @contextmanager @@ -57,19 +57,20 @@ def _skip_bad_nodes(transport): try: yield transport except (IOError, httplib.HTTPException) as e: - if is_pbc_retryable(e) or is_http_retryable(e): + if is_retryable(e): transport._node.error_rate.incr(1) skip_nodes.append(transport._node) raise BadResource(e) else: raise e except BadResource as br: - if retry < (self.RETRY_COUNT - 1): - continue - else: - raise br.args[0] + continue # These will be set or redefined by the RiakClient initializer protocol = 'http' _http_pool = None _pb_pool = None + + +def is_retryable(error): + return is_pbc_retryable(error) or is_http_retryable(error) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index d64108cc..4bd7712c 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -18,8 +18,6 @@ under the License. """ import urllib -from riak_object import RiakObject -from bucket import RiakBucket from collections import Iterable @@ -241,6 +239,8 @@ def _normalize_query(self): # If there are no phases, return the keys as links if num_phases is 0: link_results_flag = True + else: + link_results_flag = False # Convert all phases to associative arrays. Also, # if none of the phases are accumulating, then set the last one to @@ -621,3 +621,6 @@ def reduce(self, *args): """ mr = RiakMapReduce(self) return apply(mr.reduce, args) + +from riak.riak_object import RiakObject +from riak.bucket import RiakBucket diff --git a/riak/riak_object.py b/riak/riak_object.py index b37e4ebd..3bad9519 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -30,12 +30,12 @@ # MD_VTAG, # MD_DELETED ) -from mapreduce import ( +from riak.mapreduce import ( RiakMapReduce, RiakLink ) from riak import RiakError -from riak_index_entry import RiakIndexEntry +from riak.riak_index_entry import RiakIndexEntry class RiakObject(object): @@ -81,6 +81,8 @@ def _set_data(self, data, content_type=None): self.content_type = "application/json" else: self.content_type = "application/octet-stream" + if content_type: + self.content_type = content_type self._data = data return self @@ -382,9 +384,6 @@ def store(self, w=None, dw=None, pw=None, return_body=True, raise RiakError("Attempting to store an invalid object," "store one of the siblings instead") - # Issue the put over our transport - # t = self.client.get_transport() - if self.key is None: key, vclock, metadata = self.client.put_new( self, w=w, dw=dw, pw=pw, diff --git a/riak/search.py b/riak/search.py index 6a25c9eb..b340ba53 100644 --- a/riak/search.py +++ b/riak/search.py @@ -22,12 +22,12 @@ def __init__(self, client, **unused_args): self._client = client def add(self, index, *docs): - self._client.fulltext_add(index, docs) + self._client.fulltext_add(index, docs=docs) index = add def delete(self, index, docs=None, queries=None): - self._client.fulltext_delete(index, docs, queries) + self._client.fulltext_delete(index, docs=docs, queries=queries) remove = delete diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 7d9611a8..2d5c5d20 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -10,9 +10,9 @@ else: import unittest -from riak import RiakClient -from riak.mapreduce import RiakLink -from riak import RiakKeyFilter, key_filter +from riak.client import RiakClient +from riak.mapreduce import RiakLink, RiakKeyFilter +from riak import key_filter from riak.test_server import TestServer @@ -104,7 +104,7 @@ def setUp(self): def test_uses_client_id_if_given(self): zero_client_id = "\0\0\0\0" c = self.create_client(client_id=zero_client_id) - self.assertEqual(zero_client_id, c.get_client_id()) + self.assertEqual(zero_client_id, c.client_id) # def test_close_underlying_socket_fails(self): # self.skipTest("TODO: No longer using connection manager, replace") diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index f3223da0..a56ec63c 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -35,61 +35,55 @@ def test_disable_search_commit_hook(self): class SolrSearchTests(object): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_document_to_index(self): - self.client.solr().add("searchbucket", + self.client.solr.add("searchbucket", {"id": "doc", "username": "tony"}) - results = self.client.solr().search("searchbucket", "username:tony") + results = self.client.solr.search("searchbucket", "username:tony") self.assertEquals("tony", results['docs'][0]['username']) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_multiple_documents_to_index(self): - self.client.solr().add("searchbucket", + self.client.solr.add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - results = self.client.solr()\ + results = self.client.solr\ .search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(2, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_id(self): - self.client.solr().add("searchbucket", + self.client.solr.add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - self.client.solr().delete("searchbucket", docs=["dizzy"]) - results = self.client.solr()\ + self.client.solr.delete("searchbucket", docs=["dizzy"]) + results = self.client.solr\ .search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): - self.client.solr().add("searchbucket", + self.client.solr.add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - self.client.solr()\ + self.client.solr\ .delete("searchbucket", queries=["username:dizzy", "username:russell"]) - results = self.client.solr()\ + results = self.client.solr\ .search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query_and_id(self): - self.client.solr().add("searchbucket", + self.client.solr.add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - self.client.solr().delete("searchbucket", + self.client.solr.delete("searchbucket", docs=["dizzy"], queries=["username:russell"]) - results = self.client.solr()\ + results = self.client.solr\ .search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results['docs'])) - def test_build_rest_path_excludes_empty_query_params(self): - self.assertEquals( - self.client.get_transport().build_rest_path( - bucket=self.client.bucket("foo"), - key="bar", params={'r': None}), "/riak/foo/bar?") - class SearchTests(object): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') @@ -110,16 +104,16 @@ def test_solr_search_with_params_from_bucket(self): def test_solr_search_with_params(self): bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr().search("searchbucket", - "username:roidrage", wt="xml") + results = self.client.solr.search("searchbucket", + "username:roidrage", wt="xml") self.assertEquals(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search(self): bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr().search("searchbucket", - "username:roidrage") + results = self.client.solr.search("searchbucket", + "username:roidrage") self.assertEquals(1, len(results["docs"])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') @@ -133,7 +127,7 @@ def test_search_integration(self): bucket.new("five", {"foo": "five", "bar": "yellow"}).store() # Run some operations... - results = self.client.solr().search("searchbucket", + results = self.client.solr.search("searchbucket", "foo:one OR foo:two") if (len(results) == 0): print "\n\nNot running test \"testSearchIntegration()\".\n" @@ -144,5 +138,5 @@ def test_search_integration(self): self.assertEqual(len(results['docs']), 2) query = "(foo:one OR foo:two OR foo:three OR foo:four) AND\ (NOT bar:green)" - results = self.client.solr().search("searchbucket", query) + results = self.client.solr.search("searchbucket", query) self.assertEqual(len(results['docs']), 3) diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index f3461e2e..d6069444 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -31,13 +31,13 @@ def __init__(self, client, **options): self.client = client self.options = options if client.protocol is 'https': - self.connection_class = httplib.HTTPConnection - else: self.connection_class = httplib.HTTPSConnection + else: + self.connection_class = httplib.HTTPConnection super(RiakHttpPool, self).__init__() def create_resource(self): - node = self.client.choose_node() + node = self.client._choose_node() return RiakHttpTransport(node=node, client=self.client, connection_class=self.connection_class, diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py index 0eeb5425..09550d21 100644 --- a/riak/transports/http/connection.py +++ b/riak/transports/http/connection.py @@ -45,6 +45,7 @@ def _request(self, method, uri, headers={}, body=''): and return a 2-tuple containing a dictionary of response headers and the response body. """ + response = None try: self._connection.request(method, uri, body, headers) response = self._connection.getresponse() @@ -56,7 +57,8 @@ def _request(self, method, uri, headers={}, body=''): # TODO: Support streaming responses response_body = response.read() finally: - response.close() + if response: + response.close() return response_headers, response_body diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index 44d1b7ff..538a4126 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -109,7 +109,7 @@ def luwak_path(self, key=None): @lazy_property def riak_kv_wm_buckets(self): - return self.resources.get('riak_kv_wm_buckets') + return self.resources.get('riak_kv_wm_index') @lazy_property def riak_kv_wm_raw(self): @@ -159,12 +159,15 @@ def mkpath(*segments, **query): pathstring = '/'.join(segments) # Remove extra slashes pathstring = re.sub('/+', '/', pathstring) + # Add the query string if it exists - if len(query) > 0: - for key in query.keys(): - if query[key] is None: - query.pop(key) - pathstring += "?" + urlencode(query).lower() + _query = {} + for key in query: + if query[key] is not None: + _query[key] = query[key] + + if len(_query) > 0: + pathstring += "?" + urlencode(_query).lower() if not pathstring.startswith('/'): pathstring = '/' + pathstring diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index 1b55b843..06010fd9 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -120,8 +120,8 @@ def _recv_msg(self, expect=None): else: raise Exception("unknown msg code %s" % msg_code) if expect and msg_code != expect: - raise RiakError("unexpected protocol buffer message code: %d" - % msg_code) + raise RiakError("unexpected protocol buffer message code: %d, %s" + % (msg_code, repr(msg))) return msg_code, msg def _recv_pkt(self): diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index f63fb645..f8d85636 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -63,7 +63,7 @@ class RiakPbcKeyStream(RiakPbcStream): _expect = MSG_CODE_LIST_KEYS_RESP def next(self): - response = super(RiakPbcKeyStream, self).__next__() + response = super(RiakPbcKeyStream, self).next() return response.keys def _is_done(self, response): diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index ebe37685..1c3727a0 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -175,7 +175,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, req.bucket = bucket.name req.key = robj.key - vclock = robj.vclock() + vclock = robj.vclock if vclock: req.vclock = vclock @@ -223,7 +223,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, - MSG_CODE_PUT_RESP) + MSG_CODE_PUT_RESP) if not resp: raise RiakError("missing response object") if len(resp.content) != 1: @@ -254,8 +254,8 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): if pw: req.pw = self.translate_rw_val(pw) - if self.tombstone_vclocks() and robj.vclock(): - req.vclock = robj.vclock() + if self.tombstone_vclocks() and robj.vclock: + req.vclock = robj.vclock req.bucket = bucket.name req.key = robj.key @@ -291,7 +291,7 @@ def get_buckets(self): Serialize bucket listing request and deserialize response """ msg_code, resp = self._request(MSG_CODE_LIST_BUCKETS_REQ, - MSG_CODE_LIST_BUCKETS_RESP) + expect=MSG_CODE_LIST_BUCKETS_RESP) return resp.buckets def get_bucket_props(self, bucket): From 6ec4a1f4ae661a7689308bc45e2751ee7b53b000 Mon Sep 17 00:00:00 2001 From: Michael Clemmons Date: Fri, 28 Dec 2012 18:01:31 -0800 Subject: [PATCH 0275/1060] made create_client use args for host, port, and transport --- riak/tests/test_all.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index c5321f9c..7be714af 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -64,8 +64,8 @@ def create_client(self, host=None, port=None, transport_class=None): host = host or self.host port = port or self.port transport_class = transport_class or self.transport_class - return RiakClient(self.host, self.port, - transport_class=self.transport_class) + return RiakClient(host, port, + transport_class=transport_class) def setUp(self): self.client = self.create_client() From 75c2e5413e3ebd23ff23729ced42d8996e24f179 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 11:18:23 -0500 Subject: [PATCH 0276/1060] Fix the iteration logic in PBC streaming. --- riak/transports/pbc/stream.py | 38 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index f8d85636..b481f811 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -31,28 +31,26 @@ class RiakPbcStream(object): _expect = None def __init__(self, transport): + self.finished = False self.transport = transport def __iter__(self): return self def next(self): - expect = self._expect - try: - resp = self.transport._recv_msg(expect) - if(self._is_done(resp)): - raise StopIteration - else: - return resp - except StopIteration: - pass - except: - # TODO: which exceptions do we expect to be generated? - # Should we raise BadResource? + if self.finished: raise StopIteration + msg_code, resp = self.transport._recv_msg(self._expect) + if(self._is_done(resp)): + self.finished = True + + return resp + def _is_done(self, response): - raise NotImplementedError + # This could break if new messages don't name the field the + # same thing. + return response.done class RiakPbcKeyStream(RiakPbcStream): @@ -64,10 +62,11 @@ class RiakPbcKeyStream(RiakPbcStream): def next(self): response = super(RiakPbcKeyStream, self).next() - return response.keys - def _is_done(self, response): - return response.done + if response.done and not response.HasField('keys'): + raise StopIteration + + return response.keys class RiakPbcMapredStream(RiakPbcStream): @@ -80,7 +79,8 @@ class RiakPbcMapredStream(RiakPbcStream): def next(self): response = super(RiakPbcMapredStream, self).next() - return response.phase, json.loads(response.response) - def _is_done(self, response): - return response.done + if response.done and not response.HasField('response'): + raise StopIteration + + return response.phase, json.loads(response.response) From f263f2f35dcb92d4498bc64f991ba92e9f8c6c9a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 11:31:34 -0500 Subject: [PATCH 0277/1060] Fix get_buckets logic. --- riak/client/operations.py | 2 +- riak/tests/test_kv.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 5a5614ab..448802c4 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -33,7 +33,7 @@ def get_buckets(self): all keys stored in a cluster. """ with self._transport() as transport: - return [RiakBucket(self, name) for name in transport.get_buckets()] + return [self.bucket(name) for name in transport.get_buckets()] def ping(self): """ diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 08a16c04..c18d621e 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -239,7 +239,7 @@ def test_list_buckets(self): bucket = self.client.bucket("list_bucket") bucket.new("one", {"foo": "one", "bar": "red"}).store() buckets = self.client.get_buckets() - self.assertTrue("list_bucket" in buckets) + self.assertTrue(bucket in buckets) class HTTPBucketPropsTest(object): From 9adb48b3b4a1e806e4e02cead4d5095dfe6b5fa2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 11:36:06 -0500 Subject: [PATCH 0278/1060] Add a print format for RiakBucket. --- riak/bucket.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/riak/bucket.py b/riak/bucket.py index 9560026f..1749e546 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -386,3 +386,6 @@ def get_index(self, index, startkey, endkey=None): Queries a secondary index over objects in this bucket, returning keys. """ return self._client.get_index(self.name, index, startkey, endkey) + + def __str__(self): + return ''.format(self.name) From 3ebede28c150e2a05440eb428f89157c2f5f3b86 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 11:39:16 -0500 Subject: [PATCH 0279/1060] Fix PBC list-keys. --- riak/transports/pbc/stream.py | 2 +- riak/transports/pbc/transport.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index b481f811..69594228 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -63,7 +63,7 @@ class RiakPbcKeyStream(RiakPbcStream): def next(self): response = super(RiakPbcKeyStream, self).next() - if response.done and not response.HasField('keys'): + if response.done and len(response.keys) is 0: raise StopIteration return response.keys diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 1c3727a0..8969b338 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -270,7 +270,8 @@ def get_keys(self, bucket): """ keys = [] for keylist in self.stream_keys(bucket): - keys = keys + keylist + for key in keylist: + keys.append(key) return keys From 87a00bde6c6cbd95fc54bc72636e2515a7e76927 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 12:00:34 -0500 Subject: [PATCH 0280/1060] Fix search bug: shouldn't lowercase AND/OR/NOT in query string --- riak/transports/http/resources.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index 538a4126..4adcceea 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -163,11 +163,13 @@ def mkpath(*segments, **query): # Add the query string if it exists _query = {} for key in query: - if query[key] is not None: + if query[key] in [False, True]: + _query[key] = str(query[key]).lower() + elif query[key] is not None: _query[key] = query[key] if len(_query) > 0: - pathstring += "?" + urlencode(_query).lower() + pathstring += "?" + urlencode(_query) if not pathstring.startswith('/'): pathstring = '/' + pathstring From 8443fa089de5954802868bcf62c3f2221fab7cdf Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 12:03:21 -0500 Subject: [PATCH 0281/1060] Fix bug in index_path. --- riak/transports/http/resources.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index 4adcceea..3b930d2f 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -82,7 +82,8 @@ def index_path(self, bucket, index, start, finish=None, **options): if finish: finish = quote_plus(str(finish)) return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), - "index", quote_plus(str(start)), finish, **options) + "index", quote_plus(index), quote_plus(str(start)), + finish, **options) def solr_select_path(self, index, query, **options): if not self.riak_solr_searcher_wm: From d3bc5e7221017a80ea979923ce483442d6e6c7a6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 12:20:50 -0500 Subject: [PATCH 0282/1060] Fix encoding differences for binary objects between HTTP/PBC, 2.6/2.7. ALL TESTS GREEN! --- riak/bucket.py | 4 ++++ riak/transports/pbc/codec.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/riak/bucket.py b/riak/bucket.py index 1749e546..2fa2d088 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -340,6 +340,10 @@ def new_binary_from_file(self, key, filename): """ binary_data = open(filename, "rb").read() mimetype, encoding = mimetypes.guess_type(filename) + if encoding: + binary_data = bytearray(binary_data, encoding) + else: + binary_data = bytearray(binary_data) if not mimetype: mimetype = 'application/octet-stream' return self.new_binary(key, binary_data, mimetype) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index ae9a67b9..b7df3eda 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -140,4 +140,4 @@ def encode_content(self, metadata, data, rpb_content): pb_link.bucket = link.get_bucket() pb_link.key = link.get_key() pb_link.tag = link.get_tag() - rpb_content.value = data + rpb_content.value = str(data) From 117f00cd04818212912f797562e771c5b47299a3 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 12:25:51 -0500 Subject: [PATCH 0283/1060] Fix a few pyflakes and pep8 issues. --- riak/client/operations.py | 1 - riak/client/transport.py | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 448802c4..036f53af 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -16,7 +16,6 @@ under the License. """ -from riak.bucket import RiakBucket from transport import RiakClientTransport diff --git a/riak/client/transport.py b/riak/client/transport.py index c6db8823..39d9aa33 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -30,6 +30,11 @@ class RiakClientTransport(object): RETRY_COUNT = 3 + # These will be set or redefined by the RiakClient initializer + protocol = 'http' + _http_pool = None + _pb_pool = None + @contextmanager def _transport(self, protocol=None): if not protocol: @@ -63,14 +68,9 @@ def _skip_bad_nodes(transport): raise BadResource(e) else: raise e - except BadResource as br: + except BadResource: continue - # These will be set or redefined by the RiakClient initializer - protocol = 'http' - _http_pool = None - _pb_pool = None - def is_retryable(error): return is_pbc_retryable(error) or is_http_retryable(error) From 6c3d67338847ab55fb31828efdfb75f10344ceca Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 08:16:25 -0500 Subject: [PATCH 0284/1060] Remove capitalized methods from RiakHttpConnection. --- riak/transports/http/connection.py | 15 ---------- riak/transports/http/transport.py | 45 ++++++++++++++++-------------- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py index 09550d21..5f0a143c 100644 --- a/riak/transports/http/connection.py +++ b/riak/transports/http/connection.py @@ -24,21 +24,6 @@ class RiakHttpConnection(object): Connection and low-level request methods for RiakHttpTransport. """ - def GET(self, uri, headers={}): - return self._request('GET', uri, headers, '') - - def POST(self, uri, headers={}, body=''): - return self._request('POST', uri, headers, body) - - def PUT(self, uri, headers={}, body=''): - return self._request('PUT', uri, headers, body) - - def DELETE(self, uri, headers={}, body=''): - return self._request('DELETE', uri, headers, body) - - def HEAD(self, uri, headers={}): - return self._request('HEAD', uri, headers, '') - def _request(self, method, uri, headers={}, body=''): """ Given a Method, URL, Headers, and Body, perform and HTTP request, diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 342188bd..183315dc 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -80,14 +80,15 @@ def ping(self): """ Check server is alive over HTTP """ - response = self.GET(self.ping_path()) + response = self._request('GET', self.ping_path()) return(response is not None) and (response[1] == 'OK') def stats(self): """ Gets performance statistics and server information """ - response = self.GET(self.stats_path(), {'Accept': 'application/json'}) + response = self._request('GET', self.stats_path(), + {'Accept': 'application/json'}) if response[0]['http_code'] is 200: return json.loads(response[1]) else: @@ -111,7 +112,7 @@ def get_resources(self): Gets a JSON mapping of server-side resource names to paths :rtype dict """ - response = self.GET('/', {'Accept': 'application/json'}) + response = self._request('GET', '/', {'Accept': 'application/json'}) if response[0]['http_code'] is 200: return json.loads(response[1]) else: @@ -125,7 +126,7 @@ def get(self, robj, r=None, pr=None, vtag=None): # unknown flags/params. params = {'r': r, 'pr': pr, 'vtag': vtag} url = self.object_path(robj.bucket.name, robj.key, **params) - response = self.GET(url) + response = self._request('GET', url) return self.parse_body(response, [200, 300, 404]) def put(self, robj, w=None, dw=None, pw=None, return_body=True, @@ -149,9 +150,9 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, def do_put(self, url, headers, content, return_body=False, key=None): if key is None: - response = self.POST(url, headers, content) + response = self._request('POST', url, headers, content) else: - response = self.PUT(url, headers, content) + response = self._request('PUT', url, headers, content) if return_body: return self.parse_body(response, [200, 201, 300]) @@ -172,7 +173,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if if_none_match: headers["If-None-Match"] = "*" content = robj.get_encoded_data() - response = self.POST(url, headers, content) + response = self._request('POST', url, headers, content) location = response[0]['location'] idx = location.rindex('/') key = location[(idx + 1):] @@ -194,7 +195,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): url = self.object_path(robj.bucket.name, robj.key, **params) if self.tombstone_vclocks() and robj.vclock is not None: headers['X-Riak-Vclock'] = robj.vclock - response = self.DELETE(url, headers) + response = self._request('DELETE', url, headers) self.check_http_code(response, [204, 404]) return self @@ -203,7 +204,7 @@ def get_keys(self, bucket): Fetch a list of keys for the bucket """ url = self.key_list_path(bucket.name) - response = self.GET(url) + response = self._request('GET', url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: @@ -217,7 +218,7 @@ def get_buckets(self): Fetch a list of all buckets """ url = self.bucket_list_path() - response = self.GET(url) + response = self._request('GET', url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: @@ -232,7 +233,7 @@ def get_bucket_props(self, bucket): """ # Run the request... url = self.bucket_properties_path(bucket.name) - response = self.GET(url) + response = self._request('GET', url) headers = response[0] encoded_props = response[1] @@ -251,7 +252,7 @@ def set_bucket_props(self, bucket, props): content = json.dumps({'props': props}) # Run the request... - response = self.PUT(url, headers, content) + response = self._request('PUT', url, headers, content) # Handle the response... if response is None: @@ -281,7 +282,7 @@ def mapred(self, inputs, query, timeout=None): # Do the request... url = self.mapred_path() headers = {'Content-Type': 'application/json'} - response = self.POST(url, headers, content) + response = self._request('POST', url, headers, content) # Make sure the expected status code came back... status = response[0]['http_code'] @@ -297,7 +298,8 @@ def get_index(self, bucket, index, startkey, endkey=None): """ Performs a secondary index query. """ - response = self.GET(self.index_path(bucket, index, startkey, endkey)) + url = self.index_path(bucket, index, startkey, endkey) + response = self._request('GET', url) headers, data = response self.check_http_code(response, [200]) jsonData = json.loads(data) @@ -316,7 +318,8 @@ def search(self, index, query, **params): options['q.op'] = op options.update(params) - response = self.GET(self.solr_select_path(index, query, **options)) + url = self.solr_select_path(index, query, **options) + response = self._request('GET', url) headers, data = response self.check_http_code(response, [200]) if 'json' in headers['content-type']: @@ -345,9 +348,9 @@ def fulltext_add(self, index, docs): root.appendChild(doc_element) xml.appendChild(root) - self.POST(self.solr_update_path(index), - {'Content-Type': 'text/xml'}, - xml.toxml()) + self._request('POST', self.solr_update_path(index), + {'Content-Type': 'text/xml'}, + xml.toxml()) def fulltext_delete(self, index, docs=None, queries=None): """ @@ -370,9 +373,9 @@ def fulltext_delete(self, index, docs=None, queries=None): xml.appendChild(root) - self.POST(self.solr_update_path(index), - {'Content-Type': 'text/xml'}, - xml.toxml()) + self._request('POST', self.solr_update_path(index), + {'Content-Type': 'text/xml'}, + xml.toxml()) def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] From 88f691cff147eab47a24159a2d3fe25b3417790c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 08:28:00 -0500 Subject: [PATCH 0285/1060] Spec out missing methods. --- riak/transports/transport.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 25e2cba1..b8a82f62 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -119,9 +119,27 @@ def set_bucket_props(self, bucket, props): """ raise NotImplementedError + def get_keys(self, bucket): + """ + Lists all keys within the given bucket. + """ + raise NotImplementedError + + def stream_keys(self, bucket): + """ + Streams the list of keys for the bucket through an iterator. + """ + raise NotImplementedError + def mapred(self, inputs, query, timeout=None): """ - Serialize map/reduce request + Sends a MapReduce request synchronously. + """ + raise NotImplementedError + + def stream_mapred(self, inputs, query, timeout=None): + """ + Streams the results of a MapReduce request through an iterator. """ raise NotImplementedError From af2facaef8ed4771e23f3c50f25231b970007648 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 08:57:52 -0500 Subject: [PATCH 0286/1060] Use a dict to match message code with protobuf message, simplifying decode logic. --- riak/transports/pbc/connection.py | 97 ++++++++----------------------- riak/transports/pbc/messages.py | 46 ++++++++++++++- 2 files changed, 69 insertions(+), 74 deletions(-) diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index 06010fd9..1f3966de 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -18,38 +18,10 @@ import socket import struct -import riak_pb from riak import RiakError from messages import ( - MSG_CODE_ERROR_RESP, - # MSG_CODE_PING_REQ, - MSG_CODE_PING_RESP, - # MSG_CODE_GET_CLIENT_ID_REQ, - MSG_CODE_GET_CLIENT_ID_RESP, - # MSG_CODE_SET_CLIENT_ID_REQ, - MSG_CODE_SET_CLIENT_ID_RESP, - # MSG_CODE_GET_SERVER_INFO_REQ, - MSG_CODE_GET_SERVER_INFO_RESP, - # MSG_CODE_GET_REQ, - MSG_CODE_GET_RESP, - # MSG_CODE_PUT_REQ, - MSG_CODE_PUT_RESP, - # MSG_CODE_DEL_REQ, - MSG_CODE_DEL_RESP, - # MSG_CODE_LIST_BUCKETS_REQ, - MSG_CODE_LIST_BUCKETS_RESP, - # MSG_CODE_LIST_KEYS_REQ, - MSG_CODE_LIST_KEYS_RESP, - # MSG_CODE_GET_BUCKET_REQ, - MSG_CODE_GET_BUCKET_RESP, - # MSG_CODE_SET_BUCKET_REQ, - MSG_CODE_SET_BUCKET_RESP, - # MSG_CODE_MAPRED_REQ, - MSG_CODE_MAPRED_RESP, - # MSG_CODE_INDEX_REQ, - MSG_CODE_INDEX_RESP, - # MSG_CODE_SEARCH_QUERY_REQ, - MSG_CODE_SEARCH_QUERY_RESP + MESSAGE_CLASSES, + MSG_CODE_ERROR_RESP ) @@ -57,6 +29,7 @@ class RiakPbcConnection(object): """ Connection-related methods for RiakPbcTransport. """ + def _encode_msg(self, msg_code, msg=None): if msg is None: return struct.pack("!iB", 1, msg_code) @@ -75,53 +48,18 @@ def _send_msg(self, msg_code, msg): def _recv_msg(self, expect=None): self._recv_pkt() msg_code, = struct.unpack("B", self._inbuf[:1]) - if msg_code == MSG_CODE_ERROR_RESP: - err = riak_pb.RpbErrorResp() - err.ParseFromString(self._inbuf[1:]) + + if msg_code is MSG_CODE_ERROR_RESP: + err = self._parse_msg(msg_code, self._inbuf[1:]) raise RiakError(err.errmsg) - elif msg_code == MSG_CODE_PING_RESP: - msg = None - elif msg_code == MSG_CODE_GET_SERVER_INFO_RESP: - msg = riak_pb.RpbGetServerInfoResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_GET_CLIENT_ID_RESP: - msg = riak_pb.RpbGetClientIdResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_SET_CLIENT_ID_RESP: - msg = None - elif msg_code == MSG_CODE_GET_RESP: - msg = riak_pb.RpbGetResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_PUT_RESP: - msg = riak_pb.RpbPutResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_DEL_RESP: - msg = None - elif msg_code == MSG_CODE_LIST_KEYS_RESP: - msg = riak_pb.RpbListKeysResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_LIST_BUCKETS_RESP: - msg = riak_pb.RpbListBucketsResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_GET_BUCKET_RESP: - msg = riak_pb.RpbGetBucketResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_SET_BUCKET_RESP: - msg = None - elif msg_code == MSG_CODE_MAPRED_RESP: - msg = riak_pb.RpbMapRedResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_INDEX_RESP: - msg = riak_pb.RpbIndexResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_SEARCH_QUERY_RESP: - msg = riak_pb.RpbSearchQueryResp() - msg.ParseFromString(self._inbuf[1:]) + elif msg_code in MESSAGE_CLASSES: + msg = self._parse_msg(msg_code, self._inbuf[1:]) else: raise Exception("unknown msg code %s" % msg_code) + if expect and msg_code != expect: - raise RiakError("unexpected protocol buffer message code: %d, %s" - % (msg_code, repr(msg))) + raise RiakError("unexpected protocol buffer message code: %d, %r" + % (msg_code, msg)) return msg_code, msg def _recv_pkt(self): @@ -150,6 +88,19 @@ def _connect(self): def close(self): self._socket.shutdown(socket.SHUT_RDWR) + def _parse_msg(self, code, packet): + try: + pbclass = MESSAGE_CLASSES[code] + except KeyError: + pbclass = None + + if pbclass is None: + return None + + pbo = pbclass() + pbo.ParseFromString(packet) + return pbo + # These are set in the RiakPbcTransport initializer _address = None _timeouts = {} diff --git a/riak/transports/pbc/messages.py b/riak/transports/pbc/messages.py index c8175867..fb9dba5c 100644 --- a/riak/transports/pbc/messages.py +++ b/riak/transports/pbc/messages.py @@ -16,7 +16,10 @@ under the License. """ -## Protocol codes +import riak_pb + + +# Protocol codes MSG_CODE_ERROR_RESP = 0 MSG_CODE_PING_REQ = 1 MSG_CODE_PING_RESP = 2 @@ -46,3 +49,44 @@ MSG_CODE_INDEX_RESP = 26 MSG_CODE_SEARCH_QUERY_REQ = 27 MSG_CODE_SEARCH_QUERY_RESP = 28 + +# These responses don't include messages +EMPTY_RESPONSES = [ + MSG_CODE_PING_RESP, + MSG_CODE_SET_CLIENT_ID_RESP, + MSG_CODE_DEL_RESP, + MSG_CODE_SET_BUCKET_RESP +] + +# Mapping from code to protobuf class +MESSAGE_CLASSES = { + MSG_CODE_ERROR_RESP: riak_pb.RpbErrorResp, + MSG_CODE_PING_REQ: None, + MSG_CODE_PING_RESP: None, + MSG_CODE_GET_CLIENT_ID_REQ: None, + MSG_CODE_GET_CLIENT_ID_RESP: riak_pb.RpbGetClientIdResp, + MSG_CODE_SET_CLIENT_ID_REQ: riak_pb.RpbSetClientIdReq, + MSG_CODE_SET_CLIENT_ID_RESP: None, + MSG_CODE_GET_SERVER_INFO_REQ: None, + MSG_CODE_GET_SERVER_INFO_RESP: riak_pb.RpbGetServerInfoResp, + MSG_CODE_GET_REQ: riak_pb.RpbGetReq, + MSG_CODE_GET_RESP: riak_pb.RpbGetResp, + MSG_CODE_PUT_REQ: riak_pb.RpbPutReq, + MSG_CODE_PUT_RESP: riak_pb.RpbPutResp, + MSG_CODE_DEL_REQ: riak_pb.RpbDelReq, + MSG_CODE_DEL_RESP: None, + MSG_CODE_LIST_BUCKETS_REQ: None, + MSG_CODE_LIST_BUCKETS_RESP: riak_pb.RpbListBucketsResp, + MSG_CODE_LIST_KEYS_REQ: riak_pb.RpbListKeysReq, + MSG_CODE_LIST_KEYS_RESP: riak_pb.RpbListKeysResp, + MSG_CODE_GET_BUCKET_REQ: riak_pb.RpbGetBucketReq, + MSG_CODE_GET_BUCKET_RESP: riak_pb.RpbGetBucketResp, + MSG_CODE_SET_BUCKET_REQ: riak_pb.RpbSetBucketReq, + MSG_CODE_SET_BUCKET_RESP: None, + MSG_CODE_MAPRED_REQ: riak_pb.RpbMapRedReq, + MSG_CODE_MAPRED_RESP: riak_pb.RpbMapRedResp, + MSG_CODE_INDEX_REQ: riak_pb.RpbIndexReq, + MSG_CODE_INDEX_RESP: riak_pb.RpbIndexResp, + MSG_CODE_SEARCH_QUERY_REQ: riak_pb.RpbSearchQueryReq, + MSG_CODE_SEARCH_QUERY_RESP: riak_pb.RpbSearchQueryResp +} From 80476eeb8693b683c743c427335968d2bcb13e4c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 09:00:05 -0500 Subject: [PATCH 0287/1060] Fix protobuf build dependency. --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3e63c8c1..a47837e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,9 @@ language: python python: - "2.6" - "2.7" -install: ./setup.py develop +install: + - ./setup.py develop + - ./setup.py easy_install protobuf script: ./setup.py test before_script: sudo /usr/sbin/search-cmd install searchbucket notifications: From 2adb47d8814a77e8c0d2a7a861c5d18f37ba69d3 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 12:18:02 -0500 Subject: [PATCH 0288/1060] Add key streaming to HTTP. --- riak/transports/http/connection.py | 12 ++++-- riak/transports/http/resources.py | 4 +- riak/transports/http/stream.py | 66 ++++++++++++++++++++++++++++++ riak/transports/http/transport.py | 13 +++++- 4 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 riak/transports/http/stream.py diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py index 5f0a143c..1a8dca10 100644 --- a/riak/transports/http/connection.py +++ b/riak/transports/http/connection.py @@ -24,7 +24,7 @@ class RiakHttpConnection(object): Connection and low-level request methods for RiakHttpTransport. """ - def _request(self, method, uri, headers={}, body=''): + def _request(self, method, uri, headers={}, body='', stream=False): """ Given a Method, URL, Headers, and Body, perform and HTTP request, and return a 2-tuple containing a dictionary of response headers @@ -39,10 +39,14 @@ def _request(self, method, uri, headers={}, body=''): for (key, value) in response.getheaders(): response_headers[key.lower()] = value - # TODO: Support streaming responses - response_body = response.read() + if stream: + # The caller is responsible for fully reading the + # response and closing it when streaming. + response_body = response + else: + response_body = response.read() finally: - if response: + if response and not stream: response.close() return response_headers, response_body diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index 3b930d2f..b551cbae 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -55,8 +55,8 @@ def bucket_properties_path(self, bucket, **options): return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), **query) def key_list_path(self, bucket, **options): - query = options.copy() - query.update(keys=True, props=False) + query = {'keys': True, 'props': False} + query.update(options) if self.riak_kv_wm_buckets: return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), "keys", **query) diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py new file mode 100644 index 00000000..46d57cac --- /dev/null +++ b/riak/transports/http/stream.py @@ -0,0 +1,66 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import json +import string + +class RiakHttpStream(object): + """ + Base class for HTTP streaming iterators. + """ + + BLOCK_SIZE = 2048 + + def __init__(self, response): + self.response = response + self.buffer = '' + self.response_done = False + + def __iter__(self): + return self + + def read(self): + chunk = self.response.read(self.BLOCK_SIZE) + if chunk is '': + self.response_done = True + self.buffer += chunk + + def next(self): + raise NotImplementedError + +class RiakHttpKeyStream(RiakHttpStream): + """ + Streaming iterator for list-keys over HTTP + """ + + def next(self): + while True: + while '}' not in self.buffer and not self.response_done: + self.read() + + if '}' in self.buffer: + idx = string.index(self.buffer, '}') + 1 + chunk = self.buffer[:idx] + self.buffer = self.buffer[idx:] + keys = json.loads(chunk)[u'keys'] + if len(keys) is 0: + continue + else: + return keys + else: + raise StopIteration diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 183315dc..ead726bd 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -28,6 +28,7 @@ from riak.transports.http.resources import RiakHttpResources from riak.transports.http.connection import RiakHttpConnection from riak.transports.http.search import XMLSearchResult +from riak.transports.http.stream import RiakHttpKeyStream from riak.metadata import ( MD_CHARSET, MD_CTYPE, @@ -207,12 +208,22 @@ def get_keys(self, bucket): response = self._request('GET', url) headers, encoded_props = response[0:2] - if headers['http_code'] == 200: + if headers['http_code'] is 200: props = json.loads(encoded_props) return props['keys'] else: raise Exception('Error listing keys.') + def stream_keys(self, bucket): + url = self.key_list_path(bucket.name, keys='stream') + headers, response = self._request('GET', url, stream=True) + + if headers['http_code'] is 200: + return RiakHttpKeyStream(response) + else: + stream.close() + raise Exception('Error listing keys.') + def get_buckets(self): """ Fetch a list of all buckets From 44a25e598bf4d8b0a34d99689a5d0ddaf84332cd Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 13:10:22 -0500 Subject: [PATCH 0289/1060] Simplify HTTP key streaming by pushing empty-result culling into the client. --- riak/client/operations.py | 3 ++- riak/transports/http/stream.py | 26 +++++++++++--------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 036f53af..e9fd8ee1 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -80,7 +80,8 @@ def stream_keys(self, bucket): """ with self._transport() as transport: for keylist in transport.stream_keys(bucket): - yield keylist + if len(keylist) > 0: + yield keylist def put(self, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=None): diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 46d57cac..77a84f6f 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -49,18 +49,14 @@ class RiakHttpKeyStream(RiakHttpStream): """ def next(self): - while True: - while '}' not in self.buffer and not self.response_done: - self.read() - - if '}' in self.buffer: - idx = string.index(self.buffer, '}') + 1 - chunk = self.buffer[:idx] - self.buffer = self.buffer[idx:] - keys = json.loads(chunk)[u'keys'] - if len(keys) is 0: - continue - else: - return keys - else: - raise StopIteration + while '}' not in self.buffer and not self.response_done: + self.read() + + if '}' in self.buffer: + idx = string.index(self.buffer, '}') + 1 + chunk = self.buffer[:idx] + self.buffer = self.buffer[idx:] + keys = json.loads(chunk)[u'keys'] + return keys + else: + raise StopIteration From 30046643b5f9fdc051e4c51c64fdf852a2c0ab5a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 13:52:56 -0500 Subject: [PATCH 0290/1060] Add unit test for key streaming. --- riak/tests/test_kv.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index c18d621e..5b4a5e90 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -72,6 +72,18 @@ def test_generate_key(self): self.assertNotIn(o.key, existing_keys) self.assertEqual(len(bucket.get_keys()), len(existing_keys) + 1) + def test_stream_keys(self): + bucket = self.client.bucket('random_key_bucket') + regular_keys = bucket.get_keys() + self.assertNotEqual(len(regular_keys), 0) + streamed_keys = [] + for keylist in bucket.stream_keys(): + self.assertNotEqual([], keylist) + for key in keylist: + self.assertIsInstance(key, basestring) + streamed_keys += keylist + self.assertEqual(sorted(regular_keys), sorted(streamed_keys)) + def test_binary_store_and_get(self): bucket = self.client.bucket('bucket') # Store as binary, retrieve as binary, then compare... From 87ba4422403ca39ebb0d42157fd3d94c2c46cb5a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 17:17:40 -0500 Subject: [PATCH 0291/1060] Add streaming MapReduce to HTTP. --- riak/transports/http/stream.py | 57 +++++++++++++++++++++++++++++++ riak/transports/http/transport.py | 30 ++++++++++------ riak/transports/pbc/transport.py | 6 +--- riak/transports/transport.py | 13 +++++++ 4 files changed, 91 insertions(+), 15 deletions(-) diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 77a84f6f..960fd3c3 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -18,6 +18,9 @@ import json import string +import re +from cgi import parse_header +from email import message_from_string class RiakHttpStream(object): """ @@ -60,3 +63,57 @@ def next(self): return keys else: raise StopIteration + +class RiakHttpMultipartStream(RiakHttpStream): + """ + Streaming iterator for multipart messages over HTTP + """ + def __init__(self, response): + super(RiakHttpMultipartStream, self).__init__(response) + ctypehdr = response.getheader('content-type') + _, params = parse_header(ctypehdr) + self.boundary_re = re.compile('\r?\n--%s(?:--)?\r?\n' % + re.escape(params['boundary'])) + self.next_boundary = None + self.seen_first = False + + def next(self): + # multipart/mixed starts with a boundary, then the first part. + if not self.seen_first: + self.read_until_boundary() + self.advance_buffer() + self.seen_first = True + + self.read_until_boundary() + + if self.next_boundary: + part = self.advance_buffer() + message = message_from_string(part) + return message + else: + raise StopIteration + + def try_match(self): + self.next_boundary = self.boundary_re.search(self.buffer) + return self.next_boundary + + def advance_buffer(self): + part = self.buffer[:self.next_boundary.start()] + self.buffer = self.buffer[self.next_boundary.end():] + self.next_boundary = None + return part + + def read_until_boundary(self): + while not self.try_match() and not self.response_done: + self.read() + + +class RiakHttpMapReduceStream(RiakHttpMultipartStream): + """ + Streaming iterator for MapReduce over HTTP + """ + + def next(self): + message = super(RiakHttpMapReduceStream, self).next() + payload = json.loads(message.get_payload()) + return payload['phase'], payload['data'] diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index ead726bd..8cbbcb7c 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -28,7 +28,10 @@ from riak.transports.http.resources import RiakHttpResources from riak.transports.http.connection import RiakHttpConnection from riak.transports.http.search import XMLSearchResult -from riak.transports.http.stream import RiakHttpKeyStream +from riak.transports.http.stream import ( + RiakHttpKeyStream, + RiakHttpMapReduceStream + ) from riak.metadata import ( MD_CHARSET, MD_CTYPE, @@ -279,16 +282,8 @@ def mapred(self, inputs, query, timeout=None): """ Run a MapReduce query. """ - if not self.phaseless_mapred() and (query is None or len(query) is 0): - raise Exception( - 'Phase-less MapReduce is not supported by Riak node') - # Construct the job, optionally set the timeout... - job = {'inputs': inputs, 'query': query} - if timeout is not None: - job['timeout'] = timeout - - content = json.dumps(job) + content = self._construct_mapred_json(inputs, query, timeout) # Do the request... url = self.mapred_path() @@ -305,6 +300,21 @@ def mapred(self, inputs, query, timeout=None): result = json.loads(response[1]) return result + def stream_mapred(self, inputs, query, timeout=None): + content = self._construct_mapred_json(inputs, query, timeout) + + url = self.mapred_path(chunked=True) + reqheaders = {'Content-Type': 'application/json'} + headers, response = self._request('POST', url, reqheaders, + content, stream=True) + + if headers['http_code'] is 200: + return RiakHttpMapReduceStream(response) + else: + raise Exception( + 'Error running MapReduce operation. Headers: %s Body: %s' % + (repr(headers), repr(response.read()))) + def get_index(self, bucket, index, startkey, endkey=None): """ Performs a secondary index query. diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 8969b338..bc60cc21 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -352,11 +352,7 @@ def mapred(self, inputs, query, timeout=None): def stream_mapred(self, inputs, query, timeout=None): # Construct the job, optionally set the timeout... - job = {'inputs': inputs, 'query': query} - if timeout is not None: - job['timeout'] = timeout - - content = json.dumps(job) + content = self._construct_mapred_json(inputs, query, timeout) req = riak_pb.RpbMapRedReq() req.request = content diff --git a/riak/transports/transport.py b/riak/transports/transport.py index b8a82f62..27d1fc52 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -22,6 +22,7 @@ import threading import platform import os +import json from feature_detect import FeatureDetection @@ -230,3 +231,15 @@ def _get_index_mapred_emu(self, bucket, index, startkey, endkey=None): 'key': startkey}, phases) return [key for bucket, key in result] + + def _construct_mapred_json(self, inputs, query, timeout=None): + if not self.phaseless_mapred() and (query is None or len(query) is 0): + raise Exception( + 'Phase-less MapReduce is not supported by Riak node') + + job = {'inputs': inputs, 'query': query} + if timeout is not None: + job['timeout'] = timeout + + content = json.dumps(job) + return content From e10e1a74e6fb11f9265ce19eaf9346175cc1ef76 Mon Sep 17 00:00:00 2001 From: "Anton (Atilla) Tsigularov" Date: Thu, 10 Jan 2013 15:41:24 +0100 Subject: [PATCH 0292/1060] Fixes in order to make TestServer work. Importing the test backends from the ruby client modules, as advised. Fixing the config file so the recycle() command will work. --- riak/erl_src/riak_kv_test_backend.beam | Bin 5984 -> 9076 bytes riak/erl_src/riak_kv_test_backend.erl | 421 +++++++++++++++------ riak/erl_src/riak_search_test_backend.beam | Bin 4440 -> 4484 bytes riak/erl_src/riak_search_test_backend.erl | 6 +- riak/test_server.py | 6 +- 5 files changed, 323 insertions(+), 110 deletions(-) diff --git a/riak/erl_src/riak_kv_test_backend.beam b/riak/erl_src/riak_kv_test_backend.beam index 485d6b75110592ff546372210b389188cc490b74..e118809d4d755a81155d0d22b96ffc4d3f5c578e 100644 GIT binary patch literal 9076 zcmbVS4R9OBb-v>QcoHB>p!KuFQH1qtSfHZVi3LEwk%CA9 zFzzTzj-Ak=Y*S9^GNJ9%Nu0>~shy~?l*rCc6SuB2o&KclB%QR=Nz#m)OedX5I+J9^ z$#go?>G$pq5-dmQq!>Qj-n+MN-}~O%x4TCS-MfFiBpKtG9Rqs@-1>whh1W`whTP_)-CZs<~O><}3=Z?aagN2)cu(rGuxOSMsX%}v#sYmeH^f?F@ph#iSUH(lwz1jm@| zI;>_tOyjwBM|~{hxb;S7#U8g^n~mCT=Xkw7HreQKo$yF$+_B|C_e9ApA1U}Ea&N?e zs)RxIabHjU@KL+$I`bOMdc%Wp9tn|Ji=(uH9d6oip(}`Z zq||5>j@aW3yV*sC*|lRg0o7({)NVy@p(k3dH|;{RS{p4)lxld|oKNBtCp_MQcZ)>8 zhLzQ_?WpZw7Dgs({3>X47j7c*c8C3tFBbqnPa>qAWbLL(P2cN)0h|k7&p;wnvyZz0 zFzx)#O1*%%c2;YSjSRJqmuA2yMYSVFf9Mh~&AMw#&;HP9E$h!HyhXo4kP2?7( zMWNm#+~-f!E0c&R#oK$;+1_8+vc}$h`5R3(0H0ZM72K@Cm;9ng*n3JBW^?_j#nk@d5o{yNd+FGxd z<`{9j96x0Ct?xB^7y8x&TKZCf2CH7NC28RlYNiUJ&kX1dkSVJjJrD8`(*bRauXu$)?EaUPE;Bd_t9TB5N{DF5*U)8~)8yN6%SZp51&w8_|CEZTBBE*_W9z@m-+3&o z>Z#L`_E=cgwbN2c48lQpQq;P1;rZEspQ206i5OT#!3+!}zYRHIgkX&)J#|BAjRdtp zk|NOBltp7ur(RZv8d^4TOO|B^v!QG%D`(?FI?F~DXSJT3CjX+NCvvMQD{A^3*8dJ` zWSFF7m}+F0p3E@Q$}o{nyrV^K=@!LpDIOID7$ITBL^9r`He@BbJq3P`h_k5A=-HlH z%VVL43brOk12KPHpD5vjL_V zX}(132k99Nnl;q|BMC-Q&m^UMCaIRw{5Go}q-QM2$yn+%7!DYYo^hnPjH8}Q@Z07h zOH+J_fmXhPud*w&kQmm)IR+y_I(JRJl-kT76+M5jNO1Pru1JfoGU+ohjL@Wqr>gW6VptE?;*A;L5j zQu%3Q*X2RR5PDg;RcMAdrOnJrqQtt)H&vkynFxJ68(EP>j(Af?xy|=v33;-l?5ZLc zSZM`T-VOoQGT!U!TfgL*K9)rcnCH{dT})P%A(AU|vZAN?GN8H4Y5>hV&@5*x9jKbq z8B5G%tlYVbRlJhm%koI4H{V1slQD56E{v4028A|EQ9b91xl>~LS>c=)<=48T?+6x3 z@H^u0KL-Do%gP;U5ikOyrP&Z=vXPB>{x27)Bth_+MX@=b2f*=YN>o^=$RA4c6|5hm zxAHr|S7?#N32vN%W*3^{vSJ9s5+Q@SKbu5HVO0S1a7geEKNgnWb}JtLHA9@Ygq#^hGWG&z;7k+yG_W-sx)7T z^jw*2AU!RlCsI@|XOgCqN#-yMR$9Pq3X>oj6Gog;UmFx`SmY}r=_dG{4qTarD{Exs zPSuUwVj@6hPt7&qm3^%}MwTBNqF>vEmPH#dc`08Y2 zDS2aQ$UYP%WUiNntt5GS^7S-R z*Dn$$`8rJ2K3Q2uL&MS#84DA<#NwP0x!6QHWql2LwFKp#^$qI_{#t^yy z4V3B-%?-bg;Y$xtNE1%|z-a?;x;xGLfm8oAoD#gBFcMfDL-1A@K0?4MpX$<; z9>Rf&A{=^vOhZ6D(_)@!+sL$SUTxlTl5drjEh3JrgczFDWHd^4X;-zRlEF%Ligq;IJsrt*i{R zz+0S+_fQsQFv=#3Vx@Toqhzj)lHeKAE}0^-S%Z4u5t|=YZf3Gy@CVWJZuH!d=7Z=t zc&%q^iBIrB(xB(sja%qrij@u)Ellts7^t4{>(`bn+o5$J-BEcgFR-Kq%2PJeE?%$`+Bes z>jB}XDz~C?yzcE&D9U?9c9z!HeKeu(g~pxGoEufax(|1|pnrdwzhB#;@%t7Do##zH zEGv0zISf?RWP;ykMEXQh8zqQ_TkMRcTi&DfX@1b!V)28Egy08E{yteb&{{qd{9rz^ z+vNw#4aqsh4?2o2oxlZCNO*YmWs{wl__Ff8cXip*-et4+?OhIzQewRyMraXrRG@u7 z+}R6v9!T&H5HAPb2dL@;Q-h5bar;4t>_ea)6h@L_kYu9`un&QKZ%$SUX@1D)2btK* zVGoBLfT24HekdP-pNE_+4nMR=n7kw zIDw1g4Icszl%){-9GLDHo29yD~^4y!!w@&f$w>;p=-;%@? zi+Z2S!*Wu22EEIq5**T6CaqVj(aYdh;8YGy*=b%yJ*r;YH759|mG91>9AOELBesBt z_Jj-tY0%q$js(l>QIul?n5*nS2}VHdvDC6^eG8g>&2%vNDq5 zN3k;gpK_xx%z|N-+1kBgzcrWORVRNZua>9rHCOJ=q3#UgAQ6~pp@L$=6!*+La?Q*^ z&&-q-O_7-?Z&83ro|RZsU}Xvg%ffMktOP7aJS%CX`JG}_cn>g-Qmq^VqWjTxJk4uB zw3ZaDeK=M^2TXWVvjn`E#_n+r-kihk5z(wo=kYaHzLVFSbNIUA{7$zf0=MeefBZgf zjl;Hy6tA=2?LCRre7EZ4Rs(M3i?Z?%&lQWv02~l0Ef@|ZX1nBJv z-o)7viGg(pYc@(|XmCcTv^zp#%%~D=t6B?A+oIH@669dC1DK_5w~jzjG-;Pi*`h-C z5_OY~BetrR*ea=1{f|}ZAv?TETBWX%TGv9jxJqJJBbKrd>roM$NvOOZiFa%>f5aFt ztbCeJA{3KeVwDjJ2ceiY`DFR-GM~gs`T&18@&SHaRvzYJYWAwXI2qDmCYE)w)4tnA zHhTtBjhtS~Pq33)`J=LOoQvFbowQabPk1)LA7v+Hq*Op%q3K;*%qtV}(|A1_|k z^2n^mSs9yT=zF{yQN*?yn`?h&TLv9@jhGiyI{Q!4WR@?`hYYOR;EbM)!8x zAr?l8>9X=fia*5&;}|Lx3&S@6_vAexS$T?5dj`H7R%PXMntzP-gRUN+z|g>;<3sCg z{j|}~pY~1-Ug=Nsr}JRs^-NxJ68vcgM+iU(Oi^zUx~=@9vhuWtagrW@B`Y6&G%S=m zH0E6l9H+$FEVxUy?tt6J$hbd%ab*Dg3^uIEPQhnoCH3_$}-qZ73)F^w^=4TP@vtG0c_k^jC0a_L#>|sX*t6ud-KJ;DE53ryLqQF` zeId;+Lcv8(0qnR4zb;zjR~~+qSM!VcWVh4^w7{M`3lv@S75xbm)nUmO)BIH^dbMTA ztFYu%awxDweUBxt`U);VK?4ddr}-r)xYSZ`2?{QGn+%wBNyCk%cZ(e6R|*?Rjdmu6 zx5v=1cq?&}U!jA|6*_Z$31_Y^%gUF6V?zq32C6P!_SS3zvmU7zV*dUEn+>cnIxQ9} zf;A@OA9rafxIz=_HCX7Vvhw<7{&m)EssoILw#E5Zu%uo8qR~IEd}Vta&%cU^b%WQH zzSvhiv9I|x2`2rPx1he@*S2GyL?!BV;Pmwre~s$$pTa5^_uuF?&0x1>#IK#s$%(jk zsB4|hzX^1|>7nZ%g}!+YV|lkZjCaoS<9J;j_g_*=J0}|8$$tm_N>B(C2FajKP#35J z6b1FPwKcR!hWN2I{>=e@A+C!8KDBRa<1e|6Px?vbj(|@(m$$VIv>Ax_t3dRj_MIT= zOaF_cvFJm3dO;*l_M1UFiE9ef8tX2!$$pYg2Yk}mAGBM1lF5LqfZuA9{vDtp5Ph9t1Xe|1_NCZSa%?pUylz((jGSr`V^ywIytieY) zjy8_9k_#f8-vH5v$#1+gf@_1?pvxfAAxq2u^DbO#AhPWXAbQ>*e{2)Q693!w?q9~0 zp39;A|60eDh~&NqA{%7sd;6cpbsR+e^B}TM?#Lb>o5n!Y|0Pf-sH2npjB?~Ch~z&9 zBHtMMuXo7rQ4q<$08*$eC!Tx|SMq`Se-cD;ozjcnr8rc*wj{lXE8){A@A~|5T#tZA z@25fYsok-B>Yw*iUCIa8F002s^Taa)^WQSNpSW^5|CN8+S^sRP{y#5&qBmOp*~uZR)-A_^)~n{pi0BDVQ;rU+h~k@PnV6dU5mVx8(Wn{OsqSJoooM7}&J-Gix@E zbzgdE(@(z}{fk|lKVJ3J#gV_d_ovH_e@gq3eq`w0pTDr@?R#>2fAbGJFa3II(?;Xi zGuoej`_W&1`{gsA>V07JazW2haf9dMJ`Qg7F{LSAs_8%nyCHO5z)03)5*ZnaBAJAbxBehqJPXE={$^i^utnDsk3ww kzUO(vIr!;H*&dU2|NiEBE?ob!nRW1ma2h!5P`su81r)UzjQ{`u literal 5984 zcmbVQTWlNGnV!)ZiX%!V9&sqi@g*L67|Ss&YZTHKN|AlUk;OQ^gchAf&5fkUkxYvs zY0gN#)GmzJ1r+Q)IJFnshdv0&1`90qA;`n-zHK3F7uaH36lnTTbRU8Z7TAZP2-*fH ziazxFheM0b1!zm)*E#1u|M}1NU(SDyX71(r2}v4R$efwEFyl2=C5iouBuNXgR<*QJ zSXnQ4j_VZ`OXU@(UJ11v*YSq^sdA&`6k65#a$&VpFD*N*{$LJ%FwLIo|wa9AeMq$-iZM1Iot<@{eQnl_>dYx9SR9}`4 z@2J*22b&bUMxj=9y#e||gAy<^$8%ZTxlYPFM{&JIv$x{Z9M56E)LUybR@Rz5o*P;! z)m%sMT5C?oX|)=yzIwweEH$tlYp!{$-ncF&+v{}u+Ug)TliEvu!SI2{cZ3C@>Kt7qp}9Cki&KmkgjoTY}>N& zGHkOSGn}v<$NgiV}OgGbfNS_2586C@Y%?2Y0M$*V6rFVEqB9(;28xf;1 zt$Em5&e#TQw(ezY@i1fOVCqO7;oAHVxt5HH_u);bnicG_cvunlJmKCK2!J26D(q7Lh|XatL{3lru@;Cb(h4eZ{+!(0*d3 zM1EEzFDLkc@|JLQ;j(ESm(>HBbyv7CQTDSog{;lDv-Vj(Yg7DLWUh(KH7Rp7h5q5XHlfogK)x80H82gwa|JW*M2HsioC)2=uo zVzvSP)Zz&`H3^)WUHlME&U3PQh_;TSGO=YT2u$Q)J2EH<>T_Y~lYkuq*B9Kdq_Fi_ znjeOP4l@JORDwDM3Kdm7OlT7n!6ShD93YRS`4K=q(hd0tK^8?MHU-YqMdUD5T9HqM z4OK(xv>j@2J1D%qA+fLlLo71H@{j*f7&9)bNBxryU$YHC_`aNFKIVf?0(44NliiTM5_GCt;%h)3BhbeI`lu$W zC(`^l0(1Pyiy^^})2(0#s@%wjq!QND!6W>nvgLc3MzVTRvrtp0gvO!p(&2 zomRHAoL?Ct$Fl5<-oyOrJl#$Np%da5-HbRsk>kDuk`4v%SXMb>qLNT8KZSiySh9Lr zv#hoyCb+oXBPW4oW|?F%7^MwawG(_+-&!U^XXz*-;Y_wnS$_r+laQE8@GK+*SlMlf z9ELAKVha156=sqGon*^JuwMfE1!T@St`)_cVdG)|Mm=k1O&9N>6i_uM@=o^}br6bx215YSTE3r|jRQ>-^?JXxXDvfHRQl~A1ilY)k zWOp*9?cvK%IR_J8dD22NYs0*YFi%#$0g){5zIul*g(853tFCq9l zC6eTCdY|CTvpBBx*XZ(4m`!CSmc*5T5*fms0#)CH$`@hCYmZ-LIxZJHVbrG+{B;(2 zNg(kkbgxqw_fr|5jInSX`p+&J2Spm?4q?E1<&B{7zlqq;)69RE#WxJNhH80;LL;kh zr1+aC)HE9cM7aM=I&NTaF+jfv}3JYc~n*%`t5MS@la zvJSHDm&iF_8Y*u1WiVfcIaeO1@I~u0S*?PdLwv8Mc@>6LeZy?L zp7UVXVAyzD7xA1a<2eKLD`gZm z8rD!>Ye{I#;bv#_IwVbHXmZJP*AA4`p<^DX8fjjKj(Qsv20&CNM7hobE5Vz2lM)Lt zE$injA4{`?^mol`dAg zNrN6Hn9(uNjRBFv>oG=Wul6{sCuxa;p1+ilMacX${W4PJzDxh_3?T%>m1zazmv)zI}m{IGrp! zh1(bSE!c6(x1(LOZ&BI#7L+V#vU>X@zrzMJZH6&a-Y@;9LKjB+klcpwofQ8T@)7@l z<-OKrG}77CCR+JEx(0}*6cGJ-LHOTHl0h8w6sRZY??Bt()BkqV9|T1~Bt!inP=`Mf z@JY`zAo@{%AE=|#=<4rBpRjcFkW3u(EQtD}Ali$5By$Kv@(IupPzS>}`W+0<2YlK$ z*(F2#j?8qx7iecdq|*k?f;u?Q1by1~EU2^g9Qq_r`X>TD^)G&*-@*S1_?`YYsDI`L z#X|ZI5c%{1h;%OneKb-9bOJ>CoCc*pB=Z8u3i^|1j|P2`9Rn#Kvh%s1PrL&`pL|KN zBANZ5y`T_iCukT%dl8-(i1z9S^??XG<%bHQAFUxC{d(rs>SW75gGlC85b3!Iq8|(W z?E?9lWQgy9NVhEC`UT14LBwx?NT)3SBK2Fe^B}_60MUGpGWy#8puH4~SJBeE+;c_W zLYoW5SJC?T*zd-Fh5pMRT7LyZdCq!w_r89<>Jgp=(Cat8UoIAlOU2@YD?cf|RQYfD zqn-Jk{}%n5#Baa*>Cb-S&VO|GzM1Fh?lm(G5XwNwi`HONf10osP^Aw0MkZ$S|pYoXUjN~cTr$8h_ zXMl2!WJq5+=-X&df(XORjOS4-FWjr)Z_kyUip194It}EyCjH~bpZ;NL_TN?S?ahDq z&v$o_0sUSF?5%fgMOhcCJA_D1FF zl@;mv@mKJ*&>eT3Qhj{ZS$ArUCVf&JZ&jPknuDgaGFGltiG{Dc1-DfuZM}^(uTpLO zRqK5#!oip26)BWDdg5q`lq&R@@k;L^&c-iGeWhq=s5Dv{^0#vvYpt@gBAxro;rb?g S@e}z`*g}4qS#&)#(w_j{AzNes diff --git a/riak/erl_src/riak_kv_test_backend.erl b/riak/erl_src/riak_kv_test_backend.erl index 4ddf7455..f03bff35 100644 --- a/riak/erl_src/riak_kv_test_backend.erl +++ b/riak/erl_src/riak_kv_test_backend.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% riak_memory_backend: storage engine using ETS tables +%% riak_kv_test_backend: storage engine using ETS tables, for use in testing. %% %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. %% @@ -32,6 +32,7 @@ %%
      %%
    • `ttl' - The time in seconds that an object should live before being expired.
    • %%
    • `max_memory' - The amount of memory in megabytes to limit the backend to.
    • +%%
    • `test' - When `true', exposes the internal ETS tables so that they can be efficiently cleared using {@link reset/3}.
    • %%
    %% @@ -40,6 +41,8 @@ %% KV Backend API -export([api_version/0, + capabilities/1, + capabilities/2, start/2, stop/1, get/3, @@ -51,18 +54,31 @@ fold_objects/4, is_empty/1, status/1, - callback/3, - reset/0]). + callback/3]). + +%% "Testing" backend API +-export([reset/0]). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). +-compile([export_all]). -endif. -define(API_VERSION, 1). --define(CAPABILITIES, [async_fold]). +-define(CAPABILITIES, [async_fold, indexes]). + +%% Macros for working with indexes +-define(DELETE_PTN(B,K), {{B,'_','_',K},'_'}). + +%% ETS table name macros so we can break encapsulation for testing +%% mode +-define(DNAME(P), list_to_atom("riak_kv_"++integer_to_list(P))). +-define(INAME(P), list_to_atom("riak_kv_"++integer_to_list(P)++"_i")). +-define(TNAME(P), list_to_atom("riak_kv_"++integer_to_list(P)++"_t")). --record(state, {data_ref :: integer() | atom(), - time_ref :: integer() | atom(), +-record(state, {data_ref :: ets:tid(), + index_ref :: ets:tid(), + time_ref :: ets:tid(), max_memory :: undefined | integer(), used_memory=0 :: integer(), ttl :: integer()}). @@ -74,38 +90,55 @@ %% Public API %% =================================================================== -%% TestServer reset - --spec reset() -> ok | {error, timeout}. -reset() -> - {ok, Ring} = riak_core_ring_manager:get_my_ring(), - [ ets:delete_all_objects(list_to_atom("kv" ++ integer_to_list(P))) || - P <- riak_core_ring:my_indices(Ring) ], - ok. - %% KV Backend API %% @doc Return the major version of the -%% current API and a capabilities list. --spec api_version() -> {integer(), [atom()]}. +%% current API. +-spec api_version() -> {ok, integer()}. api_version() -> - {?API_VERSION, ?CAPABILITIES}. + case lists:member({capabilities, 1}, riak_kv_backend:behaviour_info(callbacks)) of + true -> % Using 1.1 API or later + {ok, ?API_VERSION}; + _ -> % Using 1.0 API + {?API_VERSION, ?CAPABILITIES} + end. + +%% @doc Return the capabilities of the backend. +-spec capabilities(state()) -> {ok, [atom()]}. +capabilities(_) -> + {ok, ?CAPABILITIES}. + +%% @doc Return the capabilities of the backend. +-spec capabilities(riak_object:bucket(), state()) -> {ok, [atom()]}. +capabilities(_, _) -> + {ok, ?CAPABILITIES}. %% @doc Start the memory backend -spec start(integer(), config()) -> {ok, state()}. +%% Bug in riak_kv_vnode in 1.0 +start(Partition, [{async_folds,_}=AFolds, Rest]) when is_list(Rest) -> + start(Partition, [AFolds|Rest]); start(Partition, Config) -> - TTL = config_value(ttl, Config), - MemoryMB = config_value(max_memory, Config), + TTL = get_prop_or_env(ttl, Config, memory_backend), + MemoryMB = get_prop_or_env(max_memory, Config, memory_backend), + TableOpts = case get_prop_or_env(test, Config, memory_backend) of + true -> + [ordered_set, public, named_table]; + _ -> + [ordered_set] + end, case MemoryMB of undefined -> MaxMemory = undefined, TimeRef = undefined; _ -> MaxMemory = MemoryMB * 1024 * 1024, - TimeRef = ets:new(list_to_atom(integer_to_list(Partition)), [ordered_set]) + TimeRef = ets:new(?TNAME(Partition), TableOpts) end, - DataRef = ets:new(list_to_atom("kv" ++ integer_to_list(Partition)), [named_table, public]), + IndexRef = ets:new(?INAME(Partition), TableOpts), + DataRef = ets:new(?DNAME(Partition), TableOpts), {ok, #state{data_ref=DataRef, + index_ref=IndexRef, max_memory=MaxMemory, time_ref=TimeRef, ttl=TTL}}. @@ -113,9 +146,11 @@ start(Partition, Config) -> %% @doc Stop the memory backend -spec stop(state()) -> ok. stop(#state{data_ref=DataRef, + index_ref=IndexRef, max_memory=MaxMemory, time_ref=TimeRef}) -> catch ets:delete(DataRef), + catch ets:delete(IndexRef), case MaxMemory of undefined -> ok; @@ -130,14 +165,27 @@ stop(#state{data_ref=DataRef, {ok, not_found, state()} | {error, term(), state()}. get(Bucket, Key, State=#state{data_ref=DataRef, + index_ref=IndexRef, + used_memory=UsedMemory, + max_memory=MaxMemory, ttl=TTL}) -> case ets:lookup(DataRef, {Bucket, Key}) of [] -> {error, not_found, State}; - [{{Bucket, Key}, {{ts, Timestamp}, Val}}] -> + [{{Bucket, Key}, {{ts, Timestamp}, Val}}=Object] -> case exceeds_ttl(Timestamp, TTL) of true -> - delete(Bucket, Key, undefined, State), - {error, not_found, State}; + %% Because we do not have the IndexSpecs, we must + %% delete the object directly and all index + %% entries blindly using match_delete. + ets:delete(DataRef, {Bucket, Key}), + ets:match_delete(IndexRef, ?DELETE_PTN(Bucket, Key)), + case MaxMemory of + undefined -> + UsedMemory1 = UsedMemory; + _ -> + UsedMemory1 = UsedMemory - object_size(Object) + end, + {error, not_found, State#state{used_memory=UsedMemory1}}; false -> {ok, Val, State} end; @@ -148,18 +196,15 @@ get(Bucket, Key, State=#state{data_ref=DataRef, end. %% @doc Insert an object into the memory backend. -%% NOTE: The memory backend does not currently -%% support secondary indexing and the _IndexSpecs -%% parameter is ignored. -type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. -spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> - {ok, state()} | - {error, term(), state()}. -put(Bucket, PrimaryKey, _IndexSpecs, Val, State=#state{data_ref=DataRef, - max_memory=MaxMemory, - time_ref=TimeRef, - ttl=TTL, - used_memory=UsedMemory}) -> + {ok, state()}. +put(Bucket, PrimaryKey, IndexSpecs, Val, State=#state{data_ref=DataRef, + index_ref=IndexRef, + max_memory=MaxMemory, + time_ref=TimeRef, + ttl=TTL, + used_memory=UsedMemory}) -> Now = now(), case TTL of undefined -> @@ -167,36 +212,29 @@ put(Bucket, PrimaryKey, _IndexSpecs, Val, State=#state{data_ref=DataRef, _ -> Val1 = {{ts, Now}, Val} end, - case do_put(Bucket, PrimaryKey, Val1, DataRef) of - {ok, Size} -> - %% If the memory is capped update timestamp table - %% and check if the memory usage is over the cap. - case MaxMemory of - undefined -> - UsedMemory1 = UsedMemory; - _ -> - time_entry(Bucket, PrimaryKey, Now, TimeRef), - Freed = trim_data_table(MaxMemory, - UsedMemory + Size, - DataRef, - TimeRef, - 0), - UsedMemory1 = UsedMemory + Size - Freed - end, - {ok, State#state{used_memory=UsedMemory1}}; - {error, Reason} -> - {error, Reason, State} - end. + {ok, Size} = do_put(Bucket, PrimaryKey, Val1, IndexSpecs, DataRef, IndexRef), + case MaxMemory of + undefined -> + UsedMemory1 = UsedMemory; + _ -> + time_entry(Bucket, PrimaryKey, Now, TimeRef), + Freed = trim_data_table(MaxMemory, + UsedMemory + Size, + DataRef, + TimeRef, + IndexRef, + 0), + UsedMemory1 = UsedMemory + Size - Freed + end, + {ok, State#state{used_memory=UsedMemory1}}. %% @doc Delete an object from the memory backend -%% NOTE: The memory backend does not currently -%% support secondary indexing and the _IndexSpecs -%% parameter is ignored. -spec delete(riak_object:bucket(), riak_object:key(), [index_spec()], state()) -> {ok, state()}. -delete(Bucket, Key, _IndexSpecs, State=#state{data_ref=DataRef, - time_ref=TimeRef, - used_memory=UsedMemory}) -> +delete(Bucket, Key, IndexSpecs, State=#state{data_ref=DataRef, + index_ref=IndexRef, + time_ref=TimeRef, + used_memory=UsedMemory}) -> case TimeRef of undefined -> UsedMemory1 = UsedMemory; @@ -213,6 +251,7 @@ delete(Bucket, Key, _IndexSpecs, State=#state{data_ref=DataRef, UsedMemory1 = UsedMemory end end, + update_indexes(Bucket, Key, IndexSpecs, IndexRef), ets:delete(DataRef, {Bucket, Key}), {ok, State#state{used_memory=UsedMemory1}}. @@ -241,15 +280,33 @@ fold_buckets(FoldBucketsFun, Acc, Opts, #state{data_ref=DataRef}) -> any(), [{atom(), term()}], state()) -> {ok, term()} | {async, fun()}. -fold_keys(FoldKeysFun, Acc, Opts, #state{data_ref=DataRef}) -> - Bucket = proplists:get_value(bucket, Opts), - FoldFun = fold_keys_fun(FoldKeysFun, Bucket), +fold_keys(FoldKeysFun, Acc, Opts, #state{data_ref=DataRef, + index_ref=IndexRef}) -> + + %% Figure out how we should limit the fold: by bucket, by + %% secondary index, or neither (fold across everything.) + Bucket = lists:keyfind(bucket, 1, Opts), + Index = lists:keyfind(index, 1, Opts), + + %% Multiple limiters may exist. Take the most specific limiter, + %% get an appropriate folder function. + Folder = if + Index /= false -> + FoldFun = fold_keys_fun(FoldKeysFun, Index), + get_index_folder(FoldFun, Acc, Index, DataRef, IndexRef); + Bucket /= false -> + FoldFun = fold_keys_fun(FoldKeysFun, Bucket), + get_folder(FoldFun, Acc, DataRef); + true -> + FoldFun = fold_keys_fun(FoldKeysFun, undefined), + get_folder(FoldFun, Acc, DataRef) + end, + case lists:member(async_fold, Opts) of true -> - {async, get_folder(FoldFun, Acc, DataRef)}; + {async, Folder}; false -> - Acc0 = ets:foldl(FoldFun, Acc, DataRef), - {ok, Acc0} + {ok, Folder()} end. %% @doc Fold over all the objects for one or all buckets. @@ -271,8 +328,10 @@ fold_objects(FoldObjectsFun, Acc, Opts, #state{data_ref=DataRef}) -> %% @doc Delete all objects from this memory backend -spec drop(state()) -> {ok, state()}. drop(State=#state{data_ref=DataRef, + index_ref=IndexRef, time_ref=TimeRef}) -> ets:delete_all_objects(DataRef), + ets:delete_all_objects(IndexRef), case TimeRef of undefined -> ok; @@ -290,14 +349,18 @@ is_empty(#state{data_ref=DataRef}) -> %% @doc Get the status information for this memory backend -spec status(state()) -> [{atom(), term()}]. status(#state{data_ref=DataRef, + index_ref=IndexRef, time_ref=TimeRef}) -> DataStatus = ets:info(DataRef), + IndexStatus = ets:info(IndexRef), case TimeRef of undefined -> - [{data_table_status, DataStatus}]; + [{data_table_status, DataStatus}, + {index_table_status, IndexStatus}]; _ -> TimeStatus = ets:info(TimeRef), [{data_table_status, DataStatus}, + {index_table_status, IndexStatus}, {time_table_status, TimeStatus}] end. @@ -306,6 +369,24 @@ status(#state{data_ref=DataRef, callback(_Ref, _Msg, State) -> {ok, State}. +%% @doc Resets state of all running memory backends on the local +%% node. The `riak_kv' environment variable `memory_backend' must +%% contain the `test' property, set to `true' for this to work. +-spec reset() -> ok | {error, reset_disabled}. +reset() -> + reset(app_helper:get_env(memory_backend, test, app_helper:get_env(riak_kv, test)), app_helper:get_env(riak_kv, storage_backend)). + +reset(true, ?MODULE) -> + {ok, Ring} = riak_core_ring_manager:get_my_ring(), + [ begin + catch ets:delete_all_objects(?DNAME(I)), + catch ets:delete_all_objects(?INAME(I)), + catch ets:delete_all_objects(?TNAME(I)) + end || I <- riak_core_ring:my_indices(Ring) ], + ok; +reset(_, _) -> + {error, reset_disabled}. + %% =================================================================== %% Internal functions %% =================================================================== @@ -331,32 +412,46 @@ fold_buckets_fun(FoldBucketsFun) -> %% Return a function to fold over keys on this backend fold_keys_fun(FoldKeysFun, undefined) -> fun({{Bucket, Key}, _}, Acc) -> - FoldKeysFun(Bucket, Key, Acc) + FoldKeysFun(Bucket, Key, Acc); + (_, Acc) -> + Acc end; -fold_keys_fun(FoldKeysFun, Bucket) -> - fun({{B, Key}, _}, Acc) -> - case B =:= Bucket of - true -> - FoldKeysFun(Bucket, Key, Acc); - false -> - Acc - end +fold_keys_fun(FoldKeysFun, {bucket, FilterBucket}) -> + fun({{Bucket, Key}, _}, Acc) when Bucket == FilterBucket -> + FoldKeysFun(Bucket, Key, Acc); + (_, Acc) -> + Acc + end; +fold_keys_fun(FoldKeysFun, {index, FilterBucket, {eq, <<"$bucket">>, _}}) -> + %% 2I exact match query on special $bucket field... + fold_keys_fun(FoldKeysFun, {bucket, FilterBucket}); +fold_keys_fun(FoldKeysFun, {index, FilterBucket, {range, <<"$key">>, _, _}}) -> + %% 2I range query on special $key field... + fold_keys_fun(FoldKeysFun, {bucket, FilterBucket}); +fold_keys_fun(FoldKeysFun, {index, FilterBucket, {eq, <<"$key">>, _}}) -> + %% 2I eq query on special $key field... + fold_keys_fun(FoldKeysFun, {bucket, FilterBucket}); +fold_keys_fun(FoldKeysFun, {index, _FilterBucket, _Query}) -> + fun({{Bucket, _FilterField, _FilterTerm, Key}, _}, Acc) -> + FoldKeysFun(Bucket, Key, Acc); + (_, Acc) -> + Acc end. + %% @private %% Return a function to fold over keys on this backend fold_objects_fun(FoldObjectsFun, undefined) -> fun({{Bucket, Key}, Value}, Acc) -> - FoldObjectsFun(Bucket, Key, Value, Acc) + FoldObjectsFun(Bucket, Key, Value, Acc); + (_, Acc) -> + Acc end; -fold_objects_fun(FoldObjectsFun, Bucket) -> - fun({{B, Key}, Value}, Acc) -> - case B =:= Bucket of - true -> - FoldObjectsFun(Bucket, Key, Value, Acc); - false -> - Acc - end +fold_objects_fun(FoldObjectsFun, FilterBucket) -> + fun({{Bucket, Key}, Value}, Acc) when Bucket == FilterBucket-> + FoldObjectsFun(Bucket, Key, Value, Acc); + (_, Acc) -> + Acc end. %% @private @@ -366,29 +461,92 @@ get_folder(FoldFun, Acc, DataRef) -> end. %% @private -do_put(Bucket, Key, Val, Ref) -> - Object = {{Bucket, Key}, Val}, - true = ets:insert(Ref, Object), - {ok, object_size(Object)}. +get_index_folder(Folder, Acc0, {index, Bucket, {eq, <<"$bucket">>, _}}, DataRef, _) -> + %% For the special $bucket index, turn it into a fold over the + %% data table. + fun() -> + key_range_folder(Folder, Acc0, DataRef, {Bucket, <<>>}, Bucket) + end; +get_index_folder(Folder, Acc0, {index, Bucket, {range, <<"$key">>, Min, Max}}, DataRef, _) -> + %% For the special range lookup on the $key index, turn it into a + %% fold on the data table + fun() -> + key_range_folder(Folder, Acc0, DataRef, {Bucket, Min}, {Bucket, Min, Max}) + end; +get_index_folder(Folder, Acc0, {index, Bucket, {eq, <<"$key">>, Val}}, DataRef, IndexRef) -> + get_index_folder(Folder, Acc0, {index, Bucket, {range, <<"$key">>, Val, Val}}, DataRef, IndexRef); +get_index_folder(Folder, Acc0, {index, Bucket, {eq, Field, Term}}, _, IndexRef) -> + fun() -> + index_range_folder(Folder, Acc0, IndexRef, {Bucket, Field, Term, undefined}, {Bucket, Field, Term, Term}) + end; +get_index_folder(Folder, Acc0, {index, Bucket, {range, Field, Min, Max}}, _, IndexRef) -> + fun() -> + index_range_folder(Folder, Acc0, IndexRef, {Bucket, Field, Min, undefined}, {Bucket, Field, Min, Max}) + end. + +%% Iterates over a range of keys, for the special $key and $bucket +%% indexes. %% @private -config_value(Key, Config) -> - config_value(Key, Config, undefined). +-spec key_range_folder(function(), term(), ets:tid(), {riak_object:bucket(), riak_object:key()}, binary() | {riak_object:bucket(), term(), term()}) -> term(). +key_range_folder(Folder, Acc0, DataRef, {B,_}=DataKey, B) -> + case ets:lookup(DataRef, DataKey) of + [] -> + key_range_folder(Folder, Acc0, DataRef, ets:next(DataRef, DataKey), B); + [Object] -> + Acc = Folder(Object, Acc0), + key_range_folder(Folder, Acc, DataRef, ets:next(DataRef, DataKey), B) + end; +key_range_folder(Folder, Acc0, DataRef, {B,K}=DataKey, {B, Min, Max}=Query) when K >= Min, K =< Max -> + case ets:lookup(DataRef, DataKey) of + [] -> + key_range_folder(Folder, Acc0, DataRef, ets:next(DataRef, DataKey), Query); + [Object] -> + Acc = Folder(Object, Acc0), + key_range_folder(Folder, Acc, DataRef, ets:next(DataRef, DataKey), Query) + end; +key_range_folder(_Folder, Acc, _DataRef, _DataKey, _Query) -> + Acc. + +%% Iterates over a range of index postings +index_range_folder(Folder, Acc0, IndexRef, {B, I, V, _K}=IndexKey, {B, I, Min, Max}=Query) when V >= Min, V =< Max -> + case ets:lookup(IndexRef, IndexKey) of + [] -> + %% This will happen on the first iteration, where the key + %% does not exist. In all other cases, ETS will give us a + %% real key from next/2. + index_range_folder(Folder, Acc0, IndexRef, ets:next(IndexRef, IndexKey), Query); + [Posting] -> + Acc = Folder(Posting, Acc0), + index_range_folder(Folder, Acc, IndexRef, ets:next(IndexRef, IndexKey), Query) + end; +index_range_folder(_Folder, Acc, _IndexRef, _IndexKey, _Query) -> + Acc. + %% @private -config_value(Key, Config, Default) -> - case proplists:get_value(Key, Config) of - undefined -> - app_helper:get_env(memory_backend, Key, Default); - Value -> - Value - end. +do_put(Bucket, Key, Val, IndexSpecs, DataRef, IndexRef) -> + Object = {{Bucket, Key}, Val}, + true = ets:insert(DataRef, Object), + update_indexes(Bucket, Key, IndexSpecs, IndexRef), + {ok, object_size(Object)}. %% Check if this timestamp is past the ttl setting. exceeds_ttl(Timestamp, TTL) -> Diff = (timer:now_diff(now(), Timestamp) / 1000 / 1000), Diff > TTL. +update_indexes(_Bucket, _Key, undefined, _IndexRef) -> + ok; +update_indexes(_Bucket, _Key, [], _IndexRef) -> + ok; +update_indexes(Bucket, Key, [{remove, Field, Value}|Rest], IndexRef) -> + true = ets:delete(IndexRef, {Bucket, Field, Value, Key}), + update_indexes(Bucket, Key, Rest, IndexRef); +update_indexes(Bucket, Key, [{add, Field, Value}|Rest], IndexRef) -> + true = ets:insert(IndexRef, {{Bucket, Field, Value, Key}, <<>>}), + update_indexes(Bucket, Key, Rest, IndexRef). + %% @private time_entry(Bucket, Key, Now, TimeRef) -> ets:insert(TimeRef, {Now, {Bucket, Key}}). @@ -396,20 +554,21 @@ time_entry(Bucket, Key, Now, TimeRef) -> %% @private %% @doc Dump some entries if the max memory size has %% been breached. -trim_data_table(MaxMemory, UsedMemory, _, _, Freed) when +trim_data_table(MaxMemory, UsedMemory, _, _, _, Freed) when (UsedMemory - Freed) =< MaxMemory -> Freed; -trim_data_table(MaxMemory, UsedMemory, DataRef, TimeRef, Freed) -> +trim_data_table(MaxMemory, UsedMemory, DataRef, TimeRef, IndexRef, Freed) -> %% Delete the oldest object - OldestSize = delete_oldest(DataRef, TimeRef), + OldestSize = delete_oldest(DataRef, TimeRef, IndexRef), trim_data_table(MaxMemory, UsedMemory, DataRef, TimeRef, + IndexRef, Freed + OldestSize). %% @private -delete_oldest(DataRef, TimeRef) -> +delete_oldest(DataRef, TimeRef, IndexRef) -> OldestTime = ets:first(TimeRef), case OldestTime of '$end_of_table' -> @@ -419,8 +578,10 @@ delete_oldest(DataRef, TimeRef) -> ets:delete(TimeRef, OldestTime), case ets:lookup(DataRef, OldestKey) of [] -> - delete_oldest(DataRef, TimeRef); + delete_oldest(DataRef, TimeRef, IndexRef); [Object] -> + {Bucket, Key} = OldestKey, + ets:match_delete(IndexRef, ?DELETE_PTN(Bucket, Key)), ets:delete(DataRef, OldestKey), object_size(Object) end @@ -436,6 +597,26 @@ object_size(Object) -> end, size(Bucket) + size(Key) + size(Val). +%% Copied from riak_core 1.2 app_helper module +%% @private +%% @doc Retrieve value for Key from Properties if it exists, otherwise +%% return from the application's env. +-spec get_prop_or_env(atom(), [{atom(), term()}], atom()) -> term(). +get_prop_or_env(Key, Properties, App) -> + get_prop_or_env(Key, Properties, App, undefined). + +%% @private +%% @doc Return the value for Key in Properties if it exists, otherwise return +%% the value from the application's env, or Default. +-spec get_prop_or_env(atom(), [{atom(), term()}], atom(), term()) -> term(). +get_prop_or_env(Key, Properties, App, Default) -> + case proplists:get_value(Key, Properties) of + undefined -> + app_helper:get_env(App, Key, Default); + Value -> + Value + end. + %% =================================================================== %% EUnit tests %% =================================================================== @@ -493,6 +674,34 @@ max_memory_test_() -> ?_assertEqual({ok, Value2, State2}, get(Bucket, Key2, State2)) ]. +regression_367_key_range_test_() -> + {ok, State} = start(142, []), + Keys = [begin + Bin = list_to_binary(integer_to_list(I)), + if I < 10 -> + <<"obj0", Bin/binary>>; + true -> <<"obj", Bin/binary>> + end + end || I <- lists:seq(1,30) ], + Bucket = <<"keyrange">>, + Value = <<"foobarbaz">>, + State1 = lists:foldl(fun(Key, IState) -> + {ok, NewState} = put(Bucket, Key, [], Value, IState), + NewState + end, State, Keys), + Folder = fun(_B, K, Acc) -> + Acc ++ [K] + end, + [ + ?_assertEqual({ok, [<<"obj01">>]}, fold_keys(Folder, [], [{index, Bucket, {range, <<"$key">>, <<"obj01">>, <<"obj01">>}}], State1)), + ?_assertEqual({ok, [<<"obj10">>,<<"obj11">>]}, fold_keys(Folder, [], [{index, Bucket, {range, <<"$key">>, <<"obj10">>, <<"obj11">>}}], State1)), + ?_assertEqual({ok, [<<"obj01">>]}, fold_keys(Folder, [], [{index, Bucket, {range, <<"$key">>, <<"obj00">>, <<"obj01">>}}], State1)), + ?_assertEqual({ok, lists:sort(Keys)}, fold_keys(Folder, [], [{index, Bucket, {range, <<"$key">>, <<"obj0">>, <<"obj31">>}}], State1)), + ?_assertEqual({ok, []}, fold_keys(Folder, [], [{index, Bucket, {range, <<"$key">>, <<"obj31">>, <<"obj32">>}}], State1)), + ?_assertEqual({ok, [<<"obj01">>]}, fold_keys(Folder, [], [{index, Bucket, {eq, <<"$key">>, <<"obj01">>}}], State1)), + ?_assertEqual(ok, stop(State1)) + ]. + -ifdef(EQC). eqc_test_() -> @@ -504,8 +713,8 @@ eqc_test_() -> [ {timeout, 60000, [?_assertEqual(true, - backend_eqc:test(?MODULE, true))]} - ]}]}]}. + backend_eqc:test(?MODULE, true))]} + ]}]}]}. setup() -> application:load(sasl), diff --git a/riak/erl_src/riak_search_test_backend.beam b/riak/erl_src/riak_search_test_backend.beam index b20a76c9de3a21a0177d94e14ef95b5192d14fa3..8c3d3723b6ade0187be8ab6c8762f3d679345d12 100644 GIT binary patch delta 223 zcmcbi)S}Gc<{xCpz#v$&k>fs-U?>9vg8&eNKnD=}Z2rN-&1vh+%%dRS$jr=QC&KH+ zE?^+wAmGYb%q;B6StQKNF5ti}z#+%T;bkx4z{1Qb!0V~N$l-0m>%lC{3^Zf23C|{G zfpwaOGXh=fs-pbrBBg8&eNKnDQLDX=`q^Yp^|ws&Us^tyNx)P=u{j55 z5K9JA3Ilt7K}lwQUNK032_(RrT%IzyU%+y5hk(uGcLFNhw-Y%Ng%U**)e}W0%L>MG OF)%nL6_)^&GXMZaoF_p5 diff --git a/riak/erl_src/riak_search_test_backend.erl b/riak/erl_src/riak_search_test_backend.erl index 477903eb..64571b1d 100644 --- a/riak/erl_src/riak_search_test_backend.erl +++ b/riak/erl_src/riak_search_test_backend.erl @@ -25,18 +25,18 @@ ]). -include_lib("riak_search/include/riak_search.hrl"). - +-define(T(P), list_to_atom("rs" ++ integer_to_list(P))). -record(state, {partition, table}). reset() -> {ok, Ring} = riak_core_ring_manager:get_my_ring(), - [ ets:delete_all_objects(list_to_atom("rs" ++ integer_to_list(P))) || + [ catch ets:delete_all_objects(?T(P)) || P <- riak_core_ring:my_indices(Ring) ], riak_search_config:clear(), ok. start(Partition, _Config) -> - Table = ets:new(list_to_atom("rs" ++ integer_to_list(Partition)), + Table = ets:new(?T(Partition), [named_table, public, ordered_set]), {ok, #state{partition=Partition, table=Table}}. diff --git a/riak/test_server.py b/riak/test_server.py index 7c6348af..7ca5c8bd 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -84,7 +84,11 @@ class TestServer(object): "js_thread_stack": 16, "riak_kv_stat": True, "map_cache_size": 0, - "vnode_cache_entries": 0 + "vnode_cache_entries": 0, + "test": True, + "memory_backend": { + "test": True, + }, }, "riak_search": { "enabled": True, From 997d6e955d2426047003d537fdd278c4402badde Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 10 Jan 2013 12:29:40 -0600 Subject: [PATCH 0293/1060] Fix some 'is' problems. --- riak/client/transport.py | 2 +- riak/transports/http/__init__.py | 2 +- riak/transports/http/stream.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/riak/client/transport.py b/riak/client/transport.py index 39d9aa33..9a000e5e 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -41,7 +41,7 @@ def _transport(self, protocol=None): protocol = self.protocol if protocol in ['http', 'https']: pool = self._http_pool - elif protocol is 'pbc': + elif protocol == 'pbc': pool = self._pb_pool else: raise ValueError("invalid protocol %s" % protocol) diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index d6069444..e7cd660e 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -30,7 +30,7 @@ class RiakHttpPool(Pool): def __init__(self, client, **options): self.client = client self.options = options - if client.protocol is 'https': + if client.protocol == 'https': self.connection_class = httplib.HTTPSConnection else: self.connection_class = httplib.HTTPConnection diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 960fd3c3..5130a0dd 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -39,7 +39,7 @@ def __iter__(self): def read(self): chunk = self.response.read(self.BLOCK_SIZE) - if chunk is '': + if chunk == '': self.response_done = True self.buffer += chunk From 730c1050c66940ac935db098fdcaa55ac5be7026 Mon Sep 17 00:00:00 2001 From: "Engel A. Sanchez" Date: Fri, 11 Jan 2013 15:22:13 -0500 Subject: [PATCH 0294/1060] Make sure protobuf comes from pypi Without this, it gets the outdated zip package from googlecode and fail. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index a375c388..06393638 100755 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ def make_docs(): package_data = {'riak' : ['erl_src/*']}, description='Python client for Riak', zip_safe=True, + options={'easy_install': {'allow_hosts': 'pypi.python.org'}}, include_package_data=True, license='Apache 2', platforms='Platform Independent', From b2261700b51523e877db1d94e7eb7af1aad39f04 Mon Sep 17 00:00:00 2001 From: Michael Clemmons Date: Fri, 28 Dec 2012 14:24:14 -0800 Subject: [PATCH 0295/1060] add rich comparison methods to client/bucket/riakobject --- riak/bucket.py | 15 +++++++ riak/client.py | 15 +++++++ riak/riak_object.py | 17 +++++++- riak/tests/test_comparison.py | 80 +++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 riak/tests/test_comparison.py diff --git a/riak/bucket.py b/riak/bucket.py index 0782a2e1..b733e472 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -56,6 +56,21 @@ def __init__(self, client, name): self._encoders = {} self._decoders = {} + def __hash__(self): + return hash((self.name, self._client)) + + def __eq__(self, other): + if isinstance(other, self.__class__): + return hash(self) == hash(other) + else: + return False + + def __nq__(self, other): + if isinstance(other, self.__class__): + return hash(self) != hash(other) + else: + return True + def get_encoder(self, content_type): """ Get the encoding function for the provided content type for diff --git a/riak/client.py b/riak/client.py index 483445a0..a281fc3d 100644 --- a/riak/client.py +++ b/riak/client.py @@ -94,6 +94,21 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', self._host = host self._port = port + def __hash__(self): + return hash(frozenset(self._cm.hostports)) + + def __eq__(self, other): + if isinstance(other, self.__class__): + return hash(self) == hash(other) + else: + return False + + def __nq__(self, other): + if isinstance(other, self.__class__): + return hash(self) != hash(other) + else: + return True + def get_transport(self): """ Get the transport instance the client is using for it's connection. diff --git a/riak/riak_object.py b/riak/riak_object.py index 4964ae49..cecb5b1e 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -57,6 +57,21 @@ def __init__(self, client, bucket, key=None): self.siblings = [] self.exists = False + def __hash__(self): + return hash((self.key, self.bucket, self.vclock)) + + def __eq__(self, other): + if isinstance(other, self.__class__): + return hash(self) == hash(other) + else: + return False + + def __nq__(self, other): + if isinstance(other, self.__class__): + return hash(self) != hash(other) + else: + return True + def _get_data(self): return self._data @@ -125,7 +140,7 @@ def _get_usermeta(self): def _set_usermeta(self, usermeta): self.metadata[MD_USERMETA] = usermeta return self - + usermeta = property(_get_usermeta, _set_usermeta, doc=""" The custom user metadata on this object. This doesn't diff --git a/riak/tests/test_comparison.py b/riak/tests/test_comparison.py new file mode 100644 index 00000000..808198d9 --- /dev/null +++ b/riak/tests/test_comparison.py @@ -0,0 +1,80 @@ +import platform + +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest + +from riak import RiakHttpTransport +from riak.client import RiakClient +from riak.riak_object import RiakObject +from riak.bucket import RiakBucket +from riak.tests.test_all import BaseTestCase + + +class RiakBucketRichComparisonTest(unittest.TestCase): + def test_bucket_eq(self): + a = RiakBucket('client', 'a') + b = RiakBucket('client', 'a') + self.assertEqual(a, b) + + def test_bucket_nq(self): + a = RiakBucket('client', 'a') + b = RiakBucket('client', 'b') + c = RiakBucket('client', 'a') + self.assertNotEqual(a, b, 'matched with a different bucket') + + def test_bucket_hash(self): + a = RiakBucket('client', 'a') + b = RiakBucket('client', 'a') + c = RiakBucket('client', 'c') + self.assertEqual(hash(a), hash(b), 'same bucket has different hashes') + self.assertNotEqual(hash(a), hash(c), 'different bucket has same hash') + + +class RiakObjectComparisonTest(unittest.TestCase): + def test_object_eq(self): + a = RiakObject(None, 'bucket', 'key') + b = RiakObject(None, 'bucket', 'key') + self.assertEqual(a, b) + + def test_object_nq(self): + a = RiakObject(None, 'bucket', 'key') + b = RiakObject(None, 'bucket', 'not key') + c = RiakObject(None, 'not bucket', 'key') + self.assertNotEqual(a, b, 'matched with different keys') + self.assertNotEqual(a, c, 'matched with different buckets') + + def test_object_hash(self): + a = RiakObject(None, 'bucket', 'key') + b = RiakObject(None, 'bucket', 'key') + c = RiakObject(None, 'bucket', 'not key') + self.assertEqual(hash(a), hash(b), 'same object has different hashes') + self.assertNotEqual(hash(a), hash(c), 'different object has same hash') + + +class RiakClientComparisonTest(unittest.TestCase, BaseTestCase): + def test_client_eq(self): + self.transport_class = RiakHttpTransport + a = self.create_client('host1', 11) + b = self.create_client('host1', 11) + self.assertEqual(a, b) + + def test_client_nq(self): + self.transport_class = RiakHttpTransport + a = self.create_client('host1', 11) + b = self.create_client('host2', 11) + c = self.create_client('host1', 12) + self.assertNotEqual(a, b, 'matched with different hosts') + self.assertNotEqual(a, c, 'matched with different ports') + + def test_client_hash(self): + self.transport_class = RiakHttpTransport + a = self.create_client('host1', 11) + b = self.create_client('host1', 11) + c = self.create_client('host2', 11) + self.assertEqual(hash(a), hash(b), 'same object has different hashes') + self.assertNotEqual(hash(a), hash(c), 'different object has same hash') + +if __name__ == '__main__': + unittest.main() From 4b8636497a7365598de3a6604650364cbdd652ac Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 29 Jan 2013 11:10:54 -0600 Subject: [PATCH 0296/1060] Rethink the automatic retries stuff. --- riak/client/operations.py | 120 ++++++++++++++++++++------------------ riak/client/transport.py | 44 ++++++++------ 2 files changed, 88 insertions(+), 76 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index e9fd8ee1..efba5128 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -16,141 +16,145 @@ under the License. """ -from transport import RiakClientTransport - +from transport import RiakClientTransport, retryable, retryableHttpOnly class RiakClientOperations(RiakClientTransport): """ Methods for RiakClient that result in requests sent to the Riak cluster. + + Note that all of these methods have an implicit 'transport' + argument that will be prepended automatically as part of the retry + logic, and does not need to be supplied by the user. """ - def get_buckets(self): + @retryable + def get_buckets(self, transport): """ Get the list of buckets as RiakBucket instances. NOTE: Do not use this in production, as it requires traversing through all keys stored in a cluster. """ - with self._transport() as transport: - return [self.bucket(name) for name in transport.get_buckets()] + return [self.bucket(name) for name in transport.get_buckets()] - def ping(self): + @retryable + def ping(self, transport): """ Check if the Riak server for this ``RiakClient`` instance is alive. :rtype: boolean """ - with self._transport() as transport: - return transport.ping() + return transport.ping() is_alive = ping - def get_index(self, bucket, index, startkey, endkey=None): + @retryable + def get_index(self, transport, bucket, index, startkey, endkey=None): """ Queries a secondary index, returning matching keys. """ - with self._transport() as transport: - return transport.get_index(bucket, index, startkey, endkey) + return transport.get_index(bucket, index, startkey, endkey) - def get_bucket_props(self, bucket): + @retryable + def get_bucket_props(self, transport, bucket): """ Fetches bucket properties for the given bucket. """ - with self._transport() as transport: - return transport.get_bucket_props(bucket) + return transport.get_bucket_props(bucket) - def set_bucket_props(self, bucket, props): + @retryable + def set_bucket_props(self, transport, bucket, props): """ Sets bucket properties for the given bucket. """ - with self._transport() as transport: - return transport.set_bucket_props(bucket, props) + return transport.set_bucket_props(bucket, props) - def get_keys(self, bucket): + @retryable + def get_keys(self, transport, bucket): """ Lists all keys in a bucket. """ - with self._transport() as transport: - return transport.get_keys(bucket) + return transport.get_keys(bucket) - def stream_keys(self, bucket): + @retryable + def stream_keys(self, transport, bucket): """ Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. """ - with self._transport() as transport: - for keylist in transport.stream_keys(bucket): - if len(keylist) > 0: - yield keylist + for keylist in transport.stream_keys(bucket): + if len(keylist) > 0: + yield keylist - def put(self, robj, w=None, dw=None, pw=None, return_body=None, + @retryable + def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=None): """ Stores an object in the Riak cluster. """ - with self._transport() as transport: - return transport.put(robj, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) + return transport.put(robj, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) - def put_new(self, robj, w=None, dw=None, pw=None, return_body=None, + @retryable + def put_new(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=None): """ Stores an object in the Riak cluster with a generated key. """ - with self._transport() as transport: - return transport.put_new(robj, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) + return transport.put_new(robj, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) - def get(self, robj, r=None, pr=None, vtag=None): + @retryable + def get(self, transport, robj, r=None, pr=None, vtag=None): """ Fetches the contents of a Riak object. """ - with self._transport() as transport: - return transport.get(robj, r=r, pr=pr, vtag=vtag) + return transport.get(robj, r=r, pr=pr, vtag=vtag) - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + @retryable + def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, pr=None, + pw=None): """ Deletes an object from Riak. """ - with self._transport() as transport: - return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, - pw=pw) + return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, + pw=pw) - def mapred(self, inputs, query, timeout): + @retryable + def mapred(self, transport, inputs, query, timeout): """ Executes a MapReduce query """ - with self._transport() as transport: - return transport.mapred(inputs, query, timeout) + return transport.mapred(inputs, query, timeout) - def stream_mapred(self, inputs, query, timeout): + @retryable + def stream_mapred(self, transport, inputs, query, timeout): """ Streams a MapReduce query as (phase, data) pairs. This is a generator method which should be iterated over. """ - with self._transport() as transport: - for phase, data in transport.stream_mapred(inputs, query, timeout): - yield phase, data + for phase, data in transport.stream_mapred(inputs, query, timeout): + yield phase, data - def fulltext_search(self, index, query, **params): + @retryableHttpOnly + def fulltext_search(self, transport, index, query, **params): """ Performs a full-text search query. """ - with self._transport() as transport: - return transport.search(index, query, **params) + return transport.search(index, query, **params) - def fulltext_add(self, index, docs): + @retryableHttpOnly + def fulltext_add(self, transport, index, docs): """ Adds documents to the full-text index. """ - with self._transport(protocol='http') as transport: - transport.fulltext_add(index, docs) + transport.fulltext_add(index, docs) - def fulltext_delete(self, index, docs=None, queries=None): + @retryableHttpOnly + def fulltext_delete(self, transport, index, docs=None, queries=None): """ Removes documents from the full-text index. """ - with self._transport(protocol='http') as transport: - transport.fulltext_delete(index, docs, queries) + transport.fulltext_delete(index, docs, queries) diff --git a/riak/client/transport.py b/riak/client/transport.py index 9a000e5e..fc46abb5 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -16,7 +16,6 @@ under the License. """ -from contextlib import contextmanager from riak.transports.pool import BadResource from riak.transports.pbc import is_retryable as is_pbc_retryable from riak.transports.http import is_retryable as is_http_retryable @@ -35,22 +34,7 @@ class RiakClientTransport(object): _http_pool = None _pb_pool = None - @contextmanager - def _transport(self, protocol=None): - if not protocol: - protocol = self.protocol - if protocol in ['http', 'https']: - pool = self._http_pool - elif protocol == 'pbc': - pool = self._pb_pool - else: - raise ValueError("invalid protocol %s" % protocol) - - with pool.take() as transport: - yield transport - - @contextmanager - def _retryable(self, pool): + def _with_retries(self, pool, fn): skip_nodes = [] def _skip_bad_nodes(transport): @@ -60,7 +44,7 @@ def _skip_bad_nodes(transport): try: with pool.take(_filter=_skip_bad_nodes) as transport: try: - yield transport + return fn(transport) except (IOError, httplib.HTTPException) as e: if is_retryable(e): transport._node.error_rate.incr(1) @@ -71,6 +55,30 @@ def _skip_bad_nodes(transport): except BadResource: continue + def _choose_pool(self, protocol=None): + if not protocol: + protocol = self.protocol + if protocol in ['http', 'https']: + pool = self._http_pool + elif protocol == 'pbc': + pool = self._pb_pool + else: + raise ValueError("invalid protocol %s" % protocol) + return pool + def is_retryable(error): return is_pbc_retryable(error) or is_http_retryable(error) + + +def retryable(fn, protocol=None): + def wrapper(self, *args, **kwargs): + pool = self._choose_pool(protocol) + def thunk(transport): + return fn(self, transport, *args, **kwargs) + return self._with_retries(pool, thunk) + return wrapper + + +def retryableHttpOnly(fn): + return retryable(fn, protocol='http') From 054a5905cad5fc60b65046b1cbe75589e042e807 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 29 Jan 2013 11:25:22 -0600 Subject: [PATCH 0297/1060] Restore _transport(), which is used when getting the client id. --- riak/client/transport.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/riak/client/transport.py b/riak/client/transport.py index fc46abb5..54e26de6 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -15,7 +15,7 @@ specific language governing permissions and limitations under the License. """ - +from contextlib import contextmanager from riak.transports.pool import BadResource from riak.transports.pbc import is_retryable as is_pbc_retryable from riak.transports.http import is_retryable as is_http_retryable @@ -34,6 +34,12 @@ class RiakClientTransport(object): _http_pool = None _pb_pool = None + @contextmanager + def _transport(self): + pool = self._choose_pool() + with pool.take() as transport: + yield transport + def _with_retries(self, pool, fn): skip_nodes = [] From add0760ca01399694e991a2caa37ad5e7f187a09 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 29 Jan 2013 12:10:43 -0600 Subject: [PATCH 0298/1060] Ensure that streams drain the socket and cleanup when there is an exception in user code. --- riak/client/operations.py | 18 +++++++++++++----- riak/tests/test_all.py | 4 +++- riak/tests/test_mapreduce.py | 33 ++++++++++++++++++++++++++++++++- riak/transports/http/stream.py | 5 +++++ riak/transports/pbc/stream.py | 14 ++++++++++++-- 5 files changed, 65 insertions(+), 9 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index efba5128..00366022 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -82,9 +82,13 @@ def stream_keys(self, transport, bucket): Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. """ - for keylist in transport.stream_keys(bucket): - if len(keylist) > 0: - yield keylist + stream = transport.stream_keys(bucket) + try: + for keylist in stream: + if len(keylist) > 0: + yield keylist + finally: + stream.close() @retryable def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, @@ -135,8 +139,12 @@ def stream_mapred(self, transport, inputs, query, timeout): Streams a MapReduce query as (phase, data) pairs. This is a generator method which should be iterated over. """ - for phase, data in transport.stream_mapred(inputs, query, timeout): - yield phase, data + stream = transport.stream_mapred(inputs, query, timeout) + try: + for phase, data in stream: + yield phase, data + finally: + stream.close() @retryableHttpOnly def fulltext_search(self, transport, index, query, **params): diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 2d5c5d20..98eeba65 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -19,7 +19,7 @@ from riak.tests.test_search import SearchTests, \ EnableSearchTests, SolrSearchTests from riak.tests.test_mapreduce import MapReduceAliasTests, \ - ErlangMapReduceTests, JSMapReduceTests, LinkTests + ErlangMapReduceTests, JSMapReduceTests, LinkTests, MapReduceStreamTests from riak.tests.test_kv import BasicKVTests, KVFileTests, \ HTTPBucketPropsTest, PbcBucketPropsTest from riak.tests.test_2i import TwoITests @@ -89,6 +89,7 @@ class RiakPbcTransportTestCase(BasicKVTests, ErlangMapReduceTests, JSMapReduceTests, MapReduceAliasTests, + MapReduceStreamTests, SearchTests, BaseTestCase, unittest.TestCase): @@ -176,6 +177,7 @@ class RiakHttpTransportTestCase(BasicKVTests, ErlangMapReduceTests, JSMapReduceTests, MapReduceAliasTests, + MapReduceStreamTests, EnableSearchTests, SolrSearchTests, SearchTests, diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 249d7590..13bb1920 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from riak.mapreduce import RiakLink +from riak.mapreduce import RiakLink, RiakMapReduce from riak import key_filter @@ -499,3 +499,34 @@ def test_filter_not_found(self): .run() self.assertEqual(sorted(result), [1, 2]) + +class MapReduceStreamTests(object): + def test_stream_results(self): + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + mr = RiakMapReduce(self.client).add('bucket', 'one').add('bucket', 'two') + mr.map_values_json() + results = [] + for phase, data in mr.stream(): + results.extend(data) + + self.assertEqual(sorted(results), [1,2]) + + def test_stream_cleanup(self): + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + mr = RiakMapReduce(self.client).add('bucket', 'one').add('bucket', 'two') + mr.map_values_json() + try: + for phase, data in mr.stream(): + raise RuntimeError("woops") + except RuntimeError: + pass + + # This should not raise an exception + obj = bucket.get('one') + self.assertEqual(1, obj.data) diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 5130a0dd..585c7489 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -46,6 +46,10 @@ def read(self): def next(self): raise NotImplementedError + def close(self): + pass + + class RiakHttpKeyStream(RiakHttpStream): """ Streaming iterator for list-keys over HTTP @@ -64,6 +68,7 @@ def next(self): else: raise StopIteration + class RiakHttpMultipartStream(RiakHttpStream): """ Streaming iterator for multipart messages over HTTP diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index 69594228..99a0d866 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -18,8 +18,9 @@ import json -from riak.transports.pbc.messages import MSG_CODE_LIST_KEYS_RESP -from riak.transports.pbc.messages import MSG_CODE_MAPRED_RESP +from riak.transports.pbc.messages import ( + MSG_CODE_LIST_KEYS_RESP, MSG_CODE_MAPRED_RESP + ) class RiakPbcStream(object): @@ -52,6 +53,15 @@ def _is_done(self, response): # same thing. return response.done + def close(self): + # We have to drain the socket to make sure that we don't get + # weird responses when the some other request comes after a + # failed/prematurely-terminated one. + try: + while self.next(): + pass + except StopIteration: + pass class RiakPbcKeyStream(RiakPbcStream): """ From efc091866da617b6c97b37afe1c8ecaacd4d4c73 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 29 Jan 2013 12:15:41 -0600 Subject: [PATCH 0299/1060] PEP8 cleanup. --- riak/client/operations.py | 9 +++++---- riak/client/transport.py | 3 +++ riak/tests/test_mapreduce.py | 11 +++++++---- riak/transports/http/stream.py | 1 + riak/transports/pbc/stream.py | 1 + 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 00366022..09747049 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -18,6 +18,7 @@ from transport import RiakClientTransport, retryable, retryableHttpOnly + class RiakClientOperations(RiakClientTransport): """ Methods for RiakClient that result in requests sent to the Riak @@ -101,8 +102,8 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=if_none_match) @retryable - def put_new(self, transport, robj, w=None, dw=None, pw=None, return_body=None, - if_none_match=None): + def put_new(self, transport, robj, w=None, dw=None, pw=None, + return_body=None, if_none_match=None): """ Stores an object in the Riak cluster with a generated key. """ @@ -118,8 +119,8 @@ def get(self, transport, robj, r=None, pr=None, vtag=None): return transport.get(robj, r=r, pr=pr, vtag=vtag) @retryable - def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, pr=None, - pw=None): + def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, + pr=None, pw=None): """ Deletes an object from Riak. """ diff --git a/riak/client/transport.py b/riak/client/transport.py index 54e26de6..ae7f749c 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -80,9 +80,12 @@ def is_retryable(error): def retryable(fn, protocol=None): def wrapper(self, *args, **kwargs): pool = self._choose_pool(protocol) + def thunk(transport): return fn(self, transport, *args, **kwargs) + return self._with_retries(pool, thunk) + return wrapper diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 13bb1920..b6e2d1b2 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -500,26 +500,29 @@ def test_filter_not_found(self): self.assertEqual(sorted(result), [1, 2]) + class MapReduceStreamTests(object): def test_stream_results(self): bucket = self.client.bucket('bucket') bucket.new('one', data=1).store() bucket.new('two', data=2).store() - mr = RiakMapReduce(self.client).add('bucket', 'one').add('bucket', 'two') + mr = RiakMapReduce(self.client).add('bucket', 'one')\ + .add('bucket', 'two') mr.map_values_json() results = [] for phase, data in mr.stream(): results.extend(data) - self.assertEqual(sorted(results), [1,2]) + self.assertEqual(sorted(results), [1, 2]) - def test_stream_cleanup(self): + def test_stream_cleanoperationsup(self): bucket = self.client.bucket('bucket') bucket.new('one', data=1).store() bucket.new('two', data=2).store() - mr = RiakMapReduce(self.client).add('bucket', 'one').add('bucket', 'two') + mr = RiakMapReduce(self.client).add('bucket', 'one')\ + .add('bucket', 'two') mr.map_values_json() try: for phase, data in mr.stream(): diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 585c7489..4e09f3fd 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -22,6 +22,7 @@ from cgi import parse_header from email import message_from_string + class RiakHttpStream(object): """ Base class for HTTP streaming iterators. diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index 99a0d866..689a26d1 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -63,6 +63,7 @@ def close(self): except StopIteration: pass + class RiakPbcKeyStream(RiakPbcStream): """ Used internally by RiakPbcTransport to implement key-list streams. From 41df3a7135a58bba1c41103d326e73e0228f80c2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 29 Jan 2013 12:18:05 -0600 Subject: [PATCH 0300/1060] Cleanup some pyflakes errors. --- riak/transports/http/transport.py | 1 - riak/transports/pbc/transport.py | 1 - 2 files changed, 2 deletions(-) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 8cbbcb7c..7f172e5e 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -224,7 +224,6 @@ def stream_keys(self, bucket): if headers['http_code'] is 200: return RiakHttpKeyStream(response) else: - stream.close() raise Exception('Error listing keys.') def get_buckets(self): diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index bc60cc21..426619b3 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -19,7 +19,6 @@ under the License. """ -import json import riak_pb from riak import RiakError from riak.transports.transport import RiakTransport From 76edabe9abeea837f0102d557cde8ad346251950 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 29 Jan 2013 13:18:11 -0600 Subject: [PATCH 0301/1060] Test aborting a key stream. --- riak/tests/test_kv.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 5b4a5e90..65862410 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -84,6 +84,20 @@ def test_stream_keys(self): streamed_keys += keylist self.assertEqual(sorted(regular_keys), sorted(streamed_keys)) + def test_stream_keys_abort(self): + bucket = self.client.bucket('random_key_bucket') + regular_keys = bucket.get_keys() + self.assertNotEqual(len(regular_keys), 0) + try: + for keylist in bucket.stream_keys(): + raise RuntimeError("abort") + except RuntimeError: + pass + + # If the stream was closed correctly, this will not error + robj = bucket.get(regular_keys[0]) + self.assertEqual(True, robj.exists) + def test_binary_store_and_get(self): bucket = self.client.bucket('bucket') # Store as binary, retrieve as binary, then compare... From 7060c30df61b17efbc659d6ccc02b038486846c7 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 30 Jan 2013 16:11:13 -0600 Subject: [PATCH 0302/1060] Document all the things --- riak/__init__.py | 3 + riak/bucket.py | 48 ++-- riak/client/__init__.py | 6 +- riak/client/operations.py | 119 +++++++++- riak/client/transport.py | 40 +++- riak/mapreduce.py | 346 +++++++++++++++++++++++------ riak/node.py | 12 + riak/riak_object.py | 24 +- riak/search.py | 17 ++ riak/tests/test_all.py | 51 ----- riak/transports/http/__init__.py | 2 + riak/transports/http/connection.py | 3 + riak/transports/pbc/__init__.py | 2 + riak/transports/pbc/codec.py | 22 ++ riak/transports/pbc/connection.py | 3 + riak/transports/pbc/stream.py | 2 +- riak/transports/pbc/transport.py | 8 +- 17 files changed, 547 insertions(+), 161 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index 0d8ab38a..2296f2a1 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -32,6 +32,9 @@ class RiakError(Exception): + """ + Base class for exceptions generated in the Riak API. + """ def __init__(self, value): self.value = value diff --git a/riak/bucket.py b/riak/bucket.py index 2fa2d088..0dae731a 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -222,7 +222,11 @@ def _set_r(self, val): def _get_r(self): return self.get_property('r') - r = property(_get_r, _set_r) + r = property(_get_r, _set_r, doc=""" + The default 'read' quorum for this bucket (how many replicas must + reply for a successful read). This should be an integer less than + the 'n_val' property, or a string of 'one', 'quorum', 'all', or + 'default'""") def _set_pr(self, val): return self.set_property('pr', val) @@ -230,7 +234,11 @@ def _set_pr(self, val): def _get_pr(self): return self.get_property('pr') - pr = property(_get_pr, _set_pr) + pr = property(_get_pr, _set_pr, doc=""" + The default 'primary read' quorum for this bucket (how many + primary replicas are required for a successful read). This should + be an integer less than the 'n_val' property, or a string of + 'one', 'quorum', 'all', or 'default'""") def _set_rw(self, val): return self.set_property('rw', val) @@ -238,7 +246,11 @@ def _set_rw(self, val): def _get_rw(self): return self.get_property('rw') - rw = property(_get_rw, _set_rw) + rw = property(_get_rw, _set_rw, doc=""" + The default 'read' and 'write' quorum for this bucket (equivalent + to 'r' and 'w' but for deletes). This should be an integer less + than the 'n_val' property, or a string of 'one', 'quorum', 'all', + or 'default'""") def _set_w(self, val): return self.set_property('w', val) @@ -246,7 +258,11 @@ def _set_w(self, val): def _get_w(self): return self.get_property('w') - w = property(_get_w, _set_w) + w = property(_get_w, _set_w, doc=""" + The default 'write' quorum for this bucket (how many replicas must + acknowledge receipt of a write). This should be an integer less + than the 'n_val' property, or a string of 'one', 'quorum', 'all', + or 'default'""") def _set_dw(self, val): return self.set_property('dw', val) @@ -254,7 +270,11 @@ def _set_dw(self, val): def _get_dw(self): return self.get_property('dw') - dw = property(_get_dw, _set_dw) + dw = property(_get_dw, _set_dw, doc=""" + The default 'durable write' quorum for this bucket (how many + replicas must commit the write). This should be an integer less + than the 'n_val' property, or a string of 'one', 'quorum', 'all', + or 'default'""") def _set_pw(self, val): return self.set_property('pw', val) @@ -262,16 +282,16 @@ def _set_pw(self, val): def _get_pw(self): return self.get_property('pw') - pw = property(_get_pw, _set_pw) + pw = property(_get_pw, _set_pw, doc=""" + The default 'primary write' quorum for this bucket (how many + primary replicas are required for a successful write). This should + be an integer less than the 'n_val' property, or a string of + 'one', 'quorum', 'all', or 'default'""") def set_property(self, key, value): """ Set a bucket property. - .. warning:: - - This should only be used if you know what you are doing. - :param key: Property to set. :type key: string :param value: Property value. @@ -296,12 +316,8 @@ def set_properties(self, props): """ Set multiple bucket properties in one call. - .. warning:: - - This should only be used if you know what you are doing. - - :param props: An associative array of key:value. - :type props: array + :param props: A dictionary of properties + :type props: dict """ self._client.set_bucket_props(self, props) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 4b816551..4fc285cf 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -93,11 +93,12 @@ def _set_protocol(self, value): self._protocol = value protocol = property(_get_protocol, _set_protocol, - doc="""which protocol to prefer""") + doc="""Which protocol to prefer, one of PROTOCOLS""") def get_transport(self): """ - Get the transport instance the client is using for it's connection. + Get the transport instance the client is using for it's + connection. DEPRECATED """ deprecated("get_transport is deprecated, use client, " + "bucket, or object methods instead") @@ -117,6 +118,7 @@ def get_client_id(self): def set_client_id(self, client_id): """ Set the client_id for this ``RiakClient`` instance. + DEPRECATED :param client_id: The new client_id. :type client_id: string diff --git a/riak/client/operations.py b/riak/client/operations.py index 09747049..d9c3f33f 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -53,6 +53,16 @@ def ping(self, transport): def get_index(self, transport, bucket, index, startkey, endkey=None): """ Queries a secondary index, returning matching keys. + + :param bucket: the bucket whose index will be queried + :type bucket: RiakBucket + :param index: the index to query + :type index: string + :param startkey: the sole key to query, or beginning of the query range + :type startkey: string, integer + :param endkey: the end of the query range (optional if equality) + :type endkey: string, integer + :rtype: list """ return transport.get_index(bucket, index, startkey, endkey) @@ -60,6 +70,10 @@ def get_index(self, transport, bucket, index, startkey, endkey=None): def get_bucket_props(self, transport, bucket): """ Fetches bucket properties for the given bucket. + + :param bucket: the bucket whose properties will be fetched + :type bucket: RiakBucket + :rtype: dict """ return transport.get_bucket_props(bucket) @@ -67,6 +81,11 @@ def get_bucket_props(self, transport, bucket): def set_bucket_props(self, transport, bucket, props): """ Sets bucket properties for the given bucket. + + :param bucket: the bucket whose properties will be set + :type bucket: RiakBucket + :param props: the properties to set + :type props: dict """ return transport.set_bucket_props(bucket, props) @@ -74,6 +93,10 @@ def set_bucket_props(self, transport, bucket, props): def get_keys(self, transport, bucket): """ Lists all keys in a bucket. + + :param bucket: the bucket whose properties will be set + :type bucket: RiakBucket + :rtype: list """ return transport.get_keys(bucket) @@ -82,6 +105,11 @@ def stream_keys(self, transport, bucket): """ Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. + + + :param bucket: the bucket whose properties will be set + :type bucket: RiakBucket + :rtype: iterator """ stream = transport.stream_keys(bucket) try: @@ -96,6 +124,21 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=None): """ Stores an object in the Riak cluster. + + :param robj: the object to store + :type robj: RiakObject + :param w: the write quorum + :type w: integer, string, None + :param dw: the durable write quorum + :type dw: integer, string, None + :param pw: the primary write quorum + :type pw: integer, string, None + :param return_body: whether to return the resulting object + after the write + :type return_body: boolean + :param if_none_match: whether to fail the write if the object + exists + :type if_none_match: boolean """ return transport.put(robj, w=w, dw=dw, pw=pw, return_body=return_body, @@ -106,6 +149,21 @@ def put_new(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=None): """ Stores an object in the Riak cluster with a generated key. + + :param robj: the object to store + :type robj: RiakObject + :param w: the write quorum + :type w: integer, string, None + :param dw: the durable write quorum + :type dw: integer, string, None + :param pw: the primary write quorum + :type pw: integer, string, None + :param return_body: whether to return the resulting object + after the write + :type return_body: boolean + :param if_none_match: whether to fail the write if the object + exists + :type if_none_match: boolean """ return transport.put_new(robj, w=w, dw=dw, pw=pw, return_body=return_body, @@ -115,6 +173,15 @@ def put_new(self, transport, robj, w=None, dw=None, pw=None, def get(self, transport, robj, r=None, pr=None, vtag=None): """ Fetches the contents of a Riak object. + + :param robj: the object to fetch + :type robj: RiakObject + :param r: the read quorum + :type r: integer, string, None + :param pr: the primary read quorum + :type pr: integer, string, None + :param vtag: the specific sibling to fetch + :type vtag: string """ return transport.get(robj, r=r, pr=pr, vtag=vtag) @@ -123,6 +190,21 @@ def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Deletes an object from Riak. + + :param robj: the object to store + :type robj: RiakObject + :param rw: the read/write (delete) quorum + :type rw: integer, string, None + :param r: the read quorum + :type r: integer, string, None + :param pr: the primary read quorum + :type pr: integer, string, None + :param w: the write quorum + :type w: integer, string, None + :param dw: the durable write quorum + :type dw: integer, string, None + :param pw: the primary write quorum + :type pw: integer, string, None """ return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) @@ -130,7 +212,15 @@ def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, @retryable def mapred(self, transport, inputs, query, timeout): """ - Executes a MapReduce query + Executes a MapReduce query. + + :param inputs: the input list/structure + :type inputs: list, dict + :param query: the list of query phases + :type query: list + :param timeout: the query timeout + :type timeout: integer, None + :rtype: mixed """ return transport.mapred(inputs, query, timeout) @@ -139,6 +229,14 @@ def stream_mapred(self, transport, inputs, query, timeout): """ Streams a MapReduce query as (phase, data) pairs. This is a generator method which should be iterated over. + + :param inputs: the input list/structure + :type inputs: list, dict + :param query: the list of query phases + :type query: list + :param timeout: the query timeout + :type timeout: integer, None + :rtype: iterator """ stream = transport.stream_mapred(inputs, query, timeout) try: @@ -151,6 +249,13 @@ def stream_mapred(self, transport, inputs, query, timeout): def fulltext_search(self, transport, index, query, **params): """ Performs a full-text search query. + + :param index: the bucket/index to search over + :type index: string + :param query: the search query + :type query: string + :param params: additional query flags + :type params: dict """ return transport.search(index, query, **params) @@ -158,6 +263,11 @@ def fulltext_search(self, transport, index, query, **params): def fulltext_add(self, transport, index, docs): """ Adds documents to the full-text index. + + :param index: the bucket/index in which to index these docs + :type index: string + :param docs: the list of documents + :type docs: list """ transport.fulltext_add(index, docs) @@ -165,5 +275,12 @@ def fulltext_add(self, transport, index, docs): def fulltext_delete(self, transport, index, docs=None, queries=None): """ Removes documents from the full-text index. + + :param index: the bucket/index from which to delete + :type index: string + :param docs: a list of documents (with ids) + :type docs: list + :param queries: a list of queries to match and delete + :type queries: list """ transport.fulltext_delete(index, docs, queries) diff --git a/riak/client/transport.py b/riak/client/transport.py index ae7f749c..5c42c55c 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -36,11 +36,23 @@ class RiakClientTransport(object): @contextmanager def _transport(self): + """ + Yields a single transport to the caller from the default pool, + without retries. + """ pool = self._choose_pool() with pool.take() as transport: yield transport def _with_retries(self, pool, fn): + """ + Performs the passed function with retries against the given pool. + + :param pool: the connection pool to use + :type pool: Pool + :param fn: the function to pass a transport + :type fn: function + """ skip_nodes = [] def _skip_bad_nodes(transport): @@ -52,7 +64,7 @@ def _skip_bad_nodes(transport): try: return fn(transport) except (IOError, httplib.HTTPException) as e: - if is_retryable(e): + if _is_retryable(e): transport._node.error_rate.incr(1) skip_nodes.append(transport._node) raise BadResource(e) @@ -62,6 +74,14 @@ def _skip_bad_nodes(transport): continue def _choose_pool(self, protocol=None): + """ + Selects a connection pool according to the default protocol + and the passed one. + + :param protocol: the protocol to use + :type protocol: string + :rtype: Pool + """ if not protocol: protocol = self.protocol if protocol in ['http', 'https']: @@ -73,11 +93,23 @@ def _choose_pool(self, protocol=None): return pool -def is_retryable(error): +def _is_retryable(error): + """ + Determines whether a given error is retryable according to the + exceptions allowed to be retried by each transport. + + :param error: the error to check + :type error: Exception + :rtype: boolean + """ return is_pbc_retryable(error) or is_http_retryable(error) def retryable(fn, protocol=None): + """ + Wraps a client operation that can be retried according to the set + RETRY_COUNT. Used internally. + """ def wrapper(self, *args, **kwargs): pool = self._choose_pool(protocol) @@ -90,4 +122,8 @@ def thunk(transport): def retryableHttpOnly(fn): + """ + Wraps a retryable client operation that is only valid over HTTP. + Used internally. + """ return retryable(fn, protocol='http') diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 4bd7712c..ebbdedff 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -24,13 +24,15 @@ class RiakMapReduce(object): """ The RiakMapReduce object allows you to build up and run a - map/reduce operation on Riak. + map/reduce operation on Riak. Most methods return the object on + which it was called, modified with new information, so you can + chain calls together to build the job. """ def __init__(self, client): """ Construct a Map/Reduce object. - @param RiakClient client - A RiakClient object. - @return RiakMapReduce + :param client: A RiakClient object. + :type client: RiakClient """ self._client = client self._phases = [] @@ -44,10 +46,14 @@ def add(self, arg1, arg2=None, arg3=None): different forms, depending on the provided inputs. You can specify either a RiakObject, a string bucket name, or a bucket, key, and additional arg. - @param mixed arg1 - RiakObject or Bucket - @param mixed arg2 - Key or List or blank - @param mixed arg3 - Arg or blank - @return RiakMapReduce + + :param arg1: the object or bucket to add + :type arg1: RiakObject, string + :param arg2: a key or list of keys to add (if a bucket is given in arg1) + :type arg2: string, list, None + :param arg3: key data for this input (must be convertible to JSON) + :type arg3: string, list, dict, None + :rtype: RiakMapReduce """ if (arg2 is None) and (arg3 is None): if isinstance(arg1, RiakObject): @@ -58,9 +64,27 @@ def add(self, arg1, arg2=None, arg3=None): return self.add_bucket_key_data(arg1, arg2, arg3) def add_object(self, obj): + """ + Adds a RiakObject to the inputs. + + :param obj: the object to add + :type obj: RiakObject + :rtype: RiakMapReduce + """ return self.add_bucket_key_data(obj._bucket._name, obj._key, None) def add_bucket_key_data(self, bucket, key, data): + """ + Adds a bucket/key/keydata triple to the inputs. + + :param bucket: the bucket + :type bucket: string + :param key: the key or list of keys + :type key: string + :param data: the key-specific data + :type data: string, list, dict, None + :rtype: RiakMapReduce + """ if self._input_mode == 'bucket': raise ValueError('Already added a bucket, can\'t add an object.') elif self._input_mode == 'query': @@ -75,11 +99,25 @@ def add_bucket_key_data(self, bucket, key, data): return self def add_bucket(self, bucket): + """ + Adds all keys in a bucket to the inputs. + + :param bucket: the bucket + :type bucket: string + :rtype: RiakMapReduce + """ self._input_mode = 'bucket' self._inputs = bucket return self def add_key_filters(self, key_filters): + """ + Adds key filters to the inputs. + + :param key_filters: a list of filters + :type key_filters: list + :rtype: RiakMapReduce + """ if self._input_mode == 'query': raise ValueError('Key filters are not supported in a query.') @@ -87,6 +125,13 @@ def add_key_filters(self, key_filters): return self def add_key_filter(self, *args): + """ + Add a single key filter to the inputs. + + :param args: a filter + :type args: list + :rtype: RiakMapReduce + """ if self._input_mode == 'query': raise ValueError('Key filters are not supported in a query.') @@ -97,8 +142,12 @@ def search(self, bucket, query): """ Begin a map/reduce operation using a Search. This command will return an error unless executed against a Riak Search cluster. - @param bucket - The bucket over which to perform the search. - @param query - The search query. + + :param bucket: The bucket over which to perform the search + :type bucket: string + :param query: The search query + :type query: string + :rtype: RiakMapReduce """ self._input_mode = 'query' self._inputs = {'module': 'riak_search', @@ -110,10 +159,16 @@ def index(self, bucket, index, startkey, endkey=None): """ Begin a map/reduce operation using a Secondary Index query. - @param bucket - The bucket over which to perform the search. - @param index - The index to use for query - @param startkey - The start key of index range - @param endkey - The end key of index range or blank + + :param bucket: The bucket over which to perform the query + :type bucket: string + :param index: The index to use for query + :type index: string + :param startkey: The start key of index range, or the + value which all entries must equal + :type startkey: string, integer + :param endkey: The end key of index range (if doing a range query) + :type endkey: string, integer, None """ self._input_mode = 'query' @@ -131,13 +186,17 @@ def index(self, bucket, index, startkey, endkey=None): def link(self, bucket='_', tag='_', keep=False): """ Add a link phase to the map/reduce operation. - @param string bucket - Bucket name (default '_', which means all + + :param bucket: Bucket name (default '_', which means all buckets) - @param string tag - Tag (default '_', which means any tag) - @param boolean keep - Flag whether to keep results from this - stage in the map/reduce. (default False, unless this is the last - step in the phase) - @return self + :type bucket: string + :param tag: Tag (default '_', which means any tag) + :type tag: string + :param keep: Flag whether to keep results from this stage in + the map/reduce. (default False, unless this is the last step + in the phase) + :type keep: boolean + :rtype: RiakMapReduce """ self._phases.append(RiakLinkPhase(bucket, tag, keep)) return self @@ -145,13 +204,16 @@ def link(self, bucket='_', tag='_', keep=False): def map(self, function, options=None): """ Add a map phase to the map/reduce operation. - @param mixed function - Either a named Javascript function (ie: - 'Riak.mapValues'), or an anonymous javascript function (ie: - 'function(...) ... ' or an array ['erlang_module', - 'function']. - @param array() options - An optional associative array - containing 'language', 'keep' flag, and/or 'arg'. - @return self + + :param function: Either a named Javascript function (ie: + 'Riak.mapValues'), or an anonymous javascript function (ie: + 'function(...) ... ' or an array ['erlang_module', + 'function']. + :type function: string, list + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + :rtype: RiakMapReduce """ if options is None: options = dict() @@ -171,12 +233,15 @@ def map(self, function, options=None): def reduce(self, function, options=None): """ Add a reduce phase to the map/reduce operation. - @param mixed function - Either a named Javascript function (ie. - 'Riak.mapValues'), or an anonymous javascript function(ie: - 'function(...) { ... }' or an array ['erlang_module', 'function']. - @param array() options - An optional associative array - containing 'language', 'keep' flag, and/or 'arg'. - @return self + + :param function: Either a named Javascript function (ie. + 'Riak.reduceSum'), or an anonymous javascript function(ie: + 'function(...) { ... }' or an array ['erlang_module', + 'function']. + :type function: string, list + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :rtype: RiakMapReduce """ if options is None: options = dict() @@ -195,10 +260,13 @@ def reduce(self, function, options=None): def run(self, timeout=None): """ - Run the map/reduce operation. Returns an array of results, or an - array of RiakLink objects if the last phase is a link phase. - @param integer timeout - Timeout in milliseconds. - @return array() + Run the map/reduce operation synchronously. Returns a list of + results, or a list of RiakLink objects if the last phase is a + link phase. + + :param timeout: Timeout in milliseconds + :type timeout: integer, None + :rtype: list """ query, link_results_flag = self._normalize_query() @@ -229,6 +297,10 @@ def run(self, timeout=None): def stream(self, timeout=None): """ Streams the MapReduce query (returns an iterator). + + :param timeout: Timeout in milliseconds + :type timeout: integer + :rtype: iterator """ query, lrf = self._normalize_query() return self._client.stream_mapred(self._inputs, query, timeout) @@ -272,21 +344,72 @@ def _normalize_query(self): # Start Shortcuts to built-ins ## def map_values(self, options=None): + """ + Adds the Javascript built-in ``Riak.mapValues`` to the query + as a map phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.map("Riak.mapValues", options=options) def map_values_json(self, options=None): + """ + Adds the Javascript built-in ``Riak.mapValuesJson`` to the + query as a map phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.map("Riak.mapValuesJson", options=options) def reduce_sum(self, options=None): + """ + Adds the Javascript built-in ``Riak.reduceSum`` to the query + as a reduce phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.reduce("Riak.reduceSum", options=options) def reduce_min(self, options=None): + """ + Adds the Javascript built-in ``Riak.reduceMin`` to the query + as a reduce phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.reduce("Riak.reduceMin", options=options) def reduce_max(self, options=None): + """ + Adds the Javascript built-in ``Riak.reduceMax`` to the query + as a reduce phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.reduce("Riak.reduceMax", options=options) def reduce_sort(self, js_cmp=None, options=None): + """ + Adds the Javascript built-in ``Riak.reduceSort`` to the query + as a reduce phase. + + :param js_cmp: A Javascript comparator function as specified by + Array.sort() + :type js_cmp: string + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ if options is None: options = dict() @@ -296,9 +419,27 @@ def reduce_sort(self, js_cmp=None, options=None): return self.reduce("Riak.reduceSort", options=options) def reduce_numeric_sort(self, options=None): + """ + Adds the Javascript built-in ``Riak.reduceNumericSort`` to the + query as a reduce phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.reduce("Riak.reduceNumericSort", options=options) def reduce_limit(self, limit, options=None): + """ + Adds the Javascript built-in ``Riak.reduceLimit`` to the query + as a reduce phase. + + :param limit: the maximum number of results to return + :type limit: integer + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ if options is None: options = dict() @@ -310,6 +451,18 @@ def reduce_limit(self, limit, options=None): return self.reduce(code, options=options) def reduce_slice(self, start, end, options=None): + """ + Adds the Javascript built-in ``Riak.reduceSlice`` to the + query as a reduce phase. + + :param start: the beginning of the slice + :type start: integer + :param end: the end of the slice + :type end: integer + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ if options is None: options = dict() @@ -317,25 +470,42 @@ def reduce_slice(self, start, end, options=None): return self.reduce("Riak.reduceSlice", options=options) def filter_not_found(self, options=None): + """ + Adds the Javascript built-in ``Riak.filterNotFound`` to the query + as a reduce phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.reduce("Riak.filterNotFound", options=options) class RiakMapReducePhase(object): """ - The RiakMapReducePhase holds information about a Map phase or - Reduce phase in a RiakMapReduce operation. + The RiakMapReducePhase holds information about a Map or Reduce + phase in a RiakMapReduce operation. + + Normally you won't need to use this object directly, but instead + call methods on RiakMapReduce objects to add instances to the + query. """ def __init__(self, type, function, language, keep, arg): """ Construct a RiakMapReducePhase object. - @param string type - 'map'placeholder149'reduce' - @param mixed function - string or array(): - @param string language - 'javascript'placeholder149'erlang' - @param boolean keep - True to return the output of this phase in - the results. - @param mixed arg - Additional value to pass into the map or - reduce function. + + :param type: the phase type - 'map', 'reduce', 'link' + :type type: string + :param function: the function to execute + :type function: string, list + :param language: 'javascript' or 'erlang' + :type language: string + :param keep: whether to return the output of this phase in the results. + :type keep: boolean + :param arg: Additional static value to pass into the map or + reduce function. + :type arg: string, dict, list """ try: if isinstance(function, basestring): @@ -351,8 +521,10 @@ def __init__(self, type, function, language, keep, arg): def to_array(self): """ - Convert the RiakMapReducePhase to an associative array. Used - internally. + Convert the RiakMapReducePhase to a format that can be output + into JSON. Used internally. + + :rtype: dict """ stepdef = {'keep': self._keep, 'language': self._language, @@ -379,14 +551,21 @@ class RiakLinkPhase(object): """ The RiakLinkPhase object holds information about a Link phase in a map/reduce operation. + + Normally you won't need to use this object directly, but instead + call ``link`` on RiakMapReduce objects to add instances to the + query. """ def __init__(self, bucket, tag, keep): """ Construct a RiakLinkPhase object. - @param string bucket - The bucket name. - @param string tag - The tag. - @param boolean keep - True to return results of this phase. + :param bucket: - The bucket name + :type bucket: string + :param tag: The tag + :type tag: string + :param keep: whether to return results of this phase. + :type keep: boolean """ self._bucket = bucket self._tag = tag @@ -394,8 +573,8 @@ def __init__(self, bucket, tag, keep): def to_array(self): """ - Convert the RiakLinkPhase to an associative array. Used - internally. + Convert the RiakLinkPhase to a format that can be output into + JSON. Used internally. """ stepdef = {'bucket': self._bucket, 'tag': self._tag, @@ -412,9 +591,13 @@ class RiakLink(object): def __init__(self, bucket, key, tag=None): """ Construct a RiakLink object. - @param string bucket - The bucket name. - @param string key - The key. - @param string tag - The tag. + + :param bucket: the bucket name + :type bucket: string + :param key: the key + :type key: string + :param tag: the tag + :type tag: string """ self._bucket = bucket self._key = key @@ -424,31 +607,38 @@ def __init__(self, bucket, key, tag=None): def get(self, r=None): """ Retrieve the RiakObject to which this link points. - @param integer r - The R-value to use. - @return RiakObject + + :param r: the read quorum to use + :type r: string, integer + :rtype: RiakObject """ return self._client.bucket(self._bucket).get(self._key, r) def get_binary(self, r=None): """ Retrieve the RiakObject to which this link points, as a binary. - @param integer r - The R-value to use. - @return RiakObject + + :param r: the read quorum to use + :type r: string, integer + :rtype: RiakObject """ return self._client.bucket(self._bucket).get_binary(self._key, r) def get_bucket(self): """ Get the bucket name of this link. - @return string + + :rtype: string """ return self._bucket def set_bucket(self, name): """ Set the bucket name of this link. - @param string name - The bucket name. - @return self + + :param name: the bucket name + :type name: string + :rtype: RiakLink """ self._bucket = name return self @@ -456,15 +646,18 @@ def set_bucket(self, name): def get_key(self): """ Get the key of this link. - @return string + + :rtype: string """ return self._key def set_key(self, key): """ Set the key of this link. - @param string key - The key. - @return self + + :param key: the key + :type key: string + :rtype: RiakLink """ self._key = key return self @@ -472,7 +665,8 @@ def set_key(self, key): def get_tag(self): """ Get the tag of this link. - @return string + + :rtype: string """ if (self._tag is None): return self._bucket @@ -482,15 +676,20 @@ def get_tag(self): def set_tag(self, tag): """ Set the tag of this link. - @param string tag - The tag. - @return self + + :param tag: the tag + :type tag: string + :rtype: RiakLink """ self._tag = tag return self def to_link_header(self, client): """ - Convert this RiakLink object to a link header string. Used internally. + Convert this RiakLink object to a link header string. Used + internally. + + :rtype: string """ link = '' link += ' Date: Wed, 30 Jan 2013 16:19:05 -0600 Subject: [PATCH 0303/1060] One last fix. --- riak/bucket.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 0dae731a..781aa4f3 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -323,9 +323,9 @@ def set_properties(self, props): def get_properties(self): """ - Retrieve an associative array of all bucket properties. + Retrieve a dict of all bucket properties. - :rtype: array + :rtype: dict """ return self._client.get_bucket_props(self) @@ -341,11 +341,13 @@ def get_keys(self): def stream_keys(self): """ - Return all keys within the bucket. + Streams all keys within the bucket through an iterator. .. warning:: At current, this is a very expensive operation. Use with caution. + + :rtype: iterator """ return self._client.stream_keys(self) From 0abef5ff0d0f564a045331b3712d6b2fec9d82e3 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 09:07:58 -0600 Subject: [PATCH 0304/1060] Remove some more 'is' in favor of '=='. --- riak/transports/http/transport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 7f172e5e..959b313a 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -211,7 +211,7 @@ def get_keys(self, bucket): response = self._request('GET', url) headers, encoded_props = response[0:2] - if headers['http_code'] is 200: + if headers['http_code'] == 200: props = json.loads(encoded_props) return props['keys'] else: @@ -221,7 +221,7 @@ def stream_keys(self, bucket): url = self.key_list_path(bucket.name, keys='stream') headers, response = self._request('GET', url, stream=True) - if headers['http_code'] is 200: + if headers['http_code'] == 200: return RiakHttpKeyStream(response) else: raise Exception('Error listing keys.') From fb4e2dd5960f52713a335526d657556ea63bef04 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 09:11:36 -0600 Subject: [PATCH 0305/1060] Remove commented imports. --- riak/riak_object.py | 6 ------ riak/transports/http/transport.py | 2 -- 2 files changed, 8 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index fbcf4a17..cdc0359e 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -19,16 +19,10 @@ """ import copy from riak.metadata import ( - # MD_CHARSET, MD_CTYPE, - # MD_ENCODING, MD_INDEX, - # MD_LASTMOD, - # MD_LASTMOD_USECS, MD_LINKS, MD_USERMETA - # MD_VTAG, - # MD_DELETED ) from riak.mapreduce import ( RiakMapReduce, diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 959b313a..9d8269d8 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -35,10 +35,8 @@ from riak.metadata import ( MD_CHARSET, MD_CTYPE, - # MD_ENCODING, MD_INDEX, MD_LASTMOD, - # MD_LASTMOD_USECS, MD_LINKS, MD_USERMETA, MD_VTAG, From 880615036d3da7ce1ae38b83d2d6b475d207be2e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 09:12:35 -0600 Subject: [PATCH 0306/1060] Stylistic fix. --- riak/riak_object.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index cdc0359e..7e1e6573 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -165,8 +165,8 @@ def add_index(self, field, value): :rtype: RiakObject """ if field[-4:] not in ("_bin", "_int"): - raise RiakError( - "Riak 2i fields must end with either '_bin' or '_int'.") + raise RiakError("Riak 2i fields must end with either '_bin'" + " or '_int'.") rie = RiakIndexEntry(field, value) if not rie in self.metadata[MD_INDEX]: From 93019a8f5b57df6e6e20427f07dc83c8d7e30e62 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 09:12:53 -0600 Subject: [PATCH 0307/1060] Make read() private on HTTP stream iterators. --- riak/transports/http/stream.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 4e09f3fd..2f1620f9 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -38,7 +38,7 @@ def __init__(self, response): def __iter__(self): return self - def read(self): + def _read(self): chunk = self.response.read(self.BLOCK_SIZE) if chunk == '': self.response_done = True @@ -58,7 +58,7 @@ class RiakHttpKeyStream(RiakHttpStream): def next(self): while '}' not in self.buffer and not self.response_done: - self.read() + self._read() if '}' in self.buffer: idx = string.index(self.buffer, '}') + 1 @@ -111,7 +111,7 @@ def advance_buffer(self): def read_until_boundary(self): while not self.try_match() and not self.response_done: - self.read() + self._read() class RiakHttpMapReduceStream(RiakHttpMultipartStream): From fdb1aa42d2da3ab7753cc436acfd5e735252921e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 09:14:18 -0600 Subject: [PATCH 0308/1060] PEP8 line-length fix. --- riak/mapreduce.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index ebbdedff..6031eeb2 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -49,7 +49,8 @@ def add(self, arg1, arg2=None, arg3=None): :param arg1: the object or bucket to add :type arg1: RiakObject, string - :param arg2: a key or list of keys to add (if a bucket is given in arg1) + :param arg2: a key or list of keys to add (if a bucket is + given in arg1) :type arg2: string, list, None :param arg3: key data for this input (must be convertible to JSON) :type arg3: string, list, dict, None From 535d0bcd14dfe729fe378e244d3f2a974a15a477 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 14:50:49 -0600 Subject: [PATCH 0309/1060] Remove retries from streaming per peer-review discussion. --- riak/client/operations.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index d9c3f33f..e28c78e3 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -100,8 +100,7 @@ def get_keys(self, transport, bucket): """ return transport.get_keys(bucket) - @retryable - def stream_keys(self, transport, bucket): + def stream_keys(self, bucket): """ Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. @@ -111,13 +110,14 @@ def stream_keys(self, transport, bucket): :type bucket: RiakBucket :rtype: iterator """ - stream = transport.stream_keys(bucket) - try: - for keylist in stream: - if len(keylist) > 0: - yield keylist - finally: - stream.close() + with self._transport() as transport: + stream = transport.stream_keys(bucket) + try: + for keylist in stream: + if len(keylist) > 0: + yield keylist + finally: + stream.close() @retryable def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, @@ -224,8 +224,7 @@ def mapred(self, transport, inputs, query, timeout): """ return transport.mapred(inputs, query, timeout) - @retryable - def stream_mapred(self, transport, inputs, query, timeout): + def stream_mapred(self, inputs, query, timeout): """ Streams a MapReduce query as (phase, data) pairs. This is a generator method which should be iterated over. @@ -238,12 +237,13 @@ def stream_mapred(self, transport, inputs, query, timeout): :type timeout: integer, None :rtype: iterator """ - stream = transport.stream_mapred(inputs, query, timeout) - try: - for phase, data in stream: - yield phase, data - finally: - stream.close() + with self._transport() as transport: + stream = transport.stream_mapred(inputs, query, timeout) + try: + for phase, data in stream: + yield phase, data + finally: + stream.close() @retryableHttpOnly def fulltext_search(self, transport, index, query, **params): From 4de107eeb2779f9e539d45e5b9556e0537ccc8b1 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 15:11:44 -0600 Subject: [PATCH 0310/1060] Fix and update client comparisons for the new structure. --- riak/client/__init__.py | 15 +++++++++++++++ riak/tests/test_comparison.py | 23 +++++++++++------------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 4fc285cf..9016e250 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -227,3 +227,18 @@ def _error_rate(node): return min(nodes, key=_error_rate) else: return random.choice(good) + + def __hash__(self): + return hash(frozenset([ (n.host, n.http_port, n.pb_port) for n in self.nodes])) + + def __eq__(self, other): + if isinstance(other, self.__class__): + return hash(self) == hash(other) + else: + return False + + def __nq__(self, other): + if isinstance(other, self.__class__): + return hash(self) != hash(other) + else: + return True diff --git a/riak/tests/test_comparison.py b/riak/tests/test_comparison.py index 808198d9..f00a273c 100644 --- a/riak/tests/test_comparison.py +++ b/riak/tests/test_comparison.py @@ -5,7 +5,6 @@ else: import unittest -from riak import RiakHttpTransport from riak.client import RiakClient from riak.riak_object import RiakObject from riak.bucket import RiakBucket @@ -55,24 +54,24 @@ def test_object_hash(self): class RiakClientComparisonTest(unittest.TestCase, BaseTestCase): def test_client_eq(self): - self.transport_class = RiakHttpTransport - a = self.create_client('host1', 11) - b = self.create_client('host1', 11) + self.protocol = 'http' + a = self.create_client(host='host1', http_port=11) + b = self.create_client(host='host1', http_port=11) self.assertEqual(a, b) def test_client_nq(self): - self.transport_class = RiakHttpTransport - a = self.create_client('host1', 11) - b = self.create_client('host2', 11) - c = self.create_client('host1', 12) + self.protocol = 'http' + a = self.create_client(host='host1', http_port=11) + b = self.create_client(host='host1', http_port=11) + c = self.create_client(host='host1', http_port=12) self.assertNotEqual(a, b, 'matched with different hosts') self.assertNotEqual(a, c, 'matched with different ports') def test_client_hash(self): - self.transport_class = RiakHttpTransport - a = self.create_client('host1', 11) - b = self.create_client('host1', 11) - c = self.create_client('host2', 11) + self.protocol = 'http' + a = self.create_client(host='host1', http_port=11) + b = self.create_client(host='host1', http_port=11) + c = self.create_client(host='host2', http_port=11) self.assertEqual(hash(a), hash(b), 'same object has different hashes') self.assertNotEqual(hash(a), hash(c), 'different object has same hash') From c2e13d45902fbf13ca91d91f5511a916b7ee10e4 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 15:12:38 -0600 Subject: [PATCH 0311/1060] pep8 and pyflakes fixes. --- riak/client/__init__.py | 3 ++- riak/tests/test_comparison.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 9016e250..7f5ce4fe 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -229,7 +229,8 @@ def _error_rate(node): return random.choice(good) def __hash__(self): - return hash(frozenset([ (n.host, n.http_port, n.pb_port) for n in self.nodes])) + return hash(frozenset([(n.host, n.http_port, n.pb_port) + for n in self.nodes])) def __eq__(self, other): if isinstance(other, self.__class__): diff --git a/riak/tests/test_comparison.py b/riak/tests/test_comparison.py index f00a273c..3e9b3aa2 100644 --- a/riak/tests/test_comparison.py +++ b/riak/tests/test_comparison.py @@ -5,7 +5,6 @@ else: import unittest -from riak.client import RiakClient from riak.riak_object import RiakObject from riak.bucket import RiakBucket from riak.tests.test_all import BaseTestCase @@ -20,7 +19,6 @@ def test_bucket_eq(self): def test_bucket_nq(self): a = RiakBucket('client', 'a') b = RiakBucket('client', 'b') - c = RiakBucket('client', 'a') self.assertNotEqual(a, b, 'matched with a different bucket') def test_bucket_hash(self): From 5a10e0fe4bd6ced3d7296015d65a56885b658cc7 Mon Sep 17 00:00:00 2001 From: evan Date: Mon, 28 Jan 2013 10:51:41 -0800 Subject: [PATCH 0312/1060] prefer simplejson when available --- riak/client/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 7f5ce4fe..9425bdba 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -19,7 +19,11 @@ under the License. """ -import json +try: + import simplejson as json +except ImportError: + import json + import random from weakref import WeakValueDictionary from riak.client.operations import RiakClientOperations From 167c4da1769cf4343eb8f9acfbd1fa5d457f58e8 Mon Sep 17 00:00:00 2001 From: evan Date: Mon, 28 Jan 2013 12:56:57 -0800 Subject: [PATCH 0313/1060] also update transport imports --- riak/tests/test_kv.py | 5 ++++- riak/transports/http/transport.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 65862410..6e4981f4 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -2,7 +2,10 @@ import os import cPickle import copy -import json +try: + import simplejson as json +except ImportError: + import json class NotJsonSerializable(object): diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 9d8269d8..f95b7760 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -19,7 +19,11 @@ under the License. """ -import json +try: + import simplejson as json +except ImportError: + import json + import urllib import re import csv From e34a44cee73e2dd132ac203981d7056bf88497c3 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Thu, 31 Jan 2013 22:59:58 -0500 Subject: [PATCH 0314/1060] Got rid of riak_index_entry Used a tuple and a wrapper function (int -> string conversion) that looks like a class. Seeking comments. --- riak/riak_index_entry.py | 65 ------------------------------- riak/riak_object.py | 30 +++++++------- riak/tests/test_2i.py | 2 +- riak/transports/http/transport.py | 10 ++--- riak/transports/pbc/codec.py | 8 ++-- riak/util.py | 1 + 6 files changed, 25 insertions(+), 91 deletions(-) delete mode 100644 riak/riak_index_entry.py diff --git a/riak/riak_index_entry.py b/riak/riak_index_entry.py deleted file mode 100644 index 85b51ac8..00000000 --- a/riak/riak_index_entry.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - - -class RiakIndexEntry(object): - def __init__(self, field, value): - self._field = field - self._value = str(value) - - def get_field(self): - return self._field - - def get_value(self): - return self._value - - def __str__(self): - return ("RiakIndexEntry(field = '%s', value='%s')" % - (self._field, self._value)) - - def __eq__(self, other): - if not isinstance(other, RiakIndexEntry): - return False - - return \ - self.get_field() == other.get_field() and \ - self.get_value() == other.get_value() - - def __cmp__(self, other): - if other == None: - raise TypeError("RiakIndexEntry cannot be compared to None") - - if not isinstance(other, RiakIndexEntry): - raise TypeError("RiakIndexEntry cannot be compared to %s" % - other.__class__.__name__) - - if self.get_field() < other.get_field(): - return -1 - - if self.get_field() > other.get_field(): - return 1 - - if self.get_value() < other.get_value(): - return -1 - - if self.get_value() > other.get_value(): - return 1 - - return 0 diff --git a/riak/riak_object.py b/riak/riak_object.py index 2ff1552c..814e7979 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -29,8 +29,7 @@ RiakLink ) from riak import RiakError -from riak.riak_index_entry import RiakIndexEntry - +from riak.util import RiakIndexEntry class RiakObject(object): """ @@ -204,35 +203,35 @@ def remove_index(self, field=None, value=None): ries = self.metadata[MD_INDEX][:] elif field and not value: ries = [x for x in self.metadata[MD_INDEX] - if x.get_field() == field] + if x[0] == field] elif field and value: ries = [RiakIndexEntry(field, value)] else: raise RiakError("Cannot pass value without a field" " name while removing index") - for rie in ries: - if rie in self.metadata[MD_INDEX]: - self.metadata[MD_INDEX].remove(rie) + + # This removes the index entries that's in the ries list. + # Done because this is preferred over metadata[MD_INDEX].remove(rie) + self.metadata[MD_INDEX] = [rie for rie in self.metadata[MD_INDEX] + if rie not in ries] return self remove_indexes = remove_index def set_indexes(self, indexes): """ - Replaces all indexes on a Riak object. Currenly supports an - iterable of 2 item tuples, (field, value) + Replaces all indexes on a Riak object. Currently supports an + iterable of 2 item tuples, (field, value). :param indexes: iterable of 2 item tuples consisting the field - and value. + and value. Both the field and the value must be a string. :rtype: RiakObject """ - new_indexes = [] - for field, value in indexes: - rie = RiakIndexEntry(field, value) - new_indexes.append(rie) + # makes a copy and does type conversion + # this seems rather slow + new_indexes = [RiakIndexEntry(field, value) for field, value in indexes] self.metadata[MD_INDEX] = new_indexes - return self def get_indexes(self, field=None): @@ -247,8 +246,7 @@ def get_indexes(self, field=None): if field == None: return self.metadata[MD_INDEX] else: - return [x.get_value() for x in self.metadata[MD_INDEX] - if x.get_field() == field] + return [v for f, v in self.metadata[MD_INDEX] if f == field] def _get_content_type(self): try: diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index e2d6c5cb..73b9d141 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -6,7 +6,7 @@ else: import unittest -from riak.riak_index_entry import RiakIndexEntry +from riak.util import RiakIndexEntry from riak import RiakError SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '0')) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index f95b7760..ab1a23c0 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -48,7 +48,7 @@ ) from riak.mapreduce import RiakLink from riak import RiakError -from riak.riak_index_entry import RiakIndexEntry +from riak.util import RiakIndexEntry from riak.multidict import MultiDict from xml.etree import ElementTree from xml.dom.minidom import Document @@ -538,12 +538,12 @@ def build_put_headers(self, robj): for key, value in robj.usermeta.iteritems(): headers['X-Riak-Meta-%s' % key] = value - for rie in robj.get_indexes(): - key = 'X-Riak-Index-%s' % rie.get_field() + for field, value in robj.get_indexes(): + key = 'X-Riak-Index-%s' % field if key in headers: - headers[key] += ", " + rie.get_value() + headers[key] += ", " + value else: - headers[key] = rie.get_value() + headers[key] = value return headers diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 2742dc20..d3518917 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -29,7 +29,7 @@ ) import riak_pb -from riak.riak_index_entry import RiakIndexEntry +from riak.util import RiakIndexEntry from riak.mapreduce import RiakLink RIAKC_RW_ONE = 4294967294 @@ -152,10 +152,10 @@ def encode_content(self, metadata, data, rpb_content): pair.key = uk pair.value = v[uk] elif k == MD_INDEX: - for rie in v: + for field, value in v: pair = rpb_content.indexes.add() - pair.key = rie.get_field() - pair.value = rie.get_value() + pair.key = field + pair.value = value elif k == MD_LINKS: for link in v: pb_link = rpb_content.links.add() diff --git a/riak/util.py b/riak/util.py index c590db53..3c5bf195 100644 --- a/riak/util.py +++ b/riak/util.py @@ -19,6 +19,7 @@ import warnings from collections import Mapping +RiakIndexEntry = lambda field, value: (field, str(value)) def quacks_like_dict(object): """Check if object is dict-like""" From 4470a6825e19ce0639ed8abac9c1b5988ef45569 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Fri, 1 Feb 2013 13:34:20 -0500 Subject: [PATCH 0315/1060] Got rid of the wrapper function. --- riak/riak_object.py | 11 +++++------ riak/tests/test_2i.py | 31 +++++++++++++++---------------- riak/transports/http/transport.py | 9 +++++---- riak/transports/pbc/codec.py | 9 ++++++--- riak/util.py | 2 -- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 814e7979..7969ef92 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -29,7 +29,6 @@ RiakLink ) from riak import RiakError -from riak.util import RiakIndexEntry class RiakObject(object): """ @@ -182,7 +181,7 @@ def add_index(self, field, value): raise RiakError("Riak 2i fields must end with either '_bin'" " or '_int'.") - rie = RiakIndexEntry(field, value) + rie = (field, value) if not rie in self.metadata[MD_INDEX]: self.metadata[MD_INDEX].append(rie) @@ -205,7 +204,7 @@ def remove_index(self, field=None, value=None): ries = [x for x in self.metadata[MD_INDEX] if x[0] == field] elif field and value: - ries = [RiakIndexEntry(field, value)] + ries = [(field, value)] else: raise RiakError("Cannot pass value without a field" " name while removing index") @@ -230,8 +229,7 @@ def set_indexes(self, indexes): """ # makes a copy and does type conversion # this seems rather slow - new_indexes = [RiakIndexEntry(field, value) for field, value in indexes] - self.metadata[MD_INDEX] = new_indexes + self.metadata[MD_INDEX] = indexes[:] return self def get_indexes(self, field=None): @@ -241,7 +239,8 @@ def get_indexes(self, field=None): :param field: The index field. :type field: string or None - :rtype: (array of RiakIndexEntry) or (array of string or integer) + :rtype: (array of 2 element tuples with field, value) or + (array of string or integer) """ if field == None: return self.metadata[MD_INDEX] diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 73b9d141..b2827590 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -6,7 +6,6 @@ else: import unittest -from riak.util import RiakIndexEntry from riak import RiakError SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '0')) @@ -39,7 +38,7 @@ def test_secondary_index_store(self): # Retrieve the object, check that the correct indexes exist... obj = bucket.get('mykey1') self.assertEqual(['val1a'], sorted(obj.get_indexes('field1_bin'))) - self.assertEqual(['1011'], sorted(obj.get_indexes('field1_int'))) + self.assertEqual([1011], sorted(obj.get_indexes('field1_int'))) # Add more indexes and save... obj.add_index('field1_bin', 'val1b') @@ -50,15 +49,15 @@ def test_secondary_index_store(self): obj = bucket.get('mykey1') self.assertEqual(['val1a', 'val1b'], sorted(obj.get_indexes('field1_bin'))) - self.assertEqual(['1011', '1012'], + self.assertEqual([1011, 1012], sorted(obj.get_indexes('field1_int'))) # Check the get_indexes() function... self.assertEqual([ - RiakIndexEntry('field1_bin', 'val1a'), - RiakIndexEntry('field1_bin', 'val1b'), - RiakIndexEntry('field1_int', 1011), - RiakIndexEntry('field1_int', 1012) + ('field1_bin', 'val1a'), + ('field1_bin', 'val1b'), + ('field1_int', 1011), + ('field1_int', 1012) ], sorted(obj.get_indexes())) # Delete an index... @@ -69,7 +68,7 @@ def test_secondary_index_store(self): # Retrieve the object, check that the correct indexes exist... obj = bucket.get('mykey1') self.assertEqual(['val1b'], sorted(obj.get_indexes('field1_bin'))) - self.assertEqual(['1012'], sorted(obj.get_indexes('field1_int'))) + self.assertEqual([1012], sorted(obj.get_indexes('field1_int'))) # Check duplicate entries... obj.add_index('field1_bin', 'val1a') @@ -80,20 +79,20 @@ def test_secondary_index_store(self): obj.add_index('field1_int', 1011) self.assertEqual([ - RiakIndexEntry('field1_bin', 'val1a'), - RiakIndexEntry('field1_bin', 'val1b'), - RiakIndexEntry('field1_int', 1011), - RiakIndexEntry('field1_int', 1012) + ('field1_bin', 'val1a'), + ('field1_bin', 'val1b'), + ('field1_int', 1011), + ('field1_int', 1012) ], sorted(obj.get_indexes())) obj.store() obj = bucket.get('mykey1') self.assertEqual([ - RiakIndexEntry('field1_bin', 'val1a'), - RiakIndexEntry('field1_bin', 'val1b'), - RiakIndexEntry('field1_int', 1011), - RiakIndexEntry('field1_int', 1012) + ('field1_bin', 'val1a'), + ('field1_bin', 'val1b'), + ('field1_int', 1011), + ('field1_int', 1012) ], sorted(obj.get_indexes())) # Clean up... diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index ab1a23c0..962fd7f6 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -48,7 +48,6 @@ ) from riak.mapreduce import RiakLink from riak import RiakError -from riak.util import RiakIndexEntry from riak.multidict import MultiDict from xml.etree import ElementTree from xml.dom.minidom import Document @@ -463,7 +462,9 @@ def parse_body(self, response, expected_statuses): reader = csv.reader([value], skipinitialspace=True) for line in reader: for token in line: - rie = RiakIndexEntry(field, token) + if field.endswith("_int"): + token = int(token) + rie = (field, token) metadata[MD_INDEX].append(rie) elif header == 'x-riak-vclock': vclock = value @@ -541,9 +542,9 @@ def build_put_headers(self, robj): for field, value in robj.get_indexes(): key = 'X-Riak-Index-%s' % field if key in headers: - headers[key] += ", " + value + headers[key] += ", " + str(value) else: - headers[key] = value + headers[key] = str(value) return headers diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index d3518917..f829f5b6 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -29,7 +29,6 @@ ) import riak_pb -from riak.util import RiakIndexEntry from riak.mapreduce import RiakLink RIAKC_RW_ONE = 4294967294 @@ -125,7 +124,11 @@ def decode_content(self, rpb_content): metadata[MD_USERMETA] = usermeta indexes = [] for index in rpb_content.indexes: - rie = RiakIndexEntry(index.key, index.value) + if index.key.endswith("_int"): + value = int(index.value) + else: + value = index.value + rie = (index.key, value) indexes.append(rie) if len(indexes) > 0: metadata[MD_INDEX] = indexes @@ -155,7 +158,7 @@ def encode_content(self, metadata, data, rpb_content): for field, value in v: pair = rpb_content.indexes.add() pair.key = field - pair.value = value + pair.value = str(value) elif k == MD_LINKS: for link in v: pb_link = rpb_content.links.add() diff --git a/riak/util.py b/riak/util.py index 3c5bf195..0c83a7b1 100644 --- a/riak/util.py +++ b/riak/util.py @@ -19,8 +19,6 @@ import warnings from collections import Mapping -RiakIndexEntry = lambda field, value: (field, str(value)) - def quacks_like_dict(object): """Check if object is dict-like""" return isinstance(object, Mapping) From bafd02193b9a8d9adaee01ca3f336fb617852a6f Mon Sep 17 00:00:00 2001 From: Shuhao Date: Fri, 1 Feb 2013 13:35:08 -0500 Subject: [PATCH 0316/1060] PEP8 fixes.. Just fixed the one that I encountered. --- riak/transports/http/transport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 962fd7f6..bcdc6c09 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -323,8 +323,8 @@ def get_index(self, bucket, index, startkey, endkey=None): response = self._request('GET', url) headers, data = response self.check_http_code(response, [200]) - jsonData = json.loads(data) - return jsonData[u'keys'][:] + json_data = json.loads(data) + return json_data[u'keys'][:] def search(self, index, query, **params): """ From 71a1b6843af08d987af6ca5ab6ba41671e32953f Mon Sep 17 00:00:00 2001 From: Shuhao Date: Fri, 1 Feb 2013 13:40:36 -0500 Subject: [PATCH 0317/1060] Got rid of apply func calls Got rid of deprecated calls and used modern approach. Also fixed what seems like a type in `RiakObject.reduce`, it should be `*args` rather than just `param` --- riak/mapreduce.py | 12 ++++++------ riak/riak_object.py | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 6031eeb2..5a3fd67c 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -769,7 +769,7 @@ def add(self, *args): :rtype: :class:`RiakMapReduce` """ mr = RiakMapReduce(self) - return apply(mr.add, args) + return mr.add(*args) def search(self, *args): """ @@ -781,7 +781,7 @@ def search(self, *args): :rtype: :class:`RiakMapReduce` """ mr = RiakMapReduce(self) - return apply(mr.search, args) + return mr.search(*args) def index(self, *args): """ @@ -791,7 +791,7 @@ def index(self, *args): :rtype: :class:`RiakMapReduce` """ mr = RiakMapReduce(self) - return apply(mr.index, args) + return mr.index(*args) def link(self, *args): """ @@ -801,7 +801,7 @@ def link(self, *args): :rtype: :class:`RiakMapReduce` """ mr = RiakMapReduce(self) - return apply(mr.link, args) + return mr.link(*args) def map(self, *args): """ @@ -811,7 +811,7 @@ def map(self, *args): :rtype: :class:`RiakMapReduce` """ mr = RiakMapReduce(self) - return apply(mr.map, args) + return mr.map(*args) def reduce(self, *args): """ @@ -821,7 +821,7 @@ def reduce(self, *args): :rtype: :class:`RiakMapReduce` """ mr = RiakMapReduce(self) - return apply(mr.reduce, args) + return mr.reduce(*args) from riak.riak_object import RiakObject from riak.bucket import RiakBucket diff --git a/riak/riak_object.py b/riak/riak_object.py index 2ff1552c..e8366f53 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -573,7 +573,7 @@ def add(self, *args): """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) - return apply(mr.add, args) + return mr.add(*args) def link(self, *args): """ @@ -584,7 +584,7 @@ def link(self, *args): """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) - return apply(mr.link, args) + return mr.link(*args) def map(self, *args): """ @@ -595,9 +595,9 @@ def map(self, *args): """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) - return apply(mr.map, args) + return mr.map(*args) - def reduce(self, params): + def reduce(self, *args): """ Start assembling a Map/Reduce operation. A shortcut for :func:`RiakMapReduce.reduce`. @@ -606,4 +606,4 @@ def reduce(self, params): """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) - return apply(mr.reduce, params) + return mr.reduce(*args) From 4d758138ed9a806dc3a7b633f83cc2e2c59d3aa4 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Wed, 6 Feb 2013 00:59:15 -0500 Subject: [PATCH 0318/1060] Fixed a typo with encoding parsing Seems like a typo with encoding parsing. Fixed. --- riak/transports/http/transport.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index bcdc6c09..d262b0c4 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -39,6 +39,7 @@ from riak.metadata import ( MD_CHARSET, MD_CTYPE, + MD_ENCODING, MD_INDEX, MD_LASTMOD, MD_LINKS, @@ -447,7 +448,7 @@ def parse_body(self, response, expected_statuses): elif header == 'charset': metadata[MD_CHARSET] = value elif header == 'content-encoding': - metadata[MD_CTYPE] = value + metadata[MD_ENCODING] = value elif header == 'etag': metadata[MD_VTAG] = value elif header == 'link': From 50331d6f3fdf4ad83bfdbfec73088e33db708dd4 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Wed, 6 Feb 2013 01:16:35 -0500 Subject: [PATCH 0319/1060] Got rid of to_link_header and changed isEqual `to_link_header` not needed as the http transport has its own function for that. `isEqual` is inconsistent with changes so far. --- riak/mapreduce.py | 23 ++++------------------- riak/riak_object.py | 2 +- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 5a3fd67c..ddafc261 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -685,22 +685,7 @@ def set_tag(self, tag): self._tag = tag return self - def to_link_header(self, client): - """ - Convert this RiakLink object to a link header string. Used - internally. - - :rtype: string - """ - link = '' - link += '; riaktag="' - link += urllib.quote_plus(self.get_tag()) + '"' - return link - - def isEqual(self, link): + def __eq__(self, other): """ Returns True if the links are equal. @@ -708,9 +693,9 @@ def isEqual(self, link): :type link: RiakLink :rtype: boolean """ - return ((self._bucket == link._bucket) and - (self._key == link._key) and - (self.get_tag() == link.get_tag())) + return ((self._bucket == other._bucket) and + (self._key == other._key) and + (self.get_tag() == other.get_tag())) class RiakKeyFilter(object): diff --git a/riak/riak_object.py b/riak/riak_object.py index 39acc4cb..2920b8ef 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -346,7 +346,7 @@ def remove_link(self, obj, tag=None): a = [] links = self.metadata.get(MD_LINKS, []) for link in links: - if not link.isEqual(oldlink): + if not link == oldlink: a.append(link) self.metadata[MD_LINKS] = a From e1b4285b42f4450b356b5ce774e7010d044e1276 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Wed, 6 Feb 2013 01:24:28 -0500 Subject: [PATCH 0320/1060] Removed get_key, get_tag, get_bucket for RiakLink Replaced the methods with just direct access to the attributes. --- riak/mapreduce.py | 76 ++++--------------------------- riak/tests/test_2i.py | 2 +- riak/tests/test_mapreduce.py | 28 ++++++------ riak/transports/http/transport.py | 4 +- riak/transports/pbc/codec.py | 6 +-- 5 files changed, 28 insertions(+), 88 deletions(-) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index ddafc261..11f0b7dc 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -600,9 +600,9 @@ def __init__(self, bucket, key, tag=None): :param tag: the tag :type tag: string """ - self._bucket = bucket - self._key = key - self._tag = tag + self.bucket = bucket + self.key = key + self.tag = tag if tag else bucket self._client = None def get(self, r=None): @@ -613,7 +613,7 @@ def get(self, r=None): :type r: string, integer :rtype: RiakObject """ - return self._client.bucket(self._bucket).get(self._key, r) + return self._client.bucket(self.bucket).get(self.key, r) def get_binary(self, r=None): """ @@ -623,67 +623,7 @@ def get_binary(self, r=None): :type r: string, integer :rtype: RiakObject """ - return self._client.bucket(self._bucket).get_binary(self._key, r) - - def get_bucket(self): - """ - Get the bucket name of this link. - - :rtype: string - """ - return self._bucket - - def set_bucket(self, name): - """ - Set the bucket name of this link. - - :param name: the bucket name - :type name: string - :rtype: RiakLink - """ - self._bucket = name - return self - - def get_key(self): - """ - Get the key of this link. - - :rtype: string - """ - return self._key - - def set_key(self, key): - """ - Set the key of this link. - - :param key: the key - :type key: string - :rtype: RiakLink - """ - self._key = key - return self - - def get_tag(self): - """ - Get the tag of this link. - - :rtype: string - """ - if (self._tag is None): - return self._bucket - else: - return self._tag - - def set_tag(self, tag): - """ - Set the tag of this link. - - :param tag: the tag - :type tag: string - :rtype: RiakLink - """ - self._tag = tag - return self + return self._client.bucket(self.bucket).get_binary(self.key, r) def __eq__(self, other): """ @@ -693,9 +633,9 @@ def __eq__(self, other): :type link: RiakLink :rtype: boolean """ - return ((self._bucket == other._bucket) and - (self._key == other._key) and - (self.get_tag() == other.get_tag())) + return ((self.bucket == other.bucket) and + (self.key == other.key) and + (self.tag == other.tag)) class RiakKeyFilter(object): diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index b2827590..7b247cfb 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -108,7 +108,7 @@ def test_set_indexes(self): foo.set_indexes((('field1_bin', 'test'), ('field2_int', 1337))).store() result = self.client.index('indexbucket', 'field2_int', 1337).run() self.assertEqual(1, len(result)) - self.assertEqual('foo', result[0].get_key()) + self.assertEqual('foo', result[0].key) result = bucket.get_index('field1_bin', 'test') self.assertEqual(1, len(result)) diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index b6e2d1b2..2238bac9 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -17,14 +17,14 @@ def test_store_and_get_links(self): links = obj.get_links() self.assertEqual(len(links), 3) for l in links: - if (l.get_key() == "foo1"): - self.assertEqual(l.get_tag(), "bucket") - elif (l.get_key() == "foo2"): - self.assertEqual(l.get_tag(), "tag") - elif (l.get_key() == "foo3"): - self.assertEqual(l.get_tag(), "tag2!@#%^&*)") + if (l.key == "foo1"): + self.assertEqual(l.tag, "bucket") + elif (l.key == "foo2"): + self.assertEqual(l.tag, "tag") + elif (l.key == "foo3"): + self.assertEqual(l.tag, "tag2!@#%^&*)") else: - self.assertEqual("unknown key", l.get_key()) + self.assertEqual("unknown key", l.key) def test_set_links(self): # Create the object @@ -33,13 +33,13 @@ def test_set_links(self): (bucket.new("foo2"), "tag"), RiakLink("bucket", "foo2", "tag2")]).store() obj = bucket.get("foo") - links = sorted(obj.get_links(), key=lambda x: x.get_key()) + links = sorted(obj.get_links(), key=lambda x: x.key) self.assertEqual(len(links), 3) - self.assertEqual(links[0].get_key(), "foo1") - self.assertEqual(links[1].get_key(), "foo2") - self.assertEqual(links[1].get_tag(), "tag") - self.assertEqual(links[2].get_key(), "foo2") - self.assertEqual(links[2].get_tag(), "tag2") + self.assertEqual(links[0].key, "foo1") + self.assertEqual(links[1].key, "foo2") + self.assertEqual(links[1].tag, "tag") + self.assertEqual(links[2].key, "foo2") + self.assertEqual(links[2].tag, "tag2") def test_set_links_all_links(self): bucket = self.client.bucket("bucket") @@ -49,7 +49,7 @@ def test_set_links_all_links(self): foo1.set_links(links, True) links = foo1.get_links() self.assertEqual(len(links), 1) - self.assertEqual(links[0].get_key(), "foo2") + self.assertEqual(links[0].key, "foo2") def test_link_walking(self): # Create the object... diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index bcdc6c09..d085bacb 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -479,8 +479,8 @@ def to_link_header(self, link): """ Convert this RiakLink object to a link header string. Used internally. """ - url = self.object_path(link.get_bucket(), link.get_key()) - header = '<%s>; riaktag="%s"' % (url, link.get_tag()) + url = self.object_path(link.bucket, link.key) + header = '<%s>; riaktag="%s"' % (url, link.tag) return header def parse_links(self, links, linkHeaders): diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index f829f5b6..526192d5 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -162,7 +162,7 @@ def encode_content(self, metadata, data, rpb_content): elif k == MD_LINKS: for link in v: pb_link = rpb_content.links.add() - pb_link.bucket = link.get_bucket() - pb_link.key = link.get_key() - pb_link.tag = link.get_tag() + pb_link.bucket = link.bucket + pb_link.key = link.key + pb_link.tag = link.tag rpb_content.value = str(data) From 9f7da875a8e22896e7067d052c5e0e18f32ee8c9 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Wed, 6 Feb 2013 01:38:16 -0500 Subject: [PATCH 0321/1060] Same typo in PBC client as well --- riak/transports/pbc/codec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index f829f5b6..8403bb08 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -148,7 +148,7 @@ def encode_content(self, metadata, data, rpb_content): elif k == MD_CHARSET: rpb_content.charset = v elif k == MD_ENCODING: - rpb_content.charset = v + rpb_content.content_encoding = v elif k == MD_USERMETA: for uk in v: pair = rpb_content.usermeta.add() From f50619b0a90debb8b73db44a96b4edad3c74a6e2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 6 Feb 2013 09:23:49 -0600 Subject: [PATCH 0322/1060] Re-add exports to the top-level. Closes #193. --- riak/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/riak/__init__.py b/riak/__init__.py index 2296f2a1..09d774f1 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -30,6 +30,9 @@ @author Jay Baird (@skatterbean) (jay@mochimedia.com) """ +__all__ = ['RiakClient', 'RiakBucket', 'RiakNode', 'RiakObject', + 'RiakMapReduce', 'RiakKeyFilter', 'RiakError', 'ONE', + 'ALL', 'QUORUM', 'key_filter'] class RiakError(Exception): """ @@ -41,7 +44,11 @@ def __init__(self, value): def __str__(self): return repr(self.value) -from mapreduce import RiakKeyFilter +from client import RiakClient +from bucket import RiakBucket +from node import RiakNode +from riak_object import RiakObject +from mapreduce import RiakKeyFilter, RiakMapReduce ONE = "one" ALL = "all" From aae3f54ef08f6f3110d68dcbde77fd7be939e855 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 6 Feb 2013 10:48:55 -0600 Subject: [PATCH 0323/1060] Add RiakLink to top-level exports and fix pep8 problems. --- riak/__init__.py | 7 ++++--- riak/riak_object.py | 5 +++-- riak/util.py | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index 09d774f1..b8e72bd1 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -31,8 +31,9 @@ """ __all__ = ['RiakClient', 'RiakBucket', 'RiakNode', 'RiakObject', - 'RiakMapReduce', 'RiakKeyFilter', 'RiakError', 'ONE', - 'ALL', 'QUORUM', 'key_filter'] + 'RiakMapReduce', 'RiakKeyFilter', 'RiakLink', 'RiakError', + 'ONE', 'ALL', 'QUORUM', 'key_filter'] + class RiakError(Exception): """ @@ -48,7 +49,7 @@ def __str__(self): from bucket import RiakBucket from node import RiakNode from riak_object import RiakObject -from mapreduce import RiakKeyFilter, RiakMapReduce +from mapreduce import RiakKeyFilter, RiakMapReduce, RiakLink ONE = "one" ALL = "all" diff --git a/riak/riak_object.py b/riak/riak_object.py index 2920b8ef..275d289a 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -30,6 +30,7 @@ ) from riak import RiakError + class RiakObject(object): """ The RiakObject holds meta information about a Riak object, plus the @@ -209,7 +210,6 @@ def remove_index(self, field=None, value=None): raise RiakError("Cannot pass value without a field" " name while removing index") - # This removes the index entries that's in the ries list. # Done because this is preferred over metadata[MD_INDEX].remove(rie) self.metadata[MD_INDEX] = [rie for rie in self.metadata[MD_INDEX] @@ -224,7 +224,8 @@ def set_indexes(self, indexes): iterable of 2 item tuples, (field, value). :param indexes: iterable of 2 item tuples consisting the field - and value. Both the field and the value must be a string. + and value. Both the field and the value must + be a string. :rtype: RiakObject """ # makes a copy and does type conversion diff --git a/riak/util.py b/riak/util.py index 0c83a7b1..c590db53 100644 --- a/riak/util.py +++ b/riak/util.py @@ -19,6 +19,7 @@ import warnings from collections import Mapping + def quacks_like_dict(object): """Check if object is dict-like""" return isinstance(object, Mapping) From 5a55f13fb2152e42413dda7789852c4a36c75d5a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 6 Feb 2013 10:57:04 -0600 Subject: [PATCH 0324/1060] Fix pyflakes warning in util.py about redefinition of 'getter' method. --- riak/util.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/riak/util.py b/riak/util.py index 0c83a7b1..cae80a51 100644 --- a/riak/util.py +++ b/riak/util.py @@ -79,14 +79,15 @@ def __deprecateQuorumAccessor(klass, parent, quorum): getter_name = "get_%s" % quorum setter_name = "set_%s" % quorum if not parent: - def getter(self, val=None): + def direct_getter(self, val=None): deprecated(QDEPMESSAGE % klass.__name__) if val: return val return getattr(self, propname, "default") + getter = direct_getter else: - def getter(self, val=None): + def parent_getter(self, val=None): deprecated(QDEPMESSAGE % klass.__name__) if val: return val @@ -94,6 +95,8 @@ def getter(self, val=None): return getattr(self, propname, getattr(parentInstance, propname, "default")) + getter = parent_getter + def setter(self, value): deprecated(QDEPMESSAGE % klass.__name__) setattr(self, propname, value) From 8673462a9fd31b0e1a49257457017104329cc76c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 6 Feb 2013 17:30:41 -0600 Subject: [PATCH 0325/1060] Use UserWarning since this is not Python stdlib code. --- riak/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/util.py b/riak/util.py index c590db53..387979e4 100644 --- a/riak/util.py +++ b/riak/util.py @@ -55,7 +55,7 @@ def deep_merge(a, b): def deprecated(message, stacklevel=3): - warnings.warn(message, DeprecationWarning, stacklevel=stacklevel) + warnings.warn(message, UserWarning, stacklevel=stacklevel) QUORUMS = ['r', 'pr', 'w', 'dw', 'pw', 'rw'] QDEPMESSAGE = """ From 856634c14e6314a08f7ad193fecdd0bb5a4fae68 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 6 Feb 2013 17:55:37 -0600 Subject: [PATCH 0326/1060] Don't silently drop the port option, instead use it for the default protocol port. Closes #194. --- riak/client/__init__.py | 12 +++++++++++- riak/node.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 9425bdba..36d89729 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -63,9 +63,19 @@ def __init__(self, protocol='http', transport_options={}, the transport constuctor :type transport_options: dict """ + unused_args = unused_args.copy() + if 'port' in unused_args: deprecated("port option is deprecated, use http_port or pb_port," - + " or the nodes option") + " or the nodes option. Your given port of %r will be " + "used as the %s port unless already set" % + (unused_args['port'], protocol)) + unused_args['already_warned_port'] = True + if (protocol in ['http', 'https'] and + 'http_port' not in unused_args): + unused_args['http_port'] = unused_args['port'] + elif protocol == 'pbc' and 'pb_port' not in unused_args: + unused_args['pb_port'] = unused_args['port'] if 'transport_class' in unused_args: deprecated( diff --git a/riak/node.py b/riak/node.py index 8113d780..08268fd9 100644 --- a/riak/node.py +++ b/riak/node.py @@ -90,7 +90,7 @@ def __init__(self, host='127.0.0.1', http_port=8098, pb_port=8087, :type pb_port: integer """ - if 'port' in unused_args: + if 'port' in unused_args and not 'already_warned_port' in unused_args: deprecated("port option is deprecated, use http_port or pb_port") self.host = host From 433f08f9eab46fcbdb045b59073a0a110a2e7d04 Mon Sep 17 00:00:00 2001 From: evan Date: Wed, 6 Feb 2013 16:17:48 -0800 Subject: [PATCH 0327/1060] convert the last few buckets --- riak/tests/test_2i.py | 2 +- riak/tests/test_mapreduce.py | 31 ++++++++++++++++--------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 700af1f1..3112dd2c 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -228,7 +228,7 @@ def test_secondary_index_invalid_name(self): if not self.is_2i_supported(): return True - bucket = self.client.bucket('indexbucket') + bucket = self.client.bucket(self.bucket_name) with self.assertRaises(RiakError): bucket.new('k', 'a').add_index('field1', 'value1') diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 74db79b8..3374c9ae 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -251,12 +251,12 @@ def test_map_reduce_from_object(self): self.assertEqual(result, [2]) def test_mr_list_add(self): - bucket = self.client.bucket("abucket") + bucket = self.client.bucket(self.bucket_name) for x in range(20): bucket.new('baz' + str(x), 'bazval' + str(x)).store() - mr = self.client.add('abucket', ['baz' + str(x) - for x in range(2, 5)]) + mr = self.client.add(self.bucket_name, ['baz' + str(x) + for x in range(2, 5)]) results = mr.map_values().run() results.sort() self.assertEqual(results, @@ -265,19 +265,20 @@ def test_mr_list_add(self): u'"bazval4"']) def test_mr_list_add_two_buckets(self): - bucket = self.client.bucket("bucket_a") + bucket = self.client.bucket(self.bucket_name) + name2 = self.randname() for x in range(10): bucket.new('foo' + str(x), 'fooval' + str(x)).store() - bucket = self.client.bucket("bucket_b") + bucket = self.client.bucket(name2) for x in range(10): bucket.new('bar' + str(x), 'barval' + str(x)).store() - mr = self.client.add('bucket_a', ['foo' + str(x) - for x in range(2, 4)]) - mr.add('bucket_b', ['bar' + str(x) - for x in range(5, 7)]) + mr = self.client.add(self.bucket_name, ['foo' + str(x) + for x in range(2, 4)]) + mr.add(name2, ['bar' + str(x) + for x in range(5, 7)]) results = mr.map_values().run() results.sort() @@ -500,12 +501,12 @@ def test_filter_not_found(self): class MapReduceStreamTests(object): def test_stream_results(self): - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() bucket.new('two', data=2).store() - mr = RiakMapReduce(self.client).add('bucket', 'one')\ - .add('bucket', 'two') + mr = RiakMapReduce(self.client).add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two') mr.map_values_json() results = [] for phase, data in mr.stream(): @@ -514,12 +515,12 @@ def test_stream_results(self): self.assertEqual(sorted(results), [1, 2]) def test_stream_cleanoperationsup(self): - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() bucket.new('two', data=2).store() - mr = RiakMapReduce(self.client).add('bucket', 'one')\ - .add('bucket', 'two') + mr = RiakMapReduce(self.client).add(self.bucket_name, 'one')\ + .add(self.bucket_name, 'two') mr.map_values_json() try: for phase, data in mr.stream(): From 570748c93384048cd7db9611b3495f1a4a337797 Mon Sep 17 00:00:00 2001 From: evan Date: Thu, 7 Feb 2013 10:49:54 -0800 Subject: [PATCH 0328/1060] add bucket property clearing. --- riak/bucket.py | 7 +++++++ riak/client/operations.py | 10 ++++++++++ riak/tests/test_all.py | 13 +++++++++++++ riak/transports/http/transport.py | 20 ++++++++++++++++++++ riak/transports/transport.py | 7 +++++++ 5 files changed, 57 insertions(+) diff --git a/riak/bucket.py b/riak/bucket.py index 193e70d6..eab085df 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -344,6 +344,13 @@ def get_properties(self): """ return self._client.get_bucket_props(self) + def clear_properties(self): + """ + Reset all bucket properties to their defaults. + + """ + return self._client.clear_bucket_props(self) + def get_keys(self): """ Return all keys within the bucket. diff --git a/riak/client/operations.py b/riak/client/operations.py index e28c78e3..9ce91dc2 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -89,6 +89,16 @@ def set_bucket_props(self, transport, bucket, props): """ return transport.set_bucket_props(bucket, props) + @retryable + def clear_bucket_props(self, transport, bucket): + """ + Resets bucket properties for the given bucket. + + :param bucket: the bucket whose properties will be set + :type bucket: RiakBucket + """ + return transport.clear_bucket_props(bucket) + @retryable def get_keys(self, transport, bucket): """ diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 9433d41f..98fdebee 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -155,6 +155,19 @@ def test_too_many_link_headers_shouldnt_break_http(self): stored_object = bucket.get("lots_of_links") self.assertEqual(len(stored_object.get_links()), 400) + def test_clear_bucket_properties(self): + bucket = self.client.bucket('bucket') + # Test setting allow mult... + bucket.allow_mult = True + self.assertTrue(bucket.allow_mult) + # Test setting nval... + bucket.n_val = 1 + self.assertEqual(bucket.n_val, 1) + # Test setting multiple properties... + bucket.clear_properties() + self.assertFalse(bucket.allow_mult) + self.assertEqual(bucket.n_val, 3) + class FilterTests(unittest.TestCase): def test_simple(self): diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 8be9c691..7bd1195a 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -279,6 +279,26 @@ def set_bucket_props(self, bucket, props): raise Exception('Error setting bucket properties.') return True + def clear_bucket_props(self, bucket): + """ + reset the properties on the bucket object given + """ + url = self.bucket_properties_path(bucket.name) + headers = {'Content-Type': 'application/json'} + + # Run the request... + response = self._request('DELETE', url, headers, None) + + # Handle the response... + if response is None: + raise Exception('Error clearing bucket properties.') + + # Check the response value... + status = response[0]['http_code'] + if status != 204: + raise Exception('Error %s clearing bucket properties.' + % status) + def mapred(self, inputs, query, timeout=None): """ Run a MapReduce query. diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 27d1fc52..2209de50 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -120,6 +120,13 @@ def set_bucket_props(self, bucket, props): """ raise NotImplementedError + def clear_bucket_props(self, bucket): + """ + Reset bucket properties to their defaults + bucket = bucket object + """ + raise NotImplementedError + def get_keys(self, bucket): """ Lists all keys within the given bucket. From fdca94feb7990d54c1fd4a5efa15b8fe37a3d792 Mon Sep 17 00:00:00 2001 From: evan Date: Thu, 7 Feb 2013 11:09:47 -0800 Subject: [PATCH 0329/1060] updates for comments. --- riak/client/operations.py | 2 +- riak/tests/test_all.py | 7 +++---- riak/transports/http/transport.py | 6 +++++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 9ce91dc2..3bde1571 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -89,7 +89,7 @@ def set_bucket_props(self, transport, bucket, props): """ return transport.set_bucket_props(bucket, props) - @retryable + @retryableHttpOnly def clear_bucket_props(self, transport, bucket): """ Resets bucket properties for the given bucket. diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 98fdebee..85f22d9d 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -157,14 +157,13 @@ def test_too_many_link_headers_shouldnt_break_http(self): def test_clear_bucket_properties(self): bucket = self.client.bucket('bucket') - # Test setting allow mult... bucket.allow_mult = True self.assertTrue(bucket.allow_mult) - # Test setting nval... bucket.n_val = 1 self.assertEqual(bucket.n_val, 1) - # Test setting multiple properties... - bucket.clear_properties() + # Test setting clearing properties... + + self.assertTrue(bucket.clear_properties()) self.assertFalse(bucket.allow_mult) self.assertEqual(bucket.n_val, 3) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 7bd1195a..d9f7d1f3 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -295,7 +295,11 @@ def clear_bucket_props(self, bucket): # Check the response value... status = response[0]['http_code'] - if status != 204: + if status == 204: + return True + elif status == 405: + return False + else: raise Exception('Error %s clearing bucket properties.' % status) From 6797fbfff915933b57dad1d877aaee6a79f18350 Mon Sep 17 00:00:00 2001 From: evan Date: Wed, 13 Feb 2013 14:04:51 -0800 Subject: [PATCH 0330/1060] clearing all properties now, some additional enhancements --- riak/client/operations.py | 6 +++--- riak/tests/test_all.py | 41 +++++++++++++++++++++++++++++---------- riak/tests/test_kv.py | 36 +++++++++++++++++++++------------- riak/tests/test_search.py | 14 ++++++++----- 4 files changed, 65 insertions(+), 32 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 3bde1571..047cc3b9 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -89,15 +89,15 @@ def set_bucket_props(self, transport, bucket, props): """ return transport.set_bucket_props(bucket, props) - @retryableHttpOnly - def clear_bucket_props(self, transport, bucket): + def clear_bucket_props(self, bucket): """ Resets bucket properties for the given bucket. :param bucket: the bucket whose properties will be set :type bucket: RiakBucket """ - return transport.clear_bucket_props(bucket) + with self._transport() as transport: + return transport.clear_bucket_props(bucket) @retryable def get_keys(self, transport, bucket): diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 762f2110..ab7620dc 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -49,7 +49,33 @@ test_server.start() testrun_search_bucket = None - +testrun_props_bucket = None +testrun_sibs_bucket = None + +def setUpModule(): + global testrun_search_bucket, testrun_props_bucket, \ + testrun_sibs_bucket + + c = RiakClient(transport='http', http_port=HTTP_PORT) + + testrun_props_bucket = 'propsbucket' + testrun_sibs_bucket = 'sibsbucket' + c.bucket(testrun_sibs_bucket).allow_mult = True + + if not int(os.environ.get('SKIP_SEARCH', '0')): + testrun_search_bucket = 'searchbucket' + b = c.bucket(testrun_search_bucket) + b.enable_search() + +def tearDownModule(): + c = RiakClient(transport='http', http_port=HTTP_PORT) + if not int(os.environ.get('SKIP_SEARCH', '0')): + b = c.bucket(testrun_search_bucket) + b.clear_properties() + b = c.bucket(testrun_sibs_bucket) + b.clear_properties() + b = c.bucket(testrun_props_bucket) + b.clear_properties() class BaseTestCase(object): @@ -80,16 +106,11 @@ def create_client(self, host=None, http_port=None, pb_port=None, pb_port=pb_port, **client_args) def setUp(self): - global testrun_search_bucket self.bucket_name = self.randname() self.key_name = self.randname() - if not testrun_search_bucket: - self.search_bucket = testrun_search_bucket = self.randname() - c = self.create_client(HTTP_HOST, http_port=HTTP_PORT) - b = c.bucket(self.search_bucket) - b.enable_search() - else: - self.search_bucket = testrun_search_bucket + self.search_bucket = testrun_search_bucket + self.sibs_bucket = testrun_sibs_bucket + self.props_bucket = testrun_props_bucket self.client = self.create_client() @@ -171,7 +192,7 @@ def test_too_many_link_headers_shouldnt_break_http(self): self.assertEqual(len(stored_object.get_links()), 400) def test_clear_bucket_properties(self): - bucket = self.client.bucket('bucket') + bucket = self.client.bucket(self.props_bucket) bucket.allow_mult = True self.assertTrue(bucket.allow_mult) bucket.n_val = 1 diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 549a9dcb..579ab2c8 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -2,6 +2,7 @@ import os import cPickle import copy + try: import simplejson as json except ImportError: @@ -168,18 +169,21 @@ def test_delete(self): self.assertFalse(obj.exists) def test_set_bucket_properties(self): - bucket = self.client.bucket(self.bucket_name) + bucket = self.client.bucket(self.props_bucket) # Test setting allow mult... bucket.allow_mult = True - self.assertTrue(bucket.allow_mult) # Test setting nval... - bucket.n_val = 3 - self.assertEqual(bucket.n_val, 3) + bucket.n_val = 1 + + bucket2 = self.create_client().bucket(self.props_bucket) + self.assertTrue(bucket2.allow_mult) + self.assertEqual(bucket2.n_val, 1) # Test setting multiple properties... bucket.set_properties({"allow_mult": False, "n_val": 2}) - self.assertFalse(bucket.allow_mult) - self.assertEqual(bucket.n_val, 2) + bucket3 = self.create_client().bucket(self.props_bucket) + self.assertFalse(bucket3.allow_mult) + self.assertEqual(bucket3.n_val, 2) def test_if_none_match(self): bucket = self.client.bucket(self.bucket_name) @@ -197,9 +201,9 @@ def test_if_none_match(self): def test_siblings(self): # Set up the bucket, clear any existing object... - bucket = self.client.bucket(self.bucket_name) - bucket.allow_mult = True + bucket = self.client.bucket(self.sibs_bucket) obj = bucket.get_binary(self.key_name) + bucket.allow_mult = True # Even if it previously existed, let's store a base resolved version # from which we can diverge by sending a stale vclock. @@ -210,7 +214,7 @@ def test_siblings(self): vals = set() for i in range(5): other_client = self.create_client() - other_bucket = other_client.bucket(self.bucket_name) + other_bucket = other_client.bucket(self.sibs_bucket) while True: randval = self.randint() if randval not in vals: @@ -223,7 +227,7 @@ def test_siblings(self): # Make sure the object has itself plus four siblings... obj.reload() - self.assertTrue(bool(obj.siblings)) + #self.assertTrue(bool(obj.siblings)) self.assertEqual(len(obj.siblings), 5) # Get each of the values - make sure they match what was assigned @@ -279,7 +283,7 @@ def test_list_buckets(self): class HTTPBucketPropsTest(object): def test_rw_settings(self): - bucket = self.client.bucket(self.bucket_name) + bucket = self.client.bucket(self.props_bucket) self.assertEqual(bucket.r, "quorum") self.assertEqual(bucket.w, "quorum") self.assertEqual(bucket.dw, "quorum") @@ -301,9 +305,10 @@ def test_rw_settings(self): 'r': 'quorum', 'dw': 'quorum', 'rw': 'quorum'}) + bucket.clear_properties() def test_primary_quora(self): - bucket = self.client.bucket(self.bucket_name) + bucket = self.client.bucket(self.props_bucket) self.assertEqual(bucket.pr, 0) self.assertEqual(bucket.pw, 0) @@ -314,11 +319,12 @@ def test_primary_quora(self): self.assertEqual(bucket.pw, "quorum") bucket.set_properties({'pr': 0, 'pw': 0}) + bucket.clear_properties() class PbcBucketPropsTest(object): def test_rw_settings(self): - bucket = self.client.bucket('rwsettings') + bucket = self.client.bucket(self.props_bucket) with self.assertRaises(NotImplementedError): bucket.r with self.assertRaises(NotImplementedError): @@ -336,9 +342,11 @@ def test_rw_settings(self): bucket.dw = 2 with self.assertRaises(NotImplementedError): bucket.rw = 2 + with self.assertRaises(NotImplementedError): + bucket.clear_properties() def test_primary_quora(self): - bucket = self.client.bucket('primary_quora') + bucket = self.client.bucket(self.props_bucket) with self.assertRaises(NotImplementedError): bucket.pr with self.assertRaises(NotImplementedError): diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index b57c213b..9316b10d 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -17,19 +17,23 @@ def test_bucket_search_enabled(self): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_enable_search_commit_hook(self): - bucket = self.client.bucket(self.bucket_name) + bucket = self.client.bucket(self.search_bucket) + bucket.clear_properties() + self.assertFalse(self.create_client().bucket(self.search_bucket).search_enabled()) bucket.enable_search() - self.assertTrue(self.client.bucket(self.bucket_name).search_enabled()) + self.assertTrue(self.create_client().bucket(self.search_bucket).search_enabled()) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_disable_search_commit_hook(self): - bucket = self.client.bucket(self.bucket_name) + bucket = self.client.bucket(self.search_bucket) + bucket.clear_properties() bucket.enable_search() - self.assertTrue(self.client.bucket(self.bucket_name)\ + self.assertTrue(self.create_client().bucket(self.search_bucket)\ .search_enabled()) bucket.disable_search() - self.assertFalse(self.client.bucket(self.bucket_name)\ + self.assertFalse(self.create_client().bucket(self.search_bucket)\ .search_enabled()) + bucket.enable_search() class SolrSearchTests(object): From 39b8128dabea6eb201d16b8c766ed225b366df30 Mon Sep 17 00:00:00 2001 From: evan Date: Thu, 14 Feb 2013 12:18:25 -0800 Subject: [PATCH 0331/1060] add 204 to the list of OK return codes when returnbody=True --- riak/transports/http/transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index d9f7d1f3..a4d6e2cc 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -161,7 +161,7 @@ def do_put(self, url, headers, content, return_body=False, key=None): response = self._request('PUT', url, headers, content) if return_body: - return self.parse_body(response, [200, 201, 300]) + return self.parse_body(response, [200, 201, 204, 300]) else: self.check_http_code(response, [204]) return None From 8dd33e29f68a4b8e145ada978720cd66872b1aff Mon Sep 17 00:00:00 2001 From: evan Date: Thu, 14 Feb 2013 15:19:05 -0800 Subject: [PATCH 0332/1060] make clear_bucket props retryable again. --- riak/client/operations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 047cc3b9..9ce91dc2 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -89,15 +89,15 @@ def set_bucket_props(self, transport, bucket, props): """ return transport.set_bucket_props(bucket, props) - def clear_bucket_props(self, bucket): + @retryable + def clear_bucket_props(self, transport, bucket): """ Resets bucket properties for the given bucket. :param bucket: the bucket whose properties will be set :type bucket: RiakBucket """ - with self._transport() as transport: - return transport.clear_bucket_props(bucket) + return transport.clear_bucket_props(bucket) @retryable def get_keys(self, transport, bucket): From a9fdf111d6c9d6d95f8f18c3ae66b1747981aab8 Mon Sep 17 00:00:00 2001 From: "Anton (Atilla) Tsigularov" Date: Fri, 15 Feb 2013 14:26:06 +0100 Subject: [PATCH 0333/1060] Switching JSON encoding to UTF-8. By default, python's JSON encoder will escape all non-ASCII characters, using a sequence of 6 bytes, for the sake of legacy non-UTF-8 compatible clients. This is a significant overhead in the case of non-ASCII string and can be reduced by using a modern encoding. --- riak/client/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 36d89729..93c3c71a 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -38,6 +38,14 @@ from riak.util import lazy_property +def default_encoder(obj): + """ + Default encoder for JSON datatypes, which returns UTF-8 encoded + json instead of the default bloated \uXXXX escaped ASCII strings. + """ + return json.dumps(obj, ensure_ascii=False).encode("utf-8") + + @deprecateQuorumAccessors class RiakClient(RiakMapReduceChain, RiakClientOperations): """ @@ -91,8 +99,8 @@ def __init__(self, protocol='http', transport_options={}, self._http_pool = RiakHttpPool(self, **transport_options) self._pb_pool = RiakPbcPool(self, **transport_options) - self._encoders = {'application/json': json.dumps, - 'text/json': json.dumps} + self._encoders = {'application/json': default_encoder, + 'text/json': default_encoder} self._decoders = {'application/json': json.loads, 'text/json': json.loads} self._buckets = WeakValueDictionary() From e4d4d8aea654d640fd1974f9eee935b6b7b25d1b Mon Sep 17 00:00:00 2001 From: evan Date: Fri, 15 Feb 2013 09:47:21 -0800 Subject: [PATCH 0334/1060] add 204 test --- riak/tests/test_kv.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 6e4981f4..dc8d412e 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -116,6 +116,16 @@ def test_binary_store_and_get(self): obj.store() obj = bucket.get_binary('foo2') self.assertEqual(data, json.loads(obj.data)) + + def test_blank_binary_204(self): + bucket = self.client.bucket(self.bucket_name) + + # this should *not* raise an error + obj = bucket.new_binary('foo2', '') + obj.store() + obj = bucket.get_binary('foo2') + self.assertTrue(obj.exists) + self.assertEqual(obj.data, '') def test_custom_bucket_encoder_decoder(self): # Teach the bucket how to pickle From 89131239ea63c94a25926420bc2b6686c3cf3e29 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Mon, 18 Feb 2013 10:53:25 -0500 Subject: [PATCH 0335/1060] Fixed typo for __ne__ We could probably also refactor `__ne__` to call `__eq__` --- riak/riak_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 275d289a..6f110068 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -74,7 +74,7 @@ def __eq__(self, other): else: return False - def __nq__(self, other): + def __ne__(self, other): if isinstance(other, self.__class__): return hash(self) != hash(other) else: From 3481c79d5d87ddeca217b18ce14c62cec2fdd8b8 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Wed, 13 Feb 2013 01:08:03 -0500 Subject: [PATCH 0336/1060] Getting rid of RiakLink in favour of tuples Please review this carefully! There are some potential issues that we need to address. Before that, some points needs to be noted: 1. `RiakLink.get` was actually never tested in code. 2. The new interface should be easier to use in a lot of scenarios where you are iterating through links (you can do `for bucket, key, tag in obj.get_links()`), it maybe less clear than a class. IMO this is not the case as a 3-item tuple is sufficient, but if the `RiakLink.get` method is actually used a lot in the wild, it might be benificial to include some sort of wrapper function. A potential issue that I identified: I've noted that `client.index` doesn't actually perform the Riak 2i operation as specified in the HTTP and PBC protocal and does a map reduce operation instead. Right now an index operation actually seems to return links, which seems weird (see line 110 in riak/test/test_2i.py) as it seems like the Riak API just returns keys and this is returning `(bucket, key, None)` --- riak/__init__.py | 4 +- riak/bucket.py | 3 +- riak/mapreduce.py | 66 +++---------------------------- riak/riak_object.py | 55 ++++++++++++-------------- riak/tests/test_2i.py | 2 +- riak/tests/test_all.py | 4 +- riak/tests/test_mapreduce.py | 36 ++++++++--------- riak/transports/http/transport.py | 15 +++---- riak/transports/pbc/codec.py | 12 +++--- 9 files changed, 70 insertions(+), 127 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index b8e72bd1..ff7b632a 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -31,7 +31,7 @@ """ __all__ = ['RiakClient', 'RiakBucket', 'RiakNode', 'RiakObject', - 'RiakMapReduce', 'RiakKeyFilter', 'RiakLink', 'RiakError', + 'RiakMapReduce', 'RiakKeyFilter', 'RiakError', 'ONE', 'ALL', 'QUORUM', 'key_filter'] @@ -49,7 +49,7 @@ def __str__(self): from bucket import RiakBucket from node import RiakNode from riak_object import RiakObject -from mapreduce import RiakKeyFilter, RiakMapReduce, RiakLink +from mapreduce import RiakKeyFilter, RiakMapReduce ONE = "one" ALL = "all" diff --git a/riak/bucket.py b/riak/bucket.py index eab085df..1f10832c 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -17,7 +17,6 @@ specific language governing permissions and limitations under the License. """ -from riak_object import RiakObject import mimetypes from riak.util import deprecateQuorumAccessors @@ -433,3 +432,5 @@ def get_index(self, index, startkey, endkey=None): def __str__(self): return ''.format(self.name) + +from riak_object import RiakObject diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 11f0b7dc..a0a36370 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -17,7 +17,7 @@ specific language governing permissions and limitations under the License. """ -import urllib + from collections import Iterable @@ -262,7 +262,7 @@ def reduce(self, function, options=None): def run(self, timeout=None): """ Run the map/reduce operation synchronously. Returns a list of - results, or a list of RiakLink objects if the last phase is a + results, or a list of links if the last phase is a link phase. :param timeout: Timeout in milliseconds @@ -283,14 +283,13 @@ def run(self, timeout=None): return [] # Otherwise, if the last phase IS a link phase, then convert the - # results to RiakLink objects. + # results to link tuples. a = [] for r in result: if (len(r) == 2): - link = RiakLink(r[0], r[1]) + link = (r[0], r[1], None) elif (len(r) == 3): - link = RiakLink(r[0], r[1], r[2]) - link._client = self._client + link = (r[0], r[1], r[2]) a.append(link) return a @@ -583,61 +582,6 @@ def to_array(self): return {'link': stepdef} -class RiakLink(object): - """ - The RiakLink object represents a link from one Riak object to - another. - """ - - def __init__(self, bucket, key, tag=None): - """ - Construct a RiakLink object. - - :param bucket: the bucket name - :type bucket: string - :param key: the key - :type key: string - :param tag: the tag - :type tag: string - """ - self.bucket = bucket - self.key = key - self.tag = tag if tag else bucket - self._client = None - - def get(self, r=None): - """ - Retrieve the RiakObject to which this link points. - - :param r: the read quorum to use - :type r: string, integer - :rtype: RiakObject - """ - return self._client.bucket(self.bucket).get(self.key, r) - - def get_binary(self, r=None): - """ - Retrieve the RiakObject to which this link points, as a binary. - - :param r: the read quorum to use - :type r: string, integer - :rtype: RiakObject - """ - return self._client.bucket(self.bucket).get_binary(self.key, r) - - def __eq__(self, other): - """ - Returns True if the links are equal. - - :param link: some other link - :type link: RiakLink - :rtype: boolean - """ - return ((self.bucket == other.bucket) and - (self.key == other.key) and - (self.tag == other.tag)) - - class RiakKeyFilter(object): def __init__(self, *args): if args: diff --git a/riak/riak_object.py b/riak/riak_object.py index 6f110068..18989c3a 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -24,10 +24,6 @@ MD_LINKS, MD_USERMETA ) -from riak.mapreduce import ( - RiakMapReduce, - RiakLink - ) from riak import RiakError @@ -285,10 +281,12 @@ def set_links(self, links, all_link=False): (RiakObject, tag). This could also be an iterable of just a RiakObject, instead of the tuple, then a tag of None would be used. Lastly, it could also be an iterable of - RiakLink. They have tags built-in. + 3 item tuples with the format of (bucket, key, tag), where tag + could be None - :param all_link: A boolean indicates if links are all RiakLink - objects This speeds up the operation. + :param all_link: A boolean indicates if links are all 3 item tuples + objects This speeds up the operation so there is no iterating + through and parsing elements. """ if all_link: self.metadata[MD_LINKS] = links @@ -296,12 +294,14 @@ def set_links(self, links, all_link=False): new_links = [] for item in links: - if isinstance(item, RiakLink): - link = item + if isinstance(item, tuple): + if len(item) == 3: + link = item + elif len(item) == 2: + link = (item[0].bucket.name, item[0].key, item[1]) elif isinstance(item, RiakObject): - link = RiakLink(item.bucket.name, item.key, None) - else: - link = RiakLink(item[0].bucket.name, item[0].key, item[1]) + link = (item.bucket.name, item.key, None) + new_links.append(link) self.metadata[MD_LINKS] = new_links @@ -311,17 +311,18 @@ def add_link(self, obj, tag=None): """ Add a link to a RiakObject. - :param obj: Either a RiakObject or a RiakLink object. + :param obj: Either a RiakObject or 3 item link tuple consisting + of (bucket, key, tag). :type obj: mixed :param tag: Optional link tag. Defaults to bucket name. It is ignored - if ``obj`` is a RiakLink instance. + if ``obj`` is a 3 item link tuple. :type tag: string :rtype: RiakObject """ - if isinstance(obj, RiakLink): + if isinstance(obj, tuple): newlink = obj else: - newlink = RiakLink(obj.bucket.name, obj.key, tag) + newlink = (obj.bucket.name, obj.key, tag) self.remove_link(newlink) links = self.metadata[MD_LINKS] @@ -332,17 +333,18 @@ def remove_link(self, obj, tag=None): """ Remove a link to a RiakObject. - :param obj: Either a RiakObject or a RiakLink object. + :param obj: Either a RiakObject or 3 item link tuple consisting + of (bucket, key, tag). :type obj: mixed :param tag: Optional link tag. Defaults to bucket name. It is ignored - if ``obj`` is a RiakLink instance. + if ``obj`` is a 3 item link tuple. :type tag: string :rtype: RiakObject """ - if isinstance(obj, RiakLink): + if isinstance(obj, tuple): oldlink = obj else: - oldlink = RiakLink(obj.bucket.name, obj.key, tag) + oldlink = (obj.bucket.name, obj.key, tag) a = [] links = self.metadata.get(MD_LINKS, []) @@ -355,18 +357,11 @@ def remove_link(self, obj, tag=None): def get_links(self): """ - Return an array of RiakLink objects. + Return an array of 3 item link tuples. :rtype: list """ - # Set the clients before returning... - if MD_LINKS in self.metadata: - links = self.metadata[MD_LINKS] - for link in links: - link._client = self.client - return links - else: - return [] + return self.metadata.get(MD_LINKS, []) def store(self, w=None, dw=None, pw=None, return_body=True, if_none_match=False): @@ -605,3 +600,5 @@ def reduce(self, *args): mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) return mr.reduce(*args) + +from riak.mapreduce import RiakMapReduce diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 3112dd2c..b5dcfee0 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -108,7 +108,7 @@ def test_set_indexes(self): foo.set_indexes((('field1_bin', 'test'), ('field2_int', 1337))).store() result = self.client.index(self.bucket_name, 'field2_int', 1337).run() self.assertEqual(1, len(result)) - self.assertEqual('foo', result[0].key) + self.assertEqual('foo', result[0][1]) result = bucket.get_index('field1_bin', 'test') self.assertEqual(1, len(result)) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index ab7620dc..13dcd2c6 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -11,7 +11,7 @@ import unittest from riak.client import RiakClient -from riak.mapreduce import RiakLink, RiakKeyFilter +from riak.mapreduce import RiakKeyFilter from riak import key_filter from riak.test_server import TestServer @@ -184,7 +184,7 @@ def test_too_many_link_headers_shouldnt_break_http(self): bucket = self.client.bucket(self.bucket_name) o = bucket.new("lots_of_links", "My god, it's full of links!") for i in range(0, 400): - link = RiakLink("other", "key%d" % i, "next") + link = ("other", "key%d" % i, "next") o.add_link(link) o.store() diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 3374c9ae..8a4bd1bc 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from riak.mapreduce import RiakLink, RiakMapReduce +from riak.mapreduce import RiakMapReduce from riak import key_filter @@ -16,40 +16,40 @@ def test_store_and_get_links(self): obj = bucket.get("test_store_and_get_links") links = obj.get_links() self.assertEqual(len(links), 3) - for l in links: - if (l.key == "foo1"): - self.assertEqual(l.tag, self.bucket_name) - elif (l.key == "foo2"): - self.assertEqual(l.tag, "tag") - elif (l.key == "foo3"): - self.assertEqual(l.tag, "tag2!@#%^&*)") + for bucket, key, tag in links: + if (key == "foo1"): + self.assertEqual(bucket, "bucket") + elif (key == "foo2"): + self.assertEqual(tag, "tag") + elif (key == "foo3"): + self.assertEqual(tag, "tag2!@#%^&*)") else: - self.assertEqual("unknown key", l.key) + self.assertEqual(key, "unknown key") def test_set_links(self): # Create the object bucket = self.client.bucket(self.bucket_name) bucket.new("foo", 2).set_links([bucket.new("foo1"), (bucket.new("foo2"), "tag"), - RiakLink(self.bucket_name, "foo2", "tag2")]).store() + ("bucket", "foo2", "tag2")]).store() obj = bucket.get("foo") - links = sorted(obj.get_links(), key=lambda x: x.key) + links = sorted(obj.get_links(), key=lambda x: x[1]) self.assertEqual(len(links), 3) - self.assertEqual(links[0].key, "foo1") - self.assertEqual(links[1].key, "foo2") - self.assertEqual(links[1].tag, "tag") - self.assertEqual(links[2].key, "foo2") - self.assertEqual(links[2].tag, "tag2") + self.assertEqual(links[0][1], "foo1") + self.assertEqual(links[1][1], "foo2") + self.assertEqual(links[1][2], "tag") + self.assertEqual(links[2][1], "foo2") + self.assertEqual(links[2][2], "tag2") def test_set_links_all_links(self): bucket = self.client.bucket(self.bucket_name) foo1 = bucket.new("foo", 1) bucket.new("foo2", 2).store() - links = [RiakLink(self.bucket_name, "foo2")] + links = [("bucket", "foo2", None)] foo1.set_links(links, True) links = foo1.get_links() self.assertEqual(len(links), 1) - self.assertEqual(links[0].key, "foo2") + self.assertEqual(links[0][1], "foo2") def test_link_walking(self): # Create the object... diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index a4d6e2cc..3c8018ff 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -47,7 +47,6 @@ MD_VTAG, MD_DELETED ) -from riak.mapreduce import RiakLink from riak import RiakError from riak.multidict import MultiDict from xml.etree import ElementTree @@ -502,10 +501,12 @@ def parse_body(self, response, expected_statuses): def to_link_header(self, link): """ - Convert this RiakLink object to a link header string. Used internally. + Convert the link tuple to a link header string. Used internally. """ - url = self.object_path(link.bucket, link.key) - header = '<%s>; riaktag="%s"' % (url, link.tag) + bucket, key, tag = link + tag = tag if tag is not None else bucket + url = self.object_path(bucket, key) + header = '<%s>; riaktag="%s"' % (url, tag) return header def parse_links(self, links, linkHeaders): @@ -520,9 +521,9 @@ def parse_links(self, links, linkHeaders): matches = (re.match(oldform, linkHeader) or re.match(newform, linkHeader)) if matches is not None: - link = RiakLink(urllib.unquote_plus(matches.group(2)), - urllib.unquote_plus(matches.group(3)), - urllib.unquote_plus(matches.group(4))) + link = (urllib.unquote_plus(matches.group(2)), + urllib.unquote_plus(matches.group(3)), + urllib.unquote_plus(matches.group(4))) links.append(link) return self diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index abbf5d9d..2328b2ab 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -29,7 +29,6 @@ ) import riak_pb -from riak.mapreduce import RiakLink RIAKC_RW_ONE = 4294967294 RIAKC_RW_QUORUM = 4294967293 @@ -110,7 +109,7 @@ def decode_content(self, rpb_content): tag = link.tag else: tag = None - links.append(RiakLink(bucket, key, tag)) + links.append((bucket, key, tag)) if links: metadata[MD_LINKS] = links if rpb_content.HasField("last_mod"): @@ -160,9 +159,10 @@ def encode_content(self, metadata, data, rpb_content): pair.key = field pair.value = str(value) elif k == MD_LINKS: - for link in v: + for bucket, key, tag in v: + tag = tag if tag is not None else bucket pb_link = rpb_content.links.add() - pb_link.bucket = link.bucket - pb_link.key = link.key - pb_link.tag = link.tag + pb_link.bucket = bucket + pb_link.key = key + pb_link.tag = tag rpb_content.value = str(data) From f9c0f87c57e3a5bddd36f06658a2b186c41c4f93 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Wed, 13 Feb 2013 12:46:01 -0500 Subject: [PATCH 0337/1060] Added a namedtuple convenience class for links Backend code do not need to change. The user is free to use RiakLink or a 3-item tuple. --- riak/__init__.py | 4 ++-- riak/mapreduce.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index ff7b632a..b8e72bd1 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -31,7 +31,7 @@ """ __all__ = ['RiakClient', 'RiakBucket', 'RiakNode', 'RiakObject', - 'RiakMapReduce', 'RiakKeyFilter', 'RiakError', + 'RiakMapReduce', 'RiakKeyFilter', 'RiakLink', 'RiakError', 'ONE', 'ALL', 'QUORUM', 'key_filter'] @@ -49,7 +49,7 @@ def __str__(self): from bucket import RiakBucket from node import RiakNode from riak_object import RiakObject -from mapreduce import RiakKeyFilter, RiakMapReduce +from mapreduce import RiakKeyFilter, RiakMapReduce, RiakLink ONE = "one" ALL = "all" diff --git a/riak/mapreduce.py b/riak/mapreduce.py index a0a36370..6c627650 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -18,8 +18,9 @@ under the License. """ -from collections import Iterable +from collections import Iterable, namedtuple +RiakLink = namedtuple("RiakLink", ("bucket", "key", "tag")) class RiakMapReduce(object): """ From 2ad677b6c6011ae65534d4c28cfee4a12178878d Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 19 Feb 2013 14:52:12 -0500 Subject: [PATCH 0338/1060] Make requests to Riak Search support non-ASCII text - Encode text as UTF-8 when making query strings - Encode assembled XML as UTF-8 - Responses are in JSON, which natively handles Unicode already --- riak/transports/http/resources.py | 5 ++++- riak/transports/http/transport.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index b551cbae..d46fb414 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -167,7 +167,10 @@ def mkpath(*segments, **query): if query[key] in [False, True]: _query[key] = str(query[key]).lower() elif query[key] is not None: - _query[key] = query[key] + if isinstance(query[key], unicode): + _query[key] = query[key].encode('utf-8') + else: + _query[key] = query[key] if len(_query) > 0: pathstring += "?" + urlencode(_query) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index a4d6e2cc..ea273686 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -396,7 +396,7 @@ def fulltext_add(self, index, docs): self._request('POST', self.solr_update_path(index), {'Content-Type': 'text/xml'}, - xml.toxml()) + xml.toxml().encode('utf-8')) def fulltext_delete(self, index, docs=None, queries=None): """ @@ -421,7 +421,7 @@ def fulltext_delete(self, index, docs=None, queries=None): self._request('POST', self.solr_update_path(index), {'Content-Type': 'text/xml'}, - xml.toxml()) + xml.toxml().encode('utf-8')) def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] From 83bb5f0877f33dda9aae7f8fca74dcbf8dfbdc6a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 19 Feb 2013 14:36:23 -0600 Subject: [PATCH 0339/1060] Fix some bucket-related breakages. --- riak/bucket.py | 2 +- riak/tests/test_mapreduce.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 1f10832c..692ac7ac 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -64,7 +64,7 @@ def __eq__(self, other): else: return False - def __nq__(self, other): + def __ne__(self, other): if isinstance(other, self.__class__): return hash(self) != hash(other) else: diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 8a4bd1bc..655050a5 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -18,7 +18,7 @@ def test_store_and_get_links(self): self.assertEqual(len(links), 3) for bucket, key, tag in links: if (key == "foo1"): - self.assertEqual(bucket, "bucket") + self.assertEqual(bucket, self.bucket_name) elif (key == "foo2"): self.assertEqual(tag, "tag") elif (key == "foo3"): From b663639bcc2bbf4c12cfd17c99d4acb78762b5d1 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 19 Feb 2013 15:07:08 -0600 Subject: [PATCH 0340/1060] Bunch of pep8 and pyflakes fixes. --- riak/client/__init__.py | 2 +- riak/mapreduce.py | 5 +-- riak/riak_object.py | 17 +++++----- riak/test_server.py | 55 +++++++++++++++---------------- riak/tests/suite.py | 2 +- riak/tests/test_2i.py | 30 ++++++++--------- riak/tests/test_all.py | 27 ++++++++------- riak/tests/test_kv.py | 7 ++-- riak/tests/test_mapreduce.py | 15 ++++++--- riak/tests/test_search.py | 38 +++++++++++---------- riak/transports/feature_detect.py | 2 +- riak/transports/http/__init__.py | 2 +- riak/transports/http/transport.py | 28 ++++++++-------- riak/transports/pbc/__init__.py | 16 ++++----- riak/transports/pbc/codec.py | 24 +++++++------- riak/transports/pbc/connection.py | 2 +- riak/transports/pbc/stream.py | 5 +-- riak/transports/pbc/transport.py | 21 +++++------- riak/transports/transport.py | 8 ++--- riak/util.py | 2 +- 20 files changed, 160 insertions(+), 148 deletions(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 93c3c71a..e26912c0 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -80,7 +80,7 @@ def __init__(self, protocol='http', transport_options={}, (unused_args['port'], protocol)) unused_args['already_warned_port'] = True if (protocol in ['http', 'https'] and - 'http_port' not in unused_args): + 'http_port' not in unused_args): unused_args['http_port'] = unused_args['port'] elif protocol == 'pbc' and 'pb_port' not in unused_args: unused_args['pb_port'] = unused_args['port'] diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 6c627650..28501e6a 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -22,6 +22,7 @@ RiakLink = namedtuple("RiakLink", ("bucket", "key", "tag")) + class RiakMapReduce(object): """ The RiakMapReduce object allows you to build up and run a @@ -174,7 +175,7 @@ def index(self, bucket, index, startkey, endkey=None): """ self._input_mode = 'query' - if endkey == None: + if endkey is None: self._inputs = {'bucket': bucket, 'index': index, 'key': startkey} @@ -280,7 +281,7 @@ def run(self, timeout=None): return result # If there are no results, then return an empty list. - if result == None: + if result is None: return [] # Otherwise, if the last phase IS a link phase, then convert the diff --git a/riak/riak_object.py b/riak/riak_object.py index 18989c3a..33a93e2a 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -19,11 +19,10 @@ """ import copy from riak.metadata import ( - MD_CTYPE, - MD_INDEX, - MD_LINKS, - MD_USERMETA - ) + MD_CTYPE, + MD_INDEX, + MD_LINKS, + MD_USERMETA) from riak import RiakError @@ -106,7 +105,7 @@ def get_encoded_data(self): :rtype: string """ - if self._encode_data == True: + if self._encode_data: content_type = self.content_type encoder = self.bucket.get_encoder(content_type) if encoder is None: @@ -130,7 +129,7 @@ def set_encoded_data(self, data): :type data: string :rtype: RiakObject """ - if self._encode_data == True: + if self._encode_data: content_type = self.content_type decoder = self.bucket.get_decoder(content_type) if decoder is None: @@ -209,7 +208,7 @@ def remove_index(self, field=None, value=None): # This removes the index entries that's in the ries list. # Done because this is preferred over metadata[MD_INDEX].remove(rie) self.metadata[MD_INDEX] = [rie for rie in self.metadata[MD_INDEX] - if rie not in ries] + if rie not in ries] return self remove_indexes = remove_index @@ -239,7 +238,7 @@ def get_indexes(self, field=None): :rtype: (array of 2 element tuples with field, value) or (array of string or integer) """ - if field == None: + if field is None: return self.metadata[MD_INDEX] else: return [v for f, v in self.metadata[MD_INDEX] if f == field] diff --git a/riak/test_server.py b/riak/test_server.py index 28502210..73df069d 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import os.path import threading import string @@ -69,33 +67,35 @@ class TestServer(object): } APP_CONFIG_DEFAULTS = { - "riak_core": { - "web_ip": "127.0.0.1", - "web_port": 9000, - "handoff_port": 9001, - "ring_creation_size": 64 - }, - "riak_kv": { - "storage_backend": Atom("riak_kv_test_backend"), - "pb_ip": "127.0.0.1", - "pb_port": 9002, - "js_vm_count": 8, - "js_max_vm_mem": 8, - "js_thread_stack": 16, - "riak_kv_stat": True, - "map_cache_size": 0, - "vnode_cache_entries": 0, - "test": True, - "memory_backend": { + "riak_core": { + "web_ip": "127.0.0.1", + "web_port": 9000, + "handoff_port": 9001, + "ring_creation_size": 64 + }, + "riak_kv": { + "storage_backend": Atom("riak_kv_test_backend"), + "pb_ip": "127.0.0.1", + "pb_port": 9002, + "js_vm_count": 8, + "js_max_vm_mem": 8, + "js_thread_stack": 16, + "riak_kv_stat": True, + "map_cache_size": 0, + "vnode_cache_entries": 0, + "test": True, + "memory_backend": { "test": True, - }, - }, - "riak_search": { - "enabled": True, - "search_backend": Atom("riak_search_test_backend") - }, + }, + }, + "riak_search": { + "enabled": True, + "search_backend": Atom("riak_search_test_backend") + }, } + DEFAULT_BASE_DIR = "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}" + _temp_bin = None _temp_etc = None _temp_log = None @@ -224,8 +224,7 @@ def write_riak_script(self): line = re.sub("(PLATFORM_DATA_DIR=)(.*)", r'\1%s' % self.temp_dir, line) - if (string.strip(line) == - "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}"): + if (string.strip(line) == self.DEFAULT_BASE_DIR): line = ("RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join(self.bin_dir, ".."))) diff --git a/riak/tests/suite.py b/riak/tests/suite.py index 6f96c84d..97f3532c 100644 --- a/riak/tests/suite.py +++ b/riak/tests/suite.py @@ -12,5 +12,5 @@ def additional_tests(): start_dir = os.path.dirname(__file__) suite = unittest.TestSuite() suite.addTest(unittest.TestLoader().discover(start_dir, - top_level_dir=top_level)) + top_level_dir=top_level)) return suite diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index b5dcfee0..a525054e 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -54,11 +54,11 @@ def test_secondary_index_store(self): # Check the get_indexes() function... self.assertEqual([ - ('field1_bin', 'val1a'), - ('field1_bin', 'val1b'), - ('field1_int', 1011), - ('field1_int', 1012) - ], sorted(obj.get_indexes())) + ('field1_bin', 'val1a'), + ('field1_bin', 'val1b'), + ('field1_int', 1011), + ('field1_int', 1012) + ], sorted(obj.get_indexes())) # Delete an index... obj.remove_index('field1_bin', 'val1a') @@ -79,21 +79,21 @@ def test_secondary_index_store(self): obj.add_index('field1_int', 1011) self.assertEqual([ - ('field1_bin', 'val1a'), - ('field1_bin', 'val1b'), - ('field1_int', 1011), - ('field1_int', 1012) - ], sorted(obj.get_indexes())) + ('field1_bin', 'val1a'), + ('field1_bin', 'val1b'), + ('field1_int', 1011), + ('field1_int', 1012) + ], sorted(obj.get_indexes())) obj.store() obj = bucket.get('mykey1') self.assertEqual([ - ('field1_bin', 'val1a'), - ('field1_bin', 'val1b'), - ('field1_int', 1011), - ('field1_int', 1012) - ], sorted(obj.get_indexes())) + ('field1_bin', 'val1a'), + ('field1_bin', 'val1b'), + ('field1_int', 1011), + ('field1_int', 1012) + ], sorted(obj.get_indexes())) # Clean up... bucket.get('mykey1').delete() diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 13dcd2c6..833447fa 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -52,6 +52,7 @@ testrun_props_bucket = None testrun_sibs_bucket = None + def setUpModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket @@ -63,10 +64,11 @@ def setUpModule(): c.bucket(testrun_sibs_bucket).allow_mult = True if not int(os.environ.get('SKIP_SEARCH', '0')): - testrun_search_bucket = 'searchbucket' + testrun_search_bucket = 'searchbucket' b = c.bucket(testrun_search_bucket) b.enable_search() + def tearDownModule(): c = RiakClient(transport='http', http_port=HTTP_PORT) if not int(os.environ.get('SKIP_SEARCH', '0')): @@ -77,6 +79,7 @@ def tearDownModule(): b = c.bucket(testrun_props_bucket) b.clear_properties() + class BaseTestCase(object): host = None @@ -134,7 +137,7 @@ def setUp(self): self.host = PB_HOST self.pb_port = PB_PORT self.protocol = 'pbc' - self.http_client = self.create_client(HTTP_HOST, + self.http_client = self.create_client(HTTP_HOST, http_port=HTTP_PORT) super(RiakPbcTransportTestCase, self).setUp() @@ -230,17 +233,17 @@ def test_multi_and(self): f3 = RiakKeyFilter("matches", "-11-") f4 = f1 & f2 & f3 self.assertEqual(list(f4), [["and", - [["starts_with", "2005-"]], - [["ends_with", "-01"]], - [["matches", "-11-"]], - ]]) + [["starts_with", "2005-"]], + [["ends_with", "-01"]], + [["matches", "-11-"]], + ]]) def test_or(self): f1 = RiakKeyFilter("starts_with", "2005-") f2 = RiakKeyFilter("ends_with", "-01") f3 = f1 | f2 self.assertEqual(list(f3), [["or", [["starts_with", "2005-"]], - [["ends_with", "-01"]]]]) + [["ends_with", "-01"]]]]) def test_multi_or(self): f1 = RiakKeyFilter("starts_with", "2005-") @@ -248,10 +251,10 @@ def test_multi_or(self): f3 = RiakKeyFilter("matches", "-11-") f4 = f1 | f2 | f3 self.assertEqual(list(f4), [["or", - [["starts_with", "2005-"]], - [["ends_with", "-01"]], - [["matches", "-11-"]], - ]]) + [["starts_with", "2005-"]], + [["ends_with", "-01"]], + [["matches", "-11-"]], + ]]) def test_chaining(self): f1 = key_filter.tokenize("-", 1).eq("2005") @@ -260,7 +263,7 @@ def test_chaining(self): self.assertEqual(list(f3), [["and", [["tokenize", "-", 1], ["eq", "2005"]], [["tokenize", "-", 2], ["eq", "05"]] - ]]) + ]]) if __name__ == '__main__': unittest.main() diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 0cc3b9b9..bb93da6f 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -118,7 +118,7 @@ def test_binary_store_and_get(self): obj.store() obj = bucket.get_binary(key2) self.assertEqual(data, json.loads(obj.data)) - + def test_blank_binary_204(self): bucket = self.client.bucket(self.bucket_name) @@ -155,7 +155,8 @@ def test_unknown_content_type_encoder_decoder(self): # Teach the bucket how to pickle bucket = self.client.bucket(self.bucket_name) data = "some funny data" - obj = bucket.new(self.key_name, data, 'application/x-frobnicator').store() + obj = bucket.new(self.key_name, data, + 'application/x-frobnicator').store() obj.store() obj2 = bucket.get(self.key_name) self.assertEqual(data, obj2.data) @@ -184,7 +185,7 @@ def test_set_bucket_properties(self): bucket.allow_mult = True # Test setting nval... bucket.n_val = 1 - + bucket2 = self.create_client().bucket(self.props_bucket) self.assertTrue(bucket2.allow_mult) self.assertEqual(bucket2.n_val, 1) diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 655050a5..4531979f 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -30,8 +30,8 @@ def test_set_links(self): # Create the object bucket = self.client.bucket(self.bucket_name) bucket.new("foo", 2).set_links([bucket.new("foo1"), - (bucket.new("foo2"), "tag"), - ("bucket", "foo2", "tag2")]).store() + (bucket.new("foo2"), "tag"), + ("bucket", "foo2", "tag2")]).store() obj = bucket.get("foo") links = sorted(obj.get_links(), key=lambda x: x[1]) self.assertEqual(len(links), 3) @@ -121,11 +121,18 @@ def test_javascript_source_map(self): # test non-ASCII-encodable unicode is rejected self.assertRaises(TypeError, mr.map, - u"function (v) { /* æ */ return [JSON.parse(v.values[0].data)]; }") + u""" + function (v) { + /* æ */ + return [JSON.parse(v.values[0].data)]; + }""") # test non-ASCII-encodable string is rejected self.assertRaises(TypeError, mr.map, - "function (v) { /* æ */ return [JSON.parse(v.values[0].data)]; }") + """function (v) { + /* æ */ + return [JSON.parse(v.values[0].data)]; + }""") def test_javascript_named_map(self): # Create the object... diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index 9316b10d..25da929d 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -19,19 +19,23 @@ def test_bucket_search_enabled(self): def test_enable_search_commit_hook(self): bucket = self.client.bucket(self.search_bucket) bucket.clear_properties() - self.assertFalse(self.create_client().bucket(self.search_bucket).search_enabled()) + self.assertFalse(self.create_client(). + bucket(self.search_bucket). + search_enabled()) bucket.enable_search() - self.assertTrue(self.create_client().bucket(self.search_bucket).search_enabled()) + self.assertTrue(self.create_client(). + bucket(self.search_bucket). + search_enabled()) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_disable_search_commit_hook(self): bucket = self.client.bucket(self.search_bucket) bucket.clear_properties() bucket.enable_search() - self.assertTrue(self.create_client().bucket(self.search_bucket)\ + self.assertTrue(self.create_client().bucket(self.search_bucket) .search_enabled()) bucket.disable_search() - self.assertFalse(self.create_client().bucket(self.search_bucket)\ + self.assertFalse(self.create_client().bucket(self.search_bucket) .search_enabled()) bucket.enable_search() @@ -40,16 +44,16 @@ class SolrSearchTests(object): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_document_to_index(self): self.client.solr.add(self.search_bucket, - {"id": "doc", "username": "tony"}) + {"id": "doc", "username": "tony"}) results = self.client.solr.search(self.search_bucket, - "username:tony") + "username:tony") self.assertEquals("tony", results['docs'][0]['username']) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_multiple_documents_to_iindex(self): self.client.solr.add(self.search_bucket, - {"id": "dizzy", "username": "dizzy"}, - {"id": "russell", "username": "russell"}) + {"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}) results = self.client.solr\ .search(self.search_bucket, "username:russell OR username:dizzy") self.assertEquals(2, len(results['docs'])) @@ -57,8 +61,8 @@ def test_add_multiple_documents_to_iindex(self): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_id(self): self.client.solr.add(self.search_bucket, - {"id": "dizzy", "username": "dizzy"}, - {"id": "russell", "username": "russell"}) + {"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}) self.client.solr.delete(self.search_bucket, docs=["dizzy"]) results = self.client.solr\ .search(self.search_bucket, "username:russell OR username:dizzy") @@ -67,8 +71,8 @@ def test_delete_documents_from_search_by_id(self): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): self.client.solr.add(self.search_bucket, - {"id": "dizzy", "username": "dizzy"}, - {"id": "russell", "username": "russell"}) + {"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}) self.client.solr\ .delete(self.search_bucket, queries=["username:dizzy", "username:russell"]) @@ -79,11 +83,11 @@ def test_delete_documents_from_search_by_query(self): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query_and_id(self): self.client.solr.add(self.search_bucket, - {"id": "dizzy", "username": "dizzy"}, - {"id": "russell", "username": "russell"}) + {"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}) self.client.solr.delete(self.search_bucket, - docs=["dizzy"], - queries=["username:russell"]) + docs=["dizzy"], + queries=["username:russell"]) results = self.client.solr\ .search(self.search_bucket, "username:russell OR username:dizzy") @@ -112,7 +116,7 @@ def test_solr_search_with_params(self): results = self.client.solr.search(self.search_bucket, "username:roidrage", wt="xml") self.assertEquals(1, len(results['docs'])) - + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search(self): bucket = self.client.bucket(self.search_bucket) diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index ae54d193..3d8acfd9 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -24,7 +24,7 @@ 1: LooseVersion("1.0.0"), 1.1: LooseVersion("1.1.0"), 1.2: LooseVersion("1.2.0") - } +} class FeatureDetection(object): diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index 21569113..20662270 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -51,7 +51,7 @@ def destroy_resource(self, transport): httplib.NotConnected, httplib.IncompleteRead, httplib.ImproperConnectionState - ) +) def is_retryable(err): diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 3c8018ff..74f598ae 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -35,18 +35,18 @@ from riak.transports.http.stream import ( RiakHttpKeyStream, RiakHttpMapReduceStream - ) +) from riak.metadata import ( - MD_CHARSET, - MD_CTYPE, - MD_ENCODING, - MD_INDEX, - MD_LASTMOD, - MD_LINKS, - MD_USERMETA, - MD_VTAG, - MD_DELETED - ) + MD_CHARSET, + MD_CTYPE, + MD_ENCODING, + MD_INDEX, + MD_LASTMOD, + MD_LINKS, + MD_USERMETA, + MD_VTAG, + MD_DELETED +) from riak import RiakError from riak.multidict import MultiDict from xml.etree import ElementTree @@ -336,8 +336,8 @@ def stream_mapred(self, inputs, query, timeout=None): return RiakHttpMapReduceStream(response) else: raise Exception( - 'Error running MapReduce operation. Headers: %s Body: %s' % - (repr(headers), repr(response.read()))) + 'Error running MapReduce operation. Headers: %s Body: %s' % + (repr(headers), repr(response.read()))) def get_index(self, bucket, index, startkey, endkey=None): """ @@ -623,7 +623,7 @@ def parse_http_headers(cls, headers): key = matches.group(1).lower() value = matches.group(2).strip() if key in retVal.keys(): - if isinstance(retVal[key], list): + if isinstance(retVal[key], list): retVal[key].append(value) else: retVal[key] = [retVal[key]].append(value) diff --git a/riak/transports/pbc/__init__.py b/riak/transports/pbc/__init__.py index d49ab6fc..fc8914b6 100644 --- a/riak/transports/pbc/__init__.py +++ b/riak/transports/pbc/__init__.py @@ -49,14 +49,14 @@ def destroy_resource(self, pbc): # usable. On seeing any of these errors, the socket # should be closed, and the connection re-established. CONN_CLOSED_ERRORS = ( - errno.EHOSTUNREACH, - errno.ECONNRESET, - errno.ECONNREFUSED, - errno.ECONNABORTED, - errno.ETIMEDOUT, - errno.EBADF, - errno.EPIPE - ) + errno.EHOSTUNREACH, + errno.ECONNRESET, + errno.ECONNREFUSED, + errno.ECONNABORTED, + errno.ETIMEDOUT, + errno.EBADF, + errno.EPIPE +) def is_retryable(err): diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 2328b2ab..d8b3549f 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -16,17 +16,17 @@ under the License. """ from riak.metadata import ( - MD_CHARSET, - MD_CTYPE, - MD_ENCODING, - MD_INDEX, - MD_LASTMOD, - MD_LASTMOD_USECS, - MD_LINKS, - MD_USERMETA, - MD_VTAG, - MD_DELETED - ) + MD_CHARSET, + MD_CTYPE, + MD_ENCODING, + MD_INDEX, + MD_LASTMOD, + MD_LASTMOD_USECS, + MD_LINKS, + MD_USERMETA, + MD_VTAG, + MD_DELETED +) import riak_pb @@ -46,7 +46,7 @@ class RiakPbcCodec(object): 'all': RIAKC_RW_ALL, 'quorum': RIAKC_RW_QUORUM, 'one': RIAKC_RW_ONE - } + } def __init__(self, **unused_args): if riak_pb is None: diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index f334fa98..53916579 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -22,7 +22,7 @@ from messages import ( MESSAGE_CLASSES, MSG_CODE_ERROR_RESP - ) +) class RiakPbcConnection(object): diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index ce0bda78..02c69918 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -19,8 +19,9 @@ import json from riak.transports.pbc.messages import ( - MSG_CODE_LIST_KEYS_RESP, MSG_CODE_MAPRED_RESP - ) + MSG_CODE_LIST_KEYS_RESP, + MSG_CODE_MAPRED_RESP +) class RiakPbcStream(object): diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 1ed44a1a..d25b2f86 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -26,7 +26,6 @@ from stream import RiakPbcKeyStream, RiakPbcMapredStream from codec import RiakPbcCodec from messages import ( - # MSG_CODE_ERROR_RESP, MSG_CODE_PING_REQ, MSG_CODE_PING_RESP, MSG_CODE_GET_CLIENT_ID_REQ, @@ -44,18 +43,16 @@ MSG_CODE_LIST_BUCKETS_REQ, MSG_CODE_LIST_BUCKETS_RESP, MSG_CODE_LIST_KEYS_REQ, - # MSG_CODE_LIST_KEYS_RESP, MSG_CODE_GET_BUCKET_REQ, MSG_CODE_GET_BUCKET_RESP, MSG_CODE_SET_BUCKET_REQ, MSG_CODE_SET_BUCKET_RESP, MSG_CODE_MAPRED_REQ, - # MSG_CODE_MAPRED_RESP, MSG_CODE_INDEX_REQ, MSG_CODE_INDEX_RESP, MSG_CODE_SEARCH_QUERY_REQ, MSG_CODE_SEARCH_QUERY_RESP - ) +) class RiakPbcTransport(RiakTransport, RiakPbcConnection, RiakPbcCodec): @@ -98,7 +95,7 @@ def get_server_info(self): Get information about the server """ msg_code, resp = self._request(MSG_CODE_GET_SERVER_INFO_REQ, - expect=MSG_CODE_GET_SERVER_INFO_RESP) + expect=MSG_CODE_GET_SERVER_INFO_RESP) return {'node': resp.node, 'server_version': resp.server_version} def _get_client_id(self): @@ -111,7 +108,7 @@ def _set_client_id(self, client_id): req.client_id = client_id msg_code, resp = self._request(MSG_CODE_SET_CLIENT_ID_REQ, req, - MSG_CODE_SET_CLIENT_ID_RESP) + MSG_CODE_SET_CLIENT_ID_RESP) self._client_id = client_id @@ -179,7 +176,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, - MSG_CODE_PUT_RESP) + MSG_CODE_PUT_RESP) if resp is not None: contents = [] for c in resp.content: @@ -256,7 +253,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): req.key = robj.key msg_code, resp = self._request(MSG_CODE_DEL_REQ, req, - MSG_CODE_DEL_RESP) + MSG_CODE_DEL_RESP) return self def get_keys(self, bucket): @@ -298,7 +295,7 @@ def get_bucket_props(self, bucket): req.bucket = bucket.name msg_code, resp = self._request(MSG_CODE_GET_BUCKET_REQ, req, - MSG_CODE_GET_BUCKET_RESP) + MSG_CODE_GET_BUCKET_RESP) props = {} if resp.props.HasField('n_val'): props['n_val'] = resp.props.n_val @@ -323,7 +320,7 @@ def set_bucket_props(self, bucket, props): req.props.allow_mult = props['allow_mult'] msg_code, resp = self._request(MSG_CODE_SET_BUCKET_REQ, req, - MSG_CODE_SET_BUCKET_RESP) + MSG_CODE_SET_BUCKET_RESP) return self def mapred(self, inputs, query, timeout=None): @@ -371,7 +368,7 @@ def get_index(self, bucket, index, startkey, endkey=None): req.key = str(startkey) msg_code, resp = self._request(MSG_CODE_INDEX_REQ, req, - MSG_CODE_INDEX_RESP) + MSG_CODE_INDEX_RESP) return resp.keys def search(self, index, query, **params): @@ -402,7 +399,7 @@ def search(self, index, query, **params): req.presort = params['presort'] msg_code, resp = self._request(MSG_CODE_SEARCH_QUERY_REQ, req, - MSG_CODE_SEARCH_QUERY_RESP) + MSG_CODE_SEARCH_QUERY_RESP) result = {} if resp.HasField('max_score'): diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 2209de50..53615853 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -45,8 +45,8 @@ def make_random_client_id(self): """ Returns a random client identifier """ - return 'py_%s' % base64.b64encode( - str(random.randint(1, 0x40000000))) + return ('py_%s' % + base64.b64encode(str(random.randint(1, 0x40000000)))) @classmethod def make_fixed_client_id(self): @@ -204,7 +204,7 @@ def _search_mapred_emu(self, index, query): mr_result = self.mapred({'module': 'riak_search', 'function': 'mapred_search', 'arg': [index, query]}, - phases) + phases) result = {'num_found': len(mr_result), 'max_score': 0.0, 'docs': []} @@ -237,7 +237,7 @@ def _get_index_mapred_emu(self, bucket, index, startkey, endkey=None): 'index': index, 'key': startkey}, phases) - return [key for bucket, key in result] + return [key for resultbucket, key in result] def _construct_mapred_json(self, inputs, query, timeout=None): if not self.phaseless_mapred() and (query is None or len(query) is 0): diff --git a/riak/util.py b/riak/util.py index 3799139f..448ee8d7 100644 --- a/riak/util.py +++ b/riak/util.py @@ -47,7 +47,7 @@ def deep_merge(a, b): current_dst[key] = current_src[key] else: if (quacks_like_dict(current_src[key]) - and quacks_like_dict(current_dst[key])): + and quacks_like_dict(current_dst[key])): stack.append((current_dst[key], current_src[key])) else: current_dst[key] = current_src[key] From dc5bd99eaa896b46a58e27d88c852e2526a8d4b9 Mon Sep 17 00:00:00 2001 From: Rob Speer Date: Tue, 19 Feb 2013 17:24:19 -0500 Subject: [PATCH 0341/1060] add httplib.BadStatusLine as a retryable error --- riak/transports/http/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index 21569113..003fc61c 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -50,7 +50,8 @@ def destroy_resource(self, transport): CONN_CLOSED_ERRORS = ( httplib.NotConnected, httplib.IncompleteRead, - httplib.ImproperConnectionState + httplib.ImproperConnectionState, + httplib.BadStatusLine ) From 1ba533e707dc99450ef73ade4ba7efda76d2d2bd Mon Sep 17 00:00:00 2001 From: evan Date: Wed, 20 Feb 2013 17:24:36 -0800 Subject: [PATCH 0342/1060] add a test & try to catch when allow_strfun is false --- riak/mapreduce.py | 10 +++++++++- riak/tests/test_mapreduce.py | 24 +++++++++++++++++++++++- riak/transports/http/transport.py | 2 +- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 8d2fa8f4..065dd4b9 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -19,6 +19,7 @@ """ from collections import Iterable, namedtuple +from riak import RiakError RiakLink = namedtuple("RiakLink", ("bucket", "key", "tag")) @@ -273,7 +274,14 @@ def run(self, timeout=None): """ query, link_results_flag = self._normalize_query() - result = self._client.mapred(self._inputs, query, timeout) + try: + result = self._client.mapred(self._inputs, query, timeout) + except RiakError as e: + for phase in self._phases: + if phase._language == 'erlang': + if type(phase._function) is str: + raise RiakError('may have tried erlang strfun when not allowed') + raise e # If the last phase is NOT a link phase, then return the result. if not (link_results_flag diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 4531979f..9bb3e6fd 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from riak.mapreduce import RiakMapReduce -from riak import key_filter +from riak import key_filter, RiakError class LinkTests(object): @@ -83,6 +83,28 @@ def test_erlang_map_reduce(self): .run() self.assertEqual(len(result), 2) + def test_erlang_source_map_reduce(self): + # Create the object... + bucket = self.client.bucket(self.bucket_name) + bucket.new("foo", 2).store() + bucket.new("bar", 3).store() + bucket.new("baz", 4).store() + strfun_allowed = True + # Run the map... + try: + result = self.client \ + .add(self.bucket_name, "foo") \ + .add(self.bucket_name, "bar") \ + .add(self.bucket_name, "baz") \ + .map("""fun(Object, _KD, _A) -> + Value = riak_object:get_value(Object), + [Value] + end.""", {'language': 'erlang'}).run() + except RiakError as e: + strfun_allowed = False + if strfun_allowed: + self.assertEqual(result, ['2', '3', '4']) + def test_client_exceptional_paths(self): bucket = self.client.bucket(self.bucket_name) bucket.new("foo", 2).store() diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 74f598ae..e17a649c 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -317,7 +317,7 @@ def mapred(self, inputs, query, timeout=None): # Make sure the expected status code came back... status = response[0]['http_code'] if status != 200: - raise Exception( + raise RiakError( 'Error running MapReduce operation. Headers: %s Body: %s' % (repr(response[0]), repr(response[1]))) From 8e16c64974c0452faa999ba251b8b67189e0c406 Mon Sep 17 00:00:00 2001 From: evan Date: Wed, 20 Feb 2013 22:24:15 -0800 Subject: [PATCH 0343/1060] tweak iterator locking to remove leak and deadlock issues. --- riak/tests/pool-grinder.py | 98 ++++++++++++++++++++++++++++++++++++++ riak/transports/pool.py | 21 +++++--- 2 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 riak/tests/pool-grinder.py diff --git a/riak/tests/pool-grinder.py b/riak/tests/pool-grinder.py new file mode 100644 index 00000000..6918aacd --- /dev/null +++ b/riak/tests/pool-grinder.py @@ -0,0 +1,98 @@ + + +from Queue import Queue +from threading import Thread, currentThread +from pool import Pool, BadResource +from random import SystemRandom +from time import sleep + +class SimplePool(Pool): + def __init__(self): + self.count = 0 + Pool.__init__(self) + + def create_resource(self): + self.count += 1 + return [self.count] + + def destroy_resource(self, resource): + del resource[:] + + +class EmptyListPool(Pool): + def create_resource(self): + return [] + +def test(): + started = Queue() + n = 1000 + threads = [] + touched = [] + pool = EmptyListPool() + rand = SystemRandom() + + def _run(): + psleep = rand.uniform(0.6, 0.75) + with pool.take() as a: + started.put(1) + started.join() + a.append(rand.uniform(0, 1)) + if psleep > 1: + print psleep + sleep(psleep) + + for i in range(n): + th = Thread(target=_run) + threads.append(th) + th.start() + + for i in range(n): + started.get() + started.task_done() + + for element in pool: + touched.append(element) + + for thr in threads: + thr.join() + + if set(pool.elements) != set(touched): + print set(pool.elements) - set(touched) + return False + else: + return True + +ret = True +count = 0 +while ret: + ret = test() + count += 1 + print count + + +# INSTRUMENTED FUNCTION + +# def __claim_elements(self): +# #print 'waiting for self lock' +# with self.lock: +# if self.__all_claimed(): # and self.unlocked: +# #print 'waiting on releaser lock' +# with self.releaser: +# print 'waiting for release' +# print 'targets', self.targets +# print 'tomb', self.targets[0].tomb +# print 'claimed', self.targets[0].claimed +# print self.releaser +# print self.lock +# print self.unlocked +# self.releaser.wait(1) +# for element in self.targets: +# if element.tomb: +# self.targets.remove(element) +# #self.unlocked.remove(element) +# continue +# if not element.claimed: +# self.targets.remove(element) +# self.unlocked.append(element) +# element.claimed = True + diff --git a/riak/transports/pool.py b/riak/transports/pool.py index 062e66e0..a94114b2 100644 --- a/riak/transports/pool.py +++ b/riak/transports/pool.py @@ -46,6 +46,8 @@ def __init__(self, obj): self.object = obj """Whether the resource is currently in use.""" self.claimed = False + """Whether the object has been deleted.""" + self.tomb = False class Pool(object): @@ -127,7 +129,7 @@ def _filter(obj): finally: with self.releaser: element.claimed = False - self.releaser.notify() + self.releaser.notify_all() def delete_element(self, element): """ @@ -140,8 +142,8 @@ def delete_element(self, element): """ with self.lock: self.elements.remove(element) + element.tomb = True self.destroy_resource(element.object) - del element def __iter__(self): """ @@ -203,23 +205,28 @@ def __iter__(self): def next(self): if len(self.targets) == 0: raise StopIteration - if len(self.unlocked) == 0: + while len(self.unlocked) == 0: self.__claim_elements() - return self.unlocked.pop(0) + ele = None + while not ele: + ele = self.unlocked.pop(0) + if ele.tomb: + ele = None + return ele def __claim_elements(self): with self.lock: if self.__all_claimed(): with self.releaser: - self.releaser.wait() - for element in self.targets[:]: + self.releaser.wait(.05) + for element in self.targets: if not element.claimed: self.targets.remove(element) self.unlocked.append(element) element.claimed = True def __all_claimed(self): - for element in self.targets[:]: + for element in self.targets: if not element.claimed: return False return True From 41ec70edd28b31a0991893a311b040952965dccf Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 21 Feb 2013 15:00:23 -0600 Subject: [PATCH 0344/1060] Make set/get_encoded_data into a real property. This changes the semantics such that accessing the 'data' property will cause the encoded data to be deserialized. The converse is also true: accessing the 'encoded_data' property causes the Python data types to be serialized. If a proper encoder or decoder is not registered for the object's content-type, an exception will be raised. This means that the assumption about 'data' as a string being equivalent to 'encoded_data' is no longer valid and existing code that relies on that assumption will break. --- riak/bucket.py | 4 +- riak/riak_object.py | 120 +++++++++++++++--------------- riak/tests/test_kv.py | 18 ++--- riak/transports/http/transport.py | 4 +- riak/transports/pbc/transport.py | 4 +- 5 files changed, 74 insertions(+), 76 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 692ac7ac..ddcacf9d 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -141,7 +141,6 @@ def new(self, key=None, data=None, content_type='application/json'): obj = RiakObject(self._client, self, key) obj.data = data obj.content_type = content_type - obj._encode_data = True return obj def new_binary(self, key, data, content_type='application/octet-stream'): @@ -160,9 +159,8 @@ def new_binary(self, key, data, content_type='application/octet-stream'): :rtype: :class:`RiakObject ` """ obj = RiakObject(self._client, self, key) - obj.data = data + obj.encoded_data = data obj.content_type = content_type - obj._encode_data = False return obj def get(self, key, r=None, pr=None): diff --git a/riak/riak_object.py b/riak/riak_object.py index 33a93e2a..3d92869f 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -24,6 +24,7 @@ MD_LINKS, MD_USERMETA) from riak import RiakError +from riak.util import deprecated class RiakObject(object): @@ -52,8 +53,8 @@ def __init__(self, client, bucket, key=None): self.client = client self.bucket = bucket self.key = key - self._encode_data = True self._data = None + self._encoded_data = None self.vclock = None self.metadata = {MD_USERMETA: {}, MD_INDEX: []} self.links = [] @@ -76,71 +77,69 @@ def __ne__(self, other): return True def _get_data(self): + if self._encoded_data is not None and self._data is None: + self._data = self._deserialize(self._encoded_data) + self._encoded_data = None return self._data - def _set_data(self, data, content_type=None): - if MD_CTYPE not in self.metadata: - if self._encode_data: - self.content_type = "application/json" - else: - self.content_type = "application/octet-stream" - if content_type: - self.content_type = content_type - self._data = data - return self + def _set_data(self, value): + self._encoded_data = None + self._data = value data = property(_get_data, _set_data, doc=""" - The data stored in this object. This data will be - JSON encoded on storage unless the object was constructed with - :func:`RiakBucket.new_binary ` or - :func:`RiakBucket.get_binary `, - in which case it will be stored as a string. On return, it shall - either be a dict or a string, depending on its storage type. - + The data stored in this object, as Python objects. For the raw + data, use the `encoded_data` property. If unset, accessing + this property will result in decoding the `encoded_data` + property into Python values. The decoding is dependent on the + `content_type` property and the bucket's registered decoders. :type mixed """) def get_encoded_data(self): - """ - Get the data encoded for storing - - :rtype: string - """ - if self._encode_data: - content_type = self.content_type - encoder = self.bucket.get_encoder(content_type) - if encoder is None: - if isinstance(self.data, basestring): - return self.data.encode() - else: - raise RiakError("No encoder for non-string data " - "with content type ${0}". - format(content_type)) - else: - return encoder(self._data) + deprecated("`get_encoded_data` is deprecated, use the `encoded_data`" + " property") + return self.encoded_data + + def set_encoded_data(self, value): + deprecated("`set_encoded_data` is deprecated, use the `encoded_data`" + " property") + self.encoded_data = value + + def _get_encoded_data(self): + if self._data is not None and self._encoded_data is None: + self._encoded_data = self._serialize(self._data) + self._data = None + return self._encoded_data + + def _set_encoded_data(self, value): + self._data = None + self._encoded_data = value + + encoded_data = property(_get_encoded_data, _set_encoded_data, doc=""" + The raw data stored in this object, essentially the encoded + form of the `data` property. If unset, accessing this property + will result in encoding the `data` property into a string. The + encoding is dependent on the `content_type` property and the + bucket's registered encoders. + :type basestring""") + + def _serialize(self, value): + encoder = self.bucket.get_encoder(self.content_type) + if encoder: + return encoder(value) + elif isinstance(value, basestring): + return value.encode() else: - return self.data - - def set_encoded_data(self, data): - """ - Set the object data from an encoded string. Make sure - the metadata has been set correctly first. - - :param data: the encoded data - :type data: string - :rtype: RiakObject - """ - if self._encode_data: - content_type = self.content_type - decoder = self.bucket.get_decoder(content_type) - if decoder is None: - # if no decoder, just set as string data for - # application to handle - self.data = data - else: - self.data = decoder(data) + raise RiakError('No encoder for non-string data ' + 'with content type "{0}"'. + format(self.content_type)) + + def _deserialize(self, value): + decoder = self.bucket.get_decoder(self.content_type) + if decoder: + return decoder(value) else: - self.data = data - return self + raise RiakError('No decoder for content type "{0}"'. + format(self.content_type)) def _get_usermeta(self): if MD_USERMETA in self.metadata: @@ -387,7 +386,8 @@ def store(self, w=None, dw=None, pw=None, return_body=True, there is no key previously defined :type if_none_match: bool :rtype: RiakObject """ - if self.siblings and not self.data and not self.vclock: + if (self.siblings and not self._data + and not self._encoded_data and not self.vclock): raise RiakError("Attempting to store an invalid object," "store one of the siblings instead") @@ -497,13 +497,13 @@ def _populate(self, result): if not MD_INDEX in metadata: metadata[MD_INDEX] = [] self.metadata = metadata - self.set_encoded_data(data) + self._encoded_data = data # Create objects for all siblings siblings = [self] for (metadata, data) in contents: sibling = copy.copy(self) sibling.metadata = metadata - sibling.data = data + sibling.encoded_data = data siblings.append(sibling) for sibling in siblings: sibling._set_siblings(siblings) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index bb93da6f..c442ed2b 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -110,14 +110,14 @@ def test_binary_store_and_get(self): obj.store() obj = bucket.get_binary(self.key_name) self.assertTrue(obj.exists) - self.assertEqual(obj.data, rand) + self.assertEqual(obj.encoded_data, rand) # Store as JSON, retrieve as binary, JSON-decode, then compare... data = [self.randint(), self.randint(), self.randint()] key2 = self.randname() obj = bucket.new(key2, data) obj.store() obj = bucket.get_binary(key2) - self.assertEqual(data, json.loads(obj.data)) + self.assertEqual(data, json.loads(obj.encoded_data)) def test_blank_binary_204(self): bucket = self.client.bucket(self.bucket_name) @@ -127,7 +127,7 @@ def test_blank_binary_204(self): obj.store() obj = bucket.get_binary('foo2') self.assertTrue(obj.exists) - self.assertEqual(obj.data, '') + self.assertEqual(obj.encoded_data, '') def test_custom_bucket_encoder_decoder(self): # Teach the bucket how to pickle @@ -159,7 +159,7 @@ def test_unknown_content_type_encoder_decoder(self): 'application/x-frobnicator').store() obj.store() obj2 = bucket.get(self.key_name) - self.assertEqual(data, obj2.data) + self.assertEqual(data, obj2.encoded_data) def test_missing_object(self): bucket = self.client.bucket(self.bucket_name) @@ -244,7 +244,7 @@ def test_siblings(self): # Get each of the values - make sure they match what was assigned vals2 = set() for i in range(5): - vals2.add(obj.get_sibling(i).data) + vals2.add(obj.get_sibling(i).encoded_data) self.assertEqual(vals, vals2) # Resolve the conflict, and then do a get... @@ -253,7 +253,7 @@ def test_siblings(self): obj.reload() self.assertEqual(len(obj.siblings), 0) - self.assertEqual(obj.data, obj3.data) + self.assertEqual(obj.encoded_data, obj3.encoded_data) def test_store_of_missing_object(self): bucket = self.client.bucket(self.bucket_name) @@ -269,10 +269,10 @@ def test_store_of_missing_object(self): # for binary objects o = bucket.get_binary(self.randname()) self.assertEqual(o.exists, False) - o.data = "1234567890" + o.encoded_data = "1234567890" o = o.store() - self.assertEqual(o.data, "1234567890") + self.assertEqual(o.encoded_data, "1234567890") self.assertEqual(o.content_type, "application/octet-stream") o.delete() @@ -376,7 +376,7 @@ def test_store_binary_object_from_file(self): obj = bucket.new_binary_from_file(self.key_name, filepath) obj.store() obj = bucket.get_binary(self.key_name) - self.assertNotEqual(obj.data, None) + self.assertNotEqual(obj.encoded_data, None) self.assertEqual(obj.content_type, "text/x-python") def test_store_binary_object_from_file_should_use_default_mimetype(self): diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 74f598ae..73459ba4 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -149,7 +149,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, # which is a superset of the if_none_match semantics. if if_none_match: headers["If-None-Match"] = "*" - content = robj.get_encoded_data() + content = robj.encoded_data return self.do_put(url, headers, content, return_body, key=robj.key) @@ -177,7 +177,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, # which is a superset of the if_none_match semantics. if if_none_match: headers["If-None-Match"] = "*" - content = robj.get_encoded_data() + content = robj.encoded_data response = self._request('POST', url, headers, content) location = response[0]['location'] idx = location.rindex('/') diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index d25b2f86..61623cdf 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -172,7 +172,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, req.vclock = vclock self.encode_content(robj.metadata, - robj.get_encoded_data(), + robj.encoded_data, req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, @@ -211,7 +211,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, req.bucket = bucket.name self.encode_content(robj.metadata, - robj.get_encoded_data(), + robj.encoded_data, req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, From fb85993387db7058e2a1c8a96edd1490f6863a7a Mon Sep 17 00:00:00 2001 From: evan Date: Thu, 21 Feb 2013 15:14:58 -0800 Subject: [PATCH 0345/1060] alter pool iterator grinder to work in its new location tweak some test parameters to test more stressfully --- riak/tests/pool-grinder.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) mode change 100644 => 100755 riak/tests/pool-grinder.py diff --git a/riak/tests/pool-grinder.py b/riak/tests/pool-grinder.py old mode 100644 new mode 100755 index 6918aacd..f1520f53 --- a/riak/tests/pool-grinder.py +++ b/riak/tests/pool-grinder.py @@ -1,7 +1,9 @@ - +#!/usr/bin/env python from Queue import Queue from threading import Thread, currentThread +import sys +sys.path.append("../transports/") from pool import Pool, BadResource from random import SystemRandom from time import sleep @@ -32,7 +34,7 @@ def test(): rand = SystemRandom() def _run(): - psleep = rand.uniform(0.6, 0.75) + psleep = rand.uniform(0.05, 0.1) with pool.take() as a: started.put(1) started.join() From 3db29c86e102a8d3ea3f128da12e978f94782d72 Mon Sep 17 00:00:00 2001 From: evan Date: Thu, 21 Feb 2013 15:43:23 -0800 Subject: [PATCH 0346/1060] enhance iterator test to be more like pool grinder --- riak/tests/test_pool.py | 54 +++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index 6021510f..3eb6e2be 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -202,37 +202,39 @@ def test_iteration(self): should eventually touch all resources (excluding ones created during iteration). """ - started = Queue() - n = 30 - threads = [] - touched = [] - pool = EmptyListPool() - rand = SystemRandom() - - def _run(): - psleep = rand.uniform(0, 0.75) - with pool.take() as a: - started.put(1) - started.join() - a.append(rand.uniform(0, 1)) - sleep(psleep) - for i in range(n): - th = Thread(target=_run) - threads.append(th) - th.start() + for i in range(50): + started = Queue() + n = 1000 + threads = [] + touched = [] + pool = EmptyListPool() + rand = SystemRandom() + + def _run(): + psleep = rand.uniform(0.05, 0.1) + with pool.take() as a: + started.put(1) + started.join() + a.append(rand.uniform(0, 1)) + sleep(psleep) - for i in range(n): - started.get() - started.task_done() + for i in range(n): + th = Thread(target=_run) + threads.append(th) + th.start() - for element in pool: - touched.append(element) + for i in range(n): + started.get() + started.task_done() - for thr in threads: - thr.join() + for element in pool: + touched.append(element) + + for thr in threads: + thr.join() - self.assertItemsEqual(pool.elements, touched) + self.assertItemsEqual(pool.elements, touched) def test_clear(self): """ From a45e91e80fa29e996ee7fbb8045a30e6fc4d70b5 Mon Sep 17 00:00:00 2001 From: evan Date: Thu, 21 Feb 2013 15:43:42 -0800 Subject: [PATCH 0347/1060] update with a more minimal fix and a better understanding of the problem. For the record: Our deadlock occurred when the __all_claimed check passed, but when another thread released the last element in the pool before we took back up the lock. Doing the check inside of the lock (thus preventing mutation of the claimed state for elements) closes the loophole for the race. Making the pool lock re-entrant and setting the condition variable might not be required, but it affords us a little bit of extra safety. --- riak/transports/pool.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/riak/transports/pool.py b/riak/transports/pool.py index a94114b2..13d5cdf7 100644 --- a/riak/transports/pool.py +++ b/riak/transports/pool.py @@ -46,8 +46,6 @@ def __init__(self, obj): self.object = obj """Whether the resource is currently in use.""" self.claimed = False - """Whether the object has been deleted.""" - self.tomb = False class Pool(object): @@ -84,8 +82,8 @@ def __init__(self): Creates a new Pool. This should be called manually if you override the __init__ method in a subclass. """ - self.lock = threading.Lock() - self.releaser = threading.Condition() + self.lock = threading.RLock() + self.releaser = threading.Condition(self.lock) self.elements = list() @contextmanager @@ -142,8 +140,8 @@ def delete_element(self, element): """ with self.lock: self.elements.remove(element) - element.tomb = True self.destroy_resource(element.object) + del element def __iter__(self): """ @@ -205,20 +203,15 @@ def __iter__(self): def next(self): if len(self.targets) == 0: raise StopIteration - while len(self.unlocked) == 0: + if len(self.unlocked) == 0: self.__claim_elements() - ele = None - while not ele: - ele = self.unlocked.pop(0) - if ele.tomb: - ele = None - return ele + return self.unlocked.pop(0) def __claim_elements(self): with self.lock: - if self.__all_claimed(): - with self.releaser: - self.releaser.wait(.05) + with self.releaser: + if self.__all_claimed(): + self.releaser.wait() for element in self.targets: if not element.claimed: self.targets.remove(element) From 1f694510d8e955efe03a6f9c9926fb3b82e5f64b Mon Sep 17 00:00:00 2001 From: evan Date: Thu, 21 Feb 2013 15:54:23 -0800 Subject: [PATCH 0348/1060] pep8 fixes, decrease iterator test time --- riak/tests/pool-grinder.py | 11 ++++++----- riak/tests/test_pool.py | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/riak/tests/pool-grinder.py b/riak/tests/pool-grinder.py index f1520f53..6765363e 100755 --- a/riak/tests/pool-grinder.py +++ b/riak/tests/pool-grinder.py @@ -8,6 +8,7 @@ from random import SystemRandom from time import sleep + class SimplePool(Pool): def __init__(self): self.count = 0 @@ -25,6 +26,7 @@ class EmptyListPool(Pool): def create_resource(self): return [] + def test(): started = Queue() n = 1000 @@ -32,7 +34,7 @@ def test(): touched = [] pool = EmptyListPool() rand = SystemRandom() - + def _run(): psleep = rand.uniform(0.05, 0.1) with pool.take() as a: @@ -42,7 +44,7 @@ def _run(): if psleep > 1: print psleep sleep(psleep) - + for i in range(n): th = Thread(target=_run) threads.append(th) @@ -67,12 +69,12 @@ def _run(): ret = True count = 0 while ret: - ret = test() + ret = test() count += 1 print count -# INSTRUMENTED FUNCTION +# INSTRUMENTED FUNCTION # def __claim_elements(self): # #print 'waiting for self lock' @@ -97,4 +99,3 @@ def _run(): # self.targets.remove(element) # self.unlocked.append(element) # element.claimed = True - diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index 3eb6e2be..61793a31 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -203,14 +203,14 @@ def test_iteration(self): during iteration). """ - for i in range(50): + for i in range(25): started = Queue() n = 1000 threads = [] touched = [] pool = EmptyListPool() rand = SystemRandom() - + def _run(): psleep = rand.uniform(0.05, 0.1) with pool.take() as a: @@ -230,7 +230,7 @@ def _run(): for element in pool: touched.append(element) - + for thr in threads: thr.join() From 8b690e13380f151e0911efa8c349336ee9f98bdc Mon Sep 17 00:00:00 2001 From: evan Date: Thu, 21 Feb 2013 16:35:14 -0800 Subject: [PATCH 0349/1060] narrow the scope of the catch so that fewer errors are masked, append the original error for user evaluation. this still masks typos in the strfun code, because they're indistinguishable from the strfun not allowed message, and it's impossible to tell, client side, which is which. --- riak/mapreduce.py | 11 +++++++---- riak/tests/test_mapreduce.py | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 065dd4b9..5792aab7 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -277,10 +277,13 @@ def run(self, timeout=None): try: result = self._client.mapred(self._inputs, query, timeout) except RiakError as e: - for phase in self._phases: - if phase._language == 'erlang': - if type(phase._function) is str: - raise RiakError('may have tried erlang strfun when not allowed') + if 'worker_startup_failed' in e.value: + for phase in self._phases: + if phase._language == 'erlang': + if type(phase._function) is str: + raise RiakError('May have tried erlang strfun ' + 'when not allowed\n' + 'original error: ' + e.value) raise e # If the last phase is NOT a link phase, then return the result. diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 9bb3e6fd..43b5bf9d 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -101,7 +101,8 @@ def test_erlang_source_map_reduce(self): [Value] end.""", {'language': 'erlang'}).run() except RiakError as e: - strfun_allowed = False + if e.value.startswith('May have tried'): + strfun_allowed = False if strfun_allowed: self.assertEqual(result, ['2', '3', '4']) From 59cd63d06b8ad1ad4c73ffb4172e972a76920bc3 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 22 Feb 2013 12:14:06 -0600 Subject: [PATCH 0350/1060] Use TypeError instead of RiakError. --- riak/riak_object.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 3d92869f..6b0cd7a7 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -129,7 +129,7 @@ def _serialize(self, value): elif isinstance(value, basestring): return value.encode() else: - raise RiakError('No encoder for non-string data ' + raise TypeError('No encoder for non-string data ' 'with content type "{0}"'. format(self.content_type)) @@ -138,7 +138,7 @@ def _deserialize(self, value): if decoder: return decoder(value) else: - raise RiakError('No decoder for content type "{0}"'. + raise TypeError('No decoder for content type "{0}"'. format(self.content_type)) def _get_usermeta(self): From f3d2fea14b074fafd9af23e6a5d4782bdc90cdd0 Mon Sep 17 00:00:00 2001 From: Hector Castro Date: Fri, 22 Feb 2013 11:28:27 -0500 Subject: [PATCH 0351/1060] Updated documentation to reflect `port` and `transport_class` configuration option deprecations to `RiakClient`. --- README.rst | 13 +++++++------ docs/tutorial.rst | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 51171234..78787881 100644 --- a/README.rst +++ b/README.rst @@ -97,14 +97,15 @@ no arguments are needed:: client = riak.RiakClient() -The constructor also configuration options such as ``host``, ``port`` & -``prefix``. Please refer to the :doc:`client` documentation for full details. +The constructor also configuration options such as ``host``, ``http_port``, +``pb_port`` & ``prefix``. Please refer to the :doc:`client` documentation +for full details. To use the Protocol Buffers interface:: import riak - client = riak.RiakClient(port=8087, transport_class=riak.RiakPbcTransport) + client = riak.RiakClient(pb_port=8087, protocol='pbc') .. warning: @@ -114,9 +115,9 @@ To use the Protocol Buffers interface:: data to the effect of ``RiakError: 'Socket returned short read 135 - expected 8192'``. -The ``transport_class`` argument indicates to the client which backend to use. -We didn't need to specify it in the HTTP example because -``riak.RiakHttpTransport`` is the default class. +The ``protocol`` argument indicates to the client which backend to use. +We didn't need to specify it in the HTTP example because ``http`` is the +default class. Available options are: ``http``, ``https``, & ``pbc``. Using Buckets diff --git a/docs/tutorial.rst b/docs/tutorial.rst index f1fe1994..3e51bfff 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -61,14 +61,15 @@ no arguments are needed:: client = riak.RiakClient() -The constructor also configuration options such as ``host``, ``port`` & -``prefix``. Please refer to the :doc:`client` documentation for full details. +The constructor also configuration options such as ``host``, ``http_port``, +``pb_port`` & ``prefix``. Please refer to the :doc:`client` documentation +for full details. To use the Protocol Buffers interface:: import riak - client = riak.RiakClient(port=8087, transport_class=riak.RiakPbcTransport) + client = riak.RiakClient(pb_port=8087, protocol='pbc') .. warning: @@ -78,9 +79,9 @@ To use the Protocol Buffers interface:: data to the effect of ``RiakError: 'Socket returned short read 135 - expected 8192'``. -The ``transport_class`` argument indicates to the client which backend to use. -We didn't need to specify it in the HTTP example because -``riak.RiakHttpTransport`` is the default class. +The ``protocol`` argument indicates to the client which backend to use. +We didn't need to specify it in the HTTP example because ``http`` is the +default class. Available options are: ``http``, ``https``, & ``pbc``. Using Buckets From bd43a804463ca8f6558376ec4af95077f8901563 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 25 Feb 2013 09:25:54 -0600 Subject: [PATCH 0352/1060] RiakObject._encode_data is no longer used. --- riak/riak_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 6b0cd7a7..bdc73797 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -246,7 +246,7 @@ def _get_content_type(self): try: return self.metadata[MD_CTYPE] except KeyError: - if self._encode_data: + if self._data: return "application/json" else: return "application/octet-stream" From b36d787853f48dfc1deb247f270d11bfad734f41 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 25 Feb 2013 10:52:20 -0600 Subject: [PATCH 0353/1060] Deprecate new_binary and new_binary_from_file. --- riak/bucket.py | 37 +++++++++++++++++++------------ riak/tests/test_kv.py | 43 +++++++++++++++++++++--------------- riak/tests/test_mapreduce.py | 9 +++++--- 3 files changed, 54 insertions(+), 35 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index ddcacf9d..dbb5dcbd 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -18,7 +18,7 @@ under the License. """ import mimetypes -from riak.util import deprecateQuorumAccessors +from riak.util import deprecateQuorumAccessors, deprecated def deprecateBucketQuorumAccessors(klass): @@ -118,7 +118,8 @@ def set_decoder(self, content_type, decoder): self._decoders[content_type] = decoder return self - def new(self, key=None, data=None, content_type='application/json'): + def new(self, key=None, data=None, content_type='application/json', + encoded_data=None): """ Create a new :class:`RiakObject ` that will be stored as JSON. A shortcut for manually @@ -139,11 +140,15 @@ def new(self, key=None, data=None, content_type='application/json'): raise TypeError('Unicode data values are not supported.') obj = RiakObject(self._client, self, key) - obj.data = data obj.content_type = content_type + if data is not None: + obj.data = data + if encoded_data is not None: + obj.encoded_data = encoded_data return obj - def new_binary(self, key, data, content_type='application/octet-stream'): + def new_binary(self, key=None, data=None, + content_type='application/octet-stream'): """ Create a new :class:`RiakObject ` that will be stored as plain text/binary. A shortcut for @@ -158,10 +163,10 @@ def new_binary(self, key, data, content_type='application/octet-stream'): :type content_type: string :rtype: :class:`RiakObject ` """ - obj = RiakObject(self._client, self, key) - obj.encoded_data = data - obj.content_type = content_type - return obj + deprecated('RiakBucket.new_binary is deprecated, ' + 'use RiakBucket.new with the encoded_data ' + 'param instead of data') + return self.new(key, encoded_data=data, content_type=content_type) def get(self, key, r=None, pr=None): """ @@ -176,7 +181,6 @@ def get(self, key, r=None, pr=None): :rtype: :class:`RiakObject ` """ obj = RiakObject(self._client, self, key) - obj._encode_data = True return obj.reload(r=r, pr=pr) def get_binary(self, key, r=None, pr=None): @@ -191,9 +195,9 @@ def get_binary(self, key, r=None, pr=None): :type pr: integer :rtype: :class:`RiakObject ` """ - obj = RiakObject(self._client, self, key) - obj._encode_data = False - return obj.reload(r=r, pr=pr) + deprecated('RiakBucket.get_binary is deprecated, ' + 'use RiakBucket.get') + return self.get(key, r=r, pr=pr) def _set_n_val(self, nval): return self.set_property('n_val', nval) @@ -370,7 +374,7 @@ def stream_keys(self): """ return self._client.stream_keys(self) - def new_binary_from_file(self, key, filename): + def new_from_file(self, key, filename): """ Create a new Riak object in the bucket, using the content of the specified file. @@ -383,7 +387,12 @@ def new_binary_from_file(self, key, filename): binary_data = bytearray(binary_data) if not mimetype: mimetype = 'application/octet-stream' - return self.new_binary(key, binary_data, mimetype) + return self.new(key, encoded_data=binary_data, content_type=mimetype) + + def new_binary_from_file(self, key, filename): + deprecated('RiakBucket.new_binary_from_file is deprecated, use ' + 'RiakBucket.new_from_file') + return self.new_from_file(key, filename) def search_enabled(self): """ diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index c442ed2b..82d18bf3 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -106,9 +106,10 @@ def test_binary_store_and_get(self): bucket = self.client.bucket(self.bucket_name) # Store as binary, retrieve as binary, then compare... rand = str(self.randint()) - obj = bucket.new_binary(self.key_name, rand) + obj = bucket.new(self.key_name, encoded_data=rand, + content_type='text/plain') obj.store() - obj = bucket.get_binary(self.key_name) + obj = bucket.get(self.key_name) self.assertTrue(obj.exists) self.assertEqual(obj.encoded_data, rand) # Store as JSON, retrieve as binary, JSON-decode, then compare... @@ -116,16 +117,16 @@ def test_binary_store_and_get(self): key2 = self.randname() obj = bucket.new(key2, data) obj.store() - obj = bucket.get_binary(key2) + obj = bucket.get(key2) self.assertEqual(data, json.loads(obj.encoded_data)) def test_blank_binary_204(self): bucket = self.client.bucket(self.bucket_name) # this should *not* raise an error - obj = bucket.new_binary('foo2', '') + obj = bucket.new('foo2', encoded_data='', content_type='text/plain') obj.store() - obj = bucket.get_binary('foo2') + obj = bucket.get('foo2') self.assertTrue(obj.exists) self.assertEqual(obj.encoded_data, '') @@ -213,12 +214,13 @@ def test_if_none_match(self): def test_siblings(self): # Set up the bucket, clear any existing object... bucket = self.client.bucket(self.sibs_bucket) - obj = bucket.get_binary(self.key_name) + obj = bucket.get(self.key_name) bucket.allow_mult = True # Even if it previously existed, let's store a base resolved version # from which we can diverge by sending a stale vclock. - obj.data = 'start' + obj.encoded_data = 'start' + obj.content_type = 'application/octet-stream' obj.store() # Store the same object five times... @@ -228,10 +230,12 @@ def test_siblings(self): other_bucket = other_client.bucket(self.sibs_bucket) while True: randval = self.randint() - if randval not in vals: + if str(randval) not in vals: break - other_obj = other_bucket.new_binary(self.key_name, str(randval)) + other_obj = other_bucket.new(key=self.key_name, + encoded_data=str(randval), + content_type='text/plain') other_obj.vclock = obj.vclock other_obj.store() vals.add(str(randval)) @@ -261,15 +265,17 @@ def test_store_of_missing_object(self): o = bucket.get(self.key_name) self.assertEqual(o.exists, False) o.data = {"foo": "bar"} + o.content_type = 'application/json' o = o.store() self.assertEqual(o.data, {"foo": "bar"}) self.assertEqual(o.content_type, "application/json") o.delete() # for binary objects - o = bucket.get_binary(self.randname()) + o = bucket.get(self.randname()) self.assertEqual(o.exists, False) o.encoded_data = "1234567890" + o.content_type = 'application/octet-stream' o = o.store() self.assertEqual(o.encoded_data, "1234567890") @@ -373,9 +379,9 @@ class KVFileTests(object): def test_store_binary_object_from_file(self): bucket = self.client.bucket(self.bucket_name) filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') - obj = bucket.new_binary_from_file(self.key_name, filepath) + obj = bucket.new_from_file(self.key_name, filepath) obj.store() - obj = bucket.get_binary(self.key_name) + obj = bucket.get(self.key_name) self.assertNotEqual(obj.encoded_data, None) self.assertEqual(obj.content_type, "text/x-python") @@ -383,14 +389,15 @@ def test_store_binary_object_from_file_should_use_default_mimetype(self): bucket = self.client.bucket(self.bucket_name) filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir, 'THANKS') - obj = bucket.new_binary_from_file(self.key_name, filepath) + obj = bucket.new_from_file(self.key_name, filepath) obj.store() - obj = bucket.get_binary(self.key_name) + obj = bucket.get(self.key_name) self.assertEqual(obj.content_type, 'application/octet-stream') def test_store_binary_object_from_file_should_fail_if_file_not_found(self): bucket = self.client.bucket(self.bucket_name) - self.assertRaises(IOError, bucket.new_binary_from_file, - 'not_found_from_file', 'FILE_NOT_FOUND') - obj = bucket.get_binary('not_found_from_file') - self.assertEqual(obj.data, None) + with self.assertRaises(IOError): + bucket.new_from_file('not_found_from_file', 'FILE_NOT_FOUND') + obj = bucket.get('not_found_from_file') + self.assertEqual(obj.encoded_data, None) + self.assertFalse(obj.exists) diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 4531979f..5be80c59 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -8,7 +8,8 @@ class LinkTests(object): def test_store_and_get_links(self): # Create the object... bucket = self.client.bucket(self.bucket_name) - bucket.new_binary("test_store_and_get_links", '2') \ + bucket.new(key="test_store_and_get_links", encoded_data='2', + content_type='application/octet-stream') \ .add_link(bucket.new("foo1")) \ .add_link(bucket.new("foo2"), "tag") \ .add_link(bucket.new("foo3"), "tag2!@#%^&*)") \ @@ -325,8 +326,10 @@ class MapReduceAliasTests(object): def test_map_values(self): # Add a value to the bucket bucket = self.client.bucket(self.bucket_name) - bucket.new_binary('one', data='value_1').store() - bucket.new_binary('two', data='value_2').store() + bucket.new('one', encoded_data='value_1', + content_type='text/plain').store() + bucket.new('two', encoded_data='value_2', + content_type='text/plain').store() # Create a map reduce object and use one and two as inputs mr = self.client.add(self.bucket_name, 'one')\ From f6189cdf7372f9ffa67a4509b688db6196c5873d Mon Sep 17 00:00:00 2001 From: Evan Vigil-McClanahan Date: Mon, 25 Feb 2013 12:45:06 -0600 Subject: [PATCH 0354/1060] Flatten and reapply #202 on top of master. --- riak/__init__.py | 2 +- riak/metadata.py | 29 ---- riak/riak_object.py | 237 +++--------------------------- riak/tests/test_2i.py | 88 ++++++----- riak/tests/test_all.py | 2 +- riak/tests/test_kv.py | 21 ++- riak/tests/test_mapreduce.py | 24 +-- riak/transports/http/transport.py | 102 ++++++------- riak/transports/pbc/codec.py | 118 +++++++-------- riak/transports/pbc/transport.py | 48 +++--- 10 files changed, 221 insertions(+), 450 deletions(-) delete mode 100644 riak/metadata.py diff --git a/riak/__init__.py b/riak/__init__.py index b8e72bd1..ba50fc00 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -30,7 +30,7 @@ @author Jay Baird (@skatterbean) (jay@mochimedia.com) """ -__all__ = ['RiakClient', 'RiakBucket', 'RiakNode', 'RiakObject', +__all__ = ['RiakBucket', 'RiakNode', 'RiakObject', 'RiakClient', 'RiakMapReduce', 'RiakKeyFilter', 'RiakLink', 'RiakError', 'ONE', 'ALL', 'QUORUM', 'key_filter'] diff --git a/riak/metadata.py b/riak/metadata.py deleted file mode 100644 index b92353d3..00000000 --- a/riak/metadata.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" -MD_CTYPE = "content-type" -MD_CHARSET = "charset" -MD_ENCODING = "content-encoding" -MD_VTAG = "vtag" -MD_LINKS = "links" -MD_LASTMOD = "lastmod" -MD_LASTMOD_USECS = "lastmod-usecs" -MD_USERMETA = "usermeta" -MD_INDEX = "index" -MD_DELETED = "deleted" diff --git a/riak/riak_object.py b/riak/riak_object.py index bdc73797..95283fe4 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -56,7 +56,11 @@ def __init__(self, client, bucket, key=None): self._data = None self._encoded_data = None self.vclock = None - self.metadata = {MD_USERMETA: {}, MD_INDEX: []} + self.charset = None + self.content_type = 'application/json' + self.content_encoding = None + self.usermeta = {} + self.indexes = set() self.links = [] self.siblings = [] self.exists = False @@ -141,26 +145,6 @@ def _deserialize(self, value): raise TypeError('No decoder for content type "{0}"'. format(self.content_type)) - def _get_usermeta(self): - if MD_USERMETA in self.metadata: - return self.metadata[MD_USERMETA] - else: - return {} - - def _set_usermeta(self, usermeta): - self.metadata[MD_USERMETA] = usermeta - return self - - usermeta = property(_get_usermeta, _set_usermeta, - doc=""" - The custom user metadata on this object. This doesn't - include things like content type and links, but only - user-defined meta attributes stored with the Riak object. - - :param userdata: The user metadata to store. - :type userdata: dict - """) - def add_index(self, field, value): """ Tag this object with the specified field/value pair for @@ -176,9 +160,7 @@ def add_index(self, field, value): raise RiakError("Riak 2i fields must end with either '_bin'" " or '_int'.") - rie = (field, value) - if not rie in self.metadata[MD_INDEX]: - self.metadata[MD_INDEX].append(rie) + self.indexes.add((field, value)) return self @@ -194,117 +176,20 @@ def remove_index(self, field=None, value=None): :rtype: RiakObject """ if not field and not value: - ries = self.metadata[MD_INDEX][:] + self.indexes.clear() elif field and not value: - ries = [x for x in self.metadata[MD_INDEX] - if x[0] == field] + for index in [x for x in self.indexes if x[0] == field]: + self.indexes.remove(index) elif field and value: - ries = [(field, value)] + self.indexes.remove((field, value)) else: raise RiakError("Cannot pass value without a field" " name while removing index") - # This removes the index entries that's in the ries list. - # Done because this is preferred over metadata[MD_INDEX].remove(rie) - self.metadata[MD_INDEX] = [rie for rie in self.metadata[MD_INDEX] - if rie not in ries] return self remove_indexes = remove_index - def set_indexes(self, indexes): - """ - Replaces all indexes on a Riak object. Currently supports an - iterable of 2 item tuples, (field, value). - - :param indexes: iterable of 2 item tuples consisting the field - and value. Both the field and the value must - be a string. - :rtype: RiakObject - """ - # makes a copy and does type conversion - # this seems rather slow - self.metadata[MD_INDEX] = indexes[:] - return self - - def get_indexes(self, field=None): - """ - Get a list of the index entries for this object. If a field is - provided, returns a list - - :param field: The index field. - :type field: string or None - :rtype: (array of 2 element tuples with field, value) or - (array of string or integer) - """ - if field is None: - return self.metadata[MD_INDEX] - else: - return [v for f, v in self.metadata[MD_INDEX] if f == field] - - def _get_content_type(self): - try: - return self.metadata[MD_CTYPE] - except KeyError: - if self._data: - return "application/json" - else: - return "application/octet-stream" - - def _set_content_type(self, content_type): - """ - Set the content type of this object. - - :param content_type: The new content type. - :type content_type: string - :rtype: self - """ - self.metadata[MD_CTYPE] = content_type - return self - - content_type = property(_get_content_type, _set_content_type, - doc=""" - The content type of this object. This is either - ``application/json``, or the provided content type if the - object was created via :func:`RiakBucket.new_binary - `. - - :rtype: string """) - - def set_links(self, links, all_link=False): - """ - Replaces all links to a RiakObject - - :param links: An iterable of 2-item tuples, consisting of - (RiakObject, tag). This could also be an iterable of just - a RiakObject, instead of the tuple, then a tag of None - would be used. Lastly, it could also be an iterable of - 3 item tuples with the format of (bucket, key, tag), where tag - could be None - - :param all_link: A boolean indicates if links are all 3 item tuples - objects This speeds up the operation so there is no iterating - through and parsing elements. - """ - if all_link: - self.metadata[MD_LINKS] = links - return self - - new_links = [] - for item in links: - if isinstance(item, tuple): - if len(item) == 3: - link = item - elif len(item) == 2: - link = (item[0].bucket.name, item[0].key, item[1]) - elif isinstance(item, RiakObject): - link = (item.bucket.name, item.key, None) - - new_links.append(link) - - self.metadata[MD_LINKS] = new_links - return self - def add_link(self, obj, tag=None): """ Add a link to a RiakObject. @@ -322,45 +207,9 @@ def add_link(self, obj, tag=None): else: newlink = (obj.bucket.name, obj.key, tag) - self.remove_link(newlink) - links = self.metadata[MD_LINKS] - links.append(newlink) - return self - - def remove_link(self, obj, tag=None): - """ - Remove a link to a RiakObject. - - :param obj: Either a RiakObject or 3 item link tuple consisting - of (bucket, key, tag). - :type obj: mixed - :param tag: Optional link tag. Defaults to bucket name. It is ignored - if ``obj`` is a 3 item link tuple. - :type tag: string - :rtype: RiakObject - """ - if isinstance(obj, tuple): - oldlink = obj - else: - oldlink = (obj.bucket.name, obj.key, tag) - - a = [] - links = self.metadata.get(MD_LINKS, []) - for link in links: - if not link == oldlink: - a.append(link) - - self.metadata[MD_LINKS] = a + self.links.append(newlink) return self - def get_links(self): - """ - Return an array of 3 item link tuples. - - :rtype: list - """ - return self.metadata.get(MD_LINKS, []) - def store(self, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """ @@ -392,14 +241,11 @@ def store(self, w=None, dw=None, pw=None, return_body=True, "store one of the siblings instead") if self.key is None: - key, vclock, metadata = self.client.put_new( + result = self.client.put_new( self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) - self.exists = True - self.key = key - self.vclock = vclock - self.metadata = metadata + self._populate(result) else: result = self.client.put(self, w=w, dw=dw, pw=pw, return_body=return_body, @@ -422,10 +268,10 @@ def reload(self, r=None, pr=None, vtag=None): """ result = self.client.get(self, r=r, pr=pr, vtag=vtag) - - self.clear() - if result is not None and result != ('', []): + if result and result != ('', []): self._populate(result) + else: + self.clear() return self @@ -483,30 +329,11 @@ def _populate(self, result): If a list of vtags is returned there are multiple sibling that need to be retrieved with get. """ - self.clear() - if result is None: + if result is None or result is self: return self - elif type(result) is list: - self._set_siblings(result) - elif type(result) is tuple: - (vclock, contents) = result - self.vclock = vclock - if len(contents) > 0: - (metadata, data) = contents.pop(0) - self.exists = True - if not MD_INDEX in metadata: - metadata[MD_INDEX] = [] - self.metadata = metadata - self._encoded_data = data - # Create objects for all siblings - siblings = [self] - for (metadata, data) in contents: - sibling = copy.copy(self) - sibling.metadata = metadata - sibling.encoded_data = data - siblings.append(sibling) - for sibling in siblings: - sibling._set_siblings(siblings) + elif type(result) is RiakObject: + self.clear() + self.__dict__ = result.__dict__.copy() else: raise RiakError("do not know how to handle type %s" % type(result)) @@ -531,31 +358,9 @@ def get_sibling(self, i, r=None, pr=None): # And make sure it knows who its siblings are self.siblings[i] = obj - obj._set_siblings(self.siblings) + obj.siblings = self.siblings return obj - def _set_siblings(self, siblings): - """ - Set the array of siblings - used internally - - .. warning:: - - Make sure this object is at index 0 so get_siblings(0) - always returns the current object - """ - try: - i = siblings.index(self) - if i != 0: - siblings.pop(i) - siblings.insert(0, self) - except ValueError: - pass - - if len(siblings) > 1: - self.siblings = siblings - else: - self.siblings = [] - def add(self, *args): """ Start assembling a Map/Reduce operation. diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index a525054e..724d4baf 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -37,8 +37,10 @@ def test_secondary_index_store(self): # Retrieve the object, check that the correct indexes exist... obj = bucket.get('mykey1') - self.assertEqual(['val1a'], sorted(obj.get_indexes('field1_bin'))) - self.assertEqual([1011], sorted(obj.get_indexes('field1_int'))) + self.assertEqual(['val1a'], [y for (x, y) in obj.indexes + if x == 'field1_bin']) + self.assertEqual([1011], [y for (x, y) in obj.indexes + if x == 'field1_int']) # Add more indexes and save... obj.add_index('field1_bin', 'val1b') @@ -48,17 +50,18 @@ def test_secondary_index_store(self): # Retrieve the object, check that the correct indexes exist... obj = bucket.get('mykey1') self.assertEqual(['val1a', 'val1b'], - sorted(obj.get_indexes('field1_bin'))) + sorted([y for (x, y) in obj.indexes + if x == 'field1_bin'])) self.assertEqual([1011, 1012], - sorted(obj.get_indexes('field1_int'))) + sorted([y for (x, y) in obj.indexes + if x == 'field1_int'])) - # Check the get_indexes() function... - self.assertEqual([ - ('field1_bin', 'val1a'), - ('field1_bin', 'val1b'), - ('field1_int', 1011), - ('field1_int', 1012) - ], sorted(obj.get_indexes())) + self.assertEqual( + [('field1_bin', 'val1a'), + ('field1_bin', 'val1b'), + ('field1_int', 1011), + ('field1_int', 1012) + ], sorted(obj.indexes)) # Delete an index... obj.remove_index('field1_bin', 'val1a') @@ -67,8 +70,10 @@ def test_secondary_index_store(self): # Retrieve the object, check that the correct indexes exist... obj = bucket.get('mykey1') - self.assertEqual(['val1b'], sorted(obj.get_indexes('field1_bin'))) - self.assertEqual([1012], sorted(obj.get_indexes('field1_int'))) + self.assertEqual(['val1b'], sorted([y for (x, y) in obj.indexes + if x == 'field1_bin'])) + self.assertEqual([1012], sorted([y for (x, y) in obj.indexes + if x == 'field1_int'])) # Check duplicate entries... obj.add_index('field1_bin', 'val1a') @@ -78,22 +83,22 @@ def test_secondary_index_store(self): obj.add_index('field1_int', 1011) obj.add_index('field1_int', 1011) - self.assertEqual([ - ('field1_bin', 'val1a'), - ('field1_bin', 'val1b'), - ('field1_int', 1011), - ('field1_int', 1012) - ], sorted(obj.get_indexes())) + self.assertEqual( + [('field1_bin', 'val1a'), + ('field1_bin', 'val1b'), + ('field1_int', 1011), + ('field1_int', 1012) + ], sorted(obj.indexes)) obj.store() obj = bucket.get('mykey1') - self.assertEqual([ - ('field1_bin', 'val1a'), - ('field1_bin', 'val1b'), - ('field1_int', 1011), - ('field1_int', 1012) - ], sorted(obj.get_indexes())) + self.assertEqual( + [('field1_bin', 'val1a'), + ('field1_bin', 'val1b'), + ('field1_int', 1011), + ('field1_int', 1012) + ], sorted(obj.indexes)) # Clean up... bucket.get('mykey1').delete() @@ -105,8 +110,10 @@ def test_set_indexes(self): bucket = self.client.bucket(self.bucket_name) foo = bucket.new('foo', 1) - foo.set_indexes((('field1_bin', 'test'), ('field2_int', 1337))).store() + foo.indexes = set([('field1_bin', 'test'), ('field2_int', 1337)]) + foo.store() result = self.client.index(self.bucket_name, 'field2_int', 1337).run() + self.assertEqual(1, len(result)) self.assertEqual('foo', result[0][1]) @@ -124,8 +131,9 @@ def test_remove_indexes(self): .add_index('bar_int', 2).add_index('baz_bin', 'baz').store() result = bucket.get_index('bar_int', 1) self.assertEqual(1, len(result)) - self.assertEqual(3, len(bar.get_indexes())) - self.assertEqual(2, len(bar.get_indexes('bar_int'))) + self.assertEqual(3, len(bar.indexes)) + self.assertEqual(2, len([x for x in bar.indexes + if x[0] == 'bar_int'])) # remove all indexes bar = bar.remove_indexes().store() @@ -133,9 +141,11 @@ def test_remove_indexes(self): self.assertEqual(0, len(result)) result = bucket.get_index('baz_bin', 'baz') self.assertEqual(0, len(result)) - self.assertEqual(0, len(bar.get_indexes())) - self.assertEqual(0, len(bar.get_indexes('bar_int'))) - self.assertEqual(0, len(bar.get_indexes('baz_bin'))) + self.assertEqual(0, len(bar.indexes)) + self.assertEqual(0, len([x for x in bar.indexes + if x[0] == 'bar_int'])) + self.assertEqual(0, len([x for x in bar.indexes + if x[0] == 'baz_bin'])) # add index again bar = bar.add_index('bar_int', 1).add_index('bar_int', 2)\ @@ -148,9 +158,11 @@ def test_remove_indexes(self): self.assertEqual(0, len(result)) result = bucket.get_index('baz_bin', 'baz') self.assertEqual(1, len(result)) - self.assertEqual(1, len(bar.get_indexes())) - self.assertEqual(0, len(bar.get_indexes('bar_int'))) - self.assertEqual(1, len(bar.get_indexes('baz_bin'))) + self.assertEqual(1, len(bar.indexes)) + self.assertEqual(0, len([x for x in bar.indexes + if x[0] == 'bar_int'])) + self.assertEqual(1, len([x for x in bar.indexes + if x[0] == 'baz_bin'])) # add index again bar = bar.add_index('bar_int', 1).add_index('bar_int', 2)\ @@ -163,9 +175,11 @@ def test_remove_indexes(self): self.assertEqual(0, len(result)) result = bucket.get_index('baz_bin', 'baz') self.assertEqual(1, len(result)) - self.assertEqual(2, len(bar.get_indexes())) - self.assertEqual(1, len(bar.get_indexes('bar_int'))) - self.assertEqual(1, len(bar.get_indexes('baz_bin'))) + self.assertEqual(2, len(bar.indexes)) + self.assertEqual(1, len([x for x in bar.indexes + if x[0] == 'bar_int'])) + self.assertEqual(1, len([x for x in bar.indexes + if x[0] == 'baz_bin'])) @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_query(self): diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 833447fa..f9602a00 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -192,7 +192,7 @@ def test_too_many_link_headers_shouldnt_break_http(self): o.store() stored_object = bucket.get("lots_of_links") - self.assertEqual(len(stored_object.get_links()), 400) + self.assertEqual(len(stored_object.links), 400) def test_clear_bucket_properties(self): bucket = self.client.bucket(self.props_bucket) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 82d18bf3..060e4cf0 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -131,8 +131,8 @@ def test_blank_binary_204(self): self.assertEqual(obj.encoded_data, '') def test_custom_bucket_encoder_decoder(self): - # Teach the bucket how to pickle bucket = self.client.bucket(self.bucket_name) + # Teach the bucket how to pickle bucket.set_encoder('application/x-pickle', cPickle.dumps) bucket.set_decoder('application/x-pickle', cPickle.loads) data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} @@ -142,8 +142,8 @@ def test_custom_bucket_encoder_decoder(self): self.assertEqual(data, obj2.data) def test_custom_client_encoder_decoder(self): - # Teach the bucket how to pickle bucket = self.client.bucket(self.bucket_name) + # Teach the client how to pickle self.client.set_encoder('application/x-pickle', cPickle.dumps) self.client.set_decoder('application/x-pickle', cPickle.loads) data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} @@ -156,8 +156,9 @@ def test_unknown_content_type_encoder_decoder(self): # Teach the bucket how to pickle bucket = self.client.bucket(self.bucket_name) data = "some funny data" - obj = bucket.new(self.key_name, data, - 'application/x-frobnicator').store() + obj = bucket.new(self.key_name, + encoded_data=data, + content_type='application/x-frobnicator') obj.store() obj2 = bucket.get(self.key_name) self.assertEqual(data, obj2.encoded_data) @@ -205,6 +206,7 @@ def test_if_none_match(self): obj.reload() self.assertFalse(obj.exists) obj.data = ["first store"] + obj.content_type = 'application/json' obj.store() obj.data = ["second store"] @@ -224,10 +226,12 @@ def test_siblings(self): obj.store() # Store the same object five times... + # First run through should overwrite the datum 'start' above + other_client = self.create_client() + other_bucket = other_client.bucket(self.sibs_bucket) + vals = set() for i in range(5): - other_client = self.create_client() - other_bucket = other_client.bucket(self.sibs_bucket) while True: randval = self.randint() if str(randval) not in vals: @@ -241,13 +245,14 @@ def test_siblings(self): vals.add(str(randval)) # Make sure the object has itself plus four siblings... + obj = bucket.get(self.key_name) obj.reload() - #self.assertTrue(bool(obj.siblings)) + self.assertTrue(bool(obj.siblings)) self.assertEqual(len(obj.siblings), 5) # Get each of the values - make sure they match what was assigned vals2 = set() - for i in range(5): + for i in xrange(len(obj.siblings)): vals2.add(obj.get_sibling(i).encoded_data) self.assertEqual(vals, vals2) diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 5be80c59..c693a789 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -15,7 +15,7 @@ def test_store_and_get_links(self): .add_link(bucket.new("foo3"), "tag2!@#%^&*)") \ .store() obj = bucket.get("test_store_and_get_links") - links = obj.get_links() + links = obj.links self.assertEqual(len(links), 3) for bucket, key, tag in links: if (key == "foo1"): @@ -30,11 +30,13 @@ def test_store_and_get_links(self): def test_set_links(self): # Create the object bucket = self.client.bucket(self.bucket_name) - bucket.new("foo", 2).set_links([bucket.new("foo1"), - (bucket.new("foo2"), "tag"), - ("bucket", "foo2", "tag2")]).store() - obj = bucket.get("foo") - links = sorted(obj.get_links(), key=lambda x: x[1]) + o = bucket.new(self.key_name, 2) + o.links = [(self.bucket_name, "foo1", None), + (self.bucket_name, "foo2", "tag"), + ("bucket", "foo2", "tag2")] + o.store() + obj = bucket.get(self.key_name) + links = sorted(obj.links, key=lambda x: x[1]) self.assertEqual(len(links), 3) self.assertEqual(links[0][1], "foo1") self.assertEqual(links[1][1], "foo2") @@ -42,16 +44,6 @@ def test_set_links(self): self.assertEqual(links[2][1], "foo2") self.assertEqual(links[2][2], "tag2") - def test_set_links_all_links(self): - bucket = self.client.bucket(self.bucket_name) - foo1 = bucket.new("foo", 1) - bucket.new("foo2", 2).store() - links = [("bucket", "foo2", None)] - foo1.set_links(links, True) - links = foo1.get_links() - self.assertEqual(len(links), 1) - self.assertEqual(links[0][1], "foo2") - def test_link_walking(self): # Create the object... bucket = self.client.bucket(self.bucket_name) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 73459ba4..2e112215 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -34,19 +34,7 @@ from riak.transports.http.search import XMLSearchResult from riak.transports.http.stream import ( RiakHttpKeyStream, - RiakHttpMapReduceStream -) -from riak.metadata import ( - MD_CHARSET, - MD_CTYPE, - MD_ENCODING, - MD_INDEX, - MD_LASTMOD, - MD_LINKS, - MD_USERMETA, - MD_VTAG, - MD_DELETED -) + RiakHttpMapReduceStream) from riak import RiakError from riak.multidict import MultiDict from xml.etree import ElementTree @@ -132,7 +120,7 @@ def get(self, robj, r=None, pr=None, vtag=None): params = {'r': r, 'pr': pr, 'vtag': vtag} url = self.object_path(robj.bucket.name, robj.key, **params) response = self._request('GET', url) - return self.parse_body(response, [200, 300, 404]) + return self.parse_body(robj, response, [200, 300, 404]) def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): @@ -143,24 +131,23 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, # unknown flags/params. params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} url = self.object_path(robj.bucket.name, robj.key, **params) - headers = self.build_put_headers(robj) + headers = self._build_put_headers(robj) # TODO: use a more general 'prevent_stale_writes' semantics, # which is a superset of the if_none_match semantics. if if_none_match: headers["If-None-Match"] = "*" content = robj.encoded_data - return self.do_put(url, headers, content, return_body, - key=robj.key) + return self.do_put(url, headers, content, robj, return_body) - def do_put(self, url, headers, content, return_body=False, key=None): - if key is None: + def do_put(self, url, headers, content, robj, return_body=False): + if robj.key is None: response = self._request('POST', url, headers, content) else: response = self._request('PUT', url, headers, content) if return_body: - return self.parse_body(response, [200, 201, 204, 300]) + return self.parse_body(robj, response, [200, 201, 204, 300]) else: self.check_http_code(response, [204]) return None @@ -172,7 +159,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, # unknown flags/params. params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} url = self.object_path(robj.bucket.name, **params) - headers = self.build_put_headers(robj) + headers = self._build_put_headers(robj) # TODO: use a more general 'prevent_stale_writes' semantics, # which is a superset of the if_none_match semantics. if if_none_match: @@ -181,13 +168,12 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, response = self._request('POST', url, headers, content) location = response[0]['location'] idx = location.rindex('/') - key = location[(idx + 1):] + robj.key = location[(idx + 1):] if return_body: - vclock, [(metadata, data)] = self.parse_body(response, [201]) - return key, vclock, metadata + return self.parse_body(robj, response, [201]) else: self.check_http_code(response, [201]) - return key, None, None + return None def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ @@ -428,13 +414,13 @@ def check_http_code(self, response, expected_statuses): raise Exception('Expected status %s, received %s : %s' % (expected_statuses, status, response[1])) - def parse_body(self, response, expected_statuses): + def parse_body(self, robj, response, expected_statuses): """ Parse the body of an object response and populate the object. """ # If no response given, then return. if response is None: - return self + return None # Make sure expected code came back self.check_http_code(response, expected_statuses) @@ -459,28 +445,32 @@ def parse_body(self, response, expected_statuses): # Parse and get rid of 'Siblings:' string in element 0 siblings = data.strip().split('\n') siblings.pop(0) - return siblings + robj.siblings = siblings + robj.exists = True + robj.vclock = headers['x-riak-vclock'] + return robj + + #no sibs + robj.siblings = [] # Parse the headers... - vclock = None - metadata = {MD_USERMETA: {}, MD_INDEX: []} links = [] for header, value in headers.iteritems(): if header == 'content-type': - metadata[MD_CTYPE] = value + robj.content_type = value elif header == 'charset': - metadata[MD_CHARSET] = value + robj.charset = value elif header == 'content-encoding': - metadata[MD_ENCODING] = value + robj.content_encoding = value elif header == 'etag': - metadata[MD_VTAG] = value + robj.etag = value elif header == 'link': - self.parse_links(links, headers['link']) + self._parse_links(links, headers['link']) elif header == 'last-modified': - metadata[MD_LASTMOD] = value + robj.last_modified = value elif header.startswith('x-riak-meta-'): metakey = header.replace('x-riak-meta-', '') - metadata[MD_USERMETA][metakey] = value + robj.usermeta[metakey] = value elif header.startswith('x-riak-index-'): field = header.replace('x-riak-index-', '') reader = csv.reader([value], skipinitialspace=True) @@ -488,32 +478,33 @@ def parse_body(self, response, expected_statuses): for token in line: if field.endswith("_int"): token = int(token) - rie = (field, token) - metadata[MD_INDEX].append(rie) + robj.add_index(field, token) elif header == 'x-riak-vclock': - vclock = value + robj.vclock = value elif header == 'x-riak-deleted': - metadata[MD_DELETED] = True + robj.deleted = True if links: - metadata[MD_LINKS] = links + robj.links = links + + robj.set_encoded_data(data) - return vclock, [(metadata, data)] + robj.exists = True + return robj def to_link_header(self, link): """ Convert the link tuple to a link header string. Used internally. """ - bucket, key, tag = link + try: + bucket, key, tag = link + except ValueError: + raise RiakError("Invalid link tuple %s" % link) tag = tag if tag is not None else bucket url = self.object_path(bucket, key) header = '<%s>; riaktag="%s"' % (url, tag) return header - def parse_links(self, links, linkHeaders): - """ - Private. - @return self - """ + def _parse_links(self, links, linkHeaders): oldform = "; ?riaktag=\"([^\"]+)\"" newform = "; ?riaktag=\"([^\"]+)\"" for linkHeader in linkHeaders.strip().split(','): @@ -525,10 +516,10 @@ def parse_links(self, links, linkHeaders): urllib.unquote_plus(matches.group(3)), urllib.unquote_plus(matches.group(4))) links.append(link) - return self + return links - def add_links_for_riak_object(self, robject, headers): - links = robject.get_links() + def _add_links_for_riak_object(self, robject, headers): + links = robject.links if links: current_header = '' for link in links: @@ -547,25 +538,24 @@ def add_links_for_riak_object(self, robject, headers): # Utility functions used by Riak library. - def build_put_headers(self, robj): + def _build_put_headers(self, robj): """Build the headers for a POST/PUT request.""" # Construct the headers... headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', 'Content-Type': robj.content_type, 'X-Riak-ClientId': self._client_id}) - # Add the vclock if it exists... if robj.vclock is not None: headers['X-Riak-Vclock'] = robj.vclock # Create the header from metadata - self.add_links_for_riak_object(robj, headers) + self._add_links_for_riak_object(robj, headers) for key, value in robj.usermeta.iteritems(): headers['X-Riak-Meta-%s' % key] = value - for field, value in robj.get_indexes(): + for field, value in robj.indexes: key = 'X-Riak-Index-%s' % field if key in headers: headers[key] += ", " + str(value) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index d8b3549f..a28da47c 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -15,20 +15,8 @@ specific language governing permissions and limitations under the License. """ -from riak.metadata import ( - MD_CHARSET, - MD_CTYPE, - MD_ENCODING, - MD_INDEX, - MD_LASTMOD, - MD_LASTMOD_USECS, - MD_LINKS, - MD_USERMETA, - MD_VTAG, - MD_DELETED -) - import riak_pb +from riak.riak_object import RiakObject RIAKC_RW_ONE = 4294967294 RIAKC_RW_QUORUM = 4294967293 @@ -70,31 +58,24 @@ def translate_rw_val(self, rw): else: return None - def decode_contents(self, rpb_contents): - """ - Decodes the multiple contents (siblings) of a RiakObject from - its protobuf representation. - """ - return [self.decode_content(rpb_c) for rpb_c in rpb_contents] - - def decode_content(self, rpb_content): + def decode_content(self, rpb_content, robj): """ Decodes a single sibling from the protobuf representation into - its metadata and value. + a RiakObject. - :rtype: (dict, string) + :rtype: (RiakObject) """ - metadata = {} + if rpb_content.HasField("deleted"): - metadata[MD_DELETED] = True + robj.deleted = True if rpb_content.HasField("content_type"): - metadata[MD_CTYPE] = rpb_content.content_type + robj.content_type = rpb_content.content_type if rpb_content.HasField("charset"): - metadata[MD_CHARSET] = rpb_content.charset + robj.charset = rpb_content.charset if rpb_content.HasField("content_encoding"): - metadata[MD_ENCODING] = rpb_content.content_encoding + robj.content_encoding = rpb_content.content_encoding if rpb_content.HasField("vtag"): - metadata[MD_VTAG] = rpb_content.vtag + robj.vtag = rpb_content.vtag links = [] for link in rpb_content.links: if link.HasField("bucket"): @@ -111,58 +92,63 @@ def decode_content(self, rpb_content): tag = None links.append((bucket, key, tag)) if links: - metadata[MD_LINKS] = links + robj.links = links if rpb_content.HasField("last_mod"): - metadata[MD_LASTMOD] = rpb_content.last_mod + robj.last_mod = rpb_content.last_mod if rpb_content.HasField("last_mod_usecs"): - metadata[MD_LASTMOD_USECS] = rpb_content.last_mod_usecs + robj.last_mod_usecs = rpb_content.last_mod_usecs usermeta = {} for usermd in rpb_content.usermeta: usermeta[usermd.key] = usermd.value if len(usermeta) > 0: - metadata[MD_USERMETA] = usermeta - indexes = [] + robj.usermeta = usermeta + indexes = set() for index in rpb_content.indexes: if index.key.endswith("_int"): - value = int(index.value) + indexes.add((index.key, int(index.value))) else: - value = index.value - rie = (index.key, value) - indexes.append(rie) + indexes.add((index.key, index.value)) + if len(indexes) > 0: - metadata[MD_INDEX] = indexes - return metadata, rpb_content.value + robj.indexes = indexes - def encode_content(self, metadata, data, rpb_content): + robj.set_encoded_data(rpb_content.value) + robj.exists = True + + return robj + + def encode_content(self, robj, rpb_content): """ Fills an RpbContent message with the appropriate data and metadata from a RiakObject. """ - # Convert the broken out fields, building up - # pbmetadata for any unknown ones - for k in metadata: - v = metadata[k] - if k == MD_CTYPE: - rpb_content.content_type = v - elif k == MD_CHARSET: - rpb_content.charset = v - elif k == MD_ENCODING: - rpb_content.content_encoding = v - elif k == MD_USERMETA: - for uk in v: - pair = rpb_content.usermeta.add() - pair.key = uk - pair.value = v[uk] - elif k == MD_INDEX: - for field, value in v: + if robj.content_type: + rpb_content.content_type = robj.content_type + if robj.charset: + rpb_content.charset = robj.charset + if robj.content_encoding: + rpb_content.content_encoding = robj.content_encoding + for uk in robj.usermeta: + pair = rpb_content.usermeta.add() + pair.key = uk + pair.value = robj.usermeta[uk] + for link in robj.links: + pb_link = rpb_content.links.add() + try: + bucket, key, tag = link + except ValueError: + raise RiakError("Invalid link tuple %s" % link) + + pb_link.bucket = bucket + pb_link.key = key + if tag: + pb_link.tag = tag + else: + pb_link.tag = '' + + for field, value in robj.indexes: pair = rpb_content.indexes.add() pair.key = field pair.value = str(value) - elif k == MD_LINKS: - for bucket, key, tag in v: - tag = tag if tag is not None else bucket - pb_link = rpb_content.links.add() - pb_link.bucket = bucket - pb_link.key = key - pb_link.tag = tag - rpb_content.value = str(data) + + rpb_content.value = str(robj.encoded_data) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 61623cdf..9b695c69 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -25,6 +25,8 @@ from connection import RiakPbcConnection from stream import RiakPbcKeyStream, RiakPbcMapredStream from codec import RiakPbcCodec +from riak.riak_object import RiakObject + from messages import ( MSG_CODE_PING_REQ, MSG_CODE_PING_RESP, @@ -115,6 +117,20 @@ def _set_client_id(self, client_id): client_id = property(_get_client_id, _set_client_id, doc="""the client ID for this connection""") + def _decoded_contents(self, resp, old_obj): + contents = [] + for c in resp.content: + new_obj = RiakObject(old_obj.client, old_obj.bucket, old_obj.key) + new_obj.vclock = resp.vclock + contents.append(self.decode_content(c, new_obj)) + if contents: + ret = contents[0] + if len(contents) > 1: + ret.siblings = contents[:] + return ret + else: + return old_obj + def get(self, robj, r=None, pr=None, vtag=None): """ Serialize get request and deserialize response @@ -138,10 +154,7 @@ def get(self, robj, r=None, pr=None, vtag=None): msg_code, resp = self._request(MSG_CODE_GET_REQ, req) if msg_code == MSG_CODE_GET_RESP: - contents = [] - for c in resp.content: - contents.append(self.decode_content(c)) - return resp.vclock, contents + return self._decoded_contents(resp, robj) else: return None @@ -167,21 +180,16 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, req.bucket = bucket.name req.key = robj.key - vclock = robj.vclock - if vclock: - req.vclock = vclock + if robj.vclock: + req.vclock = robj.vclock - self.encode_content(robj.metadata, - robj.encoded_data, - req.content) + self.encode_content(robj, req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, MSG_CODE_PUT_RESP) + contents = [] if resp is not None: - contents = [] - for c in resp.content: - contents.append(self.decode_content(c)) - return resp.vclock, contents + return self._decoded_contents(resp, robj) def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): @@ -190,7 +198,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, If return_meta is False, then the vlock and metadata return values will be None. - @return (key, vclock, metadata) + @return robj """ # Note that this won't work on 0.14 nodes. bucket = robj.bucket @@ -210,9 +218,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, req.bucket = bucket.name - self.encode_content(robj.metadata, - robj.encoded_data, - req.content) + self.encode_content(robj, req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, MSG_CODE_PUT_RESP) @@ -221,8 +227,10 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if len(resp.content) != 1: raise RiakError("siblings were returned from object creation") - metadata, content = self.decode_content(resp.content[0]) - return resp.key, resp.vclock, metadata + robj.key = resp.key + robj.vclock = resp.vclock + content = self.decode_content(resp.content[0], robj) + return content def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ From 974fa6caa7255668e3274e71568140d6ed82a2b3 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 25 Feb 2013 13:08:51 -0600 Subject: [PATCH 0355/1060] Fix a few more oversights. --- riak/riak_object.py | 5 ----- riak/transports/http/transport.py | 2 +- riak/transports/pbc/codec.py | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 95283fe4..d1bb0ffb 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -18,11 +18,6 @@ under the License. """ import copy -from riak.metadata import ( - MD_CTYPE, - MD_INDEX, - MD_LINKS, - MD_USERMETA) from riak import RiakError from riak.util import deprecated diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 2e112215..f9d86db8 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -486,7 +486,7 @@ def parse_body(self, robj, response, expected_statuses): if links: robj.links = links - robj.set_encoded_data(data) + robj.encoded_data = data robj.exists = True return robj diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index a28da47c..d41d8ed6 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -112,7 +112,7 @@ def decode_content(self, rpb_content, robj): if len(indexes) > 0: robj.indexes = indexes - robj.set_encoded_data(rpb_content.value) + robj.encoded_data = rpb_content.value robj.exists = True return robj From 89b62ab0bcbb507a5e0d6d8df760fe8b89274868 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 25 Feb 2013 13:14:33 -0600 Subject: [PATCH 0356/1060] Fix nq typo. --- riak/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index e26912c0..acfbfb46 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -260,7 +260,7 @@ def __eq__(self, other): else: return False - def __nq__(self, other): + def __ne__(self, other): if isinstance(other, self.__class__): return hash(self) != hash(other) else: From 79f916f045844fb6a99cc447009876166d5f9cf4 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 25 Feb 2013 13:23:15 -0600 Subject: [PATCH 0357/1060] Fix comparison test for client objects. --- riak/tests/test_comparison.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/tests/test_comparison.py b/riak/tests/test_comparison.py index 3e9b3aa2..e6f888c9 100644 --- a/riak/tests/test_comparison.py +++ b/riak/tests/test_comparison.py @@ -60,7 +60,7 @@ def test_client_eq(self): def test_client_nq(self): self.protocol = 'http' a = self.create_client(host='host1', http_port=11) - b = self.create_client(host='host1', http_port=11) + b = self.create_client(host='host2', http_port=11) c = self.create_client(host='host1', http_port=12) self.assertNotEqual(a, b, 'matched with different hosts') self.assertNotEqual(a, c, 'matched with different ports') From e5d990ef8f95532518de1b31ceb0e85d5c044071 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 25 Feb 2013 13:35:59 -0600 Subject: [PATCH 0358/1060] Documents returning from Riak Search always use UTF-8. See also basho/riak-ruby-client#75. --- riak/transports/pbc/transport.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 61623cdf..335e2b90 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -410,7 +410,9 @@ def search(self, index, query, **params): for doc in resp.docs: resultdoc = {} for pair in doc.fields: - resultdoc[pair.key] = pair.value + ukey = unicode(pair.key, 'utf-8') + uval = unicode(pair.value, 'utf-8') + resultdoc[ukey] = uval docs.append(resultdoc) result['docs'] = docs return result From 97aba46c372b23c241395590ebcf22ed5088d3b5 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 25 Feb 2013 13:54:54 -0600 Subject: [PATCH 0359/1060] PEP8 and pyflakes fixes. --- riak/tests/pool-grinder.py | 4 ++-- riak/transports/http/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/riak/tests/pool-grinder.py b/riak/tests/pool-grinder.py index 6765363e..a7d73c83 100755 --- a/riak/tests/pool-grinder.py +++ b/riak/tests/pool-grinder.py @@ -1,10 +1,10 @@ #!/usr/bin/env python from Queue import Queue -from threading import Thread, currentThread +from threading import Thread #, currentThread import sys sys.path.append("../transports/") -from pool import Pool, BadResource +from pool import Pool #, BadResource from random import SystemRandom from time import sleep diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index 003fc61c..50ed56bb 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -52,7 +52,7 @@ def destroy_resource(self, transport): httplib.IncompleteRead, httplib.ImproperConnectionState, httplib.BadStatusLine - ) +) def is_retryable(err): From 0921cc0712d7a3e67b56e905c2a889f4cbb98ae6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 25 Feb 2013 13:55:47 -0600 Subject: [PATCH 0360/1060] Disallow 'None' as a key on fetch. Closes #124. --- riak/client/operations.py | 4 ++++ riak/tests/test_kv.py | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/riak/client/operations.py b/riak/client/operations.py index 9ce91dc2..516b2d26 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -193,6 +193,10 @@ def get(self, transport, robj, r=None, pr=None, vtag=None): :param vtag: the specific sibling to fetch :type vtag: string """ + if not isinstance(robj.key, basestring): + raise TypeError( + 'key must be a string, instead got {0}'.format(repr(robj.key))) + return transport.get(robj, r=r, pr=pr, vtag=vtag) @retryable diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 82d18bf3..97916063 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -102,6 +102,18 @@ def test_stream_keys_abort(self): robj = bucket.get(regular_keys[0]) self.assertEqual(True, robj.exists) + def test_bad_key(self): + bucket = self.client.bucket(self.bucket_name) + obj = bucket.new() + with self.assertRaises(TypeError): + bucket.get(None) + + with self.assertRaises(TypeError): + self.client.get(obj) + + with self.assertRaises(TypeError): + bucket.get(1) + def test_binary_store_and_get(self): bucket = self.client.bucket(self.bucket_name) # Store as binary, retrieve as binary, then compare... From 522527fdc0bacd002a2a207f9d99603ee868f838 Mon Sep 17 00:00:00 2001 From: Kenji Rikitake Date: Tue, 12 Mar 2013 14:34:36 +0900 Subject: [PATCH 0361/1060] Changed 'localhost' to '127.0.0.1' in riak/tests/test_all.py --- riak/tests/test_all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index f9602a00..8708165e 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -30,7 +30,7 @@ except ImportError: HAVE_PROTO = False -HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') +HOST = os.environ.get('RIAK_TEST_HOST', '127.0.0.1') PB_HOST = os.environ.get('RIAK_TEST_PB_HOST', HOST) PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) From 3952ce4d48c3c4bc124d9d3b688cef59104b5ed2 Mon Sep 17 00:00:00 2001 From: Kenji Rikitake Date: Tue, 12 Mar 2013 15:33:37 +0900 Subject: [PATCH 0362/1060] Update riak/tests/test_all.py * Explicitly set http=HTTP_HOST to riak.RiakClient() So that setUpModule() and tearDownModule() will succeed to non-localhost addresses when running `python setup.py test` --- riak/tests/test_all.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index f9602a00..f902e3b5 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -57,7 +57,7 @@ def setUpModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket - c = RiakClient(transport='http', http_port=HTTP_PORT) + c = RiakClient(transport='http', host=HTTP_HOST, http_port=HTTP_PORT) testrun_props_bucket = 'propsbucket' testrun_sibs_bucket = 'sibsbucket' @@ -70,7 +70,7 @@ def setUpModule(): def tearDownModule(): - c = RiakClient(transport='http', http_port=HTTP_PORT) + c = RiakClient(transport='http', host=HTTP_HOST, http_port=HTTP_PORT) if not int(os.environ.get('SKIP_SEARCH', '0')): b = c.bucket(testrun_search_bucket) b.clear_properties() From 5424f0f88e58524a2ea23f116aebaa341ce767b2 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 12 Mar 2013 16:41:59 -0500 Subject: [PATCH 0363/1060] Innocuous addition of newline --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 78787881..27ec95ac 100644 --- a/README.rst +++ b/README.rst @@ -586,3 +586,4 @@ suites or in subsequent test runs, be sure to call cleanup() before starting or after stopping it. .. _Ripple: https://github.com/seancribbs/ripple + From 97fe3a580fc9d92580cc269566e0e05b83fba219 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 12 Mar 2013 16:44:06 -0500 Subject: [PATCH 0364/1060] Removed annoying newline --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index 27ec95ac..78787881 100644 --- a/README.rst +++ b/README.rst @@ -586,4 +586,3 @@ suites or in subsequent test runs, be sure to call cleanup() before starting or after stopping it. .. _Ripple: https://github.com/seancribbs/ripple - From bfb638c4ec4abd54dc9e4fa4478a603b910dc1ed Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Thu, 14 Mar 2013 16:37:31 -0500 Subject: [PATCH 0365/1060] Issue #150: Removed connection and request timeouts with a single timeout parameter --- riak/transports/pbc/connection.py | 8 +++++--- riak/transports/pbc/transport.py | 6 ++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index 53916579..86e409b4 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -82,8 +82,10 @@ def _recv_pkt(self): % (len(self._inbuf), self._inbuf_len)) def _connect(self): - self._socket = socket.create_connection(self._address, - self._timeouts['connect']) + if self._timeout: + self._socket = socket.create_connection(self._address, self._timeout) + else: + self._socket = socket.create_connection(self._address) def close(self): """ @@ -106,4 +108,4 @@ def _parse_msg(self, code, packet): # These are set in the RiakPbcTransport initializer _address = None - _timeouts = {} + _timeout = None diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 94c5dff9..72be6a8c 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -63,8 +63,7 @@ class RiakPbcTransport(RiakTransport, RiakPbcConnection, RiakPbcCodec): buffers interface on the riak server. """ - def __init__(self, node=None, client=None, connect_timeout=None, - request_timeout=None, **unused_options): + def __init__(self, node=None, client=None, timeout=None, *unused_options): """ Construct a new RiakPbcTransport object. """ @@ -73,8 +72,7 @@ def __init__(self, node=None, client=None, connect_timeout=None, self._client = client self._node = node self._address = (node.host, node.pb_port) - self._timeouts = {'connect': connect_timeout, - 'request': request_timeout} + self._timeout = timeout self._connect() # FeatureDetection API From 9dd1594f928ef9eb0140a520b88b85b78892da6d Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 15 Mar 2013 11:31:48 -0500 Subject: [PATCH 0366/1060] Issue #150: Added comments to release notes about timeout flag for PBC --- RELEASE_NOTES.md | 5 +++++ riak/client/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index b229baa0..fb5a56d5 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,10 @@ # Riak Python Client Release Notes +## 1.5.2 Patch Release + +* Added optional `timeout` parameter to `transport_options` dictionary + when creating a RiakClient object with Protocol Buffers. + ## 1.5.1 Patch Release - 2012-10-24 Release 1.5.1 fixes one bug and some documentation errors. diff --git a/riak/client/__init__.py b/riak/client/__init__.py index acfbfb46..2dea7a62 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -68,7 +68,7 @@ def __init__(self, protocol='http', transport_options={}, 'host', 'http_port', and 'pb_port' :type nodes: list :param transport_options: Optional key-value args to pass to - the transport constuctor + the transport constructor :type transport_options: dict """ unused_args = unused_args.copy() From 9b96b6d03f9395a23c2e4abfad204bcade1f39fd Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 26 Mar 2013 10:03:20 -0500 Subject: [PATCH 0367/1060] Issue #228: Do not allow the creation of an empty RiakObject key --- riak/riak_object.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/riak/riak_object.py b/riak/riak_object.py index d1bb0ffb..7c3d04b6 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -45,6 +45,9 @@ def __init__(self, client, bucket, key=None): except UnicodeError: raise TypeError('Unicode keys are not supported.') + if key!=None and len(key) == 0: + raise ValueError('Key name must either be "None" or a non-empty string.') + self.client = client self.bucket = bucket self.key = key From 0d24f80d1b6e96d0250b4775ad3ab952e6237e4d Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 26 Mar 2013 11:08:57 -0500 Subject: [PATCH 0368/1060] Issue #228/#231: Clean up key comparison to None value --- riak/riak_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 7c3d04b6..944a9203 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -45,7 +45,7 @@ def __init__(self, client, bucket, key=None): except UnicodeError: raise TypeError('Unicode keys are not supported.') - if key!=None and len(key) == 0: + if key is not None and len(key) == 0: raise ValueError('Key name must either be "None" or a non-empty string.') self.client = client From cca6fdd982391547cea5da3e65074009b7e7f4aa Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 26 Mar 2013 11:33:53 -0500 Subject: [PATCH 0369/1060] Issue #228/#231: Clean up PEP8 warning on line length --- riak/riak_object.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 944a9203..e97b4942 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -46,7 +46,8 @@ def __init__(self, client, bucket, key=None): raise TypeError('Unicode keys are not supported.') if key is not None and len(key) == 0: - raise ValueError('Key name must either be "None" or a non-empty string.') + raise ValueError('Key name must either be "None"' + ' or a non-empty string.') self.client = client self.bucket = bucket From 48d4068ffbc22a2ab2c1d8876139ba2a7111341e Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 26 Mar 2013 12:12:14 -0500 Subject: [PATCH 0370/1060] Issue #228/#231: Removed unused "import copy" which pyflakes complained about --- riak/riak_object.py | 1 - 1 file changed, 1 deletion(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index e97b4942..0a860d97 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -17,7 +17,6 @@ specific language governing permissions and limitations under the License. """ -import copy from riak import RiakError from riak.util import deprecated From 5525f4f33bdbd61a72ee173f800125ffa94d901d Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 26 Mar 2013 17:20:13 -0500 Subject: [PATCH 0371/1060] Issue #228: Add unit tests to invalid object key issue --- riak/tests/test_comparison.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/riak/tests/test_comparison.py b/riak/tests/test_comparison.py index e6f888c9..297ca5fa 100644 --- a/riak/tests/test_comparison.py +++ b/riak/tests/test_comparison.py @@ -49,6 +49,15 @@ def test_object_hash(self): self.assertEqual(hash(a), hash(b), 'same object has different hashes') self.assertNotEqual(hash(a), hash(c), 'different object has same hash') + def test_object_valid_key(self): + a = RiakObject(None, 'bucket', 'key') + self.assertIsInstance(a, RiakObject, 'valid key name is rejected') + try: + b = RiakObject(None, 'bucket', '') + except ValueError: + b = None + self.assertIsNone(b, 'empty object key not allowed') + class RiakClientComparisonTest(unittest.TestCase, BaseTestCase): def test_client_eq(self): From 32601bd7e4829d8fc508f38165f35813c64fd6a1 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 2 Apr 2013 15:41:40 -0500 Subject: [PATCH 0372/1060] Issue #227: Allow charset encoding in content-type field --- riak/riak_object.py | 27 +++++++++++++++++++++++---- riak/tests/test_all.py | 5 ++--- riak/tests/test_kv.py | 5 +++++ riak/transports/http/transport.py | 2 +- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 0a860d97..ebf20daf 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -19,6 +19,7 @@ """ from riak import RiakError from riak.util import deprecated +import re class RiakObject(object): @@ -125,7 +126,8 @@ def _set_encoded_data(self, value): :type basestring""") def _serialize(self, value): - encoder = self.bucket.get_encoder(self.content_type) + content_type, charset = self._parse_content_type(self.content_type, self.charset) + encoder = self.bucket.get_encoder(content_type) if encoder: return encoder(value) elif isinstance(value, basestring): @@ -133,15 +135,16 @@ def _serialize(self, value): else: raise TypeError('No encoder for non-string data ' 'with content type "{0}"'. - format(self.content_type)) + format(content_type)) def _deserialize(self, value): - decoder = self.bucket.get_decoder(self.content_type) + content_type, charset = self._parse_content_type(self.content_type, self.charset) + decoder = self.bucket.get_decoder(content_type) if decoder: return decoder(value) else: raise TypeError('No decoder for content type "{0}"'. - format(self.content_type)) + format(content_type)) def add_index(self, field, value): """ @@ -403,4 +406,20 @@ def reduce(self, *args): mr.add(self.bucket.name, self.key) return mr.reduce(*args) + def _parse_content_type(self, value, oldcharset): + """ + Determine if the charset is embeded in the content-type. + If so, then break it out into the appropriate field + """ + charset = oldcharset + charpattern = "^(?P[A-Za-z0-9_/]+);\W*(charset|CHARSET)=(?P[A-Za-z0-9_-]+)" + matches = re.match(charpattern, value) + if matches is not None: + content_type = matches.group('content') + charset = matches.group('charset') + else: + content_type = value + + return content_type, charset + from riak.mapreduce import RiakMapReduce diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 9c41718a..17fcc52f 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -57,7 +57,7 @@ def setUpModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket - c = RiakClient(transport='http', host=HTTP_HOST, http_port=HTTP_PORT) + c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT) testrun_props_bucket = 'propsbucket' testrun_sibs_bucket = 'sibsbucket' @@ -68,9 +68,8 @@ def setUpModule(): b = c.bucket(testrun_search_bucket) b.enable_search() - def tearDownModule(): - c = RiakClient(transport='http', host=HTTP_HOST, http_port=HTTP_PORT) + c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT) if not int(os.environ.get('SKIP_SEARCH', '0')): b = c.bucket(testrun_search_bucket) b.clear_properties() diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index b18e0d43..126ae7fe 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -63,6 +63,11 @@ def test_store_and_get(self): self.assertRaises(TypeError, bucket.new, 'foo', u'éå') self.assertRaises(TypeError, bucket.new, 'foo', u'éå') + obj2 = bucket.new('baz', rand, 'application/json; charset=UTF-8') + obj2.store() + obj2 = bucket.get('baz') + self.assertEqual(obj2.data, rand) + def test_generate_key(self): # Ensure that Riak generates a random key when # the key passed to bucket.new() is None. diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 48aba9af..76824ed5 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -457,7 +457,7 @@ def parse_body(self, robj, response, expected_statuses): links = [] for header, value in headers.iteritems(): if header == 'content-type': - robj.content_type = value + robj.content_type, robj.charset = robj._parse_content_type(value, robj.charset) elif header == 'charset': robj.charset = value elif header == 'content-encoding': From 7abcb16774e4556403d0f5bc4e740637a3a12ec8 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Wed, 3 Apr 2013 11:10:52 -0500 Subject: [PATCH 0373/1060] Issue #233: Move content subtype removal into RiakClient --- riak/client/__init__.py | 8 ++++++-- riak/riak_object.py | 27 ++++----------------------- riak/transports/http/transport.py | 2 +- 3 files changed, 11 insertions(+), 26 deletions(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 2dea7a62..e472087f 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -25,6 +25,7 @@ import json import random +from email.message import Message from weakref import WeakValueDictionary from riak.client.operations import RiakClientOperations from riak.node import RiakNode @@ -104,6 +105,7 @@ def __init__(self, protocol='http', transport_options={}, self._decoders = {'application/json': json.loads, 'text/json': json.loads} self._buckets = WeakValueDictionary() + self._message = Message() def _get_protocol(self): return self._protocol @@ -167,7 +169,8 @@ def get_encoder(self, content_type): """ Get the encoding function for the provided content type. """ - return self._encoders.get(content_type) + self._message.set_type(content_type) + return self._encoders.get(self._message.get_content_type()) def set_encoder(self, content_type, encoder): """ @@ -182,7 +185,8 @@ def get_decoder(self, content_type): """ Get the decoding function for the provided content type. """ - return self._decoders.get(content_type) + self._message.set_type(content_type) + return self._decoders.get(self._message.get_content_type()) def set_decoder(self, content_type, decoder): """ diff --git a/riak/riak_object.py b/riak/riak_object.py index ebf20daf..0a860d97 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -19,7 +19,6 @@ """ from riak import RiakError from riak.util import deprecated -import re class RiakObject(object): @@ -126,8 +125,7 @@ def _set_encoded_data(self, value): :type basestring""") def _serialize(self, value): - content_type, charset = self._parse_content_type(self.content_type, self.charset) - encoder = self.bucket.get_encoder(content_type) + encoder = self.bucket.get_encoder(self.content_type) if encoder: return encoder(value) elif isinstance(value, basestring): @@ -135,16 +133,15 @@ def _serialize(self, value): else: raise TypeError('No encoder for non-string data ' 'with content type "{0}"'. - format(content_type)) + format(self.content_type)) def _deserialize(self, value): - content_type, charset = self._parse_content_type(self.content_type, self.charset) - decoder = self.bucket.get_decoder(content_type) + decoder = self.bucket.get_decoder(self.content_type) if decoder: return decoder(value) else: raise TypeError('No decoder for content type "{0}"'. - format(content_type)) + format(self.content_type)) def add_index(self, field, value): """ @@ -406,20 +403,4 @@ def reduce(self, *args): mr.add(self.bucket.name, self.key) return mr.reduce(*args) - def _parse_content_type(self, value, oldcharset): - """ - Determine if the charset is embeded in the content-type. - If so, then break it out into the appropriate field - """ - charset = oldcharset - charpattern = "^(?P[A-Za-z0-9_/]+);\W*(charset|CHARSET)=(?P[A-Za-z0-9_-]+)" - matches = re.match(charpattern, value) - if matches is not None: - content_type = matches.group('content') - charset = matches.group('charset') - else: - content_type = value - - return content_type, charset - from riak.mapreduce import RiakMapReduce diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 76824ed5..48aba9af 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -457,7 +457,7 @@ def parse_body(self, robj, response, expected_statuses): links = [] for header, value in headers.iteritems(): if header == 'content-type': - robj.content_type, robj.charset = robj._parse_content_type(value, robj.charset) + robj.content_type = value elif header == 'charset': robj.charset = value elif header == 'content-encoding': From cc5f9caba982ba900da1eb7230375585c2c93ab1 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Wed, 3 Apr 2013 19:19:27 -0500 Subject: [PATCH 0374/1060] Issue #227: Moved management of content-type back into RiakObject --- riak/client/__init__.py | 8 ++------ riak/riak_object.py | 25 +++++++++++++++++++++++++ riak/transports/http/transport.py | 6 ++---- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index e472087f..2dea7a62 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -25,7 +25,6 @@ import json import random -from email.message import Message from weakref import WeakValueDictionary from riak.client.operations import RiakClientOperations from riak.node import RiakNode @@ -105,7 +104,6 @@ def __init__(self, protocol='http', transport_options={}, self._decoders = {'application/json': json.loads, 'text/json': json.loads} self._buckets = WeakValueDictionary() - self._message = Message() def _get_protocol(self): return self._protocol @@ -169,8 +167,7 @@ def get_encoder(self, content_type): """ Get the encoding function for the provided content type. """ - self._message.set_type(content_type) - return self._encoders.get(self._message.get_content_type()) + return self._encoders.get(content_type) def set_encoder(self, content_type, encoder): """ @@ -185,8 +182,7 @@ def get_decoder(self, content_type): """ Get the decoding function for the provided content type. """ - self._message.set_type(content_type) - return self._decoders.get(self._message.get_content_type()) + return self._decoders.get(content_type) def set_decoder(self, content_type, decoder): """ diff --git a/riak/riak_object.py b/riak/riak_object.py index 0a860d97..1f112552 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -17,6 +17,7 @@ specific language governing permissions and limitations under the License. """ +from email.message import Message from riak import RiakError from riak.util import deprecated @@ -56,6 +57,7 @@ def __init__(self, client, bucket, key=None): self.vclock = None self.charset = None self.content_type = 'application/json' + self.full_content_type = 'application/json' self.content_encoding = None self.usermeta = {} self.indexes = set() @@ -124,6 +126,29 @@ def _set_encoded_data(self, value): bucket's registered encoders. :type basestring""") + def _get_content_type(self): + return self.content_type + + def _set_content_type(self, value): + """ + Split the content-type header into two parts: + 1) Actual main/sub encoding type + 2) charset + + :param value: Complete MIME content-type string + """ + message = Message() + message.set_type(value) + + self.full_content_type = value + self.content_type = message.get_content_type() + self.charset = message.get_content_charset(None) + + content_type = property(_get_content_type, _set_content_type, doc=""" + The MIME `content-type` field less the `charset` property. + If set, it will set the internal `charset` field on the + RiakObject.""") + def _serialize(self, value): encoder = self.bucket.get_encoder(self.content_type) if encoder: diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 48aba9af..ae411a06 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -458,8 +458,6 @@ def parse_body(self, robj, response, expected_statuses): for header, value in headers.iteritems(): if header == 'content-type': robj.content_type = value - elif header == 'charset': - robj.charset = value elif header == 'content-encoding': robj.content_encoding = value elif header == 'etag': @@ -543,7 +541,7 @@ def _build_put_headers(self, robj): # Construct the headers... headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', - 'Content-Type': robj.content_type, + 'Content-Type': robj.full_content_type, 'X-Riak-ClientId': self._client_id}) # Add the vclock if it exists... if robj.vclock is not None: @@ -601,7 +599,7 @@ def build_headers(cls, headers): @classmethod def parse_http_headers(cls, headers): """ - Parse an HTTP Header string into an asssociative array of + Parse an HTTP Header string into an associative array of response headers. """ retVal = {} From 0848da75c945f80e0a588fe8954b1c2f71e3fbfb Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Thu, 4 Apr 2013 08:44:58 -0500 Subject: [PATCH 0375/1060] Issue #227: Re-re-reimplmented charset breakout in the http transport layer --- riak/riak_object.py | 25 ------------------------- riak/transports/http/transport.py | 25 +++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 1f112552..0a860d97 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -17,7 +17,6 @@ specific language governing permissions and limitations under the License. """ -from email.message import Message from riak import RiakError from riak.util import deprecated @@ -57,7 +56,6 @@ def __init__(self, client, bucket, key=None): self.vclock = None self.charset = None self.content_type = 'application/json' - self.full_content_type = 'application/json' self.content_encoding = None self.usermeta = {} self.indexes = set() @@ -126,29 +124,6 @@ def _set_encoded_data(self, value): bucket's registered encoders. :type basestring""") - def _get_content_type(self): - return self.content_type - - def _set_content_type(self, value): - """ - Split the content-type header into two parts: - 1) Actual main/sub encoding type - 2) charset - - :param value: Complete MIME content-type string - """ - message = Message() - message.set_type(value) - - self.full_content_type = value - self.content_type = message.get_content_type() - self.charset = message.get_content_charset(None) - - content_type = property(_get_content_type, _set_content_type, doc=""" - The MIME `content-type` field less the `charset` property. - If set, it will set the internal `charset` field on the - RiakObject.""") - def _serialize(self, value): encoder = self.bucket.get_encoder(self.content_type) if encoder: diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index ae411a06..daf320f6 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -28,6 +28,7 @@ import re import csv import httplib +from email.message import Message from riak.transports.transport import RiakTransport from riak.transports.http.resources import RiakHttpResources from riak.transports.http.connection import RiakHttpConnection @@ -457,7 +458,7 @@ def parse_body(self, robj, response, expected_statuses): links = [] for header, value in headers.iteritems(): if header == 'content-type': - robj.content_type = value + robj.content_type, robj.charset = self._parse_content_type(value) elif header == 'content-encoding': robj.content_encoding = value elif header == 'etag': @@ -540,8 +541,12 @@ def _build_put_headers(self, robj): """Build the headers for a POST/PUT request.""" # Construct the headers... + if robj.charset is not None: + content_type = "%s; charset=%s" % (robj.content_type, robj.charset) + else: + content_type = robj.content_type headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', - 'Content-Type': robj.full_content_type, + 'Content-Type': content_type, 'X-Riak-ClientId': self._client_id}) # Add the vclock if it exists... if robj.vclock is not None: @@ -591,6 +596,22 @@ def _normalize_xml_search_response(self, xml): parser.feed(xml) return parser.close() + def _parse_content_type(self, value): + """ + Split the content-type header into two parts: + 1) Actual main/sub encoding type + 2) charset + + :param value: Complete MIME content-type string + """ + message = Message() + message.set_type(value) + + content_type = message.get_content_type() + charset = message.get_content_charset(None) + + return content_type, charset + @classmethod def build_headers(cls, headers): return ['%s: %s' % (header, value) From 0e8d20a463626aa596d68a769ac5ee2c9c8338c9 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Thu, 4 Apr 2013 09:05:24 -0500 Subject: [PATCH 0376/1060] Issue #227: Clean up pep8 and pyflakes (and @seancribbs) warnings --- riak/tests/test_all.py | 1 + riak/transports/http/transport.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 17fcc52f..1894cba9 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -68,6 +68,7 @@ def setUpModule(): b = c.bucket(testrun_search_bucket) b.enable_search() + def tearDownModule(): c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT) if not int(os.environ.get('SKIP_SEARCH', '0')): diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index daf320f6..75a6e5d1 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -458,7 +458,8 @@ def parse_body(self, robj, response, expected_statuses): links = [] for header, value in headers.iteritems(): if header == 'content-type': - robj.content_type, robj.charset = self._parse_content_type(value) + robj.content_type, robj.charset = \ + self._parse_content_type(value) elif header == 'content-encoding': robj.content_encoding = value elif header == 'etag': @@ -542,7 +543,8 @@ def _build_put_headers(self, robj): # Construct the headers... if robj.charset is not None: - content_type = "%s; charset=%s" % (robj.content_type, robj.charset) + content_type = ("%s; charset='%s'" % + (robj.content_type, robj.charset)) else: content_type = robj.content_type headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', From 6efdb8ae0104a41dd6918b78bc8c36c9c3651b5e Mon Sep 17 00:00:00 2001 From: Shuhao Date: Thu, 4 Apr 2013 10:15:57 -0400 Subject: [PATCH 0377/1060] Added bucket.delete, fixes #236 --- riak/bucket.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/riak/bucket.py b/riak/bucket.py index dbb5dcbd..c3ef4533 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -437,6 +437,16 @@ def get_index(self, index, startkey, endkey=None): """ return self._client.get_index(self.name, index, startkey, endkey) + def delete(self, key, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + """Deletes an object from riak. + + Short hand for bucket.new(key).delete() + :param key: The key for the object + :type key: string + :rtype: RiakObject + """ + return self.new(key).delete(rw, r, w, dw, pr, pw) + def __str__(self): return ''.format(self.name) From e27759c46fbebe4fb7f706a724b3c04f12c9239a Mon Sep 17 00:00:00 2001 From: Shuhao Date: Thu, 4 Apr 2013 10:22:09 -0400 Subject: [PATCH 0378/1060] Unittest for bucket.delete --- riak/tests/test_kv.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index b18e0d43..c9c11f46 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -193,6 +193,16 @@ def test_delete(self): obj.reload() self.assertFalse(obj.exists) + def test_bucket_delete(self): + bucket = self.client.bucket(self.bucket_name) + rand = self.randint() + obj = bucket.new(self.key_name, rand) + obj.store() + + bucket.delete(self.key_name) + obj.reload() + self.assertFalse(obj.exists) + def test_set_bucket_properties(self): bucket = self.client.bucket(self.props_bucket) # Test setting allow mult... From 8868921fdd809e2b9a3c799650242b80f5c979b7 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Thu, 4 Apr 2013 11:11:40 -0400 Subject: [PATCH 0379/1060] Used kwargs instead of positional arguments --- riak/bucket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index c3ef4533..3b255885 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -437,7 +437,7 @@ def get_index(self, index, startkey, endkey=None): """ return self._client.get_index(self.name, index, startkey, endkey) - def delete(self, key, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + def delete(self, key, **kwargs): """Deletes an object from riak. Short hand for bucket.new(key).delete() @@ -445,7 +445,7 @@ def delete(self, key, rw=None, r=None, w=None, dw=None, pr=None, pw=None): :type key: string :rtype: RiakObject """ - return self.new(key).delete(rw, r, w, dw, pr, pw) + return self.new(key).delete(**kwargs) def __str__(self): return ''.format(self.name) From 5167cef163fcf10096f001d04bb168d54d2b8439 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Thu, 4 Apr 2013 11:12:12 -0400 Subject: [PATCH 0380/1060] Found an unused variable. Removed. Pyflake warning was active. --- riak/transports/pbc/transport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 72be6a8c..5f9b855a 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -185,7 +185,6 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, MSG_CODE_PUT_RESP) - contents = [] if resp is not None: return self._decoded_contents(resp, robj) From c4c6eda82f201f81ce9dddbfaa4dc6c44a9c6389 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Thu, 4 Apr 2013 10:52:43 -0500 Subject: [PATCH 0381/1060] Issue #227: Fix unit test and RFC-compliant quoting of charset --- riak/tests/test_kv.py | 3 ++- riak/transports/http/transport.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 126ae7fe..acc53367 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -63,7 +63,8 @@ def test_store_and_get(self): self.assertRaises(TypeError, bucket.new, 'foo', u'éå') self.assertRaises(TypeError, bucket.new, 'foo', u'éå') - obj2 = bucket.new('baz', rand, 'application/json; charset=UTF-8') + obj2 = bucket.new('baz', rand, 'application/json') + obj2.charset = 'UTF-8' obj2.store() obj2 = bucket.get('baz') self.assertEqual(obj2.data, rand) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 75a6e5d1..c75fec5f 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -543,7 +543,8 @@ def _build_put_headers(self, robj): # Construct the headers... if robj.charset is not None: - content_type = ("%s; charset='%s'" % +# content_type = ('%s; charset="%s"' % + content_type = ("%s; charset='%s" % (robj.content_type, robj.charset)) else: content_type = robj.content_type From f093f88b3c8799c194f08a06bcc7dbb7b5248f40 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Thu, 4 Apr 2013 10:55:22 -0500 Subject: [PATCH 0382/1060] Issue #227: Commit correct file version --- riak/transports/http/transport.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index c75fec5f..ee147f3c 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -543,8 +543,7 @@ def _build_put_headers(self, robj): # Construct the headers... if robj.charset is not None: -# content_type = ('%s; charset="%s"' % - content_type = ("%s; charset='%s" % + content_type = ('%s; charset="%s"' % (robj.content_type, robj.charset)) else: content_type = robj.content_type From da569259c58d143a2283ee7b907b96161dd01e18 Mon Sep 17 00:00:00 2001 From: Shuhao Date: Thu, 4 Apr 2013 15:18:55 -0400 Subject: [PATCH 0383/1060] Unittest workaround for underlying transport bug --- riak/tests/test_kv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index c9c11f46..63fa0c99 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -200,7 +200,7 @@ def test_bucket_delete(self): obj.store() bucket.delete(self.key_name) - obj.reload() + obj = bucket.get(self.key_name) self.assertFalse(obj.exists) def test_set_bucket_properties(self): From ee54fd427c06794acc8b9eab40ba2ed8ea925f76 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 5 Apr 2013 13:14:30 -0500 Subject: [PATCH 0384/1060] Fix issue with reloading deleted objects over PBC. * Reinstated @shuhaowu's obj.reload() in bucket delete test. * Set obj.exists to False if there are no contents, resolving reload problem. --- riak/tests/test_kv.py | 2 +- riak/transports/pbc/transport.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index ca063904..6fb644c1 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -206,7 +206,7 @@ def test_bucket_delete(self): obj.store() bucket.delete(self.key_name) - obj = bucket.get(self.key_name) + obj.reload() self.assertFalse(obj.exists) def test_set_bucket_properties(self): diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 5f9b855a..542e97fb 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -127,6 +127,7 @@ def _decoded_contents(self, resp, old_obj): ret.siblings = contents[:] return ret else: + old_obj.exists = False return old_obj def get(self, robj, r=None, pr=None, vtag=None): From 44d72fef1676de53101f6ebbc3ba9c53abc6cfa9 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 25 Apr 2013 09:45:16 +0200 Subject: [PATCH 0385/1060] Added set_index. It allows to ensure that there is no other index on given field. --- riak/riak_object.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/riak/riak_object.py b/riak/riak_object.py index 0a860d97..dc057c36 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -186,6 +186,21 @@ def remove_index(self, field=None, value=None): return self + def set_index(self, field, value): + """ + Works like add_index, but ensures that there is only one index on given field. + If other found, then removes it first. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: RiakObject + """ + to_rem = set((x for x in self.indexes if x[0] == field)) + self.indexes.difference_update(to_rem) + return self.add_index(field, value) + remove_indexes = remove_index def add_link(self, obj, tag=None): From b1125fdff644053a592e1359b8fc2951d0c5c8ec Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 25 Apr 2013 13:21:59 +0200 Subject: [PATCH 0386/1060] Added test suite for set_index --- riak/tests/test_2i.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 724d4baf..6f674287 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -246,3 +246,21 @@ def test_secondary_index_invalid_name(self): with self.assertRaises(RiakError): bucket.new('k', 'a').add_index('field1', 'value1') + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_set_index(self): + if not self.is_2i_supported(): + return True + + bucket = self.client.bucket(self.bucket_name) + obj = bucket.new('bar', 1) + obj.set_index('bar_int', 1) + obj.set_index('bar2_int', 1) + self.assertEqual(2, len(obj.indexes)) + self.assertEqual(set(('bar_int', 1), ('bar2_int', 1)), obj.indexes) + + obj.set_index('bar_int', 3) + self.assertEqual(2, len(obj.indexes)) + self.assertEqual(set(('bar_int', 3), ('bar2_int', 1)), obj.indexes) + obj.set_index('bar2_int', 10) + self.assertEqual(set(('bar_int', 3), ('bar2_int', 10)), obj.indexes) From 2921353d3b602cb952652b2c68775d24d19ef898 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 25 Apr 2013 13:24:59 +0200 Subject: [PATCH 0387/1060] fix for set initialization in test_2i --- riak/tests/test_2i.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 6f674287..d303eb4b 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -257,10 +257,10 @@ def test_set_index(self): obj.set_index('bar_int', 1) obj.set_index('bar2_int', 1) self.assertEqual(2, len(obj.indexes)) - self.assertEqual(set(('bar_int', 1), ('bar2_int', 1)), obj.indexes) + self.assertEqual(set((('bar_int', 1), ('bar2_int', 1))), obj.indexes) obj.set_index('bar_int', 3) self.assertEqual(2, len(obj.indexes)) - self.assertEqual(set(('bar_int', 3), ('bar2_int', 1)), obj.indexes) + self.assertEqual(set((('bar_int', 3), ('bar2_int', 1))), obj.indexes) obj.set_index('bar2_int', 10) - self.assertEqual(set(('bar_int', 3), ('bar2_int', 10)), obj.indexes) + self.assertEqual(set((('bar_int', 3), ('bar2_int', 10))), obj.indexes) From e6f5e0cb04992597766a85b836c84ff24a63f47e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 28 Apr 2013 05:26:27 -0500 Subject: [PATCH 0388/1060] Refactor model and handling of siblings. In order to... 1. Better detect deleted objects and sibling tombstones 2. Reduce server roundtrips to fetch object siblings (which was an inconsistency between HTTP and PBC). ...this refactor introduces breaking changes. This introduces the `RiakContent` class which models a single sibling. An `RiakObject` that is NOT in conflict will appear to have one "sibling". Objects that are strictly "not found" will have ZERO siblings. Otherwise, the object will have as many siblings as are returned by Riak. This is more consistent with Riak's model of objects (values). Most existing methods and properties that were on `RiakObject` have been proxied in such a manner that if the object is not in conflict, the corresponding property or method on the solitary sibling will be invoked. When in conflict, these properties and methods with raise the new `riak.ConflictError` exception. Except in the case of conflict, this will ease transition of existing code to the new behavior. The `get_sibling` method and behavior has been broken to reflect the fact that all siblings are always requested and handled appropriately. The method now produces a deprecation warning and simply returns the requested sibling already in the object. The HTTP transport no longer supports retrieving the "text" format of objects in conflict. This commit also completes a refactoring of the HTTP transport to break-out transport codec methods into a separate class/file, for clarity and consistency with the PBC transport. --- riak/__init__.py | 12 +- riak/client/operations.py | 6 +- riak/content.py | 187 ++++++++++++++ riak/riak_object.py | 290 +++++++-------------- riak/tests/test_kv.py | 33 +-- riak/tests/test_mapreduce.py | 6 +- riak/transports/http/codec.py | 255 +++++++++++++++++++ riak/transports/http/connection.py | 14 +- riak/transports/http/transport.py | 391 +++++------------------------ riak/transports/pbc/codec.py | 115 +++++---- riak/transports/pbc/connection.py | 3 +- riak/transports/pbc/transport.py | 70 +----- riak/transports/transport.py | 2 +- 13 files changed, 713 insertions(+), 671 deletions(-) create mode 100644 riak/content.py create mode 100644 riak/transports/http/codec.py diff --git a/riak/__init__.py b/riak/__init__.py index ba50fc00..25a41535 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -32,7 +32,7 @@ __all__ = ['RiakBucket', 'RiakNode', 'RiakObject', 'RiakClient', 'RiakMapReduce', 'RiakKeyFilter', 'RiakLink', 'RiakError', - 'ONE', 'ALL', 'QUORUM', 'key_filter'] + 'ConflictError', 'ONE', 'ALL', 'QUORUM', 'key_filter'] class RiakError(Exception): @@ -45,6 +45,16 @@ def __init__(self, value): def __str__(self): return repr(self.value) + +class ConflictError(RiakError): + """ + Raised when an operation is attempted on a RiakObject that has + more than one sibling. + """ + def __init__(self, message="Object in conflict"): + super(ConflictError, self).__init__(message) + + from client import RiakClient from bucket import RiakBucket from node import RiakNode diff --git a/riak/client/operations.py b/riak/client/operations.py index 516b2d26..cf4cc46f 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -180,7 +180,7 @@ def put_new(self, transport, robj, w=None, dw=None, pw=None, if_none_match=if_none_match) @retryable - def get(self, transport, robj, r=None, pr=None, vtag=None): + def get(self, transport, robj, r=None, pr=None): """ Fetches the contents of a Riak object. @@ -190,14 +190,12 @@ def get(self, transport, robj, r=None, pr=None, vtag=None): :type r: integer, string, None :param pr: the primary read quorum :type pr: integer, string, None - :param vtag: the specific sibling to fetch - :type vtag: string """ if not isinstance(robj.key, basestring): raise TypeError( 'key must be a string, instead got {0}'.format(repr(robj.key))) - return transport.get(robj, r=r, pr=pr, vtag=vtag) + return transport.get(robj, r=r, pr=pr) @retryable def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, diff --git a/riak/content.py b/riak/content.py new file mode 100644 index 00000000..f19606b7 --- /dev/null +++ b/riak/content.py @@ -0,0 +1,187 @@ +""" +Copyright 2013 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" +from riak import RiakError +from riak.util import deprecated + + +class RiakContent(object): + """ + The RiakContent holds the metadata and value of a single sibling + within a RiakObject. RiakObjects that have more than one sibling + are considered to be in conflict. + """ + def __init__(self, robject, data=None, encoded_data=None, charset=None, + content_type='application/json', content_encoding=None, + etag=None, usermeta=None, links=None, indexes=None, + exists=False): + self._robject = robject + self._data = data + self._encoded_data = encoded_data + self.charset = charset + self.content_type = content_type + self.content_encoding = content_encoding + self.etag = etag + self.usermeta = usermeta or {} + self.links = links or [] + self.indexes = indexes or set() + self.exists = exists + + def _get_data(self): + if self._encoded_data is not None and self._data is None: + self._data = self._deserialize(self._encoded_data) + self._encoded_data = None + return self._data + + def _set_data(self, value): + self._encoded_data = None + self._data = value + + data = property(_get_data, _set_data, doc=""" + The data stored in this object, as Python objects. For the raw + data, use the `encoded_data` property. If unset, accessing + this property will result in decoding the `encoded_data` + property into Python values. The decoding is dependent on the + `content_type` property and the bucket's registered decoders. + :type mixed """) + + def get_encoded_data(self): + deprecated("`get_encoded_data` is deprecated, use the `encoded_data`" + " property") + return self.encoded_data + + def set_encoded_data(self, value): + deprecated("`set_encoded_data` is deprecated, use the `encoded_data`" + " property") + self.encoded_data = value + + def _get_encoded_data(self): + if self._data is not None and self._encoded_data is None: + self._encoded_data = self._serialize(self._data) + self._data = None + return self._encoded_data + + def _set_encoded_data(self, value): + self._data = None + self._encoded_data = value + + encoded_data = property(_get_encoded_data, _set_encoded_data, doc=""" + The raw data stored in this object, essentially the encoded + form of the `data` property. If unset, accessing this property + will result in encoding the `data` property into a string. The + encoding is dependent on the `content_type` property and the + bucket's registered encoders. + :type basestring""") + + def _serialize(self, value): + encoder = self._robject.bucket.get_encoder(self.content_type) + if encoder: + return encoder(value) + elif isinstance(value, basestring): + return value.encode() + else: + raise TypeError('No encoder for non-string data ' + 'with content type "{0}"'. + format(self.content_type)) + + def _deserialize(self, value): + decoder = self._robject.bucket.get_decoder(self.content_type) + if decoder: + return decoder(value) + else: + raise TypeError('No decoder for content type "{0}"'. + format(self.content_type)) + + def add_index(self, field, value): + """ + Tag this object with the specified field/value pair for + indexing. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: RiakObject + """ + if field[-4:] not in ("_bin", "_int"): + raise RiakError("Riak 2i fields must end with either '_bin'" + " or '_int'.") + + self.indexes.add((field, value)) + + return self._robject + + def remove_index(self, field=None, value=None): + """ + Remove the specified field/value pair as an index on this + object. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: RiakObject + """ + if not field and not value: + self.indexes.clear() + elif field and not value: + for index in [x for x in self.indexes if x[0] == field]: + self.indexes.remove(index) + elif field and value: + self.indexes.remove((field, value)) + else: + raise RiakError("Cannot pass value without a field" + " name while removing index") + + return self._robject + + remove_indexes = remove_index + + def set_index(self, field, value): + """ + Works like add_index, but ensures that there is only one index + on given field. If other found, then removes it first. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: RiakObject + """ + to_rem = set((x for x in self.indexes if x[0] == field)) + self.indexes.difference_update(to_rem) + return self.add_index(field, value) + + def add_link(self, obj, tag=None): + """ + Add a link to a RiakObject. + + :param obj: Either a RiakObject or 3 item link tuple consisting + of (bucket, key, tag). + :type obj: mixed + :param tag: Optional link tag. Defaults to bucket name. It is ignored + if ``obj`` is a 3 item link tuple. + :type tag: string + :rtype: RiakObject + """ + if isinstance(obj, tuple): + newlink = obj + else: + newlink = (obj.bucket.name, obj.key, tag) + + self.links.append(newlink) + return self._robject diff --git a/riak/riak_object.py b/riak/riak_object.py index dc057c36..eee42db8 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -1,4 +1,5 @@ """ +Copyright 2012-2013 Basho Technologies Copyright 2010 Rusty Klophaus Copyright 2010 Justin Sheehy Copyright 2009 Jay Baird @@ -17,10 +18,46 @@ specific language governing permissions and limitations under the License. """ -from riak import RiakError +from riak import ConflictError +from riak.content import RiakContent from riak.util import deprecated +def content_property(name, doc=None): + """ + Delegates a property to the first sibling in a RiakObject, raising + an error when the object is in conflict. + """ + def _setter(self, value): + if len(self.siblings) == 0: + # In this case, assume that what the user wants is to + # create a new sibling inside an empty object. + self.siblings = [RiakContent(self)] + if len(self.siblings) != 1: + raise ConflictError() + setattr(self.siblings[0], name, value) + + def _getter(self): + if len(self.siblings) != 1: + raise ConflictError() + return getattr(self.siblings[0], name) + + return property(_getter, _setter, doc=doc) + + +def content_method(name): + """ + Delegates a method to the first sibling in a RiakObject, raising + an error when the object is in conflict. + """ + def _delegate(self, *args, **kwargs): + if len(self.siblings) != 1: + raise ConflictError() + return getattr(self.siblings[0], name).__call__(*args, **kwargs) + + return _delegate + + class RiakObject(object): """ The RiakObject holds meta information about a Riak object, plus the @@ -51,17 +88,8 @@ def __init__(self, client, bucket, key=None): self.client = client self.bucket = bucket self.key = key - self._data = None - self._encoded_data = None self.vclock = None - self.charset = None - self.content_type = 'application/json' - self.content_encoding = None - self.usermeta = {} - self.indexes = set() - self.links = [] - self.siblings = [] - self.exists = False + self.siblings = [RiakContent(self)] def __hash__(self): return hash((self.key, self.bucket, self.vclock)) @@ -78,45 +106,14 @@ def __ne__(self, other): else: return True - def _get_data(self): - if self._encoded_data is not None and self._data is None: - self._data = self._deserialize(self._encoded_data) - self._encoded_data = None - return self._data - - def _set_data(self, value): - self._encoded_data = None - self._data = value - - data = property(_get_data, _set_data, doc=""" + data = content_property('data', doc=""" The data stored in this object, as Python objects. For the raw data, use the `encoded_data` property. If unset, accessing this property will result in decoding the `encoded_data` property into Python values. The decoding is dependent on the `content_type` property and the bucket's registered decoders. :type mixed """) - - def get_encoded_data(self): - deprecated("`get_encoded_data` is deprecated, use the `encoded_data`" - " property") - return self.encoded_data - - def set_encoded_data(self, value): - deprecated("`set_encoded_data` is deprecated, use the `encoded_data`" - " property") - self.encoded_data = value - - def _get_encoded_data(self): - if self._data is not None and self._encoded_data is None: - self._encoded_data = self._serialize(self._data) - self._data = None - return self._encoded_data - - def _set_encoded_data(self, value): - self._data = None - self._encoded_data = value - - encoded_data = property(_get_encoded_data, _set_encoded_data, doc=""" + encoded_data = content_property('encoded_data', doc=""" The raw data stored in this object, essentially the encoded form of the `data` property. If unset, accessing this property will result in encoding the `data` property into a string. The @@ -124,104 +121,56 @@ def _set_encoded_data(self, value): bucket's registered encoders. :type basestring""") - def _serialize(self, value): - encoder = self.bucket.get_encoder(self.content_type) - if encoder: - return encoder(value) - elif isinstance(value, basestring): - return value.encode() - else: - raise TypeError('No encoder for non-string data ' - 'with content type "{0}"'. - format(self.content_type)) - - def _deserialize(self, value): - decoder = self.bucket.get_decoder(self.content_type) - if decoder: - return decoder(value) - else: - raise TypeError('No decoder for content type "{0}"'. - format(self.content_type)) + charset = content_property('charset', doc=""" + The character set of the encoded data + :type string""") - def add_index(self, field, value): - """ - Tag this object with the specified field/value pair for - indexing. + content_type = content_property('content_type', doc=""" + The MIME media type of the encoded data + :type string""") - :param field: The index field. - :type field: string - :param value: The index value. - :type value: string or integer - :rtype: RiakObject - """ - if field[-4:] not in ("_bin", "_int"): - raise RiakError("Riak 2i fields must end with either '_bin'" - " or '_int'.") + content_encoding = content_property('content_encoding') - self.indexes.add((field, value)) + usermeta = content_property('usermeta', doc=""" + Arbitrary user-defined metadata, mapping strings to strings. + :type dict""") - return self + links = content_property('links', doc=""" + A collection of bucket/key/tag 3-tuples representing links to + other keys. + :type set""") - def remove_index(self, field=None, value=None): - """ - Remove the specified field/value pair as an index on this - object. - - :param field: The index field. - :type field: string - :param value: The index value. - :type value: string or integer - :rtype: RiakObject - """ - if not field and not value: - self.indexes.clear() - elif field and not value: - for index in [x for x in self.indexes if x[0] == field]: - self.indexes.remove(index) - elif field and value: - self.indexes.remove((field, value)) - else: - raise RiakError("Cannot pass value without a field" - " name while removing index") - - return self - - def set_index(self, field, value): - """ - Works like add_index, but ensures that there is only one index on given field. - If other found, then removes it first. - - :param field: The index field. - :type field: string - :param value: The index value. - :type value: string or integer - :rtype: RiakObject - """ - to_rem = set((x for x in self.indexes if x[0] == field)) - self.indexes.difference_update(to_rem) - return self.add_index(field, value) + indexes = content_property('indexes', doc=""" + The set of secondary index entries, consisting of + index-name/value tuples + :type set""") + get_encoded_data = content_method('get_encoded_data') + set_encoded_data = content_method('set_encoded_data') + add_index = content_method('add_index') + remove_index = content_method('remove_index') remove_indexes = remove_index + set_index = content_method('set_index') + add_link = content_method('add_link') - def add_link(self, obj, tag=None): - """ - Add a link to a RiakObject. - - :param obj: Either a RiakObject or 3 item link tuple consisting - of (bucket, key, tag). - :type obj: mixed - :param tag: Optional link tag. Defaults to bucket name. It is ignored - if ``obj`` is a 3 item link tuple. - :type tag: string - :rtype: RiakObject - """ - if isinstance(obj, tuple): - newlink = obj + def _exists(self): + if len(self.siblings) == 0: + return False + elif len(self.siblings) > 1: + # Even if all of the siblings are tombstones, the object + # essentially exists. + return True else: - newlink = (obj.bucket.name, obj.key, tag) + return self.siblings[0].exists - self.links.append(newlink) - return self + exists = property(_exists, None, doc=""" + Whether the object exists. This is only true when there is a + single sibling and it is neither a tombstone nor unsaved.""") + + def get_sibling(self, index): + deprecated("RiakObject.get_sibling is deprecated, use the " + "siblings property instead") + return self.siblings[index] def store(self, w=None, dw=None, pw=None, return_body=True, if_none_match=False): @@ -248,27 +197,23 @@ def store(self, w=None, dw=None, pw=None, return_body=True, there is no key previously defined :type if_none_match: bool :rtype: RiakObject """ - if (self.siblings and not self._data - and not self._encoded_data and not self.vclock): - raise RiakError("Attempting to store an invalid object," - "store one of the siblings instead") + if len(self.siblings) != 1: + raise ConflictError("Attempting to store an invalid object, " + "resolve the siblings first") if self.key is None: - result = self.client.put_new( + self.client.put_new( self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) - self._populate(result) else: - result = self.client.put(self, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) - if result is not None and result != ('', []): - self._populate(result) + self.client.put(self, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) return self - def reload(self, r=None, pr=None, vtag=None): + def reload(self, r=None, pr=None): """ Reload the object from Riak. When this operation completes, the object could contain new metadata and a new value, if the object @@ -277,15 +222,14 @@ def reload(self, r=None, pr=None, vtag=None): :param r: R-Value, wait for this many partitions to respond before returning to client. :type r: integer + :param pr: PR-value, require this many primary partitions to + be available before performing the read that + precedes the put + :type pr: integer :rtype: RiakObject """ - result = self.client.get(self, r=r, pr=pr, vtag=vtag) - if result and result != ('', []): - self._populate(result) - else: - self.clear() - + self.client.get(self, r=r, pr=pr) return self def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): @@ -325,55 +269,9 @@ def clear(self): :rtype: RiakObject """ - self.headers = [] - self.links = [] - self.data = None - self.exists = False self.siblings = [] return self - def _populate(self, result): - """ - Populate the object based on the return from get. - - If None returned, then object is not found - If a tuple of vclock, contents then one or more - whole revisions of the key were found - If a list of vtags is returned there are multiple - sibling that need to be retrieved with get. - """ - if result is None or result is self: - return self - elif type(result) is RiakObject: - self.clear() - self.__dict__ = result.__dict__.copy() - else: - raise RiakError("do not know how to handle type %s" % type(result)) - - def get_sibling(self, i, r=None, pr=None): - """ - Retrieve a sibling by sibling number. - - :param i: Sibling number. - :type i: integer - :param r: R-Value. Wait until this many partitions - have responded before returning to client. - :type r: integer - :rtype: RiakObject. - """ - if isinstance(self.siblings[i], RiakObject): - return self.siblings[i] - else: - # Run the request... - vtag = self.siblings[i] - obj = RiakObject(self.client, self.bucket, self.key) - obj.reload(r=r, pr=pr, vtag=vtag) - - # And make sure it knows who its siblings are - self.siblings[i] = obj - obj.siblings = self.siblings - return obj - def add(self, *args): """ Start assembling a Map/Reduce operation. diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 6fb644c1..880245bc 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -2,6 +2,7 @@ import os import cPickle import copy +from riak import ConflictError try: import simplejson as json @@ -106,6 +107,7 @@ def test_stream_keys_abort(self): # If the stream was closed correctly, this will not error robj = bucket.get(regular_keys[0]) + self.assertEqual(len(robj.siblings), 1) self.assertEqual(True, robj.exists) def test_bad_key(self): @@ -185,7 +187,6 @@ def test_missing_object(self): bucket = self.client.bucket(self.bucket_name) obj = bucket.get(self.key_name) self.assertFalse(obj.exists) - self.assertEqual(obj.data, None) def test_delete(self): bucket = self.client.bucket(self.bucket_name) @@ -272,25 +273,29 @@ def test_siblings(self): other_obj.store() vals.add(str(randval)) - # Make sure the object has itself plus four siblings... + # Make sure the object has five siblings... obj = bucket.get(self.key_name) obj.reload() - self.assertTrue(bool(obj.siblings)) self.assertEqual(len(obj.siblings), 5) - # Get each of the values - make sure they match what was assigned - vals2 = set() - for i in xrange(len(obj.siblings)): - vals2.add(obj.get_sibling(i).encoded_data) + # When the object is in conflict, using the shortcut methods + # should raise the ConflictError + with self.assertRaises(ConflictError): + obj.data + + # Get each of the values - make sure they match what was + # assigned + vals2 = set([sibling.encoded_data for sibling in obj.siblings]) self.assertEqual(vals, vals2) # Resolve the conflict, and then do a get... - obj3 = obj.get_sibling(3) - obj3.store() + resolved_sibling = obj.siblings[3] + obj.siblings = [resolved_sibling] + obj.store() obj.reload() - self.assertEqual(len(obj.siblings), 0) - self.assertEqual(obj.encoded_data, obj3.encoded_data) + self.assertEqual(len(obj.siblings), 1) + self.assertEqual(obj.encoded_data, resolved_sibling.encoded_data) def test_store_of_missing_object(self): bucket = self.client.bucket(self.bucket_name) @@ -430,7 +435,7 @@ def test_store_binary_object_from_file_should_use_default_mimetype(self): def test_store_binary_object_from_file_should_fail_if_file_not_found(self): bucket = self.client.bucket(self.bucket_name) with self.assertRaises(IOError): - bucket.new_from_file('not_found_from_file', 'FILE_NOT_FOUND') - obj = bucket.get('not_found_from_file') - self.assertEqual(obj.encoded_data, None) + bucket.new_from_file(self.key_name, 'FILE_NOT_FOUND') + obj = bucket.get(self.key_name) + # self.assertEqual(obj.encoded_data, None) self.assertFalse(obj.exists) diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 8f2ad92a..cb7e3759 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -8,13 +8,13 @@ class LinkTests(object): def test_store_and_get_links(self): # Create the object... bucket = self.client.bucket(self.bucket_name) - bucket.new(key="test_store_and_get_links", encoded_data='2', + bucket.new(key=self.key_name, encoded_data='2', content_type='application/octet-stream') \ .add_link(bucket.new("foo1")) \ .add_link(bucket.new("foo2"), "tag") \ .add_link(bucket.new("foo3"), "tag2!@#%^&*)") \ .store() - obj = bucket.get("test_store_and_get_links") + obj = bucket.get(self.key_name) links = obj.links self.assertEqual(len(links), 3) for bucket, key, tag in links: @@ -555,4 +555,4 @@ def test_stream_cleanoperationsup(self): # This should not raise an exception obj = bucket.get('one') - self.assertEqual(1, obj.data) + self.assertEqual('1', obj.encoded_data) diff --git a/riak/transports/http/codec.py b/riak/transports/http/codec.py new file mode 100644 index 00000000..c2896f3f --- /dev/null +++ b/riak/transports/http/codec.py @@ -0,0 +1,255 @@ +""" +Copyright 2012 Basho Technologies, Inc. +Copyright 2010 Rusty Klophaus +Copyright 2010 Justin Sheehy +Copyright 2009 Jay Baird + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +# subtract length of "Link: " header string and newline +MAX_LINK_HEADER_SIZE = 8192 - 8 + + +import re +import csv +import urllib +from cgi import parse_header +from email import message_from_string +from xml.etree import ElementTree +from riak import RiakError +from riak.content import RiakContent +from riak.multidict import MultiDict +from riak.transports.http.search import XMLSearchResult + + +class RiakHttpCodec(object): + """ + Methods for HTTP transport that marshals and unmarshals HTTP + messages. + """ + + def _parse_body(self, robj, response, expected_statuses): + """ + Parse the body of an object response and populate the object. + """ + # If no response given, then return. + if response is None: + return None + + status, headers, data = response + + # Check if the server is down(status==0) + if not status: + m = 'Could not contact Riak Server: http://{0}:{1}!'.format( + self._node.host, self._node.http_port) + raise RiakError(m) + + # Make sure expected code came back + self.check_http_code(status, expected_statuses) + + if 'x-riak-vclock' in headers: + robj.vclock = headers['x-riak-vclock'] + + # If 404(Not Found), then clear the object. + if status == 404: + robj.siblings = [] + return None + # If 201 Created, we need to extract the location and set the + # key on the object. + elif status == 201: + robj.key = headers['location'].strip().split('/')[-1] + # If 300(Siblings), apply the siblings to the object + elif status == 300: + ctype, params = parse_header(headers['content-type']) + if ctype == 'multipart/mixed': + boundary = re.compile('\r?\n--%s(?:--)?\r?\n' % + re.escape(params['boundary'])) + parts = [message_from_string(p) + for p in re.split(boundary, data)[1:-1]] + robj.siblings = [self._parse_sibling(RiakContent(robj), + part.items(), + part.get_payload()) + for part in parts] + return robj + else: + raise Exception('unexpected sibling response format: {0}'. + format(ctype)) + + robj.siblings = [self._parse_sibling(RiakContent(robj), + headers.items(), data)] + return robj + + def _parse_sibling(self, sibling, headers, data): + """ + Parses a single sibling out of a response. + """ + + sibling.exists = True + + # Parse the headers... + for header, value in headers: + header = header.lower() + if header == 'content-type': + sibling.content_type, sibling.charset = \ + self._parse_content_type(value) + elif header == 'etag': + sibling.etag = value + elif header == 'link': + sibling.links = self._parse_links(value) + elif header == 'last-modified': + sibling.last_modified = value + elif header.startswith('x-riak-meta-'): + metakey = header.replace('x-riak-meta-', '') + sibling.usermeta[metakey] = value + elif header.startswith('x-riak-index-'): + field = header.replace('x-riak-index-', '') + reader = csv.reader([value], skipinitialspace=True) + for line in reader: + for token in line: + if field.endswith("_int"): + token = int(token) + sibling.add_index(field, token) + elif header == 'x-riak-deleted': + sibling.exists = False + + sibling.encoded_data = data + + return sibling + + def _to_link_header(self, link): + """ + Convert the link tuple to a link header string. Used internally. + """ + try: + bucket, key, tag = link + except ValueError: + raise RiakError("Invalid link tuple %s" % link) + tag = tag if tag is not None else bucket + url = self.object_path(bucket, key) + header = '<%s>; riaktag="%s"' % (url, tag) + return header + + def _parse_links(self, linkHeaders): + links = [] + oldform = "; ?riaktag=\"([^\"]+)\"" + newform = "; ?riaktag=\"([^\"]+)\"" + for linkHeader in linkHeaders.strip().split(','): + linkHeader = linkHeader.strip() + matches = (re.match(oldform, linkHeader) or + re.match(newform, linkHeader)) + if matches is not None: + link = (urllib.unquote_plus(matches.group(2)), + urllib.unquote_plus(matches.group(3)), + urllib.unquote_plus(matches.group(4))) + links.append(link) + return links + + def _add_links_for_riak_object(self, robject, headers): + links = robject.links + if links: + current_header = '' + for link in links: + header = self._to_link_header(link) + if len(current_header + header) > MAX_LINK_HEADER_SIZE: + headers.add('Link', current_header) + current_header = '' + + if current_header != '': + header = ', ' + header + current_header += header + + headers.add('Link', current_header) + + return headers + + def _build_put_headers(self, robj, if_none_match=False): + """Build the headers for a POST/PUT request.""" + + # Construct the headers... + if robj.charset is not None: + content_type = ('%s; charset="%s"' % + (robj.content_type, robj.charset)) + else: + content_type = robj.content_type + + headers = MultiDict({'Content-Type': content_type, + 'X-Riak-ClientId': self._client_id}) + + # Add the vclock if it exists... + if robj.vclock is not None: + headers['X-Riak-Vclock'] = robj.vclock + + # Create the header from metadata + self._add_links_for_riak_object(robj, headers) + + for key, value in robj.usermeta.iteritems(): + headers['X-Riak-Meta-%s' % key] = value + + for field, value in robj.indexes: + key = 'X-Riak-Index-%s' % field + if key in headers: + headers[key] += ", " + str(value) + else: + headers[key] = str(value) + + if if_none_match: + headers['If-None-Match'] = '*' + + return headers + + def _normalize_json_search_response(self, json): + """ + Normalizes a JSON search response so that PB and HTTP have the + same return value + """ + result = {} + if u'response' in json: + result['num_found'] = json[u'response'][u'numFound'] + result['max_score'] = float(json[u'response'][u'maxScore']) + docs = [] + for doc in json[u'response'][u'docs']: + resdoc = {u'id': doc[u'id']} + if u'fields' in doc: + for k, v in doc[u'fields'].iteritems(): + resdoc[k] = v + docs.append(resdoc) + result['docs'] = docs + return result + + def _normalize_xml_search_response(self, xml): + """ + Normalizes an XML search response so that PB and HTTP have the + same return value + """ + target = XMLSearchResult() + parser = ElementTree.XMLParser(target=target) + parser.feed(xml) + return parser.close() + + def _parse_content_type(self, value): + """ + Split the content-type header into two parts: + 1) Actual main/sub encoding type + 2) charset + + :param value: Complete MIME content-type string + """ + content_type, params = parse_header(value) + if 'charset' in params: + charset = params['charset'] + else: + charset = None + return content_type, charset diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py index 3262b8d3..59a13ea9 100644 --- a/riak/transports/http/connection.py +++ b/riak/transports/http/connection.py @@ -26,19 +26,17 @@ class RiakHttpConnection(object): def _request(self, method, uri, headers={}, body='', stream=False): """ - Given a Method, URL, Headers, and Body, perform and HTTP request, - and return a 2-tuple containing a dictionary of response headers - and the response body. + Given a Method, URL, Headers, and Body, perform and HTTP + request, and return a 3-tuple containing the response status, + response headers (as httplib.HTTPMessage), and response body. """ response = None + headers.setdefault('Accept', + 'multipart/mixed, application/json, */*;q=0.5') try: self._connection.request(method, uri, body, headers) response = self._connection.getresponse() - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value - if stream: # The caller is responsible for fully reading the # response and closing it when streaming. @@ -49,7 +47,7 @@ def _request(self, method, uri, headers={}, body='', stream=False): if response and not stream: response.close() - return response_headers, response_body + return response.status, response.msg, response_body def _connect(self): self._connection = self._connection_class(self._node.host, diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index ee147f3c..460f85de 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -24,29 +24,21 @@ except ImportError: import json -import urllib -import re -import csv + import httplib -from email.message import Message +from xml.dom.minidom import Document from riak.transports.transport import RiakTransport from riak.transports.http.resources import RiakHttpResources from riak.transports.http.connection import RiakHttpConnection -from riak.transports.http.search import XMLSearchResult +from riak.transports.http.codec import RiakHttpCodec from riak.transports.http.stream import ( RiakHttpKeyStream, RiakHttpMapReduceStream) from riak import RiakError -from riak.multidict import MultiDict -from xml.etree import ElementTree -from xml.dom.minidom import Document -# subtract length of "Link: " header string and newline -MAX_LINK_HEADER_SIZE = 8192 - 8 - - -class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakTransport): +class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakHttpCodec, + RiakTransport): """ The RiakHttpTransport object holds information necessary to connect to Riak via HTTP. @@ -74,17 +66,17 @@ def ping(self): """ Check server is alive over HTTP """ - response = self._request('GET', self.ping_path()) - return(response is not None) and (response[1] == 'OK') + status, _, body = self._request('GET', self.ping_path()) + return(status is not None) and (body == 'OK') def stats(self): """ Gets performance statistics and server information """ - response = self._request('GET', self.stats_path(), - {'Accept': 'application/json'}) - if response[0]['http_code'] is 200: - return json.loads(response[1]) + status, _, body = self._request('GET', self.stats_path(), + {'Accept': 'application/json'}) + if status == 200: + return json.loads(body) else: return None @@ -106,75 +98,51 @@ def get_resources(self): Gets a JSON mapping of server-side resource names to paths :rtype dict """ - response = self._request('GET', '/', {'Accept': 'application/json'}) - if response[0]['http_code'] is 200: - return json.loads(response[1]) + status, _, body = self._request('GET', '/', + {'Accept': 'application/json'}) + if status == 200: + return json.loads(body) else: return {} - def get(self, robj, r=None, pr=None, vtag=None): + def get(self, robj, r=None, pr=None): """ Get a bucket/key from the server """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'r': r, 'pr': pr, 'vtag': vtag} + params = {'r': r, 'pr': pr} url = self.object_path(robj.bucket.name, robj.key, **params) response = self._request('GET', url) - return self.parse_body(robj, response, [200, 300, 404]) + return self._parse_body(robj, response, [200, 300, 404]) def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """ - Serialize put request and deserialize response + Puts a (possibly new) object. """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} url = self.object_path(robj.bucket.name, robj.key, **params) - headers = self._build_put_headers(robj) - - # TODO: use a more general 'prevent_stale_writes' semantics, - # which is a superset of the if_none_match semantics. - if if_none_match: - headers["If-None-Match"] = "*" + headers = self._build_put_headers(robj, if_none_match=if_none_match) content = robj.encoded_data - return self.do_put(url, headers, content, robj, return_body) - def do_put(self, url, headers, content, robj, return_body=False): if robj.key is None: - response = self._request('POST', url, headers, content) + expect = [201] + method = 'POST' else: - response = self._request('PUT', url, headers, content) + expect = [204] + method = 'PUT' + response = self._request(method, url, headers, content) if return_body: - return self.parse_body(robj, response, [200, 201, 204, 300]) + return self._parse_body(robj, response, [200, 201, 204, 300]) else: - self.check_http_code(response, [204]) + self.check_http_code(response[0], expect) return None - def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """Put a new object into the Riak store, returning its (new) key.""" - # We could detect quorum_controls here but HTTP ignores - # unknown flags/params. - params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} - url = self.object_path(robj.bucket.name, **params) - headers = self._build_put_headers(robj) - # TODO: use a more general 'prevent_stale_writes' semantics, - # which is a superset of the if_none_match semantics. - if if_none_match: - headers["If-None-Match"] = "*" - content = robj.encoded_data - response = self._request('POST', url, headers, content) - location = response[0]['location'] - idx = location.rindex('/') - robj.key = location[(idx + 1):] - if return_body: - return self.parse_body(robj, response, [201]) - else: - self.check_http_code(response, [201]) - return None + put_new = put def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ @@ -188,7 +156,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): if self.tombstone_vclocks() and robj.vclock is not None: headers['X-Riak-Vclock'] = robj.vclock response = self._request('DELETE', url, headers) - self.check_http_code(response, [204, 404]) + self.check_http_code(response[0], [204, 404]) return self def get_keys(self, bucket): @@ -196,20 +164,19 @@ def get_keys(self, bucket): Fetch a list of keys for the bucket """ url = self.key_list_path(bucket.name) - response = self._request('GET', url) + status, _, body = self._request('GET', url) - headers, encoded_props = response[0:2] - if headers['http_code'] == 200: - props = json.loads(encoded_props) + if status == 200: + props = json.loads(body) return props['keys'] else: raise Exception('Error listing keys.') def stream_keys(self, bucket): url = self.key_list_path(bucket.name, keys='stream') - headers, response = self._request('GET', url, stream=True) + status, headers, response = self._request('GET', url, stream=True) - if headers['http_code'] == 200: + if status == 200: return RiakHttpKeyStream(response) else: raise Exception('Error listing keys.') @@ -219,11 +186,10 @@ def get_buckets(self): Fetch a list of all buckets """ url = self.bucket_list_path() - response = self._request('GET', url) + status, headers, body = self._request('GET', url) - headers, encoded_props = response[0:2] - if headers['http_code'] == 200: - props = json.loads(encoded_props) + if status == 200: + props = json.loads(body) return props['buckets'] else: raise Exception('Error getting buckets.') @@ -234,12 +200,10 @@ def get_bucket_props(self, bucket): """ # Run the request... url = self.bucket_properties_path(bucket.name) - response = self._request('GET', url) + status, headers, body = self._request('GET', url) - headers = response[0] - encoded_props = response[1] - if headers['http_code'] == 200: - props = json.loads(encoded_props) + if status == 200: + props = json.loads(body) return props['props'] else: raise Exception('Error getting bucket properties.') @@ -253,14 +217,8 @@ def set_bucket_props(self, bucket, props): content = json.dumps({'props': props}) # Run the request... - response = self._request('PUT', url, headers, content) - - # Handle the response... - if response is None: - raise Exception('Error setting bucket properties.') + status, _, _ = self._request('PUT', url, headers, content) - # Check the response value... - status = response[0]['http_code'] if status != 204: raise Exception('Error setting bucket properties.') return True @@ -273,14 +231,8 @@ def clear_bucket_props(self, bucket): headers = {'Content-Type': 'application/json'} # Run the request... - response = self._request('DELETE', url, headers, None) + status, _, _ = self._request('DELETE', url, headers, None) - # Handle the response... - if response is None: - raise Exception('Error clearing bucket properties.') - - # Check the response value... - status = response[0]['http_code'] if status == 204: return True elif status == 405: @@ -299,16 +251,15 @@ def mapred(self, inputs, query, timeout=None): # Do the request... url = self.mapred_path() headers = {'Content-Type': 'application/json'} - response = self._request('POST', url, headers, content) + status, headers, body = self._request('POST', url, headers, content) # Make sure the expected status code came back... - status = response[0]['http_code'] if status != 200: raise RiakError( 'Error running MapReduce operation. Headers: %s Body: %s' % - (repr(response[0]), repr(response[1]))) + (repr(headers), repr(body))) - result = json.loads(response[1]) + result = json.loads(body) return result def stream_mapred(self, inputs, query, timeout=None): @@ -316,10 +267,10 @@ def stream_mapred(self, inputs, query, timeout=None): url = self.mapred_path(chunked=True) reqheaders = {'Content-Type': 'application/json'} - headers, response = self._request('POST', url, reqheaders, - content, stream=True) + status, headers, response = self._request('POST', url, reqheaders, + content, stream=True) - if headers['http_code'] is 200: + if status == 200: return RiakHttpMapReduceStream(response) else: raise Exception( @@ -331,10 +282,9 @@ def get_index(self, bucket, index, startkey, endkey=None): Performs a secondary index query. """ url = self.index_path(bucket, index, startkey, endkey) - response = self._request('GET', url) - headers, data = response - self.check_http_code(response, [200]) - json_data = json.loads(data) + status, headers, body = self._request('GET', url) + self.check_http_code(status, [200]) + json_data = json.loads(body) return json_data[u'keys'][:] def search(self, index, query, **params): @@ -351,9 +301,8 @@ def search(self, index, query, **params): options.update(params) url = self.solr_select_path(index, query, **options) - response = self._request('GET', url) - headers, data = response - self.check_http_code(response, [200]) + status, headers, data = self._request('GET', url) + self.check_http_code(status, [200]) if 'json' in headers['content-type']: results = json.loads(data) return self._normalize_json_search_response(results) @@ -409,235 +358,7 @@ def fulltext_delete(self, index, docs=None, queries=None): {'Content-Type': 'text/xml'}, xml.toxml().encode('utf-8')) - def check_http_code(self, response, expected_statuses): - status = response[0]['http_code'] + def check_http_code(self, status, expected_statuses): if not status in expected_statuses: - raise Exception('Expected status %s, received %s : %s' % - (expected_statuses, status, response[1])) - - def parse_body(self, robj, response, expected_statuses): - """ - Parse the body of an object response and populate the object. - """ - # If no response given, then return. - if response is None: - return None - - # Make sure expected code came back - self.check_http_code(response, expected_statuses) - - # Update the object... - headers = response[0] - data = response[1] - status = headers['http_code'] - - # Check if the server is down(status==0) - if not status: - ### we need the host/port that was used. - m = 'Could not contact Riak Server: http://$HOST:$PORT !' - raise RiakError(m) - - # If 404(Not Found), then clear the object. - if status == 404: - return None - - # If 300(Siblings), then return the list of siblings - elif status == 300: - # Parse and get rid of 'Siblings:' string in element 0 - siblings = data.strip().split('\n') - siblings.pop(0) - robj.siblings = siblings - robj.exists = True - robj.vclock = headers['x-riak-vclock'] - return robj - - #no sibs - robj.siblings = [] - - # Parse the headers... - links = [] - for header, value in headers.iteritems(): - if header == 'content-type': - robj.content_type, robj.charset = \ - self._parse_content_type(value) - elif header == 'content-encoding': - robj.content_encoding = value - elif header == 'etag': - robj.etag = value - elif header == 'link': - self._parse_links(links, headers['link']) - elif header == 'last-modified': - robj.last_modified = value - elif header.startswith('x-riak-meta-'): - metakey = header.replace('x-riak-meta-', '') - robj.usermeta[metakey] = value - elif header.startswith('x-riak-index-'): - field = header.replace('x-riak-index-', '') - reader = csv.reader([value], skipinitialspace=True) - for line in reader: - for token in line: - if field.endswith("_int"): - token = int(token) - robj.add_index(field, token) - elif header == 'x-riak-vclock': - robj.vclock = value - elif header == 'x-riak-deleted': - robj.deleted = True - if links: - robj.links = links - - robj.encoded_data = data - - robj.exists = True - return robj - - def to_link_header(self, link): - """ - Convert the link tuple to a link header string. Used internally. - """ - try: - bucket, key, tag = link - except ValueError: - raise RiakError("Invalid link tuple %s" % link) - tag = tag if tag is not None else bucket - url = self.object_path(bucket, key) - header = '<%s>; riaktag="%s"' % (url, tag) - return header - - def _parse_links(self, links, linkHeaders): - oldform = "; ?riaktag=\"([^\"]+)\"" - newform = "; ?riaktag=\"([^\"]+)\"" - for linkHeader in linkHeaders.strip().split(','): - linkHeader = linkHeader.strip() - matches = (re.match(oldform, linkHeader) or - re.match(newform, linkHeader)) - if matches is not None: - link = (urllib.unquote_plus(matches.group(2)), - urllib.unquote_plus(matches.group(3)), - urllib.unquote_plus(matches.group(4))) - links.append(link) - return links - - def _add_links_for_riak_object(self, robject, headers): - links = robject.links - if links: - current_header = '' - for link in links: - header = self.to_link_header(link) - if len(current_header + header) > MAX_LINK_HEADER_SIZE: - headers.add('Link', current_header) - current_header = '' - - if current_header != '': - header = ', ' + header - current_header += header - - headers.add('Link', current_header) - - return headers - - # Utility functions used by Riak library. - - def _build_put_headers(self, robj): - """Build the headers for a POST/PUT request.""" - - # Construct the headers... - if robj.charset is not None: - content_type = ('%s; charset="%s"' % - (robj.content_type, robj.charset)) - else: - content_type = robj.content_type - headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', - 'Content-Type': content_type, - 'X-Riak-ClientId': self._client_id}) - # Add the vclock if it exists... - if robj.vclock is not None: - headers['X-Riak-Vclock'] = robj.vclock - - # Create the header from metadata - self._add_links_for_riak_object(robj, headers) - - for key, value in robj.usermeta.iteritems(): - headers['X-Riak-Meta-%s' % key] = value - - for field, value in robj.indexes: - key = 'X-Riak-Index-%s' % field - if key in headers: - headers[key] += ", " + str(value) - else: - headers[key] = str(value) - - return headers - - def _normalize_json_search_response(self, json): - """ - Normalizes a JSON search response so that PB and HTTP have the - same return value - """ - result = {} - if u'response' in json: - result['num_found'] = json[u'response'][u'numFound'] - result['max_score'] = float(json[u'response'][u'maxScore']) - docs = [] - for doc in json[u'response'][u'docs']: - resdoc = {u'id': doc[u'id']} - if u'fields' in doc: - for k, v in doc[u'fields'].iteritems(): - resdoc[k] = v - docs.append(resdoc) - result['docs'] = docs - return result - - def _normalize_xml_search_response(self, xml): - """ - Normalizes an XML search response so that PB and HTTP have the - same return value - """ - target = XMLSearchResult() - parser = ElementTree.XMLParser(target=target) - parser.feed(xml) - return parser.close() - - def _parse_content_type(self, value): - """ - Split the content-type header into two parts: - 1) Actual main/sub encoding type - 2) charset - - :param value: Complete MIME content-type string - """ - message = Message() - message.set_type(value) - - content_type = message.get_content_type() - charset = message.get_content_charset(None) - - return content_type, charset - - @classmethod - def build_headers(cls, headers): - return ['%s: %s' % (header, value) - for header, value in headers.iteritems()] - - @classmethod - def parse_http_headers(cls, headers): - """ - Parse an HTTP Header string into an associative array of - response headers. - """ - retVal = {} - fields = headers.split("\n") - for field in fields: - matches = re.match("([^:]+):(.+)", field) - if matches is None: - continue - key = matches.group(1).lower() - value = matches.group(2).strip() - if key in retVal.keys(): - if isinstance(retVal[key], list): - retVal[key].append(value) - else: - retVal[key] = [retVal[key]].append(value) - else: - retVal[key] = value - return retVal + raise Exception('Expected status %s, received %s' % + (expected_statuses, status)) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index d41d8ed6..e24b8697 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -16,7 +16,9 @@ under the License. """ import riak_pb -from riak.riak_object import RiakObject +from riak import RiakError +from riak.content import RiakContent +from riak.mapreduce import RiakLink RIAKC_RW_ONE = 4294967294 RIAKC_RW_QUORUM = 4294967293 @@ -58,7 +60,17 @@ def translate_rw_val(self, rw): else: return None - def decode_content(self, rpb_content, robj): + def _decoded_contents(self, resp, obj): + if type(resp) == riak_pb.RpbPutResp and resp.HasField('key'): + obj.key = resp.key + if resp.HasField("vclock"): + obj.vclock = resp.vclock + + obj.siblings = [self._decode_content(c, RiakContent(obj)) + for c in resp.content] + return obj + + def _decode_content(self, rpb_content, sibling): """ Decodes a single sibling from the protobuf representation into a RiakObject. @@ -66,58 +78,38 @@ def decode_content(self, rpb_content, robj): :rtype: (RiakObject) """ - if rpb_content.HasField("deleted"): - robj.deleted = True + if rpb_content.HasField("deleted") and rpb_content.deleted: + sibling.exists = False + else: + sibling.exists = True if rpb_content.HasField("content_type"): - robj.content_type = rpb_content.content_type + sibling.content_type = rpb_content.content_type if rpb_content.HasField("charset"): - robj.charset = rpb_content.charset + sibling.charset = rpb_content.charset if rpb_content.HasField("content_encoding"): - robj.content_encoding = rpb_content.content_encoding + sibling.content_encoding = rpb_content.content_encoding if rpb_content.HasField("vtag"): - robj.vtag = rpb_content.vtag - links = [] - for link in rpb_content.links: - if link.HasField("bucket"): - bucket = link.bucket - else: - bucket = None - if link.HasField("key"): - key = link.key - else: - key = None - if link.HasField("tag"): - tag = link.tag - else: - tag = None - links.append((bucket, key, tag)) - if links: - robj.links = links + sibling.etag = rpb_content.vtag + + sibling.links = [self._decode_link(link) + for link in rpb_content.links] if rpb_content.HasField("last_mod"): - robj.last_mod = rpb_content.last_mod + sibling.last_mod = rpb_content.last_mod if rpb_content.HasField("last_mod_usecs"): - robj.last_mod_usecs = rpb_content.last_mod_usecs - usermeta = {} - for usermd in rpb_content.usermeta: - usermeta[usermd.key] = usermd.value - if len(usermeta) > 0: - robj.usermeta = usermeta - indexes = set() - for index in rpb_content.indexes: - if index.key.endswith("_int"): - indexes.add((index.key, int(index.value))) - else: - indexes.add((index.key, index.value)) + sibling.last_mod_usecs = rpb_content.last_mod_usecs - if len(indexes) > 0: - robj.indexes = indexes + sibling.usermeta = dict([(usermd.key, usermd.value) + for usermd in rpb_content.usermeta]) + sibling.indexes = set([(index.key, + self._decode_index_value(index.key, + index.value)) + for index in rpb_content.indexes]) - robj.encoded_data = rpb_content.value - robj.exists = True + sibling.encoded_data = rpb_content.value - return robj + return sibling - def encode_content(self, robj, rpb_content): + def _encode_content(self, robj, rpb_content): """ Fills an RpbContent message with the appropriate data and metadata from a RiakObject. @@ -147,8 +139,37 @@ def encode_content(self, robj, rpb_content): pb_link.tag = '' for field, value in robj.indexes: - pair = rpb_content.indexes.add() - pair.key = field - pair.value = str(value) + pair = rpb_content.indexes.add() + pair.key = field + pair.value = str(value) rpb_content.value = str(robj.encoded_data) + + def _decode_link(self, link): + """ + Decodes an RpbLink message into a RiakLink named tuple + """ + + if link.HasField("bucket"): + bucket = link.bucket + else: + bucket = None + if link.HasField("key"): + key = link.key + else: + key = None + if link.HasField("tag"): + tag = link.tag + else: + tag = None + + return RiakLink(bucket, key, tag) + + def _decode_index_value(self, index, value): + """ + Decodes a secondary index value into the correct Python type. + """ + if index.endswith("_int"): + return int(value) + else: + return value diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index 86e409b4..a461558f 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -83,7 +83,8 @@ def _recv_pkt(self): def _connect(self): if self._timeout: - self._socket = socket.create_connection(self._address, self._timeout) + self._socket = socket.create_connection(self._address, + self._timeout) else: self._socket = socket.create_connection(self._address) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 542e97fb..fe668009 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -25,7 +25,6 @@ from connection import RiakPbcConnection from stream import RiakPbcKeyStream, RiakPbcMapredStream from codec import RiakPbcCodec -from riak.riak_object import RiakObject from messages import ( MSG_CODE_PING_REQ, @@ -115,28 +114,10 @@ def _set_client_id(self, client_id): client_id = property(_get_client_id, _set_client_id, doc="""the client ID for this connection""") - def _decoded_contents(self, resp, old_obj): - contents = [] - for c in resp.content: - new_obj = RiakObject(old_obj.client, old_obj.bucket, old_obj.key) - new_obj.vclock = resp.vclock - contents.append(self.decode_content(c, new_obj)) - if contents: - ret = contents[0] - if len(contents) > 1: - ret.siblings = contents[:] - return ret - else: - old_obj.exists = False - return old_obj - - def get(self, robj, r=None, pr=None, vtag=None): + def get(self, robj, r=None, pr=None): """ Serialize get request and deserialize response """ - if vtag is not None: - raise RiakError("PB transport does not support vtags") - bucket = robj.bucket req = riak_pb.RpbGetReq() @@ -178,57 +159,24 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, req.if_none_match = 1 req.bucket = bucket.name - req.key = robj.key + if robj.key: + req.key = robj.key if robj.vclock: req.vclock = robj.vclock - self.encode_content(robj, req.content) + self._encode_content(robj, req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, MSG_CODE_PUT_RESP) + if resp is not None: return self._decoded_contents(resp, robj) - - def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """Put a new object into the Riak store, returning its (new) key. - - If return_meta is False, then the vlock and metadata return values - will be None. - - @return robj - """ - # Note that this won't work on 0.14 nodes. - bucket = robj.bucket - - req = riak_pb.RpbPutReq() - if w: - req.w = self.translate_rw_val(w) - if dw: - req.dw = self.translate_rw_val(dw) - if self.quorum_controls() and pw: - req.pw = self.translate_rw_val(pw) - - if return_body: - req.return_body = 1 - if if_none_match: - req.if_none_match = 1 - - req.bucket = bucket.name - - self.encode_content(robj, req.content) - - msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, - MSG_CODE_PUT_RESP) - if not resp: + elif not robj.key: raise RiakError("missing response object") - if len(resp.content) != 1: - raise RiakError("siblings were returned from object creation") + else: + return robj - robj.key = resp.key - robj.vclock = resp.vclock - content = self.decode_content(resp.content[0], robj) - return content + put_new = put def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 53615853..390a83a8 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -65,7 +65,7 @@ def ping(self): """ raise NotImplementedError - def get(self, robj, r=None, vtag=None): + def get(self, robj, r=None): """ Serialize get request and deserialize response @return (vclock=None, [(metadata, value)]=None) From 8d3a083f1194025829ab8bdaf15dd5384680a094 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 28 Apr 2013 06:13:20 -0500 Subject: [PATCH 0389/1060] Correct usage and handling of last_modified metadata. --- riak/content.py | 5 +++-- riak/riak_object.py | 12 ++++++++++-- riak/transports/http/codec.py | 3 ++- riak/transports/pbc/codec.py | 6 +++--- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/riak/content.py b/riak/content.py index f19606b7..dce93a4e 100644 --- a/riak/content.py +++ b/riak/content.py @@ -27,14 +27,15 @@ class RiakContent(object): """ def __init__(self, robject, data=None, encoded_data=None, charset=None, content_type='application/json', content_encoding=None, - etag=None, usermeta=None, links=None, indexes=None, - exists=False): + last_modified=None, etag=None, usermeta=None, links=None, + indexes=None, exists=False): self._robject = robject self._data = data self._encoded_data = encoded_data self.charset = charset self.content_type = content_type self.content_encoding = content_encoding + self.last_modified = last_modified self.etag = etag self.usermeta = usermeta or {} self.links = links or [] diff --git a/riak/riak_object.py b/riak/riak_object.py index eee42db8..faac0822 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -124,12 +124,20 @@ def __ne__(self, other): charset = content_property('charset', doc=""" The character set of the encoded data :type string""") - content_type = content_property('content_type', doc=""" The MIME media type of the encoded data :type string""") + content_encoding = content_property('content_encoding', doc=""" + The encoding (compression) of the encoded data. Valid values + are identity, deflate, gzip + :type string""") - content_encoding = content_property('content_encoding') + last_modified = content_property('last_modified', """ + The UNIX timestamp of the modification time of this value. + :type float""") + etag = content_property('etag', """ + A unique entity-tag for the value. + :type string""") usermeta = content_property('usermeta', doc=""" Arbitrary user-defined metadata, mapping strings to strings. diff --git a/riak/transports/http/codec.py b/riak/transports/http/codec.py index c2896f3f..a884634c 100644 --- a/riak/transports/http/codec.py +++ b/riak/transports/http/codec.py @@ -28,6 +28,7 @@ import urllib from cgi import parse_header from email import message_from_string +from rfc822 import parsedate_tz, mktime_tz from xml.etree import ElementTree from riak import RiakError from riak.content import RiakContent @@ -110,7 +111,7 @@ def _parse_sibling(self, sibling, headers, data): elif header == 'link': sibling.links = self._parse_links(value) elif header == 'last-modified': - sibling.last_modified = value + sibling.last_modified = mktime_tz(parsedate_tz(value)) elif header.startswith('x-riak-meta-'): metakey = header.replace('x-riak-meta-', '') sibling.usermeta[metakey] = value diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index e24b8697..61202e48 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -94,9 +94,9 @@ def _decode_content(self, rpb_content, sibling): sibling.links = [self._decode_link(link) for link in rpb_content.links] if rpb_content.HasField("last_mod"): - sibling.last_mod = rpb_content.last_mod - if rpb_content.HasField("last_mod_usecs"): - sibling.last_mod_usecs = rpb_content.last_mod_usecs + sibling.last_modified = float(rpb_content.last_mod) + if rpb_content.HasField("last_mod_usecs"): + sibling.last_modified += rpb_content.last_mod_usecs / 1000000.0 sibling.usermeta = dict([(usermd.key, usermd.value) for usermd in rpb_content.usermeta]) From cce4cae176d1cc2c185185dd9046c72d1b9ca1a6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 28 Apr 2013 06:49:31 -0500 Subject: [PATCH 0390/1060] Normalize handling of vector clocks across transports. This addresses the possibility that an object might be fetched from one interface/transport and stored in the other. Instead, we wrap the raw value in an object that can handle decoding and encoding the format that the transport requires. --- riak/riak_object.py | 31 +++++++++++++++++++++++++++++++ riak/transports/http/codec.py | 5 +++-- riak/transports/http/transport.py | 2 +- riak/transports/pbc/codec.py | 3 ++- riak/transports/pbc/transport.py | 4 ++-- 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index faac0822..c9afe978 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -21,6 +21,7 @@ from riak import ConflictError from riak.content import RiakContent from riak.util import deprecated +import base64 def content_property(name, doc=None): @@ -58,6 +59,36 @@ def _delegate(self, *args, **kwargs): return _delegate +class VClock(object): + """ + A representation of a vector clock received from Riak. + """ + + _decoders = { + 'base64': base64.b64decode, + 'binary': str + } + + _encoders = { + 'base64': base64.b64encode, + 'binary': str + } + + def __init__(self, value, encoding): + self._vclock = self._decoders[encoding].__call__(value) + + def encode(self, encoding): + if encoding in self._encoders: + return self._encoders[encoding].__call__(self._vclock) + else: + raise ValueError('{} is not a valid vector clock encoding'. + format(encoding)) + + def __repr__(self): + return '<{} {}>'.format(self.__class__.__name__, + self.encode('base64')) + + class RiakObject(object): """ The RiakObject holds meta information about a Riak object, plus the diff --git a/riak/transports/http/codec.py b/riak/transports/http/codec.py index a884634c..1ef30129 100644 --- a/riak/transports/http/codec.py +++ b/riak/transports/http/codec.py @@ -32,6 +32,7 @@ from xml.etree import ElementTree from riak import RiakError from riak.content import RiakContent +from riak.riak_object import VClock from riak.multidict import MultiDict from riak.transports.http.search import XMLSearchResult @@ -62,7 +63,7 @@ def _parse_body(self, robj, response, expected_statuses): self.check_http_code(status, expected_statuses) if 'x-riak-vclock' in headers: - robj.vclock = headers['x-riak-vclock'] + robj.vclock = VClock(headers['x-riak-vclock'], 'base64') # If 404(Not Found), then clear the object. if status == 404: @@ -191,7 +192,7 @@ def _build_put_headers(self, robj, if_none_match=False): # Add the vclock if it exists... if robj.vclock is not None: - headers['X-Riak-Vclock'] = robj.vclock + headers['X-Riak-Vclock'] = robj.vclock.encode('base64') # Create the header from metadata self._add_links_for_riak_object(robj, headers) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 460f85de..c80ceb33 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -154,7 +154,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): headers = {} url = self.object_path(robj.bucket.name, robj.key, **params) if self.tombstone_vclocks() and robj.vclock is not None: - headers['X-Riak-Vclock'] = robj.vclock + headers['X-Riak-Vclock'] = robj.vclock.encode('base64') response = self._request('DELETE', url, headers) self.check_http_code(response[0], [204, 404]) return self diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 61202e48..7b2b08f0 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -17,6 +17,7 @@ """ import riak_pb from riak import RiakError +from riak.riak_object import VClock from riak.content import RiakContent from riak.mapreduce import RiakLink @@ -64,7 +65,7 @@ def _decoded_contents(self, resp, obj): if type(resp) == riak_pb.RpbPutResp and resp.HasField('key'): obj.key = resp.key if resp.HasField("vclock"): - obj.vclock = resp.vclock + obj.vclock = VClock(resp.vclock, 'binary') obj.siblings = [self._decode_content(c, RiakContent(obj)) for c in resp.content] diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index fe668009..e43b347b 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -162,7 +162,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, if robj.key: req.key = robj.key if robj.vclock: - req.vclock = robj.vclock + req.vclock = robj.vclock.encode('binary') self._encode_content(robj, req.content) @@ -201,7 +201,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): req.pw = self.translate_rw_val(pw) if self.tombstone_vclocks() and robj.vclock: - req.vclock = robj.vclock + req.vclock = robj.vclock.encode('binary') req.bucket = bucket.name req.key = robj.key From 428842f380fd7e31827d06aa46a5707fcb358b9a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 30 Apr 2013 11:12:53 -0500 Subject: [PATCH 0391/1060] Remove put_new from transports. --- riak/client/operations.py | 25 ------------------------- riak/riak_object.py | 12 +++--------- riak/transports/http/transport.py | 2 -- riak/transports/pbc/transport.py | 2 -- riak/transports/transport.py | 10 ---------- 5 files changed, 3 insertions(+), 48 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index cf4cc46f..3c812e59 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -154,31 +154,6 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, return_body=return_body, if_none_match=if_none_match) - @retryable - def put_new(self, transport, robj, w=None, dw=None, pw=None, - return_body=None, if_none_match=None): - """ - Stores an object in the Riak cluster with a generated key. - - :param robj: the object to store - :type robj: RiakObject - :param w: the write quorum - :type w: integer, string, None - :param dw: the durable write quorum - :type dw: integer, string, None - :param pw: the primary write quorum - :type pw: integer, string, None - :param return_body: whether to return the resulting object - after the write - :type return_body: boolean - :param if_none_match: whether to fail the write if the object - exists - :type if_none_match: boolean - """ - return transport.put_new(robj, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) - @retryable def get(self, transport, robj, r=None, pr=None): """ diff --git a/riak/riak_object.py b/riak/riak_object.py index c9afe978..247f092e 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -240,15 +240,9 @@ def store(self, w=None, dw=None, pw=None, return_body=True, raise ConflictError("Attempting to store an invalid object, " "resolve the siblings first") - if self.key is None: - self.client.put_new( - self, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) - else: - self.client.put(self, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) + self.client.put(self, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) return self diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index c80ceb33..4824d0b9 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -142,8 +142,6 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, self.check_http_code(response[0], expect) return None - put_new = put - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Delete an object. diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index e43b347b..738c67e1 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -176,8 +176,6 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, else: return robj - put_new = put - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Serialize get request and deserialize response diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 390a83a8..c6d581c7 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -80,16 +80,6 @@ def put(self, robj, w=None, dw=None, return_body=True): """ raise NotImplementedError - def put_new(self, robj, w=None, dw=None, return_meta=True): - """Put a new object into the Riak store, returning its (new) key. - - If return_meta is False, then the vlock and metadata return values - will be None. - - @return (key, vclock, metadata) - """ - raise NotImplementedError - def delete(self, robj, rw=None): """ Serialize delete request and deserialize response From 09993fab769420fda3caa1def178091b25487450 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 1 May 2013 10:02:46 -0500 Subject: [PATCH 0392/1060] Normalize handling of get/put responses in PBC to prevent clobbering of object data. --- riak/transports/pbc/codec.py | 10 ++-------- riak/transports/pbc/transport.py | 32 +++++++++++++++++++++++++------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 7b2b08f0..c5a4dd9b 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -17,7 +17,6 @@ """ import riak_pb from riak import RiakError -from riak.riak_object import VClock from riak.content import RiakContent from riak.mapreduce import RiakLink @@ -61,14 +60,9 @@ def translate_rw_val(self, rw): else: return None - def _decoded_contents(self, resp, obj): - if type(resp) == riak_pb.RpbPutResp and resp.HasField('key'): - obj.key = resp.key - if resp.HasField("vclock"): - obj.vclock = VClock(resp.vclock, 'binary') - + def _decode_contents(self, contents, obj): obj.siblings = [self._decode_content(c, RiakContent(obj)) - for c in resp.content] + for c in contents] return obj def _decode_content(self, rpb_content, sibling): diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 738c67e1..c59c30d9 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -22,6 +22,7 @@ import riak_pb from riak import RiakError from riak.transports.transport import RiakTransport +from riak.riak_object import VClock from connection import RiakPbcConnection from stream import RiakPbcKeyStream, RiakPbcMapredStream from codec import RiakPbcCodec @@ -132,11 +133,23 @@ def get(self, robj, r=None, pr=None): req.bucket = bucket.name req.key = robj.key - msg_code, resp = self._request(MSG_CODE_GET_REQ, req) - if msg_code == MSG_CODE_GET_RESP: - return self._decoded_contents(resp, robj) + msg_code, resp = self._request(MSG_CODE_GET_REQ, req, + MSG_CODE_GET_RESP) + + # TODO: support if_modified flag + + if resp is not None: + if resp.HasField('vclock'): + robj.vclock = VClock(resp.vclock, 'binary') + # We should do this even if there are no contents, i.e. + # the object is tombstoned + self._decode_contents(resp.content, robj) else: - return None + # "not found" returns an empty message, + # so let's make sure to clear the siblings + robj.siblings = [] + + return robj def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): @@ -170,11 +183,16 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, MSG_CODE_PUT_RESP) if resp is not None: - return self._decoded_contents(resp, robj) + if resp.HasField('key'): + robj.key = resp.key + if resp.HasField("vclock"): + robj.vclock = VClock(resp.vclock, 'binary') + if resp.content: + self._decode_contents(resp.content, robj) elif not robj.key: raise RiakError("missing response object") - else: - return robj + + return robj def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ From 6f678e1b283f08ff4a622450c378ca87389e9944 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 1 May 2013 10:05:15 -0500 Subject: [PATCH 0393/1060] HTTP uses a plain tuple for links, so should PBC. --- riak/transports/pbc/codec.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index c5a4dd9b..db8647b4 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -18,7 +18,6 @@ import riak_pb from riak import RiakError from riak.content import RiakContent -from riak.mapreduce import RiakLink RIAKC_RW_ONE = 4294967294 RIAKC_RW_QUORUM = 4294967293 @@ -158,7 +157,7 @@ def _decode_link(self, link): else: tag = None - return RiakLink(bucket, key, tag) + return (bucket, key, tag) def _decode_index_value(self, index, value): """ From effc1d84506ce2640b3f6b9df22e594b2f675c70 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 1 May 2013 10:08:35 -0500 Subject: [PATCH 0394/1060] Correct documentation on exists property. --- riak/riak_object.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 247f092e..ec4e8786 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -203,8 +203,11 @@ def _exists(self): return self.siblings[0].exists exists = property(_exists, None, doc=""" - Whether the object exists. This is only true when there is a - single sibling and it is neither a tombstone nor unsaved.""") + Whether the object exists. This is only False when there are no + siblings (the object was not found), or the solitary sibling is + a tombstone. + :type bool + """) def get_sibling(self, index): deprecated("RiakObject.get_sibling is deprecated, use the " From 1f5337787df425a68a22f490c03f59247ae603a9 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 3 May 2013 11:36:15 -0500 Subject: [PATCH 0395/1060] Correct documentation of _decode_link method. It returns a bare tuple, not a named RiakLink tuple. --- riak/transports/pbc/codec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index db8647b4..7ee0b0e0 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -141,7 +141,7 @@ def _encode_content(self, robj, rpb_content): def _decode_link(self, link): """ - Decodes an RpbLink message into a RiakLink named tuple + Decodes an RpbLink message into a tuple """ if link.HasField("bucket"): From a3f70a038ce7946d832f6276b57e294006537155 Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 3 May 2013 22:34:47 -0700 Subject: [PATCH 0396/1060] Updating README and tutorial to reflect that get_data() is no longer used --- README.rst | 10 +++++----- docs/tutorial.rst | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 78787881..c93a5937 100644 --- a/README.rst +++ b/README.rst @@ -249,7 +249,7 @@ data out looks like:: # You've now got a ``RiakObject``. To get at the values in a dictionary # form, call: - johndoe_dict = johndoe.get_data() + johndoe_dict = johndoe.data Getting binary data out looks like:: @@ -261,7 +261,7 @@ Getting binary data out looks like:: johndoe = user_photo_bucket.get_binary('johndoe') # You've now got a ``RiakObject``. To get at the binary data, call: - johndoe_headshot = johndoe.get_data() + johndoe_headshot = johndoe.data Manually fetching data is also possible:: @@ -280,7 +280,7 @@ Manually fetching data is also possible:: first_post_status.reload(r) # Finally, pull out the data. - message = first_post_status.get_data()['message'] + message = first_post_status.data['message'] Fetching Data Via Map/Reduce @@ -391,7 +391,7 @@ Fetching the data is equally simple:: # Since what we get back are lightweight ``RiakLink`` objects, we need to # get the associated ``RiakObject`` to access its data. status = status_link.get() - print status.get_data()['message'] + print status.data['message'] Using Search @@ -414,7 +414,7 @@ tutorial, but usage of this feature looks like:: for result in search_query.run(): # You get ``RiakLink`` objects back. user = result.get() - user_data = user.get_data() + user_data = user.data print "%s %s" % (user_data['first_name'], user_data['last_name']) # Results in something like: diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 3e51bfff..2506d114 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -213,7 +213,7 @@ data out looks like:: # You've now got a ``RiakObject``. To get at the values in a dictionary # form, call: - johndoe_dict = johndoe.get_data() + johndoe_dict = johndoe.data Getting binary data out looks like:: @@ -225,7 +225,7 @@ Getting binary data out looks like:: johndoe = user_photo_bucket.get_binary('johndoe') # You've now got a ``RiakObject``. To get at the binary data, call: - johndoe_headshot = johndoe.get_data() + johndoe_headshot = johndoe.data Manually fetching data is also possible:: @@ -244,7 +244,7 @@ Manually fetching data is also possible:: first_post_status.reload(r) # Finally, pull out the data. - message = first_post_status.get_data()['message'] + message = first_post_status.data['message'] Fetching Data Via Map/Reduce @@ -355,7 +355,7 @@ Fetching the data is equally simple:: # Since what we get back are lightweight ``RiakLink`` objects, we need to # get the associated ``RiakObject`` to access its data. status = status_link.get() - print status.get_data()['message'] + print status.data['message'] Using Search @@ -378,7 +378,7 @@ tutorial, but usage of this feature looks like:: for result in search_query.run(): # You get ``RiakLink`` objects back. user = result.get() - user_data = user.get_data() + user_data = user.data print "%s %s" % (user_data['first_name'], user_data['last_name']) # Results in something like: From 3afe011b4102c12874bb97cae81d70884f8d6eff Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 7 May 2013 14:42:48 -0500 Subject: [PATCH 0397/1060] Add test for sibling tombstones. --- riak/tests/test_kv.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 880245bc..5eaf9325 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -297,6 +297,42 @@ def test_siblings(self): self.assertEqual(len(obj.siblings), 1) self.assertEqual(obj.encoded_data, resolved_sibling.encoded_data) + def test_tombstone_siblings(self): + # Set up the bucket, clear any existing object... + bucket = self.client.bucket(self.sibs_bucket) + obj = bucket.get(self.key_name) + bucket.allow_mult = True + + obj.encoded_data = 'start' + obj.content_type = 'application/octet-stream' + obj.store(return_body=True) + + vclock = obj.vclock + obj.delete() + + vals = set() + for i in range(4): + while True: + randval = self.randint() + if str(randval) not in vals: + break + + other_obj = bucket.new(key=self.key_name, + encoded_data=str(randval), + content_type='text/plain') + other_obj.vclock = vclock + other_obj.store() + vals.add(str(randval)) + + obj = bucket.get(self.key_name) + self.assertEqual(len(obj.siblings), 5) + non_tombstones = 0 + for sib in obj.siblings: + if sib.exists: + non_tombstones += 1 + self.assertTrue(sib.encoded_data in vals or not sib.exists) + self.assertEqual(non_tombstones, 4) + def test_store_of_missing_object(self): bucket = self.client.bucket(self.bucket_name) # for json objects From ce58986dbbf282c7055b922aaa3b458503014c59 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 7 May 2013 14:43:31 -0500 Subject: [PATCH 0398/1060] whitespace cleanup --- riak/riak_object.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/riak/riak_object.py b/riak/riak_object.py index ec4e8786..92d6ec78 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -144,6 +144,7 @@ def __ne__(self, other): property into Python values. The decoding is dependent on the `content_type` property and the bucket's registered decoders. :type mixed """) + encoded_data = content_property('encoded_data', doc=""" The raw data stored in this object, essentially the encoded form of the `data` property. If unset, accessing this property @@ -155,9 +156,11 @@ def __ne__(self, other): charset = content_property('charset', doc=""" The character set of the encoded data :type string""") + content_type = content_property('content_type', doc=""" The MIME media type of the encoded data :type string""") + content_encoding = content_property('content_encoding', doc=""" The encoding (compression) of the encoded data. Valid values are identity, deflate, gzip @@ -166,6 +169,7 @@ def __ne__(self, other): last_modified = content_property('last_modified', """ The UNIX timestamp of the modification time of this value. :type float""") + etag = content_property('etag', """ A unique entity-tag for the value. :type string""") From 434394346d39c9124c69448d505791c823f61bad Mon Sep 17 00:00:00 2001 From: William Kral Date: Thu, 23 May 2013 12:40:08 -0700 Subject: [PATCH 0399/1060] Added text/plain encoding handling by default - The old version of the client handled this out of the box - Added a test to ensure it works --- .gitignore | 3 +++ riak/client/__init__.py | 6 ++++-- riak/tests/test_kv.py | 8 ++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 34e7a5bb..29341d74 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ riak.egg-info/ #*# *~ + +Vagrantfile +.vagrant* diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 2dea7a62..63c50202 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -100,9 +100,11 @@ def __init__(self, protocol='http', transport_options={}, self._pb_pool = RiakPbcPool(self, **transport_options) self._encoders = {'application/json': default_encoder, - 'text/json': default_encoder} + 'text/json': default_encoder, + 'text/plain': unicode} self._decoders = {'application/json': json.loads, - 'text/json': json.loads} + 'text/json': json.loads, + 'text/plain': unicode} self._buckets = WeakValueDictionary() def _get_protocol(self): diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 5eaf9325..6b93131b 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -183,6 +183,14 @@ def test_unknown_content_type_encoder_decoder(self): obj2 = bucket.get(self.key_name) self.assertEqual(data, obj2.encoded_data) + def test_text_plain_encoder_decoder(self): + bucket = self.client.bucket(self.bucket_name) + data = "some funny data" + obj = bucket.new(self.key_name, data, content_type='text/plain') + obj.store() + obj2 = bucket.get(self.key_name) + self.assertEqual(data, obj2.data) + def test_missing_object(self): bucket = self.client.bucket(self.bucket_name) obj = bucket.get(self.key_name) From fd18c4da7b22be57010fe683ce398ae3171fdd58 Mon Sep 17 00:00:00 2001 From: William Kral Date: Thu, 23 May 2013 14:40:34 -0700 Subject: [PATCH 0400/1060] Speed up test server start - Seemed to slow down with riak 1.3 which seems to be because of much more text output when starting - wait_for_erlang_prompt was reading one character at a time and running a long regex against the output which was causing the output of riak console to stall - Changed to read a line at a time speeds up from 34 seconds to 2.4 on my system --- riak/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/test_server.py b/riak/test_server.py index 73df069d..a4978e3f 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -199,7 +199,7 @@ def wait_for_erlang_prompt(self): prompted = False buffer = "" while not prompted: - line = self._server.stdout.read(1) + line = self._server.stdout.readline() if len(line) > 0: buffer += line if re.search(r"\(%s\)\d+>" % self.vm_args["-name"], buffer): From 0be3ff06406eb70a514cf5f0e86f11437f864a89 Mon Sep 17 00:00:00 2001 From: William Kral Date: Wed, 29 May 2013 15:36:12 -0700 Subject: [PATCH 0401/1060] Moving ignored personal stuff to .git/info/exclude --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index 29341d74..34e7a5bb 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,3 @@ riak.egg-info/ #*# *~ - -Vagrantfile -.vagrant* From c6bbe884ebfb06370fa95276c2cc4b13e0eec6eb Mon Sep 17 00:00:00 2001 From: William Kral Date: Wed, 29 May 2013 19:17:38 -0700 Subject: [PATCH 0402/1060] Changed decoder to simple str and removed encoder --- riak/client/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 63c50202..3af3a555 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -100,11 +100,10 @@ def __init__(self, protocol='http', transport_options={}, self._pb_pool = RiakPbcPool(self, **transport_options) self._encoders = {'application/json': default_encoder, - 'text/json': default_encoder, - 'text/plain': unicode} + 'text/json': default_encoder} self._decoders = {'application/json': json.loads, 'text/json': json.loads, - 'text/plain': unicode} + 'text/plain': str} self._buckets = WeakValueDictionary() def _get_protocol(self): From 68d2d87289c98ccad675d80ba8b2c1d7c39e008a Mon Sep 17 00:00:00 2001 From: William Kral Date: Wed, 29 May 2013 19:33:07 -0700 Subject: [PATCH 0403/1060] Fixed RiakHttpTransport with non-ascii streams - Encoded data that has octets outside the 128 byte range fail in python's httplib - Using a bytearray for the message body causes the entire stream to be converted to a bytearray when a str object is added to it and the message is transported correctly - Added two tests one with unicode data in a json object and one unicode data in a string --- riak/tests/test_kv.py | 18 ++++++++++++++++++ riak/transports/http/transport.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 5eaf9325..6da5f4d1 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -70,6 +70,24 @@ def test_store_and_get(self): obj2 = bucket.get('baz') self.assertEqual(obj2.data, rand) + def test_store_obj_with_unicode(self): + bucket = self.client.bucket(self.bucket_name) + data = {u'føø': u'éå'} + obj = bucket.new('foo', data) + obj.store() + obj = bucket.get('foo') + self.assertEqual(obj.data, data) + + def test_store_unicode_string(self): + bucket = self.client.bucket(self.bucket_name) + data = u"some unicode data: \u00c6" + obj = bucket.new(self.key_name, encoded_data=data.encode('utf-8'), + content_type='text/plain') + obj.charset = 'utf-8' + obj.store() + obj2 = bucket.get(self.key_name) + self.assertEqual(data, obj2.encoded_data.decode('utf-8')) + def test_generate_key(self): # Ensure that Riak generates a random key when # the key passed to bucket.new() is None. diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 4824d0b9..632f37bf 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -126,7 +126,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} url = self.object_path(robj.bucket.name, robj.key, **params) headers = self._build_put_headers(robj, if_none_match=if_none_match) - content = robj.encoded_data + content = bytearray(robj.encoded_data) if robj.key is None: expect = [201] From 609388736fd3ed7d9d0ed4919b7caa10f09e6e59 Mon Sep 17 00:00:00 2001 From: William Kral Date: Thu, 30 May 2013 10:34:34 -0700 Subject: [PATCH 0404/1060] Added encoder back for symmetry --- riak/client/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 3af3a555..c49b5e4b 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -100,7 +100,8 @@ def __init__(self, protocol='http', transport_options={}, self._pb_pool = RiakPbcPool(self, **transport_options) self._encoders = {'application/json': default_encoder, - 'text/json': default_encoder} + 'text/json': default_encoder, + 'text/plain': str} self._decoders = {'application/json': json.loads, 'text/json': json.loads, 'text/plain': str} From 05166ce50c02a4d8aa977cdbe7732a0d091a3edd Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 31 May 2013 17:19:29 -0500 Subject: [PATCH 0405/1060] Allow RiakClientOperations.fulltext_search/add/delete to run under PBC as well as HTTP --- riak/client/operations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 3c812e59..ee8535f5 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -232,7 +232,7 @@ def stream_mapred(self, inputs, query, timeout): finally: stream.close() - @retryableHttpOnly + @retryable def fulltext_search(self, transport, index, query, **params): """ Performs a full-text search query. @@ -246,7 +246,7 @@ def fulltext_search(self, transport, index, query, **params): """ return transport.search(index, query, **params) - @retryableHttpOnly + @retryable def fulltext_add(self, transport, index, docs): """ Adds documents to the full-text index. @@ -258,7 +258,7 @@ def fulltext_add(self, transport, index, docs): """ transport.fulltext_add(index, docs) - @retryableHttpOnly + @retryable def fulltext_delete(self, transport, index, docs=None, queries=None): """ Removes documents from the full-text index. From d1bba86f26b0f5f65545d2a34eff2fd9b23ca5fb Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 31 May 2013 17:32:16 -0500 Subject: [PATCH 0406/1060] Attempt to install latest Riak on Travis builder. --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a47837e0..15055442 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,12 @@ language: python python: - "2.6" - "2.7" +before_install: + - "curl http://apt.basho.com/gpg/basho.apt.key | sudo apt-key add -" + - 'sudo bash -c "echo deb http://apt.basho.com $(lsb_release -sc) main > /etc/apt/sources.list.d/basho.list"' + - "sudo apt-get update" + - "sudo apt-get upgrade riak" + - "sudo service start riak" install: - ./setup.py develop - ./setup.py easy_install protobuf @@ -9,5 +15,5 @@ script: ./setup.py test before_script: sudo /usr/sbin/search-cmd install searchbucket notifications: email: clients@basho.com -services: - - riak +# services: +# - riak From 8192ce9bd2efbe1ab4198fb44ecdb51f040621cd Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 31 May 2013 17:46:38 -0500 Subject: [PATCH 0407/1060] Don't override the existing Riak configs. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 15055442..859e7f32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ before_install: - "curl http://apt.basho.com/gpg/basho.apt.key | sudo apt-key add -" - 'sudo bash -c "echo deb http://apt.basho.com $(lsb_release -sc) main > /etc/apt/sources.list.d/basho.list"' - "sudo apt-get update" - - "sudo apt-get upgrade riak" + - "sudo apt-get install riak --assume-no" - "sudo service start riak" install: - ./setup.py develop From e7526452d512566faed0bccbcb0a2676d6cdcb07 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 31 May 2013 17:50:58 -0500 Subject: [PATCH 0408/1060] Pipe `yes n` to the apt-get install command --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 859e7f32..b5cec402 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ before_install: - "curl http://apt.basho.com/gpg/basho.apt.key | sudo apt-key add -" - 'sudo bash -c "echo deb http://apt.basho.com $(lsb_release -sc) main > /etc/apt/sources.list.d/basho.list"' - "sudo apt-get update" - - "sudo apt-get install riak --assume-no" + - "yes n | sudo apt-get install riak" - "sudo service start riak" install: - ./setup.py develop From af75328b7eeb343f08a443569a9231bf20bcddcb Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 31 May 2013 17:52:55 -0500 Subject: [PATCH 0409/1060] Transpose arguments of the service command. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b5cec402..e23e20ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ before_install: - 'sudo bash -c "echo deb http://apt.basho.com $(lsb_release -sc) main > /etc/apt/sources.list.d/basho.list"' - "sudo apt-get update" - "yes n | sudo apt-get install riak" - - "sudo service start riak" + - "sudo service riak start" install: - ./setup.py develop - ./setup.py easy_install protobuf From bb5db6e61595e17831dae15cd00a527fd63b94c7 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 31 May 2013 17:59:10 -0500 Subject: [PATCH 0410/1060] Cleanup the builder config. [ci skip] --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e23e20ae..4ce6d1f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,5 +15,3 @@ script: ./setup.py test before_script: sudo /usr/sbin/search-cmd install searchbucket notifications: email: clients@basho.com -# services: -# - riak From ed5297aeb13f6ace574936212627333f67db600c Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 31 May 2013 19:18:35 -0500 Subject: [PATCH 0411/1060] Switch fulltext_add/delete back to @retryableHttpOnly --- riak/client/operations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index ee8535f5..c928128f 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -246,7 +246,7 @@ def fulltext_search(self, transport, index, query, **params): """ return transport.search(index, query, **params) - @retryable + @retryableHttpOnly def fulltext_add(self, transport, index, docs): """ Adds documents to the full-text index. @@ -258,7 +258,7 @@ def fulltext_add(self, transport, index, docs): """ transport.fulltext_add(index, docs) - @retryable + @retryableHttpOnly def fulltext_delete(self, transport, index, docs=None, queries=None): """ Removes documents from the full-text index. From 9e47e0e154a9da52d33127119e6b979455e950b2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 13 Jun 2013 08:22:04 -0500 Subject: [PATCH 0412/1060] Add sibling-resolution functions. A resolver function takes a RiakObject and (hopefully) resolves its siblings into a single sibling. Resolvers are invoked automatically when fetching an object returns siblings. The resolver can be set on the client, the bucket, or the individual object. If the object in conflict has no resolver, the bucket's resolver will be used, and then the client's resolver if the bucket has none. The default resolver does no resolution. Also included (but not assigned) is a resolver that selects the latest sibling based on the `last_modified` property; see riak/resolver.py for more details. --- riak/bucket.py | 18 +++++++++ riak/client/__init__.py | 3 +- riak/resolver.py | 42 +++++++++++++++++++++ riak/riak_object.py | 18 +++++++++ riak/tests/test_kv.py | 70 ++++++++++++++++++++++++++++++++++- riak/transports/http/codec.py | 6 +++ riak/transports/pbc/codec.py | 3 ++ 7 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 riak/resolver.py diff --git a/riak/bucket.py b/riak/bucket.py index 3b255885..4aeb54d0 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -54,6 +54,7 @@ def __init__(self, client, name): self.name = name self._encoders = {} self._decoders = {} + self._resolver = None def __hash__(self): return hash((self.name, self._client)) @@ -199,6 +200,23 @@ def get_binary(self, key, r=None, pr=None): 'use RiakBucket.get') return self.get(key, r=r, pr=pr) + def _get_resolver(self): + if callable(self._resolver): + return self._resolver + elif self._resolver is None: + return self._client.resolver + else: + raise TypeError("resolver is not a function") + + def _set_resolver(self, value): + self._resolver = value + + resolver = property(_get_resolver, _set_resolver, doc= + """The sibling-resolution function for this + bucket. If the resolver is not set, the + client's resolver will be used. :type + callable""") + def _set_n_val(self, nval): return self.set_property('n_val', nval) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index c49b5e4b..1ca2c018 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -30,6 +30,7 @@ from riak.node import RiakNode from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduceChain +from riak.resolver import default_resolver from riak.search import RiakSearch from riak.transports.http import RiakHttpPool from riak.transports.pbc import RiakPbcPool @@ -95,7 +96,7 @@ def __init__(self, protocol='http', transport_options={}, self.nodes = [self._create_node(n) for n in nodes] self.protocol = protocol or 'http' - + self.resolver = default_resolver self._http_pool = RiakHttpPool(self, **transport_options) self._pb_pool = RiakPbcPool(self, **transport_options) diff --git a/riak/resolver.py b/riak/resolver.py new file mode 100644 index 00000000..30bfcc73 --- /dev/null +++ b/riak/resolver.py @@ -0,0 +1,42 @@ +""" +Copyright 2013 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + + +def default_resolver(riak_object): + """ + The default conflict-resolution function, which does nothing. To + implement a resolver, define a function that sets the ``siblings`` + property on the passed ``RiakObject`` instance to a list + containing a single ``RiakContent`` object. + + :param riak_object: an object-in-conflict that will be resolved + :type riak_object: RiakObject + """ + pass + + +def last_written_resolver(riak_object): + """ + A conflict-resolution function that resolves by selecting the most + recently-modified sibling by timestamp. + + :param riak_object: an object-in-conflict that will be resolved + :type riak_object: RiakObject + """ + lm = lambda x: x.last_modified + riak_object.siblings = [max(riak_object.siblings, key=lm), ] diff --git a/riak/riak_object.py b/riak/riak_object.py index 92d6ec78..bcc4559b 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -116,6 +116,7 @@ def __init__(self, client, bucket, key=None): raise ValueError('Key name must either be "None"' ' or a non-empty string.') + self._resolver = None self.client = client self.bucket = bucket self.key = key @@ -213,6 +214,23 @@ def _exists(self): :type bool """) + def _get_resolver(self): + if callable(self._resolver): + return self._resolver + elif self._resolver is None: + return self.bucket.resolver + else: + raise TypeError("resolver is not a function") + + def _set_resolver(self, value): + self._resolver = value + + resolver = property(_get_resolver, _set_resolver, doc= + """The sibling-resolution function for this + object. If the resolver is not set, the + bucket's resolver will be used. :type + callable""") + def get_sibling(self, index): deprecated("RiakObject.get_sibling is deprecated, use the " "siblings property instead") diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index b42a1ed4..fe55c2ee 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -2,8 +2,9 @@ import os import cPickle import copy +from time import sleep from riak import ConflictError - +from riak.resolver import default_resolver, last_written_resolver try: import simplejson as json except ImportError: @@ -323,6 +324,73 @@ def test_siblings(self): self.assertEqual(len(obj.siblings), 1) self.assertEqual(obj.encoded_data, resolved_sibling.encoded_data) + def test_resolution(self): + bucket = self.client.bucket(self.sibs_bucket) + obj = bucket.get(self.key_name) + bucket.allow_mult = True + + # Even if it previously existed, let's store a base resolved version + # from which we can diverge by sending a stale vclock. + obj.encoded_data = 'start' + obj.content_type = 'text/plain' + obj.store() + + # Store the same object five times... + # First run through should overwrite the datum 'start' above + other_client = self.create_client() + other_bucket = other_client.bucket(self.sibs_bucket) + + vals = [] + for i in range(5): + while True: + randval = self.randint() + if str(randval) not in vals: + break + + other_obj = other_bucket.new(key=self.key_name, + encoded_data=str(randval), + content_type='text/plain') + other_obj.vclock = obj.vclock + other_obj.store() + vals.append(str(randval)) + # TODO: This sleep exists so that last_written_resolver + # will find timestamps in different seconds. HTTP dates do + # not have enough significant digits for sub-second + # differences. + sleep(0.75) + + # Make sure the object has five siblings when using the + # default resolver + obj = bucket.get(self.key_name) + obj.reload() + self.assertEqual(len(obj.siblings), 5) + + # Setting the resolver on the client object to use the + # "last-write-wins" behavior + self.client.resolver = last_written_resolver + obj.reload() + self.assertEqual(obj.resolver, last_written_resolver) + self.assertEqual(1, len(obj.siblings)) + self.assertEqual(obj.data, vals[-1]) + + # Set the resolver on the bucket to the default resolver, + # overriding the resolver on the client + bucket.resolver = default_resolver + obj.reload() + self.assertEqual(obj.resolver, default_resolver) + self.assertEqual(len(obj.siblings), 5) + + # Define our own custom resolver on the object that returns + # the maximum value, overriding the bucket and client resolvers + def max_value_resolver(obj): + datafun = lambda s: s.data + obj.siblings = [max(obj.siblings, key=datafun), ] + + obj.resolver = max_value_resolver + obj.reload() + self.assertEqual(obj.resolver, max_value_resolver) + self.assertEqual(obj.data, max(vals)) + def test_tombstone_siblings(self): # Set up the bucket, clear any existing object... bucket = self.client.bucket(self.sibs_bucket) diff --git a/riak/transports/http/codec.py b/riak/transports/http/codec.py index 1ef30129..25bb7dab 100644 --- a/riak/transports/http/codec.py +++ b/riak/transports/http/codec.py @@ -85,6 +85,11 @@ def _parse_body(self, robj, response, expected_statuses): part.items(), part.get_payload()) for part in parts] + + # Invoke sibling-resolution logic + if robj.resolver is not None: + robj.resolver(robj) + return robj else: raise Exception('unexpected sibling response format: {0}'. @@ -92,6 +97,7 @@ def _parse_body(self, robj, response, expected_statuses): robj.siblings = [self._parse_sibling(RiakContent(robj), headers.items(), data)] + return robj def _parse_sibling(self, sibling, headers, data): diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 7ee0b0e0..c91dda20 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -62,6 +62,9 @@ def translate_rw_val(self, rw): def _decode_contents(self, contents, obj): obj.siblings = [self._decode_content(c, RiakContent(obj)) for c in contents] + # Invoke sibling-resolution logic + if len(obj.siblings) > 1 and obj.resolver is not None: + obj.resolver(obj) return obj def _decode_content(self, rpb_content, sibling): From 9fa4f81d45bca6f9e05975774ecf7b6a6933daa0 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 13 Jun 2013 15:26:32 -0500 Subject: [PATCH 0413/1060] Type-check when setting the resolver on bucket and object. --- riak/bucket.py | 5 ++++- riak/riak_object.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 4aeb54d0..29861f45 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -209,7 +209,10 @@ def _get_resolver(self): raise TypeError("resolver is not a function") def _set_resolver(self, value): - self._resolver = value + if value is None or callable(value): + self._resolver = value + else: + raise TypeError("resolver is not a function") resolver = property(_get_resolver, _set_resolver, doc= """The sibling-resolution function for this diff --git a/riak/riak_object.py b/riak/riak_object.py index bcc4559b..48d9b779 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -223,7 +223,10 @@ def _get_resolver(self): raise TypeError("resolver is not a function") def _set_resolver(self, value): - self._resolver = value + if value is None or callable(value): + self._resolver = value + else: + raise TypeError("resolver is not a function") resolver = property(_get_resolver, _set_resolver, doc= """The sibling-resolution function for this From afa3619859a3e8239a779015157d5392bf641f9f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 13 Jun 2013 16:04:13 -0500 Subject: [PATCH 0414/1060] Factor out the sibling generation into a helper method. --- riak/tests/test_kv.py | 76 ++++++++++++------------------------------- 1 file changed, 21 insertions(+), 55 deletions(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index fe55c2ee..52c1bde6 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -281,24 +281,7 @@ def test_siblings(self): obj.content_type = 'application/octet-stream' obj.store() - # Store the same object five times... - # First run through should overwrite the datum 'start' above - other_client = self.create_client() - other_bucket = other_client.bucket(self.sibs_bucket) - - vals = set() - for i in range(5): - while True: - randval = self.randint() - if str(randval) not in vals: - break - - other_obj = other_bucket.new(key=self.key_name, - encoded_data=str(randval), - content_type='text/plain') - other_obj.vclock = obj.vclock - other_obj.store() - vals.add(str(randval)) + vals = set(self.generate_siblings(obj, count=5)) # Make sure the object has five siblings... obj = bucket.get(self.key_name) @@ -335,29 +318,7 @@ def test_resolution(self): obj.content_type = 'text/plain' obj.store() - # Store the same object five times... - # First run through should overwrite the datum 'start' above - other_client = self.create_client() - other_bucket = other_client.bucket(self.sibs_bucket) - - vals = [] - for i in range(5): - while True: - randval = self.randint() - if str(randval) not in vals: - break - - other_obj = other_bucket.new(key=self.key_name, - encoded_data=str(randval), - content_type='text/plain') - other_obj.vclock = obj.vclock - other_obj.store() - vals.append(str(randval)) - # TODO: This sleep exists so that last_written_resolver - # will find timestamps in different seconds. HTTP dates do - # not have enough significant digits for sub-second - # differences. - sleep(0.75) + vals = self.generate_siblings(obj, count=5, delay=0.75) # Make sure the object has five siblings when using the # default resolver @@ -401,22 +362,9 @@ def test_tombstone_siblings(self): obj.content_type = 'application/octet-stream' obj.store(return_body=True) - vclock = obj.vclock obj.delete() - vals = set() - for i in range(4): - while True: - randval = self.randint() - if str(randval) not in vals: - break - - other_obj = bucket.new(key=self.key_name, - encoded_data=str(randval), - content_type='text/plain') - other_obj.vclock = vclock - other_obj.store() - vals.add(str(randval)) + vals = set(self.generate_siblings(obj, count=4)) obj = bucket.get(self.key_name) self.assertEqual(len(obj.siblings), 5) @@ -465,6 +413,24 @@ def test_list_buckets(self): buckets = self.client.get_buckets() self.assertTrue(self.bucket_name in [x.name for x in buckets]) + def generate_siblings(self, original, count=5, delay=None): + vals = [] + for i in range(count): + while True: + randval = self.randint() + if str(randval) not in vals: + break + + other_obj = original.bucket.new(key=original.key, + encoded_data=str(randval), + content_type='text/plain') + other_obj.vclock = original.vclock + other_obj.store() + vals.append(str(randval)) + if delay: + sleep(delay) + return vals + class HTTPBucketPropsTest(object): def test_rw_settings(self): From f3740561249fdc99fa129324122dd64b4367dd3f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 13:47:28 -0500 Subject: [PATCH 0415/1060] Touch the ssl_distribution.args_file by running `riak chkconfig`. Closes #245. --- riak/test_server.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/riak/test_server.py b/riak/test_server.py index a4978e3f..8ad93a15 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -125,6 +125,7 @@ def __init__(self, tmp_dir="/tmp/riak/test_server", def prepare(self): if not self._prepared: + self.touch_ssl_distribution_args() self.create_temp_directories() self._riak_script = os.path.join(self._temp_bin, "riak") self.write_riak_script() @@ -243,6 +244,14 @@ def write_app_config(self): app_config.write(erlang_config(self.app_config)) app_config.write(".") + def touch_ssl_distribution_args(self): + # To make sure that the ssl_distribution.args file is present, + # the control script in the source node has to have been run at + # least once. Running the `chkconfig` command is innocuous + # enough to accomplish this without other side-effects. + script = os.path.join(self.bin_dir, "riak") + Popen([script, "chkconfig"]).wait() + def _kv_backend(self): return self.app_config["riak_kv"]["storage_backend"] From d6659941ca3c8884ac7915fd5dff52912bc6b036 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 14:24:40 -0500 Subject: [PATCH 0416/1060] Add documentation about protocol selection and the lazy-connection semantics. Closes #241. --- riak/client/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 1ca2c018..503109de 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -118,7 +118,19 @@ def _set_protocol(self, value): self._protocol = value protocol = property(_get_protocol, _set_protocol, - doc="""Which protocol to prefer, one of PROTOCOLS""") + doc= + """ + Which protocol to prefer, one of PROTOCOLS. + Please note that when one protocol is + selected, the other protocols MAY NOT attempt + to connect. Changing to another protocol will + cause a connection on the next request. + + Some requests are only valid over 'http' or + 'https', and will always be sent via those + transports, regardless of which protocol is + preferred. + """) def get_transport(self): """ From f2f469956b8b5736e3950dfc9766c2a9d6d9d439 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 15:21:56 -0500 Subject: [PATCH 0417/1060] Don't open the PBC connection until a request is sent. This prevents exceptions from occuring while trying to create a new transport in the pool, which happens outside a try/except. The result is the same behavior as HTTP, which delays opening a connection until the first request. --- riak/transports/pbc/connection.py | 15 +++++++++------ riak/transports/pbc/transport.py | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index a461558f..c805d26d 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -39,6 +39,7 @@ def _encode_msg(self, msg_code, msg=None): return hdr + msgstr def _request(self, msg_code, msg=None, expect=None): + self._connect() self._send_msg(msg_code, msg) return self._recv_msg(expect) @@ -82,17 +83,19 @@ def _recv_pkt(self): % (len(self._inbuf), self._inbuf_len)) def _connect(self): - if self._timeout: - self._socket = socket.create_connection(self._address, - self._timeout) - else: - self._socket = socket.create_connection(self._address) + if not self._socket: + if self._timeout: + self._socket = socket.create_connection(self._address, + self._timeout) + else: + self._socket = socket.create_connection(self._address) def close(self): """ Closes the underlying socket of the PB connection. """ - self._socket.shutdown(socket.SHUT_RDWR) + if self._socket: + self._socket.shutdown(socket.SHUT_RDWR) def _parse_msg(self, code, packet): try: diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index c59c30d9..31575d16 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -73,7 +73,7 @@ def __init__(self, node=None, client=None, timeout=None, *unused_options): self._node = node self._address = (node.host, node.pb_port) self._timeout = timeout - self._connect() + self._socket = None # FeatureDetection API def _server_version(self): From 7ad11e9124d33d67abc6648ac88798f81279847e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 15:34:51 -0500 Subject: [PATCH 0418/1060] Raise the final exception when the retry limit has been reached. This prevents masking of errors when the loop falls through to its completion, all tries raising BadResource. The previous behavior was that the function doesn't return any value, essentially returning None. --- riak/client/transport.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/riak/client/transport.py b/riak/client/transport.py index 5c42c55c..4be2bce8 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -58,18 +58,20 @@ def _with_retries(self, pool, fn): def _skip_bad_nodes(transport): return transport._node not in skip_nodes - for retry in range(self.RETRY_COUNT): + retry_count = self.RETRY_COUNT + + for retry in range(retry_count): try: with pool.take(_filter=_skip_bad_nodes) as transport: try: return fn(transport) except (IOError, httplib.HTTPException) as e: - if _is_retryable(e): - transport._node.error_rate.incr(1) + transport._node.error_rate.incr(1) + if retry < (retry_count - 1) and _is_retryable(e): skip_nodes.append(transport._node) raise BadResource(e) else: - raise e + raise except BadResource: continue From 9e0b2061eeea1a76ceed481c0e656b6865a6aa0b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 15:44:01 -0500 Subject: [PATCH 0419/1060] Move _connect() call into _send_msg() so that streaming doesn't break. --- riak/transports/pbc/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index c805d26d..8609c370 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -39,11 +39,11 @@ def _encode_msg(self, msg_code, msg=None): return hdr + msgstr def _request(self, msg_code, msg=None, expect=None): - self._connect() self._send_msg(msg_code, msg) return self._recv_msg(expect) def _send_msg(self, msg_code, msg): + self._connect() self._socket.send(self._encode_msg(msg_code, msg)) def _recv_msg(self, expect=None): From 61bed7068be6b402150bdf3dc84f53c923997155 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 16:05:13 -0500 Subject: [PATCH 0420/1060] Test that retry logic re-raises the original exception when tries are exhausted. --- riak/tests/test_all.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 1894cba9..cbccbc98 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -118,6 +118,17 @@ def setUp(self): self.client = self.create_client() +class ClientTests(object): + def test_request_retries(self): + # We guess at some ports that will be unused by Riak or + # anything else. + client = self.create_client(http_port=1023, pb_port=1022) + + # If retries are exhausted, the final result should also be an + # error. + self.assertRaises(IOError, client.ping) + + class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, PbcBucketPropsTest, @@ -128,6 +139,7 @@ class RiakPbcTransportTestCase(BasicKVTests, MapReduceAliasTests, MapReduceStreamTests, SearchTests, + ClientTests, BaseTestCase, unittest.TestCase): @@ -169,6 +181,7 @@ class RiakHttpTransportTestCase(BasicKVTests, EnableSearchTests, SolrSearchTests, SearchTests, + ClientTests, BaseTestCase, unittest.TestCase): From ee94304bb4fad92ee9e3c88b94863b3e30a4f754 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 16:10:40 -0500 Subject: [PATCH 0421/1060] Eject the bad connection from the pool even on the last try. Also, don't penalize a node that has some unspecified unretryable exception. --- riak/client/transport.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/riak/client/transport.py b/riak/client/transport.py index 4be2bce8..38f4b272 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -66,14 +66,18 @@ def _skip_bad_nodes(transport): try: return fn(transport) except (IOError, httplib.HTTPException) as e: - transport._node.error_rate.incr(1) - if retry < (retry_count - 1) and _is_retryable(e): + if _is_retryable(e): + transport._node.error_rate.incr(1) skip_nodes.append(transport._node) raise BadResource(e) else: raise - except BadResource: - continue + except BadResource as e: + if retry < (retry_count - 1): + continue + else: + # Re-raise the inner exception + raise e.args[0] def _choose_pool(self, protocol=None): """ From c15b22f52f892010892a3fa3b9f13190cdfbf5fd Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 18 Jun 2013 09:45:19 -0500 Subject: [PATCH 0422/1060] Use pipes for stdio when running chkconfig. --- riak/test_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/test_server.py b/riak/test_server.py index 8ad93a15..a0a5ec44 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -250,7 +250,8 @@ def touch_ssl_distribution_args(self): # least once. Running the `chkconfig` command is innocuous # enough to accomplish this without other side-effects. script = os.path.join(self.bin_dir, "riak") - Popen([script, "chkconfig"]).wait() + Popen([script, "chkconfig"], + stdin=PIPE, stdout=PIPE, stderr=PIPE).communicate() def _kv_backend(self): return self.app_config["riak_kv"]["storage_backend"] From 612ceababed1ad30a869e21f1b62dfc8f56cc325 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 20 Jun 2013 14:31:01 -0500 Subject: [PATCH 0423/1060] Increased the delay between sibling writes so as to avoid races in resolution test. Also added an option to skip the resolution test since it runs for ~5 seconds each invocation. This should fix the non-deterministic failures in riak_test like http://giddyup.basho.com/#/projects/riak/scorecards/35/35-387-client_python_verify-ubuntu-1004-64-eleveldb/14385/artifacts/76166 /cc @engelsanchez @javajolt --- riak/tests/test_kv.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 52c1bde6..1fb50fce 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -2,6 +2,7 @@ import os import cPickle import copy +import platform from time import sleep from riak import ConflictError from riak.resolver import default_resolver, last_written_resolver @@ -10,6 +11,11 @@ except ImportError: import json +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest + class NotJsonSerializable(object): @@ -307,6 +313,8 @@ def test_siblings(self): self.assertEqual(len(obj.siblings), 1) self.assertEqual(obj.encoded_data, resolved_sibling.encoded_data) + @unittest.skipIf(os.environ.get('SKIP_RESOLVE', '0') == '1', + "skip requested for resolvers test") def test_resolution(self): bucket = self.client.bucket(self.sibs_bucket) obj = bucket.get(self.key_name) @@ -318,7 +326,7 @@ def test_resolution(self): obj.content_type = 'text/plain' obj.store() - vals = self.generate_siblings(obj, count=5, delay=0.75) + vals = self.generate_siblings(obj, count=5, delay=1.01) # Make sure the object has five siblings when using the # default resolver From fca898b06821c965170f61af8b00b2a93fdf92a6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 25 Jun 2013 10:24:41 -0500 Subject: [PATCH 0424/1060] Add multiget implementation with a static worker pool. --- riak/client/multiget.py | 167 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 riak/client/multiget.py diff --git a/riak/client/multiget.py b/riak/client/multiget.py new file mode 100644 index 00000000..149d7599 --- /dev/null +++ b/riak/client/multiget.py @@ -0,0 +1,167 @@ +""" +Copyright 2013 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from collections import namedtuple +from Queue import Queue +from threading import Thread, Lock, Event +from multiprocessing import cpu_count + +__all__ = ['multiget'] + + +try: + POOL_SIZE = cpu_count() * 2 +except NotImplementedError: + # Make an educated guess + POOL_SIZE = 6 + + +Task = namedtuple('Task', ['client', 'outq', 'bucket', 'key', 'options']) + + +class MultiGetPool(object): + """ + Encapsulates a pool of fetcher threads. These threads can be used + across many multi-get requests. + """ + + def __init__(self, size=POOL_SIZE): + self._inq = Queue() + self._size = size + self._started = Event() + self._stop = Event() + self._lock = Lock() + self._workers = [] + + def enq(self, task): + """ + Enqueues a fetch task to the pool of workers. This will raise + a RuntimeError if the pool is stopped or in the process of + stopping. + + :param task: the Task object + :type task: Task + """ + if not self._stop.is_set(): + self._inq.put(task) + else: + raise RuntimeError("Attempted to enqueue a fetch operation while " + "multi-get pool was shutdown!") + + def start(self): + """ + Starts the worker threads if they are not already started. + This method is thread-safe and will be called automatically + when executing a MultiGet operation. + """ + # Check whether we are already started, skip if we are. + if not self._started.is_set(): + # If we are not started, try to capture the lock. + if self._lock.acquire(False): + # If we got the lock, go ahead and start the worker + # threads, set the started flag, and release the lock. + for i in range(self._size): + name = "riak.client.multiget-worker-{0}".format(i) + worker = Thread(target=self._fetcher, args=(name,), + name=name) + worker.daemon = True + worker.start() + self._workers.append(worker) + self._started.set() + self._lock.release() + else: + # We didn't get the lock, so someone else is already + # starting the worker threads. Wait until they have + # signaled that the threads are started. + self._started.wait() + + def stop(self): + """ + Signals the worker threads to exit and waits on them. + """ + self._stop.set() + for worker in self._workers: + worker.join() + + def stopped(self): + """ + Detects whether this pool has been stopped. + """ + return self._stop.is_set() + + def __del__(self): + # Ensure that all work in the queue is processed before + # shutting down. + self.stop() + + def _fetcher(self, name): + """ + The body of the multi-get worker. + """ + while not self._should_quit(): + task = self._inq.get() + try: + obj = task.client.bucket(task.bucket).get(task.key, + **task.options) + task.outq.put(obj) + except KeyboardInterrupt: + raise + except StandardError as err: + task.outq.put((task.bucket, task.key, err), ) + finally: + self._inq.task_done() + + def _should_quit(self): + """ + Worker threads should exit when the stop flag is set and the + input queue is empty. Once the stop flag is set, new enqueues + are disallowed, meaning that the workers can safely drain the + queue before exiting. + :rtype boolean + """ + return self.stopped() and self._inq.empty() + + +RIAK_MULTIGET_POOL = MultiGetPool() + + +def multiget(client, keys, **options): + """ + Executes a parallel-fetch across multiple threads. Returns a list + containing RiakObject instances, or 3-tuples of bucket, key, and + the exception raised. + + :rtype list + """ + outq = Queue() + + RIAK_MULTIGET_POOL.start() + for bucket, key in keys: + task = Task(client=client, outq=outq, options=options, + bucket=bucket, key=key) + RIAK_MULTIGET_POOL.enq(task) + + results = [] + for _ in range(len(keys)): + if RIAK_MULTIGET_POOL.stopped(): + raise RuntimeError("Multi-get operation interrupted by pool " + "stopping!") + results.append(outq.get()) + outq.task_done() + + return results From a518e9009354524bd0fcdf9884169e05a94fb353 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 25 Jun 2013 13:10:20 -0500 Subject: [PATCH 0425/1060] Add multiget benchmark. --- riak/benchmark.py | 165 ++++++++++++++++++++++++++++++++++++++++ riak/client/multiget.py | 36 +++++++++ 2 files changed, 201 insertions(+) create mode 100644 riak/benchmark.py diff --git a/riak/benchmark.py b/riak/benchmark.py new file mode 100644 index 00000000..735d76e5 --- /dev/null +++ b/riak/benchmark.py @@ -0,0 +1,165 @@ +""" +Copyright 2013 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import os +import gc + +__all__ = ['bm', 'bmbm'] + + +def bmbm(): + """ + Runs a benchmark when used as an iterator, injecting a garbage + collection between iterations. Example: + + for b in benchmark.bmbm(): + with b.report("pow"): + for _ in range(10000): + math.pow(2,10000) + with b.report("factorial"): + for i in range(100): + math.factorial(i) + """ + return Benchmark(True) + + +def bm(): + """ + Runs a benchmark once when used as a context manager. Example: + + with benchmark.bm() as b: + with b.report("pow"): + for _ in range(10000): + math.pow(2,10000) + with b.report("factorial"): + for i in range(100): + math.factorial(i) + """ + return Benchmark() + + +class Benchmark(object): + """ + A benchmarking run, which may consist of multiple steps. See + bmbm() and bm() for examples. + """ + def __init__(self, rehearse=False): + """ + Creates a new benchmark reporter. + + :param rehearse: whether to run twice to take counter the effects + of garbage collection + :type rehearse: boolean + """ + self.rehearse = rehearse + if rehearse: + self.count = 2 + else: + self.count = 1 + self._report = None + + def __enter__(self): + if self.rehearse: + raise ValueError("bmbm() cannot be used in with statements, " + "use bm() or the for..in statement") + print_header() + self._report = BenchmarkReport() + self._report.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._report: + return self._report.__exit__(exc_type, exc_val, exc_tb) + else: + print + return True + + def __iter__(self): + return self + + def next(self): + """ + Runs the next iteration of the benchmark. + """ + if self.count == 0: + raise StopIteration + elif self.count > 1: + print_rehearsal_header() + else: + if self.rehearse: + gc.collect() + print "-----------------------------------------------------------\n" + print_header() + + self.count -= 1 + return self + + def report(self, name): + """ + Returns a report for the current step of the benchmark. + """ + self._report = None + return BenchmarkReport(name) + + +def print_rehearsal_header(): + """ + Prints the header for the rehearsal phase of a benchmark. + """ + print + print "Rehearsal -------------------------------------------------" + + +def print_report(label, user, system, real): + """ + Prints the report of one step of a benchmark. + """ + print "{:<12s} {:12f} {:12f} ( {:12f} )".format(label, user, system, real) + + +def print_header(): + """ + Prints the header for the normal phase of a benchmark. + """ + print "{:<12s} {:<12s} {:<12s} ( {:<12s} )".format('', 'user', 'system', 'real') + + +class BenchmarkReport(object): + """ + A labeled step in a benchmark. Acts as a context-manager, printing + its timing results when the context exits. + """ + def __init__(self, name='benchmark'): + self.name = name + self.start = None + + def __enter__(self): + self.start = os.times() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not exc_type: + user1, system1, _, _, real1 = self.start + user2, system2, _, _, real2 = os.times() + print_report(self.name, user2 - user1, system2 - system1, + real2 - real1) + elif exc_type is KeyboardInterrupt: + return False + else: + print "EXCEPTION! %r" % ((exc_type, exc_val, exc_tb),) + return True diff --git a/riak/client/multiget.py b/riak/client/multiget.py index 149d7599..a6e4e94f 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -165,3 +165,39 @@ def multiget(client, keys, **options): outq.task_done() return results + +if __name__ == '__main__': + # Run a benchmark! + from riak import RiakClient + import riak.benchmark as benchmark + client = RiakClient() + bkeys = [ ('multiget', str(key)) for key in xrange(10000) ] + + print "Benchmarking multiget:" + print " CPUs: {0}".format(cpu_count()) + print " Threads: {0}".format(POOL_SIZE) + print " Keys: {0}".format(len(bkeys)) + print + + with benchmark.bm() as b: + with b.report('populate'): + for bucket, key in bkeys: + client.bucket(bucket).new(key, encoded_data=key, + content_type='text/plain' + ).store() + for b in benchmark.bmbm(): + client.protocol = 'http' + with b.report('http seq'): + for bucket, key in bkeys: + client.bucket(bucket).get(key) + + with b.report('http multi'): + multiget(client, bkeys) + + client.protocol = 'pbc' + with b.report('pbc seq'): + for bucket, key in bkeys: + client.bucket(bucket).get(key) + + with b.report('pbc multi'): + multiget(client, bkeys) From 0294e49c0133bdc2471051831e66a3985cc63ddb Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 25 Jun 2013 13:20:13 -0500 Subject: [PATCH 0426/1060] Expose multiget operations on client and bucket objects. --- riak/bucket.py | 15 +++++++++++++++ riak/client/operations.py | 14 +++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/riak/bucket.py b/riak/bucket.py index 29861f45..b39ec338 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -200,6 +200,21 @@ def get_binary(self, key, r=None, pr=None): 'use RiakBucket.get') return self.get(key, r=r, pr=pr) + def multiget(self, keys, r=None, pr=None): + """ + Retrieves a list of keys belonging to this bucket in parallel. + + :param keys: the keys to fetch + :type keys: list + :param r: R-Value for the requests (defaults to bucket's R) + :type r: integer + :param pr: PR-Value for the requests (defaults to bucket's PR) + :type pr: integer + :rtype list of :class:`RiakObject ` + """ + bkeys = [(self.name, key) for key in keys] + return self._client.multiget(bkeys, r=r, pr=pr) + def _get_resolver(self): if callable(self._resolver): return self._resolver diff --git a/riak/client/operations.py b/riak/client/operations.py index c928128f..d9044699 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -17,7 +17,7 @@ """ from transport import RiakClientTransport, retryable, retryableHttpOnly - +from multiget import multiget class RiakClientOperations(RiakClientTransport): """ @@ -271,3 +271,15 @@ def fulltext_delete(self, transport, index, docs=None, queries=None): :type queries: list """ transport.fulltext_delete(index, docs, queries) + + def multiget(self, pairs, **params): + """ + Fetches many keys in parallel via threads. + + :param pairs: list of bucket/key tuple pairs + :type pairs: list + :param params: additional request flags, e.g. r, pr + :type params: dict + :rtype list + """ + return multiget(self, pairs, **params) From dac83361384bc4fd13395f007a984ad409b6401d Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 25 Jun 2013 13:42:07 -0500 Subject: [PATCH 0427/1060] Use CPU count as the pool size, no difference apparent above that. --- riak/client/multiget.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/riak/client/multiget.py b/riak/client/multiget.py index a6e4e94f..845170c0 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -25,7 +25,7 @@ try: - POOL_SIZE = cpu_count() * 2 + POOL_SIZE = cpu_count() except NotImplementedError: # Make an educated guess POOL_SIZE = 6 @@ -170,9 +170,11 @@ def multiget(client, keys, **options): # Run a benchmark! from riak import RiakClient import riak.benchmark as benchmark - client = RiakClient() + client = RiakClient(protocol='pbc') bkeys = [ ('multiget', str(key)) for key in xrange(10000) ] + data = open(__file__).read() + print "Benchmarking multiget:" print " CPUs: {0}".format(cpu_count()) print " Threads: {0}".format(POOL_SIZE) @@ -182,7 +184,7 @@ def multiget(client, keys, **options): with benchmark.bm() as b: with b.report('populate'): for bucket, key in bkeys: - client.bucket(bucket).new(key, encoded_data=key, + client.bucket(bucket).new(key, encoded_data=data, content_type='text/plain' ).store() for b in benchmark.bmbm(): From 753a0a4dbb0e71dcfd64cbf6e7a4e91421cfdea6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 25 Jun 2013 14:07:48 -0500 Subject: [PATCH 0428/1060] Add some tests for multiget. --- riak/tests/test_all.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index cbccbc98..b868ddb5 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -13,6 +13,7 @@ from riak.client import RiakClient from riak.mapreduce import RiakKeyFilter from riak import key_filter +from riak.riak_object import RiakObject from riak.test_server import TestServer @@ -128,6 +129,45 @@ def test_request_retries(self): # error. self.assertRaises(IOError, client.ping) + def test_multiget_bucket(self): + """ + Multiget operations can be invoked on buckets. + """ + keys = [self.key_name, self.randname(), self.randname()] + for key in keys: + self.client.bucket(self.bucket_name)\ + .new(key, encoded_data=key, content_type="text/plain")\ + .store() + results = self.client.bucket(self.bucket_name).multiget(keys) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertTrue(obj.exists) + self.assertEqual(obj.key, obj.encoded_data) + + def test_multiget_errors(self): + """ + Unrecoverable errors are captured along with the bucket/key + and not propagated. + """ + keys = [self.key_name, self.randname(), self.randname()] + client = self.create_client(http_port=1023, pb_port=1024) + results = client.bucket(self.bucket_name).multiget(keys) + for failure in results: + self.assertIsInstance(failure, tuple) + self.assertEqual(failure[0], self.bucket_name) + self.assertIn(failure[1], keys) + self.assertIsInstance(failure[2], StandardError) + + def test_multiget_notfounds(self): + """ + Not founds work in multiget just the same as get. + """ + keys = [(self.bucket_name, self.key_name), + (self.bucket_name, self.randname())] + results = self.client.multiget(keys) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertFalse(obj.exists) class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, From 3fdc469b32f5ded71ad4436b91ea74c4a626036f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 08:54:57 -0500 Subject: [PATCH 0429/1060] Rename bmbm and friends and fix minor bugs in multiget. --- riak/benchmark.py | 23 +++++++++++++---------- riak/client/multiget.py | 12 +++++------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/riak/benchmark.py b/riak/benchmark.py index 735d76e5..c3eade41 100644 --- a/riak/benchmark.py +++ b/riak/benchmark.py @@ -19,15 +19,15 @@ import os import gc -__all__ = ['bm', 'bmbm'] +__all__ = ['measure', 'measure_with_rehearsal'] -def bmbm(): +def measure_with_rehearsal(): """ Runs a benchmark when used as an iterator, injecting a garbage collection between iterations. Example: - for b in benchmark.bmbm(): + for b in benchmark.measure_with_rehearsal(): with b.report("pow"): for _ in range(10000): math.pow(2,10000) @@ -38,11 +38,11 @@ def bmbm(): return Benchmark(True) -def bm(): +def measure(): """ Runs a benchmark once when used as a context manager. Example: - with benchmark.bm() as b: + with benchmark.measure() as b: with b.report("pow"): for _ in range(10000): math.pow(2,10000) @@ -56,7 +56,7 @@ def bm(): class Benchmark(object): """ A benchmarking run, which may consist of multiple steps. See - bmbm() and bm() for examples. + measure_with_rehearsal() and measure() for examples. """ def __init__(self, rehearse=False): """ @@ -75,8 +75,9 @@ def __init__(self, rehearse=False): def __enter__(self): if self.rehearse: - raise ValueError("bmbm() cannot be used in with statements, " - "use bm() or the for..in statement") + raise ValueError("measure_with_rehearsal() cannot be used in with " + "statements, use measure() or the for..in " + "statement") print_header() self._report = BenchmarkReport() self._report.__enter__() @@ -103,7 +104,8 @@ def next(self): else: if self.rehearse: gc.collect() - print "-----------------------------------------------------------\n" + print ("-" * 59) + print print_header() self.count -= 1 @@ -136,7 +138,8 @@ def print_header(): """ Prints the header for the normal phase of a benchmark. """ - print "{:<12s} {:<12s} {:<12s} ( {:<12s} )".format('', 'user', 'system', 'real') + print "{:<12s} {:<12s} {:<12s} ( {:<12s} )"\ + .format('', 'user', 'system', 'real') class BenchmarkReport(object): diff --git a/riak/client/multiget.py b/riak/client/multiget.py index 845170c0..9995133b 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -77,8 +77,7 @@ def start(self): # threads, set the started flag, and release the lock. for i in range(self._size): name = "riak.client.multiget-worker-{0}".format(i) - worker = Thread(target=self._fetcher, args=(name,), - name=name) + worker = Thread(target=self._fetcher, name=name) worker.daemon = True worker.start() self._workers.append(worker) @@ -109,7 +108,7 @@ def __del__(self): # shutting down. self.stop() - def _fetcher(self, name): + def _fetcher(self): """ The body of the multi-get worker. """ @@ -152,8 +151,7 @@ def multiget(client, keys, **options): RIAK_MULTIGET_POOL.start() for bucket, key in keys: - task = Task(client=client, outq=outq, options=options, - bucket=bucket, key=key) + task = Task(client, outq, bucket, key, options) RIAK_MULTIGET_POOL.enq(task) results = [] @@ -181,13 +179,13 @@ def multiget(client, keys, **options): print " Keys: {0}".format(len(bkeys)) print - with benchmark.bm() as b: + with benchmark.measure() as b: with b.report('populate'): for bucket, key in bkeys: client.bucket(bucket).new(key, encoded_data=data, content_type='text/plain' ).store() - for b in benchmark.bmbm(): + for b in benchmark.measure_with_rehearsal(): client.protocol = 'http' with b.report('http seq'): for bucket, key in bkeys: From 61a05b41e2b621a10a4cb5a790d096f6b64909a9 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 09:31:23 -0500 Subject: [PATCH 0430/1060] Bump client version and riak_pb dep for Riak 1.4 features. --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 06393638..7a982009 100755 --- a/setup.py +++ b/setup.py @@ -12,15 +12,15 @@ def make_docs(): for name in glob.glob('*.html'): os.rename(name, 'docs/%s' % name) -install_requires = ["riak_pb >=1.2.0, < 1.3.0"] -requires = ["riak_pb(>=1.2.0,<1.3.0)"] +install_requires = ["riak_pb >=1.4.0, < 1.5.0"] +requires = ["riak_pb(>=1.4.0,<1.5.0)"] tests_require = [] if platform.python_version() < '2.7': tests_require.append("unittest2") setup( name='riak', - version='1.5.1', + version='2.0.0a', packages = find_packages(), requires = requires, install_requires = install_requires, From 5f4b588a596eaa3400b55acfe36b6b9ef741df88 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 10:28:14 -0500 Subject: [PATCH 0431/1060] Unify the bucket properties tests since PBC will support all properties. --- riak/tests/test_all.py | 30 ++++------------------------ riak/tests/test_kv.py | 44 ++++++++++-------------------------------- 2 files changed, 14 insertions(+), 60 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index cbccbc98..e2fd3832 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -21,7 +21,7 @@ from riak.tests.test_mapreduce import MapReduceAliasTests, \ ErlangMapReduceTests, JSMapReduceTests, LinkTests, MapReduceStreamTests from riak.tests.test_kv import BasicKVTests, KVFileTests, \ - HTTPBucketPropsTest, PbcBucketPropsTest + BucketPropsTest from riak.tests.test_2i import TwoITests try: @@ -131,13 +131,14 @@ def test_request_retries(self): class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, - PbcBucketPropsTest, + BucketPropsTest, TwoITests, LinkTests, ErlangMapReduceTests, JSMapReduceTests, MapReduceAliasTests, MapReduceStreamTests, + EnableSearchTests, SearchTests, ClientTests, BaseTestCase, @@ -158,20 +159,10 @@ def test_uses_client_id_if_given(self): c = self.create_client(client_id=zero_client_id) self.assertEqual(zero_client_id, c.client_id) - def test_bucket_search_enabled(self): - with self.assertRaises(NotImplementedError): - bucket = self.client.bucket(self.bucket_name) - bucket.search_enabled() - - def test_enable_search_commit_hook(self): - with self.assertRaises(NotImplementedError): - bucket = self.client.bucket(self.bucket_name) - bucket.enable_search() - class RiakHttpTransportTestCase(BasicKVTests, KVFileTests, - HTTPBucketPropsTest, + BucketPropsTest, TwoITests, LinkTests, ErlangMapReduceTests, @@ -207,19 +198,6 @@ def test_too_many_link_headers_shouldnt_break_http(self): stored_object = bucket.get("lots_of_links") self.assertEqual(len(stored_object.links), 400) - def test_clear_bucket_properties(self): - bucket = self.client.bucket(self.props_bucket) - bucket.allow_mult = True - self.assertTrue(bucket.allow_mult) - bucket.n_val = 1 - self.assertEqual(bucket.n_val, 1) - # Test setting clearing properties... - - self.assertTrue(bucket.clear_properties()) - self.assertFalse(bucket.allow_mult) - self.assertEqual(bucket.n_val, 3) - - class FilterTests(unittest.TestCase): def test_simple(self): f1 = RiakKeyFilter("tokenize", "-", 1) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 1fb50fce..ac72748f 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -440,7 +440,7 @@ def generate_siblings(self, original, count=5, delay=None): return vals -class HTTPBucketPropsTest(object): +class BucketPropsTest(object): def test_rw_settings(self): bucket = self.client.bucket(self.props_bucket) self.assertEqual(bucket.r, "quorum") @@ -480,41 +480,17 @@ def test_primary_quora(self): bucket.set_properties({'pr': 0, 'pw': 0}) bucket.clear_properties() - -class PbcBucketPropsTest(object): - def test_rw_settings(self): + def test_clear_bucket_properties(self): bucket = self.client.bucket(self.props_bucket) - with self.assertRaises(NotImplementedError): - bucket.r - with self.assertRaises(NotImplementedError): - bucket.w - with self.assertRaises(NotImplementedError): - bucket.dw - with self.assertRaises(NotImplementedError): - bucket.rw - - with self.assertRaises(NotImplementedError): - bucket.r = 2 - with self.assertRaises(NotImplementedError): - bucket.w = 2 - with self.assertRaises(NotImplementedError): - bucket.dw = 2 - with self.assertRaises(NotImplementedError): - bucket.rw = 2 - with self.assertRaises(NotImplementedError): - bucket.clear_properties() + bucket.allow_mult = True + self.assertTrue(bucket.allow_mult) + bucket.n_val = 1 + self.assertEqual(bucket.n_val, 1) + # Test setting clearing properties... - def test_primary_quora(self): - bucket = self.client.bucket(self.props_bucket) - with self.assertRaises(NotImplementedError): - bucket.pr - with self.assertRaises(NotImplementedError): - bucket.pw - - with self.assertRaises(NotImplementedError): - bucket.pr = 2 - with self.assertRaises(NotImplementedError): - bucket.pw = 2 + self.assertTrue(bucket.clear_properties()) + self.assertFalse(bucket.allow_mult) + self.assertEqual(bucket.n_val, 3) class KVFileTests(object): From cdcab150a259bb906ab8319b3681ab48c6fdb7b4 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 22:57:02 -0500 Subject: [PATCH 0432/1060] Add support for all bucket properties and clearing in PB. Also, some code reorganization was done in codec.py, translate_rw_val was renamed to _encode_quorum. --- riak/transports/pbc/codec.py | 203 +++++++++++++++++++++++++++++-- riak/transports/pbc/messages.py | 9 +- riak/transports/pbc/transport.py | 54 ++++---- 3 files changed, 225 insertions(+), 41 deletions(-) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index c91dda20..f42f630c 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -19,30 +19,55 @@ from riak import RiakError from riak.content import RiakContent +def _invert(d): + out = {} + for key in d: + value = d[key] + out[value] = key + return out + +REPL_TO_PY = { + riak_pb.RpbBucketProps.FALSE: False, + riak_pb.RpbBucketProps.TRUE: True, + riak_pb.RpbBucketProps.REALTIME: 'realtime', + riak_pb.RpbBucketProps.FULLSYNC: 'fullsync' + } + +REPL_TO_PB = _invert(REPL_TO_PY) + RIAKC_RW_ONE = 4294967294 RIAKC_RW_QUORUM = 4294967293 RIAKC_RW_ALL = 4294967292 RIAKC_RW_DEFAULT = 4294967291 +QUORUM_TO_PB = { + 'default': RIAKC_RW_DEFAULT, + 'all': RIAKC_RW_ALL, + 'quorum': RIAKC_RW_QUORUM, + 'one': RIAKC_RW_ONE + } + +QUORUM_TO_PY = _invert(QUORUM_TO_PB) + +NORMAL_PROPS = ['n_val', 'allow_mult', 'last_write_wins', 'old_vclock', + 'young_vclock','big_vclock', 'small_vclock', + 'basic_quorum', 'notfound_ok', 'search', 'backend'] +COMMIT_HOOK_PROPS = ['precommit', 'postcommit'] +MODFUN_PROPS = ['chash_keyfun', 'linkfun'] +QUORUM_PROPS = ['r', 'pr', 'w', 'pw', 'dw', 'rw'] + class RiakPbcCodec(object): """ Protobuffs Encoding and decoding methods for RiakPbcTransport. """ - rw_names = { - 'default': RIAKC_RW_DEFAULT, - 'all': RIAKC_RW_ALL, - 'quorum': RIAKC_RW_QUORUM, - 'one': RIAKC_RW_ONE - } - def __init__(self, **unused_args): if riak_pb is None: raise NotImplementedError("this transport is not available") super(RiakPbcCodec, self).__init__(**unused_args) - def translate_rw_val(self, rw): + def _encode_quorum(self, rw): """ Converts a symbolic quorum value into its on-the-wire equivalent. @@ -51,14 +76,26 @@ def translate_rw_val(self, rw): :type rw: string, integer :rtype: integer """ - val = self.rw_names.get(rw) - if val is None: - return rw + if rw in QUORUM_TO_PB: + return QUORUM_TO_PB[rw] elif type(rw) is int and rw >= 0: - return val + return rw else: return None + def _decode_quorum(self, rw): + """ + Converts a protobuf quorum value to a symbolic value if + necessary. + + :param rw: the quorum + :type rw: int + :rtype int or string + """ + if rw in QUORUM_TO_PY: + return QUORUM_TO_PY[rw] + else: + return rw def _decode_contents(self, contents, obj): obj.siblings = [self._decode_content(c, RiakContent(obj)) for c in contents] @@ -170,3 +207,145 @@ def _decode_index_value(self, index, value): return int(value) else: return value + + def _encode_bucket_props(self, props, msg): + """ + Encodes a dict of bucket properties into the protobuf message. + + :param props: bucket properties + :type props: dict + :param msg: the protobuf message to fill + :type msg: riak_pb.RpbSetBucketReq + """ + msg.props.has_precommit = False + msg.props.has_postcommit = False + for prop in NORMAL_PROPS: + if prop in props and props[prop] is not None: + setattr(msg.props, prop, props[prop]) + for prop in COMMIT_HOOK_PROPS: + if prop in props: + setattr(msg.props, 'has_' + prop, True) + self._encode_hooklist(props[prop], getattr(msg.props, prop)) + for prop in MODFUN_PROPS: + if prop in props and props[prop] is not None: + self._encode_modfun(props[prop], getattr(msg.props, prop)) + for prop in QUORUM_PROPS: + if prop in props and props[prop] not in (None, 'default'): + value = self._encode_quorum(props[prop]) + if value is not None: + setattr(msg.props, prop, value) + if 'repl' in props: + msg.props.repl = REPL_TO_PY[props['repl']] + + return msg + + def _decode_bucket_props(self, msg): + """ + Decodes the protobuf bucket properties message into a dict. + + :param msg: the protobuf message to decode + :type msg: riak_pb.RpbBucketProps + :rtype dict + """ + props = {} + + for prop in NORMAL_PROPS: + if msg.HasField(prop): + props[prop] = getattr(msg, prop) + for prop in COMMIT_HOOK_PROPS: + if getattr(msg, 'has_' + prop): + props[prop] = self._decode_hooklist(getattr(msg, prop)) + for prop in MODFUN_PROPS: + if msg.HasField(prop): + props[prop] = self._decode_modfun(getattr(msg, prop)) + for prop in QUORUM_PROPS: + if msg.HasField(prop): + props[prop] = self._decode_quorum(getattr(msg, prop)) + if msg.HasField('repl'): + props['repl'] = REPL_TO_PY[msg.repl] + + return props + + def _decode_modfun(self, modfun): + """ + Decodes a protobuf modfun pair into a dict with 'mod' and + 'fun' keys. Used in bucket properties. + + :param modfun: the protobuf message to decode + :type modfun: riak_pb.RpbModFun + :rtype dict + """ + return {'mod': modfun.module, + 'fun': modfun.function} + + def _encode_modfun(self, props, msg=None): + """ + Encodes a dict with 'mod' and 'fun' keys into a protobuf + modfun pair. Used in bucket properties. + + :param props: the module/function pair + :type props: dict + :param msg: the protobuf message to fill + :type msg: riak_pb.RpbModFun + :rtype riak_pb.RpbModFun + """ + if msg is None: + msg = riak_pb.RpbModFun() + msg.module = props['mod'] + msg.function = props['fun'] + return msg + + def _decode_hooklist(self, hooklist): + """ + Decodes a list of protobuf commit hooks into their python + equivalents. Used in bucket properties. + + :param hooklist: a list of protobuf commit hooks + :type hooklist: list + :rtype list + """ + return [ self._decode_hook(hook) for hook in hooklist ] + + def _encode_hooklist(self, hooklist, msg): + """ + Encodes a list of commit hooks into their protobuf equivalent. + Used in bucket properties. + + :param hooklist: a list of commit hooks + :type hooklist: list + :param msg: a protobuf field that is a list of commit hooks + """ + for hook in hooklist: + pbhook = msg.add() + self._encode_hook(hook, pbhook) + + def _decode_hook(self, hook): + """ + Decodes a protobuf commit hook message into a dict. Used in + bucket properties. + + :param hook: the hook to decode + :type hook: riak_pb.RpbCommitHook + :rtype dict + """ + if hook.HasField('modfun'): + return self._decode_modfun(hook.modfun) + else: + return {'name': hook.name} + + def _encode_hook(self, hook, msg): + """ + Encodes a commit hook dict into the protobuf message. Used in + bucket properties. + + :param hook: the hook to encode + :type hook: dict + :param msg: the protobuf message to fill + :type msg: riak_pb.RpbCommitHook + :rtype riak_pb.RpbCommitHook + """ + if 'name' in hook: + msg.name = name + else: + self._encode_modfun(hook, msg.modfun) + return msg diff --git a/riak/transports/pbc/messages.py b/riak/transports/pbc/messages.py index fb9dba5c..d4b867c8 100644 --- a/riak/transports/pbc/messages.py +++ b/riak/transports/pbc/messages.py @@ -49,13 +49,16 @@ MSG_CODE_INDEX_RESP = 26 MSG_CODE_SEARCH_QUERY_REQ = 27 MSG_CODE_SEARCH_QUERY_RESP = 28 +MSG_CODE_RESET_BUCKET_REQ = 29 +MSG_CODE_RESET_BUCKET_RESP = 30 # These responses don't include messages EMPTY_RESPONSES = [ MSG_CODE_PING_RESP, MSG_CODE_SET_CLIENT_ID_RESP, MSG_CODE_DEL_RESP, - MSG_CODE_SET_BUCKET_RESP + MSG_CODE_SET_BUCKET_RESP, + MSG_CODE_RESET_BUCKET_RESP ] # Mapping from code to protobuf class @@ -88,5 +91,7 @@ MSG_CODE_INDEX_REQ: riak_pb.RpbIndexReq, MSG_CODE_INDEX_RESP: riak_pb.RpbIndexResp, MSG_CODE_SEARCH_QUERY_REQ: riak_pb.RpbSearchQueryReq, - MSG_CODE_SEARCH_QUERY_RESP: riak_pb.RpbSearchQueryResp + MSG_CODE_SEARCH_QUERY_RESP: riak_pb.RpbSearchQueryResp, + MSG_CODE_RESET_BUCKET_REQ: riak_pb.RpbResetBucketReq, + MSG_CODE_RESET_BUCKET_RESP: None } diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 31575d16..6d4cd00f 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -53,7 +53,9 @@ MSG_CODE_INDEX_REQ, MSG_CODE_INDEX_RESP, MSG_CODE_SEARCH_QUERY_REQ, - MSG_CODE_SEARCH_QUERY_RESP + MSG_CODE_SEARCH_QUERY_RESP, + MSG_CODE_RESET_BUCKET_REQ, + MSG_CODE_RESET_BUCKET_RESP ) @@ -123,9 +125,9 @@ def get(self, robj, r=None, pr=None): req = riak_pb.RpbGetReq() if r: - req.r = self.translate_rw_val(r) + req.r = self._encode_quorum(r) if self.quorum_controls() and pr: - req.pr = self.translate_rw_val(pr) + req.pr = self._encode_quorum(pr) if self.tombstone_vclocks(): req.deletedvclock = 1 @@ -160,11 +162,11 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, req = riak_pb.RpbPutReq() if w: - req.w = self.translate_rw_val(w) + req.w = self._encode_quorum(w) if dw: - req.dw = self.translate_rw_val(dw) + req.dw = self._encode_quorum(dw) if self.quorum_controls() and pw: - req.pw = self.translate_rw_val(pw) + req.pw = self._encode_quorum(pw) if return_body: req.return_body = 1 @@ -202,19 +204,19 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): req = riak_pb.RpbDelReq() if rw: - req.rw = self.translate_rw_val(rw) + req.rw = self._encode_quorum(rw) if r: - req.r = self.translate_rw_val(r) + req.r = self._encode_quorum(r) if w: - req.w = self.translate_rw_val(w) + req.w = self._encode_quorum(w) if dw: - req.dw = self.translate_rw_val(dw) + req.dw = self._encode_quorum(dw) if self.quorum_controls(): if pr: - req.pr = self.translate_rw_val(pr) + req.pr = self._encode_quorum(pr) if pw: - req.pw = self.translate_rw_val(pw) + req.pw = self._encode_quorum(pw) if self.tombstone_vclocks() and robj.vclock: req.vclock = robj.vclock.encode('binary') @@ -266,13 +268,8 @@ def get_bucket_props(self, bucket): msg_code, resp = self._request(MSG_CODE_GET_BUCKET_REQ, req, MSG_CODE_GET_BUCKET_RESP) - props = {} - if resp.props.HasField('n_val'): - props['n_val'] = resp.props.n_val - if resp.props.HasField('allow_mult'): - props['allow_mult'] = resp.props.allow_mult - return props + return self._decode_bucket_props(resp.props) def set_bucket_props(self, bucket, props): """ @@ -280,18 +277,21 @@ def set_bucket_props(self, bucket, props): """ req = riak_pb.RpbSetBucketReq() req.bucket = bucket.name - for key in props: - if key not in ['n_val', 'allow_mult']: - raise NotImplementedError - - if 'n_val' in props: - req.props.n_val = props['n_val'] - if 'allow_mult' in props: - req.props.allow_mult = props['allow_mult'] + self._encode_bucket_props(props, req) msg_code, resp = self._request(MSG_CODE_SET_BUCKET_REQ, req, MSG_CODE_SET_BUCKET_RESP) - return self + return True + + def clear_bucket_props(self, bucket): + """ + Clear bucket properties, resetting them to their defaults + """ + req = riak_pb.RpbResetBucketReq() + req.bucket = bucket.name + msg_code = self._request(MSG_CODE_RESET_BUCKET_REQ, req, + MSG_CODE_RESET_BUCKET_RESP) + return True def mapred(self, inputs, query, timeout=None): # dictionary of phase results - each content should be an encoded array From 43b663e1bb7d21fe479f22e457d25d0b2f2f156b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 22:57:19 -0500 Subject: [PATCH 0433/1060] Add some much needed documentation for other PB codec methods. --- riak/transports/pbc/codec.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index f42f630c..3d4782a8 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -96,7 +96,18 @@ def _decode_quorum(self, rw): return QUORUM_TO_PY[rw] else: return rw + def _decode_contents(self, contents, obj): + """ + Decodes the list of siblings from the protobuf representation + into the object. + + :param contents: a list of RpbContent messages + :type contents: list + :param obj: a RiakObject + :type obj: RiakObject + :rtype RiakObject + """ obj.siblings = [self._decode_content(c, RiakContent(obj)) for c in contents] # Invoke sibling-resolution logic @@ -109,7 +120,11 @@ def _decode_content(self, rpb_content, sibling): Decodes a single sibling from the protobuf representation into a RiakObject. - :rtype: (RiakObject) + :param rpb_content: a single RpbContent message + :type rpb_content: riak_pb.RpbContent + :param sibling: a RiakContent sibling container + :type sibling: RiakContent + :rtype: RiakContent """ if rpb_content.HasField("deleted") and rpb_content.deleted: @@ -147,6 +162,11 @@ def _encode_content(self, robj, rpb_content): """ Fills an RpbContent message with the appropriate data and metadata from a RiakObject. + + :param robj: a RiakObject + :type robj: RiakObject + :param rpb_content: the protobuf message to fill + :type rpb_content: riak_pb.RpbContent """ if robj.content_type: rpb_content.content_type = robj.content_type @@ -182,6 +202,10 @@ def _encode_content(self, robj, rpb_content): def _decode_link(self, link): """ Decodes an RpbLink message into a tuple + + :param link: an RpbLink message + :type link: riak_pb.RpbLink + :rtype tuple """ if link.HasField("bucket"): @@ -202,6 +226,11 @@ def _decode_link(self, link): def _decode_index_value(self, index, value): """ Decodes a secondary index value into the correct Python type. + :param index: the name of the index + :type index: str + :param value: the value of the index entry + :type value: str + :rtype str or int """ if index.endswith("_int"): return int(value) From caf7fafe8137a17a6e9dacfdf56459230eeed9bd Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 22:58:32 -0500 Subject: [PATCH 0434/1060] Cleanup bucket properties shortcuts with a helper function. --- riak/bucket.py | 72 +++++++++++--------------------------------------- 1 file changed, 16 insertions(+), 56 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 29861f45..6021d140 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -24,6 +24,14 @@ def deprecateBucketQuorumAccessors(klass): return deprecateQuorumAccessors(klass, parent='_client') +def bucket_property(name, doc=None): + def _prop_getter(self): + return self.get_property(name) + + def _prop_setter(self, value): + return self.set_property(name, value) + + return property(_prop_getter, _prop_setter, doc=doc) @deprecateBucketQuorumAccessors class RiakBucket(object): @@ -220,13 +228,7 @@ def _set_resolver(self, value): client's resolver will be used. :type callable""") - def _set_n_val(self, nval): - return self.set_property('n_val', nval) - - def _get_n_val(self): - return self.get_property('n_val') - - n_val = property(_get_n_val, _set_n_val, doc=""" + n_val = bucket_property('n_val', doc=""" N-value for this bucket, which is the number of replicas that will be written of each object in the bucket. @@ -239,13 +241,7 @@ def _get_n_val(self): :type nval: integer """) - def _set_allow_mult(self, bool): - return self.set_property('allow_mult', bool) - - def _get_allow_mult(self): - return self.get_property('allow_mult') - - allow_mult = property(_get_allow_mult, _set_allow_mult, doc=""" + allow_mult = bucket_property('allow_mult', doc=""" If set to True, then writes with conflicting data will be stored and returned to the client. This situation can be detected by calling has_siblings() and get_siblings(). @@ -253,73 +249,37 @@ def _get_allow_mult(self): :type bool: boolean """) - def _set_r(self, val): - return self.set_property('r', val) - - def _get_r(self): - return self.get_property('r') - - r = property(_get_r, _set_r, doc=""" + r = bucket_property('r', doc=""" The default 'read' quorum for this bucket (how many replicas must reply for a successful read). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_pr(self, val): - return self.set_property('pr', val) - - def _get_pr(self): - return self.get_property('pr') - - pr = property(_get_pr, _set_pr, doc=""" + pr = bucket_property('pr', doc=""" The default 'primary read' quorum for this bucket (how many primary replicas are required for a successful read). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_rw(self, val): - return self.set_property('rw', val) - - def _get_rw(self): - return self.get_property('rw') - - rw = property(_get_rw, _set_rw, doc=""" + rw = bucket_property('rw', doc=""" The default 'read' and 'write' quorum for this bucket (equivalent to 'r' and 'w' but for deletes). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_w(self, val): - return self.set_property('w', val) - - def _get_w(self): - return self.get_property('w') - - w = property(_get_w, _set_w, doc=""" + w = bucket_property('w', doc=""" The default 'write' quorum for this bucket (how many replicas must acknowledge receipt of a write). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_dw(self, val): - return self.set_property('dw', val) - - def _get_dw(self): - return self.get_property('dw') - - dw = property(_get_dw, _set_dw, doc=""" + dw = bucket_property('dw', doc=""" The default 'durable write' quorum for this bucket (how many replicas must commit the write). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_pw(self, val): - return self.set_property('pw', val) - - def _get_pw(self): - return self.get_property('pw') - - pw = property(_get_pw, _set_pw, doc=""" + pw = bucket_property('pw', doc=""" The default 'primary write' quorum for this bucket (how many primary replicas are required for a successful write). This should be an integer less than the 'n_val' property, or a string of From d94a16fa947b0c9fd87b4f42e024056330fd4bfa Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 23:00:07 -0500 Subject: [PATCH 0435/1060] Use the search property instead of directly modifying precommit. The bucket fixup for search has been available since 1.0, it is much cleaner and more reliable to use it instead. --- riak/bucket.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 6021d140..4ff1dbed 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -304,10 +304,7 @@ def get_property(self, key): :type key: string :rtype: mixed """ - try: - return self.get_properties()[key] - except KeyError: - raise NotImplementedError + return self.get_properties()[key] def set_properties(self, props): """ @@ -380,19 +377,15 @@ def search_enabled(self): Returns True if the search precommit hook is enabled for this bucket. """ - return self.SEARCH_PRECOMMIT_HOOK in (self.get_property("precommit") or - []) + return self.get_properties().get('search', False) def enable_search(self): """ Enable search for this bucket by installing the precommit hook to index objects in it. """ - precommit_hooks = self.get_property("precommit") or [] - if self.SEARCH_PRECOMMIT_HOOK not in precommit_hooks: - self.set_properties({"precommit": - precommit_hooks + - [self.SEARCH_PRECOMMIT_HOOK]}) + if not self.search_enabled(): + self.set_property('search', True) return True def disable_search(self): @@ -400,10 +393,8 @@ def disable_search(self): Disable search for this bucket by removing the precommit hook to index objects in it. """ - precommit_hooks = self.get_property("precommit") or [] - if self.SEARCH_PRECOMMIT_HOOK in precommit_hooks: - precommit_hooks.remove(self.SEARCH_PRECOMMIT_HOOK) - self.set_properties({"precommit": precommit_hooks}) + if self.search_enabled(): + self.set_property('search', False) return True def search(self, query, **params): From a26a6998a6ad5006deeccac631c68dc9a8d51837 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 23:00:22 -0500 Subject: [PATCH 0436/1060] Fix a few doc string problems in bucket.py. --- riak/bucket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 4ff1dbed..b8c256ba 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -179,7 +179,7 @@ def new_binary(self, key=None, data=None, def get(self, key, r=None, pr=None): """ - Retrieve a JSON-encoded object from Riak. + Retrieve an object from Riak. :param key: Name of the key. :type key: string @@ -194,7 +194,7 @@ def get(self, key, r=None, pr=None): def get_binary(self, key, r=None, pr=None): """ - Retrieve a binary/string object from Riak. + Retrieve a binary/string object from Riak. DEPRECATED :param key: Name of the key. :type key: string From cc54a053e0675747d13bd2b63c083dac157af55d Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Jun 2013 08:26:03 -0500 Subject: [PATCH 0437/1060] Predicate the bucket properties features on version detection. --- riak/tests/test_feature_detection.py | 24 ++++++++++++++++++++++-- riak/transports/feature_detect.py | 19 ++++++++++++++++++- riak/transports/pbc/codec.py | 2 -- riak/transports/pbc/transport.py | 10 ++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py index 0ab7fde4..4048c0d5 100644 --- a/riak/tests/test_feature_detection.py +++ b/riak/tests/test_feature_detection.py @@ -42,9 +42,8 @@ class FeatureDetectionTest(unittest.TestCase): def test_implements_server_version(self): t = IncompleteTransport() - def get_server_version(): + with self.assertRaises(NotImplementedError): t.server_version - self.assertRaises(NotImplementedError, get_server_version) def test_pre_10(self): t = DummyTransport("0.14.2") @@ -55,6 +54,8 @@ def test_pre_10(self): self.assertFalse(t.quorum_controls()) self.assertFalse(t.tombstone_vclocks()) self.assertFalse(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) def test_10(self): t = DummyTransport("1.0.3") @@ -65,6 +66,8 @@ def test_10(self): self.assertTrue(t.quorum_controls()) self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) def test_11(self): t = DummyTransport("1.1.4") @@ -75,6 +78,8 @@ def test_11(self): self.assertTrue(t.quorum_controls()) self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) def test_12(self): t = DummyTransport("1.2.0") @@ -85,6 +90,8 @@ def test_12(self): self.assertTrue(t.quorum_controls()) self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) def test_12_loose(self): t = DummyTransport("1.2.1p3") @@ -95,7 +102,20 @@ def test_12_loose(self): self.assertTrue(t.quorum_controls()) self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) + def test_14(self): + t = DummyTransport("1.4.0rc1") + self.assertTrue(t.phaseless_mapred()) + self.assertTrue(t.pb_indexes()) + self.assertTrue(t.pb_search()) + self.assertTrue(t.pb_conditionals()) + self.assertTrue(t.quorum_controls()) + self.assertTrue(t.tombstone_vclocks()) + self.assertTrue(t.pb_head()) + self.assertTrue(t.pb_clear_bucket_props()) + self.assertTrue(t.pb_all_bucket_props()) if __name__ == '__main__': unittest.main() diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 3d8acfd9..3712ec8b 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -23,7 +23,8 @@ versions = { 1: LooseVersion("1.0.0"), 1.1: LooseVersion("1.1.0"), - 1.2: LooseVersion("1.2.0") + 1.2: LooseVersion("1.2.0"), + 1.4: LooseVersion("1.4.0") } @@ -90,6 +91,22 @@ def pb_head(self): """ return self.server_version >= versions[1] + def pb_clear_bucket_props(self): + """ + Whether bucket properties can be cleared over Protocol + Buffers. + :rtype bool + """ + return self.server_version >= versions[1.4] + + def pb_all_bucket_props(self): + """ + Whether all normal bucket properties are supported over + Protocol Buffers. + :rtype bool + """ + return self.server_version >= versions[1.4] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 3d4782a8..70d093f2 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -246,8 +246,6 @@ def _encode_bucket_props(self, props, msg): :param msg: the protobuf message to fill :type msg: riak_pb.RpbSetBucketReq """ - msg.props.has_precommit = False - msg.props.has_postcommit = False for prop in NORMAL_PROPS: if prop in props and props[prop] is not None: setattr(msg.props, prop, props[prop]) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 6d4cd00f..da7bb64e 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -277,6 +277,13 @@ def set_bucket_props(self, bucket, props): """ req = riak_pb.RpbSetBucketReq() req.bucket = bucket.name + + if not self.pb_all_bucket_props(): + for key in props: + if key not in ('n_val', 'allow_mult'): + raise NotImplementedError('Server only supports n_val and ' + 'allow_mult properties over PBC') + self._encode_bucket_props(props, req) msg_code, resp = self._request(MSG_CODE_SET_BUCKET_REQ, req, @@ -287,6 +294,9 @@ def clear_bucket_props(self, bucket): """ Clear bucket properties, resetting them to their defaults """ + if not self.pb_clear_bucket_props(): + return False + req = riak_pb.RpbResetBucketReq() req.bucket = bucket.name msg_code = self._request(MSG_CODE_RESET_BUCKET_REQ, req, From a31efe131e7d3f329a8a1e01993ebcdde3a1a4a2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Jun 2013 08:32:44 -0500 Subject: [PATCH 0438/1060] Apply some PEP8 and pyflakes fixes. --- riak/bucket.py | 2 ++ riak/tests/test_all.py | 1 + riak/transports/pbc/codec.py | 29 +++++++++++++---------------- riak/transports/pbc/transport.py | 4 ++-- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index b8c256ba..966ebe68 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -24,6 +24,7 @@ def deprecateBucketQuorumAccessors(klass): return deprecateQuorumAccessors(klass, parent='_client') + def bucket_property(name, doc=None): def _prop_getter(self): return self.get_property(name) @@ -33,6 +34,7 @@ def _prop_setter(self, value): return property(_prop_getter, _prop_setter, doc=doc) + @deprecateBucketQuorumAccessors class RiakBucket(object): """ diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index e2fd3832..592ecdf1 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -198,6 +198,7 @@ def test_too_many_link_headers_shouldnt_break_http(self): stored_object = bucket.get("lots_of_links") self.assertEqual(len(stored_object.links), 400) + class FilterTests(unittest.TestCase): def test_simple(self): f1 = RiakKeyFilter("tokenize", "-", 1) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 70d093f2..250995aa 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -19,6 +19,7 @@ from riak import RiakError from riak.content import RiakContent + def _invert(d): out = {} for key in d: @@ -26,12 +27,10 @@ def _invert(d): out[value] = key return out -REPL_TO_PY = { - riak_pb.RpbBucketProps.FALSE: False, - riak_pb.RpbBucketProps.TRUE: True, - riak_pb.RpbBucketProps.REALTIME: 'realtime', - riak_pb.RpbBucketProps.FULLSYNC: 'fullsync' - } +REPL_TO_PY = {riak_pb.RpbBucketProps.FALSE: False, + riak_pb.RpbBucketProps.TRUE: True, + riak_pb.RpbBucketProps.REALTIME: 'realtime', + riak_pb.RpbBucketProps.FULLSYNC: 'fullsync'} REPL_TO_PB = _invert(REPL_TO_PY) @@ -40,18 +39,16 @@ def _invert(d): RIAKC_RW_ALL = 4294967292 RIAKC_RW_DEFAULT = 4294967291 -QUORUM_TO_PB = { - 'default': RIAKC_RW_DEFAULT, - 'all': RIAKC_RW_ALL, - 'quorum': RIAKC_RW_QUORUM, - 'one': RIAKC_RW_ONE - } +QUORUM_TO_PB = {'default': RIAKC_RW_DEFAULT, + 'all': RIAKC_RW_ALL, + 'quorum': RIAKC_RW_QUORUM, + 'one': RIAKC_RW_ONE} QUORUM_TO_PY = _invert(QUORUM_TO_PB) NORMAL_PROPS = ['n_val', 'allow_mult', 'last_write_wins', 'old_vclock', - 'young_vclock','big_vclock', 'small_vclock', - 'basic_quorum', 'notfound_ok', 'search', 'backend'] + 'young_vclock', 'big_vclock', 'small_vclock', 'basic_quorum', + 'notfound_ok', 'search', 'backend'] COMMIT_HOOK_PROPS = ['precommit', 'postcommit'] MODFUN_PROPS = ['chash_keyfun', 'linkfun'] QUORUM_PROPS = ['r', 'pr', 'w', 'pw', 'dw', 'rw'] @@ -331,7 +328,7 @@ def _decode_hooklist(self, hooklist): :type hooklist: list :rtype list """ - return [ self._decode_hook(hook) for hook in hooklist ] + return [self._decode_hook(hook) for hook in hooklist] def _encode_hooklist(self, hooklist, msg): """ @@ -372,7 +369,7 @@ def _encode_hook(self, hook, msg): :rtype riak_pb.RpbCommitHook """ if 'name' in hook: - msg.name = name + msg.name = hook['name'] else: self._encode_modfun(hook, msg.modfun) return msg diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index da7bb64e..0b62689a 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -299,8 +299,8 @@ def clear_bucket_props(self, bucket): req = riak_pb.RpbResetBucketReq() req.bucket = bucket.name - msg_code = self._request(MSG_CODE_RESET_BUCKET_REQ, req, - MSG_CODE_RESET_BUCKET_RESP) + self._request(MSG_CODE_RESET_BUCKET_REQ, req, + MSG_CODE_RESET_BUCKET_RESP) return True def mapred(self, inputs, query, timeout=None): From b0efb68fca9d504feda81f9abde4c4c38a84d55a Mon Sep 17 00:00:00 2001 From: Pavel Kirienko Date: Sat, 29 Jun 2013 13:26:01 +0400 Subject: [PATCH 0439/1060] Fixes ConflictError on non-existent objects --- riak/riak_object.py | 2 ++ riak/tests/test_kv.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/riak/riak_object.py b/riak/riak_object.py index 48d9b779..f6a4aa8f 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -39,6 +39,8 @@ def _setter(self, value): setattr(self.siblings[0], name, value) def _getter(self): + if len(self.siblings) == 0: + return if len(self.siblings) != 1: raise ConflictError() return getattr(self.siblings[0], name) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 1fb50fce..c7be3646 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -220,6 +220,8 @@ def test_missing_object(self): bucket = self.client.bucket(self.bucket_name) obj = bucket.get(self.key_name) self.assertFalse(obj.exists) + # Object with no siblings should not raise the ConflictError + self.assertIsNone(obj.data) def test_delete(self): bucket = self.client.bucket(self.bucket_name) From be7791a351f2d81360675cd9b28e0b314734f66c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 28 Jun 2013 08:57:43 -0500 Subject: [PATCH 0440/1060] Add surface API for 1.4 counters. --- riak/bucket.py | 22 +++++++++++++++ riak/client/operations.py | 54 ++++++++++++++++++++++++++++++++++++ riak/transports/transport.py | 14 ++++++++++ 3 files changed, 90 insertions(+) diff --git a/riak/bucket.py b/riak/bucket.py index 966ebe68..993505f6 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -421,6 +421,28 @@ def delete(self, key, **kwargs): """ return self.new(key).delete(**kwargs) + def get_counter(self, key, **kwargs): + """ + Gets the value of a counter stored in this bucket. + + :param key: the key of the counter + :type key: string + :rtype int + """ + return self._client.get_counter(self, key, **kwargs) + + def update_counter(self, key, value, **kwargs): + """ + Updates the value of a counter stored in this bucket. Positive + values increment the counter, negative values decrement. + + :param key: the key of the counter + :type key: string + :param value: the amount to increment or decrement + :type value: integer + """ + return self._client.update_counter(self, key, value, **kwargs) + def __str__(self): return ''.format(self.name) diff --git a/riak/client/operations.py b/riak/client/operations.py index c928128f..f1220c7a 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -271,3 +271,57 @@ def fulltext_delete(self, transport, index, docs=None, queries=None): :type queries: list """ transport.fulltext_delete(index, docs, queries) + + @retryable + def get_counter(self, transport, bucket, key, r=None, pr=None, + basic_quorum=None, notfound_ok=None): + """ + Gets the value of a counter. + + :param bucket: the bucket of the counter + :type bucket: RiakBucket + :param key: the key of the counter + :type key: string + :param r: the read quorum + :type r: integer, string, None + :param pr: the primary read quorum + :type pr: integer, string, None + :param basic_quorum: whether to use the "basic quorum" policy + for not-founds + :type basic_quorum: bool + :param notfound_ok: whether to treat not-found responses as successful + :type notfound_ok: bool + :rtype integer + """ + return transport.get_counter(bucket, key, r=r, pr=pr) + + def update_counter(self, bucket, key, value, w=None, dw=None, pw=None, + returnvalue=False): + """ + Updates a counter by the given value. This operation is not + idempotent and so should not be retried automatically. + + :param bucket: the bucket of the counter + :type bucket: RiakBucket + :param key: the key of the counter + :type key: string + :param value: the amount to increment or decrement + :type value: integer + :param w: the write quorum + :type w: integer, string, None + :param dw: the durable write quorum + :type dw: integer, string, None + :param pw: the primary write quorum + :type pw: integer, string, None + :param returnvalue: whether to return the updated value of the counter + :type returnvalue: bool + """ + if type(value) is not int: + raise TypeError("Counter update amount must be an integer") + if value == 0: + raise ValueError("Cannot increment counter by 0") + + with self._transport() as transport: + return transport.update_counter(bucket, key, value, + w=w, dw=dw, pw=pw, + returnvalue=returnvalue) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index c6d581c7..a61d2ee3 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -179,6 +179,20 @@ def fulltext_delete(self, index, docs=None, queries=None): """ raise NotImplementedError + def get_counter(self, bucket, key, r=None, pr=None, basic_quorum=None, + notfound_ok=None): + """ + Gets the value of a counter. + """ + raise NotImplementedError + + def update_counter(self, bucket, key, value, w=None, dw=None, pw=None, + returnvalue=False): + """ + Updates a counter by the given value. + """ + raise NotImplementedError + def _search_mapred_emu(self, index, query): """ Emulates a search request via MapReduce. Used in the case From 5deacf893f7946d89f865689672cbe0c68faada7 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 28 Jun 2013 17:09:08 -0500 Subject: [PATCH 0441/1060] Implement counter ops in transports and add tests. --- riak/tests/test_all.py | 4 ++- riak/tests/test_kv.py | 28 ++++++++++++++++ riak/transports/feature_detect.py | 7 ++++ riak/transports/http/resources.py | 11 +++++++ riak/transports/http/transport.py | 29 +++++++++++++++++ riak/transports/pbc/messages.py | 11 ++++++- riak/transports/pbc/transport.py | 53 ++++++++++++++++++++++++++++++- 7 files changed, 140 insertions(+), 3 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 592ecdf1..4fe3755f 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -21,7 +21,7 @@ from riak.tests.test_mapreduce import MapReduceAliasTests, \ ErlangMapReduceTests, JSMapReduceTests, LinkTests, MapReduceStreamTests from riak.tests.test_kv import BasicKVTests, KVFileTests, \ - BucketPropsTest + BucketPropsTest, CounterTests from riak.tests.test_2i import TwoITests try: @@ -141,6 +141,7 @@ class RiakPbcTransportTestCase(BasicKVTests, EnableSearchTests, SearchTests, ClientTests, + CounterTests, BaseTestCase, unittest.TestCase): @@ -173,6 +174,7 @@ class RiakHttpTransportTestCase(BasicKVTests, SolrSearchTests, SearchTests, ClientTests, + CounterTests, BaseTestCase, unittest.TestCase): diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index ac72748f..017dc3f6 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -519,3 +519,31 @@ def test_store_binary_object_from_file_should_fail_if_file_not_found(self): obj = bucket.get(self.key_name) # self.assertEqual(obj.encoded_data, None) self.assertFalse(obj.exists) + + +class CounterTests(object): + def test_counter_requires_allow_mult(self): + bucket = self.client.bucket(self.bucket_name) + self.assertFalse(bucket.allow_mult) + + with self.assertRaises(Exception): + bucket.update_counter(self.key_name, 10) + + def test_counter_ops(self): + bucket = self.client.bucket(self.sibs_bucket) + self.assertTrue(bucket.allow_mult) + + # Non-existent counter has no value + self.assertEqual(None, bucket.get_counter(self.key_name)) + + # Update the counter + bucket.update_counter(self.key_name, 10) + self.assertEqual(10, bucket.get_counter(self.key_name)) + + # Update with returning the value + self.assertEqual(15, bucket.update_counter(self.key_name, 5, + returnvalue=True)) + + # Now try decrementing + self.assertEqual(10, bucket.update_counter(self.key_name, -5, + returnvalue=True)) diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 3712ec8b..16ab2016 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -107,6 +107,13 @@ def pb_all_bucket_props(self): """ return self.server_version >= versions[1.4] + def counters(self): + """ + Whether CRDT counters are supported. + :rtype bool + """ + return self.server_version >= versions[1.4] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index d46fb414..9942e06d 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -108,6 +108,13 @@ def luwak_path(self, key=None): key = quote_plus(key) return mkpath(self.luwak_wm_file, key) + def counters_path(self, bucket, key, **options): + if not self.riak_kv_wm_counter: + raise RiakError("Counters are unsupported by this Riak node") + + return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), "counters", + quote_plus(key), **options) + @lazy_property def riak_kv_wm_buckets(self): return self.resources.get('riak_kv_wm_index') @@ -144,6 +151,10 @@ def riak_solr_indexer_wm(self): def luwak_wm_file(self): return self.resources.get('luwak_wm_file') + @lazy_property + def riak_kv_wm_counter(self): + return self.resources.get('riak_kv_wm_counter') + @lazy_property def resources(self): return self.get_resources() diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 632f37bf..99ffc2f3 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -356,6 +356,35 @@ def fulltext_delete(self, index, docs=None, queries=None): {'Content-Type': 'text/xml'}, xml.toxml().encode('utf-8')) + def get_counter(self, bucket, key, **options): + if not self.counters(): + raise NotImplementedError("Counters are not supported") + + url = self.counters_path(bucket.name, key, **options) + status, headers, body = self._request('GET', url) + + self.check_http_code(status, [200, 404]) + if status == 200: + return long(body.strip()) + elif status == 404: + return None + + def update_counter(self, bucket, key, amount, **options): + if not self.counters(): + raise NotImplementedError("Counters are not supported") + + return_value = 'returnvalue' in options and options['returnvalue'] + headers = {'Content-Type': 'text/plain'} + url = self.counters_path(bucket.name, key, **options) + status, headers, body = self._request('POST', url, headers, + str(amount)) + if return_value and status == 200: + return long(body.strip()) + elif status == 204: + return True + else: + self.check_http_code(status, [200, 204]) + def check_http_code(self, status, expected_statuses): if not status in expected_statuses: raise Exception('Expected status %s, received %s' % diff --git a/riak/transports/pbc/messages.py b/riak/transports/pbc/messages.py index d4b867c8..487b611a 100644 --- a/riak/transports/pbc/messages.py +++ b/riak/transports/pbc/messages.py @@ -51,6 +51,10 @@ MSG_CODE_SEARCH_QUERY_RESP = 28 MSG_CODE_RESET_BUCKET_REQ = 29 MSG_CODE_RESET_BUCKET_RESP = 30 +MSG_CODE_COUNTER_UPDATE_REQ = 50 +MSG_CODE_COUNTER_UPDATE_RESP = 51 +MSG_CODE_COUNTER_GET_REQ = 52 +MSG_CODE_COUNTER_GET_RESP = 53 # These responses don't include messages EMPTY_RESPONSES = [ @@ -93,5 +97,10 @@ MSG_CODE_SEARCH_QUERY_REQ: riak_pb.RpbSearchQueryReq, MSG_CODE_SEARCH_QUERY_RESP: riak_pb.RpbSearchQueryResp, MSG_CODE_RESET_BUCKET_REQ: riak_pb.RpbResetBucketReq, - MSG_CODE_RESET_BUCKET_RESP: None + MSG_CODE_RESET_BUCKET_RESP: None, + MSG_CODE_COUNTER_UPDATE_REQ: riak_pb.RpbCounterUpdateReq, + MSG_CODE_COUNTER_UPDATE_RESP: riak_pb.RpbCounterUpdateResp, + MSG_CODE_COUNTER_GET_REQ: riak_pb.RpbCounterGetReq, + MSG_CODE_COUNTER_GET_RESP: riak_pb.RpbCounterGetResp + } diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 0b62689a..d4f4aeba 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -55,7 +55,11 @@ MSG_CODE_SEARCH_QUERY_REQ, MSG_CODE_SEARCH_QUERY_RESP, MSG_CODE_RESET_BUCKET_REQ, - MSG_CODE_RESET_BUCKET_RESP + MSG_CODE_RESET_BUCKET_RESP, + MSG_CODE_COUNTER_UPDATE_REQ, + MSG_CODE_COUNTER_UPDATE_RESP, + MSG_CODE_COUNTER_GET_REQ, + MSG_CODE_COUNTER_GET_RESP ) @@ -396,3 +400,50 @@ def search(self, index, query, **params): docs.append(resultdoc) result['docs'] = docs return result + + def get_counter(self, bucket, key, **params): + if not self.counters(): + raise NotImplementedError("Counters are not supported") + + req = riak_pb.RpbCounterGetReq() + req.bucket = bucket.name + req.key = key + if params.get('r') is not None: + req.r = self._encode_quorum(params['r']) + if params.get('pr') is not None: + req.pr = self._encode_quorum(params['pr']) + if params.get('basic_quorum') is not None: + req.basic_quorum = params['basic_quorum'] + if params.get('notfound_ok') is not None: + req.notfound_ok = params['notfound_ok'] + + msg_code, resp = self._request(MSG_CODE_COUNTER_GET_REQ, req, + MSG_CODE_COUNTER_GET_RESP) + if resp.HasField('value'): + return resp.value + else: + return None + + def update_counter(self, bucket, key, value, **params): + if not self.counters(): + raise NotImplementedError("Counters are not supported") + + req = riak_pb.RpbCounterUpdateReq() + req.bucket = bucket.name + req.key = key + req.amount = value + if params.get('w') is not None: + req.w = self._encode_quorum(params['w']) + if params.get('dw') is not None: + req.dw = self._encode_quorum(params['dw']) + if params.get('pw') is not None: + req.pw = self._encode_quorum(params['pw']) + if params.get('returnvalue') is not None: + req.returnvalue = params['returnvalue'] + + msg_code, resp = self._request(MSG_CODE_COUNTER_UPDATE_REQ, req, + MSG_CODE_COUNTER_UPDATE_RESP) + if resp.HasField('value'): + return resp.value + else: + return True From abab89256df45b62abf8ce0cb6d852b65a4c28b2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 11:38:03 -0500 Subject: [PATCH 0442/1060] Allow longs for update amounts, since transport returns longs. --- riak/client/operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index f1220c7a..4597fd06 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -316,7 +316,7 @@ def update_counter(self, bucket, key, value, w=None, dw=None, pw=None, :param returnvalue: whether to return the updated value of the counter :type returnvalue: bool """ - if type(value) is not int: + if type(value) not in (int, long): raise TypeError("Counter update amount must be an integer") if value == 0: raise ValueError("Cannot increment counter by 0") From 6b85a04fadee837d324f9104baf85ceb7800c16a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 14:16:57 -0500 Subject: [PATCH 0443/1060] Add surface API for streaming list-buckets. --- riak/client/operations.py | 21 +++++++++++++++++++++ riak/transports/transport.py | 7 ++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index c928128f..b6b8e877 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -35,9 +35,30 @@ def get_buckets(self, transport): Get the list of buckets as RiakBucket instances. NOTE: Do not use this in production, as it requires traversing through all keys stored in a cluster. + + :rtype list of RiakBucket instances """ return [self.bucket(name) for name in transport.get_buckets()] + def stream_buckets(self): + """ + Streams the list of buckets. This is a generator method that + should be iterated over. NOTE: Do not use this in production, + as it requires traversing through all keys stored in a + cluster. + + :rtype iterator + """ + with self._transport() as transport: + stream = transport.stream_buckets() + try: + for bucket_list in stream: + bucket_list = [self.bucket(name) for name in bucket_list] + if len(bucket_list) > 0: + yield bucket_list + finally: + stream.close() + @retryable def ping(self, transport): """ diff --git a/riak/transports/transport.py b/riak/transports/transport.py index c6d581c7..11185791 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -90,7 +90,12 @@ def delete(self, robj, rw=None): def get_buckets(self): """ Serialize get buckets request and deserialize response - @return dict() + """ + raise NotImplementedError + + def stream_buckets(self): + """ + Streams the list of buckets through an iterator """ raise NotImplementedError From a70ab9b6e1c6b1714c40922c713e861e7b04927d Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 15:00:51 -0500 Subject: [PATCH 0444/1060] Add implementation of streaming list-buckets. * Feature detection for streaming list-buckets was added. * HTTP bucket_list_path was modified to allow overriding the 'buckets' qs param. * Since streaming list-buckets and list-keys use roughly the same JSON format, the common bits were factored out. --- riak/transports/feature_detect.py | 7 +++++++ riak/transports/http/resources.py | 4 ++-- riak/transports/http/stream.py | 24 ++++++++++++++++++------ riak/transports/http/transport.py | 19 ++++++++++++++++++- riak/transports/pbc/stream.py | 19 ++++++++++++++++++- riak/transports/pbc/transport.py | 18 +++++++++++++++++- 6 files changed, 80 insertions(+), 11 deletions(-) diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 3712ec8b..5dac0180 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -107,6 +107,13 @@ def pb_all_bucket_props(self): """ return self.server_version >= versions[1.4] + def bucket_stream(self): + """ + Whether streaming bucket lists are supported. + :rtype bool + """ + return self.server_version >= versions[1.4] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index d46fb414..92c52032 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -38,8 +38,8 @@ def mapred_path(self, **options): return mkpath(self.riak_kv_wm_mapred, **options) def bucket_list_path(self, **options): - query = options.copy() - query.update(buckets=True) + query = {'buckets': True} + query.update(options) if self.riak_kv_wm_buckets: return mkpath(self.riak_kv_wm_buckets, **query) else: diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 2f1620f9..ec49eee4 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -51,10 +51,8 @@ def close(self): pass -class RiakHttpKeyStream(RiakHttpStream): - """ - Streaming iterator for list-keys over HTTP - """ +class RiakHttpJsonStream(RiakHttpStream): + _json_field = None def next(self): while '}' not in self.buffer and not self.response_done: @@ -64,12 +62,26 @@ def next(self): idx = string.index(self.buffer, '}') + 1 chunk = self.buffer[:idx] self.buffer = self.buffer[idx:] - keys = json.loads(chunk)[u'keys'] - return keys + field = json.loads(chunk)[self._json_field] + return field else: raise StopIteration +class RiakHttpKeyStream(RiakHttpJsonStream): + """ + Streaming iterator for list-keys over HTTP + """ + _json_field = u'keys' + + +class RiakHttpBucketStream(RiakHttpJsonStream): + """ + Streaming iterator for list-buckets over HTTP + """ + _json_field = u'buckets' + + class RiakHttpMultipartStream(RiakHttpStream): """ Streaming iterator for multipart messages over HTTP diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 632f37bf..91fd910d 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -33,7 +33,8 @@ from riak.transports.http.codec import RiakHttpCodec from riak.transports.http.stream import ( RiakHttpKeyStream, - RiakHttpMapReduceStream) + RiakHttpMapReduceStream, + RiakHttpBucketStream) from riak import RiakError @@ -192,6 +193,22 @@ def get_buckets(self): else: raise Exception('Error getting buckets.') + def stream_buckets(self): + """ + Stream list of buckets through an iterator + """ + if not self.bucket_stream(): + raise NotImplementedError('Streaming list-buckets is not ' + 'supported') + + url = self.bucket_list_path(buckets="stream") + status, headers, response = self._request('GET', url, stream=True) + + if status == 200: + return RiakHttpBucketStream(response) + else: + raise Exception('Error listing buckets.') + def get_bucket_props(self, bucket): """ Get properties for a bucket diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index 02c69918..b292f276 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -20,7 +20,8 @@ import json from riak.transports.pbc.messages import ( MSG_CODE_LIST_KEYS_RESP, - MSG_CODE_MAPRED_RESP + MSG_CODE_MAPRED_RESP, + MSG_CODE_LIST_BUCKETS_RESP ) @@ -96,3 +97,19 @@ def next(self): raise StopIteration return response.phase, json.loads(response.response) + + +class RiakPbcBucketStream(RiakPbcStream): + """ + Used internally by RiakPbcTransport to implement key-list streams. + """ + + _expect = MSG_CODE_LIST_BUCKETS_RESP + + def next(self): + response = super(RiakPbcBucketStream, self).next() + + if response.done and len(response.buckets) is 0: + raise StopIteration + + return response.buckets diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 0b62689a..c4ded703 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -24,7 +24,7 @@ from riak.transports.transport import RiakTransport from riak.riak_object import VClock from connection import RiakPbcConnection -from stream import RiakPbcKeyStream, RiakPbcMapredStream +from stream import RiakPbcKeyStream, RiakPbcMapredStream, RiakPbcBucketStream from codec import RiakPbcCodec from messages import ( @@ -259,6 +259,22 @@ def get_buckets(self): expect=MSG_CODE_LIST_BUCKETS_RESP) return resp.buckets + def stream_buckets(self): + """ + Stream list of buckets through an iterator + """ + + if not self.bucket_stream(): + raise NotImplementedError('Streaming list-buckets is not ' + 'supported') + + req = riak_pb.RpbListBucketsReq() + req.stream = True + + self._send_msg(MSG_CODE_LIST_BUCKETS_REQ, req) + + return RiakPbcBucketStream(self) + def get_bucket_props(self, bucket): """ Serialize bucket property request and deserialize response From 156d1e5445afcec1b6fcf497e5ade739289f9e4a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 15:50:25 -0500 Subject: [PATCH 0445/1060] Modify surface API for timeouts and update a bunch of docstrings. --- riak/bucket.py | 12 ++++++---- riak/client/operations.py | 45 ++++++++++++++++++++++++------------ riak/riak_object.py | 21 ++++++++++++----- riak/transports/transport.py | 37 ++++++++++++----------------- 4 files changed, 68 insertions(+), 47 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 966ebe68..d8ff0832 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -179,7 +179,7 @@ def new_binary(self, key=None, data=None, 'param instead of data') return self.new(key, encoded_data=data, content_type=content_type) - def get(self, key, r=None, pr=None): + def get(self, key, r=None, pr=None, timeout=None): """ Retrieve an object from Riak. @@ -189,12 +189,14 @@ def get(self, key, r=None, pr=None): :type r: integer :param pr: PR-Value of the request (defaults to bucket's PR) :type pr: integer + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: :class:`RiakObject ` """ obj = RiakObject(self._client, self, key) - return obj.reload(r=r, pr=pr) + return obj.reload(r=r, pr=pr, timeout=timeout) - def get_binary(self, key, r=None, pr=None): + def get_binary(self, key, r=None, pr=None, timeout=None): """ Retrieve a binary/string object from Riak. DEPRECATED @@ -204,11 +206,13 @@ def get_binary(self, key, r=None, pr=None): :type r: integer :param pr: PR-Value of the request (defaults to bucket's PR) :type pr: integer + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: :class:`RiakObject ` """ deprecated('RiakBucket.get_binary is deprecated, ' 'use RiakBucket.get') - return self.get(key, r=r, pr=pr) + return self.get(key, r=r, pr=pr, timeout=timeout) def _get_resolver(self): if callable(self._resolver): diff --git a/riak/client/operations.py b/riak/client/operations.py index b6b8e877..09295422 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -24,33 +24,37 @@ class RiakClientOperations(RiakClientTransport): Methods for RiakClient that result in requests sent to the Riak cluster. - Note that all of these methods have an implicit 'transport' + Note that many of these methods have an implicit 'transport' argument that will be prepended automatically as part of the retry logic, and does not need to be supplied by the user. """ @retryable - def get_buckets(self, transport): + def get_buckets(self, transport, timeout=None): """ Get the list of buckets as RiakBucket instances. NOTE: Do not use this in production, as it requires traversing through all keys stored in a cluster. + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype list of RiakBucket instances """ - return [self.bucket(name) for name in transport.get_buckets()] + return [self.bucket(name) for name in transport.get_buckets(timeout=timeout)] - def stream_buckets(self): + def stream_buckets(self, timeout=None): """ Streams the list of buckets. This is a generator method that should be iterated over. NOTE: Do not use this in production, as it requires traversing through all keys stored in a cluster. + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype iterator """ with self._transport() as transport: - stream = transport.stream_buckets() + stream = transport.stream_buckets(timeout=timeout) try: for bucket_list in stream: bucket_list = [self.bucket(name) for name in bucket_list] @@ -121,17 +125,19 @@ def clear_bucket_props(self, transport, bucket): return transport.clear_bucket_props(bucket) @retryable - def get_keys(self, transport, bucket): + def get_keys(self, transport, bucket, timeout=None): """ Lists all keys in a bucket. :param bucket: the bucket whose properties will be set :type bucket: RiakBucket + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: list """ - return transport.get_keys(bucket) + return transport.get_keys(bucket, timeout=timeout) - def stream_keys(self, bucket): + def stream_keys(self, bucket, timeout=None): """ Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. @@ -139,10 +145,12 @@ def stream_keys(self, bucket): :param bucket: the bucket whose properties will be set :type bucket: RiakBucket + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: iterator """ with self._transport() as transport: - stream = transport.stream_keys(bucket) + stream = transport.stream_keys(bucket, timeout=timeout) try: for keylist in stream: if len(keylist) > 0: @@ -152,7 +160,7 @@ def stream_keys(self, bucket): @retryable def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, - if_none_match=None): + if_none_match=None, timeout=None): """ Stores an object in the Riak cluster. @@ -170,13 +178,16 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, :param if_none_match: whether to fail the write if the object exists :type if_none_match: boolean + :param timeout: a timeout value in milliseconds + :type timeout: int """ return transport.put(robj, w=w, dw=dw, pw=pw, return_body=return_body, - if_none_match=if_none_match) + if_none_match=if_none_match, + timeout=timeout) @retryable - def get(self, transport, robj, r=None, pr=None): + def get(self, transport, robj, r=None, pr=None, timeout=None): """ Fetches the contents of a Riak object. @@ -186,16 +197,18 @@ def get(self, transport, robj, r=None, pr=None): :type r: integer, string, None :param pr: the primary read quorum :type pr: integer, string, None + :param timeout: a timeout value in milliseconds + :type timeout: int """ if not isinstance(robj.key, basestring): raise TypeError( 'key must be a string, instead got {0}'.format(repr(robj.key))) - return transport.get(robj, r=r, pr=pr) + return transport.get(robj, r=r, pr=pr, timeout=timeout) @retryable def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, - pr=None, pw=None): + pr=None, pw=None, timeout=None): """ Deletes an object from Riak. @@ -213,9 +226,11 @@ def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, :type dw: integer, string, None :param pw: the primary write quorum :type pw: integer, string, None + :param timeout: a timeout value in milliseconds + :type timeout: int """ return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, - pw=pw) + pw=pw, timeout=timeout) @retryable def mapred(self, transport, inputs, query, timeout): diff --git a/riak/riak_object.py b/riak/riak_object.py index f6a4aa8f..b74d61c7 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -242,7 +242,7 @@ def get_sibling(self, index): return self.siblings[index] def store(self, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): + if_none_match=False, timeout=None): """ Store the object in Riak. When this operation completes, the object could contain new metadata and possibly new data if Riak @@ -265,6 +265,8 @@ def store(self, w=None, dw=None, pw=None, return_body=True, :param if_none_match: Should the object be stored only if there is no key previously defined :type if_none_match: bool + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: RiakObject """ if len(self.siblings) != 1: raise ConflictError("Attempting to store an invalid object, " @@ -272,11 +274,12 @@ def store(self, w=None, dw=None, pw=None, return_body=True, self.client.put(self, w=w, dw=dw, pw=pw, return_body=return_body, - if_none_match=if_none_match) + if_none_match=if_none_match, + timeout=timeout) return self - def reload(self, r=None, pr=None): + def reload(self, r=None, pr=None, timeout=None): """ Reload the object from Riak. When this operation completes, the object could contain new metadata and a new value, if the object @@ -289,13 +292,16 @@ def reload(self, r=None, pr=None): be available before performing the read that precedes the put :type pr: integer + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: RiakObject """ - self.client.get(self, r=r, pr=pr) + self.client.get(self, r=r, pr=pr, timeout=timeout) return self - def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None, + timeout=None): """ Delete this object from Riak. @@ -319,10 +325,13 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): :param pw: PW-value, require this many primary partitions to be available before performing the put :type pw: integer + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: RiakObject """ - self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) + self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw, + timeout=timeout) self.clear() return self diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 11185791..e022d02a 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -61,39 +61,37 @@ def make_fixed_client_id(self): def ping(self): """ Ping the remote server - @return boolean """ raise NotImplementedError - def get(self, robj, r=None): + def get(self, robj, r=None, pr=None, timeout=None): """ - Serialize get request and deserialize response - @return (vclock=None, [(metadata, value)]=None) + Fetches an object. """ raise NotImplementedError - def put(self, robj, w=None, dw=None, return_body=True): + def put(self, robj, w=None, dw=None, pw=None, return_body=None, + if_none_match=None, timeout=None): """ - Serialize put request and deserialize response - if 'content' - is true, retrieve the updated metadata/content - @return (vclock=None, [(metadata, value)]=None) + Stores an object. """ raise NotImplementedError - def delete(self, robj, rw=None): + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, + pw=None, timeout=None): """ - Serialize delete request and deserialize response + Deletes an object. @return true """ raise NotImplementedError - def get_buckets(self): + def get_buckets(self, timeout=None): """ - Serialize get buckets request and deserialize response + Gets the list of buckets as strings. """ raise NotImplementedError - def stream_buckets(self): + def stream_buckets(self, timeout=None): """ Streams the list of buckets through an iterator """ @@ -101,34 +99,29 @@ def stream_buckets(self): def get_bucket_props(self, bucket): """ - Serialize get bucket property request and deserialize response - @return dict() + Fetches properties for the given bucket. """ raise NotImplementedError def set_bucket_props(self, bucket, props): """ - Serialize set bucket property request and deserialize response - bucket = bucket object - props = dictionary of properties - @return boolean + Sets properties on the given bucket. """ raise NotImplementedError def clear_bucket_props(self, bucket): """ Reset bucket properties to their defaults - bucket = bucket object """ raise NotImplementedError - def get_keys(self, bucket): + def get_keys(self, bucket, timeout=None): """ Lists all keys within the given bucket. """ raise NotImplementedError - def stream_keys(self, bucket): + def stream_keys(self, bucket, timeout=None): """ Streams the list of keys for the bucket through an iterator. """ From a501cd287171f63730a7f5fd9b6317e9487c39eb Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 16:49:13 -0500 Subject: [PATCH 0446/1060] Add assertions for counters feature detection. --- riak/tests/test_feature_detection.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py index 4048c0d5..fad2372e 100644 --- a/riak/tests/test_feature_detection.py +++ b/riak/tests/test_feature_detection.py @@ -56,6 +56,7 @@ def test_pre_10(self): self.assertFalse(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) def test_10(self): t = DummyTransport("1.0.3") @@ -68,6 +69,7 @@ def test_10(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) def test_11(self): t = DummyTransport("1.1.4") @@ -80,6 +82,7 @@ def test_11(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) def test_12(self): t = DummyTransport("1.2.0") @@ -92,6 +95,7 @@ def test_12(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) def test_12_loose(self): t = DummyTransport("1.2.1p3") @@ -104,6 +108,7 @@ def test_12_loose(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) def test_14(self): t = DummyTransport("1.4.0rc1") @@ -116,6 +121,7 @@ def test_14(self): self.assertTrue(t.pb_head()) self.assertTrue(t.pb_clear_bucket_props()) self.assertTrue(t.pb_all_bucket_props()) + self.assertTrue(t.counters()) if __name__ == '__main__': unittest.main() From 8249d288d908e1f4dc86527641b641055c51fb5b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 19:01:32 -0500 Subject: [PATCH 0447/1060] Validate timeout parameters. --- riak/client/operations.py | 22 +++++++++++++++++++++- riak/tests/test_all.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 09295422..638f7243 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -40,7 +40,9 @@ def get_buckets(self, transport, timeout=None): :type timeout: int :rtype list of RiakBucket instances """ - return [self.bucket(name) for name in transport.get_buckets(timeout=timeout)] + _validate_timeout(timeout) + return [self.bucket(name) for name in + transport.get_buckets(timeout=timeout)] def stream_buckets(self, timeout=None): """ @@ -53,6 +55,7 @@ def stream_buckets(self, timeout=None): :type timeout: int :rtype iterator """ + _validate_timeout(timeout) with self._transport() as transport: stream = transport.stream_buckets(timeout=timeout) try: @@ -135,6 +138,7 @@ def get_keys(self, transport, bucket, timeout=None): :type timeout: int :rtype: list """ + _validate_timeout(timeout) return transport.get_keys(bucket, timeout=timeout) def stream_keys(self, bucket, timeout=None): @@ -149,6 +153,7 @@ def stream_keys(self, bucket, timeout=None): :type timeout: int :rtype: iterator """ + _validate_timeout(timeout) with self._transport() as transport: stream = transport.stream_keys(bucket, timeout=timeout) try: @@ -181,6 +186,7 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, :param timeout: a timeout value in milliseconds :type timeout: int """ + _validate_timeout(timeout) return transport.put(robj, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match, @@ -200,6 +206,7 @@ def get(self, transport, robj, r=None, pr=None, timeout=None): :param timeout: a timeout value in milliseconds :type timeout: int """ + _validate_timeout(timeout) if not isinstance(robj.key, basestring): raise TypeError( 'key must be a string, instead got {0}'.format(repr(robj.key))) @@ -229,6 +236,7 @@ def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, :param timeout: a timeout value in milliseconds :type timeout: int """ + _validate_timeout(timeout) return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw, timeout=timeout) @@ -245,6 +253,7 @@ def mapred(self, transport, inputs, query, timeout): :type timeout: integer, None :rtype: mixed """ + _validate_timeout(timeout) return transport.mapred(inputs, query, timeout) def stream_mapred(self, inputs, query, timeout): @@ -260,6 +269,7 @@ def stream_mapred(self, inputs, query, timeout): :type timeout: integer, None :rtype: iterator """ + _validate_timeout(timeout) with self._transport() as transport: stream = transport.stream_mapred(inputs, query, timeout) try: @@ -307,3 +317,13 @@ def fulltext_delete(self, transport, index, docs=None, queries=None): :type queries: list """ transport.fulltext_delete(index, docs, queries) + + +def _validate_timeout(timeout): + """ + Raises an exception if the given timeout is an invalid value. + """ + if not (timeout is None or + (type(timeout) in (int, long) and + timeout > 0)): + raise ValueError("timeout must be a positive integer") diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 592ecdf1..89172760 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -128,6 +128,40 @@ def test_request_retries(self): # error. self.assertRaises(IOError, client.ping) + def test_timeout_validation(self): + bucket = self.client.bucket(self.bucket_name) + key = self.key_name + obj = bucket.new(key) + for bad in [0, -1, False, "foo"]: + with self.assertRaises(ValueError): + self.client.get_buckets(timeout=bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_buckets(timeout=bad): + pass + + with self.assertRaises(ValueError): + self.client.get_keys(bucket, timeout=bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_keys(bucket, timeout=bad): + pass + + with self.assertRaises(ValueError): + self.client.put(obj, timeout=bad) + + with self.assertRaises(ValueError): + self.client.get(obj, timeout=bad) + + with self.assertRaises(ValueError): + self.client.delete(obj, timeout=bad) + + with self.assertRaises(ValueError): + self.client.mapred([], [], bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_mapred([], [], bad): + pass class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, From f110966c4175e4a7a30bccc4e35408403fa9c6b5 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 19:02:17 -0500 Subject: [PATCH 0448/1060] Add timeouts to the backend and test streaming buckets. --- riak/tests/test_all.py | 1 + riak/tests/test_kv.py | 24 +++++++++++++++++++ riak/transports/feature_detect.py | 7 ++++++ riak/transports/http/resources.py | 2 +- riak/transports/http/transport.py | 31 +++++++++++++----------- riak/transports/pbc/messages.py | 2 +- riak/transports/pbc/transport.py | 40 ++++++++++++++++++++++--------- 7 files changed, 80 insertions(+), 27 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 89172760..c484e158 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -163,6 +163,7 @@ def test_timeout_validation(self): for i in self.client.stream_mapred([], [], bad): pass + class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, BucketPropsTest, diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index b78046e9..de0ea790 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -423,6 +423,30 @@ def test_list_buckets(self): buckets = self.client.get_buckets() self.assertTrue(self.bucket_name in [x.name for x in buckets]) + def test_stream_buckets(self): + bucket = self.client.bucket(self.bucket_name) + bucket.new(self.key_name, data={"foo": "one", + "bar": "baz"}).store() + buckets = [] + for bucket_list in self.client.stream_buckets(): + buckets.extend(bucket_list) + + self.assertTrue(self.bucket_name in [x.name for x in buckets]) + + def test_stream_buckets_abort(self): + bucket = self.client.bucket(self.bucket_name) + bucket.new(self.key_name, data={"foo": "one", + "bar": "baz"}).store() + try: + for bucket_list in self.client.stream_buckets(): + raise RuntimeError("abort") + except RuntimeError: + pass + + robj = bucket.get(self.key_name) + self.assertTrue(robj.exists) + self.assertEqual(len(robj.siblings), 1) + def generate_siblings(self, original, count=5, delay=None): vals = [] for i in range(count): diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 5dac0180..2ffd2652 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -114,6 +114,13 @@ def bucket_stream(self): """ return self.server_version >= versions[1.4] + def client_timeouts(self): + """ + Whether client-supplied timeouts are supported. + :rtype bool + """ + return self.server_version >= versions[1.4] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index 92c52032..28ac59f2 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -51,7 +51,7 @@ def bucket_properties_path(self, bucket, **options): "props", **options) else: query = options.copy() - query.update(props=True, keys=True) + query.update(props=True, keys=False) return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), **query) def key_list_path(self, bucket, **options): diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 91fd910d..43300fb5 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -106,25 +106,26 @@ def get_resources(self): else: return {} - def get(self, robj, r=None, pr=None): + def get(self, robj, r=None, pr=None, timeout=None): """ Get a bucket/key from the server """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'r': r, 'pr': pr} + params = {'r': r, 'pr': pr, 'timeout': timeout} url = self.object_path(robj.bucket.name, robj.key, **params) response = self._request('GET', url) return self._parse_body(robj, response, [200, 300, 404]) def put(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): + if_none_match=False, timeout=None): """ Puts a (possibly new) object. """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} + params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw, + 'timeout': timeout} url = self.object_path(robj.bucket.name, robj.key, **params) headers = self._build_put_headers(robj, if_none_match=if_none_match) content = bytearray(robj.encoded_data) @@ -143,13 +144,15 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, self.check_http_code(response[0], expect) return None - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None, + timeout=None): """ Delete an object. """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} + params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw, + 'timeout': timeout} headers = {} url = self.object_path(robj.bucket.name, robj.key, **params) if self.tombstone_vclocks() and robj.vclock is not None: @@ -158,11 +161,11 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): self.check_http_code(response[0], [204, 404]) return self - def get_keys(self, bucket): + def get_keys(self, bucket, timeout=None): """ Fetch a list of keys for the bucket """ - url = self.key_list_path(bucket.name) + url = self.key_list_path(bucket.name, timeout=timeout) status, _, body = self._request('GET', url) if status == 200: @@ -171,8 +174,8 @@ def get_keys(self, bucket): else: raise Exception('Error listing keys.') - def stream_keys(self, bucket): - url = self.key_list_path(bucket.name, keys='stream') + def stream_keys(self, bucket, timeout=None): + url = self.key_list_path(bucket.name, keys='stream', timeout=timeout) status, headers, response = self._request('GET', url, stream=True) if status == 200: @@ -180,11 +183,11 @@ def stream_keys(self, bucket): else: raise Exception('Error listing keys.') - def get_buckets(self): + def get_buckets(self, timeout=None): """ Fetch a list of all buckets """ - url = self.bucket_list_path() + url = self.bucket_list_path(timeout=timeout) status, headers, body = self._request('GET', url) if status == 200: @@ -193,7 +196,7 @@ def get_buckets(self): else: raise Exception('Error getting buckets.') - def stream_buckets(self): + def stream_buckets(self, timeout=None): """ Stream list of buckets through an iterator """ @@ -201,7 +204,7 @@ def stream_buckets(self): raise NotImplementedError('Streaming list-buckets is not ' 'supported') - url = self.bucket_list_path(buckets="stream") + url = self.bucket_list_path(buckets="stream", timeout=timeout) status, headers, response = self._request('GET', url, stream=True) if status == 200: diff --git a/riak/transports/pbc/messages.py b/riak/transports/pbc/messages.py index d4b867c8..58a2adb4 100644 --- a/riak/transports/pbc/messages.py +++ b/riak/transports/pbc/messages.py @@ -78,7 +78,7 @@ MSG_CODE_PUT_RESP: riak_pb.RpbPutResp, MSG_CODE_DEL_REQ: riak_pb.RpbDelReq, MSG_CODE_DEL_RESP: None, - MSG_CODE_LIST_BUCKETS_REQ: None, + MSG_CODE_LIST_BUCKETS_REQ: riak_pb.RpbListBucketsReq, MSG_CODE_LIST_BUCKETS_RESP: riak_pb.RpbListBucketsResp, MSG_CODE_LIST_KEYS_REQ: riak_pb.RpbListKeysReq, MSG_CODE_LIST_KEYS_RESP: riak_pb.RpbListKeysResp, diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index c4ded703..25bb70da 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -117,7 +117,7 @@ def _set_client_id(self, client_id): client_id = property(_get_client_id, _set_client_id, doc="""the client ID for this connection""") - def get(self, robj, r=None, pr=None): + def get(self, robj, r=None, pr=None, timeout=None): """ Serialize get request and deserialize response """ @@ -128,7 +128,8 @@ def get(self, robj, r=None, pr=None): req.r = self._encode_quorum(r) if self.quorum_controls() and pr: req.pr = self._encode_quorum(pr) - + if self.client_timeouts() and timeout: + req.timeout = timeout if self.tombstone_vclocks(): req.deletedvclock = 1 @@ -154,7 +155,7 @@ def get(self, robj, r=None, pr=None): return robj def put(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): + if_none_match=False, timeout=None): """ Serialize get request and deserialize response """ @@ -172,6 +173,8 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, req.return_body = 1 if if_none_match: req.if_none_match = 1 + if self.client_timeouts() and timeout: + req.timeout = timeout req.bucket = bucket.name if robj.key: @@ -196,7 +199,8 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, return robj - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None, + timeout=None): """ Serialize get request and deserialize response """ @@ -218,6 +222,9 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): if pw: req.pw = self._encode_quorum(pw) + if self.client_timeouts() and timeout: + req.timeout = timeout + if self.tombstone_vclocks() and robj.vclock: req.vclock = robj.vclock.encode('binary') @@ -228,38 +235,45 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): MSG_CODE_DEL_RESP) return self - def get_keys(self, bucket): + def get_keys(self, bucket, timeout=None): """ Lists all keys within a bucket. """ keys = [] - for keylist in self.stream_keys(bucket): + for keylist in self.stream_keys(bucket, timeout=timeout): for key in keylist: keys.append(key) return keys - def stream_keys(self, bucket): + def stream_keys(self, bucket, timeout=None): """ Streams keys from a bucket, returning an iterator that yields lists of keys. """ req = riak_pb.RpbListKeysReq() req.bucket = bucket.name + if self.client_timeouts() and timeout: + req.timeout = timeout self._send_msg(MSG_CODE_LIST_KEYS_REQ, req) return RiakPbcKeyStream(self) - def get_buckets(self): + def get_buckets(self, timeout=None): """ Serialize bucket listing request and deserialize response """ - msg_code, resp = self._request(MSG_CODE_LIST_BUCKETS_REQ, - expect=MSG_CODE_LIST_BUCKETS_RESP) + req = None + if self.client_timeouts() and timeout: + req = riak_pb.RpbListBucketsReq() + req.timeout = timeout + + msg_code, resp = self._request(MSG_CODE_LIST_BUCKETS_REQ, req, + MSG_CODE_LIST_BUCKETS_RESP) return resp.buckets - def stream_buckets(self): + def stream_buckets(self, timeout=None): """ Stream list of buckets through an iterator """ @@ -270,6 +284,10 @@ def stream_buckets(self): req = riak_pb.RpbListBucketsReq() req.stream = True + # Bucket streaming landed in the same release as timeouts, so + # we don't need to check the capability. + if timeout: + req.timeout = timeout self._send_msg(MSG_CODE_LIST_BUCKETS_REQ, req) From 3c5430bf393d6dd07caac11b36784f492c8ab978 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 19:39:50 -0500 Subject: [PATCH 0449/1060] Add failing test for streaming 2i. --- riak/tests/test_2i.py | 50 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index d303eb4b..60a97fb6 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -15,7 +15,7 @@ class TwoITests(object): def is_2i_supported(self): # Immediate test to see if 2i is even supported w/ the backend try: - self.client.index('foo', 'bar_bin', 'baz').run() + self.client.get_index('foo', 'bar_bin', 'baz') return True except Exception as e: if "indexes_not_supported" in str(e): @@ -25,7 +25,7 @@ def is_2i_supported(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_store(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") # Create a new object with indexes... bucket = self.client.bucket(self.bucket_name) @@ -106,7 +106,7 @@ def test_secondary_index_store(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_set_indexes(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) foo = bucket.new('foo', 1) @@ -124,7 +124,7 @@ def test_set_indexes(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_remove_indexes(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) bar = bucket.new('bar', 1).add_index('bar_int', 1)\ @@ -184,7 +184,7 @@ def test_remove_indexes(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_query(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) @@ -240,7 +240,7 @@ def test_secondary_index_query(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_invalid_name(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) @@ -250,7 +250,7 @@ def test_secondary_index_invalid_name(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') def test_set_index(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) obj = bucket.new('bar', 1) @@ -264,3 +264,39 @@ def test_set_index(self): self.assertEqual(set((('bar_int', 3), ('bar2_int', 1))), obj.indexes) obj.set_index('bar2_int', 10) self.assertEqual(set((('bar_int', 3), ('bar2_int', 10))), obj.indexes) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_stream_index(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I not supported") + + bucket = self.client.bucket(self.bucket_name) + + o1 = bucket.\ + new(self.key_name, 'data1').\ + add_index('field1_bin', 'val1').\ + add_index('field2_int', 1001).\ + store() + o2 = bucket.\ + new(self.randname(), 'data1').\ + add_index('field1_bin', 'val2').\ + add_index('field2_int', 1002).\ + store() + o3 = bucket.\ + new(self.randname(), 'data1').\ + add_index('field1_bin', 'val3').\ + add_index('field2_int', 1003).\ + store() + o4 = bucket.\ + new(self.randname(), 'data1').\ + add_index('field1_bin', 'val4').\ + add_index('field2_int', 1004).\ + store() + + keys = [] + for entries in self.client.stream_index(bucket, 'field1_bin', + 'val1', 'val3'): + keys.append(entries) + + # Riak 1.4 ensures that entries come back in-order + self.assertEqual([o1.key, o2.key, o3.key], keys) From 46740ce5ee863bc9a84c9fd1ff9254d3e5ffe691 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 20:08:30 -0500 Subject: [PATCH 0450/1060] Add front-end to stream_index operation. --- riak/bucket.py | 7 +++++++ riak/client/operations.py | 23 +++++++++++++++++++++++ riak/tests/test_feature_detection.py | 6 ++++++ riak/transports/feature_detect.py | 7 +++++++ riak/transports/transport.py | 6 ++++++ 5 files changed, 49 insertions(+) diff --git a/riak/bucket.py b/riak/bucket.py index 966ebe68..4c54c6f0 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -411,6 +411,13 @@ def get_index(self, index, startkey, endkey=None): """ return self._client.get_index(self.name, index, startkey, endkey) + def stream_index(self, index, startkey, endkey=None): + """ + Queries a secondary index over objects in this bucket, + streaming keys via an iterator. + """ + return self._client.stream_index(self.name, index, startkey, endkey) + def delete(self, key, **kwargs): """Deletes an object from riak. diff --git a/riak/client/operations.py b/riak/client/operations.py index c928128f..d6004318 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -66,6 +66,29 @@ def get_index(self, transport, bucket, index, startkey, endkey=None): """ return transport.get_index(bucket, index, startkey, endkey) + def stream_index(self, bucket, index, startkey, endkey=None): + """ + Queries a secondary index, streaming matching keys through an + iterator. + + :param bucket: the bucket whose index will be queried + :type bucket: RiakBucket + :param index: the index to query + :type index: string + :param startkey: the sole key to query, or beginning of the query range + :type startkey: string, integer + :param endkey: the end of the query range (optional if equality) + :type endkey: string, integer + :rtype: iterable + """ + with self._transport() as transport: + stream = transport.stream_index(bucket, index, startkey, endkey) + try: + for item in stream: + yield item + finally: + stream.close() + @retryable def get_bucket_props(self, transport, bucket): """ diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py index 4048c0d5..a8f0fed2 100644 --- a/riak/tests/test_feature_detection.py +++ b/riak/tests/test_feature_detection.py @@ -56,6 +56,7 @@ def test_pre_10(self): self.assertFalse(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.stream_indexes()) def test_10(self): t = DummyTransport("1.0.3") @@ -68,6 +69,7 @@ def test_10(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.stream_indexes()) def test_11(self): t = DummyTransport("1.1.4") @@ -80,6 +82,7 @@ def test_11(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.stream_indexes()) def test_12(self): t = DummyTransport("1.2.0") @@ -92,6 +95,7 @@ def test_12(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.stream_indexes()) def test_12_loose(self): t = DummyTransport("1.2.1p3") @@ -104,6 +108,7 @@ def test_12_loose(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.stream_indexes()) def test_14(self): t = DummyTransport("1.4.0rc1") @@ -116,6 +121,7 @@ def test_14(self): self.assertTrue(t.pb_head()) self.assertTrue(t.pb_clear_bucket_props()) self.assertTrue(t.pb_all_bucket_props()) + self.assertTrue(t.stream_indexes()) if __name__ == '__main__': unittest.main() diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 3712ec8b..220b6446 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -107,6 +107,13 @@ def pb_all_bucket_props(self): """ return self.server_version >= versions[1.4] + def stream_indexes(self): + """ + Whether secondary indexes support streaming responses. + :rtype bool + """ + return self.server_version >= versions[1.4] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index c6d581c7..d373152e 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -167,6 +167,12 @@ def get_index(self, bucket, index, startkey, endkey=None): """ raise NotImplementedError + def stream_index(self, bucket, index, startkey, endkey=None): + """ + Streams a secondary index query. + """ + raise NotImplementedError + def fulltext_add(self, index, *docs): """ Adds documents to the full-text index. From e48ab19b1852258aa3ee37e55cfcc711c2238455 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 21:10:59 -0500 Subject: [PATCH 0451/1060] Fix bug in test. --- riak/tests/test_2i.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 60a97fb6..1ea5cc4b 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -294,9 +294,8 @@ def test_stream_index(self): store() keys = [] - for entries in self.client.stream_index(bucket, 'field1_bin', - 'val1', 'val3'): - keys.append(entries) + for entries in bucket.stream_index('field1_bin', 'val1', 'val3'): + keys.extend(entries) # Riak 1.4 ensures that entries come back in-order self.assertEqual([o1.key, o2.key, o3.key], keys) From a10fd6b607936ac8b43bbf67e1cc2c5cb9fffc2e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 21:12:25 -0500 Subject: [PATCH 0452/1060] Implement basic streaming of secondary index responses. There are edge cases around the response format in the stream iterator classes that I hope to address in future commits. They are helpfully marked by "WAT". --- riak/transports/http/stream.py | 20 ++++++++++++++++++++ riak/transports/http/transport.py | 19 ++++++++++++++++++- riak/transports/pbc/codec.py | 23 +++++++++++++++++++++++ riak/transports/pbc/stream.py | 26 +++++++++++++++++++++++++- riak/transports/pbc/transport.py | 22 +++++++++++++--------- 5 files changed, 99 insertions(+), 11 deletions(-) diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 2f1620f9..668413c9 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -123,3 +123,23 @@ def next(self): message = super(RiakHttpMapReduceStream, self).next() payload = json.loads(message.get_payload()) return payload['phase'], payload['data'] + + +class RiakHttpIndexStream(RiakHttpMultipartStream): + """ + Streaming iterator for secondary indexes over HTTP + """ + + def next(self): + message = super(RiakHttpIndexStream, self).next() + payload = json.loads(message.get_payload()) + if u'keys' in payload: + return payload[u'keys'] + elif u'results' in payload: + structs = payload[u'results'] + # Format is {"results":[{"2ikey":"primarykey"}, ...]} + munged = [ d.items()[0] for d in structs ] + return munged + else: + # WAT + self.next() diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 632f37bf..ac19741c 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -33,7 +33,8 @@ from riak.transports.http.codec import RiakHttpCodec from riak.transports.http.stream import ( RiakHttpKeyStream, - RiakHttpMapReduceStream) + RiakHttpMapReduceStream, + RiakHttpIndexStream) from riak import RiakError @@ -285,6 +286,22 @@ def get_index(self, bucket, index, startkey, endkey=None): json_data = json.loads(body) return json_data[u'keys'][:] + def stream_index(self, bucket, index, startkey, endkey=None): + """ + Streams a secondary index query. + """ + if not self.stream_indexes(): + raise NotImplementedError("Secondary index streaming is not " + "supported") + + url = self.index_path(bucket, index, startkey, endkey, stream=True) + status, headers, response = self._request('GET', url, stream=True) + + if status == 200: + return RiakHttpIndexStream(response) + else: + raise Exception('Error streaming secondary index.') + def search(self, index, query, **params): """ Performs a search query. diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 250995aa..a2f3c916 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -373,3 +373,26 @@ def _encode_hook(self, hook, msg): else: self._encode_modfun(hook, msg.modfun) return msg + + def _encode_index_req(self, bucket, index, startkey, endkey=None): + """ + Encodes a secondary index request into the protobuf message. + + :param bucket: the bucket whose index to query + :type bucket: string + :param index: the index to query + :type index: string + :param startkey: the value or beginning of the range + :type startkey: integer, string + :param endkey: the end of the range + :type endkey: integer, string + """ + req = riak_pb.RpbIndexReq(bucket=bucket, index=index) + if endkey: + req.qtype = riak_pb.RpbIndexReq.range + req.range_min = str(startkey) + req.range_max = str(endkey) + else: + req.qtype = riak_pb.RpbIndexReq.eq + req.key = str(startkey) + return req diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index 02c69918..3224ee41 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -20,7 +20,8 @@ import json from riak.transports.pbc.messages import ( MSG_CODE_LIST_KEYS_RESP, - MSG_CODE_MAPRED_RESP + MSG_CODE_MAPRED_RESP, + MSG_CODE_INDEX_RESP ) @@ -96,3 +97,26 @@ def next(self): raise StopIteration return response.phase, json.loads(response.response) + + +class RiakPbcIndexStream(RiakPbcStream): + """ + Used internally by RiakPbcTransport to implement Secondary Index + streams. + """ + + _expect = MSG_CODE_INDEX_RESP + + def next(self): + response = super(RiakPbcIndexStream, self).next() + + if response.done and not (response.keys or response.results): + raise StopIteration + + if response.keys: + return response.keys + elif response.results: + return [(r.key, r.value) for r in response.results] + else: + # WAT + return self.next() diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 0b62689a..52fb27dd 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -24,7 +24,7 @@ from riak.transports.transport import RiakTransport from riak.riak_object import VClock from connection import RiakPbcConnection -from stream import RiakPbcKeyStream, RiakPbcMapredStream +from stream import RiakPbcKeyStream, RiakPbcMapredStream, RiakPbcIndexStream from codec import RiakPbcCodec from messages import ( @@ -338,19 +338,23 @@ def get_index(self, bucket, index, startkey, endkey=None): if not self.pb_indexes(): return self._get_index_mapred_emu(bucket, index, startkey, endkey) - req = riak_pb.RpbIndexReq(bucket=bucket, index=index) - if endkey: - req.qtype = riak_pb.RpbIndexReq.range - req.range_min = str(startkey) - req.range_max = str(endkey) - else: - req.qtype = riak_pb.RpbIndexReq.eq - req.key = str(startkey) + req = self._encode_index_req(bucket, index, startkey, endkey) msg_code, resp = self._request(MSG_CODE_INDEX_REQ, req, MSG_CODE_INDEX_RESP) return resp.keys + def stream_index(self, bucket, index, startkey, endkey=None): + if not self.stream_indexes(): + raise NotImplementedError("Secondary index streaming is not " + "supported") + req = self._encode_index_req(bucket, index, startkey, endkey) + req.stream = True + + self._send_msg(MSG_CODE_INDEX_REQ, req) + + return RiakPbcIndexStream(self) + def search(self, index, query, **params): if not self.pb_search(): return self._search_mapred_emu(index, query) From 5fcbea162247933a4584498ffe6d569b8a732f9f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 2 Jul 2013 12:44:27 -0500 Subject: [PATCH 0453/1060] Add return_terms option. --- riak/bucket.py | 10 ++-- riak/client/operations.py | 16 ++++-- riak/tests/test_2i.py | 81 ++++++++++++++++--------------- riak/transports/http/codec.py | 4 +- riak/transports/http/stream.py | 12 ++++- riak/transports/http/transport.py | 24 ++++++--- riak/transports/pbc/codec.py | 12 +++-- riak/transports/pbc/stream.py | 18 +++++-- riak/transports/pbc/transport.py | 22 ++++++--- riak/util.py | 7 +++ 10 files changed, 136 insertions(+), 70 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 4c54c6f0..d8a03db8 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -405,18 +405,20 @@ def search(self, query, **params): """ return self._client.solr.search(self.name, query, **params) - def get_index(self, index, startkey, endkey=None): + def get_index(self, index, startkey, endkey=None, return_terms=None): """ Queries a secondary index over objects in this bucket, returning keys. """ - return self._client.get_index(self.name, index, startkey, endkey) + return self._client.get_index(self.name, index, startkey, endkey, + return_terms=return_terms) - def stream_index(self, index, startkey, endkey=None): + def stream_index(self, index, startkey, endkey=None, return_terms=None): """ Queries a secondary index over objects in this bucket, streaming keys via an iterator. """ - return self._client.stream_index(self.name, index, startkey, endkey) + return self._client.stream_index(self.name, index, startkey, endkey, + return_terms=return_terms) def delete(self, key, **kwargs): """Deletes an object from riak. diff --git a/riak/client/operations.py b/riak/client/operations.py index d6004318..6fd4956c 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -50,7 +50,8 @@ def ping(self, transport): is_alive = ping @retryable - def get_index(self, transport, bucket, index, startkey, endkey=None): + def get_index(self, transport, bucket, index, startkey, endkey=None, + return_terms=None): """ Queries a secondary index, returning matching keys. @@ -62,11 +63,15 @@ def get_index(self, transport, bucket, index, startkey, endkey=None): :type startkey: string, integer :param endkey: the end of the query range (optional if equality) :type endkey: string, integer + :param return_terms: whether to include the secondary index value + :type return_terms: boolean :rtype: list """ - return transport.get_index(bucket, index, startkey, endkey) + return transport.get_index(bucket, index, startkey, endkey, + return_terms=return_terms) - def stream_index(self, bucket, index, startkey, endkey=None): + def stream_index(self, bucket, index, startkey, endkey=None, + return_terms=None): """ Queries a secondary index, streaming matching keys through an iterator. @@ -79,10 +84,13 @@ def stream_index(self, bucket, index, startkey, endkey=None): :type startkey: string, integer :param endkey: the end of the query range (optional if equality) :type endkey: string, integer + :param return_terms: whether to include the secondary index value + :type return_terms: boolean :rtype: iterable """ with self._transport() as transport: - stream = transport.stream_index(bucket, index, startkey, endkey) + stream = transport.stream_index(bucket, index, startkey, endkey, + return_terms=return_terms) try: for item in stream: yield item diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 1ea5cc4b..996741c6 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -186,56 +186,29 @@ def test_secondary_index_query(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") - bucket = self.client.bucket(self.bucket_name) - - bucket.\ - new('mykey1', 'data1').\ - add_index('field1_bin', 'val1').\ - add_index('field2_int', 1001).\ - store() - bucket.\ - new('mykey2', 'data1').\ - add_index('field1_bin', 'val2').\ - add_index('field2_int', 1002).\ - store() - bucket.\ - new('mykey3', 'data1').\ - add_index('field1_bin', 'val3').\ - add_index('field2_int', 1003).\ - store() - bucket.\ - new('mykey4', 'data1').\ - add_index('field1_bin', 'val4').\ - add_index('field2_int', 1004).\ - store() + bucket, o1, o2, o3, o4 = self._create_index_objects() # Test an equality query... results = bucket.get_index('field1_bin', 'val2') self.assertEquals(1, len(results)) - self.assertEquals('mykey2', str(results[0])) + self.assertEquals(o2.key, str(results[0])) # Test a range query... results = bucket.get_index('field1_bin', 'val2', 'val4') vals = set([str(key) for key in results]) self.assertEquals(3, len(results)) - self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) + self.assertEquals(set([o2.key, o3.key, o4.key]), vals) # Test an equality query... results = bucket.get_index('field2_int', 1002) self.assertEquals(1, len(results)) - self.assertEquals('mykey2', str(results[0])) + self.assertEquals(o2.key, str(results[0])) # Test a range query... results = bucket.get_index('field2_int', 1002, 1004) vals = set([str(key) for key in results]) self.assertEquals(3, len(results)) - self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) - - # Clean up... - bucket.get('mykey1').delete() - bucket.get('mykey2').delete() - bucket.get('mykey3').delete() - bucket.get('mykey4').delete() + self.assertEquals(set([o2.key, o3.key, o4.key]), vals) @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_invalid_name(self): @@ -270,6 +243,43 @@ def test_stream_index(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") + bucket, o1, o2, o3, o4 = self._create_index_objects() + + keys = [] + for entries in bucket.stream_index('field1_bin', 'val1', 'val3'): + keys.extend(entries) + + # Riak 1.4 ensures that entries come back in-order + self.assertEqual([o1.key, o2.key, o3.key], keys) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # Test synchronous index query + pairs = bucket.get_index('field1_bin', 'val2', 'val4', + return_terms=True) + + self.assertEqual([('val2', o2.key), + ('val3', o3.key), + ('val4', o4.key)], pairs) + + # Test streaming index query + spairs = [] + for chunk in bucket.stream_index('field2_int', 1002, 1004, + return_terms=True): + spairs.extend(chunk) + + self.assertEqual([(1002, o2.key), (1003, o3.key), (1004, o4.key)], + spairs) + + def _create_index_objects(self): + """ + Creates a number of index objects to be used in 2i test + """ bucket = self.client.bucket(self.bucket_name) o1 = bucket.\ @@ -293,9 +303,4 @@ def test_stream_index(self): add_index('field2_int', 1004).\ store() - keys = [] - for entries in bucket.stream_index('field1_bin', 'val1', 'val3'): - keys.extend(entries) - - # Riak 1.4 ensures that entries come back in-order - self.assertEqual([o1.key, o2.key, o3.key], keys) + return bucket, o1, o2, o3, o4 diff --git a/riak/transports/http/codec.py b/riak/transports/http/codec.py index 25bb7dab..7b9e2076 100644 --- a/riak/transports/http/codec.py +++ b/riak/transports/http/codec.py @@ -35,6 +35,7 @@ from riak.riak_object import VClock from riak.multidict import MultiDict from riak.transports.http.search import XMLSearchResult +from riak.util import decode_index_value class RiakHttpCodec(object): @@ -127,8 +128,7 @@ def _parse_sibling(self, sibling, headers, data): reader = csv.reader([value], skipinitialspace=True) for line in reader: for token in line: - if field.endswith("_int"): - token = int(token) + token = decode_index_value(field, token) sibling.add_index(field, token) elif header == 'x-riak-deleted': sibling.exists = False diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 668413c9..949d64d0 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -21,6 +21,7 @@ import re from cgi import parse_header from email import message_from_string +from riak.util import decode_index_value class RiakHttpStream(object): @@ -130,6 +131,11 @@ class RiakHttpIndexStream(RiakHttpMultipartStream): Streaming iterator for secondary indexes over HTTP """ + def __init__(self, response, index, return_terms): + super(RiakHttpIndexStream, self).__init__(response) + self.index = index + self.return_terms = return_terms + def next(self): message = super(RiakHttpIndexStream, self).next() payload = json.loads(message.get_payload()) @@ -138,8 +144,10 @@ def next(self): elif u'results' in payload: structs = payload[u'results'] # Format is {"results":[{"2ikey":"primarykey"}, ...]} - munged = [ d.items()[0] for d in structs ] - return munged + return [self._decode_pair(d.items()[0]) for d in structs] else: # WAT self.next() + + def _decode_pair(self, pair): + return (decode_index_value(self.index, pair[0]), pair[1]) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index ac19741c..76cc3a96 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -36,6 +36,7 @@ RiakHttpMapReduceStream, RiakHttpIndexStream) from riak import RiakError +from riak.util import decode_index_value class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakHttpCodec, @@ -276,17 +277,27 @@ def stream_mapred(self, inputs, query, timeout=None): 'Error running MapReduce operation. Headers: %s Body: %s' % (repr(headers), repr(response.read()))) - def get_index(self, bucket, index, startkey, endkey=None): + def get_index(self, bucket, index, startkey, endkey=None, + return_terms=None): """ Performs a secondary index query. """ - url = self.index_path(bucket, index, startkey, endkey) + params = {'return_terms': return_terms} + url = self.index_path(bucket, index, startkey, endkey, **params) status, headers, body = self._request('GET', url) self.check_http_code(status, [200]) json_data = json.loads(body) - return json_data[u'keys'][:] + if return_terms: + results = [] + for result in json_data[u'results'][:]: + term, key = result.items()[0] + results.append((decode_index_value(index, term), key),) + return results + else: + return json_data[u'keys'][:] - def stream_index(self, bucket, index, startkey, endkey=None): + def stream_index(self, bucket, index, startkey, endkey=None, + return_terms=None): """ Streams a secondary index query. """ @@ -294,11 +305,12 @@ def stream_index(self, bucket, index, startkey, endkey=None): raise NotImplementedError("Secondary index streaming is not " "supported") - url = self.index_path(bucket, index, startkey, endkey, stream=True) + params = {'return_terms': return_terms, 'stream': True} + url = self.index_path(bucket, index, startkey, endkey, **params) status, headers, response = self._request('GET', url, stream=True) if status == 200: - return RiakHttpIndexStream(response) + return RiakHttpIndexStream(response, index, return_terms) else: raise Exception('Error streaming secondary index.') diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index a2f3c916..ddf61a98 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -18,6 +18,7 @@ import riak_pb from riak import RiakError from riak.content import RiakContent +from riak.util import decode_index_value def _invert(d): @@ -147,8 +148,7 @@ def _decode_content(self, rpb_content, sibling): sibling.usermeta = dict([(usermd.key, usermd.value) for usermd in rpb_content.usermeta]) sibling.indexes = set([(index.key, - self._decode_index_value(index.key, - index.value)) + decode_index_value(index.key, index.value)) for index in rpb_content.indexes]) sibling.encoded_data = rpb_content.value @@ -374,7 +374,8 @@ def _encode_hook(self, hook, msg): self._encode_modfun(hook, msg.modfun) return msg - def _encode_index_req(self, bucket, index, startkey, endkey=None): + def _encode_index_req(self, bucket, index, startkey, endkey=None, + return_terms=None): """ Encodes a secondary index request into the protobuf message. @@ -386,6 +387,9 @@ def _encode_index_req(self, bucket, index, startkey, endkey=None): :type startkey: integer, string :param endkey: the end of the range :type endkey: integer, string + :param return_terms: whether to return the index term with the key + :type return_terms: bool + :rtype riak_pb.RpbIndexReq """ req = riak_pb.RpbIndexReq(bucket=bucket, index=index) if endkey: @@ -395,4 +399,6 @@ def _encode_index_req(self, bucket, index, startkey, endkey=None): else: req.qtype = riak_pb.RpbIndexReq.eq req.key = str(startkey) + if return_terms is not None: + req.return_terms = return_terms return req diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index 3224ee41..ff8345e6 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -107,16 +107,24 @@ class RiakPbcIndexStream(RiakPbcStream): _expect = MSG_CODE_INDEX_RESP + def __init__(self, transport, index, return_terms=False): + super(RiakPbcIndexStream, self).__init__(transport) + self.index = index + self.return_terms = return_terms + def next(self): response = super(RiakPbcIndexStream, self).next() if response.done and not (response.keys or response.results): raise StopIteration - if response.keys: + if self.return_terms and response.results: + return [(self._coerce(r.key), r.value) for r in response.results] + elif response.keys: return response.keys - elif response.results: - return [(r.key, r.value) for r in response.results] + + def _coerce(self, index_value): + if "_int" in self.index: + return long(index_value) else: - # WAT - return self.next() + return str(index_value) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 52fb27dd..1ef91560 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -23,6 +23,7 @@ from riak import RiakError from riak.transports.transport import RiakTransport from riak.riak_object import VClock +from riak.util import decode_index_value from connection import RiakPbcConnection from stream import RiakPbcKeyStream, RiakPbcMapredStream, RiakPbcIndexStream from codec import RiakPbcCodec @@ -334,26 +335,35 @@ def stream_mapred(self, inputs, query, timeout=None): return RiakPbcMapredStream(self) - def get_index(self, bucket, index, startkey, endkey=None): + def get_index(self, bucket, index, startkey, endkey=None, + return_terms=None): if not self.pb_indexes(): return self._get_index_mapred_emu(bucket, index, startkey, endkey) - req = self._encode_index_req(bucket, index, startkey, endkey) + req = self._encode_index_req(bucket, index, startkey, endkey, + return_terms=return_terms) msg_code, resp = self._request(MSG_CODE_INDEX_REQ, req, MSG_CODE_INDEX_RESP) - return resp.keys + if return_terms: + return [(decode_index_value(index, pair.key), pair.value) + for pair in resp.results] + else: + return resp.keys - def stream_index(self, bucket, index, startkey, endkey=None): + def stream_index(self, bucket, index, startkey, endkey=None, + return_terms=None): if not self.stream_indexes(): raise NotImplementedError("Secondary index streaming is not " "supported") - req = self._encode_index_req(bucket, index, startkey, endkey) + + req = self._encode_index_req(bucket, index, startkey, endkey, + return_terms=return_terms) req.stream = True self._send_msg(MSG_CODE_INDEX_REQ, req) - return RiakPbcIndexStream(self) + return RiakPbcIndexStream(self, index, return_terms) def search(self, index, query, **params): if not self.pb_search(): diff --git a/riak/util.py b/riak/util.py index 448ee8d7..f096caf7 100644 --- a/riak/util.py +++ b/riak/util.py @@ -123,3 +123,10 @@ def __get__(self, obj, cls): value = self.fget(obj) setattr(obj, self.func_name, value) return value + + +def decode_index_value(index, value): + if "_int" in index: + return long(value) + else: + return str(value) From 66cef408387a247946c889dfe8249ffcd229bab9 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 3 Jul 2013 15:00:45 -0500 Subject: [PATCH 0454/1060] Add pagination to secondary indexes, commit the first. This introduces the IndexPage class, which is necessary both to wrap the results into a sensible API, and to provide consistency between single-roundtrip and streaming options. The IndexPage class can appear as if it were a list, is iterable for streaming and regular iteration purposes. Regardless of whether pagination is used, an IndexPage will be returned (thanks to the quirkiness of Python's generators). This leaves equality/return-terms dangling, to be addressed in the next commit. Sending an equality query with the return-terms flag on does not result in pairs being returned, but simply keys. Obviously, the client knows what the index term is and so can inject it into the result (which is what we will do for consistency). --- riak/bucket.py | 14 +++- riak/client/index_page.py | 122 ++++++++++++++++++++++++++++++ riak/client/operations.py | 53 +++++++++---- riak/tests/test_2i.py | 122 ++++++++++++++++++++++++++++++ riak/transports/http/stream.py | 6 +- riak/transports/http/transport.py | 20 +++-- riak/transports/pbc/codec.py | 12 ++- riak/transports/pbc/stream.py | 17 +++-- riak/transports/pbc/transport.py | 22 ++++-- riak/transports/transport.py | 6 +- 10 files changed, 347 insertions(+), 47 deletions(-) create mode 100644 riak/client/index_page.py diff --git a/riak/bucket.py b/riak/bucket.py index d8a03db8..74aaa380 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -405,20 +405,26 @@ def search(self, query, **params): """ return self._client.solr.search(self.name, query, **params) - def get_index(self, index, startkey, endkey=None, return_terms=None): + def get_index(self, index, startkey, endkey=None, return_terms=None, + max_results=None, continuation=None): """ Queries a secondary index over objects in this bucket, returning keys. """ return self._client.get_index(self.name, index, startkey, endkey, - return_terms=return_terms) + return_terms=return_terms, + max_results=max_results, + continuation=continuation) - def stream_index(self, index, startkey, endkey=None, return_terms=None): + def stream_index(self, index, startkey, endkey=None, return_terms=None, + max_results=None, continuation=None): """ Queries a secondary index over objects in this bucket, streaming keys via an iterator. """ return self._client.stream_index(self.name, index, startkey, endkey, - return_terms=return_terms) + return_terms=return_terms, + max_results=max_results, + continuation=continuation) def delete(self, key, **kwargs): """Deletes an object from riak. diff --git a/riak/client/index_page.py b/riak/client/index_page.py new file mode 100644 index 00000000..9a8725cc --- /dev/null +++ b/riak/client/index_page.py @@ -0,0 +1,122 @@ +""" +Copyright 2013 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from collections import namedtuple, Sequence + + +CONTINUATION = namedtuple('Continuation', ['c']) + + +class IndexPage(Sequence, object): + """ + Encapsulates a single page of results from a secondary index + query, with the ability to iterate over results (if not streamed), + capture the page marker (continuation), and automatically fetch + the next page. + + While users will interact with this object, it will be created + automatically by the client and does not need to be instantiated + elsewhere. + """ + def __init__(self, client, bucket, index, startkey, endkey, return_terms, + max_results): + self.client = client + self.bucket = bucket + self.index = index + self.startkey = startkey + self.endkey = endkey + self.return_terms = return_terms + self.max_results = max_results + self.results = None + self.continuation = None + self.stream = False + + def __iter__(self): + if self.results: + try: + for result in self.results: + if self.stream and isinstance(result, CONTINUATION): + self.continuation = result.c + else: + yield result + finally: + if self.stream: + self.results.close() + else: + raise ValueError("No index results to iterate") + + def __len__(self): + if not self.stream and self.results is not None: + return len(self.results) + else: + raise ValueError("Streamed index page has no length") + + def __getitem__(self, index): + if not self.stream and self.results is not None: + return self.results[index] + else: + raise ValueError("Streamed index page has no entries") + + def __eq__(self, other): + if isinstance(other, list) and not (self.stream or + self.results is None): + return self.results == other + elif isinstance(other, IndexPage): + return other.__dict__ == self.__dict__ + else: + return False + + def __ne__(self, other): + return not self.__eq__(other) + + def has_next_page(self): + """ + Whether there is another page available, i.e. the response + included a continuation. + """ + return self.continuation is not None + + def next_page(self, stream=None): + """ + Fetches the next page using the same parameters as the + original query. + + Note that if streaming was used before, it will be used again + unless overridden. + + :param stream: whether to enable streaming. `True` enables, + `False` disables, `None` uses previous value. + :type stream: boolean + """ + if not self.continuation: + raise ValueError("Cannot get next index page, no continuation") + + if stream is not None: + self.stream = stream + + args = {'bucket': self.bucket, + 'index': self.index, + 'startkey': self.startkey, + 'endkey': self.endkey, + 'return_terms': self.return_terms, + 'max_results': self.max_results, + 'continuation': self.continuation} + if self.stream: + return self.client.stream_index(**args) + else: + return self.client.get_index(**args) diff --git a/riak/client/operations.py b/riak/client/operations.py index 6fd4956c..a49b24e1 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -17,6 +17,7 @@ """ from transport import RiakClientTransport, retryable, retryableHttpOnly +from index_page import IndexPage class RiakClientOperations(RiakClientTransport): @@ -51,7 +52,7 @@ def ping(self, transport): @retryable def get_index(self, transport, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, continuation=None): """ Queries a secondary index, returning matching keys. @@ -65,13 +66,29 @@ def get_index(self, transport, bucket, index, startkey, endkey=None, :type endkey: string, integer :param return_terms: whether to include the secondary index value :type return_terms: boolean - :rtype: list + :param max_results: the maximum number of results to return (page size) + :type max_results: integer + :param continuation: the opaque continuation returned from a + previous paginated request + :type continuation: string + :rtype: :class:`riak.client.index_page.IndexPage` """ - return transport.get_index(bucket, index, startkey, endkey, - return_terms=return_terms) + if return_terms and endkey is None: + raise ValueError("Cannot use return_terms with an equality query") + + page = IndexPage(self, bucket, index, startkey, endkey, + return_terms, max_results) + + results, continuation = transport.get_index( + bucket, index, startkey, endkey, return_terms=return_terms, + max_results=max_results, continuation=continuation) + + page.results = results + page.continuation = continuation + return page def stream_index(self, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, continuation=None): """ Queries a secondary index, streaming matching keys through an iterator. @@ -86,16 +103,24 @@ def stream_index(self, bucket, index, startkey, endkey=None, :type endkey: string, integer :param return_terms: whether to include the secondary index value :type return_terms: boolean - :rtype: iterable - """ + :param max_results: the maximum number of results to return (page size) + :type max_results: integer + :param continuation: the opaque continuation returned from a + previous paginated request + :type continuation: string + :rtype: :class:`riak.client.index_page.IndexPage` + """ + if return_terms and endkey is None: + raise ValueError("Cannot use return_terms with an equality query") + + page = IndexPage(self, bucket, index, startkey, endkey, + return_terms, max_results) with self._transport() as transport: - stream = transport.stream_index(bucket, index, startkey, endkey, - return_terms=return_terms) - try: - for item in stream: - yield item - finally: - stream.close() + page.stream = True + page.results = transport.stream_index( + bucket, index, startkey, endkey, return_terms=return_terms, + max_results=max_results, continuation=continuation) + return page @retryable def get_bucket_props(self, transport, bucket): diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 996741c6..aada2192 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -276,6 +276,128 @@ def test_index_return_terms(self): self.assertEqual([(1002, o2.key), (1003, o3.key), (1004, o4.key)], spairs) + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_pagination(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + results = bucket.get_index('field1_bin', 'val0', 'val5', + max_results=2) + # Number of results =< page size + self.assertLessEqual(2, len(results)) + # Results are in-order + self.assertEqual([o1.key, o2.key], results) + + # Continuation/next page present when page size smaller than + # total results size + self.assertIsNotNone(results.continuation) + self.assertTrue(results.has_next_page()) + + # Retrieving next page gets more results + page2 = results.next_page() + self.assertLessEqual(2, len(page2)) + self.assertEqual([o3.key, o4.key], page2) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_pagination_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # ========= Above steps work for return-terms ========== + results = bucket.get_index('field1_bin', 'val0', 'val5', + max_results=2, return_terms=True) + # Number of results =< page size + self.assertLessEqual(2, len(results)) + # Results are in-order + self.assertEqual([('val1', o1.key), ('val2', o2.key)], results) + + # Continuation/next page present when page size smaller than + # total results size + self.assertIsNotNone(results.continuation) + self.assertTrue(results.has_next_page()) + + # Retrieving next page gets more results + page2 = results.next_page() + self.assertLessEqual(2, len(results)) + self.assertEqual([('val3', o3.key), ('val4', o4.key)], page2) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_pagination_stream(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # ========= Above steps work for streaming ========== + stream = bucket.stream_index('field1_bin', 'val0', 'val5', + max_results=2) + results = [] + for result in stream: + results.extend(result) + + # Number of results =< page size + self.assertLessEqual(2, len(results)) + # Results are in-order + self.assertEqual([o1.key, o2.key], results) + + # Continuation/next page present when page size smaller than + # total results size + self.assertIsNotNone(stream.continuation) + self.assertTrue(stream.has_next_page()) + + # Retrieving next page gets more results + results = [] + for result in stream.next_page(): + results.extend(result) + self.assertLessEqual(2, len(results)) + self.assertEqual([o3.key, o4.key], results) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_pagination_stream_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # ========= Above steps work for streaming with return-terms ========== + + stream = bucket.stream_index('field1_bin', 'val0', 'val5', + max_results=2, return_terms=True) + results = [] + for result in stream: + results.extend(result) + + # Number of results =< page size + self.assertLessEqual(2, len(results)) + # Results are in-order + self.assertEqual([('val1', o1.key), ('val2', o2.key)], results) + + # Continuation/next page present when page size smaller than + # total results size + self.assertIsNotNone(stream.continuation) + self.assertTrue(stream.has_next_page()) + + # Retrieving next page gets more results + results = [] + for result in stream.next_page(): + results.extend(result) + self.assertLessEqual(2, len(results)) + self.assertEqual([('val3', o3.key), ('val4', o4.key)], results) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_eq_query_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + results = bucket.get_index('field2_int', 1001, return_terms=True) + self.assertEqual([(1001, o1.key)], results) + def _create_index_objects(self): """ Creates a number of index objects to be used in 2i test diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 949d64d0..d442cd3e 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -22,6 +22,7 @@ from cgi import parse_header from email import message_from_string from riak.util import decode_index_value +from riak.client.index_page import CONTINUATION class RiakHttpStream(object): @@ -145,9 +146,8 @@ def next(self): structs = payload[u'results'] # Format is {"results":[{"2ikey":"primarykey"}, ...]} return [self._decode_pair(d.items()[0]) for d in structs] - else: - # WAT - self.next() + elif u'continuation' in payload: + return CONTINUATION(payload[u'continuation']) def _decode_pair(self, pair): return (decode_index_value(self.index, pair[0]), pair[1]) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 76cc3a96..ffc70d5e 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -278,26 +278,31 @@ def stream_mapred(self, inputs, query, timeout=None): (repr(headers), repr(response.read()))) def get_index(self, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, continuation=None): """ Performs a secondary index query. """ - params = {'return_terms': return_terms} + params = {'return_terms': return_terms, 'max_results': max_results, + 'continuation': continuation} url = self.index_path(bucket, index, startkey, endkey, **params) status, headers, body = self._request('GET', url) self.check_http_code(status, [200]) json_data = json.loads(body) - if return_terms: + if return_terms and u'results' in json_data: results = [] for result in json_data[u'results'][:]: term, key = result.items()[0] results.append((decode_index_value(index, term), key),) - return results else: - return json_data[u'keys'][:] + results = json_data[u'keys'][:] + + if max_results and u'continuation' in json_data: + return (results, json_data[u'continuation']) + else: + return (results, None) def stream_index(self, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, continuation=None): """ Streams a secondary index query. """ @@ -305,7 +310,8 @@ def stream_index(self, bucket, index, startkey, endkey=None, raise NotImplementedError("Secondary index streaming is not " "supported") - params = {'return_terms': return_terms, 'stream': True} + params = {'return_terms': return_terms, 'stream': True, + 'max_results': max_results, 'continuation': continuation} url = self.index_path(bucket, index, startkey, endkey, **params) status, headers, response = self._request('GET', url, stream=True) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index ddf61a98..9a849665 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -375,7 +375,8 @@ def _encode_hook(self, hook, msg): return msg def _encode_index_req(self, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, + continuation=None): """ Encodes a secondary index request into the protobuf message. @@ -389,6 +390,11 @@ def _encode_index_req(self, bucket, index, startkey, endkey=None, :type endkey: integer, string :param return_terms: whether to return the index term with the key :type return_terms: bool + :param max_results: the maximum number of results to return (page size) + :type max_results: integer + :param continuation: the opaque continuation returned from a + previous paginated request + :type continuation: string :rtype riak_pb.RpbIndexReq """ req = riak_pb.RpbIndexReq(bucket=bucket, index=index) @@ -401,4 +407,8 @@ def _encode_index_req(self, bucket, index, startkey, endkey=None, req.key = str(startkey) if return_terms is not None: req.return_terms = return_terms + if max_results: + req.max_results = max_results + if continuation: + req.continuation = continuation return req diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index ff8345e6..b46b8227 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -23,6 +23,8 @@ MSG_CODE_MAPRED_RESP, MSG_CODE_INDEX_RESP ) +from riak.util import decode_index_value +from riak.client.index_page import CONTINUATION class RiakPbcStream(object): @@ -115,16 +117,15 @@ def __init__(self, transport, index, return_terms=False): def next(self): response = super(RiakPbcIndexStream, self).next() - if response.done and not (response.keys or response.results): + if response.done and not (response.keys or + response.results or + response.continuation): raise StopIteration if self.return_terms and response.results: - return [(self._coerce(r.key), r.value) for r in response.results] + return [(decode_index_value(self.index, r.key), r.value) + for r in response.results] elif response.keys: return response.keys - - def _coerce(self, index_value): - if "_int" in self.index: - return long(index_value) - else: - return str(index_value) + elif response.continuation: + return CONTINUATION(response.continuation) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 1ef91560..3ce914fe 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -336,29 +336,35 @@ def stream_mapred(self, inputs, query, timeout=None): return RiakPbcMapredStream(self) def get_index(self, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, continuation=None): if not self.pb_indexes(): return self._get_index_mapred_emu(bucket, index, startkey, endkey) req = self._encode_index_req(bucket, index, startkey, endkey, - return_terms=return_terms) + return_terms, max_results, continuation) msg_code, resp = self._request(MSG_CODE_INDEX_REQ, req, MSG_CODE_INDEX_RESP) - if return_terms: - return [(decode_index_value(index, pair.key), pair.value) - for pair in resp.results] + + if return_terms and resp.results: + results = [(decode_index_value(index, pair.key), pair.value) + for pair in resp.results] + else: + results = resp.keys + + if max_results: + return (results, resp.continuation) else: - return resp.keys + return (results, None) def stream_index(self, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, continuation=None): if not self.stream_indexes(): raise NotImplementedError("Secondary index streaming is not " "supported") req = self._encode_index_req(bucket, index, startkey, endkey, - return_terms=return_terms) + return_terms, max_results, continuation) req.stream = True self._send_msg(MSG_CODE_INDEX_REQ, req) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index d373152e..64605fa8 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -161,13 +161,15 @@ def search(self, index, query, **params): """ raise NotImplementedError - def get_index(self, bucket, index, startkey, endkey=None): + def get_index(self, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, continuation=None): """ Performs a secondary index query. """ raise NotImplementedError - def stream_index(self, bucket, index, startkey, endkey=None): + def stream_index(self, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, continuation=None): """ Streams a secondary index query. """ From 45d266fc54633259d6204376417ba0c2bf40d31b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 3 Jul 2013 16:22:06 -0500 Subject: [PATCH 0455/1060] Add pagination to secondary indexes, commit the second. This refactors a bit of the IndexPage class so as to support the equality/return-terms issue left dangling in the last commit. Along the way, I added a test for streaming with the eq/rt combo, and discovered an issue where the PBC interface returned the `keys` field from the response directly when it was expected to be a list. This wouldn't normally be an issue, but was necessitated by the eq/rt issue. This completes the work for #252. --- riak/client/index_page.py | 81 +++++++++++++++++++++++++------- riak/client/operations.py | 6 --- riak/tests/test_2i.py | 13 +++++ riak/transports/pbc/stream.py | 2 +- riak/transports/pbc/transport.py | 2 +- 5 files changed, 80 insertions(+), 24 deletions(-) diff --git a/riak/client/index_page.py b/riak/client/index_page.py index 9a8725cc..11416c18 100644 --- a/riak/client/index_page.py +++ b/riak/client/index_page.py @@ -47,41 +47,60 @@ def __init__(self, client, bucket, index, startkey, endkey, return_terms, self.stream = False def __iter__(self): - if self.results: - try: - for result in self.results: - if self.stream and isinstance(result, CONTINUATION): - self.continuation = result.c - else: - yield result - finally: - if self.stream: - self.results.close() - else: + """ + Emulates the iterator interface. When streaming, this means + delegating to the stream, otherwise iterating over the + existing result set. + """ + if self.results is None: raise ValueError("No index results to iterate") + try: + for result in self.results: + if self.stream and isinstance(result, CONTINUATION): + self.continuation = result.c + else: + yield self._inject_term(result) + finally: + if self.stream: + self.results.close() + def __len__(self): - if not self.stream and self.results is not None: + """ + Returns the length of the captured results. + """ + if self._has_results(): return len(self.results) else: raise ValueError("Streamed index page has no length") def __getitem__(self, index): - if not self.stream and self.results is not None: + """ + Fetches an item by index from the captured results. + """ + if self._has_results(): return self.results[index] else: raise ValueError("Streamed index page has no entries") def __eq__(self, other): - if isinstance(other, list) and not (self.stream or - self.results is None): - return self.results == other + """ + An IndexPage can pretend to be equal to a list when it has + captured results by simply comparing the internal results to + the passed list. Otherwise the other object needs to be an + equivalent IndexPage. + """ + if isinstance(other, list) and self._has_results(): + return self._inject_term(self.results) == other elif isinstance(other, IndexPage): return other.__dict__ == self.__dict__ else: return False def __ne__(self, other): + """ + Converse of __eq__. + """ return not self.__eq__(other) def has_next_page(self): @@ -120,3 +139,33 @@ def next_page(self, stream=None): return self.client.stream_index(**args) else: return self.client.get_index(**args) + + def _has_results(self): + """ + When not streaming, have results been assigned? + """ + return not (self.stream or self.results is None) + + def _should_inject_term(self, term): + """ + The index term should be injected when using an equality query + and the return terms option. If the term is already a tuple, + it can be skipped. + """ + return self.return_terms and not self.endkey + + def _inject_term(self, result): + """ + Upgrades a result (streamed or not) to include the index term + when an equality query is used with return_terms. + """ + if self._should_inject_term(result): + if type(result) is list: + return [ (self.startkey, r) for r in result ] + else: + return (self.startkey, result) + else: + return result + + def __repr__(self): + return "<{!s} {!r}>".format(self.__class__.__name__, self.__dict__) diff --git a/riak/client/operations.py b/riak/client/operations.py index a49b24e1..88b70df6 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -73,9 +73,6 @@ def get_index(self, transport, bucket, index, startkey, endkey=None, :type continuation: string :rtype: :class:`riak.client.index_page.IndexPage` """ - if return_terms and endkey is None: - raise ValueError("Cannot use return_terms with an equality query") - page = IndexPage(self, bucket, index, startkey, endkey, return_terms, max_results) @@ -110,9 +107,6 @@ def stream_index(self, bucket, index, startkey, endkey=None, :type continuation: string :rtype: :class:`riak.client.index_page.IndexPage` """ - if return_terms and endkey is None: - raise ValueError("Cannot use return_terms with an equality query") - page = IndexPage(self, bucket, index, startkey, endkey, return_terms, max_results) with self._transport() as transport: diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index aada2192..9a83eb49 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -398,6 +398,19 @@ def test_index_eq_query_return_terms(self): results = bucket.get_index('field2_int', 1001, return_terms=True) self.assertEqual([(1001, o1.key)], results) + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_eq_query_stream_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + results = [] + for item in bucket.stream_index('field2_int', 1001, return_terms=True): + results.extend(item) + + self.assertEqual([(1001, o1.key)], results) + def _create_index_objects(self): """ Creates a number of index objects to be used in 2i test diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index b46b8227..fd9766ba 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -126,6 +126,6 @@ def next(self): return [(decode_index_value(self.index, r.key), r.value) for r in response.results] elif response.keys: - return response.keys + return response.keys[:] elif response.continuation: return CONTINUATION(response.continuation) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 3ce914fe..7eb5e51c 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -350,7 +350,7 @@ def get_index(self, bucket, index, startkey, endkey=None, results = [(decode_index_value(index, pair.key), pair.value) for pair in resp.results] else: - results = resp.keys + results = resp.keys[:] if max_results: return (results, resp.continuation) From 7c1a7c4aca9e9ea2d5453031747e7855b04f10a9 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 23 Jul 2013 14:34:42 -0500 Subject: [PATCH 0456/1060] Fix some pep8, pyflakes and merge bugs. --- riak/client/index_page.py | 2 +- riak/client/multiget.py | 4 ++-- riak/tests/test_all.py | 29 +++++++++++++++-------------- riak/transports/pbc/transport.py | 3 ++- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/riak/client/index_page.py b/riak/client/index_page.py index 11416c18..5314f4db 100644 --- a/riak/client/index_page.py +++ b/riak/client/index_page.py @@ -161,7 +161,7 @@ def _inject_term(self, result): """ if self._should_inject_term(result): if type(result) is list: - return [ (self.startkey, r) for r in result ] + return [(self.startkey, r) for r in result] else: return (self.startkey, result) else: diff --git a/riak/client/multiget.py b/riak/client/multiget.py index 9995133b..665eba77 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -158,7 +158,7 @@ def multiget(client, keys, **options): for _ in range(len(keys)): if RIAK_MULTIGET_POOL.stopped(): raise RuntimeError("Multi-get operation interrupted by pool " - "stopping!") + "stopping!") results.append(outq.get()) outq.task_done() @@ -169,7 +169,7 @@ def multiget(client, keys, **options): from riak import RiakClient import riak.benchmark as benchmark client = RiakClient(protocol='pbc') - bkeys = [ ('multiget', str(key)) for key in xrange(10000) ] + bkeys = [('multiget', str(key)) for key in xrange(10000)] data = open(__file__).read() diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 66cf5823..b8220189 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -136,20 +136,6 @@ def test_timeout_validation(self): for bad in [0, -1, False, "foo"]: with self.assertRaises(ValueError): self.client.get_buckets(timeout=bad) - def test_multiget_bucket(self): - """ - Multiget operations can be invoked on buckets. - """ - keys = [self.key_name, self.randname(), self.randname()] - for key in keys: - self.client.bucket(self.bucket_name)\ - .new(key, encoded_data=key, content_type="text/plain")\ - .store() - results = self.client.bucket(self.bucket_name).multiget(keys) - for obj in results: - self.assertIsInstance(obj, RiakObject) - self.assertTrue(obj.exists) - self.assertEqual(obj.key, obj.encoded_data) with self.assertRaises(ValueError): for i in self.client.stream_buckets(timeout=bad): @@ -178,6 +164,20 @@ def test_multiget_bucket(self): for i in self.client.stream_mapred([], [], bad): pass + def test_multiget_bucket(self): + """ + Multiget operations can be invoked on buckets. + """ + keys = [self.key_name, self.randname(), self.randname()] + for key in keys: + self.client.bucket(self.bucket_name)\ + .new(key, encoded_data=key, content_type="text/plain")\ + .store() + results = self.client.bucket(self.bucket_name).multiget(keys) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertTrue(obj.exists) + self.assertEqual(obj.key, obj.encoded_data) def test_multiget_errors(self): """ @@ -204,6 +204,7 @@ def test_multiget_notfounds(self): self.assertIsInstance(obj, RiakObject) self.assertFalse(obj.exists) + class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, BucketPropsTest, diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 35468d70..9a760bbe 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -25,7 +25,8 @@ from riak.riak_object import VClock from riak.util import decode_index_value from connection import RiakPbcConnection -from stream import RiakPbcKeyStream, RiakPbcMapredStream, RiakPbcBucketStream, RiakPbcIndexStream +from stream import (RiakPbcKeyStream, RiakPbcMapredStream, RiakPbcBucketStream, + RiakPbcIndexStream) from codec import RiakPbcCodec from messages import ( From cf2ecf40d620974123f69f87e65c0b93e058392c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 23 Jul 2013 14:59:39 -0500 Subject: [PATCH 0457/1060] Update version to 2.0.0 in docs. --- docs/conf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d8e2e07d..c8bba0b5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,16 +41,16 @@ # General information about the project. project = u'Riak Python Client' -copyright = u'2010-2012, Basho Technologies' +copyright = u'2010-2013, Basho Technologies' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '1.4.1' +version = '2.0.0' # The full version, including alpha/beta/rc tags. -release = '1.4.1' +release = '2.0.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 461fc55cc5440873ce828215220083a6f0329e1f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 23 Jul 2013 15:00:05 -0500 Subject: [PATCH 0458/1060] Add autodoc for RiakContent class. --- docs/content.rst | 9 +++++++++ docs/index.rst | 1 + 2 files changed, 10 insertions(+) create mode 100644 docs/content.rst diff --git a/docs/content.rst b/docs/content.rst new file mode 100644 index 00000000..04880600 --- /dev/null +++ b/docs/content.rst @@ -0,0 +1,9 @@ +.. ref-content: + +=========== +RiakContent +=========== + +.. currentmodule:: riak.content + +.. autoclass:: riak.content.RiakContent diff --git a/docs/index.rst b/docs/index.rst index f8261594..d5b2e16a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,6 +26,7 @@ Contents: client bucket riak_object + content mapreduce Indices and tables From 01f6ef86cf5d7c49602bae03255423c6d6404e39 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 23 Jul 2013 15:00:32 -0500 Subject: [PATCH 0459/1060] Fix some typos in RiakBucket docs. --- docs/bucket.rst | 8 ++++---- riak/bucket.py | 15 ++++++--------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/bucket.rst b/docs/bucket.rst index 297d2b46..aceaa730 100644 --- a/docs/bucket.rst +++ b/docs/bucket.rst @@ -1,9 +1,9 @@ .. ref-bucket: -========== -RiakBucket -========== +=========================== +Bucket Objects (RiakBucket) +=========================== .. currentmodule:: riak.bucket -.. autoclass:: riak.bucket.RiakBucket +.. autoclass:: RiakBucket diff --git a/riak/bucket.py b/riak/bucket.py index 98690409..7770f3e6 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -224,7 +224,7 @@ def multiget(self, keys, r=None, pr=None): :type r: integer :param pr: PR-Value for the requests (defaults to bucket's PR) :type pr: integer - :rtype list of :class:`RiakObject ` + :rtype: list of :class:`RiakObject ` """ bkeys = [(self.name, key) for key in keys] return self._client.multiget(bkeys, r=r, pr=pr) @@ -253,13 +253,10 @@ def _set_resolver(self, value): N-value for this bucket, which is the number of replicas that will be written of each object in the bucket. - .. warning:: - - Set this once before you write any data to the bucket, and never - change it again, otherwise unpredictable things could happen. - This should only be used if you know what you are doing. - - :type nval: integer + .. warning:: Set this once before you write any data to the + bucket, and never change it again, otherwise unpredictable + things could happen. This should only be used if you know what + you are doing. """) allow_mult = bucket_property('allow_mult', doc=""" @@ -461,7 +458,7 @@ def get_counter(self, key, **kwargs): :param key: the key of the counter :type key: string - :rtype int + :rtype: int """ return self._client.get_counter(self, key, **kwargs) From 26ccf8f8047a00197e18d45697aff99b8b73117a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 08:59:06 -0500 Subject: [PATCH 0460/1060] Rewrite client docs page, breaking into sections. * Changes default autodoc options. * Improve documentation of some methods, adjusting the signature as necessary. * Ensure the retryable decorator maintains the docstring. --- docs/client.rst | 149 ++++++++++++++++++++++++++++++++++++-- docs/conf.py | 4 +- docs/index.rst | 2 +- riak/client/__init__.py | 44 ++++++++--- riak/client/operations.py | 110 +++++++++++++++++++++++++--- riak/client/transport.py | 3 + 6 files changed, 281 insertions(+), 31 deletions(-) diff --git a/docs/client.rst b/docs/client.rst index dff87c9f..ebeb664e 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -1,9 +1,148 @@ -.. ref-client: +==================== +Client & Connections +==================== -========== -RiakClient -========== +-------- +Overview +-------- + +To connect to a Riak cluster, you must create a +:py:class:`~riak.client.RiakClient` object. The default configuration +connects to a single Riak node on ``localhost`` with the default +ports. The below instantiation statements are all equivalent:: + + from riak import RiakClient, RiakNode + + RiakClient() + RiakClient(protocol='http', host='127.0.0.1', http_port=8098) + RiakClient(nodes=[{'host':'127.0.0.1','http_port':8098}]) + RiakClient(protocol='http', nodes=[RiakNode()]) + + +.. note:: Connections are not established until you attempt to perform + an operation. If the host or port are incorrect, you will not get + an error raised immediately. + +The client maintains a connection pool behind the scenes, one for each +protocol. Connections are opened as-needed; a random node is selected +when a new connection is requested. + +-------------- +RiakClient API +-------------- .. currentmodule:: riak.client +.. autoclass:: RiakClient + + .. autoattribute:: PROTOCOLS + .. autoattribute:: protocol + .. autoattribute:: client_id + .. attribute:: nodes + + The list of :class:`nodes ` that this + client will connect to. It is best not to modify this property + directly, as it is not thread-safe. + + .. attribute:: RETRY_COUNT + + The maximum number of times to retry requests where it is + permitted, default is 3. Retries will attempt to select nodes + with better error rates, excluding nodes where the request + failed. + +^^^^^ +Nodes +^^^^^ + +The ``nodes`` attribute of ``RiakClient`` objects is a list of +``RiakNode`` objects. If you include multiple host specifications in +the ``RiakClient`` constructor, they will be turned into this type. + +.. autoclass:: riak.node.RiakNode + :members: + +^^^^^^^^^^^^^^^^^^^^^^^ +Client-level Operations +^^^^^^^^^^^^^^^^^^^^^^^ + +Some operations are not scoped by buckets and can be performed on the +client directly: + +.. automethod:: RiakClient.ping +.. automethod:: RiakClient.get_buckets +.. automethod:: RiakClient.stream_buckets + +^^^^^^^^^^^^^^^^^ +Accessing Buckets +^^^^^^^^^^^^^^^^^ + +Most client operations are on :py:class:`bucket objects +` or keys within those buckets. Use the +``bucket`` method for creating buckets that will proxy operations to +the called client. + +.. automethod:: RiakClient.bucket + +^^^^^^^^^^^^^^^^^^^^^^^ +Bucket-level Operations +^^^^^^^^^^^^^^^^^^^^^^^ + +.. automethod:: RiakClient.get_bucket_props +.. automethod:: RiakClient.set_bucket_props +.. automethod:: RiakClient.clear_bucket_props +.. automethod:: RiakClient.get_keys +.. automethod:: RiakClient.stream_keys + +^^^^^^^^^^^^^^^^^^^^ +Key-level Operations +^^^^^^^^^^^^^^^^^^^^ + +.. automethod:: RiakClient.get +.. automethod:: RiakClient.put +.. automethod:: RiakClient.delete +.. automethod:: RiakClient.multiget +.. automethod:: RiakClient.get_counter +.. automethod:: RiakClient.update_counter + +^^^^^^^^^^^^^^^^ +Query Operations +^^^^^^^^^^^^^^^^ + +.. automethod:: RiakClient.mapred +.. automethod:: RiakClient.stream_mapred +.. automethod:: RiakClient.get_index +.. automethod:: RiakClient.stream_index +.. automethod:: RiakClient.fulltext_search +.. automethod:: RiakClient.fulltext_add +.. automethod:: RiakClient.fulltext_delete + +^^^^^^^^^^^^^ +Serialization +^^^^^^^^^^^^^ + +The client supports automatic transformation of Riak responses into +Python types if encoders and decoders are registered for the +media-types. Supported by default are ``application/json`` and +``text/plain``. + +.. autofunction:: default_encoder +.. automethod:: RiakClient.get_encoder +.. automethod:: RiakClient.set_encoder +.. automethod:: RiakClient.get_decoder +.. automethod:: RiakClient.set_decoder + + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Deprecated Methods and Properties +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. warning:: These methods exist solely for backwards-compatibility and should not + be used unless code is being ported from an older version. + +.. automethod:: RiakClient.get_transport +.. automethod:: RiakClient.get_client_id +.. automethod:: RiakClient.set_client_id +.. attribute:: RiakClient.solr -.. autoclass:: riak.client.RiakClient + Returns a RiakSearch object which can access search indexes. + DEPRECATED diff --git a/docs/conf.py b/docs/conf.py index c8bba0b5..bbc2064f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -256,6 +256,6 @@ #epub_tocdup = True # Autodoc settings -autodoc_default_flags = ['members', 'undoc-members'] -autodoc_member_order = 'bysource' +autodoc_default_flags = ['no-undoc-members'] +autodoc_member_order = 'groupwise' autoclass_content = 'both' diff --git a/docs/index.rst b/docs/index.rst index d5b2e16a..7ca823a0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,7 +14,7 @@ Installation .. _Pip: http://pip.openplans.org/ .. _easy_install: http://pypi.python.org/pypi/setuptools -.. _PyPI: http://pypi.python.org/pypi/riak/1.4.0 +.. _PyPI: http://pypi.python.org/pypi/riak/ Contents: diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 503109de..c3de41b8 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -55,6 +55,7 @@ class RiakClient(RiakMapReduceChain, RiakClientOperations): or by using the methods on related objects. """ + #: The supported protocols PROTOCOLS = ['http', 'https', 'pbc'] def __init__(self, protocol='http', transport_options={}, @@ -120,16 +121,18 @@ def _set_protocol(self, value): protocol = property(_get_protocol, _set_protocol, doc= """ - Which protocol to prefer, one of PROTOCOLS. - Please note that when one protocol is - selected, the other protocols MAY NOT attempt - to connect. Changing to another protocol will - cause a connection on the next request. - - Some requests are only valid over 'http' or - 'https', and will always be sent via those - transports, regardless of which protocol is - preferred. + Which protocol to prefer, one of + :attr:`PROTOCOLS + `. Please + note that when one protocol is selected, the + other protocols MAY NOT attempt to connect. + Changing to another protocol will cause a + connection on the next request. + + Some requests are only valid over ``'http'`` + or ``'https'``, and will always be sent via + those transports, regardless of which protocol + is preferred. """) def get_transport(self): @@ -181,6 +184,10 @@ def _set_client_id(self, client_id): def get_encoder(self, content_type): """ Get the encoding function for the provided content type. + + :param content_type: the requested media type + :type content_type: str + :rtype: function """ return self._encoders.get(content_type) @@ -188,7 +195,10 @@ def set_encoder(self, content_type, encoder): """ Set the encoding function for the provided content type. - :param encoder: + :param content_type: the requested media type + :type content_type: str + :param encoder: an encoding function, takes a single object + argument and returns a string :type encoder: function """ self._encoders[content_type] = encoder @@ -196,6 +206,10 @@ def set_encoder(self, content_type, encoder): def get_decoder(self, content_type): """ Get the decoding function for the provided content type. + + :param content_type: the requested media type + :type content_type: str + :rtype: function """ return self._decoders.get(content_type) @@ -203,7 +217,10 @@ def set_decoder(self, content_type, decoder): """ Set the decoding function for the provided content type. - :param decoder: + :param content_type: the requested media type + :type content_type: str + :param decoder: a decoding function, takes a string and + returns a Python type :type decoder: function """ self._decoders[content_type] = decoder @@ -226,7 +243,10 @@ def bucket(self, name): def solr(self): """ Returns a RiakSearch object which can access search indexes. + DEPRECATED """ + deprecated("``solr`` is deprecated, use ``fulltext_search``," + " ``fulltext_add`` and ``fulltext_delete`` directly") return RiakSearch(self) def _create_node(self, n): diff --git a/riak/client/operations.py b/riak/client/operations.py index a1906e01..22fc49b3 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -34,13 +34,20 @@ class RiakClientOperations(RiakClientTransport): @retryable def get_buckets(self, transport, timeout=None): """ - Get the list of buckets as RiakBucket instances. - NOTE: Do not use this in production, as it requires traversing through - all keys stored in a cluster. + get_buckets(timeout=None) + + Get the list of buckets as :class:`RiakBucket + ` instances. + + .. warning:: Do not use this in production, as it requires + traversing through all keys stored in a cluster. + + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. :param timeout: a timeout value in milliseconds :type timeout: int - :rtype list of RiakBucket instances + :rtype: list of :class:`RiakBucket ` instances """ _validate_timeout(timeout) return [self.bucket(name) for name in @@ -49,13 +56,15 @@ def get_buckets(self, transport, timeout=None): def stream_buckets(self, timeout=None): """ Streams the list of buckets. This is a generator method that - should be iterated over. NOTE: Do not use this in production, - as it requires traversing through all keys stored in a - cluster. + should be iterated over. + + .. warning:: Do not use this in production, as it requires + traversing through all keys stored in a cluster. :param timeout: a timeout value in milliseconds :type timeout: int - :rtype iterator + :rtype: iterator that yields lists of :class:`RiakBucket + ` instances """ _validate_timeout(timeout) with self._transport() as transport: @@ -71,8 +80,13 @@ def stream_buckets(self, timeout=None): @retryable def ping(self, transport): """ + ping() + Check if the Riak server for this ``RiakClient`` instance is alive. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :rtype: boolean """ return transport.ping() @@ -83,8 +97,14 @@ def ping(self, transport): def get_index(self, transport, bucket, index, startkey, endkey=None, return_terms=None, max_results=None, continuation=None): """ + get_index(bucket, index, startkey, endkey=None, return_terms=None,\ + max_results=None, continuation=None) + Queries a secondary index, returning matching keys. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param bucket: the bucket whose index will be queried :type bucket: RiakBucket :param index: the index to query @@ -148,8 +168,13 @@ def stream_index(self, bucket, index, startkey, endkey=None, @retryable def get_bucket_props(self, transport, bucket): """ + get_bucket_props(bucket) + Fetches bucket properties for the given bucket. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param bucket: the bucket whose properties will be fetched :type bucket: RiakBucket :rtype: dict @@ -159,8 +184,13 @@ def get_bucket_props(self, transport, bucket): @retryable def set_bucket_props(self, transport, bucket, props): """ + set_bucket_props(bucket, props) + Sets bucket properties for the given bucket. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param bucket: the bucket whose properties will be set :type bucket: RiakBucket :param props: the properties to set @@ -171,8 +201,13 @@ def set_bucket_props(self, transport, bucket, props): @retryable def clear_bucket_props(self, transport, bucket): """ + clear_bucket_props(bucket) + Resets bucket properties for the given bucket. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param bucket: the bucket whose properties will be set :type bucket: RiakBucket """ @@ -181,8 +216,13 @@ def clear_bucket_props(self, transport, bucket): @retryable def get_keys(self, transport, bucket, timeout=None): """ + get_keys(bucket, timeout=None) + Lists all keys in a bucket. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param bucket: the bucket whose properties will be set :type bucket: RiakBucket :param timeout: a timeout value in milliseconds @@ -197,7 +237,6 @@ def stream_keys(self, bucket, timeout=None): Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. - :param bucket: the bucket whose properties will be set :type bucket: RiakBucket :param timeout: a timeout value in milliseconds @@ -218,8 +257,14 @@ def stream_keys(self, bucket, timeout=None): def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=None, timeout=None): """ + put(robj, w=None, dw=None, pw=None, return_body=None,\ + if_none_match=None, timeout=None) + Stores an object in the Riak cluster. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param robj: the object to store :type robj: RiakObject :param w: the write quorum @@ -246,8 +291,13 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, @retryable def get(self, transport, robj, r=None, pr=None, timeout=None): """ + get(robj, r=None, pr=None, timeout=None) + Fetches the contents of a Riak object. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param robj: the object to fetch :type robj: RiakObject :param r: the read quorum @@ -268,8 +318,14 @@ def get(self, transport, robj, r=None, pr=None, timeout=None): def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None, timeout=None): """ + delete(robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None,\ + timeout=None) + Deletes an object from Riak. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param robj: the object to store :type robj: RiakObject :param rw: the read/write (delete) quorum @@ -294,8 +350,13 @@ def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, @retryable def mapred(self, transport, inputs, query, timeout): """ + mapred(inputs, query, timeout) + Executes a MapReduce query. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param inputs: the input list/structure :type inputs: list, dict :param query: the list of query phases @@ -332,8 +393,13 @@ def stream_mapred(self, inputs, query, timeout): @retryable def fulltext_search(self, transport, index, query, **params): """ + fulltext_search(index, query, **params) + Performs a full-text search query. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param index: the bucket/index to search over :type index: string :param query: the search query @@ -346,8 +412,14 @@ def fulltext_search(self, transport, index, query, **params): @retryableHttpOnly def fulltext_add(self, transport, index, docs): """ + fulltext_add(index, docs) + Adds documents to the full-text index. + .. note:: This request is automatically retried + :attr:`RETRY_COUNT` times if it fails due to network error. + Only HTTP will be used for this request. + :param index: the bucket/index in which to index these docs :type index: string :param docs: the list of documents @@ -358,8 +430,14 @@ def fulltext_add(self, transport, index, docs): @retryableHttpOnly def fulltext_delete(self, transport, index, docs=None, queries=None): """ + fulltext_delete(index, docs=None, queries=None) + Removes documents from the full-text index. + .. note:: This request is automatically retried + :attr:`RETRY_COUNT` times if it fails due to network error. + Only HTTP will be used for this request. + :param index: the bucket/index from which to delete :type index: string :param docs: a list of documents (with ids) @@ -377,7 +455,8 @@ def multiget(self, pairs, **params): :type pairs: list :param params: additional request flags, e.g. r, pr :type params: dict - :rtype list + :rtype: list of :class:`RiakObject ` + instances """ return multiget(self, pairs, **params) @@ -385,8 +464,14 @@ def multiget(self, pairs, **params): def get_counter(self, transport, bucket, key, r=None, pr=None, basic_quorum=None, notfound_ok=None): """ + get_counter(bucket, key, r=None, pr=None, basic_quorum=None,\ + notfound_ok=None) + Gets the value of a counter. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param bucket: the bucket of the counter :type bucket: RiakBucket :param key: the key of the counter @@ -400,13 +485,16 @@ def get_counter(self, transport, bucket, key, r=None, pr=None, :type basic_quorum: bool :param notfound_ok: whether to treat not-found responses as successful :type notfound_ok: bool - :rtype integer + :rtype: integer """ return transport.get_counter(bucket, key, r=r, pr=pr) def update_counter(self, bucket, key, value, w=None, dw=None, pw=None, returnvalue=False): """ + update_counter(bucket, key, value, w=None, dw=None, pw=None,\ + returnvalue=False) + Updates a counter by the given value. This operation is not idempotent and so should not be retried automatically. diff --git a/riak/client/transport.py b/riak/client/transport.py index 38f4b272..eb2b7481 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -124,6 +124,9 @@ def thunk(transport): return self._with_retries(pool, thunk) + wrapper.__doc__ = fn.__doc__ + wrapper.__repr__ = fn.__repr__ + return wrapper From 1353f1231cfbc3bdcf7c8eed8f292919e625d987 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 10:11:00 -0500 Subject: [PATCH 0461/1060] Remove the tutorial. --- docs/index.rst | 2 - docs/tutorial.rst | 424 ---------------------------------------------- 2 files changed, 426 deletions(-) delete mode 100644 docs/tutorial.rst diff --git a/docs/index.rst b/docs/index.rst index 7ca823a0..9b1ed90a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,8 +21,6 @@ Contents: .. toctree:: :maxdepth: 2 - tutorial - client bucket riak_object diff --git a/docs/tutorial.rst b/docs/tutorial.rst deleted file mode 100644 index 2506d114..00000000 --- a/docs/tutorial.rst +++ /dev/null @@ -1,424 +0,0 @@ -.. ref-tutorial: - -======== -Tutorial -======== - -This tutorial assumes basic working knowledge of how Riak works & what it can -do. If you need a more comprehensive overview how to use Riak, please check out -the `Riak Fast Track`_. - -.. _`Riak Fast Track`: http://wiki.basho.com/The-Riak-Fast-Track.html - - -Quick Start -=========== - -For the impatient, simple usage of the official Python binding for Riak looks -like:: - - import riak - - # Connect to Riak. - client = riak.RiakClient() - - # Choose the bucket to store data in. - bucket = client.bucket('test') - - - # Supply a key to store data under. - # The ``data`` can be any data Python's ``json`` encoder can handle. - person = bucket.new('riak_developer_1', data={ - 'name': 'John Smith', - 'age': 28, - 'company': 'Mr. Startup!', - }) - # Save the object to Riak. - person.store() - - -Connecting To Riak -================== - -There are two supported ways to connect to Riak, the HTTP interface & the -`Protocol Buffers`_ interface. Both provide the same API & full access to -Riak. - -The HTTP interface is easier to setup & is well suited for development use. It -is the slower of the two interfaces, but if you are only making a handful of -requests, it is more than capable. - -The Protocol Buffers (also called ``protobuf``) is more difficult to setup but -is significantly faster (2-3x) and is more suitable for production use. This -interface is better suited to a higher number of requests. - -.. _`Protocol Buffers`: http://code.google.com/p/protobuf/ - -To use the HTTP interface and connecting to a local Riak on the default port, -no arguments are needed:: - - import riak - - client = riak.RiakClient() - -The constructor also configuration options such as ``host``, ``http_port``, -``pb_port`` & ``prefix``. Please refer to the :doc:`client` documentation -for full details. - -To use the Protocol Buffers interface:: - - import riak - - client = riak.RiakClient(pb_port=8087, protocol='pbc') - -.. warning: - - Riak's default port is 8098. However, when using the Protocol Buffers, the - Riak listens on port 8087. If you forget this, you will *NOT* get an - immediate error, but will instead receive an error when fetching or storing - data to the effect of ``RiakError: 'Socket returned short read 135 - - expected 8192'``. - -The ``protocol`` argument indicates to the client which backend to use. -We didn't need to specify it in the HTTP example because ``http`` is the -default class. Available options are: ``http``, ``https``, & ``pbc``. - - -Using Buckets -============= - -Buckets in Riak's terminology are segmented keyspaces. They are a way to -categorize different types of data and are roughly analogous to tables in an -RDBMS. - -Once you have a ``client``, selecting a bucket is simple. Provide a string of -the name of the bucket to use:: - - test_bucket = client.bucket('test') - -If the bucket does not exist, Riak will create it for you. You can also open -as many buckets as you need:: - - user_bucket = client.bucket('user') - profile_bucket = client.bucket('profile') - status_bucket = client.bucket('status') - -If needed, you can also manually instantiate a bucket like so:: - - user_bucket = riak.RiakBucket(client, 'user') - -The buckets themselves provide many different methods. The most commonly used -are: - -* ``get`` - Fetches a key's value (decoded from JSON). -* ``get_binary`` - Also fetches a key's raw value (plain text or binary). -* ``new`` - Creates a new key/value pair (encoded in JSON). -* ``new_binary`` - Creates a new key/raw value pair. - -See the full :doc:`bucket` documentation for the other methods. - - -Storing Keys/Values -=================== - -Once you've got a working client/bucket, the next task at hand is storing data. -Riak provides several ways to store your data, but the most common are a -JSON-encoded structure or a binary blob. - -To store JSON-encoded data, you'd do something like the following:: - - import riak - - client = riak.RiakClient() - user_bucket = client.bucket('user') - - # We're creating the user data & keying off their username. - new_user = user_bucket.new('johndoe', data={ - 'first_name': 'John', - 'last_name': 'Doe', - 'gender': 'm', - 'website': 'http://example.com/', - 'is_active': True, - }) - # Note that the user hasn't been stored in Riak yet. - new_user.store() - -Note that any data Python's ``json`` (or ``simplejson``) encoder can handle is -fair game. - -As mentioned, Riak can also handle binary data, such as images, audio files, -etc. Storing binary data looks almost identical:: - - import riak - - client = riak.RiakClient() - user_photo_bucket = client.bucket('user_photo') - - # For example purposes, we'll read a file off the filesystem, but you can get - # the data from anywhere. - the_photo_data = open('/tmp/johndoe_headshot.jpg', 'rb').read() - - # We're storing the photo in a different bucket but keyed off the same - # username. - new_user = user_photo_bucket.new_binary('johndoe', data=the_photo_data, content_type='image/jpeg') - new_user.store() - -You can also manually store data by using ``RiakObject``:: - - import riak - import time - import uuid - - client = riak.RiakClient() - status_bucket = client.bucket('status') - - # We use ``uuid.uuid1().hex`` here to create a unique identifier for the status. - post_uuid = uuid.uuid1().hex - new_status = riak.RiakObject(client, status_bucket, post_uuid) - - # Add in the data you want to store. - new_status.set_data({ - 'message': 'First post!', - 'created': time.time(), - 'is_public': True, - }) - - # Set the content type. - new_status.set_content_type('application/json') - - # We want to do JSON-encoding on the value. - new_status._encode_data = True - - # Again, make sure you save it. - new_status.store() - - -Getting Single Values Out -========================= - -Storing data is all well and good, but you'll need to get that data out at a -later date. - -Riak provides several ways to get data out, though fetching single key/value -pairs is the easiest. Just like storing the data, you can pull the data out -in either the JSON-decoded form or a binary blob. Getting the JSON-decoded -data out looks like:: - - import riak - - client = riak.RiakClient() - user_bucket = client.bucket('user') - - johndoe = user_bucket.get('johndoe') - - # You've now got a ``RiakObject``. To get at the values in a dictionary - # form, call: - johndoe_dict = johndoe.data - -Getting binary data out looks like:: - - import riak - - client = riak.RiakClient() - user_photo_bucket = client.bucket('user_photo') - - johndoe = user_photo_bucket.get_binary('johndoe') - - # You've now got a ``RiakObject``. To get at the binary data, call: - johndoe_headshot = johndoe.data - -Manually fetching data is also possible:: - - import riak - - client = riak.RiakClient() - status_bucket = client.bucket('status') - - # We're using the UUID generated from the above section. - first_post_status = riak.RiakObject(client, status_bucket, post_uuid) - first_post_status._encode_data = True - r = status_bucket.get_r() - - # Calling ``reload`` will cause the ``RiakObject`` instance to load fresh - # data/metadata from Riak. - first_post_status.reload(r) - - # Finally, pull out the data. - message = first_post_status.data['message'] - - -Fetching Data Via Map/Reduce -============================ - -When you need to work with larger sets of data, one of the tools at your -disposal is MapReduce_. This technique iterates over all of the data, returning -data from the map phase & combining all the different maps in the reduce -phase(s). - -.. _MapReduce: http://wiki.basho.com/MapReduce.html - -To perform a map operation, such as returning all active users, you can do -something like:: - - import riak - - client = riak.RiakClient() - # First, you need to ``add`` the bucket you want to MapReduce on. - query = client.add('user') - # Then, you supply a Javascript map function as the code to be executed. - query.map("function(v) { var data = JSON.parse(v.values[0].data); if(data.is_active == true) { return [[v.key, data]]; } return []; }") - - for result in query.run(): - # Print the key (``v.key``) and the value for that key (``data``). - print "%s - %s" % (result[0], result[1]) - - # Results in something like: - # - # mr_smith - {'first_name': 'Mister', 'last_name': 'Smith', 'is_active': True} - # johndoe - {'first_name': 'John', 'last_name': 'Doe', 'is_active': True} - # annabody - {'first_name': 'Anna', 'last_name': 'Body', 'is_active': True} - -You can also do this manually:: - - import riak - - client = riak.RiakClient() - query = riak.RiakMapReduce(client).add('user') - query.map("function(v) { var data = JSON.parse(v.values[0].data); if(data.is_active == true) { return [[v.key, data]]; } return []; }") - - for result in query.run(): - print "%s - %s" % (result[0], result[1]) - -Adding a reduce phase, say to sort by username (key), looks almost identical:: - - import riak - - client = riak.RiakClient() - query = client.add('user') - query.map("function(v) { var data = JSON.parse(v.values[0].data); if(data.is_active == true) { return [[v.key, data]]; } return []; }") - query.reduce("function(values) { return values.sort(); }") - - for result in query.run(): - # Print the key (``v.key``) and the value for that key (``data``). - print "%s - %s" % (result[0], result[1]) - - # Results in something like: - # - # annabody - {'first_name': 'Anna', 'last_name': 'Body', 'is_active': True} - # johndoe - {'first_name': 'John', 'last_name': 'Doe', 'is_active': True} - # mr_smith - {'first_name': 'Mister', 'last_name': 'Smith', 'is_active': True} - - -Working With Related Data Via Links -=================================== - -Links_ are powerful concept in Riak that allow, within the key/value pair's -metadata, relations between objects. - -.. _Links: http://wiki.basho.com/Links.html - -Adding them to your data is relatively trivial. For instance, we'll link a -user's statuses to their user data:: - - import riak - import uuid - - client = riak.RiakClient() - user_bucket = client.bucket('user') - status_bucket = client.bucket('status') - - johndoe = user_bucket.get('johndoe') - - new_status = status_bucket.new(uuid.uuid1().hex, data={ - 'message': 'First post!', - 'created': time.time(), - 'is_public': True, - }) - # Add one direction (from status to user)... - new_status.add_link(johndoe) - new_status.store() - - # ... Then add the other direction. - johndoe.add_link(new_status) - johndoe.store() - -Fetching the data is equally simple:: - - import riak - - client = riak.RiakClient() - user_bucket = client.bucket('user') - - johndoe = user_bucket.get('johndoe') - - for status_link in johndoe.get_links(): - # Since what we get back are lightweight ``RiakLink`` objects, we need to - # get the associated ``RiakObject`` to access its data. - status = status_link.get() - print status.data['message'] - - -Using Search -============ - -`Riak Search`_ is a new feature available as of Riak 0.13. It allows you to create -queries that filter on data in the values without writing a MapReduce. It takes -inspiration from Lucene_, a popular Java-based search library, and incorporates -a Solr-like interface into Riak. The setup of this is outside the realm of this -tutorial, but usage of this feature looks like:: - - import riak - - client = riak.RiakClient() - - # First parameter is the bucket we want to search within, the second - # is the query we want to perform. - search_query = client.search('user', 'first_name:[Anna TO John]') - - for result in search_query.run(): - # You get ``RiakLink`` objects back. - user = result.get() - user_data = user.data - print "%s %s" % (user_data['first_name'], user_data['last_name']) - - # Results in something like: - # - # John Doe - # Anna Body - -.. _`Riak Search`: http://wiki.basho.com/Riak-Search.html -.. _Lucene: http://lucene.apache.org/ - -Using Secondary Indexes -======================= - -Secondary Indexes is a new feature available as of Riak 1.0. It -allows you to tag an object with index metadata, and then later find -the object by querying the metadata, returning a list of matching keys. - -Your Riak cluster must have Secondary Indexes enabled. See the Riak -documentation for details. - -Usage of this feature looks like:: - - import riak - - client = riak.RiakClient() - bucket = client.bucket('mybucket') - - # Create and store the object with indexes... - obj = bucket.new('mykey1', 'mydata') - obj.add_index('field1_bin', 'val1') - obj.add_index('field2_int', 1001) - obj.store() - - # Query the indexes. The return value is a list of ``RiakLink`` objects. - results = client.index('mybucket', 'field1_bin', 'val1').run() - - # Query the indexes using a range... - results = client.index('mybucket', 'field1_bin', 'val1', 'val5').run() - - # Remove an index entry... - obj = bucket.get('mykey1') - obj.remove_index('field1_bin', 'val1') - obj.store() From 9263fcd6482d0584649abedd1860e6527696c7a0 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 11:34:39 -0500 Subject: [PATCH 0462/1060] Document more deprecated methods on client. --- docs/client.rst | 25 ++++++++++++++++++++++--- riak/util.py | 20 ++++++++++++++++++-- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/docs/client.rst b/docs/client.rst index ebeb664e..531cfc1d 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -37,6 +37,11 @@ RiakClient API .. autoattribute:: PROTOCOLS .. autoattribute:: protocol .. autoattribute:: client_id + .. attribute:: resolver + + The sibling-resolution function for this client. Defaults + to :func:`riak.resolver.default_resolver`. + .. attribute:: nodes The list of :class:`nodes ` that this @@ -136,8 +141,9 @@ media-types. Supported by default are ``application/json`` and Deprecated Methods and Properties ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. warning:: These methods exist solely for backwards-compatibility and should not - be used unless code is being ported from an older version. +.. warning:: These methods and attributes exist solely for + backwards-compatibility and should not be used unless code is being + ported from an older version. .. automethod:: RiakClient.get_transport .. automethod:: RiakClient.get_client_id @@ -145,4 +151,17 @@ Deprecated Methods and Properties .. attribute:: RiakClient.solr Returns a RiakSearch object which can access search indexes. - DEPRECATED + **DEPRECATED** + +.. automethod:: RiakClient.get_r +.. automethod:: RiakClient.set_r +.. automethod:: RiakClient.get_pr +.. automethod:: RiakClient.set_pr +.. automethod:: RiakClient.get_w +.. automethod:: RiakClient.set_w +.. automethod:: RiakClient.get_dw +.. automethod:: RiakClient.set_dw +.. automethod:: RiakClient.get_pw +.. automethod:: RiakClient.set_pw +.. automethod:: RiakClient.get_rw +.. automethod:: RiakClient.set_rw diff --git a/riak/util.py b/riak/util.py index f096caf7..9d371b92 100644 --- a/riak/util.py +++ b/riak/util.py @@ -80,7 +80,7 @@ def __deprecateQuorumAccessor(klass, parent, quorum): getter_name = "get_%s" % quorum setter_name = "set_%s" % quorum if not parent: - def direct_getter(self, val=None): + def direct_getter(self, value=None): deprecated(QDEPMESSAGE % klass.__name__) if val: return val @@ -88,7 +88,7 @@ def direct_getter(self, val=None): getter = direct_getter else: - def parent_getter(self, val=None): + def parent_getter(self, value=None): deprecated(QDEPMESSAGE % klass.__name__) if val: return val @@ -103,6 +103,22 @@ def setter(self, value): setattr(self, propname, value) return self + getter.__doc__ = """ + Gets the value used in requests for the {!r} quorum. + If not set, returns the passed value. **DEPRECATED** + + :param value: the value to use if not set + :type value: mixed + :rtype: mixed""".format(quorum) + + setter.__doc__ = """ + Sets the value used in requests for the {!r} quorum. + **DEPRECATED** + + :param value: the value to use if not set + :type value: mixed + """ + setattr(klass, getter_name, getter) setattr(klass, setter_name, setter) From 6cbbb96e05c5461f0c457280ddb8f7bac2e11d6b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 12:13:00 -0500 Subject: [PATCH 0463/1060] Update bucket docs. --- docs/bucket.rst | 161 ++++++++++++++++++++++++++++++++++++++++++++++-- riak/bucket.py | 100 +++++++++++++++++++----------- 2 files changed, 219 insertions(+), 42 deletions(-) diff --git a/docs/bucket.rst b/docs/bucket.rst index aceaa730..9aa81df7 100644 --- a/docs/bucket.rst +++ b/docs/bucket.rst @@ -1,9 +1,160 @@ -.. ref-bucket: - -=========================== -Bucket Objects (RiakBucket) -=========================== +======= +Buckets +======= .. currentmodule:: riak.bucket +-------- +Overview +-------- + +Buckets are both namespaces for the key-value pairs you store in Riak, +and containers for properties that apply to that namespace. Buckets +should be created via the :meth:`bucket() +` method on the client object, like so:: + + import riak + + client = riak.RiakClient() + mybucket = client.bucket('mybucket') + +-------------- +RiakBucket API +-------------- + .. autoclass:: RiakBucket + + .. attribute:: name + + The name of the bucket, a string. + + .. autoattribute:: resolver + +^^^^^^^^^^^^^^^^^ +Bucket properties +^^^^^^^^^^^^^^^^^ + +Bucket properties are flags and defaults that apply to all keys in the +bucket. + +.. automethod:: RiakBucket.get_properties +.. automethod:: RiakBucket.set_properties +.. automethod:: RiakBucket.clear_properties +.. automethod:: RiakBucket.get_property +.. automethod:: RiakBucket.set_property + +""""""""""""""""""""""""""""""" +Shortcuts for common properties +""""""""""""""""""""""""""""""" + +Some of the most commonly-used bucket properties are exposed as object +properties as well. The getters and setters simply call +:meth:`RiakBucket.get_property` and :meth:`RiakBucket.set_property` +respectively. + +.. autoattribute:: RiakBucket.n_val +.. autoattribute:: RiakBucket.allow_mult +.. autoattribute:: RiakBucket.r +.. autoattribute:: RiakBucket.pr +.. autoattribute:: RiakBucket.w +.. autoattribute:: RiakBucket.dw +.. autoattribute:: RiakBucket.pw +.. autoattribute:: RiakBucket.rw + +"""""""""""""""""""" +Shortcuts for search +"""""""""""""""""""" + +When Riak Search is enabled on the server, you can toggle which +buckets have automatic indexing turned on using the ``search`` bucket +property (and on older versions, the ``precommit`` property). These +methods simplify interacting with that configuration. + +.. automethod:: RiakBucket.search_enabled +.. automethod:: RiakBucket.enable_search +.. automethod:: RiakBucket.disable_search + +^^^^^^^^^^^^^^^^^ +Working with keys +^^^^^^^^^^^^^^^^^ + +The primary purpose of buckets is to act as namespaces for keys. As +such, you can use the bucket object to create, fetch and delete +:class:`objects `. + +.. automethod:: RiakBucket.new +.. automethod:: RiakBucket.new_from_file +.. automethod:: RiakBucket.get +.. automethod:: RiakBucket.multiget +.. automethod:: RiakBucket.delete + +"""""""" +Counters +"""""""" + +Rather than returning objects, the counter operations new to Riak 1.4 +act directly on the value of the counter. + +.. automethod:: RiakBucket.get_counter +.. automethod:: RiakBucket.update_counter + +^^^^^^^^^^^^^^^^ +Query operations +^^^^^^^^^^^^^^^^ + +.. automethod:: RiakBucket.search +.. automethod:: RiakBucket.get_index +.. automethod:: RiakBucket.stream_index + + +^^^^^^^^^^^^^ +Serialization +^^^^^^^^^^^^^ + +Similar to :class:`RiakClient `, buckets can +register custom transformation functions for media-types. When +undefined on the bucket, :meth:`RiakBucket.get_encoder` and +:meth:`RiakBucket.get_decoder` will delegate to the client associated +with the bucket. + +.. automethod:: RiakBucket.get_encoder +.. automethod:: RiakBucket.set_encoder +.. automethod:: RiakBucket.get_decoder +.. automethod:: RiakBucket.set_decoder + + +^^^^^^^^^^^^ +Listing keys +^^^^^^^^^^^^ + +Shortcuts for :meth:`RiakClient.get_keys() +` and +:meth:`RiakClient.stream_keys() +` are exposed on the bucket +object. The same admonitions for these operations apply. + +.. automethod:: RiakBucket.get_keys +.. automethod:: RiakBucket.stream_keys + +^^^^^^^^^^^^^^^^^^ +Deprecated methods +^^^^^^^^^^^^^^^^^^ + +.. warning:: These methods exist solely for backwards-compatibility and should not + be used unless code is being ported from an older version. + +.. automethod:: RiakBucket.new_binary +.. automethod:: RiakBucket.new_binary_from_file +.. automethod:: RiakBucket.get_binary +.. automethod:: RiakBucket.get_r +.. automethod:: RiakBucket.set_r +.. automethod:: RiakBucket.get_pr +.. automethod:: RiakBucket.set_pr +.. automethod:: RiakBucket.get_w +.. automethod:: RiakBucket.set_w +.. automethod:: RiakBucket.get_dw +.. automethod:: RiakBucket.set_dw +.. automethod:: RiakBucket.get_pw +.. automethod:: RiakBucket.set_pw +.. automethod:: RiakBucket.get_rw +.. automethod:: RiakBucket.set_rw diff --git a/riak/bucket.py b/riak/bucket.py index 7770f3e6..22dc3cb9 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -43,8 +43,6 @@ class RiakBucket(object): objects within the bucket. """ - SEARCH_PRECOMMIT_HOOK = {"mod": "riak_search_kv_hook", "fun": "precommit"} - def __init__(self, client, name): """ Returns a new ``RiakBucket`` instance. @@ -86,6 +84,8 @@ def get_encoder(self, content_type): Get the encoding function for the provided content type for this bucket. + :param content_type: the requested media type + :type content_type: str :param content_type: Content type requested """ if content_type in self._encoders: @@ -98,9 +98,11 @@ def set_encoder(self, content_type, encoder): Set the encoding function for the provided content type for this bucket. - :param content_type: Content type for encoder - :param encoder: Function to encode with - will be called with - data as single argument. + :param content_type: the requested media type + :type content_type: str + :param encoder: an encoding function, takes a single object + argument and returns a string data as single argument. + :type encoder: function """ self._encoders[content_type] = encoder return self @@ -110,7 +112,9 @@ def get_decoder(self, content_type): Get the decoding function for the provided content type for this bucket. - :param content_type: Content type for decoder + :param content_type: the requested media type + :type content_type: str + :rtype: function """ if content_type in self._decoders: return self._decoders[content_type] @@ -122,9 +126,11 @@ def set_decoder(self, content_type, decoder): Set the decoding function for the provided content type for this bucket. - :param content_type: Content type for decoder - :param decoder: Function to decode with - will be called with - string + :param content_type: the requested media type + :type content_type: str + :param decoder: a decoding function, takes a string and + returns a Python type + :type decoder: function """ self._decoders[content_type] = decoder return self @@ -164,7 +170,7 @@ def new_binary(self, key=None, data=None, Create a new :class:`RiakObject ` that will be stored as plain text/binary. A shortcut for manually instantiating a :class:`RiakObject - `. + `. **DEPRECATED** :param key: Name of the key. :type key: string @@ -198,7 +204,7 @@ def get(self, key, r=None, pr=None, timeout=None): def get_binary(self, key, r=None, pr=None, timeout=None): """ - Retrieve a binary/string object from Riak. DEPRECATED + Retrieve a binary/string object from Riak. **DEPRECATED** :param key: Name of the key. :type key: string @@ -246,8 +252,7 @@ def _set_resolver(self, value): resolver = property(_get_resolver, _set_resolver, doc= """The sibling-resolution function for this bucket. If the resolver is not set, the - client's resolver will be used. :type - callable""") + client's resolver will be used.""") n_val = bucket_property('n_val', doc=""" N-value for this bucket, which is the number of replicas @@ -261,8 +266,7 @@ def _set_resolver(self, value): allow_mult = bucket_property('allow_mult', doc=""" If set to True, then writes with conflicting data will be stored - and returned to the client. This situation can be detected by - calling has_siblings() and get_siblings(). + and returned to the client. :type bool: boolean """) @@ -344,7 +348,6 @@ def get_properties(self): def clear_properties(self): """ Reset all bucket properties to their defaults. - """ return self._client.clear_bucket_props(self) @@ -352,9 +355,7 @@ def get_keys(self): """ Return all keys within the bucket. - .. warning:: - - At current, this is a very expensive operation. Use with caution. + :rtype: list of keys """ return self._client.get_keys(self) @@ -362,18 +363,21 @@ def stream_keys(self): """ Streams all keys within the bucket through an iterator. - .. warning:: - - At current, this is a very expensive operation. Use with caution. - :rtype: iterator """ return self._client.stream_keys(self) def new_from_file(self, key, filename): """ - Create a new Riak object in the bucket, using the content of - the specified file. + Create a new Riak object in the bucket, using the contents of + the specified file. This is a shortcut for :meth:`new`, where the + ``encoded_data`` and ``content_type`` are set for you. + + :param key: the key of the new object + :type key: string + :param filename: the file to read the contents from + :type filename: string + :rtype: :class:`RiakObject ` """ binary_data = open(filename, "rb").read() mimetype, encoding = mimetypes.guess_type(filename) @@ -386,21 +390,31 @@ def new_from_file(self, key, filename): return self.new(key, encoded_data=binary_data, content_type=mimetype) def new_binary_from_file(self, key, filename): + """ + Create a new Riak object in the bucket, using the contents of + the specified file. This is a shortcut for :meth:`new`, where the + ``encoded_data`` and ``content_type`` are set for you. **DEPRECATED** + + :param key: the key of the new object + :type key: string + :param filename: the file to read the contents from + :type filename: string + :rtype: :class:`RiakObject ` + """ deprecated('RiakBucket.new_binary_from_file is deprecated, use ' 'RiakBucket.new_from_file') return self.new_from_file(key, filename) def search_enabled(self): """ - Returns True if the search precommit hook is enabled for this + Returns True if search indexing is enabled for this bucket. """ return self.get_properties().get('search', False) def enable_search(self): """ - Enable search for this bucket by installing the precommit hook to - index objects in it. + Enable search indexing for this bucket. """ if not self.search_enabled(): self.set_property('search', True) @@ -408,8 +422,7 @@ def enable_search(self): def disable_search(self): """ - Disable search for this bucket by removing the precommit hook to - index objects in it. + Disable search indexing for this bucket. """ if self.search_enabled(): self.set_property('search', False) @@ -417,14 +430,19 @@ def disable_search(self): def search(self, query, **params): """ - Queries a search index over objects in this bucket/index. + Queries a search index over objects in this bucket/index. See + :meth:`RiakClient.fulltext_search() + ` for more details. """ return self._client.solr.search(self.name, query, **params) def get_index(self, index, startkey, endkey=None, return_terms=None, max_results=None, continuation=None): """ - Queries a secondary index over objects in this bucket, returning keys. + Queries a secondary index over objects in this bucket, + returning keys or index/key pairs. See + :meth:`RiakClient.get_index() + ` for more details. """ return self._client.get_index(self.name, index, startkey, endkey, return_terms=return_terms, @@ -435,7 +453,9 @@ def stream_index(self, index, startkey, endkey=None, return_terms=None, max_results=None, continuation=None): """ Queries a secondary index over objects in this bucket, - streaming keys via an iterator. + streaming keys or index/key pairs via an iterator. See + :meth:`RiakClient.stream_index() + ` for more details. """ return self._client.stream_index(self.name, index, startkey, endkey, return_terms=return_terms, @@ -443,9 +463,10 @@ def stream_index(self, index, startkey, endkey=None, return_terms=None, continuation=continuation) def delete(self, key, **kwargs): - """Deletes an object from riak. + """Deletes an object from riak. Short hand for + bucket.new(key).delete(). See :meth:`RiakClient.delete() + ` for options. - Short hand for bucket.new(key).delete() :param key: The key for the object :type key: string :rtype: RiakObject @@ -454,7 +475,9 @@ def delete(self, key, **kwargs): def get_counter(self, key, **kwargs): """ - Gets the value of a counter stored in this bucket. + Gets the value of a counter stored in this bucket. See + :meth:`RiakClient.get_counter() + ` for options. :param key: the key of the counter :type key: string @@ -465,7 +488,10 @@ def get_counter(self, key, **kwargs): def update_counter(self, key, value, **kwargs): """ Updates the value of a counter stored in this bucket. Positive - values increment the counter, negative values decrement. + values increment the counter, negative values decrement. See + :meth:`RiakClient.update_counter() + ` for options. + :param key: the key of the counter :type key: string From 6ec6f6a619d46345f43f447041c545e17b652c6f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 12:32:47 -0500 Subject: [PATCH 0464/1060] Use the bootstrap theme. This requires installing `sphinx-bootstrap-theme` from PyPi. --- docs/conf.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index bbc2064f..4ea6f92b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ # serve to show the default. import sys, os +import sphinx_bootstrap_theme # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -81,7 +82,7 @@ #show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = 'friendly' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] @@ -91,15 +92,20 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'bootstrap' +# html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +html_theme_options = { + 'navbar_site_name':"Documentation", + 'globaltoc_depth': 2, + 'bootswatch_theme': 'cerulean' +} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +html_theme_path = sphinx_bootstrap_theme.get_html_theme_path() # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". From 6fecbfb5b191d2fde303f33a185c1feed77104d6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 15:04:56 -0500 Subject: [PATCH 0465/1060] Move contents into a floating well, rename/remove some files. --- docs/content.rst | 9 --------- docs/index.rst | 30 +++++++++++++++++++--------- docs/{riak_object.rst => object.rst} | 0 3 files changed, 21 insertions(+), 18 deletions(-) delete mode 100644 docs/content.rst rename docs/{riak_object.rst => object.rst} (100%) diff --git a/docs/content.rst b/docs/content.rst deleted file mode 100644 index 04880600..00000000 --- a/docs/content.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. ref-content: - -=========== -RiakContent -=========== - -.. currentmodule:: riak.content - -.. autoclass:: riak.content.RiakContent diff --git a/docs/index.rst b/docs/index.rst index 9b1ed90a..5fb8c0dd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,5 +1,18 @@ Riak Python Client -===================== +================== + +.. cssclass:: well pull-right +.. compound:: + + **Contents** + + .. toctree:: + :maxdepth: 2 + + client + bucket + object + mapreduce Installation ------------ @@ -16,16 +29,15 @@ Installation .. _easy_install: http://pypi.python.org/pypi/setuptools .. _PyPI: http://pypi.python.org/pypi/riak/ -Contents: +Development +----------- + +All development is done on Github_. Use Issues_ to report +problems or submit contributions. -.. toctree:: - :maxdepth: 2 +.. _Github: https://github.com/basho/riak-python-client/ +.. _Issues: https://github.com/basho/riak-python-client/issues - client - bucket - riak_object - content - mapreduce Indices and tables ------------------ diff --git a/docs/riak_object.rst b/docs/object.rst similarity index 100% rename from docs/riak_object.rst rename to docs/object.rst From 51379b605f0fcf4cfd97eac45d665a17c08f63d6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 22:02:32 -0500 Subject: [PATCH 0466/1060] Tweak some headers formatting. Also, fix a bug where deprecated quorum accessor docs weren't being formatted properly. --- docs/bucket.rst | 44 +++++++++++++++++++++----------------------- docs/client.rst | 36 ++++++++++++++++-------------------- riak/util.py | 2 +- 3 files changed, 38 insertions(+), 44 deletions(-) diff --git a/docs/bucket.rst b/docs/bucket.rst index 9aa81df7..8f0ea833 100644 --- a/docs/bucket.rst +++ b/docs/bucket.rst @@ -4,10 +4,6 @@ Buckets .. currentmodule:: riak.bucket --------- -Overview --------- - Buckets are both namespaces for the key-value pairs you store in Riak, and containers for properties that apply to that namespace. Buckets should be created via the :meth:`bucket() @@ -19,7 +15,7 @@ should be created via the :meth:`bucket() mybucket = client.bucket('mybucket') -------------- -RiakBucket API +Bucket objects -------------- .. autoclass:: RiakBucket @@ -30,9 +26,9 @@ RiakBucket API .. autoattribute:: resolver -^^^^^^^^^^^^^^^^^ +----------------- Bucket properties -^^^^^^^^^^^^^^^^^ +----------------- Bucket properties are flags and defaults that apply to all keys in the bucket. @@ -43,9 +39,9 @@ bucket. .. automethod:: RiakBucket.get_property .. automethod:: RiakBucket.set_property -""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Shortcuts for common properties -""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Some of the most commonly-used bucket properties are exposed as object properties as well. The getters and setters simply call @@ -61,9 +57,9 @@ respectively. .. autoattribute:: RiakBucket.pw .. autoattribute:: RiakBucket.rw -"""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^ Shortcuts for search -"""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^ When Riak Search is enabled on the server, you can toggle which buckets have automatic indexing turned on using the ``search`` bucket @@ -74,9 +70,9 @@ methods simplify interacting with that configuration. .. automethod:: RiakBucket.enable_search .. automethod:: RiakBucket.disable_search -^^^^^^^^^^^^^^^^^ +----------------- Working with keys -^^^^^^^^^^^^^^^^^ +----------------- The primary purpose of buckets is to act as namespaces for keys. As such, you can use the bucket object to create, fetch and delete @@ -88,9 +84,11 @@ such, you can use the bucket object to create, fetch and delete .. automethod:: RiakBucket.multiget .. automethod:: RiakBucket.delete -"""""""" +.. _counters: + +^^^^^^^^ Counters -"""""""" +^^^^^^^^ Rather than returning objects, the counter operations new to Riak 1.4 act directly on the value of the counter. @@ -98,18 +96,18 @@ act directly on the value of the counter. .. automethod:: RiakBucket.get_counter .. automethod:: RiakBucket.update_counter -^^^^^^^^^^^^^^^^ +---------------- Query operations -^^^^^^^^^^^^^^^^ +---------------- .. automethod:: RiakBucket.search .. automethod:: RiakBucket.get_index .. automethod:: RiakBucket.stream_index -^^^^^^^^^^^^^ +------------- Serialization -^^^^^^^^^^^^^ +------------- Similar to :class:`RiakClient `, buckets can register custom transformation functions for media-types. When @@ -123,9 +121,9 @@ with the bucket. .. automethod:: RiakBucket.set_decoder -^^^^^^^^^^^^ +------------ Listing keys -^^^^^^^^^^^^ +------------ Shortcuts for :meth:`RiakClient.get_keys() ` and @@ -136,9 +134,9 @@ object. The same admonitions for these operations apply. .. automethod:: RiakBucket.get_keys .. automethod:: RiakBucket.stream_keys -^^^^^^^^^^^^^^^^^^ +------------------ Deprecated methods -^^^^^^^^^^^^^^^^^^ +------------------ .. warning:: These methods exist solely for backwards-compatibility and should not be used unless code is being ported from an older version. diff --git a/docs/client.rst b/docs/client.rst index 531cfc1d..053dcd78 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -2,10 +2,6 @@ Client & Connections ==================== --------- -Overview --------- - To connect to a Riak cluster, you must create a :py:class:`~riak.client.RiakClient` object. The default configuration connects to a single Riak node on ``localhost`` with the default @@ -28,7 +24,7 @@ protocol. Connections are opened as-needed; a random node is selected when a new connection is requested. -------------- -RiakClient API +Client objects -------------- .. currentmodule:: riak.client @@ -66,9 +62,9 @@ the ``RiakClient`` constructor, they will be turned into this type. .. autoclass:: riak.node.RiakNode :members: -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- Client-level Operations -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- Some operations are not scoped by buckets and can be performed on the client directly: @@ -77,9 +73,9 @@ client directly: .. automethod:: RiakClient.get_buckets .. automethod:: RiakClient.stream_buckets -^^^^^^^^^^^^^^^^^ +----------------- Accessing Buckets -^^^^^^^^^^^^^^^^^ +----------------- Most client operations are on :py:class:`bucket objects ` or keys within those buckets. Use the @@ -88,9 +84,9 @@ the called client. .. automethod:: RiakClient.bucket -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- Bucket-level Operations -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- .. automethod:: RiakClient.get_bucket_props .. automethod:: RiakClient.set_bucket_props @@ -98,9 +94,9 @@ Bucket-level Operations .. automethod:: RiakClient.get_keys .. automethod:: RiakClient.stream_keys -^^^^^^^^^^^^^^^^^^^^ +-------------------- Key-level Operations -^^^^^^^^^^^^^^^^^^^^ +-------------------- .. automethod:: RiakClient.get .. automethod:: RiakClient.put @@ -109,9 +105,9 @@ Key-level Operations .. automethod:: RiakClient.get_counter .. automethod:: RiakClient.update_counter -^^^^^^^^^^^^^^^^ +---------------- Query Operations -^^^^^^^^^^^^^^^^ +---------------- .. automethod:: RiakClient.mapred .. automethod:: RiakClient.stream_mapred @@ -121,9 +117,9 @@ Query Operations .. automethod:: RiakClient.fulltext_add .. automethod:: RiakClient.fulltext_delete -^^^^^^^^^^^^^ +------------- Serialization -^^^^^^^^^^^^^ +------------- The client supports automatic transformation of Riak responses into Python types if encoders and decoders are registered for the @@ -137,9 +133,9 @@ media-types. Supported by default are ``application/json`` and .. automethod:: RiakClient.set_decoder -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Deprecated Methods and Properties -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------ +Deprecated Methods +------------------ .. warning:: These methods and attributes exist solely for backwards-compatibility and should not be used unless code is being diff --git a/riak/util.py b/riak/util.py index 9d371b92..c8f79777 100644 --- a/riak/util.py +++ b/riak/util.py @@ -117,7 +117,7 @@ def setter(self, value): :param value: the value to use if not set :type value: mixed - """ + """.format(quorum) setattr(klass, getter_name, getter) setattr(klass, setter_name, setter) From 3fbd8bf2c69512a20bf52b09323908ab398b75a1 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 22:03:18 -0500 Subject: [PATCH 0467/1060] WIP Object docs, with explanations of siblings and resolvers. --- docs/object.rst | 138 ++++++++++++++++++++++++++++++++++++++++++-- riak/content.py | 21 +++++-- riak/resolver.py | 12 ++-- riak/riak_object.py | 55 ++++++++++-------- 4 files changed, 186 insertions(+), 40 deletions(-) diff --git a/docs/object.rst b/docs/object.rst index b85fcbb1..4c9d015a 100644 --- a/docs/object.rst +++ b/docs/object.rst @@ -1,9 +1,137 @@ -.. ref-riak-object: +============== +Keys & Objects +============== -========== +.. currentmodule:: riak.riak_object + +Keys in Riak are namespaced into :class:`buckets +`, and their associated values are represented +by :class:`objects `, not to be confused with Python +"objects". A :class:`RiakObject` is a container for the key, the +:ref:`vclock`, the value(s) and any metadata associated with the +value(s). + +---------- RiakObject -========== +---------- -.. currentmodule:: riak.riak_object +.. autoclass:: RiakObject + + .. attribute:: key + + The key of this object, a string. If not present, the server + will generate a key the first time this object is stored. + + .. attribute:: bucket + + The :class:`bucket ` to which this + object belongs. + + .. autoattribute:: resolver + .. attribute:: vclock + + The :ref:`vclock` for this object. + + .. autoattribute:: exists + +.. _vclock: + +^^^^^^^^^^^^ +Vector clock +^^^^^^^^^^^^ + +Vector clocks are Riak's means of tracking the relationships between +writes to a key. It is best practice to fetch the latest version of a +key before attempting to modify or overwrite the value; if you do not, +you may create :ref:`siblings` or lose data! The content of a vector +clock is essentially opaque to the user. + +.. autoclass:: VClock + +----------- +Persistence +----------- + +Fetching, storing, and deleting keys are the bread-and-butter of Riak. + +.. automethod:: RiakObject.store +.. automethod:: RiakObject.reload +.. automethod:: RiakObject.delete + +.. _object_accessors: + +------------------ +Value and Metadata +------------------ + +Unless you have enabled :ref:`siblings` via the :attr:`allow_mult +` bucket property, you can +inspect and manipulate the value and metadata of an object directly using these +properties and methods: + +.. autoattribute:: RiakObject.data +.. autoattribute:: RiakObject.encoded_data +.. autoattribute:: RiakObject.content_type +.. autoattribute:: RiakObject.charset +.. autoattribute:: RiakObject.content_encoding +.. autoattribute:: RiakObject.last_modified +.. autoattribute:: RiakObject.etag +.. autoattribute:: RiakObject.usermeta +.. autoattribute:: RiakObject.links +.. autoattribute:: RiakObject.indexes +.. automethod:: RiakObject.add_index +.. automethod:: RiakObject.remove_index +.. automethod:: RiakObject.set_index +.. automethod:: RiakObject.add_link + +.. _siblings: + +-------- +Siblings +-------- + +Because Riak's consistency model is "eventual" (and not linearizable), +there is no way for it to disambiguate writes that happen +concurrently. The :ref:`vclock` helps establish a +"happens after" relationships so that concurrent writes can be +detected, but with the exception of :ref:`counters`, Riak has no way +to determine which write has the correct value. + +Instead, when :attr:`allow_mult ` +is ``True``, Riak keeps all writes that appear to be concurrent. Thus, +the contents of a key's value may, in fact, be multiple values, which +are called "siblings". Siblings are modeled in :class:`RiakContent +` objects, which contain all of the same +:ref:`object_accessors` methods and attributes as the parent object. + +.. autoattribute:: RiakObject.siblings + +.. autoclass:: riak.content.RiakContent + +You do not typically have to create :class:`RiakContent +` objects yourself, but they will be created +for you when :meth:`fetching ` objects from Riak. + +.. note:: The :ref:`object_accessors` accessors on :class:`RiakObject` + are actually proxied to the first sibling when the object has only + one. + + +^^^^^^^^^^^^^^^^^^^^^^^ +Conflicts and Resolvers +^^^^^^^^^^^^^^^^^^^^^^^ + +When an object is *not* in conflict, it has only one sibling. When it +is in conflict, you will have to resolve the conflict before it can be +written again. How you choose to resolve the conflict is up to you, +but you can automate the process using a :attr:`resolver +` function. + +.. autofunction:: riak.resolver.default_resolver +.. autofunction:: riak.resolver.last_written_resolver + +If you do not supply a resolver function, or your resolver leaves +multiple siblings present, accessing the :ref:`object_accessors` will +result in a :exc:`ConflictError ` being raised. -.. autoclass:: riak.riak_object.RiakObject +.. autoexception:: riak.ConflictError diff --git a/riak/content.py b/riak/content.py index dce93a4e..1768c219 100644 --- a/riak/content.py +++ b/riak/content.py @@ -109,6 +109,8 @@ def _deserialize(self, value): def add_index(self, field, value): """ + add_index(field, value) + Tag this object with the specified field/value pair for indexing. @@ -116,7 +118,7 @@ def add_index(self, field, value): :type field: string :param value: The index value. :type value: string or integer - :rtype: RiakObject + :rtype: :class:`RiakObject ` """ if field[-4:] not in ("_bin", "_int"): raise RiakError("Riak 2i fields must end with either '_bin'" @@ -128,6 +130,8 @@ def add_index(self, field, value): def remove_index(self, field=None, value=None): """ + remove_index(field=None, value=None) + Remove the specified field/value pair as an index on this object. @@ -135,7 +139,7 @@ def remove_index(self, field=None, value=None): :type field: string :param value: The index value. :type value: string or integer - :rtype: RiakObject + :rtype: :class:`RiakObject ` """ if not field and not value: self.indexes.clear() @@ -154,14 +158,17 @@ def remove_index(self, field=None, value=None): def set_index(self, field, value): """ - Works like add_index, but ensures that there is only one index - on given field. If other found, then removes it first. + set_index(field, value) + + Works like :meth:`add_index`, but ensures that there is only + one index on given field. If other found, then removes it + first. :param field: The index field. :type field: string :param value: The index value. :type value: string or integer - :rtype: RiakObject + :rtype: :class:`RiakObject ` """ to_rem = set((x for x in self.indexes if x[0] == field)) self.indexes.difference_update(to_rem) @@ -169,6 +176,8 @@ def set_index(self, field, value): def add_link(self, obj, tag=None): """ + add_link(obj, tag=None) + Add a link to a RiakObject. :param obj: Either a RiakObject or 3 item link tuple consisting @@ -177,7 +186,7 @@ def add_link(self, obj, tag=None): :param tag: Optional link tag. Defaults to bucket name. It is ignored if ``obj`` is a 3 item link tuple. :type tag: string - :rtype: RiakObject + :rtype: :class:`RiakObject ` """ if isinstance(obj, tuple): newlink = obj diff --git a/riak/resolver.py b/riak/resolver.py index 30bfcc73..c54779ca 100644 --- a/riak/resolver.py +++ b/riak/resolver.py @@ -20,12 +20,14 @@ def default_resolver(riak_object): """ The default conflict-resolution function, which does nothing. To - implement a resolver, define a function that sets the ``siblings`` - property on the passed ``RiakObject`` instance to a list - containing a single ``RiakContent`` object. + implement a resolver, define a function that sets the + :attr:`siblings ` property + on the passed :class:`RiakObject ` + instance to a list containing a single :class:`RiakContent + ` object. :param riak_object: an object-in-conflict that will be resolved - :type riak_object: RiakObject + :type riak_object: :class:`RiakObject ` """ pass @@ -36,7 +38,7 @@ def last_written_resolver(riak_object): recently-modified sibling by timestamp. :param riak_object: an object-in-conflict that will be resolved - :type riak_object: RiakObject + :type riak_object: :class:`RiakObject ` """ lm = lambda x: x.last_modified riak_object.siblings = [max(riak_object.siblings, key=lm), ] diff --git a/riak/riak_object.py b/riak/riak_object.py index b74d61c7..0c9573a9 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -58,6 +58,8 @@ def _delegate(self, *args, **kwargs): raise ConflictError() return getattr(self.siblings[0], name).__call__(*args, **kwargs) + _delegate.__doc__ = getattr(RiakContent, name).__doc__ + return _delegate @@ -125,6 +127,9 @@ def __init__(self, client, bucket, key=None): self.vclock = None self.siblings = [RiakContent(self)] + #: The list of sibling values contained in this object + siblings = [] + def __hash__(self): return hash((self.key, self.bucket, self.vclock)) @@ -146,7 +151,7 @@ def __ne__(self, other): this property will result in decoding the `encoded_data` property into Python values. The decoding is dependent on the `content_type` property and the bucket's registered decoders. - :type mixed """) + """) encoded_data = content_property('encoded_data', doc=""" The raw data stored in this object, essentially the encoded @@ -154,42 +159,42 @@ def __ne__(self, other): will result in encoding the `data` property into a string. The encoding is dependent on the `content_type` property and the bucket's registered encoders. - :type basestring""") + """) charset = content_property('charset', doc=""" - The character set of the encoded data - :type string""") + The character set of the encoded data as a string + """) content_type = content_property('content_type', doc=""" - The MIME media type of the encoded data - :type string""") + The MIME media type of the encoded data as a string + """) content_encoding = content_property('content_encoding', doc=""" The encoding (compression) of the encoded data. Valid values are identity, deflate, gzip - :type string""") + """) last_modified = content_property('last_modified', """ The UNIX timestamp of the modification time of this value. - :type float""") + """) etag = content_property('etag', """ A unique entity-tag for the value. - :type string""") + """) usermeta = content_property('usermeta', doc=""" - Arbitrary user-defined metadata, mapping strings to strings. - :type dict""") + Arbitrary user-defined metadata dict, mapping strings to strings. + """) links = content_property('links', doc=""" - A collection of bucket/key/tag 3-tuples representing links to - other keys. - :type set""") + A set of bucket/key/tag 3-tuples representing links to other + keys. + """) indexes = content_property('indexes', doc=""" The set of secondary index entries, consisting of index-name/value tuples - :type set""") + """) get_encoded_data = content_method('get_encoded_data') set_encoded_data = content_method('set_encoded_data') @@ -210,10 +215,9 @@ def _exists(self): return self.siblings[0].exists exists = property(_exists, None, doc=""" - Whether the object exists. This is only False when there are no - siblings (the object was not found), or the solitary sibling is - a tombstone. - :type bool + Whether the object exists. This is only ``False`` when there + are no siblings (the object was not found), or the solitary + sibling is a tombstone. """) def _get_resolver(self): @@ -233,8 +237,7 @@ def _set_resolver(self, value): resolver = property(_get_resolver, _set_resolver, doc= """The sibling-resolution function for this object. If the resolver is not set, the - bucket's resolver will be used. :type - callable""") + bucket's resolver will be used.""") def get_sibling(self, index): deprecated("RiakObject.get_sibling is deprecated, use the " @@ -267,7 +270,7 @@ def store(self, w=None, dw=None, pw=None, return_body=True, :type if_none_match: bool :param timeout: a timeout value in milliseconds :type timeout: int - :rtype: RiakObject """ + :rtype: :class:`RiakObject` """ if len(self.siblings) != 1: raise ConflictError("Attempting to store an invalid object, " "resolve the siblings first") @@ -285,6 +288,10 @@ def reload(self, r=None, pr=None, timeout=None): object could contain new metadata and a new value, if the object was updated in Riak since it was last retrieved. + .. note:: Even if the key is not found in Riak, this will + return a :class:`RiakObject`. Check the :attr:`exists` + property to see if the key was found. + :param r: R-Value, wait for this many partitions to respond before returning to client. :type r: integer @@ -294,7 +301,7 @@ def reload(self, r=None, pr=None, timeout=None): :type pr: integer :param timeout: a timeout value in milliseconds :type timeout: int - :rtype: RiakObject + :rtype: :class:`RiakObject` """ self.client.get(self, r=r, pr=pr, timeout=timeout) @@ -327,7 +334,7 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None, :type pw: integer :param timeout: a timeout value in milliseconds :type timeout: int - :rtype: RiakObject + :rtype: :class:`RiakObject` """ self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw, From 6dcc74de347a504a4598cdc97d74b04050539458 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 26 Jul 2013 11:05:28 -0500 Subject: [PATCH 0468/1060] Make some template tweaks. --- docs/_static/custom.css | 17 +++++++++++++ docs/_templates/layout.html | 49 +++++++++++++++++++++++++++++++++++++ docs/conf.py | 9 ++++--- 3 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 docs/_static/custom.css create mode 100644 docs/_templates/layout.html diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 00000000..d8a66dbe --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,17 @@ +div.admonition p { + margin: 0; +} + +p.admonition-title { + float: left; + margin-right: 0.5em ! important; +} + +p.admonition-title:after { + content: ":"; + font-weight: bold; +} + +div.alert-info a { + color: #555; +} \ No newline at end of file diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 00000000..91c92523 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,49 @@ +{% extends "!layout.html" %} + +{% block sidebarrel %}{% endblock %} + +{%- block footer %} +
    + +
    +
    +
    +

    + Back to top + {% if theme_source_link_position == "footer" %} +
    + {% include "sourcelink.html" %} + {% endif %} +

    +

    + {%- if show_copyright %} + {%- if hasdoc('copyright') %} + {% trans path=pathto('copyright'), copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %}
    + {%- else %} + {% trans copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %}
    + {%- endif %} + {%- endif %} + {%- if last_updated %} + {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %}
    + {%- endif %} + {%- if show_sphinx %} + {% trans sphinx_version=sphinx_version|e %}Created using Sphinx {{ sphinx_version }}.{% endtrans %}
    + {%- endif %} +

    +
    +
    +{%- endblock %} + + +{% set css_files = css_files + ['_static/custom.css'] %} diff --git a/docs/conf.py b/docs/conf.py index 4ea6f92b..38b7abc2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -82,7 +82,7 @@ #show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'friendly' +pygments_style = 'tango' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] @@ -99,9 +99,10 @@ # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - 'navbar_site_name':"Documentation", + 'bootswatch_theme': 'cerulean', + 'navbar_site_name':"Docs", 'globaltoc_depth': 2, - 'bootswatch_theme': 'cerulean' + 'source_link_position':'footer' } # Add any paths that contain custom themes here, relative to this directory. @@ -126,7 +127,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -#html_static_path = ['_static'] +html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. From f31b4c3b586461f815404f67e006841aaa915db7 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 26 Jul 2013 15:45:27 -0500 Subject: [PATCH 0469/1060] Add "Advanced" page with all the miscellaneous and implementation details. --- docs/advanced.rst | 142 ++++++++++++++++++++++++++++++ docs/index.rst | 1 + riak/benchmark.py | 8 +- riak/client/multiget.py | 31 +++++-- riak/client/transport.py | 2 + riak/transports/feature_detect.py | 51 ++++++++--- riak/transports/pool.py | 26 +++--- riak/transports/transport.py | 3 +- riak/util.py | 8 +- 9 files changed, 233 insertions(+), 39 deletions(-) create mode 100644 docs/advanced.rst diff --git a/docs/advanced.rst b/docs/advanced.rst new file mode 100644 index 00000000..8bb3a9dc --- /dev/null +++ b/docs/advanced.rst @@ -0,0 +1,142 @@ +============================ + Advanced Usage & Internals +============================ + +This page contains documentation for aspects of library internals that +you will rarely need to interact with, but are important for +understanding how it works and development purposes. + +--------------- +Connection pool +--------------- + +.. currentmodule:: riak.transports.pool + +.. autoexception:: BadResource +.. autoclass:: Element + :members: +.. autoclass:: Pool + :members: + +.. autoclass:: PoolIterator + :members: + :special-members: + +----------- +Retry logic +----------- + +.. currentmodule:: riak.client.transport + +.. autoclass:: RiakClientTransport + :members: + :private-members: + +.. autofunction:: _is_retryable + +.. autofunction:: retryable + +.. autofunction:: retryableHttpOnly + +-------- +Multiget +-------- + +.. currentmodule:: riak.client.multiget + +.. autodata:: POOL_SIZE + +.. autoclass:: Task + +.. autoclass:: MultiGetPool + :members: + :private-members: + +.. autodata:: RIAK_MULTIGET_POOL + +.. autofunction:: multiget + +---------- +Transports +---------- + +.. currentmodule:: riak.transports.transport + +.. autoclass:: RiakTransport + :members: + :private-members: + +.. currentmodule:: riak.transports.feature_detect + +.. autoclass:: FeatureDetection + :members: + :private-members: + +^^^^^^^^^^^^^^ +HTTP Transport +^^^^^^^^^^^^^^ + +.. currentmodule:: riak.transports.http + +.. autoclass:: RiakHttpPool + +.. autofunction:: is_retryable + +.. autoclass:: RiakHttpTransport + :members: + +^^^^^^^^^^^^^^^^^^^^^^^^^^ +Protocol Buffers Transport +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. currentmodule:: riak.transports.pbc + +.. autoclass:: RiakPbcTransport + :members: + +--------- +Utilities +--------- + +^^^^^^^^^^^^^^^^^ +Multi-valued Dict +^^^^^^^^^^^^^^^^^ + +.. currentmodule:: riak.multidict + +.. autoclass:: MultiDict + + .. automethod:: add + .. automethod:: getall + .. automethod:: getone + .. automethod:: mixed + .. automethod:: dict_of_lists + +^^^^^^^^^^^^^^^^^^ +Micro-benchmarking +^^^^^^^^^^^^^^^^^^ + +.. currentmodule:: riak.benchmark + +.. autofunction:: measure + +.. autofunction:: measure_with_rehearsal + +.. autoclass:: Benchmark + :members: + +^^^^^^^^^^^^^ +Miscellaneous +^^^^^^^^^^^^^ + +.. currentmodule:: riak.util + +.. autofunction:: quacks_like_dict + +.. autofunction:: deep_merge + +.. autofunction:: deprecated + +.. autofunction:: deprecateQuorumAccessors + +.. autoclass:: lazy_property diff --git a/docs/index.rst b/docs/index.rst index 5fb8c0dd..43589f77 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,6 +13,7 @@ Riak Python Client bucket object mapreduce + advanced Installation ------------ diff --git a/riak/benchmark.py b/riak/benchmark.py index c3eade41..7372a68c 100644 --- a/riak/benchmark.py +++ b/riak/benchmark.py @@ -25,9 +25,9 @@ def measure_with_rehearsal(): """ Runs a benchmark when used as an iterator, injecting a garbage - collection between iterations. Example: + collection between iterations. Example:: - for b in benchmark.measure_with_rehearsal(): + for b in riak.benchmark.measure_with_rehearsal(): with b.report("pow"): for _ in range(10000): math.pow(2,10000) @@ -40,9 +40,9 @@ def measure_with_rehearsal(): def measure(): """ - Runs a benchmark once when used as a context manager. Example: + Runs a benchmark once when used as a context manager. Example:: - with benchmark.measure() as b: + with riak.benchmark.measure() as b: with b.report("pow"): for _ in range(10000): math.pow(2,10000) diff --git a/riak/client/multiget.py b/riak/client/multiget.py index 665eba77..1af248d4 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -25,15 +25,17 @@ try: + #: The default size of the worker pool, either based on the number + #: of CPUS or defaulting to 6 POOL_SIZE = cpu_count() except NotImplementedError: # Make an educated guess POOL_SIZE = 6 - +#: A :class:`namedtuple` for tasks that are fed to workers in the +#: multiget pool. Task = namedtuple('Task', ['client', 'outq', 'bucket', 'key', 'options']) - class MultiGetPool(object): """ Encapsulates a pool of fetcher threads. These threads can be used @@ -41,6 +43,11 @@ class MultiGetPool(object): """ def __init__(self, size=POOL_SIZE): + """ + :param size: the desired size of the worker pool + :type size: int + """ + self._inq = Queue() self._size = size self._started = Event() @@ -110,7 +117,10 @@ def __del__(self): def _fetcher(self): """ - The body of the multi-get worker. + The body of the multi-get worker. Loops until + :meth:`_should_quit` returns ``True``, taking tasks off the + input queue, fetching the object, and putting them on the + output queue. """ while not self._should_quit(): task = self._inq.get() @@ -131,21 +141,26 @@ def _should_quit(self): input queue is empty. Once the stop flag is set, new enqueues are disallowed, meaning that the workers can safely drain the queue before exiting. - :rtype boolean + :rtype: boolean """ return self.stopped() and self._inq.empty() +#: The default pool is automatically created and stored in this constant. RIAK_MULTIGET_POOL = MultiGetPool() def multiget(client, keys, **options): """ Executes a parallel-fetch across multiple threads. Returns a list - containing RiakObject instances, or 3-tuples of bucket, key, and - the exception raised. - - :rtype list + containing :class:`~riak.riak_object.RiakObject` instances, or + 3-tuples of bucket, key, and the exception raised. + + :param client: the client to use + :type client: :class:`~riak.client.RiakClient` + :param keys: the bucket/key pairs to fetch in parallel + :type keys: list of two-tuples -- bucket/key pairs + :rtype: list """ outq = Queue() diff --git a/riak/client/transport.py b/riak/client/transport.py index eb2b7481..ccef4813 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -37,6 +37,8 @@ class RiakClientTransport(object): @contextmanager def _transport(self): """ + _transport() + Yields a single transport to the caller from the default pool, without retries. """ diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index ab31d886..fcbc43b1 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -29,18 +29,30 @@ class FeatureDetection(object): + """ + Implements boolean methods that can be checked for the presence of + specific server-side features. Subclasses must implement the + :meth:`_server_version` method to use this functionality, which + should return the server's version as a string. + + :class:`FeatureDetection` is a parent class of + :class:`RiakTransport `. + """ + def _server_version(self): """ Gets the server version from the server. To be implemented by the individual transport class. - :rtype string + + :rtype: string """ raise NotImplementedError def phaseless_mapred(self): """ Whether MapReduce requests can be submitted without phases. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.1] @@ -49,14 +61,15 @@ def pb_indexes(self): Whether secondary index queries are supported over Protocol Buffers - :rtype bool + :rtype: bool """ return self.server_version >= versions[1.2] def pb_search(self): """ Whether search queries are supported over Protocol Buffers - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.2] @@ -64,7 +77,8 @@ def pb_conditionals(self): """ Whether conditional fetch/store semantics are supported over Protocol Buffers - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1] @@ -72,14 +86,16 @@ def quorum_controls(self): """ Whether additional quorums and FSM controls are available, e.g. primary quorums, basic_quorum, notfound_ok - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1] def tombstone_vclocks(self): """ Whether 'not found' responses might include vclocks - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1] @@ -87,7 +103,8 @@ def pb_head(self): """ Whether partial-fetches (vclock and metadata only) are supported over Protocol Buffers - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1] @@ -95,7 +112,8 @@ def pb_clear_bucket_props(self): """ Whether bucket properties can be cleared over Protocol Buffers. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.4] @@ -103,35 +121,40 @@ def pb_all_bucket_props(self): """ Whether all normal bucket properties are supported over Protocol Buffers. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.4] def counters(self): """ Whether CRDT counters are supported. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.4] def bucket_stream(self): """ Whether streaming bucket lists are supported. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.4] def client_timeouts(self): """ Whether client-supplied timeouts are supported. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.4] def stream_indexes(self): """ Whether secondary indexes support streaming responses. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.4] diff --git a/riak/transports/pool.py b/riak/transports/pool.py index 13d5cdf7..b150808a 100644 --- a/riak/transports/pool.py +++ b/riak/transports/pool.py @@ -23,29 +23,33 @@ # This file is a rough port of the Innertube Ruby library class BadResource(StandardError): """ - Users of a Pool should raise this error when the pool element - currently in-use is bad and should be removed from the pool. + Users of a :class:`Pool` should raise this error when the pool + element currently in-use is bad and should be removed from the + pool. """ pass class Element(object): """ - A member of the Pool, a container for the actual resource being - pooled and a marker for whether the resource is currently claimed. + A member of the :class:`Pool`, a container for the actual resource + being pooled and a marker for whether the resource is currently + claimed. """ def __init__(self, obj): """ Creates a new Element, wrapping the passed object as the pooled resource. + :param obj: the resource to wrap :type obj: object """ - """The wrapped pool resource.""" self.object = obj - """Whether the resource is currently in use.""" + """The wrapped pool resource.""" + self.claimed = False + """Whether the resource is currently in use.""" class Pool(object): @@ -59,7 +63,7 @@ class Pool(object): method also allows filtering of the pool and supplying a default value to be used as the resource if no elements are free. - Example: + Example:: from riak.Pool import Pool, BadResource class ListPool(Pool): @@ -80,7 +84,7 @@ def destroy_resource(self): def __init__(self): """ Creates a new Pool. This should be called manually if you - override the __init__ method in a subclass. + override the :meth:`__init__` method in a subclass. """ self.lock = threading.RLock() self.releaser = threading.Condition(self.lock) @@ -89,6 +93,8 @@ def __init__(self): @contextmanager def take(self, _filter=None, default=None): """ + take(_filter=None, default=None) + Claims a resource from the pool for use in a thread-safe, reentrant manner (as part of a with statement). Resources are created as needed when all members of the pool are claimed or @@ -98,7 +104,7 @@ def take(self, _filter=None, default=None): of the pool :type _filter: callable :param default: a value that will be used instead of calling - create_resource if a new resource needs to be created + :meth:`create_resource` if a new resource needs to be created """ if not _filter: def _filter(obj): @@ -151,7 +157,7 @@ def __iter__(self): def clear(self): """ - Removes all resources from the pool, calling delete_element + Removes all resources from the pool, calling :meth:`delete_element` with each one so that the resources are cleaned up. """ for element in self: diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 6a98f2bf..7feb087c 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -28,7 +28,8 @@ class RiakTransport(FeatureDetection): """ - Class to encapsulate transport details + Class to encapsulate transport details and methods. All protocol + transports are subclasses of this class. """ def _get_client_id(self): diff --git a/riak/util.py b/riak/util.py index c8f79777..4b69bfd7 100644 --- a/riak/util.py +++ b/riak/util.py @@ -55,6 +55,9 @@ def deep_merge(a, b): def deprecated(message, stacklevel=3): + """ + Prints a deprecation warning to the console. + """ warnings.warn(message, UserWarning, stacklevel=stacklevel) QUORUMS = ['r', 'pr', 'w', 'dw', 'pw', 'rw'] @@ -125,8 +128,9 @@ def setter(self, value): class lazy_property(object): ''' - meant to be used for lazy evaluation of an object attribute. - property should represent non-mutable data, as it replaces itself. + A method decorator meant to be used for lazy evaluation and + memoization of an object attribute. The property should represent + immutable data, as it replaces itself on first access. ''' def __init__(self, fget): From a6984188839578b7b7c3bcf2e89ebf22e0884800 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 29 Jul 2013 10:48:52 -0500 Subject: [PATCH 0470/1060] Add query page. * Remove mapreduce page, merge and expand into query page. * Move `RiakLink` to advanced page. * Fix some docstrings in mapreduce and index_page. --- docs/advanced.rst | 6 ++ docs/index.rst | 2 +- docs/mapreduce.rst | 15 --- docs/query.rst | 186 ++++++++++++++++++++++++++++++++++++++ riak/client/index_page.py | 9 +- riak/mapreduce.py | 44 +++++---- 6 files changed, 226 insertions(+), 36 deletions(-) delete mode 100644 docs/mapreduce.rst create mode 100644 docs/query.rst diff --git a/docs/advanced.rst b/docs/advanced.rst index 8bb3a9dc..5bfcfb34 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -98,6 +98,12 @@ Protocol Buffers Transport Utilities --------- +^^^^^^^^^^^^^^^^^^ +Link wrapper class +^^^^^^^^^^^^^^^^^^ + +.. autoclass:: riak.mapreduce.RiakLink + ^^^^^^^^^^^^^^^^^ Multi-valued Dict ^^^^^^^^^^^^^^^^^ diff --git a/docs/index.rst b/docs/index.rst index 43589f77..62ae5571 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,7 +12,7 @@ Riak Python Client client bucket object - mapreduce + query advanced Installation diff --git a/docs/mapreduce.rst b/docs/mapreduce.rst deleted file mode 100644 index 19f5bb57..00000000 --- a/docs/mapreduce.rst +++ /dev/null @@ -1,15 +0,0 @@ -.. ref-mapreduce: - -============= -RiakMapReduce -============= - -.. currentmodule:: riak.mapreduce - -.. autoclass:: riak.mapreduce.RiakMapReduce - -.. autoclass:: riak.mapreduce.RiakMapReducePhase - -.. autoclass:: riak.mapreduce.RiakLinkPhase - -.. autoclass:: riak.mapreduce.RiakLink diff --git a/docs/query.rst b/docs/query.rst new file mode 100644 index 00000000..22fad8e3 --- /dev/null +++ b/docs/query.rst @@ -0,0 +1,186 @@ +============= +Query Methods +============= + +Although most operations you will do involve directly interacting with +known buckets and keys, there are additional ways to get information +out of Riak. + +----------------- +Secondary Indexes +----------------- + +:ref:`Objects ` can be :meth:`tagged +` with :attr:`secondary index +entries `. Those entries can then +be queried over :meth:`the bucket ` +for equality or across ranges.:: + + bucket = client.bucket("index_test") + + # Tag an object with indexes and save + sean = bucket.new("seancribbs") + sean.add_index("fname_bin", "Sean") + sean.add_index("byear_int", 1979) + sean.store() + + # Performs an equality query + seans = bucket.get_index("fname_bin", "Sean") + + # Performs a range query + eighties = bucket.get_index("byear_int", 1980, 1989) + +Secondary indexes are also available via :meth:`MapReduce +`. + +^^^^^^^^^^^^^^^^^^ +Riak 1.4+ Features +^^^^^^^^^^^^^^^^^^ + +.. note:: The features below will raise ``NotImplementedError`` if + requested against a server that does not support them. + +Sometimes the number of results from such a query is too great to +process in one payload, so you can also :meth:`stream the results +`:: + + for keys in bucket.stream_index("bmonth_int", 1): + # keys is a list of matching keys + print keys + +Both the regular :meth:`~riak.bucket.RiakBucket.get_index` method and +the :meth:`~riak.bucket.RiakBucket.stream_index` method allow you to +return the index entry along with the matching key as tuples using the +``return_terms`` option:: + + bucket.get_index("byear_int", 1970, 1990, return_terms=True) + # => [(1979, 'seancribbs')] + +You can also limit the number of results using the ``max_results`` +option, which enables pagination:: + + results = bucket.get_index("fname_bin", "S", "T", max_results=20) + +All of these features are implemented using the +:class:`~riak.client.index_page.IndexPage` class, which emulates a +list but also supports streaming and capturing the +:attr:`~riak.client.index_page.IndexPage.continuation`, which is a +sort of pointer to the next page of results:: + + # Detect whether there are more results + if results.has_next_page(): + + # Fetch the next page of results manually + more = bucket.get_index("fname_bin", "S", "T", max_results=20, + continuation=results.continuation) + + # Fetch the next page of results automatically + more = results.next_page() + +.. currentmodule:: riak.client.index_page + +.. autoclass:: IndexPage + + .. autoattribute:: continuation + .. automethod:: has_next_page + .. automethod:: next_page + .. automethod:: __eq__ + .. automethod:: __iter__ + .. automethod:: __getitem__ + +--------------- +Fulltext Search +--------------- + +If Riak Search is enabled, you can query an index via the bucket's +:meth:`~riak.bucket.RiakBucket.search` method:: + + bucket.enable_search() + bucket.new("one", data={'value':'one'}, + content_type="application/json").store() + + bucket.search('value=one') + +To manually add and remove documents from an index (without an +associated key), use the :class:`~riak.client.RiakClient` +:meth:`~riak.client.RiakClient.fulltext_add` and +:meth:`~riak.client.RiakClient.fulltext_delete` methods directly. + +--------- +MapReduce +--------- + +.. currentmodule:: riak.mapreduce + +:class:`MapReduce` allows you to construct query-processing jobs that +are performed mostly in-parallel around the Riak cluster. You can +think of it as a pipeline, where inputs are fed in one end, they pass +through a number of ``map`` and ``reduce`` phases, and then are +returned to the client. + +^^^^^^^^^^^^^^^^^^^^^^ +Constructing the query +^^^^^^^^^^^^^^^^^^^^^^ + +.. autoclass:: RiakMapReduce + +^^^^^^ +Inputs +^^^^^^ + +The first step is to identify the inputs that should be processed. +They can be: + +#. An entire :meth:`bucket ` +#. An entire bucket, with the :meth:`keys filtered by criteria ` +#. A :meth:`list of bucket/key pairs ` or bucket/key/data triples +#. A :meth:`fulltext search query ` +#. A :meth:`secondary-index query ` + +Adding inputs always returns the ``RiakMapReduce`` object so that you +can chain the construction of the query job. + +.. automethod:: RiakMapReduce.add_bucket +.. automethod:: RiakMapReduce.add_key_filters +.. automethod:: RiakMapReduce.add_key_filter + +.. automethod:: RiakMapReduce.add +.. automethod:: RiakMapReduce.add_object +.. automethod:: RiakMapReduce.add_bucket_key_data + +.. automethod:: RiakMapReduce.search +.. automethod:: RiakMapReduce.index + +^^^^^^ +Phases +^^^^^^ + +The second step is to add processing phases to the query. ``map`` +phases load and process individual keys, returning one or more +results, while ``reduce`` phases operate over collections of results +from previous phases. ``link`` phases are a special type of ``map`` +phase that extract matching :attr:`~riak.riak_object.RiakObject.links` +from the object, usually so they can be used in a subsequent ``map`` +phase. + +Any number of phases can return results directly to the client by +passing ``keep=True``. + +.. automethod:: RiakMapReduce.map +.. automethod:: RiakMapReduce.reduce +.. automethod:: RiakMapReduce.link + +.. autoclass:: RiakMapReducePhase + +.. autoclass:: RiakLinkPhase + +^^^^^^^^^ +Execution +^^^^^^^^^ + +Query results can either be executed in one round-trip, or streamed +back to the client. The format of results will depend on the structure +of the ``map`` and ``reduce`` phases the query contains. + +.. automethod:: RiakMapReduce.run +.. automethod:: RiakMapReduce.stream diff --git a/riak/client/index_page.py b/riak/client/index_page.py index 5314f4db..b4c26e27 100644 --- a/riak/client/index_page.py +++ b/riak/client/index_page.py @@ -43,9 +43,16 @@ def __init__(self, client, bucket, index, startkey, endkey, return_terms, self.return_terms = return_terms self.max_results = max_results self.results = None - self.continuation = None self.stream = False + continuation = None + """ + The opaque page marker that is used when fetching the next chunk + of results. The user can simply call :meth:`next_page` to do so, + or pass this to the :meth:`~riak.client.RiakClient.get_index` + method using the ``continuation`` option. + """ + def __iter__(self): """ Emulates the iterator interface. When streaming, this means diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 5792aab7..17f3eab3 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -21,6 +21,8 @@ from collections import Iterable, namedtuple from riak import RiakError +#: Links are just bucket/key/tag tuples, this class provides a +#: backwards-compatible format: ``RiakLink(bucket, key, tag)`` RiakLink = namedtuple("RiakLink", ("bucket", "key", "tag")) @@ -34,8 +36,9 @@ class RiakMapReduce(object): def __init__(self, client): """ Construct a Map/Reduce object. - :param client: A RiakClient object. - :type client: RiakClient + + :param client: the client that will perform the query + :type client: :class:`~riak.client.RiakClient` """ self._client = client self._phases = [] @@ -57,7 +60,7 @@ def add(self, arg1, arg2=None, arg3=None): :type arg2: string, list, None :param arg3: key data for this input (must be convertible to JSON) :type arg3: string, list, dict, None - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if (arg2 is None) and (arg3 is None): if isinstance(arg1, RiakObject): @@ -73,7 +76,7 @@ def add_object(self, obj): :param obj: the object to add :type obj: RiakObject - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ return self.add_bucket_key_data(obj._bucket._name, obj._key, None) @@ -87,7 +90,7 @@ def add_bucket_key_data(self, bucket, key, data): :type key: string :param data: the key-specific data :type data: string, list, dict, None - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if self._input_mode == 'bucket': raise ValueError('Already added a bucket, can\'t add an object.') @@ -108,7 +111,7 @@ def add_bucket(self, bucket): :param bucket: the bucket :type bucket: string - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ self._input_mode = 'bucket' self._inputs = bucket @@ -120,7 +123,7 @@ def add_key_filters(self, key_filters): :param key_filters: a list of filters :type key_filters: list - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if self._input_mode == 'query': raise ValueError('Key filters are not supported in a query.') @@ -134,7 +137,7 @@ def add_key_filter(self, *args): :param args: a filter :type args: list - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if self._input_mode == 'query': raise ValueError('Key filters are not supported in a query.') @@ -151,7 +154,7 @@ def search(self, bucket, query): :type bucket: string :param query: The search query :type query: string - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ self._input_mode = 'query' self._inputs = {'module': 'riak_search', @@ -173,6 +176,7 @@ def index(self, bucket, index, startkey, endkey=None): :type startkey: string, integer :param endkey: The end key of index range (if doing a range query) :type endkey: string, integer, None + :rtype: :class:`RiakMapReduce` """ self._input_mode = 'query' @@ -192,7 +196,7 @@ def link(self, bucket='_', tag='_', keep=False): Add a link phase to the map/reduce operation. :param bucket: Bucket name (default '_', which means all - buckets) + buckets) :type bucket: string :param tag: Tag (default '_', which means any tag) :type tag: string @@ -200,7 +204,7 @@ def link(self, bucket='_', tag='_', keep=False): the map/reduce. (default False, unless this is the last step in the phase) :type keep: boolean - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ self._phases.append(RiakLinkPhase(bucket, tag, keep)) return self @@ -217,7 +221,7 @@ def map(self, function, options=None): :param options: phase options, containing 'language', 'keep' flag, and/or 'arg'. :type options: dict - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if options is None: options = dict() @@ -245,7 +249,7 @@ def reduce(self, function, options=None): :type function: string, list :param options: phase options, containing 'language', 'keep' flag, and/or 'arg'. - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if options is None: options = dict() @@ -265,8 +269,8 @@ def reduce(self, function, options=None): def run(self, timeout=None): """ Run the map/reduce operation synchronously. Returns a list of - results, or a list of links if the last phase is a - link phase. + results, or a list of links if the last phase is a link phase. + Shortcut for :meth:`riak.client.RiakClient.mapred`. :param timeout: Timeout in milliseconds :type timeout: integer, None @@ -309,11 +313,12 @@ def run(self, timeout=None): def stream(self, timeout=None): """ - Streams the MapReduce query (returns an iterator). + Streams the MapReduce query (returns an iterator). Shortcut + for :meth:`riak.client.RiakClient.stream_mapred`. :param timeout: Timeout in milliseconds :type timeout: integer - :rtype: iterator + :rtype: iterator that yields (phase_num, data) tuples """ query, lrf = self._normalize_query() return self._client.stream_mapred(self._inputs, query, timeout) @@ -569,13 +574,14 @@ class RiakLinkPhase(object): map/reduce operation. Normally you won't need to use this object directly, but instead - call ``link`` on RiakMapReduce objects to add instances to the - query. + call :meth:`RiakMapReduce.link` on RiakMapReduce objects to add + instances to the query. """ def __init__(self, bucket, tag, keep): """ Construct a RiakLinkPhase object. + :param bucket: - The bucket name :type bucket: string :param tag: The tag From 51169f598420e37ed0d691c5944b046f74d12087 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 29 Jul 2013 10:49:18 -0500 Subject: [PATCH 0471/1060] Tweak layout to put pagers at top and bottom. --- docs/_templates/layout.html | 41 +++++++++++++------------------------ 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 91c92523..31ae631c 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -2,7 +2,8 @@ {% block sidebarrel %}{% endblock %} -{%- block footer %} +{%- block content %} +{{ navBar() }}
      {%- if prev %} @@ -16,34 +17,20 @@ title="{{ _('next chapter') }}">{{ next.title }}{{"→"|safe}} {%- endif %}
    -
    -
    -
    -

    - Back to top - {% if theme_source_link_position == "footer" %} -
    - {% include "sourcelink.html" %} - {% endif %} -

    -

    - {%- if show_copyright %} - {%- if hasdoc('copyright') %} - {% trans path=pathto('copyright'), copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %}
    - {%- else %} - {% trans copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %}
    - {%- endif %} - {%- endif %} - {%- if last_updated %} - {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %}
    + {% block body %}{% endblock %} +

    -
    + + {%- endblock %} - {% set css_files = css_files + ['_static/custom.css'] %} From 7507fb8f25e629fe6ad0c0723024a0c7628e3693 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 29 Jul 2013 10:50:56 -0500 Subject: [PATCH 0472/1060] Fix some PEP8 and pyflakes things. --- riak/client/multiget.py | 1 + riak/util.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/riak/client/multiget.py b/riak/client/multiget.py index 1af248d4..ac096189 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -36,6 +36,7 @@ #: multiget pool. Task = namedtuple('Task', ['client', 'outq', 'bucket', 'key', 'options']) + class MultiGetPool(object): """ Encapsulates a pool of fetcher threads. These threads can be used diff --git a/riak/util.py b/riak/util.py index 4b69bfd7..20f6d129 100644 --- a/riak/util.py +++ b/riak/util.py @@ -85,16 +85,16 @@ def __deprecateQuorumAccessor(klass, parent, quorum): if not parent: def direct_getter(self, value=None): deprecated(QDEPMESSAGE % klass.__name__) - if val: - return val + if value: + return value return getattr(self, propname, "default") getter = direct_getter else: def parent_getter(self, value=None): deprecated(QDEPMESSAGE % klass.__name__) - if val: - return val + if value: + return value parentInstance = getattr(self, parent) return getattr(self, propname, getattr(parentInstance, propname, "default")) From d59a5d760bb2cb148042a4c0fff099b4c8b2da29 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 29 Jul 2013 11:14:14 -0500 Subject: [PATCH 0473/1060] Document some of the mapred shortcut methods. --- docs/query.rst | 35 +++++++++++++++++++++++++++++++++++ riak/riak_object.py | 16 ++++++++-------- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/docs/query.rst b/docs/query.rst index 22fad8e3..40587b3c 100644 --- a/docs/query.rst +++ b/docs/query.rst @@ -151,6 +151,8 @@ can chain the construction of the query job. .. automethod:: RiakMapReduce.search .. automethod:: RiakMapReduce.index +.. autoclass:: RiakKeyFilter + ^^^^^^ Phases ^^^^^^ @@ -174,6 +176,24 @@ passing ``keep=True``. .. autoclass:: RiakLinkPhase +""""""""""""""" +Phase shortcuts +""""""""""""""" + +A number of commonly-used phases are also available as shortcut +methods: + +.. automethod:: RiakMapReduce.map_values +.. automethod:: RiakMapReduce.map_values_json +.. automethod:: RiakMapReduce.reduce_sum +.. automethod:: RiakMapReduce.reduce_min +.. automethod:: RiakMapReduce.reduce_max +.. automethod:: RiakMapReduce.reduce_sort +.. automethod:: RiakMapReduce.reduce_numeric_sort +.. automethod:: RiakMapReduce.reduce_limit +.. automethod:: RiakMapReduce.reduce_slice +.. automethod:: RiakMapReduce.filter_not_found + ^^^^^^^^^ Execution ^^^^^^^^^ @@ -184,3 +204,18 @@ of the ``map`` and ``reduce`` phases the query contains. .. automethod:: RiakMapReduce.run .. automethod:: RiakMapReduce.stream + +^^^^^^^^^^^^^^^^^^^^^ +Shortcut constructors +^^^^^^^^^^^^^^^^^^^^^ + +:class:`~riak.riak_object.RiakObject` contains some shortcut methods +that make it more convenient to begin constructing +:class:`RiakMapReduce` queries. + +.. currentmodule:: riak.riak_object + +.. automethod:: RiakObject.add +.. automethod:: RiakObject.link +.. automethod:: RiakObject.map +.. automethod:: RiakObject.reduce diff --git a/riak/riak_object.py b/riak/riak_object.py index 0c9573a9..7115a9c0 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -354,9 +354,9 @@ def clear(self): def add(self, *args): """ Start assembling a Map/Reduce operation. - A shortcut for :func:`RiakMapReduce.add`. + A shortcut for :meth:`~riak.mapreduce.RiakMapReduce.add`. - :rtype: RiakMapReduce + :rtype: :class:`~riak.mapreduce.RiakMapReduce` """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) @@ -365,9 +365,9 @@ def add(self, *args): def link(self, *args): """ Start assembling a Map/Reduce operation. - A shortcut for :func:`RiakMapReduce.link`. + A shortcut for :meth:`~riak.mapreduce.RiakMapReduce.link`. - :rtype: RiakMapReduce + :rtype: :class:`~riak.mapreduce.RiakMapReduce` """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) @@ -376,9 +376,9 @@ def link(self, *args): def map(self, *args): """ Start assembling a Map/Reduce operation. - A shortcut for :func:`RiakMapReduce.map`. + A shortcut for :meth:`~riak.mapreduce.RiakMapReduce.map`. - :rtype: RiakMapReduce + :rtype: :class:`~riak.mapreduce.RiakMapReduce` """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) @@ -387,9 +387,9 @@ def map(self, *args): def reduce(self, *args): """ Start assembling a Map/Reduce operation. - A shortcut for :func:`RiakMapReduce.reduce`. + A shortcut for :meth:`~riak.mapreduce.RiakMapReduce.reduce`. - :rtype: RiakMapReduce + :rtype: :class:`~riak.mapreduce.RiakMapReduce` """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) From f4d658d92c4491f80eef73085d60b12cb31a08a5 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 29 Jul 2013 11:21:58 -0500 Subject: [PATCH 0474/1060] Fixup the pager on the index page and point at the tutorial docs. --- docs/_templates/layout.html | 1 + docs/index.rst | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 31ae631c..61243358 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -18,6 +18,7 @@ {%- endif %} {% block body %}{% endblock %} +
     
      {%- if prev %}