From e6f5e0cb04992597766a85b836c84ff24a63f47e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 28 Apr 2013 05:26:27 -0500 Subject: [PATCH 001/672] Refactor model and handling of siblings. In order to... 1. Better detect deleted objects and sibling tombstones 2. Reduce server roundtrips to fetch object siblings (which was an inconsistency between HTTP and PBC). ...this refactor introduces breaking changes. This introduces the `RiakContent` class which models a single sibling. An `RiakObject` that is NOT in conflict will appear to have one "sibling". Objects that are strictly "not found" will have ZERO siblings. Otherwise, the object will have as many siblings as are returned by Riak. This is more consistent with Riak's model of objects (values). Most existing methods and properties that were on `RiakObject` have been proxied in such a manner that if the object is not in conflict, the corresponding property or method on the solitary sibling will be invoked. When in conflict, these properties and methods with raise the new `riak.ConflictError` exception. Except in the case of conflict, this will ease transition of existing code to the new behavior. The `get_sibling` method and behavior has been broken to reflect the fact that all siblings are always requested and handled appropriately. The method now produces a deprecation warning and simply returns the requested sibling already in the object. The HTTP transport no longer supports retrieving the "text" format of objects in conflict. This commit also completes a refactoring of the HTTP transport to break-out transport codec methods into a separate class/file, for clarity and consistency with the PBC transport. --- riak/__init__.py | 12 +- riak/client/operations.py | 6 +- riak/content.py | 187 ++++++++++++++ riak/riak_object.py | 290 +++++++-------------- riak/tests/test_kv.py | 33 +-- riak/tests/test_mapreduce.py | 6 +- riak/transports/http/codec.py | 255 +++++++++++++++++++ riak/transports/http/connection.py | 14 +- riak/transports/http/transport.py | 391 +++++------------------------ riak/transports/pbc/codec.py | 115 +++++---- riak/transports/pbc/connection.py | 3 +- riak/transports/pbc/transport.py | 70 +----- riak/transports/transport.py | 2 +- 13 files changed, 713 insertions(+), 671 deletions(-) create mode 100644 riak/content.py create mode 100644 riak/transports/http/codec.py diff --git a/riak/__init__.py b/riak/__init__.py index ba50fc00..25a41535 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -32,7 +32,7 @@ __all__ = ['RiakBucket', 'RiakNode', 'RiakObject', 'RiakClient', 'RiakMapReduce', 'RiakKeyFilter', 'RiakLink', 'RiakError', - 'ONE', 'ALL', 'QUORUM', 'key_filter'] + 'ConflictError', 'ONE', 'ALL', 'QUORUM', 'key_filter'] class RiakError(Exception): @@ -45,6 +45,16 @@ def __init__(self, value): def __str__(self): return repr(self.value) + +class ConflictError(RiakError): + """ + Raised when an operation is attempted on a RiakObject that has + more than one sibling. + """ + def __init__(self, message="Object in conflict"): + super(ConflictError, self).__init__(message) + + from client import RiakClient from bucket import RiakBucket from node import RiakNode diff --git a/riak/client/operations.py b/riak/client/operations.py index 516b2d26..cf4cc46f 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -180,7 +180,7 @@ def put_new(self, transport, robj, w=None, dw=None, pw=None, if_none_match=if_none_match) @retryable - def get(self, transport, robj, r=None, pr=None, vtag=None): + def get(self, transport, robj, r=None, pr=None): """ Fetches the contents of a Riak object. @@ -190,14 +190,12 @@ def get(self, transport, robj, r=None, pr=None, vtag=None): :type r: integer, string, None :param pr: the primary read quorum :type pr: integer, string, None - :param vtag: the specific sibling to fetch - :type vtag: string """ if not isinstance(robj.key, basestring): raise TypeError( 'key must be a string, instead got {0}'.format(repr(robj.key))) - return transport.get(robj, r=r, pr=pr, vtag=vtag) + return transport.get(robj, r=r, pr=pr) @retryable def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, diff --git a/riak/content.py b/riak/content.py new file mode 100644 index 00000000..f19606b7 --- /dev/null +++ b/riak/content.py @@ -0,0 +1,187 @@ +""" +Copyright 2013 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" +from riak import RiakError +from riak.util import deprecated + + +class RiakContent(object): + """ + The RiakContent holds the metadata and value of a single sibling + within a RiakObject. RiakObjects that have more than one sibling + are considered to be in conflict. + """ + def __init__(self, robject, data=None, encoded_data=None, charset=None, + content_type='application/json', content_encoding=None, + etag=None, usermeta=None, links=None, indexes=None, + exists=False): + self._robject = robject + self._data = data + self._encoded_data = encoded_data + self.charset = charset + self.content_type = content_type + self.content_encoding = content_encoding + self.etag = etag + self.usermeta = usermeta or {} + self.links = links or [] + self.indexes = indexes or set() + self.exists = exists + + def _get_data(self): + if self._encoded_data is not None and self._data is None: + self._data = self._deserialize(self._encoded_data) + self._encoded_data = None + return self._data + + def _set_data(self, value): + self._encoded_data = None + self._data = value + + data = property(_get_data, _set_data, doc=""" + The data stored in this object, as Python objects. For the raw + data, use the `encoded_data` property. If unset, accessing + this property will result in decoding the `encoded_data` + property into Python values. The decoding is dependent on the + `content_type` property and the bucket's registered decoders. + :type mixed """) + + def get_encoded_data(self): + deprecated("`get_encoded_data` is deprecated, use the `encoded_data`" + " property") + return self.encoded_data + + def set_encoded_data(self, value): + deprecated("`set_encoded_data` is deprecated, use the `encoded_data`" + " property") + self.encoded_data = value + + def _get_encoded_data(self): + if self._data is not None and self._encoded_data is None: + self._encoded_data = self._serialize(self._data) + self._data = None + return self._encoded_data + + def _set_encoded_data(self, value): + self._data = None + self._encoded_data = value + + encoded_data = property(_get_encoded_data, _set_encoded_data, doc=""" + The raw data stored in this object, essentially the encoded + form of the `data` property. If unset, accessing this property + will result in encoding the `data` property into a string. The + encoding is dependent on the `content_type` property and the + bucket's registered encoders. + :type basestring""") + + def _serialize(self, value): + encoder = self._robject.bucket.get_encoder(self.content_type) + if encoder: + return encoder(value) + elif isinstance(value, basestring): + return value.encode() + else: + raise TypeError('No encoder for non-string data ' + 'with content type "{0}"'. + format(self.content_type)) + + def _deserialize(self, value): + decoder = self._robject.bucket.get_decoder(self.content_type) + if decoder: + return decoder(value) + else: + raise TypeError('No decoder for content type "{0}"'. + format(self.content_type)) + + def add_index(self, field, value): + """ + Tag this object with the specified field/value pair for + indexing. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: RiakObject + """ + if field[-4:] not in ("_bin", "_int"): + raise RiakError("Riak 2i fields must end with either '_bin'" + " or '_int'.") + + self.indexes.add((field, value)) + + return self._robject + + def remove_index(self, field=None, value=None): + """ + Remove the specified field/value pair as an index on this + object. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: RiakObject + """ + if not field and not value: + self.indexes.clear() + elif field and not value: + for index in [x for x in self.indexes if x[0] == field]: + self.indexes.remove(index) + elif field and value: + self.indexes.remove((field, value)) + else: + raise RiakError("Cannot pass value without a field" + " name while removing index") + + return self._robject + + remove_indexes = remove_index + + def set_index(self, field, value): + """ + Works like add_index, but ensures that there is only one index + on given field. If other found, then removes it first. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: RiakObject + """ + to_rem = set((x for x in self.indexes if x[0] == field)) + self.indexes.difference_update(to_rem) + return self.add_index(field, value) + + def add_link(self, obj, tag=None): + """ + Add a link to a RiakObject. + + :param obj: Either a RiakObject or 3 item link tuple consisting + of (bucket, key, tag). + :type obj: mixed + :param tag: Optional link tag. Defaults to bucket name. It is ignored + if ``obj`` is a 3 item link tuple. + :type tag: string + :rtype: RiakObject + """ + if isinstance(obj, tuple): + newlink = obj + else: + newlink = (obj.bucket.name, obj.key, tag) + + self.links.append(newlink) + return self._robject diff --git a/riak/riak_object.py b/riak/riak_object.py index dc057c36..eee42db8 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -1,4 +1,5 @@ """ +Copyright 2012-2013 Basho Technologies Copyright 2010 Rusty Klophaus Copyright 2010 Justin Sheehy Copyright 2009 Jay Baird @@ -17,10 +18,46 @@ specific language governing permissions and limitations under the License. """ -from riak import RiakError +from riak import ConflictError +from riak.content import RiakContent from riak.util import deprecated +def content_property(name, doc=None): + """ + Delegates a property to the first sibling in a RiakObject, raising + an error when the object is in conflict. + """ + def _setter(self, value): + if len(self.siblings) == 0: + # In this case, assume that what the user wants is to + # create a new sibling inside an empty object. + self.siblings = [RiakContent(self)] + if len(self.siblings) != 1: + raise ConflictError() + setattr(self.siblings[0], name, value) + + def _getter(self): + if len(self.siblings) != 1: + raise ConflictError() + return getattr(self.siblings[0], name) + + return property(_getter, _setter, doc=doc) + + +def content_method(name): + """ + Delegates a method to the first sibling in a RiakObject, raising + an error when the object is in conflict. + """ + def _delegate(self, *args, **kwargs): + if len(self.siblings) != 1: + raise ConflictError() + return getattr(self.siblings[0], name).__call__(*args, **kwargs) + + return _delegate + + class RiakObject(object): """ The RiakObject holds meta information about a Riak object, plus the @@ -51,17 +88,8 @@ def __init__(self, client, bucket, key=None): self.client = client self.bucket = bucket self.key = key - self._data = None - self._encoded_data = None self.vclock = None - self.charset = None - self.content_type = 'application/json' - self.content_encoding = None - self.usermeta = {} - self.indexes = set() - self.links = [] - self.siblings = [] - self.exists = False + self.siblings = [RiakContent(self)] def __hash__(self): return hash((self.key, self.bucket, self.vclock)) @@ -78,45 +106,14 @@ def __ne__(self, other): else: return True - def _get_data(self): - if self._encoded_data is not None and self._data is None: - self._data = self._deserialize(self._encoded_data) - self._encoded_data = None - return self._data - - def _set_data(self, value): - self._encoded_data = None - self._data = value - - data = property(_get_data, _set_data, doc=""" + data = content_property('data', doc=""" The data stored in this object, as Python objects. For the raw data, use the `encoded_data` property. If unset, accessing this property will result in decoding the `encoded_data` property into Python values. The decoding is dependent on the `content_type` property and the bucket's registered decoders. :type mixed """) - - def get_encoded_data(self): - deprecated("`get_encoded_data` is deprecated, use the `encoded_data`" - " property") - return self.encoded_data - - def set_encoded_data(self, value): - deprecated("`set_encoded_data` is deprecated, use the `encoded_data`" - " property") - self.encoded_data = value - - def _get_encoded_data(self): - if self._data is not None and self._encoded_data is None: - self._encoded_data = self._serialize(self._data) - self._data = None - return self._encoded_data - - def _set_encoded_data(self, value): - self._data = None - self._encoded_data = value - - encoded_data = property(_get_encoded_data, _set_encoded_data, doc=""" + encoded_data = content_property('encoded_data', doc=""" The raw data stored in this object, essentially the encoded form of the `data` property. If unset, accessing this property will result in encoding the `data` property into a string. The @@ -124,104 +121,56 @@ def _set_encoded_data(self, value): bucket's registered encoders. :type basestring""") - def _serialize(self, value): - encoder = self.bucket.get_encoder(self.content_type) - if encoder: - return encoder(value) - elif isinstance(value, basestring): - return value.encode() - else: - raise TypeError('No encoder for non-string data ' - 'with content type "{0}"'. - format(self.content_type)) - - def _deserialize(self, value): - decoder = self.bucket.get_decoder(self.content_type) - if decoder: - return decoder(value) - else: - raise TypeError('No decoder for content type "{0}"'. - format(self.content_type)) + charset = content_property('charset', doc=""" + The character set of the encoded data + :type string""") - def add_index(self, field, value): - """ - Tag this object with the specified field/value pair for - indexing. + content_type = content_property('content_type', doc=""" + The MIME media type of the encoded data + :type string""") - :param field: The index field. - :type field: string - :param value: The index value. - :type value: string or integer - :rtype: RiakObject - """ - if field[-4:] not in ("_bin", "_int"): - raise RiakError("Riak 2i fields must end with either '_bin'" - " or '_int'.") + content_encoding = content_property('content_encoding') - self.indexes.add((field, value)) + usermeta = content_property('usermeta', doc=""" + Arbitrary user-defined metadata, mapping strings to strings. + :type dict""") - return self + links = content_property('links', doc=""" + A collection of bucket/key/tag 3-tuples representing links to + other keys. + :type set""") - def remove_index(self, field=None, value=None): - """ - Remove the specified field/value pair as an index on this - object. - - :param field: The index field. - :type field: string - :param value: The index value. - :type value: string or integer - :rtype: RiakObject - """ - if not field and not value: - self.indexes.clear() - elif field and not value: - for index in [x for x in self.indexes if x[0] == field]: - self.indexes.remove(index) - elif field and value: - self.indexes.remove((field, value)) - else: - raise RiakError("Cannot pass value without a field" - " name while removing index") - - return self - - def set_index(self, field, value): - """ - Works like add_index, but ensures that there is only one index on given field. - If other found, then removes it first. - - :param field: The index field. - :type field: string - :param value: The index value. - :type value: string or integer - :rtype: RiakObject - """ - to_rem = set((x for x in self.indexes if x[0] == field)) - self.indexes.difference_update(to_rem) - return self.add_index(field, value) + indexes = content_property('indexes', doc=""" + The set of secondary index entries, consisting of + index-name/value tuples + :type set""") + get_encoded_data = content_method('get_encoded_data') + set_encoded_data = content_method('set_encoded_data') + add_index = content_method('add_index') + remove_index = content_method('remove_index') remove_indexes = remove_index + set_index = content_method('set_index') + add_link = content_method('add_link') - def add_link(self, obj, tag=None): - """ - Add a link to a RiakObject. - - :param obj: Either a RiakObject or 3 item link tuple consisting - of (bucket, key, tag). - :type obj: mixed - :param tag: Optional link tag. Defaults to bucket name. It is ignored - if ``obj`` is a 3 item link tuple. - :type tag: string - :rtype: RiakObject - """ - if isinstance(obj, tuple): - newlink = obj + def _exists(self): + if len(self.siblings) == 0: + return False + elif len(self.siblings) > 1: + # Even if all of the siblings are tombstones, the object + # essentially exists. + return True else: - newlink = (obj.bucket.name, obj.key, tag) + return self.siblings[0].exists - self.links.append(newlink) - return self + exists = property(_exists, None, doc=""" + Whether the object exists. This is only true when there is a + single sibling and it is neither a tombstone nor unsaved.""") + + def get_sibling(self, index): + deprecated("RiakObject.get_sibling is deprecated, use the " + "siblings property instead") + return self.siblings[index] def store(self, w=None, dw=None, pw=None, return_body=True, if_none_match=False): @@ -248,27 +197,23 @@ def store(self, w=None, dw=None, pw=None, return_body=True, there is no key previously defined :type if_none_match: bool :rtype: RiakObject """ - if (self.siblings and not self._data - and not self._encoded_data and not self.vclock): - raise RiakError("Attempting to store an invalid object," - "store one of the siblings instead") + if len(self.siblings) != 1: + raise ConflictError("Attempting to store an invalid object, " + "resolve the siblings first") if self.key is None: - result = self.client.put_new( + self.client.put_new( self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) - self._populate(result) else: - result = self.client.put(self, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) - if result is not None and result != ('', []): - self._populate(result) + self.client.put(self, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) return self - def reload(self, r=None, pr=None, vtag=None): + def reload(self, r=None, pr=None): """ Reload the object from Riak. When this operation completes, the object could contain new metadata and a new value, if the object @@ -277,15 +222,14 @@ def reload(self, r=None, pr=None, vtag=None): :param r: R-Value, wait for this many partitions to respond before returning to client. :type r: integer + :param pr: PR-value, require this many primary partitions to + be available before performing the read that + precedes the put + :type pr: integer :rtype: RiakObject """ - result = self.client.get(self, r=r, pr=pr, vtag=vtag) - if result and result != ('', []): - self._populate(result) - else: - self.clear() - + self.client.get(self, r=r, pr=pr) return self def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): @@ -325,55 +269,9 @@ def clear(self): :rtype: RiakObject """ - self.headers = [] - self.links = [] - self.data = None - self.exists = False self.siblings = [] return self - def _populate(self, result): - """ - Populate the object based on the return from get. - - If None returned, then object is not found - If a tuple of vclock, contents then one or more - whole revisions of the key were found - If a list of vtags is returned there are multiple - sibling that need to be retrieved with get. - """ - if result is None or result is self: - return self - elif type(result) is RiakObject: - self.clear() - self.__dict__ = result.__dict__.copy() - else: - raise RiakError("do not know how to handle type %s" % type(result)) - - def get_sibling(self, i, r=None, pr=None): - """ - Retrieve a sibling by sibling number. - - :param i: Sibling number. - :type i: integer - :param r: R-Value. Wait until this many partitions - have responded before returning to client. - :type r: integer - :rtype: RiakObject. - """ - if isinstance(self.siblings[i], RiakObject): - return self.siblings[i] - else: - # Run the request... - vtag = self.siblings[i] - obj = RiakObject(self.client, self.bucket, self.key) - obj.reload(r=r, pr=pr, vtag=vtag) - - # And make sure it knows who its siblings are - self.siblings[i] = obj - obj.siblings = self.siblings - return obj - def add(self, *args): """ Start assembling a Map/Reduce operation. diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 6fb644c1..880245bc 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -2,6 +2,7 @@ import os import cPickle import copy +from riak import ConflictError try: import simplejson as json @@ -106,6 +107,7 @@ def test_stream_keys_abort(self): # If the stream was closed correctly, this will not error robj = bucket.get(regular_keys[0]) + self.assertEqual(len(robj.siblings), 1) self.assertEqual(True, robj.exists) def test_bad_key(self): @@ -185,7 +187,6 @@ def test_missing_object(self): bucket = self.client.bucket(self.bucket_name) obj = bucket.get(self.key_name) self.assertFalse(obj.exists) - self.assertEqual(obj.data, None) def test_delete(self): bucket = self.client.bucket(self.bucket_name) @@ -272,25 +273,29 @@ def test_siblings(self): other_obj.store() vals.add(str(randval)) - # Make sure the object has itself plus four siblings... + # Make sure the object has five siblings... obj = bucket.get(self.key_name) obj.reload() - self.assertTrue(bool(obj.siblings)) self.assertEqual(len(obj.siblings), 5) - # Get each of the values - make sure they match what was assigned - vals2 = set() - for i in xrange(len(obj.siblings)): - vals2.add(obj.get_sibling(i).encoded_data) + # When the object is in conflict, using the shortcut methods + # should raise the ConflictError + with self.assertRaises(ConflictError): + obj.data + + # Get each of the values - make sure they match what was + # assigned + vals2 = set([sibling.encoded_data for sibling in obj.siblings]) self.assertEqual(vals, vals2) # Resolve the conflict, and then do a get... - obj3 = obj.get_sibling(3) - obj3.store() + resolved_sibling = obj.siblings[3] + obj.siblings = [resolved_sibling] + obj.store() obj.reload() - self.assertEqual(len(obj.siblings), 0) - self.assertEqual(obj.encoded_data, obj3.encoded_data) + self.assertEqual(len(obj.siblings), 1) + self.assertEqual(obj.encoded_data, resolved_sibling.encoded_data) def test_store_of_missing_object(self): bucket = self.client.bucket(self.bucket_name) @@ -430,7 +435,7 @@ def test_store_binary_object_from_file_should_use_default_mimetype(self): def test_store_binary_object_from_file_should_fail_if_file_not_found(self): bucket = self.client.bucket(self.bucket_name) with self.assertRaises(IOError): - bucket.new_from_file('not_found_from_file', 'FILE_NOT_FOUND') - obj = bucket.get('not_found_from_file') - self.assertEqual(obj.encoded_data, None) + bucket.new_from_file(self.key_name, 'FILE_NOT_FOUND') + obj = bucket.get(self.key_name) + # self.assertEqual(obj.encoded_data, None) self.assertFalse(obj.exists) diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 8f2ad92a..cb7e3759 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -8,13 +8,13 @@ class LinkTests(object): def test_store_and_get_links(self): # Create the object... bucket = self.client.bucket(self.bucket_name) - bucket.new(key="test_store_and_get_links", encoded_data='2', + bucket.new(key=self.key_name, encoded_data='2', content_type='application/octet-stream') \ .add_link(bucket.new("foo1")) \ .add_link(bucket.new("foo2"), "tag") \ .add_link(bucket.new("foo3"), "tag2!@#%^&*)") \ .store() - obj = bucket.get("test_store_and_get_links") + obj = bucket.get(self.key_name) links = obj.links self.assertEqual(len(links), 3) for bucket, key, tag in links: @@ -555,4 +555,4 @@ def test_stream_cleanoperationsup(self): # This should not raise an exception obj = bucket.get('one') - self.assertEqual(1, obj.data) + self.assertEqual('1', obj.encoded_data) diff --git a/riak/transports/http/codec.py b/riak/transports/http/codec.py new file mode 100644 index 00000000..c2896f3f --- /dev/null +++ b/riak/transports/http/codec.py @@ -0,0 +1,255 @@ +""" +Copyright 2012 Basho Technologies, Inc. +Copyright 2010 Rusty Klophaus +Copyright 2010 Justin Sheehy +Copyright 2009 Jay Baird + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +# subtract length of "Link: " header string and newline +MAX_LINK_HEADER_SIZE = 8192 - 8 + + +import re +import csv +import urllib +from cgi import parse_header +from email import message_from_string +from xml.etree import ElementTree +from riak import RiakError +from riak.content import RiakContent +from riak.multidict import MultiDict +from riak.transports.http.search import XMLSearchResult + + +class RiakHttpCodec(object): + """ + Methods for HTTP transport that marshals and unmarshals HTTP + messages. + """ + + def _parse_body(self, robj, response, expected_statuses): + """ + Parse the body of an object response and populate the object. + """ + # If no response given, then return. + if response is None: + return None + + status, headers, data = response + + # Check if the server is down(status==0) + if not status: + m = 'Could not contact Riak Server: http://{0}:{1}!'.format( + self._node.host, self._node.http_port) + raise RiakError(m) + + # Make sure expected code came back + self.check_http_code(status, expected_statuses) + + if 'x-riak-vclock' in headers: + robj.vclock = headers['x-riak-vclock'] + + # If 404(Not Found), then clear the object. + if status == 404: + robj.siblings = [] + return None + # If 201 Created, we need to extract the location and set the + # key on the object. + elif status == 201: + robj.key = headers['location'].strip().split('/')[-1] + # If 300(Siblings), apply the siblings to the object + elif status == 300: + ctype, params = parse_header(headers['content-type']) + if ctype == 'multipart/mixed': + boundary = re.compile('\r?\n--%s(?:--)?\r?\n' % + re.escape(params['boundary'])) + parts = [message_from_string(p) + for p in re.split(boundary, data)[1:-1]] + robj.siblings = [self._parse_sibling(RiakContent(robj), + part.items(), + part.get_payload()) + for part in parts] + return robj + else: + raise Exception('unexpected sibling response format: {0}'. + format(ctype)) + + robj.siblings = [self._parse_sibling(RiakContent(robj), + headers.items(), data)] + return robj + + def _parse_sibling(self, sibling, headers, data): + """ + Parses a single sibling out of a response. + """ + + sibling.exists = True + + # Parse the headers... + for header, value in headers: + header = header.lower() + if header == 'content-type': + sibling.content_type, sibling.charset = \ + self._parse_content_type(value) + elif header == 'etag': + sibling.etag = value + elif header == 'link': + sibling.links = self._parse_links(value) + elif header == 'last-modified': + sibling.last_modified = value + elif header.startswith('x-riak-meta-'): + metakey = header.replace('x-riak-meta-', '') + sibling.usermeta[metakey] = value + elif header.startswith('x-riak-index-'): + field = header.replace('x-riak-index-', '') + reader = csv.reader([value], skipinitialspace=True) + for line in reader: + for token in line: + if field.endswith("_int"): + token = int(token) + sibling.add_index(field, token) + elif header == 'x-riak-deleted': + sibling.exists = False + + sibling.encoded_data = data + + return sibling + + def _to_link_header(self, link): + """ + Convert the link tuple to a link header string. Used internally. + """ + try: + bucket, key, tag = link + except ValueError: + raise RiakError("Invalid link tuple %s" % link) + tag = tag if tag is not None else bucket + url = self.object_path(bucket, key) + header = '<%s>; riaktag="%s"' % (url, tag) + return header + + def _parse_links(self, linkHeaders): + links = [] + oldform = "; ?riaktag=\"([^\"]+)\"" + newform = "; ?riaktag=\"([^\"]+)\"" + for linkHeader in linkHeaders.strip().split(','): + linkHeader = linkHeader.strip() + matches = (re.match(oldform, linkHeader) or + re.match(newform, linkHeader)) + if matches is not None: + link = (urllib.unquote_plus(matches.group(2)), + urllib.unquote_plus(matches.group(3)), + urllib.unquote_plus(matches.group(4))) + links.append(link) + return links + + def _add_links_for_riak_object(self, robject, headers): + links = robject.links + if links: + current_header = '' + for link in links: + header = self._to_link_header(link) + if len(current_header + header) > MAX_LINK_HEADER_SIZE: + headers.add('Link', current_header) + current_header = '' + + if current_header != '': + header = ', ' + header + current_header += header + + headers.add('Link', current_header) + + return headers + + def _build_put_headers(self, robj, if_none_match=False): + """Build the headers for a POST/PUT request.""" + + # Construct the headers... + if robj.charset is not None: + content_type = ('%s; charset="%s"' % + (robj.content_type, robj.charset)) + else: + content_type = robj.content_type + + headers = MultiDict({'Content-Type': content_type, + 'X-Riak-ClientId': self._client_id}) + + # Add the vclock if it exists... + if robj.vclock is not None: + headers['X-Riak-Vclock'] = robj.vclock + + # Create the header from metadata + self._add_links_for_riak_object(robj, headers) + + for key, value in robj.usermeta.iteritems(): + headers['X-Riak-Meta-%s' % key] = value + + for field, value in robj.indexes: + key = 'X-Riak-Index-%s' % field + if key in headers: + headers[key] += ", " + str(value) + else: + headers[key] = str(value) + + if if_none_match: + headers['If-None-Match'] = '*' + + return headers + + def _normalize_json_search_response(self, json): + """ + Normalizes a JSON search response so that PB and HTTP have the + same return value + """ + result = {} + if u'response' in json: + result['num_found'] = json[u'response'][u'numFound'] + result['max_score'] = float(json[u'response'][u'maxScore']) + docs = [] + for doc in json[u'response'][u'docs']: + resdoc = {u'id': doc[u'id']} + if u'fields' in doc: + for k, v in doc[u'fields'].iteritems(): + resdoc[k] = v + docs.append(resdoc) + result['docs'] = docs + return result + + def _normalize_xml_search_response(self, xml): + """ + Normalizes an XML search response so that PB and HTTP have the + same return value + """ + target = XMLSearchResult() + parser = ElementTree.XMLParser(target=target) + parser.feed(xml) + return parser.close() + + def _parse_content_type(self, value): + """ + Split the content-type header into two parts: + 1) Actual main/sub encoding type + 2) charset + + :param value: Complete MIME content-type string + """ + content_type, params = parse_header(value) + if 'charset' in params: + charset = params['charset'] + else: + charset = None + return content_type, charset diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py index 3262b8d3..59a13ea9 100644 --- a/riak/transports/http/connection.py +++ b/riak/transports/http/connection.py @@ -26,19 +26,17 @@ class RiakHttpConnection(object): def _request(self, method, uri, headers={}, body='', stream=False): """ - Given a Method, URL, Headers, and Body, perform and HTTP request, - and return a 2-tuple containing a dictionary of response headers - and the response body. + Given a Method, URL, Headers, and Body, perform and HTTP + request, and return a 3-tuple containing the response status, + response headers (as httplib.HTTPMessage), and response body. """ response = None + headers.setdefault('Accept', + 'multipart/mixed, application/json, */*;q=0.5') try: self._connection.request(method, uri, body, headers) response = self._connection.getresponse() - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value - if stream: # The caller is responsible for fully reading the # response and closing it when streaming. @@ -49,7 +47,7 @@ def _request(self, method, uri, headers={}, body='', stream=False): if response and not stream: response.close() - return response_headers, response_body + return response.status, response.msg, response_body def _connect(self): self._connection = self._connection_class(self._node.host, diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index ee147f3c..460f85de 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -24,29 +24,21 @@ except ImportError: import json -import urllib -import re -import csv + import httplib -from email.message import Message +from xml.dom.minidom import Document from riak.transports.transport import RiakTransport from riak.transports.http.resources import RiakHttpResources from riak.transports.http.connection import RiakHttpConnection -from riak.transports.http.search import XMLSearchResult +from riak.transports.http.codec import RiakHttpCodec from riak.transports.http.stream import ( RiakHttpKeyStream, RiakHttpMapReduceStream) from riak import RiakError -from riak.multidict import MultiDict -from xml.etree import ElementTree -from xml.dom.minidom import Document -# subtract length of "Link: " header string and newline -MAX_LINK_HEADER_SIZE = 8192 - 8 - - -class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakTransport): +class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakHttpCodec, + RiakTransport): """ The RiakHttpTransport object holds information necessary to connect to Riak via HTTP. @@ -74,17 +66,17 @@ def ping(self): """ Check server is alive over HTTP """ - response = self._request('GET', self.ping_path()) - return(response is not None) and (response[1] == 'OK') + status, _, body = self._request('GET', self.ping_path()) + return(status is not None) and (body == 'OK') def stats(self): """ Gets performance statistics and server information """ - response = self._request('GET', self.stats_path(), - {'Accept': 'application/json'}) - if response[0]['http_code'] is 200: - return json.loads(response[1]) + status, _, body = self._request('GET', self.stats_path(), + {'Accept': 'application/json'}) + if status == 200: + return json.loads(body) else: return None @@ -106,75 +98,51 @@ def get_resources(self): Gets a JSON mapping of server-side resource names to paths :rtype dict """ - response = self._request('GET', '/', {'Accept': 'application/json'}) - if response[0]['http_code'] is 200: - return json.loads(response[1]) + status, _, body = self._request('GET', '/', + {'Accept': 'application/json'}) + if status == 200: + return json.loads(body) else: return {} - def get(self, robj, r=None, pr=None, vtag=None): + def get(self, robj, r=None, pr=None): """ Get a bucket/key from the server """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'r': r, 'pr': pr, 'vtag': vtag} + params = {'r': r, 'pr': pr} url = self.object_path(robj.bucket.name, robj.key, **params) response = self._request('GET', url) - return self.parse_body(robj, response, [200, 300, 404]) + return self._parse_body(robj, response, [200, 300, 404]) def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """ - Serialize put request and deserialize response + Puts a (possibly new) object. """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} url = self.object_path(robj.bucket.name, robj.key, **params) - headers = self._build_put_headers(robj) - - # TODO: use a more general 'prevent_stale_writes' semantics, - # which is a superset of the if_none_match semantics. - if if_none_match: - headers["If-None-Match"] = "*" + headers = self._build_put_headers(robj, if_none_match=if_none_match) content = robj.encoded_data - return self.do_put(url, headers, content, robj, return_body) - def do_put(self, url, headers, content, robj, return_body=False): if robj.key is None: - response = self._request('POST', url, headers, content) + expect = [201] + method = 'POST' else: - response = self._request('PUT', url, headers, content) + expect = [204] + method = 'PUT' + response = self._request(method, url, headers, content) if return_body: - return self.parse_body(robj, response, [200, 201, 204, 300]) + return self._parse_body(robj, response, [200, 201, 204, 300]) else: - self.check_http_code(response, [204]) + self.check_http_code(response[0], expect) return None - def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """Put a new object into the Riak store, returning its (new) key.""" - # We could detect quorum_controls here but HTTP ignores - # unknown flags/params. - params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} - url = self.object_path(robj.bucket.name, **params) - headers = self._build_put_headers(robj) - # TODO: use a more general 'prevent_stale_writes' semantics, - # which is a superset of the if_none_match semantics. - if if_none_match: - headers["If-None-Match"] = "*" - content = robj.encoded_data - response = self._request('POST', url, headers, content) - location = response[0]['location'] - idx = location.rindex('/') - robj.key = location[(idx + 1):] - if return_body: - return self.parse_body(robj, response, [201]) - else: - self.check_http_code(response, [201]) - return None + put_new = put def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ @@ -188,7 +156,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): if self.tombstone_vclocks() and robj.vclock is not None: headers['X-Riak-Vclock'] = robj.vclock response = self._request('DELETE', url, headers) - self.check_http_code(response, [204, 404]) + self.check_http_code(response[0], [204, 404]) return self def get_keys(self, bucket): @@ -196,20 +164,19 @@ def get_keys(self, bucket): Fetch a list of keys for the bucket """ url = self.key_list_path(bucket.name) - response = self._request('GET', url) + status, _, body = self._request('GET', url) - headers, encoded_props = response[0:2] - if headers['http_code'] == 200: - props = json.loads(encoded_props) + if status == 200: + props = json.loads(body) return props['keys'] else: raise Exception('Error listing keys.') def stream_keys(self, bucket): url = self.key_list_path(bucket.name, keys='stream') - headers, response = self._request('GET', url, stream=True) + status, headers, response = self._request('GET', url, stream=True) - if headers['http_code'] == 200: + if status == 200: return RiakHttpKeyStream(response) else: raise Exception('Error listing keys.') @@ -219,11 +186,10 @@ def get_buckets(self): Fetch a list of all buckets """ url = self.bucket_list_path() - response = self._request('GET', url) + status, headers, body = self._request('GET', url) - headers, encoded_props = response[0:2] - if headers['http_code'] == 200: - props = json.loads(encoded_props) + if status == 200: + props = json.loads(body) return props['buckets'] else: raise Exception('Error getting buckets.') @@ -234,12 +200,10 @@ def get_bucket_props(self, bucket): """ # Run the request... url = self.bucket_properties_path(bucket.name) - response = self._request('GET', url) + status, headers, body = self._request('GET', url) - headers = response[0] - encoded_props = response[1] - if headers['http_code'] == 200: - props = json.loads(encoded_props) + if status == 200: + props = json.loads(body) return props['props'] else: raise Exception('Error getting bucket properties.') @@ -253,14 +217,8 @@ def set_bucket_props(self, bucket, props): content = json.dumps({'props': props}) # Run the request... - response = self._request('PUT', url, headers, content) - - # Handle the response... - if response is None: - raise Exception('Error setting bucket properties.') + status, _, _ = self._request('PUT', url, headers, content) - # Check the response value... - status = response[0]['http_code'] if status != 204: raise Exception('Error setting bucket properties.') return True @@ -273,14 +231,8 @@ def clear_bucket_props(self, bucket): headers = {'Content-Type': 'application/json'} # Run the request... - response = self._request('DELETE', url, headers, None) + status, _, _ = self._request('DELETE', url, headers, None) - # Handle the response... - if response is None: - raise Exception('Error clearing bucket properties.') - - # Check the response value... - status = response[0]['http_code'] if status == 204: return True elif status == 405: @@ -299,16 +251,15 @@ def mapred(self, inputs, query, timeout=None): # Do the request... url = self.mapred_path() headers = {'Content-Type': 'application/json'} - response = self._request('POST', url, headers, content) + status, headers, body = self._request('POST', url, headers, content) # Make sure the expected status code came back... - status = response[0]['http_code'] if status != 200: raise RiakError( 'Error running MapReduce operation. Headers: %s Body: %s' % - (repr(response[0]), repr(response[1]))) + (repr(headers), repr(body))) - result = json.loads(response[1]) + result = json.loads(body) return result def stream_mapred(self, inputs, query, timeout=None): @@ -316,10 +267,10 @@ def stream_mapred(self, inputs, query, timeout=None): url = self.mapred_path(chunked=True) reqheaders = {'Content-Type': 'application/json'} - headers, response = self._request('POST', url, reqheaders, - content, stream=True) + status, headers, response = self._request('POST', url, reqheaders, + content, stream=True) - if headers['http_code'] is 200: + if status == 200: return RiakHttpMapReduceStream(response) else: raise Exception( @@ -331,10 +282,9 @@ def get_index(self, bucket, index, startkey, endkey=None): Performs a secondary index query. """ url = self.index_path(bucket, index, startkey, endkey) - response = self._request('GET', url) - headers, data = response - self.check_http_code(response, [200]) - json_data = json.loads(data) + status, headers, body = self._request('GET', url) + self.check_http_code(status, [200]) + json_data = json.loads(body) return json_data[u'keys'][:] def search(self, index, query, **params): @@ -351,9 +301,8 @@ def search(self, index, query, **params): options.update(params) url = self.solr_select_path(index, query, **options) - response = self._request('GET', url) - headers, data = response - self.check_http_code(response, [200]) + status, headers, data = self._request('GET', url) + self.check_http_code(status, [200]) if 'json' in headers['content-type']: results = json.loads(data) return self._normalize_json_search_response(results) @@ -409,235 +358,7 @@ def fulltext_delete(self, index, docs=None, queries=None): {'Content-Type': 'text/xml'}, xml.toxml().encode('utf-8')) - def check_http_code(self, response, expected_statuses): - status = response[0]['http_code'] + def check_http_code(self, status, expected_statuses): if not status in expected_statuses: - raise Exception('Expected status %s, received %s : %s' % - (expected_statuses, status, response[1])) - - def parse_body(self, robj, response, expected_statuses): - """ - Parse the body of an object response and populate the object. - """ - # If no response given, then return. - if response is None: - return None - - # Make sure expected code came back - self.check_http_code(response, expected_statuses) - - # Update the object... - headers = response[0] - data = response[1] - status = headers['http_code'] - - # Check if the server is down(status==0) - if not status: - ### we need the host/port that was used. - m = 'Could not contact Riak Server: http://$HOST:$PORT !' - raise RiakError(m) - - # If 404(Not Found), then clear the object. - if status == 404: - return None - - # If 300(Siblings), then return the list of siblings - elif status == 300: - # Parse and get rid of 'Siblings:' string in element 0 - siblings = data.strip().split('\n') - siblings.pop(0) - robj.siblings = siblings - robj.exists = True - robj.vclock = headers['x-riak-vclock'] - return robj - - #no sibs - robj.siblings = [] - - # Parse the headers... - links = [] - for header, value in headers.iteritems(): - if header == 'content-type': - robj.content_type, robj.charset = \ - self._parse_content_type(value) - elif header == 'content-encoding': - robj.content_encoding = value - elif header == 'etag': - robj.etag = value - elif header == 'link': - self._parse_links(links, headers['link']) - elif header == 'last-modified': - robj.last_modified = value - elif header.startswith('x-riak-meta-'): - metakey = header.replace('x-riak-meta-', '') - robj.usermeta[metakey] = value - elif header.startswith('x-riak-index-'): - field = header.replace('x-riak-index-', '') - reader = csv.reader([value], skipinitialspace=True) - for line in reader: - for token in line: - if field.endswith("_int"): - token = int(token) - robj.add_index(field, token) - elif header == 'x-riak-vclock': - robj.vclock = value - elif header == 'x-riak-deleted': - robj.deleted = True - if links: - robj.links = links - - robj.encoded_data = data - - robj.exists = True - return robj - - def to_link_header(self, link): - """ - Convert the link tuple to a link header string. Used internally. - """ - try: - bucket, key, tag = link - except ValueError: - raise RiakError("Invalid link tuple %s" % link) - tag = tag if tag is not None else bucket - url = self.object_path(bucket, key) - header = '<%s>; riaktag="%s"' % (url, tag) - return header - - def _parse_links(self, links, linkHeaders): - oldform = "; ?riaktag=\"([^\"]+)\"" - newform = "; ?riaktag=\"([^\"]+)\"" - for linkHeader in linkHeaders.strip().split(','): - linkHeader = linkHeader.strip() - matches = (re.match(oldform, linkHeader) or - re.match(newform, linkHeader)) - if matches is not None: - link = (urllib.unquote_plus(matches.group(2)), - urllib.unquote_plus(matches.group(3)), - urllib.unquote_plus(matches.group(4))) - links.append(link) - return links - - def _add_links_for_riak_object(self, robject, headers): - links = robject.links - if links: - current_header = '' - for link in links: - header = self.to_link_header(link) - if len(current_header + header) > MAX_LINK_HEADER_SIZE: - headers.add('Link', current_header) - current_header = '' - - if current_header != '': - header = ', ' + header - current_header += header - - headers.add('Link', current_header) - - return headers - - # Utility functions used by Riak library. - - def _build_put_headers(self, robj): - """Build the headers for a POST/PUT request.""" - - # Construct the headers... - if robj.charset is not None: - content_type = ('%s; charset="%s"' % - (robj.content_type, robj.charset)) - else: - content_type = robj.content_type - headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', - 'Content-Type': content_type, - 'X-Riak-ClientId': self._client_id}) - # Add the vclock if it exists... - if robj.vclock is not None: - headers['X-Riak-Vclock'] = robj.vclock - - # Create the header from metadata - self._add_links_for_riak_object(robj, headers) - - for key, value in robj.usermeta.iteritems(): - headers['X-Riak-Meta-%s' % key] = value - - for field, value in robj.indexes: - key = 'X-Riak-Index-%s' % field - if key in headers: - headers[key] += ", " + str(value) - else: - headers[key] = str(value) - - return headers - - def _normalize_json_search_response(self, json): - """ - Normalizes a JSON search response so that PB and HTTP have the - same return value - """ - result = {} - if u'response' in json: - result['num_found'] = json[u'response'][u'numFound'] - result['max_score'] = float(json[u'response'][u'maxScore']) - docs = [] - for doc in json[u'response'][u'docs']: - resdoc = {u'id': doc[u'id']} - if u'fields' in doc: - for k, v in doc[u'fields'].iteritems(): - resdoc[k] = v - docs.append(resdoc) - result['docs'] = docs - return result - - def _normalize_xml_search_response(self, xml): - """ - Normalizes an XML search response so that PB and HTTP have the - same return value - """ - target = XMLSearchResult() - parser = ElementTree.XMLParser(target=target) - parser.feed(xml) - return parser.close() - - def _parse_content_type(self, value): - """ - Split the content-type header into two parts: - 1) Actual main/sub encoding type - 2) charset - - :param value: Complete MIME content-type string - """ - message = Message() - message.set_type(value) - - content_type = message.get_content_type() - charset = message.get_content_charset(None) - - return content_type, charset - - @classmethod - def build_headers(cls, headers): - return ['%s: %s' % (header, value) - for header, value in headers.iteritems()] - - @classmethod - def parse_http_headers(cls, headers): - """ - Parse an HTTP Header string into an associative array of - response headers. - """ - retVal = {} - fields = headers.split("\n") - for field in fields: - matches = re.match("([^:]+):(.+)", field) - if matches is None: - continue - key = matches.group(1).lower() - value = matches.group(2).strip() - if key in retVal.keys(): - if isinstance(retVal[key], list): - retVal[key].append(value) - else: - retVal[key] = [retVal[key]].append(value) - else: - retVal[key] = value - return retVal + raise Exception('Expected status %s, received %s' % + (expected_statuses, status)) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index d41d8ed6..e24b8697 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -16,7 +16,9 @@ under the License. """ import riak_pb -from riak.riak_object import RiakObject +from riak import RiakError +from riak.content import RiakContent +from riak.mapreduce import RiakLink RIAKC_RW_ONE = 4294967294 RIAKC_RW_QUORUM = 4294967293 @@ -58,7 +60,17 @@ def translate_rw_val(self, rw): else: return None - def decode_content(self, rpb_content, robj): + def _decoded_contents(self, resp, obj): + if type(resp) == riak_pb.RpbPutResp and resp.HasField('key'): + obj.key = resp.key + if resp.HasField("vclock"): + obj.vclock = resp.vclock + + obj.siblings = [self._decode_content(c, RiakContent(obj)) + for c in resp.content] + return obj + + def _decode_content(self, rpb_content, sibling): """ Decodes a single sibling from the protobuf representation into a RiakObject. @@ -66,58 +78,38 @@ def decode_content(self, rpb_content, robj): :rtype: (RiakObject) """ - if rpb_content.HasField("deleted"): - robj.deleted = True + if rpb_content.HasField("deleted") and rpb_content.deleted: + sibling.exists = False + else: + sibling.exists = True if rpb_content.HasField("content_type"): - robj.content_type = rpb_content.content_type + sibling.content_type = rpb_content.content_type if rpb_content.HasField("charset"): - robj.charset = rpb_content.charset + sibling.charset = rpb_content.charset if rpb_content.HasField("content_encoding"): - robj.content_encoding = rpb_content.content_encoding + sibling.content_encoding = rpb_content.content_encoding if rpb_content.HasField("vtag"): - robj.vtag = rpb_content.vtag - links = [] - for link in rpb_content.links: - if link.HasField("bucket"): - bucket = link.bucket - else: - bucket = None - if link.HasField("key"): - key = link.key - else: - key = None - if link.HasField("tag"): - tag = link.tag - else: - tag = None - links.append((bucket, key, tag)) - if links: - robj.links = links + sibling.etag = rpb_content.vtag + + sibling.links = [self._decode_link(link) + for link in rpb_content.links] if rpb_content.HasField("last_mod"): - robj.last_mod = rpb_content.last_mod + sibling.last_mod = rpb_content.last_mod if rpb_content.HasField("last_mod_usecs"): - robj.last_mod_usecs = rpb_content.last_mod_usecs - usermeta = {} - for usermd in rpb_content.usermeta: - usermeta[usermd.key] = usermd.value - if len(usermeta) > 0: - robj.usermeta = usermeta - indexes = set() - for index in rpb_content.indexes: - if index.key.endswith("_int"): - indexes.add((index.key, int(index.value))) - else: - indexes.add((index.key, index.value)) + sibling.last_mod_usecs = rpb_content.last_mod_usecs - if len(indexes) > 0: - robj.indexes = indexes + sibling.usermeta = dict([(usermd.key, usermd.value) + for usermd in rpb_content.usermeta]) + sibling.indexes = set([(index.key, + self._decode_index_value(index.key, + index.value)) + for index in rpb_content.indexes]) - robj.encoded_data = rpb_content.value - robj.exists = True + sibling.encoded_data = rpb_content.value - return robj + return sibling - def encode_content(self, robj, rpb_content): + def _encode_content(self, robj, rpb_content): """ Fills an RpbContent message with the appropriate data and metadata from a RiakObject. @@ -147,8 +139,37 @@ def encode_content(self, robj, rpb_content): pb_link.tag = '' for field, value in robj.indexes: - pair = rpb_content.indexes.add() - pair.key = field - pair.value = str(value) + pair = rpb_content.indexes.add() + pair.key = field + pair.value = str(value) rpb_content.value = str(robj.encoded_data) + + def _decode_link(self, link): + """ + Decodes an RpbLink message into a RiakLink named tuple + """ + + if link.HasField("bucket"): + bucket = link.bucket + else: + bucket = None + if link.HasField("key"): + key = link.key + else: + key = None + if link.HasField("tag"): + tag = link.tag + else: + tag = None + + return RiakLink(bucket, key, tag) + + def _decode_index_value(self, index, value): + """ + Decodes a secondary index value into the correct Python type. + """ + if index.endswith("_int"): + return int(value) + else: + return value diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index 86e409b4..a461558f 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -83,7 +83,8 @@ def _recv_pkt(self): def _connect(self): if self._timeout: - self._socket = socket.create_connection(self._address, self._timeout) + self._socket = socket.create_connection(self._address, + self._timeout) else: self._socket = socket.create_connection(self._address) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 542e97fb..fe668009 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -25,7 +25,6 @@ from connection import RiakPbcConnection from stream import RiakPbcKeyStream, RiakPbcMapredStream from codec import RiakPbcCodec -from riak.riak_object import RiakObject from messages import ( MSG_CODE_PING_REQ, @@ -115,28 +114,10 @@ def _set_client_id(self, client_id): client_id = property(_get_client_id, _set_client_id, doc="""the client ID for this connection""") - def _decoded_contents(self, resp, old_obj): - contents = [] - for c in resp.content: - new_obj = RiakObject(old_obj.client, old_obj.bucket, old_obj.key) - new_obj.vclock = resp.vclock - contents.append(self.decode_content(c, new_obj)) - if contents: - ret = contents[0] - if len(contents) > 1: - ret.siblings = contents[:] - return ret - else: - old_obj.exists = False - return old_obj - - def get(self, robj, r=None, pr=None, vtag=None): + def get(self, robj, r=None, pr=None): """ Serialize get request and deserialize response """ - if vtag is not None: - raise RiakError("PB transport does not support vtags") - bucket = robj.bucket req = riak_pb.RpbGetReq() @@ -178,57 +159,24 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, req.if_none_match = 1 req.bucket = bucket.name - req.key = robj.key + if robj.key: + req.key = robj.key if robj.vclock: req.vclock = robj.vclock - self.encode_content(robj, req.content) + self._encode_content(robj, req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, MSG_CODE_PUT_RESP) + if resp is not None: return self._decoded_contents(resp, robj) - - def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """Put a new object into the Riak store, returning its (new) key. - - If return_meta is False, then the vlock and metadata return values - will be None. - - @return robj - """ - # Note that this won't work on 0.14 nodes. - bucket = robj.bucket - - req = riak_pb.RpbPutReq() - if w: - req.w = self.translate_rw_val(w) - if dw: - req.dw = self.translate_rw_val(dw) - if self.quorum_controls() and pw: - req.pw = self.translate_rw_val(pw) - - if return_body: - req.return_body = 1 - if if_none_match: - req.if_none_match = 1 - - req.bucket = bucket.name - - self.encode_content(robj, req.content) - - msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, - MSG_CODE_PUT_RESP) - if not resp: + elif not robj.key: raise RiakError("missing response object") - if len(resp.content) != 1: - raise RiakError("siblings were returned from object creation") + else: + return robj - robj.key = resp.key - robj.vclock = resp.vclock - content = self.decode_content(resp.content[0], robj) - return content + put_new = put def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 53615853..390a83a8 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -65,7 +65,7 @@ def ping(self): """ raise NotImplementedError - def get(self, robj, r=None, vtag=None): + def get(self, robj, r=None): """ Serialize get request and deserialize response @return (vclock=None, [(metadata, value)]=None) From 8d3a083f1194025829ab8bdaf15dd5384680a094 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 28 Apr 2013 06:13:20 -0500 Subject: [PATCH 002/672] Correct usage and handling of last_modified metadata. --- riak/content.py | 5 +++-- riak/riak_object.py | 12 ++++++++++-- riak/transports/http/codec.py | 3 ++- riak/transports/pbc/codec.py | 6 +++--- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/riak/content.py b/riak/content.py index f19606b7..dce93a4e 100644 --- a/riak/content.py +++ b/riak/content.py @@ -27,14 +27,15 @@ class RiakContent(object): """ def __init__(self, robject, data=None, encoded_data=None, charset=None, content_type='application/json', content_encoding=None, - etag=None, usermeta=None, links=None, indexes=None, - exists=False): + last_modified=None, etag=None, usermeta=None, links=None, + indexes=None, exists=False): self._robject = robject self._data = data self._encoded_data = encoded_data self.charset = charset self.content_type = content_type self.content_encoding = content_encoding + self.last_modified = last_modified self.etag = etag self.usermeta = usermeta or {} self.links = links or [] diff --git a/riak/riak_object.py b/riak/riak_object.py index eee42db8..faac0822 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -124,12 +124,20 @@ def __ne__(self, other): charset = content_property('charset', doc=""" The character set of the encoded data :type string""") - content_type = content_property('content_type', doc=""" The MIME media type of the encoded data :type string""") + content_encoding = content_property('content_encoding', doc=""" + The encoding (compression) of the encoded data. Valid values + are identity, deflate, gzip + :type string""") - content_encoding = content_property('content_encoding') + last_modified = content_property('last_modified', """ + The UNIX timestamp of the modification time of this value. + :type float""") + etag = content_property('etag', """ + A unique entity-tag for the value. + :type string""") usermeta = content_property('usermeta', doc=""" Arbitrary user-defined metadata, mapping strings to strings. diff --git a/riak/transports/http/codec.py b/riak/transports/http/codec.py index c2896f3f..a884634c 100644 --- a/riak/transports/http/codec.py +++ b/riak/transports/http/codec.py @@ -28,6 +28,7 @@ import urllib from cgi import parse_header from email import message_from_string +from rfc822 import parsedate_tz, mktime_tz from xml.etree import ElementTree from riak import RiakError from riak.content import RiakContent @@ -110,7 +111,7 @@ def _parse_sibling(self, sibling, headers, data): elif header == 'link': sibling.links = self._parse_links(value) elif header == 'last-modified': - sibling.last_modified = value + sibling.last_modified = mktime_tz(parsedate_tz(value)) elif header.startswith('x-riak-meta-'): metakey = header.replace('x-riak-meta-', '') sibling.usermeta[metakey] = value diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index e24b8697..61202e48 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -94,9 +94,9 @@ def _decode_content(self, rpb_content, sibling): sibling.links = [self._decode_link(link) for link in rpb_content.links] if rpb_content.HasField("last_mod"): - sibling.last_mod = rpb_content.last_mod - if rpb_content.HasField("last_mod_usecs"): - sibling.last_mod_usecs = rpb_content.last_mod_usecs + sibling.last_modified = float(rpb_content.last_mod) + if rpb_content.HasField("last_mod_usecs"): + sibling.last_modified += rpb_content.last_mod_usecs / 1000000.0 sibling.usermeta = dict([(usermd.key, usermd.value) for usermd in rpb_content.usermeta]) From cce4cae176d1cc2c185185dd9046c72d1b9ca1a6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 28 Apr 2013 06:49:31 -0500 Subject: [PATCH 003/672] Normalize handling of vector clocks across transports. This addresses the possibility that an object might be fetched from one interface/transport and stored in the other. Instead, we wrap the raw value in an object that can handle decoding and encoding the format that the transport requires. --- riak/riak_object.py | 31 +++++++++++++++++++++++++++++++ riak/transports/http/codec.py | 5 +++-- riak/transports/http/transport.py | 2 +- riak/transports/pbc/codec.py | 3 ++- riak/transports/pbc/transport.py | 4 ++-- 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index faac0822..c9afe978 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -21,6 +21,7 @@ from riak import ConflictError from riak.content import RiakContent from riak.util import deprecated +import base64 def content_property(name, doc=None): @@ -58,6 +59,36 @@ def _delegate(self, *args, **kwargs): return _delegate +class VClock(object): + """ + A representation of a vector clock received from Riak. + """ + + _decoders = { + 'base64': base64.b64decode, + 'binary': str + } + + _encoders = { + 'base64': base64.b64encode, + 'binary': str + } + + def __init__(self, value, encoding): + self._vclock = self._decoders[encoding].__call__(value) + + def encode(self, encoding): + if encoding in self._encoders: + return self._encoders[encoding].__call__(self._vclock) + else: + raise ValueError('{} is not a valid vector clock encoding'. + format(encoding)) + + def __repr__(self): + return '<{} {}>'.format(self.__class__.__name__, + self.encode('base64')) + + class RiakObject(object): """ The RiakObject holds meta information about a Riak object, plus the diff --git a/riak/transports/http/codec.py b/riak/transports/http/codec.py index a884634c..1ef30129 100644 --- a/riak/transports/http/codec.py +++ b/riak/transports/http/codec.py @@ -32,6 +32,7 @@ from xml.etree import ElementTree from riak import RiakError from riak.content import RiakContent +from riak.riak_object import VClock from riak.multidict import MultiDict from riak.transports.http.search import XMLSearchResult @@ -62,7 +63,7 @@ def _parse_body(self, robj, response, expected_statuses): self.check_http_code(status, expected_statuses) if 'x-riak-vclock' in headers: - robj.vclock = headers['x-riak-vclock'] + robj.vclock = VClock(headers['x-riak-vclock'], 'base64') # If 404(Not Found), then clear the object. if status == 404: @@ -191,7 +192,7 @@ def _build_put_headers(self, robj, if_none_match=False): # Add the vclock if it exists... if robj.vclock is not None: - headers['X-Riak-Vclock'] = robj.vclock + headers['X-Riak-Vclock'] = robj.vclock.encode('base64') # Create the header from metadata self._add_links_for_riak_object(robj, headers) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 460f85de..c80ceb33 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -154,7 +154,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): headers = {} url = self.object_path(robj.bucket.name, robj.key, **params) if self.tombstone_vclocks() and robj.vclock is not None: - headers['X-Riak-Vclock'] = robj.vclock + headers['X-Riak-Vclock'] = robj.vclock.encode('base64') response = self._request('DELETE', url, headers) self.check_http_code(response[0], [204, 404]) return self diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 61202e48..7b2b08f0 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -17,6 +17,7 @@ """ import riak_pb from riak import RiakError +from riak.riak_object import VClock from riak.content import RiakContent from riak.mapreduce import RiakLink @@ -64,7 +65,7 @@ def _decoded_contents(self, resp, obj): if type(resp) == riak_pb.RpbPutResp and resp.HasField('key'): obj.key = resp.key if resp.HasField("vclock"): - obj.vclock = resp.vclock + obj.vclock = VClock(resp.vclock, 'binary') obj.siblings = [self._decode_content(c, RiakContent(obj)) for c in resp.content] diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index fe668009..e43b347b 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -162,7 +162,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, if robj.key: req.key = robj.key if robj.vclock: - req.vclock = robj.vclock + req.vclock = robj.vclock.encode('binary') self._encode_content(robj, req.content) @@ -201,7 +201,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): req.pw = self.translate_rw_val(pw) if self.tombstone_vclocks() and robj.vclock: - req.vclock = robj.vclock + req.vclock = robj.vclock.encode('binary') req.bucket = bucket.name req.key = robj.key From 428842f380fd7e31827d06aa46a5707fcb358b9a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 30 Apr 2013 11:12:53 -0500 Subject: [PATCH 004/672] Remove put_new from transports. --- riak/client/operations.py | 25 ------------------------- riak/riak_object.py | 12 +++--------- riak/transports/http/transport.py | 2 -- riak/transports/pbc/transport.py | 2 -- riak/transports/transport.py | 10 ---------- 5 files changed, 3 insertions(+), 48 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index cf4cc46f..3c812e59 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -154,31 +154,6 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, return_body=return_body, if_none_match=if_none_match) - @retryable - def put_new(self, transport, robj, w=None, dw=None, pw=None, - return_body=None, if_none_match=None): - """ - Stores an object in the Riak cluster with a generated key. - - :param robj: the object to store - :type robj: RiakObject - :param w: the write quorum - :type w: integer, string, None - :param dw: the durable write quorum - :type dw: integer, string, None - :param pw: the primary write quorum - :type pw: integer, string, None - :param return_body: whether to return the resulting object - after the write - :type return_body: boolean - :param if_none_match: whether to fail the write if the object - exists - :type if_none_match: boolean - """ - return transport.put_new(robj, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) - @retryable def get(self, transport, robj, r=None, pr=None): """ diff --git a/riak/riak_object.py b/riak/riak_object.py index c9afe978..247f092e 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -240,15 +240,9 @@ def store(self, w=None, dw=None, pw=None, return_body=True, raise ConflictError("Attempting to store an invalid object, " "resolve the siblings first") - if self.key is None: - self.client.put_new( - self, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) - else: - self.client.put(self, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) + self.client.put(self, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) return self diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index c80ceb33..4824d0b9 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -142,8 +142,6 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, self.check_http_code(response[0], expect) return None - put_new = put - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Delete an object. diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index e43b347b..738c67e1 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -176,8 +176,6 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, else: return robj - put_new = put - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Serialize get request and deserialize response diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 390a83a8..c6d581c7 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -80,16 +80,6 @@ def put(self, robj, w=None, dw=None, return_body=True): """ raise NotImplementedError - def put_new(self, robj, w=None, dw=None, return_meta=True): - """Put a new object into the Riak store, returning its (new) key. - - If return_meta is False, then the vlock and metadata return values - will be None. - - @return (key, vclock, metadata) - """ - raise NotImplementedError - def delete(self, robj, rw=None): """ Serialize delete request and deserialize response From 09993fab769420fda3caa1def178091b25487450 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 1 May 2013 10:02:46 -0500 Subject: [PATCH 005/672] Normalize handling of get/put responses in PBC to prevent clobbering of object data. --- riak/transports/pbc/codec.py | 10 ++-------- riak/transports/pbc/transport.py | 32 +++++++++++++++++++++++++------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 7b2b08f0..c5a4dd9b 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -17,7 +17,6 @@ """ import riak_pb from riak import RiakError -from riak.riak_object import VClock from riak.content import RiakContent from riak.mapreduce import RiakLink @@ -61,14 +60,9 @@ def translate_rw_val(self, rw): else: return None - def _decoded_contents(self, resp, obj): - if type(resp) == riak_pb.RpbPutResp and resp.HasField('key'): - obj.key = resp.key - if resp.HasField("vclock"): - obj.vclock = VClock(resp.vclock, 'binary') - + def _decode_contents(self, contents, obj): obj.siblings = [self._decode_content(c, RiakContent(obj)) - for c in resp.content] + for c in contents] return obj def _decode_content(self, rpb_content, sibling): diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 738c67e1..c59c30d9 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -22,6 +22,7 @@ import riak_pb from riak import RiakError from riak.transports.transport import RiakTransport +from riak.riak_object import VClock from connection import RiakPbcConnection from stream import RiakPbcKeyStream, RiakPbcMapredStream from codec import RiakPbcCodec @@ -132,11 +133,23 @@ def get(self, robj, r=None, pr=None): req.bucket = bucket.name req.key = robj.key - msg_code, resp = self._request(MSG_CODE_GET_REQ, req) - if msg_code == MSG_CODE_GET_RESP: - return self._decoded_contents(resp, robj) + msg_code, resp = self._request(MSG_CODE_GET_REQ, req, + MSG_CODE_GET_RESP) + + # TODO: support if_modified flag + + if resp is not None: + if resp.HasField('vclock'): + robj.vclock = VClock(resp.vclock, 'binary') + # We should do this even if there are no contents, i.e. + # the object is tombstoned + self._decode_contents(resp.content, robj) else: - return None + # "not found" returns an empty message, + # so let's make sure to clear the siblings + robj.siblings = [] + + return robj def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): @@ -170,11 +183,16 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, MSG_CODE_PUT_RESP) if resp is not None: - return self._decoded_contents(resp, robj) + if resp.HasField('key'): + robj.key = resp.key + if resp.HasField("vclock"): + robj.vclock = VClock(resp.vclock, 'binary') + if resp.content: + self._decode_contents(resp.content, robj) elif not robj.key: raise RiakError("missing response object") - else: - return robj + + return robj def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ From 6f678e1b283f08ff4a622450c378ca87389e9944 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 1 May 2013 10:05:15 -0500 Subject: [PATCH 006/672] HTTP uses a plain tuple for links, so should PBC. --- riak/transports/pbc/codec.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index c5a4dd9b..db8647b4 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -18,7 +18,6 @@ import riak_pb from riak import RiakError from riak.content import RiakContent -from riak.mapreduce import RiakLink RIAKC_RW_ONE = 4294967294 RIAKC_RW_QUORUM = 4294967293 @@ -158,7 +157,7 @@ def _decode_link(self, link): else: tag = None - return RiakLink(bucket, key, tag) + return (bucket, key, tag) def _decode_index_value(self, index, value): """ From effc1d84506ce2640b3f6b9df22e594b2f675c70 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 1 May 2013 10:08:35 -0500 Subject: [PATCH 007/672] Correct documentation on exists property. --- riak/riak_object.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index 247f092e..ec4e8786 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -203,8 +203,11 @@ def _exists(self): return self.siblings[0].exists exists = property(_exists, None, doc=""" - Whether the object exists. This is only true when there is a - single sibling and it is neither a tombstone nor unsaved.""") + Whether the object exists. This is only False when there are no + siblings (the object was not found), or the solitary sibling is + a tombstone. + :type bool + """) def get_sibling(self, index): deprecated("RiakObject.get_sibling is deprecated, use the " From 1f5337787df425a68a22f490c03f59247ae603a9 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 3 May 2013 11:36:15 -0500 Subject: [PATCH 008/672] Correct documentation of _decode_link method. It returns a bare tuple, not a named RiakLink tuple. --- riak/transports/pbc/codec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index db8647b4..7ee0b0e0 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -141,7 +141,7 @@ def _encode_content(self, robj, rpb_content): def _decode_link(self, link): """ - Decodes an RpbLink message into a RiakLink named tuple + Decodes an RpbLink message into a tuple """ if link.HasField("bucket"): From 3afe011b4102c12874bb97cae81d70884f8d6eff Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 7 May 2013 14:42:48 -0500 Subject: [PATCH 009/672] Add test for sibling tombstones. --- riak/tests/test_kv.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 880245bc..5eaf9325 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -297,6 +297,42 @@ def test_siblings(self): self.assertEqual(len(obj.siblings), 1) self.assertEqual(obj.encoded_data, resolved_sibling.encoded_data) + def test_tombstone_siblings(self): + # Set up the bucket, clear any existing object... + bucket = self.client.bucket(self.sibs_bucket) + obj = bucket.get(self.key_name) + bucket.allow_mult = True + + obj.encoded_data = 'start' + obj.content_type = 'application/octet-stream' + obj.store(return_body=True) + + vclock = obj.vclock + obj.delete() + + vals = set() + for i in range(4): + while True: + randval = self.randint() + if str(randval) not in vals: + break + + other_obj = bucket.new(key=self.key_name, + encoded_data=str(randval), + content_type='text/plain') + other_obj.vclock = vclock + other_obj.store() + vals.add(str(randval)) + + obj = bucket.get(self.key_name) + self.assertEqual(len(obj.siblings), 5) + non_tombstones = 0 + for sib in obj.siblings: + if sib.exists: + non_tombstones += 1 + self.assertTrue(sib.encoded_data in vals or not sib.exists) + self.assertEqual(non_tombstones, 4) + def test_store_of_missing_object(self): bucket = self.client.bucket(self.bucket_name) # for json objects From ce58986dbbf282c7055b922aaa3b458503014c59 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 7 May 2013 14:43:31 -0500 Subject: [PATCH 010/672] whitespace cleanup --- riak/riak_object.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/riak/riak_object.py b/riak/riak_object.py index ec4e8786..92d6ec78 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -144,6 +144,7 @@ def __ne__(self, other): property into Python values. The decoding is dependent on the `content_type` property and the bucket's registered decoders. :type mixed """) + encoded_data = content_property('encoded_data', doc=""" The raw data stored in this object, essentially the encoded form of the `data` property. If unset, accessing this property @@ -155,9 +156,11 @@ def __ne__(self, other): charset = content_property('charset', doc=""" The character set of the encoded data :type string""") + content_type = content_property('content_type', doc=""" The MIME media type of the encoded data :type string""") + content_encoding = content_property('content_encoding', doc=""" The encoding (compression) of the encoded data. Valid values are identity, deflate, gzip @@ -166,6 +169,7 @@ def __ne__(self, other): last_modified = content_property('last_modified', """ The UNIX timestamp of the modification time of this value. :type float""") + etag = content_property('etag', """ A unique entity-tag for the value. :type string""") From 434394346d39c9124c69448d505791c823f61bad Mon Sep 17 00:00:00 2001 From: William Kral Date: Thu, 23 May 2013 12:40:08 -0700 Subject: [PATCH 011/672] Added text/plain encoding handling by default - The old version of the client handled this out of the box - Added a test to ensure it works --- .gitignore | 3 +++ riak/client/__init__.py | 6 ++++-- riak/tests/test_kv.py | 8 ++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 34e7a5bb..29341d74 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ riak.egg-info/ #*# *~ + +Vagrantfile +.vagrant* diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 2dea7a62..63c50202 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -100,9 +100,11 @@ def __init__(self, protocol='http', transport_options={}, self._pb_pool = RiakPbcPool(self, **transport_options) self._encoders = {'application/json': default_encoder, - 'text/json': default_encoder} + 'text/json': default_encoder, + 'text/plain': unicode} self._decoders = {'application/json': json.loads, - 'text/json': json.loads} + 'text/json': json.loads, + 'text/plain': unicode} self._buckets = WeakValueDictionary() def _get_protocol(self): diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 5eaf9325..6b93131b 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -183,6 +183,14 @@ def test_unknown_content_type_encoder_decoder(self): obj2 = bucket.get(self.key_name) self.assertEqual(data, obj2.encoded_data) + def test_text_plain_encoder_decoder(self): + bucket = self.client.bucket(self.bucket_name) + data = "some funny data" + obj = bucket.new(self.key_name, data, content_type='text/plain') + obj.store() + obj2 = bucket.get(self.key_name) + self.assertEqual(data, obj2.data) + def test_missing_object(self): bucket = self.client.bucket(self.bucket_name) obj = bucket.get(self.key_name) From fd18c4da7b22be57010fe683ce398ae3171fdd58 Mon Sep 17 00:00:00 2001 From: William Kral Date: Thu, 23 May 2013 14:40:34 -0700 Subject: [PATCH 012/672] Speed up test server start - Seemed to slow down with riak 1.3 which seems to be because of much more text output when starting - wait_for_erlang_prompt was reading one character at a time and running a long regex against the output which was causing the output of riak console to stall - Changed to read a line at a time speeds up from 34 seconds to 2.4 on my system --- riak/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/test_server.py b/riak/test_server.py index 73df069d..a4978e3f 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -199,7 +199,7 @@ def wait_for_erlang_prompt(self): prompted = False buffer = "" while not prompted: - line = self._server.stdout.read(1) + line = self._server.stdout.readline() if len(line) > 0: buffer += line if re.search(r"\(%s\)\d+>" % self.vm_args["-name"], buffer): From 0be3ff06406eb70a514cf5f0e86f11437f864a89 Mon Sep 17 00:00:00 2001 From: William Kral Date: Wed, 29 May 2013 15:36:12 -0700 Subject: [PATCH 013/672] Moving ignored personal stuff to .git/info/exclude --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index 29341d74..34e7a5bb 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,3 @@ riak.egg-info/ #*# *~ - -Vagrantfile -.vagrant* From c6bbe884ebfb06370fa95276c2cc4b13e0eec6eb Mon Sep 17 00:00:00 2001 From: William Kral Date: Wed, 29 May 2013 19:17:38 -0700 Subject: [PATCH 014/672] Changed decoder to simple str and removed encoder --- riak/client/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 63c50202..3af3a555 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -100,11 +100,10 @@ def __init__(self, protocol='http', transport_options={}, self._pb_pool = RiakPbcPool(self, **transport_options) self._encoders = {'application/json': default_encoder, - 'text/json': default_encoder, - 'text/plain': unicode} + 'text/json': default_encoder} self._decoders = {'application/json': json.loads, 'text/json': json.loads, - 'text/plain': unicode} + 'text/plain': str} self._buckets = WeakValueDictionary() def _get_protocol(self): From 68d2d87289c98ccad675d80ba8b2c1d7c39e008a Mon Sep 17 00:00:00 2001 From: William Kral Date: Wed, 29 May 2013 19:33:07 -0700 Subject: [PATCH 015/672] Fixed RiakHttpTransport with non-ascii streams - Encoded data that has octets outside the 128 byte range fail in python's httplib - Using a bytearray for the message body causes the entire stream to be converted to a bytearray when a str object is added to it and the message is transported correctly - Added two tests one with unicode data in a json object and one unicode data in a string --- riak/tests/test_kv.py | 18 ++++++++++++++++++ riak/transports/http/transport.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 5eaf9325..6da5f4d1 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -70,6 +70,24 @@ def test_store_and_get(self): obj2 = bucket.get('baz') self.assertEqual(obj2.data, rand) + def test_store_obj_with_unicode(self): + bucket = self.client.bucket(self.bucket_name) + data = {u'føø': u'éå'} + obj = bucket.new('foo', data) + obj.store() + obj = bucket.get('foo') + self.assertEqual(obj.data, data) + + def test_store_unicode_string(self): + bucket = self.client.bucket(self.bucket_name) + data = u"some unicode data: \u00c6" + obj = bucket.new(self.key_name, encoded_data=data.encode('utf-8'), + content_type='text/plain') + obj.charset = 'utf-8' + obj.store() + obj2 = bucket.get(self.key_name) + self.assertEqual(data, obj2.encoded_data.decode('utf-8')) + def test_generate_key(self): # Ensure that Riak generates a random key when # the key passed to bucket.new() is None. diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 4824d0b9..632f37bf 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -126,7 +126,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} url = self.object_path(robj.bucket.name, robj.key, **params) headers = self._build_put_headers(robj, if_none_match=if_none_match) - content = robj.encoded_data + content = bytearray(robj.encoded_data) if robj.key is None: expect = [201] From 609388736fd3ed7d9d0ed4919b7caa10f09e6e59 Mon Sep 17 00:00:00 2001 From: William Kral Date: Thu, 30 May 2013 10:34:34 -0700 Subject: [PATCH 016/672] Added encoder back for symmetry --- riak/client/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 3af3a555..c49b5e4b 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -100,7 +100,8 @@ def __init__(self, protocol='http', transport_options={}, self._pb_pool = RiakPbcPool(self, **transport_options) self._encoders = {'application/json': default_encoder, - 'text/json': default_encoder} + 'text/json': default_encoder, + 'text/plain': str} self._decoders = {'application/json': json.loads, 'text/json': json.loads, 'text/plain': str} From 05166ce50c02a4d8aa977cdbe7732a0d091a3edd Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 31 May 2013 17:19:29 -0500 Subject: [PATCH 017/672] Allow RiakClientOperations.fulltext_search/add/delete to run under PBC as well as HTTP --- riak/client/operations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 3c812e59..ee8535f5 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -232,7 +232,7 @@ def stream_mapred(self, inputs, query, timeout): finally: stream.close() - @retryableHttpOnly + @retryable def fulltext_search(self, transport, index, query, **params): """ Performs a full-text search query. @@ -246,7 +246,7 @@ def fulltext_search(self, transport, index, query, **params): """ return transport.search(index, query, **params) - @retryableHttpOnly + @retryable def fulltext_add(self, transport, index, docs): """ Adds documents to the full-text index. @@ -258,7 +258,7 @@ def fulltext_add(self, transport, index, docs): """ transport.fulltext_add(index, docs) - @retryableHttpOnly + @retryable def fulltext_delete(self, transport, index, docs=None, queries=None): """ Removes documents from the full-text index. From d1bba86f26b0f5f65545d2a34eff2fd9b23ca5fb Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 31 May 2013 17:32:16 -0500 Subject: [PATCH 018/672] Attempt to install latest Riak on Travis builder. --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a47837e0..15055442 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,12 @@ language: python python: - "2.6" - "2.7" +before_install: + - "curl http://apt.basho.com/gpg/basho.apt.key | sudo apt-key add -" + - 'sudo bash -c "echo deb http://apt.basho.com $(lsb_release -sc) main > /etc/apt/sources.list.d/basho.list"' + - "sudo apt-get update" + - "sudo apt-get upgrade riak" + - "sudo service start riak" install: - ./setup.py develop - ./setup.py easy_install protobuf @@ -9,5 +15,5 @@ script: ./setup.py test before_script: sudo /usr/sbin/search-cmd install searchbucket notifications: email: clients@basho.com -services: - - riak +# services: +# - riak From 8192ce9bd2efbe1ab4198fb44ecdb51f040621cd Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 31 May 2013 17:46:38 -0500 Subject: [PATCH 019/672] Don't override the existing Riak configs. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 15055442..859e7f32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ before_install: - "curl http://apt.basho.com/gpg/basho.apt.key | sudo apt-key add -" - 'sudo bash -c "echo deb http://apt.basho.com $(lsb_release -sc) main > /etc/apt/sources.list.d/basho.list"' - "sudo apt-get update" - - "sudo apt-get upgrade riak" + - "sudo apt-get install riak --assume-no" - "sudo service start riak" install: - ./setup.py develop From e7526452d512566faed0bccbcb0a2676d6cdcb07 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 31 May 2013 17:50:58 -0500 Subject: [PATCH 020/672] Pipe `yes n` to the apt-get install command --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 859e7f32..b5cec402 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ before_install: - "curl http://apt.basho.com/gpg/basho.apt.key | sudo apt-key add -" - 'sudo bash -c "echo deb http://apt.basho.com $(lsb_release -sc) main > /etc/apt/sources.list.d/basho.list"' - "sudo apt-get update" - - "sudo apt-get install riak --assume-no" + - "yes n | sudo apt-get install riak" - "sudo service start riak" install: - ./setup.py develop From af75328b7eeb343f08a443569a9231bf20bcddcb Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 31 May 2013 17:52:55 -0500 Subject: [PATCH 021/672] Transpose arguments of the service command. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b5cec402..e23e20ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ before_install: - 'sudo bash -c "echo deb http://apt.basho.com $(lsb_release -sc) main > /etc/apt/sources.list.d/basho.list"' - "sudo apt-get update" - "yes n | sudo apt-get install riak" - - "sudo service start riak" + - "sudo service riak start" install: - ./setup.py develop - ./setup.py easy_install protobuf From bb5db6e61595e17831dae15cd00a527fd63b94c7 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 31 May 2013 17:59:10 -0500 Subject: [PATCH 022/672] Cleanup the builder config. [ci skip] --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e23e20ae..4ce6d1f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,5 +15,3 @@ script: ./setup.py test before_script: sudo /usr/sbin/search-cmd install searchbucket notifications: email: clients@basho.com -# services: -# - riak From ed5297aeb13f6ace574936212627333f67db600c Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 31 May 2013 19:18:35 -0500 Subject: [PATCH 023/672] Switch fulltext_add/delete back to @retryableHttpOnly --- riak/client/operations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index ee8535f5..c928128f 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -246,7 +246,7 @@ def fulltext_search(self, transport, index, query, **params): """ return transport.search(index, query, **params) - @retryable + @retryableHttpOnly def fulltext_add(self, transport, index, docs): """ Adds documents to the full-text index. @@ -258,7 +258,7 @@ def fulltext_add(self, transport, index, docs): """ transport.fulltext_add(index, docs) - @retryable + @retryableHttpOnly def fulltext_delete(self, transport, index, docs=None, queries=None): """ Removes documents from the full-text index. From 9e47e0e154a9da52d33127119e6b979455e950b2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 13 Jun 2013 08:22:04 -0500 Subject: [PATCH 024/672] Add sibling-resolution functions. A resolver function takes a RiakObject and (hopefully) resolves its siblings into a single sibling. Resolvers are invoked automatically when fetching an object returns siblings. The resolver can be set on the client, the bucket, or the individual object. If the object in conflict has no resolver, the bucket's resolver will be used, and then the client's resolver if the bucket has none. The default resolver does no resolution. Also included (but not assigned) is a resolver that selects the latest sibling based on the `last_modified` property; see riak/resolver.py for more details. --- riak/bucket.py | 18 +++++++++ riak/client/__init__.py | 3 +- riak/resolver.py | 42 +++++++++++++++++++++ riak/riak_object.py | 18 +++++++++ riak/tests/test_kv.py | 70 ++++++++++++++++++++++++++++++++++- riak/transports/http/codec.py | 6 +++ riak/transports/pbc/codec.py | 3 ++ 7 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 riak/resolver.py diff --git a/riak/bucket.py b/riak/bucket.py index 3b255885..4aeb54d0 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -54,6 +54,7 @@ def __init__(self, client, name): self.name = name self._encoders = {} self._decoders = {} + self._resolver = None def __hash__(self): return hash((self.name, self._client)) @@ -199,6 +200,23 @@ def get_binary(self, key, r=None, pr=None): 'use RiakBucket.get') return self.get(key, r=r, pr=pr) + def _get_resolver(self): + if callable(self._resolver): + return self._resolver + elif self._resolver is None: + return self._client.resolver + else: + raise TypeError("resolver is not a function") + + def _set_resolver(self, value): + self._resolver = value + + resolver = property(_get_resolver, _set_resolver, doc= + """The sibling-resolution function for this + bucket. If the resolver is not set, the + client's resolver will be used. :type + callable""") + def _set_n_val(self, nval): return self.set_property('n_val', nval) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index c49b5e4b..1ca2c018 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -30,6 +30,7 @@ from riak.node import RiakNode from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduceChain +from riak.resolver import default_resolver from riak.search import RiakSearch from riak.transports.http import RiakHttpPool from riak.transports.pbc import RiakPbcPool @@ -95,7 +96,7 @@ def __init__(self, protocol='http', transport_options={}, self.nodes = [self._create_node(n) for n in nodes] self.protocol = protocol or 'http' - + self.resolver = default_resolver self._http_pool = RiakHttpPool(self, **transport_options) self._pb_pool = RiakPbcPool(self, **transport_options) diff --git a/riak/resolver.py b/riak/resolver.py new file mode 100644 index 00000000..30bfcc73 --- /dev/null +++ b/riak/resolver.py @@ -0,0 +1,42 @@ +""" +Copyright 2013 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + + +def default_resolver(riak_object): + """ + The default conflict-resolution function, which does nothing. To + implement a resolver, define a function that sets the ``siblings`` + property on the passed ``RiakObject`` instance to a list + containing a single ``RiakContent`` object. + + :param riak_object: an object-in-conflict that will be resolved + :type riak_object: RiakObject + """ + pass + + +def last_written_resolver(riak_object): + """ + A conflict-resolution function that resolves by selecting the most + recently-modified sibling by timestamp. + + :param riak_object: an object-in-conflict that will be resolved + :type riak_object: RiakObject + """ + lm = lambda x: x.last_modified + riak_object.siblings = [max(riak_object.siblings, key=lm), ] diff --git a/riak/riak_object.py b/riak/riak_object.py index 92d6ec78..bcc4559b 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -116,6 +116,7 @@ def __init__(self, client, bucket, key=None): raise ValueError('Key name must either be "None"' ' or a non-empty string.') + self._resolver = None self.client = client self.bucket = bucket self.key = key @@ -213,6 +214,23 @@ def _exists(self): :type bool """) + def _get_resolver(self): + if callable(self._resolver): + return self._resolver + elif self._resolver is None: + return self.bucket.resolver + else: + raise TypeError("resolver is not a function") + + def _set_resolver(self, value): + self._resolver = value + + resolver = property(_get_resolver, _set_resolver, doc= + """The sibling-resolution function for this + object. If the resolver is not set, the + bucket's resolver will be used. :type + callable""") + def get_sibling(self, index): deprecated("RiakObject.get_sibling is deprecated, use the " "siblings property instead") diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index b42a1ed4..fe55c2ee 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -2,8 +2,9 @@ import os import cPickle import copy +from time import sleep from riak import ConflictError - +from riak.resolver import default_resolver, last_written_resolver try: import simplejson as json except ImportError: @@ -323,6 +324,73 @@ def test_siblings(self): self.assertEqual(len(obj.siblings), 1) self.assertEqual(obj.encoded_data, resolved_sibling.encoded_data) + def test_resolution(self): + bucket = self.client.bucket(self.sibs_bucket) + obj = bucket.get(self.key_name) + bucket.allow_mult = True + + # Even if it previously existed, let's store a base resolved version + # from which we can diverge by sending a stale vclock. + obj.encoded_data = 'start' + obj.content_type = 'text/plain' + obj.store() + + # Store the same object five times... + # First run through should overwrite the datum 'start' above + other_client = self.create_client() + other_bucket = other_client.bucket(self.sibs_bucket) + + vals = [] + for i in range(5): + while True: + randval = self.randint() + if str(randval) not in vals: + break + + other_obj = other_bucket.new(key=self.key_name, + encoded_data=str(randval), + content_type='text/plain') + other_obj.vclock = obj.vclock + other_obj.store() + vals.append(str(randval)) + # TODO: This sleep exists so that last_written_resolver + # will find timestamps in different seconds. HTTP dates do + # not have enough significant digits for sub-second + # differences. + sleep(0.75) + + # Make sure the object has five siblings when using the + # default resolver + obj = bucket.get(self.key_name) + obj.reload() + self.assertEqual(len(obj.siblings), 5) + + # Setting the resolver on the client object to use the + # "last-write-wins" behavior + self.client.resolver = last_written_resolver + obj.reload() + self.assertEqual(obj.resolver, last_written_resolver) + self.assertEqual(1, len(obj.siblings)) + self.assertEqual(obj.data, vals[-1]) + + # Set the resolver on the bucket to the default resolver, + # overriding the resolver on the client + bucket.resolver = default_resolver + obj.reload() + self.assertEqual(obj.resolver, default_resolver) + self.assertEqual(len(obj.siblings), 5) + + # Define our own custom resolver on the object that returns + # the maximum value, overriding the bucket and client resolvers + def max_value_resolver(obj): + datafun = lambda s: s.data + obj.siblings = [max(obj.siblings, key=datafun), ] + + obj.resolver = max_value_resolver + obj.reload() + self.assertEqual(obj.resolver, max_value_resolver) + self.assertEqual(obj.data, max(vals)) + def test_tombstone_siblings(self): # Set up the bucket, clear any existing object... bucket = self.client.bucket(self.sibs_bucket) diff --git a/riak/transports/http/codec.py b/riak/transports/http/codec.py index 1ef30129..25bb7dab 100644 --- a/riak/transports/http/codec.py +++ b/riak/transports/http/codec.py @@ -85,6 +85,11 @@ def _parse_body(self, robj, response, expected_statuses): part.items(), part.get_payload()) for part in parts] + + # Invoke sibling-resolution logic + if robj.resolver is not None: + robj.resolver(robj) + return robj else: raise Exception('unexpected sibling response format: {0}'. @@ -92,6 +97,7 @@ def _parse_body(self, robj, response, expected_statuses): robj.siblings = [self._parse_sibling(RiakContent(robj), headers.items(), data)] + return robj def _parse_sibling(self, sibling, headers, data): diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 7ee0b0e0..c91dda20 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -62,6 +62,9 @@ def translate_rw_val(self, rw): def _decode_contents(self, contents, obj): obj.siblings = [self._decode_content(c, RiakContent(obj)) for c in contents] + # Invoke sibling-resolution logic + if len(obj.siblings) > 1 and obj.resolver is not None: + obj.resolver(obj) return obj def _decode_content(self, rpb_content, sibling): From 9fa4f81d45bca6f9e05975774ecf7b6a6933daa0 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 13 Jun 2013 15:26:32 -0500 Subject: [PATCH 025/672] Type-check when setting the resolver on bucket and object. --- riak/bucket.py | 5 ++++- riak/riak_object.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 4aeb54d0..29861f45 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -209,7 +209,10 @@ def _get_resolver(self): raise TypeError("resolver is not a function") def _set_resolver(self, value): - self._resolver = value + if value is None or callable(value): + self._resolver = value + else: + raise TypeError("resolver is not a function") resolver = property(_get_resolver, _set_resolver, doc= """The sibling-resolution function for this diff --git a/riak/riak_object.py b/riak/riak_object.py index bcc4559b..48d9b779 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -223,7 +223,10 @@ def _get_resolver(self): raise TypeError("resolver is not a function") def _set_resolver(self, value): - self._resolver = value + if value is None or callable(value): + self._resolver = value + else: + raise TypeError("resolver is not a function") resolver = property(_get_resolver, _set_resolver, doc= """The sibling-resolution function for this From afa3619859a3e8239a779015157d5392bf641f9f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 13 Jun 2013 16:04:13 -0500 Subject: [PATCH 026/672] Factor out the sibling generation into a helper method. --- riak/tests/test_kv.py | 76 ++++++++++++------------------------------- 1 file changed, 21 insertions(+), 55 deletions(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index fe55c2ee..52c1bde6 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -281,24 +281,7 @@ def test_siblings(self): obj.content_type = 'application/octet-stream' obj.store() - # Store the same object five times... - # First run through should overwrite the datum 'start' above - other_client = self.create_client() - other_bucket = other_client.bucket(self.sibs_bucket) - - vals = set() - for i in range(5): - while True: - randval = self.randint() - if str(randval) not in vals: - break - - other_obj = other_bucket.new(key=self.key_name, - encoded_data=str(randval), - content_type='text/plain') - other_obj.vclock = obj.vclock - other_obj.store() - vals.add(str(randval)) + vals = set(self.generate_siblings(obj, count=5)) # Make sure the object has five siblings... obj = bucket.get(self.key_name) @@ -335,29 +318,7 @@ def test_resolution(self): obj.content_type = 'text/plain' obj.store() - # Store the same object five times... - # First run through should overwrite the datum 'start' above - other_client = self.create_client() - other_bucket = other_client.bucket(self.sibs_bucket) - - vals = [] - for i in range(5): - while True: - randval = self.randint() - if str(randval) not in vals: - break - - other_obj = other_bucket.new(key=self.key_name, - encoded_data=str(randval), - content_type='text/plain') - other_obj.vclock = obj.vclock - other_obj.store() - vals.append(str(randval)) - # TODO: This sleep exists so that last_written_resolver - # will find timestamps in different seconds. HTTP dates do - # not have enough significant digits for sub-second - # differences. - sleep(0.75) + vals = self.generate_siblings(obj, count=5, delay=0.75) # Make sure the object has five siblings when using the # default resolver @@ -401,22 +362,9 @@ def test_tombstone_siblings(self): obj.content_type = 'application/octet-stream' obj.store(return_body=True) - vclock = obj.vclock obj.delete() - vals = set() - for i in range(4): - while True: - randval = self.randint() - if str(randval) not in vals: - break - - other_obj = bucket.new(key=self.key_name, - encoded_data=str(randval), - content_type='text/plain') - other_obj.vclock = vclock - other_obj.store() - vals.add(str(randval)) + vals = set(self.generate_siblings(obj, count=4)) obj = bucket.get(self.key_name) self.assertEqual(len(obj.siblings), 5) @@ -465,6 +413,24 @@ def test_list_buckets(self): buckets = self.client.get_buckets() self.assertTrue(self.bucket_name in [x.name for x in buckets]) + def generate_siblings(self, original, count=5, delay=None): + vals = [] + for i in range(count): + while True: + randval = self.randint() + if str(randval) not in vals: + break + + other_obj = original.bucket.new(key=original.key, + encoded_data=str(randval), + content_type='text/plain') + other_obj.vclock = original.vclock + other_obj.store() + vals.append(str(randval)) + if delay: + sleep(delay) + return vals + class HTTPBucketPropsTest(object): def test_rw_settings(self): From f3740561249fdc99fa129324122dd64b4367dd3f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 13:47:28 -0500 Subject: [PATCH 027/672] Touch the ssl_distribution.args_file by running `riak chkconfig`. Closes #245. --- riak/test_server.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/riak/test_server.py b/riak/test_server.py index a4978e3f..8ad93a15 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -125,6 +125,7 @@ def __init__(self, tmp_dir="/tmp/riak/test_server", def prepare(self): if not self._prepared: + self.touch_ssl_distribution_args() self.create_temp_directories() self._riak_script = os.path.join(self._temp_bin, "riak") self.write_riak_script() @@ -243,6 +244,14 @@ def write_app_config(self): app_config.write(erlang_config(self.app_config)) app_config.write(".") + def touch_ssl_distribution_args(self): + # To make sure that the ssl_distribution.args file is present, + # the control script in the source node has to have been run at + # least once. Running the `chkconfig` command is innocuous + # enough to accomplish this without other side-effects. + script = os.path.join(self.bin_dir, "riak") + Popen([script, "chkconfig"]).wait() + def _kv_backend(self): return self.app_config["riak_kv"]["storage_backend"] From d6659941ca3c8884ac7915fd5dff52912bc6b036 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 14:24:40 -0500 Subject: [PATCH 028/672] Add documentation about protocol selection and the lazy-connection semantics. Closes #241. --- riak/client/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 1ca2c018..503109de 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -118,7 +118,19 @@ def _set_protocol(self, value): self._protocol = value protocol = property(_get_protocol, _set_protocol, - doc="""Which protocol to prefer, one of PROTOCOLS""") + doc= + """ + Which protocol to prefer, one of PROTOCOLS. + Please note that when one protocol is + selected, the other protocols MAY NOT attempt + to connect. Changing to another protocol will + cause a connection on the next request. + + Some requests are only valid over 'http' or + 'https', and will always be sent via those + transports, regardless of which protocol is + preferred. + """) def get_transport(self): """ From f2f469956b8b5736e3950dfc9766c2a9d6d9d439 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 15:21:56 -0500 Subject: [PATCH 029/672] Don't open the PBC connection until a request is sent. This prevents exceptions from occuring while trying to create a new transport in the pool, which happens outside a try/except. The result is the same behavior as HTTP, which delays opening a connection until the first request. --- riak/transports/pbc/connection.py | 15 +++++++++------ riak/transports/pbc/transport.py | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index a461558f..c805d26d 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -39,6 +39,7 @@ def _encode_msg(self, msg_code, msg=None): return hdr + msgstr def _request(self, msg_code, msg=None, expect=None): + self._connect() self._send_msg(msg_code, msg) return self._recv_msg(expect) @@ -82,17 +83,19 @@ def _recv_pkt(self): % (len(self._inbuf), self._inbuf_len)) def _connect(self): - if self._timeout: - self._socket = socket.create_connection(self._address, - self._timeout) - else: - self._socket = socket.create_connection(self._address) + if not self._socket: + if self._timeout: + self._socket = socket.create_connection(self._address, + self._timeout) + else: + self._socket = socket.create_connection(self._address) def close(self): """ Closes the underlying socket of the PB connection. """ - self._socket.shutdown(socket.SHUT_RDWR) + if self._socket: + self._socket.shutdown(socket.SHUT_RDWR) def _parse_msg(self, code, packet): try: diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index c59c30d9..31575d16 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -73,7 +73,7 @@ def __init__(self, node=None, client=None, timeout=None, *unused_options): self._node = node self._address = (node.host, node.pb_port) self._timeout = timeout - self._connect() + self._socket = None # FeatureDetection API def _server_version(self): From 7ad11e9124d33d67abc6648ac88798f81279847e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 15:34:51 -0500 Subject: [PATCH 030/672] Raise the final exception when the retry limit has been reached. This prevents masking of errors when the loop falls through to its completion, all tries raising BadResource. The previous behavior was that the function doesn't return any value, essentially returning None. --- riak/client/transport.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/riak/client/transport.py b/riak/client/transport.py index 5c42c55c..4be2bce8 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -58,18 +58,20 @@ def _with_retries(self, pool, fn): def _skip_bad_nodes(transport): return transport._node not in skip_nodes - for retry in range(self.RETRY_COUNT): + retry_count = self.RETRY_COUNT + + for retry in range(retry_count): try: with pool.take(_filter=_skip_bad_nodes) as transport: try: return fn(transport) except (IOError, httplib.HTTPException) as e: - if _is_retryable(e): - transport._node.error_rate.incr(1) + transport._node.error_rate.incr(1) + if retry < (retry_count - 1) and _is_retryable(e): skip_nodes.append(transport._node) raise BadResource(e) else: - raise e + raise except BadResource: continue From 9e0b2061eeea1a76ceed481c0e656b6865a6aa0b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 15:44:01 -0500 Subject: [PATCH 031/672] Move _connect() call into _send_msg() so that streaming doesn't break. --- riak/transports/pbc/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index c805d26d..8609c370 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -39,11 +39,11 @@ def _encode_msg(self, msg_code, msg=None): return hdr + msgstr def _request(self, msg_code, msg=None, expect=None): - self._connect() self._send_msg(msg_code, msg) return self._recv_msg(expect) def _send_msg(self, msg_code, msg): + self._connect() self._socket.send(self._encode_msg(msg_code, msg)) def _recv_msg(self, expect=None): From 61bed7068be6b402150bdf3dc84f53c923997155 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 16:05:13 -0500 Subject: [PATCH 032/672] Test that retry logic re-raises the original exception when tries are exhausted. --- riak/tests/test_all.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 1894cba9..cbccbc98 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -118,6 +118,17 @@ def setUp(self): self.client = self.create_client() +class ClientTests(object): + def test_request_retries(self): + # We guess at some ports that will be unused by Riak or + # anything else. + client = self.create_client(http_port=1023, pb_port=1022) + + # If retries are exhausted, the final result should also be an + # error. + self.assertRaises(IOError, client.ping) + + class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, PbcBucketPropsTest, @@ -128,6 +139,7 @@ class RiakPbcTransportTestCase(BasicKVTests, MapReduceAliasTests, MapReduceStreamTests, SearchTests, + ClientTests, BaseTestCase, unittest.TestCase): @@ -169,6 +181,7 @@ class RiakHttpTransportTestCase(BasicKVTests, EnableSearchTests, SolrSearchTests, SearchTests, + ClientTests, BaseTestCase, unittest.TestCase): From ee94304bb4fad92ee9e3c88b94863b3e30a4f754 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 14 Jun 2013 16:10:40 -0500 Subject: [PATCH 033/672] Eject the bad connection from the pool even on the last try. Also, don't penalize a node that has some unspecified unretryable exception. --- riak/client/transport.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/riak/client/transport.py b/riak/client/transport.py index 4be2bce8..38f4b272 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -66,14 +66,18 @@ def _skip_bad_nodes(transport): try: return fn(transport) except (IOError, httplib.HTTPException) as e: - transport._node.error_rate.incr(1) - if retry < (retry_count - 1) and _is_retryable(e): + if _is_retryable(e): + transport._node.error_rate.incr(1) skip_nodes.append(transport._node) raise BadResource(e) else: raise - except BadResource: - continue + except BadResource as e: + if retry < (retry_count - 1): + continue + else: + # Re-raise the inner exception + raise e.args[0] def _choose_pool(self, protocol=None): """ From c15b22f52f892010892a3fa3b9f13190cdfbf5fd Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 18 Jun 2013 09:45:19 -0500 Subject: [PATCH 034/672] Use pipes for stdio when running chkconfig. --- riak/test_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/test_server.py b/riak/test_server.py index 8ad93a15..a0a5ec44 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -250,7 +250,8 @@ def touch_ssl_distribution_args(self): # least once. Running the `chkconfig` command is innocuous # enough to accomplish this without other side-effects. script = os.path.join(self.bin_dir, "riak") - Popen([script, "chkconfig"]).wait() + Popen([script, "chkconfig"], + stdin=PIPE, stdout=PIPE, stderr=PIPE).communicate() def _kv_backend(self): return self.app_config["riak_kv"]["storage_backend"] From 612ceababed1ad30a869e21f1b62dfc8f56cc325 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 20 Jun 2013 14:31:01 -0500 Subject: [PATCH 035/672] Increased the delay between sibling writes so as to avoid races in resolution test. Also added an option to skip the resolution test since it runs for ~5 seconds each invocation. This should fix the non-deterministic failures in riak_test like http://giddyup.basho.com/#/projects/riak/scorecards/35/35-387-client_python_verify-ubuntu-1004-64-eleveldb/14385/artifacts/76166 /cc @engelsanchez @javajolt --- riak/tests/test_kv.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 52c1bde6..1fb50fce 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -2,6 +2,7 @@ import os import cPickle import copy +import platform from time import sleep from riak import ConflictError from riak.resolver import default_resolver, last_written_resolver @@ -10,6 +11,11 @@ except ImportError: import json +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest + class NotJsonSerializable(object): @@ -307,6 +313,8 @@ def test_siblings(self): self.assertEqual(len(obj.siblings), 1) self.assertEqual(obj.encoded_data, resolved_sibling.encoded_data) + @unittest.skipIf(os.environ.get('SKIP_RESOLVE', '0') == '1', + "skip requested for resolvers test") def test_resolution(self): bucket = self.client.bucket(self.sibs_bucket) obj = bucket.get(self.key_name) @@ -318,7 +326,7 @@ def test_resolution(self): obj.content_type = 'text/plain' obj.store() - vals = self.generate_siblings(obj, count=5, delay=0.75) + vals = self.generate_siblings(obj, count=5, delay=1.01) # Make sure the object has five siblings when using the # default resolver From fca898b06821c965170f61af8b00b2a93fdf92a6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 25 Jun 2013 10:24:41 -0500 Subject: [PATCH 036/672] Add multiget implementation with a static worker pool. --- riak/client/multiget.py | 167 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 riak/client/multiget.py diff --git a/riak/client/multiget.py b/riak/client/multiget.py new file mode 100644 index 00000000..149d7599 --- /dev/null +++ b/riak/client/multiget.py @@ -0,0 +1,167 @@ +""" +Copyright 2013 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from collections import namedtuple +from Queue import Queue +from threading import Thread, Lock, Event +from multiprocessing import cpu_count + +__all__ = ['multiget'] + + +try: + POOL_SIZE = cpu_count() * 2 +except NotImplementedError: + # Make an educated guess + POOL_SIZE = 6 + + +Task = namedtuple('Task', ['client', 'outq', 'bucket', 'key', 'options']) + + +class MultiGetPool(object): + """ + Encapsulates a pool of fetcher threads. These threads can be used + across many multi-get requests. + """ + + def __init__(self, size=POOL_SIZE): + self._inq = Queue() + self._size = size + self._started = Event() + self._stop = Event() + self._lock = Lock() + self._workers = [] + + def enq(self, task): + """ + Enqueues a fetch task to the pool of workers. This will raise + a RuntimeError if the pool is stopped or in the process of + stopping. + + :param task: the Task object + :type task: Task + """ + if not self._stop.is_set(): + self._inq.put(task) + else: + raise RuntimeError("Attempted to enqueue a fetch operation while " + "multi-get pool was shutdown!") + + def start(self): + """ + Starts the worker threads if they are not already started. + This method is thread-safe and will be called automatically + when executing a MultiGet operation. + """ + # Check whether we are already started, skip if we are. + if not self._started.is_set(): + # If we are not started, try to capture the lock. + if self._lock.acquire(False): + # If we got the lock, go ahead and start the worker + # threads, set the started flag, and release the lock. + for i in range(self._size): + name = "riak.client.multiget-worker-{0}".format(i) + worker = Thread(target=self._fetcher, args=(name,), + name=name) + worker.daemon = True + worker.start() + self._workers.append(worker) + self._started.set() + self._lock.release() + else: + # We didn't get the lock, so someone else is already + # starting the worker threads. Wait until they have + # signaled that the threads are started. + self._started.wait() + + def stop(self): + """ + Signals the worker threads to exit and waits on them. + """ + self._stop.set() + for worker in self._workers: + worker.join() + + def stopped(self): + """ + Detects whether this pool has been stopped. + """ + return self._stop.is_set() + + def __del__(self): + # Ensure that all work in the queue is processed before + # shutting down. + self.stop() + + def _fetcher(self, name): + """ + The body of the multi-get worker. + """ + while not self._should_quit(): + task = self._inq.get() + try: + obj = task.client.bucket(task.bucket).get(task.key, + **task.options) + task.outq.put(obj) + except KeyboardInterrupt: + raise + except StandardError as err: + task.outq.put((task.bucket, task.key, err), ) + finally: + self._inq.task_done() + + def _should_quit(self): + """ + Worker threads should exit when the stop flag is set and the + input queue is empty. Once the stop flag is set, new enqueues + are disallowed, meaning that the workers can safely drain the + queue before exiting. + :rtype boolean + """ + return self.stopped() and self._inq.empty() + + +RIAK_MULTIGET_POOL = MultiGetPool() + + +def multiget(client, keys, **options): + """ + Executes a parallel-fetch across multiple threads. Returns a list + containing RiakObject instances, or 3-tuples of bucket, key, and + the exception raised. + + :rtype list + """ + outq = Queue() + + RIAK_MULTIGET_POOL.start() + for bucket, key in keys: + task = Task(client=client, outq=outq, options=options, + bucket=bucket, key=key) + RIAK_MULTIGET_POOL.enq(task) + + results = [] + for _ in range(len(keys)): + if RIAK_MULTIGET_POOL.stopped(): + raise RuntimeError("Multi-get operation interrupted by pool " + "stopping!") + results.append(outq.get()) + outq.task_done() + + return results From a518e9009354524bd0fcdf9884169e05a94fb353 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 25 Jun 2013 13:10:20 -0500 Subject: [PATCH 037/672] Add multiget benchmark. --- riak/benchmark.py | 165 ++++++++++++++++++++++++++++++++++++++++ riak/client/multiget.py | 36 +++++++++ 2 files changed, 201 insertions(+) create mode 100644 riak/benchmark.py diff --git a/riak/benchmark.py b/riak/benchmark.py new file mode 100644 index 00000000..735d76e5 --- /dev/null +++ b/riak/benchmark.py @@ -0,0 +1,165 @@ +""" +Copyright 2013 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import os +import gc + +__all__ = ['bm', 'bmbm'] + + +def bmbm(): + """ + Runs a benchmark when used as an iterator, injecting a garbage + collection between iterations. Example: + + for b in benchmark.bmbm(): + with b.report("pow"): + for _ in range(10000): + math.pow(2,10000) + with b.report("factorial"): + for i in range(100): + math.factorial(i) + """ + return Benchmark(True) + + +def bm(): + """ + Runs a benchmark once when used as a context manager. Example: + + with benchmark.bm() as b: + with b.report("pow"): + for _ in range(10000): + math.pow(2,10000) + with b.report("factorial"): + for i in range(100): + math.factorial(i) + """ + return Benchmark() + + +class Benchmark(object): + """ + A benchmarking run, which may consist of multiple steps. See + bmbm() and bm() for examples. + """ + def __init__(self, rehearse=False): + """ + Creates a new benchmark reporter. + + :param rehearse: whether to run twice to take counter the effects + of garbage collection + :type rehearse: boolean + """ + self.rehearse = rehearse + if rehearse: + self.count = 2 + else: + self.count = 1 + self._report = None + + def __enter__(self): + if self.rehearse: + raise ValueError("bmbm() cannot be used in with statements, " + "use bm() or the for..in statement") + print_header() + self._report = BenchmarkReport() + self._report.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._report: + return self._report.__exit__(exc_type, exc_val, exc_tb) + else: + print + return True + + def __iter__(self): + return self + + def next(self): + """ + Runs the next iteration of the benchmark. + """ + if self.count == 0: + raise StopIteration + elif self.count > 1: + print_rehearsal_header() + else: + if self.rehearse: + gc.collect() + print "-----------------------------------------------------------\n" + print_header() + + self.count -= 1 + return self + + def report(self, name): + """ + Returns a report for the current step of the benchmark. + """ + self._report = None + return BenchmarkReport(name) + + +def print_rehearsal_header(): + """ + Prints the header for the rehearsal phase of a benchmark. + """ + print + print "Rehearsal -------------------------------------------------" + + +def print_report(label, user, system, real): + """ + Prints the report of one step of a benchmark. + """ + print "{:<12s} {:12f} {:12f} ( {:12f} )".format(label, user, system, real) + + +def print_header(): + """ + Prints the header for the normal phase of a benchmark. + """ + print "{:<12s} {:<12s} {:<12s} ( {:<12s} )".format('', 'user', 'system', 'real') + + +class BenchmarkReport(object): + """ + A labeled step in a benchmark. Acts as a context-manager, printing + its timing results when the context exits. + """ + def __init__(self, name='benchmark'): + self.name = name + self.start = None + + def __enter__(self): + self.start = os.times() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not exc_type: + user1, system1, _, _, real1 = self.start + user2, system2, _, _, real2 = os.times() + print_report(self.name, user2 - user1, system2 - system1, + real2 - real1) + elif exc_type is KeyboardInterrupt: + return False + else: + print "EXCEPTION! %r" % ((exc_type, exc_val, exc_tb),) + return True diff --git a/riak/client/multiget.py b/riak/client/multiget.py index 149d7599..a6e4e94f 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -165,3 +165,39 @@ def multiget(client, keys, **options): outq.task_done() return results + +if __name__ == '__main__': + # Run a benchmark! + from riak import RiakClient + import riak.benchmark as benchmark + client = RiakClient() + bkeys = [ ('multiget', str(key)) for key in xrange(10000) ] + + print "Benchmarking multiget:" + print " CPUs: {0}".format(cpu_count()) + print " Threads: {0}".format(POOL_SIZE) + print " Keys: {0}".format(len(bkeys)) + print + + with benchmark.bm() as b: + with b.report('populate'): + for bucket, key in bkeys: + client.bucket(bucket).new(key, encoded_data=key, + content_type='text/plain' + ).store() + for b in benchmark.bmbm(): + client.protocol = 'http' + with b.report('http seq'): + for bucket, key in bkeys: + client.bucket(bucket).get(key) + + with b.report('http multi'): + multiget(client, bkeys) + + client.protocol = 'pbc' + with b.report('pbc seq'): + for bucket, key in bkeys: + client.bucket(bucket).get(key) + + with b.report('pbc multi'): + multiget(client, bkeys) From 0294e49c0133bdc2471051831e66a3985cc63ddb Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 25 Jun 2013 13:20:13 -0500 Subject: [PATCH 038/672] Expose multiget operations on client and bucket objects. --- riak/bucket.py | 15 +++++++++++++++ riak/client/operations.py | 14 +++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/riak/bucket.py b/riak/bucket.py index 29861f45..b39ec338 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -200,6 +200,21 @@ def get_binary(self, key, r=None, pr=None): 'use RiakBucket.get') return self.get(key, r=r, pr=pr) + def multiget(self, keys, r=None, pr=None): + """ + Retrieves a list of keys belonging to this bucket in parallel. + + :param keys: the keys to fetch + :type keys: list + :param r: R-Value for the requests (defaults to bucket's R) + :type r: integer + :param pr: PR-Value for the requests (defaults to bucket's PR) + :type pr: integer + :rtype list of :class:`RiakObject ` + """ + bkeys = [(self.name, key) for key in keys] + return self._client.multiget(bkeys, r=r, pr=pr) + def _get_resolver(self): if callable(self._resolver): return self._resolver diff --git a/riak/client/operations.py b/riak/client/operations.py index c928128f..d9044699 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -17,7 +17,7 @@ """ from transport import RiakClientTransport, retryable, retryableHttpOnly - +from multiget import multiget class RiakClientOperations(RiakClientTransport): """ @@ -271,3 +271,15 @@ def fulltext_delete(self, transport, index, docs=None, queries=None): :type queries: list """ transport.fulltext_delete(index, docs, queries) + + def multiget(self, pairs, **params): + """ + Fetches many keys in parallel via threads. + + :param pairs: list of bucket/key tuple pairs + :type pairs: list + :param params: additional request flags, e.g. r, pr + :type params: dict + :rtype list + """ + return multiget(self, pairs, **params) From dac83361384bc4fd13395f007a984ad409b6401d Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 25 Jun 2013 13:42:07 -0500 Subject: [PATCH 039/672] Use CPU count as the pool size, no difference apparent above that. --- riak/client/multiget.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/riak/client/multiget.py b/riak/client/multiget.py index a6e4e94f..845170c0 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -25,7 +25,7 @@ try: - POOL_SIZE = cpu_count() * 2 + POOL_SIZE = cpu_count() except NotImplementedError: # Make an educated guess POOL_SIZE = 6 @@ -170,9 +170,11 @@ def multiget(client, keys, **options): # Run a benchmark! from riak import RiakClient import riak.benchmark as benchmark - client = RiakClient() + client = RiakClient(protocol='pbc') bkeys = [ ('multiget', str(key)) for key in xrange(10000) ] + data = open(__file__).read() + print "Benchmarking multiget:" print " CPUs: {0}".format(cpu_count()) print " Threads: {0}".format(POOL_SIZE) @@ -182,7 +184,7 @@ def multiget(client, keys, **options): with benchmark.bm() as b: with b.report('populate'): for bucket, key in bkeys: - client.bucket(bucket).new(key, encoded_data=key, + client.bucket(bucket).new(key, encoded_data=data, content_type='text/plain' ).store() for b in benchmark.bmbm(): From 753a0a4dbb0e71dcfd64cbf6e7a4e91421cfdea6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 25 Jun 2013 14:07:48 -0500 Subject: [PATCH 040/672] Add some tests for multiget. --- riak/tests/test_all.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index cbccbc98..b868ddb5 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -13,6 +13,7 @@ from riak.client import RiakClient from riak.mapreduce import RiakKeyFilter from riak import key_filter +from riak.riak_object import RiakObject from riak.test_server import TestServer @@ -128,6 +129,45 @@ def test_request_retries(self): # error. self.assertRaises(IOError, client.ping) + def test_multiget_bucket(self): + """ + Multiget operations can be invoked on buckets. + """ + keys = [self.key_name, self.randname(), self.randname()] + for key in keys: + self.client.bucket(self.bucket_name)\ + .new(key, encoded_data=key, content_type="text/plain")\ + .store() + results = self.client.bucket(self.bucket_name).multiget(keys) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertTrue(obj.exists) + self.assertEqual(obj.key, obj.encoded_data) + + def test_multiget_errors(self): + """ + Unrecoverable errors are captured along with the bucket/key + and not propagated. + """ + keys = [self.key_name, self.randname(), self.randname()] + client = self.create_client(http_port=1023, pb_port=1024) + results = client.bucket(self.bucket_name).multiget(keys) + for failure in results: + self.assertIsInstance(failure, tuple) + self.assertEqual(failure[0], self.bucket_name) + self.assertIn(failure[1], keys) + self.assertIsInstance(failure[2], StandardError) + + def test_multiget_notfounds(self): + """ + Not founds work in multiget just the same as get. + """ + keys = [(self.bucket_name, self.key_name), + (self.bucket_name, self.randname())] + results = self.client.multiget(keys) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertFalse(obj.exists) class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, From 3fdc469b32f5ded71ad4436b91ea74c4a626036f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 08:54:57 -0500 Subject: [PATCH 041/672] Rename bmbm and friends and fix minor bugs in multiget. --- riak/benchmark.py | 23 +++++++++++++---------- riak/client/multiget.py | 12 +++++------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/riak/benchmark.py b/riak/benchmark.py index 735d76e5..c3eade41 100644 --- a/riak/benchmark.py +++ b/riak/benchmark.py @@ -19,15 +19,15 @@ import os import gc -__all__ = ['bm', 'bmbm'] +__all__ = ['measure', 'measure_with_rehearsal'] -def bmbm(): +def measure_with_rehearsal(): """ Runs a benchmark when used as an iterator, injecting a garbage collection between iterations. Example: - for b in benchmark.bmbm(): + for b in benchmark.measure_with_rehearsal(): with b.report("pow"): for _ in range(10000): math.pow(2,10000) @@ -38,11 +38,11 @@ def bmbm(): return Benchmark(True) -def bm(): +def measure(): """ Runs a benchmark once when used as a context manager. Example: - with benchmark.bm() as b: + with benchmark.measure() as b: with b.report("pow"): for _ in range(10000): math.pow(2,10000) @@ -56,7 +56,7 @@ def bm(): class Benchmark(object): """ A benchmarking run, which may consist of multiple steps. See - bmbm() and bm() for examples. + measure_with_rehearsal() and measure() for examples. """ def __init__(self, rehearse=False): """ @@ -75,8 +75,9 @@ def __init__(self, rehearse=False): def __enter__(self): if self.rehearse: - raise ValueError("bmbm() cannot be used in with statements, " - "use bm() or the for..in statement") + raise ValueError("measure_with_rehearsal() cannot be used in with " + "statements, use measure() or the for..in " + "statement") print_header() self._report = BenchmarkReport() self._report.__enter__() @@ -103,7 +104,8 @@ def next(self): else: if self.rehearse: gc.collect() - print "-----------------------------------------------------------\n" + print ("-" * 59) + print print_header() self.count -= 1 @@ -136,7 +138,8 @@ def print_header(): """ Prints the header for the normal phase of a benchmark. """ - print "{:<12s} {:<12s} {:<12s} ( {:<12s} )".format('', 'user', 'system', 'real') + print "{:<12s} {:<12s} {:<12s} ( {:<12s} )"\ + .format('', 'user', 'system', 'real') class BenchmarkReport(object): diff --git a/riak/client/multiget.py b/riak/client/multiget.py index 845170c0..9995133b 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -77,8 +77,7 @@ def start(self): # threads, set the started flag, and release the lock. for i in range(self._size): name = "riak.client.multiget-worker-{0}".format(i) - worker = Thread(target=self._fetcher, args=(name,), - name=name) + worker = Thread(target=self._fetcher, name=name) worker.daemon = True worker.start() self._workers.append(worker) @@ -109,7 +108,7 @@ def __del__(self): # shutting down. self.stop() - def _fetcher(self, name): + def _fetcher(self): """ The body of the multi-get worker. """ @@ -152,8 +151,7 @@ def multiget(client, keys, **options): RIAK_MULTIGET_POOL.start() for bucket, key in keys: - task = Task(client=client, outq=outq, options=options, - bucket=bucket, key=key) + task = Task(client, outq, bucket, key, options) RIAK_MULTIGET_POOL.enq(task) results = [] @@ -181,13 +179,13 @@ def multiget(client, keys, **options): print " Keys: {0}".format(len(bkeys)) print - with benchmark.bm() as b: + with benchmark.measure() as b: with b.report('populate'): for bucket, key in bkeys: client.bucket(bucket).new(key, encoded_data=data, content_type='text/plain' ).store() - for b in benchmark.bmbm(): + for b in benchmark.measure_with_rehearsal(): client.protocol = 'http' with b.report('http seq'): for bucket, key in bkeys: From 61a05b41e2b621a10a4cb5a790d096f6b64909a9 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 09:31:23 -0500 Subject: [PATCH 042/672] Bump client version and riak_pb dep for Riak 1.4 features. --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 06393638..7a982009 100755 --- a/setup.py +++ b/setup.py @@ -12,15 +12,15 @@ def make_docs(): for name in glob.glob('*.html'): os.rename(name, 'docs/%s' % name) -install_requires = ["riak_pb >=1.2.0, < 1.3.0"] -requires = ["riak_pb(>=1.2.0,<1.3.0)"] +install_requires = ["riak_pb >=1.4.0, < 1.5.0"] +requires = ["riak_pb(>=1.4.0,<1.5.0)"] tests_require = [] if platform.python_version() < '2.7': tests_require.append("unittest2") setup( name='riak', - version='1.5.1', + version='2.0.0a', packages = find_packages(), requires = requires, install_requires = install_requires, From 5f4b588a596eaa3400b55acfe36b6b9ef741df88 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 10:28:14 -0500 Subject: [PATCH 043/672] Unify the bucket properties tests since PBC will support all properties. --- riak/tests/test_all.py | 30 ++++------------------------ riak/tests/test_kv.py | 44 ++++++++++-------------------------------- 2 files changed, 14 insertions(+), 60 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index cbccbc98..e2fd3832 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -21,7 +21,7 @@ from riak.tests.test_mapreduce import MapReduceAliasTests, \ ErlangMapReduceTests, JSMapReduceTests, LinkTests, MapReduceStreamTests from riak.tests.test_kv import BasicKVTests, KVFileTests, \ - HTTPBucketPropsTest, PbcBucketPropsTest + BucketPropsTest from riak.tests.test_2i import TwoITests try: @@ -131,13 +131,14 @@ def test_request_retries(self): class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, - PbcBucketPropsTest, + BucketPropsTest, TwoITests, LinkTests, ErlangMapReduceTests, JSMapReduceTests, MapReduceAliasTests, MapReduceStreamTests, + EnableSearchTests, SearchTests, ClientTests, BaseTestCase, @@ -158,20 +159,10 @@ def test_uses_client_id_if_given(self): c = self.create_client(client_id=zero_client_id) self.assertEqual(zero_client_id, c.client_id) - def test_bucket_search_enabled(self): - with self.assertRaises(NotImplementedError): - bucket = self.client.bucket(self.bucket_name) - bucket.search_enabled() - - def test_enable_search_commit_hook(self): - with self.assertRaises(NotImplementedError): - bucket = self.client.bucket(self.bucket_name) - bucket.enable_search() - class RiakHttpTransportTestCase(BasicKVTests, KVFileTests, - HTTPBucketPropsTest, + BucketPropsTest, TwoITests, LinkTests, ErlangMapReduceTests, @@ -207,19 +198,6 @@ def test_too_many_link_headers_shouldnt_break_http(self): stored_object = bucket.get("lots_of_links") self.assertEqual(len(stored_object.links), 400) - def test_clear_bucket_properties(self): - bucket = self.client.bucket(self.props_bucket) - bucket.allow_mult = True - self.assertTrue(bucket.allow_mult) - bucket.n_val = 1 - self.assertEqual(bucket.n_val, 1) - # Test setting clearing properties... - - self.assertTrue(bucket.clear_properties()) - self.assertFalse(bucket.allow_mult) - self.assertEqual(bucket.n_val, 3) - - class FilterTests(unittest.TestCase): def test_simple(self): f1 = RiakKeyFilter("tokenize", "-", 1) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 1fb50fce..ac72748f 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -440,7 +440,7 @@ def generate_siblings(self, original, count=5, delay=None): return vals -class HTTPBucketPropsTest(object): +class BucketPropsTest(object): def test_rw_settings(self): bucket = self.client.bucket(self.props_bucket) self.assertEqual(bucket.r, "quorum") @@ -480,41 +480,17 @@ def test_primary_quora(self): bucket.set_properties({'pr': 0, 'pw': 0}) bucket.clear_properties() - -class PbcBucketPropsTest(object): - def test_rw_settings(self): + def test_clear_bucket_properties(self): bucket = self.client.bucket(self.props_bucket) - with self.assertRaises(NotImplementedError): - bucket.r - with self.assertRaises(NotImplementedError): - bucket.w - with self.assertRaises(NotImplementedError): - bucket.dw - with self.assertRaises(NotImplementedError): - bucket.rw - - with self.assertRaises(NotImplementedError): - bucket.r = 2 - with self.assertRaises(NotImplementedError): - bucket.w = 2 - with self.assertRaises(NotImplementedError): - bucket.dw = 2 - with self.assertRaises(NotImplementedError): - bucket.rw = 2 - with self.assertRaises(NotImplementedError): - bucket.clear_properties() + bucket.allow_mult = True + self.assertTrue(bucket.allow_mult) + bucket.n_val = 1 + self.assertEqual(bucket.n_val, 1) + # Test setting clearing properties... - def test_primary_quora(self): - bucket = self.client.bucket(self.props_bucket) - with self.assertRaises(NotImplementedError): - bucket.pr - with self.assertRaises(NotImplementedError): - bucket.pw - - with self.assertRaises(NotImplementedError): - bucket.pr = 2 - with self.assertRaises(NotImplementedError): - bucket.pw = 2 + self.assertTrue(bucket.clear_properties()) + self.assertFalse(bucket.allow_mult) + self.assertEqual(bucket.n_val, 3) class KVFileTests(object): From cdcab150a259bb906ab8319b3681ab48c6fdb7b4 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 22:57:02 -0500 Subject: [PATCH 044/672] Add support for all bucket properties and clearing in PB. Also, some code reorganization was done in codec.py, translate_rw_val was renamed to _encode_quorum. --- riak/transports/pbc/codec.py | 203 +++++++++++++++++++++++++++++-- riak/transports/pbc/messages.py | 9 +- riak/transports/pbc/transport.py | 54 ++++---- 3 files changed, 225 insertions(+), 41 deletions(-) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index c91dda20..f42f630c 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -19,30 +19,55 @@ from riak import RiakError from riak.content import RiakContent +def _invert(d): + out = {} + for key in d: + value = d[key] + out[value] = key + return out + +REPL_TO_PY = { + riak_pb.RpbBucketProps.FALSE: False, + riak_pb.RpbBucketProps.TRUE: True, + riak_pb.RpbBucketProps.REALTIME: 'realtime', + riak_pb.RpbBucketProps.FULLSYNC: 'fullsync' + } + +REPL_TO_PB = _invert(REPL_TO_PY) + RIAKC_RW_ONE = 4294967294 RIAKC_RW_QUORUM = 4294967293 RIAKC_RW_ALL = 4294967292 RIAKC_RW_DEFAULT = 4294967291 +QUORUM_TO_PB = { + 'default': RIAKC_RW_DEFAULT, + 'all': RIAKC_RW_ALL, + 'quorum': RIAKC_RW_QUORUM, + 'one': RIAKC_RW_ONE + } + +QUORUM_TO_PY = _invert(QUORUM_TO_PB) + +NORMAL_PROPS = ['n_val', 'allow_mult', 'last_write_wins', 'old_vclock', + 'young_vclock','big_vclock', 'small_vclock', + 'basic_quorum', 'notfound_ok', 'search', 'backend'] +COMMIT_HOOK_PROPS = ['precommit', 'postcommit'] +MODFUN_PROPS = ['chash_keyfun', 'linkfun'] +QUORUM_PROPS = ['r', 'pr', 'w', 'pw', 'dw', 'rw'] + class RiakPbcCodec(object): """ Protobuffs Encoding and decoding methods for RiakPbcTransport. """ - rw_names = { - 'default': RIAKC_RW_DEFAULT, - 'all': RIAKC_RW_ALL, - 'quorum': RIAKC_RW_QUORUM, - 'one': RIAKC_RW_ONE - } - def __init__(self, **unused_args): if riak_pb is None: raise NotImplementedError("this transport is not available") super(RiakPbcCodec, self).__init__(**unused_args) - def translate_rw_val(self, rw): + def _encode_quorum(self, rw): """ Converts a symbolic quorum value into its on-the-wire equivalent. @@ -51,14 +76,26 @@ def translate_rw_val(self, rw): :type rw: string, integer :rtype: integer """ - val = self.rw_names.get(rw) - if val is None: - return rw + if rw in QUORUM_TO_PB: + return QUORUM_TO_PB[rw] elif type(rw) is int and rw >= 0: - return val + return rw else: return None + def _decode_quorum(self, rw): + """ + Converts a protobuf quorum value to a symbolic value if + necessary. + + :param rw: the quorum + :type rw: int + :rtype int or string + """ + if rw in QUORUM_TO_PY: + return QUORUM_TO_PY[rw] + else: + return rw def _decode_contents(self, contents, obj): obj.siblings = [self._decode_content(c, RiakContent(obj)) for c in contents] @@ -170,3 +207,145 @@ def _decode_index_value(self, index, value): return int(value) else: return value + + def _encode_bucket_props(self, props, msg): + """ + Encodes a dict of bucket properties into the protobuf message. + + :param props: bucket properties + :type props: dict + :param msg: the protobuf message to fill + :type msg: riak_pb.RpbSetBucketReq + """ + msg.props.has_precommit = False + msg.props.has_postcommit = False + for prop in NORMAL_PROPS: + if prop in props and props[prop] is not None: + setattr(msg.props, prop, props[prop]) + for prop in COMMIT_HOOK_PROPS: + if prop in props: + setattr(msg.props, 'has_' + prop, True) + self._encode_hooklist(props[prop], getattr(msg.props, prop)) + for prop in MODFUN_PROPS: + if prop in props and props[prop] is not None: + self._encode_modfun(props[prop], getattr(msg.props, prop)) + for prop in QUORUM_PROPS: + if prop in props and props[prop] not in (None, 'default'): + value = self._encode_quorum(props[prop]) + if value is not None: + setattr(msg.props, prop, value) + if 'repl' in props: + msg.props.repl = REPL_TO_PY[props['repl']] + + return msg + + def _decode_bucket_props(self, msg): + """ + Decodes the protobuf bucket properties message into a dict. + + :param msg: the protobuf message to decode + :type msg: riak_pb.RpbBucketProps + :rtype dict + """ + props = {} + + for prop in NORMAL_PROPS: + if msg.HasField(prop): + props[prop] = getattr(msg, prop) + for prop in COMMIT_HOOK_PROPS: + if getattr(msg, 'has_' + prop): + props[prop] = self._decode_hooklist(getattr(msg, prop)) + for prop in MODFUN_PROPS: + if msg.HasField(prop): + props[prop] = self._decode_modfun(getattr(msg, prop)) + for prop in QUORUM_PROPS: + if msg.HasField(prop): + props[prop] = self._decode_quorum(getattr(msg, prop)) + if msg.HasField('repl'): + props['repl'] = REPL_TO_PY[msg.repl] + + return props + + def _decode_modfun(self, modfun): + """ + Decodes a protobuf modfun pair into a dict with 'mod' and + 'fun' keys. Used in bucket properties. + + :param modfun: the protobuf message to decode + :type modfun: riak_pb.RpbModFun + :rtype dict + """ + return {'mod': modfun.module, + 'fun': modfun.function} + + def _encode_modfun(self, props, msg=None): + """ + Encodes a dict with 'mod' and 'fun' keys into a protobuf + modfun pair. Used in bucket properties. + + :param props: the module/function pair + :type props: dict + :param msg: the protobuf message to fill + :type msg: riak_pb.RpbModFun + :rtype riak_pb.RpbModFun + """ + if msg is None: + msg = riak_pb.RpbModFun() + msg.module = props['mod'] + msg.function = props['fun'] + return msg + + def _decode_hooklist(self, hooklist): + """ + Decodes a list of protobuf commit hooks into their python + equivalents. Used in bucket properties. + + :param hooklist: a list of protobuf commit hooks + :type hooklist: list + :rtype list + """ + return [ self._decode_hook(hook) for hook in hooklist ] + + def _encode_hooklist(self, hooklist, msg): + """ + Encodes a list of commit hooks into their protobuf equivalent. + Used in bucket properties. + + :param hooklist: a list of commit hooks + :type hooklist: list + :param msg: a protobuf field that is a list of commit hooks + """ + for hook in hooklist: + pbhook = msg.add() + self._encode_hook(hook, pbhook) + + def _decode_hook(self, hook): + """ + Decodes a protobuf commit hook message into a dict. Used in + bucket properties. + + :param hook: the hook to decode + :type hook: riak_pb.RpbCommitHook + :rtype dict + """ + if hook.HasField('modfun'): + return self._decode_modfun(hook.modfun) + else: + return {'name': hook.name} + + def _encode_hook(self, hook, msg): + """ + Encodes a commit hook dict into the protobuf message. Used in + bucket properties. + + :param hook: the hook to encode + :type hook: dict + :param msg: the protobuf message to fill + :type msg: riak_pb.RpbCommitHook + :rtype riak_pb.RpbCommitHook + """ + if 'name' in hook: + msg.name = name + else: + self._encode_modfun(hook, msg.modfun) + return msg diff --git a/riak/transports/pbc/messages.py b/riak/transports/pbc/messages.py index fb9dba5c..d4b867c8 100644 --- a/riak/transports/pbc/messages.py +++ b/riak/transports/pbc/messages.py @@ -49,13 +49,16 @@ MSG_CODE_INDEX_RESP = 26 MSG_CODE_SEARCH_QUERY_REQ = 27 MSG_CODE_SEARCH_QUERY_RESP = 28 +MSG_CODE_RESET_BUCKET_REQ = 29 +MSG_CODE_RESET_BUCKET_RESP = 30 # These responses don't include messages EMPTY_RESPONSES = [ MSG_CODE_PING_RESP, MSG_CODE_SET_CLIENT_ID_RESP, MSG_CODE_DEL_RESP, - MSG_CODE_SET_BUCKET_RESP + MSG_CODE_SET_BUCKET_RESP, + MSG_CODE_RESET_BUCKET_RESP ] # Mapping from code to protobuf class @@ -88,5 +91,7 @@ MSG_CODE_INDEX_REQ: riak_pb.RpbIndexReq, MSG_CODE_INDEX_RESP: riak_pb.RpbIndexResp, MSG_CODE_SEARCH_QUERY_REQ: riak_pb.RpbSearchQueryReq, - MSG_CODE_SEARCH_QUERY_RESP: riak_pb.RpbSearchQueryResp + MSG_CODE_SEARCH_QUERY_RESP: riak_pb.RpbSearchQueryResp, + MSG_CODE_RESET_BUCKET_REQ: riak_pb.RpbResetBucketReq, + MSG_CODE_RESET_BUCKET_RESP: None } diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 31575d16..6d4cd00f 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -53,7 +53,9 @@ MSG_CODE_INDEX_REQ, MSG_CODE_INDEX_RESP, MSG_CODE_SEARCH_QUERY_REQ, - MSG_CODE_SEARCH_QUERY_RESP + MSG_CODE_SEARCH_QUERY_RESP, + MSG_CODE_RESET_BUCKET_REQ, + MSG_CODE_RESET_BUCKET_RESP ) @@ -123,9 +125,9 @@ def get(self, robj, r=None, pr=None): req = riak_pb.RpbGetReq() if r: - req.r = self.translate_rw_val(r) + req.r = self._encode_quorum(r) if self.quorum_controls() and pr: - req.pr = self.translate_rw_val(pr) + req.pr = self._encode_quorum(pr) if self.tombstone_vclocks(): req.deletedvclock = 1 @@ -160,11 +162,11 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, req = riak_pb.RpbPutReq() if w: - req.w = self.translate_rw_val(w) + req.w = self._encode_quorum(w) if dw: - req.dw = self.translate_rw_val(dw) + req.dw = self._encode_quorum(dw) if self.quorum_controls() and pw: - req.pw = self.translate_rw_val(pw) + req.pw = self._encode_quorum(pw) if return_body: req.return_body = 1 @@ -202,19 +204,19 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): req = riak_pb.RpbDelReq() if rw: - req.rw = self.translate_rw_val(rw) + req.rw = self._encode_quorum(rw) if r: - req.r = self.translate_rw_val(r) + req.r = self._encode_quorum(r) if w: - req.w = self.translate_rw_val(w) + req.w = self._encode_quorum(w) if dw: - req.dw = self.translate_rw_val(dw) + req.dw = self._encode_quorum(dw) if self.quorum_controls(): if pr: - req.pr = self.translate_rw_val(pr) + req.pr = self._encode_quorum(pr) if pw: - req.pw = self.translate_rw_val(pw) + req.pw = self._encode_quorum(pw) if self.tombstone_vclocks() and robj.vclock: req.vclock = robj.vclock.encode('binary') @@ -266,13 +268,8 @@ def get_bucket_props(self, bucket): msg_code, resp = self._request(MSG_CODE_GET_BUCKET_REQ, req, MSG_CODE_GET_BUCKET_RESP) - props = {} - if resp.props.HasField('n_val'): - props['n_val'] = resp.props.n_val - if resp.props.HasField('allow_mult'): - props['allow_mult'] = resp.props.allow_mult - return props + return self._decode_bucket_props(resp.props) def set_bucket_props(self, bucket, props): """ @@ -280,18 +277,21 @@ def set_bucket_props(self, bucket, props): """ req = riak_pb.RpbSetBucketReq() req.bucket = bucket.name - for key in props: - if key not in ['n_val', 'allow_mult']: - raise NotImplementedError - - if 'n_val' in props: - req.props.n_val = props['n_val'] - if 'allow_mult' in props: - req.props.allow_mult = props['allow_mult'] + self._encode_bucket_props(props, req) msg_code, resp = self._request(MSG_CODE_SET_BUCKET_REQ, req, MSG_CODE_SET_BUCKET_RESP) - return self + return True + + def clear_bucket_props(self, bucket): + """ + Clear bucket properties, resetting them to their defaults + """ + req = riak_pb.RpbResetBucketReq() + req.bucket = bucket.name + msg_code = self._request(MSG_CODE_RESET_BUCKET_REQ, req, + MSG_CODE_RESET_BUCKET_RESP) + return True def mapred(self, inputs, query, timeout=None): # dictionary of phase results - each content should be an encoded array From 43b663e1bb7d21fe479f22e457d25d0b2f2f156b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 22:57:19 -0500 Subject: [PATCH 045/672] Add some much needed documentation for other PB codec methods. --- riak/transports/pbc/codec.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index f42f630c..3d4782a8 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -96,7 +96,18 @@ def _decode_quorum(self, rw): return QUORUM_TO_PY[rw] else: return rw + def _decode_contents(self, contents, obj): + """ + Decodes the list of siblings from the protobuf representation + into the object. + + :param contents: a list of RpbContent messages + :type contents: list + :param obj: a RiakObject + :type obj: RiakObject + :rtype RiakObject + """ obj.siblings = [self._decode_content(c, RiakContent(obj)) for c in contents] # Invoke sibling-resolution logic @@ -109,7 +120,11 @@ def _decode_content(self, rpb_content, sibling): Decodes a single sibling from the protobuf representation into a RiakObject. - :rtype: (RiakObject) + :param rpb_content: a single RpbContent message + :type rpb_content: riak_pb.RpbContent + :param sibling: a RiakContent sibling container + :type sibling: RiakContent + :rtype: RiakContent """ if rpb_content.HasField("deleted") and rpb_content.deleted: @@ -147,6 +162,11 @@ def _encode_content(self, robj, rpb_content): """ Fills an RpbContent message with the appropriate data and metadata from a RiakObject. + + :param robj: a RiakObject + :type robj: RiakObject + :param rpb_content: the protobuf message to fill + :type rpb_content: riak_pb.RpbContent """ if robj.content_type: rpb_content.content_type = robj.content_type @@ -182,6 +202,10 @@ def _encode_content(self, robj, rpb_content): def _decode_link(self, link): """ Decodes an RpbLink message into a tuple + + :param link: an RpbLink message + :type link: riak_pb.RpbLink + :rtype tuple """ if link.HasField("bucket"): @@ -202,6 +226,11 @@ def _decode_link(self, link): def _decode_index_value(self, index, value): """ Decodes a secondary index value into the correct Python type. + :param index: the name of the index + :type index: str + :param value: the value of the index entry + :type value: str + :rtype str or int """ if index.endswith("_int"): return int(value) From caf7fafe8137a17a6e9dacfdf56459230eeed9bd Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 22:58:32 -0500 Subject: [PATCH 046/672] Cleanup bucket properties shortcuts with a helper function. --- riak/bucket.py | 72 +++++++++++--------------------------------------- 1 file changed, 16 insertions(+), 56 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 29861f45..6021d140 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -24,6 +24,14 @@ def deprecateBucketQuorumAccessors(klass): return deprecateQuorumAccessors(klass, parent='_client') +def bucket_property(name, doc=None): + def _prop_getter(self): + return self.get_property(name) + + def _prop_setter(self, value): + return self.set_property(name, value) + + return property(_prop_getter, _prop_setter, doc=doc) @deprecateBucketQuorumAccessors class RiakBucket(object): @@ -220,13 +228,7 @@ def _set_resolver(self, value): client's resolver will be used. :type callable""") - def _set_n_val(self, nval): - return self.set_property('n_val', nval) - - def _get_n_val(self): - return self.get_property('n_val') - - n_val = property(_get_n_val, _set_n_val, doc=""" + n_val = bucket_property('n_val', doc=""" N-value for this bucket, which is the number of replicas that will be written of each object in the bucket. @@ -239,13 +241,7 @@ def _get_n_val(self): :type nval: integer """) - def _set_allow_mult(self, bool): - return self.set_property('allow_mult', bool) - - def _get_allow_mult(self): - return self.get_property('allow_mult') - - allow_mult = property(_get_allow_mult, _set_allow_mult, doc=""" + allow_mult = bucket_property('allow_mult', doc=""" If set to True, then writes with conflicting data will be stored and returned to the client. This situation can be detected by calling has_siblings() and get_siblings(). @@ -253,73 +249,37 @@ def _get_allow_mult(self): :type bool: boolean """) - def _set_r(self, val): - return self.set_property('r', val) - - def _get_r(self): - return self.get_property('r') - - r = property(_get_r, _set_r, doc=""" + r = bucket_property('r', doc=""" The default 'read' quorum for this bucket (how many replicas must reply for a successful read). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_pr(self, val): - return self.set_property('pr', val) - - def _get_pr(self): - return self.get_property('pr') - - pr = property(_get_pr, _set_pr, doc=""" + pr = bucket_property('pr', doc=""" The default 'primary read' quorum for this bucket (how many primary replicas are required for a successful read). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_rw(self, val): - return self.set_property('rw', val) - - def _get_rw(self): - return self.get_property('rw') - - rw = property(_get_rw, _set_rw, doc=""" + rw = bucket_property('rw', doc=""" The default 'read' and 'write' quorum for this bucket (equivalent to 'r' and 'w' but for deletes). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_w(self, val): - return self.set_property('w', val) - - def _get_w(self): - return self.get_property('w') - - w = property(_get_w, _set_w, doc=""" + w = bucket_property('w', doc=""" The default 'write' quorum for this bucket (how many replicas must acknowledge receipt of a write). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_dw(self, val): - return self.set_property('dw', val) - - def _get_dw(self): - return self.get_property('dw') - - dw = property(_get_dw, _set_dw, doc=""" + dw = bucket_property('dw', doc=""" The default 'durable write' quorum for this bucket (how many replicas must commit the write). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_pw(self, val): - return self.set_property('pw', val) - - def _get_pw(self): - return self.get_property('pw') - - pw = property(_get_pw, _set_pw, doc=""" + pw = bucket_property('pw', doc=""" The default 'primary write' quorum for this bucket (how many primary replicas are required for a successful write). This should be an integer less than the 'n_val' property, or a string of From d94a16fa947b0c9fd87b4f42e024056330fd4bfa Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 23:00:07 -0500 Subject: [PATCH 047/672] Use the search property instead of directly modifying precommit. The bucket fixup for search has been available since 1.0, it is much cleaner and more reliable to use it instead. --- riak/bucket.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 6021d140..4ff1dbed 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -304,10 +304,7 @@ def get_property(self, key): :type key: string :rtype: mixed """ - try: - return self.get_properties()[key] - except KeyError: - raise NotImplementedError + return self.get_properties()[key] def set_properties(self, props): """ @@ -380,19 +377,15 @@ def search_enabled(self): Returns True if the search precommit hook is enabled for this bucket. """ - return self.SEARCH_PRECOMMIT_HOOK in (self.get_property("precommit") or - []) + return self.get_properties().get('search', False) def enable_search(self): """ Enable search for this bucket by installing the precommit hook to index objects in it. """ - precommit_hooks = self.get_property("precommit") or [] - if self.SEARCH_PRECOMMIT_HOOK not in precommit_hooks: - self.set_properties({"precommit": - precommit_hooks + - [self.SEARCH_PRECOMMIT_HOOK]}) + if not self.search_enabled(): + self.set_property('search', True) return True def disable_search(self): @@ -400,10 +393,8 @@ def disable_search(self): Disable search for this bucket by removing the precommit hook to index objects in it. """ - precommit_hooks = self.get_property("precommit") or [] - if self.SEARCH_PRECOMMIT_HOOK in precommit_hooks: - precommit_hooks.remove(self.SEARCH_PRECOMMIT_HOOK) - self.set_properties({"precommit": precommit_hooks}) + if self.search_enabled(): + self.set_property('search', False) return True def search(self, query, **params): From a26a6998a6ad5006deeccac631c68dc9a8d51837 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 26 Jun 2013 23:00:22 -0500 Subject: [PATCH 048/672] Fix a few doc string problems in bucket.py. --- riak/bucket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 4ff1dbed..b8c256ba 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -179,7 +179,7 @@ def new_binary(self, key=None, data=None, def get(self, key, r=None, pr=None): """ - Retrieve a JSON-encoded object from Riak. + Retrieve an object from Riak. :param key: Name of the key. :type key: string @@ -194,7 +194,7 @@ def get(self, key, r=None, pr=None): def get_binary(self, key, r=None, pr=None): """ - Retrieve a binary/string object from Riak. + Retrieve a binary/string object from Riak. DEPRECATED :param key: Name of the key. :type key: string From cc54a053e0675747d13bd2b63c083dac157af55d Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Jun 2013 08:26:03 -0500 Subject: [PATCH 049/672] Predicate the bucket properties features on version detection. --- riak/tests/test_feature_detection.py | 24 ++++++++++++++++++++++-- riak/transports/feature_detect.py | 19 ++++++++++++++++++- riak/transports/pbc/codec.py | 2 -- riak/transports/pbc/transport.py | 10 ++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py index 0ab7fde4..4048c0d5 100644 --- a/riak/tests/test_feature_detection.py +++ b/riak/tests/test_feature_detection.py @@ -42,9 +42,8 @@ class FeatureDetectionTest(unittest.TestCase): def test_implements_server_version(self): t = IncompleteTransport() - def get_server_version(): + with self.assertRaises(NotImplementedError): t.server_version - self.assertRaises(NotImplementedError, get_server_version) def test_pre_10(self): t = DummyTransport("0.14.2") @@ -55,6 +54,8 @@ def test_pre_10(self): self.assertFalse(t.quorum_controls()) self.assertFalse(t.tombstone_vclocks()) self.assertFalse(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) def test_10(self): t = DummyTransport("1.0.3") @@ -65,6 +66,8 @@ def test_10(self): self.assertTrue(t.quorum_controls()) self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) def test_11(self): t = DummyTransport("1.1.4") @@ -75,6 +78,8 @@ def test_11(self): self.assertTrue(t.quorum_controls()) self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) def test_12(self): t = DummyTransport("1.2.0") @@ -85,6 +90,8 @@ def test_12(self): self.assertTrue(t.quorum_controls()) self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) def test_12_loose(self): t = DummyTransport("1.2.1p3") @@ -95,7 +102,20 @@ def test_12_loose(self): self.assertTrue(t.quorum_controls()) self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) + def test_14(self): + t = DummyTransport("1.4.0rc1") + self.assertTrue(t.phaseless_mapred()) + self.assertTrue(t.pb_indexes()) + self.assertTrue(t.pb_search()) + self.assertTrue(t.pb_conditionals()) + self.assertTrue(t.quorum_controls()) + self.assertTrue(t.tombstone_vclocks()) + self.assertTrue(t.pb_head()) + self.assertTrue(t.pb_clear_bucket_props()) + self.assertTrue(t.pb_all_bucket_props()) if __name__ == '__main__': unittest.main() diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 3d8acfd9..3712ec8b 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -23,7 +23,8 @@ versions = { 1: LooseVersion("1.0.0"), 1.1: LooseVersion("1.1.0"), - 1.2: LooseVersion("1.2.0") + 1.2: LooseVersion("1.2.0"), + 1.4: LooseVersion("1.4.0") } @@ -90,6 +91,22 @@ def pb_head(self): """ return self.server_version >= versions[1] + def pb_clear_bucket_props(self): + """ + Whether bucket properties can be cleared over Protocol + Buffers. + :rtype bool + """ + return self.server_version >= versions[1.4] + + def pb_all_bucket_props(self): + """ + Whether all normal bucket properties are supported over + Protocol Buffers. + :rtype bool + """ + return self.server_version >= versions[1.4] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 3d4782a8..70d093f2 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -246,8 +246,6 @@ def _encode_bucket_props(self, props, msg): :param msg: the protobuf message to fill :type msg: riak_pb.RpbSetBucketReq """ - msg.props.has_precommit = False - msg.props.has_postcommit = False for prop in NORMAL_PROPS: if prop in props and props[prop] is not None: setattr(msg.props, prop, props[prop]) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 6d4cd00f..da7bb64e 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -277,6 +277,13 @@ def set_bucket_props(self, bucket, props): """ req = riak_pb.RpbSetBucketReq() req.bucket = bucket.name + + if not self.pb_all_bucket_props(): + for key in props: + if key not in ('n_val', 'allow_mult'): + raise NotImplementedError('Server only supports n_val and ' + 'allow_mult properties over PBC') + self._encode_bucket_props(props, req) msg_code, resp = self._request(MSG_CODE_SET_BUCKET_REQ, req, @@ -287,6 +294,9 @@ def clear_bucket_props(self, bucket): """ Clear bucket properties, resetting them to their defaults """ + if not self.pb_clear_bucket_props(): + return False + req = riak_pb.RpbResetBucketReq() req.bucket = bucket.name msg_code = self._request(MSG_CODE_RESET_BUCKET_REQ, req, From a31efe131e7d3f329a8a1e01993ebcdde3a1a4a2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Jun 2013 08:32:44 -0500 Subject: [PATCH 050/672] Apply some PEP8 and pyflakes fixes. --- riak/bucket.py | 2 ++ riak/tests/test_all.py | 1 + riak/transports/pbc/codec.py | 29 +++++++++++++---------------- riak/transports/pbc/transport.py | 4 ++-- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index b8c256ba..966ebe68 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -24,6 +24,7 @@ def deprecateBucketQuorumAccessors(klass): return deprecateQuorumAccessors(klass, parent='_client') + def bucket_property(name, doc=None): def _prop_getter(self): return self.get_property(name) @@ -33,6 +34,7 @@ def _prop_setter(self, value): return property(_prop_getter, _prop_setter, doc=doc) + @deprecateBucketQuorumAccessors class RiakBucket(object): """ diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index e2fd3832..592ecdf1 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -198,6 +198,7 @@ def test_too_many_link_headers_shouldnt_break_http(self): stored_object = bucket.get("lots_of_links") self.assertEqual(len(stored_object.links), 400) + class FilterTests(unittest.TestCase): def test_simple(self): f1 = RiakKeyFilter("tokenize", "-", 1) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 70d093f2..250995aa 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -19,6 +19,7 @@ from riak import RiakError from riak.content import RiakContent + def _invert(d): out = {} for key in d: @@ -26,12 +27,10 @@ def _invert(d): out[value] = key return out -REPL_TO_PY = { - riak_pb.RpbBucketProps.FALSE: False, - riak_pb.RpbBucketProps.TRUE: True, - riak_pb.RpbBucketProps.REALTIME: 'realtime', - riak_pb.RpbBucketProps.FULLSYNC: 'fullsync' - } +REPL_TO_PY = {riak_pb.RpbBucketProps.FALSE: False, + riak_pb.RpbBucketProps.TRUE: True, + riak_pb.RpbBucketProps.REALTIME: 'realtime', + riak_pb.RpbBucketProps.FULLSYNC: 'fullsync'} REPL_TO_PB = _invert(REPL_TO_PY) @@ -40,18 +39,16 @@ def _invert(d): RIAKC_RW_ALL = 4294967292 RIAKC_RW_DEFAULT = 4294967291 -QUORUM_TO_PB = { - 'default': RIAKC_RW_DEFAULT, - 'all': RIAKC_RW_ALL, - 'quorum': RIAKC_RW_QUORUM, - 'one': RIAKC_RW_ONE - } +QUORUM_TO_PB = {'default': RIAKC_RW_DEFAULT, + 'all': RIAKC_RW_ALL, + 'quorum': RIAKC_RW_QUORUM, + 'one': RIAKC_RW_ONE} QUORUM_TO_PY = _invert(QUORUM_TO_PB) NORMAL_PROPS = ['n_val', 'allow_mult', 'last_write_wins', 'old_vclock', - 'young_vclock','big_vclock', 'small_vclock', - 'basic_quorum', 'notfound_ok', 'search', 'backend'] + 'young_vclock', 'big_vclock', 'small_vclock', 'basic_quorum', + 'notfound_ok', 'search', 'backend'] COMMIT_HOOK_PROPS = ['precommit', 'postcommit'] MODFUN_PROPS = ['chash_keyfun', 'linkfun'] QUORUM_PROPS = ['r', 'pr', 'w', 'pw', 'dw', 'rw'] @@ -331,7 +328,7 @@ def _decode_hooklist(self, hooklist): :type hooklist: list :rtype list """ - return [ self._decode_hook(hook) for hook in hooklist ] + return [self._decode_hook(hook) for hook in hooklist] def _encode_hooklist(self, hooklist, msg): """ @@ -372,7 +369,7 @@ def _encode_hook(self, hook, msg): :rtype riak_pb.RpbCommitHook """ if 'name' in hook: - msg.name = name + msg.name = hook['name'] else: self._encode_modfun(hook, msg.modfun) return msg diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index da7bb64e..0b62689a 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -299,8 +299,8 @@ def clear_bucket_props(self, bucket): req = riak_pb.RpbResetBucketReq() req.bucket = bucket.name - msg_code = self._request(MSG_CODE_RESET_BUCKET_REQ, req, - MSG_CODE_RESET_BUCKET_RESP) + self._request(MSG_CODE_RESET_BUCKET_REQ, req, + MSG_CODE_RESET_BUCKET_RESP) return True def mapred(self, inputs, query, timeout=None): From b0efb68fca9d504feda81f9abde4c4c38a84d55a Mon Sep 17 00:00:00 2001 From: Pavel Kirienko Date: Sat, 29 Jun 2013 13:26:01 +0400 Subject: [PATCH 051/672] Fixes ConflictError on non-existent objects --- riak/riak_object.py | 2 ++ riak/tests/test_kv.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/riak/riak_object.py b/riak/riak_object.py index 48d9b779..f6a4aa8f 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -39,6 +39,8 @@ def _setter(self, value): setattr(self.siblings[0], name, value) def _getter(self): + if len(self.siblings) == 0: + return if len(self.siblings) != 1: raise ConflictError() return getattr(self.siblings[0], name) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 1fb50fce..c7be3646 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -220,6 +220,8 @@ def test_missing_object(self): bucket = self.client.bucket(self.bucket_name) obj = bucket.get(self.key_name) self.assertFalse(obj.exists) + # Object with no siblings should not raise the ConflictError + self.assertIsNone(obj.data) def test_delete(self): bucket = self.client.bucket(self.bucket_name) From be7791a351f2d81360675cd9b28e0b314734f66c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 28 Jun 2013 08:57:43 -0500 Subject: [PATCH 052/672] Add surface API for 1.4 counters. --- riak/bucket.py | 22 +++++++++++++++ riak/client/operations.py | 54 ++++++++++++++++++++++++++++++++++++ riak/transports/transport.py | 14 ++++++++++ 3 files changed, 90 insertions(+) diff --git a/riak/bucket.py b/riak/bucket.py index 966ebe68..993505f6 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -421,6 +421,28 @@ def delete(self, key, **kwargs): """ return self.new(key).delete(**kwargs) + def get_counter(self, key, **kwargs): + """ + Gets the value of a counter stored in this bucket. + + :param key: the key of the counter + :type key: string + :rtype int + """ + return self._client.get_counter(self, key, **kwargs) + + def update_counter(self, key, value, **kwargs): + """ + Updates the value of a counter stored in this bucket. Positive + values increment the counter, negative values decrement. + + :param key: the key of the counter + :type key: string + :param value: the amount to increment or decrement + :type value: integer + """ + return self._client.update_counter(self, key, value, **kwargs) + def __str__(self): return ''.format(self.name) diff --git a/riak/client/operations.py b/riak/client/operations.py index c928128f..f1220c7a 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -271,3 +271,57 @@ def fulltext_delete(self, transport, index, docs=None, queries=None): :type queries: list """ transport.fulltext_delete(index, docs, queries) + + @retryable + def get_counter(self, transport, bucket, key, r=None, pr=None, + basic_quorum=None, notfound_ok=None): + """ + Gets the value of a counter. + + :param bucket: the bucket of the counter + :type bucket: RiakBucket + :param key: the key of the counter + :type key: string + :param r: the read quorum + :type r: integer, string, None + :param pr: the primary read quorum + :type pr: integer, string, None + :param basic_quorum: whether to use the "basic quorum" policy + for not-founds + :type basic_quorum: bool + :param notfound_ok: whether to treat not-found responses as successful + :type notfound_ok: bool + :rtype integer + """ + return transport.get_counter(bucket, key, r=r, pr=pr) + + def update_counter(self, bucket, key, value, w=None, dw=None, pw=None, + returnvalue=False): + """ + Updates a counter by the given value. This operation is not + idempotent and so should not be retried automatically. + + :param bucket: the bucket of the counter + :type bucket: RiakBucket + :param key: the key of the counter + :type key: string + :param value: the amount to increment or decrement + :type value: integer + :param w: the write quorum + :type w: integer, string, None + :param dw: the durable write quorum + :type dw: integer, string, None + :param pw: the primary write quorum + :type pw: integer, string, None + :param returnvalue: whether to return the updated value of the counter + :type returnvalue: bool + """ + if type(value) is not int: + raise TypeError("Counter update amount must be an integer") + if value == 0: + raise ValueError("Cannot increment counter by 0") + + with self._transport() as transport: + return transport.update_counter(bucket, key, value, + w=w, dw=dw, pw=pw, + returnvalue=returnvalue) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index c6d581c7..a61d2ee3 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -179,6 +179,20 @@ def fulltext_delete(self, index, docs=None, queries=None): """ raise NotImplementedError + def get_counter(self, bucket, key, r=None, pr=None, basic_quorum=None, + notfound_ok=None): + """ + Gets the value of a counter. + """ + raise NotImplementedError + + def update_counter(self, bucket, key, value, w=None, dw=None, pw=None, + returnvalue=False): + """ + Updates a counter by the given value. + """ + raise NotImplementedError + def _search_mapred_emu(self, index, query): """ Emulates a search request via MapReduce. Used in the case From 5deacf893f7946d89f865689672cbe0c68faada7 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 28 Jun 2013 17:09:08 -0500 Subject: [PATCH 053/672] Implement counter ops in transports and add tests. --- riak/tests/test_all.py | 4 ++- riak/tests/test_kv.py | 28 ++++++++++++++++ riak/transports/feature_detect.py | 7 ++++ riak/transports/http/resources.py | 11 +++++++ riak/transports/http/transport.py | 29 +++++++++++++++++ riak/transports/pbc/messages.py | 11 ++++++- riak/transports/pbc/transport.py | 53 ++++++++++++++++++++++++++++++- 7 files changed, 140 insertions(+), 3 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 592ecdf1..4fe3755f 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -21,7 +21,7 @@ from riak.tests.test_mapreduce import MapReduceAliasTests, \ ErlangMapReduceTests, JSMapReduceTests, LinkTests, MapReduceStreamTests from riak.tests.test_kv import BasicKVTests, KVFileTests, \ - BucketPropsTest + BucketPropsTest, CounterTests from riak.tests.test_2i import TwoITests try: @@ -141,6 +141,7 @@ class RiakPbcTransportTestCase(BasicKVTests, EnableSearchTests, SearchTests, ClientTests, + CounterTests, BaseTestCase, unittest.TestCase): @@ -173,6 +174,7 @@ class RiakHttpTransportTestCase(BasicKVTests, SolrSearchTests, SearchTests, ClientTests, + CounterTests, BaseTestCase, unittest.TestCase): diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index ac72748f..017dc3f6 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -519,3 +519,31 @@ def test_store_binary_object_from_file_should_fail_if_file_not_found(self): obj = bucket.get(self.key_name) # self.assertEqual(obj.encoded_data, None) self.assertFalse(obj.exists) + + +class CounterTests(object): + def test_counter_requires_allow_mult(self): + bucket = self.client.bucket(self.bucket_name) + self.assertFalse(bucket.allow_mult) + + with self.assertRaises(Exception): + bucket.update_counter(self.key_name, 10) + + def test_counter_ops(self): + bucket = self.client.bucket(self.sibs_bucket) + self.assertTrue(bucket.allow_mult) + + # Non-existent counter has no value + self.assertEqual(None, bucket.get_counter(self.key_name)) + + # Update the counter + bucket.update_counter(self.key_name, 10) + self.assertEqual(10, bucket.get_counter(self.key_name)) + + # Update with returning the value + self.assertEqual(15, bucket.update_counter(self.key_name, 5, + returnvalue=True)) + + # Now try decrementing + self.assertEqual(10, bucket.update_counter(self.key_name, -5, + returnvalue=True)) diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 3712ec8b..16ab2016 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -107,6 +107,13 @@ def pb_all_bucket_props(self): """ return self.server_version >= versions[1.4] + def counters(self): + """ + Whether CRDT counters are supported. + :rtype bool + """ + return self.server_version >= versions[1.4] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index d46fb414..9942e06d 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -108,6 +108,13 @@ def luwak_path(self, key=None): key = quote_plus(key) return mkpath(self.luwak_wm_file, key) + def counters_path(self, bucket, key, **options): + if not self.riak_kv_wm_counter: + raise RiakError("Counters are unsupported by this Riak node") + + return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), "counters", + quote_plus(key), **options) + @lazy_property def riak_kv_wm_buckets(self): return self.resources.get('riak_kv_wm_index') @@ -144,6 +151,10 @@ def riak_solr_indexer_wm(self): def luwak_wm_file(self): return self.resources.get('luwak_wm_file') + @lazy_property + def riak_kv_wm_counter(self): + return self.resources.get('riak_kv_wm_counter') + @lazy_property def resources(self): return self.get_resources() diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 632f37bf..99ffc2f3 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -356,6 +356,35 @@ def fulltext_delete(self, index, docs=None, queries=None): {'Content-Type': 'text/xml'}, xml.toxml().encode('utf-8')) + def get_counter(self, bucket, key, **options): + if not self.counters(): + raise NotImplementedError("Counters are not supported") + + url = self.counters_path(bucket.name, key, **options) + status, headers, body = self._request('GET', url) + + self.check_http_code(status, [200, 404]) + if status == 200: + return long(body.strip()) + elif status == 404: + return None + + def update_counter(self, bucket, key, amount, **options): + if not self.counters(): + raise NotImplementedError("Counters are not supported") + + return_value = 'returnvalue' in options and options['returnvalue'] + headers = {'Content-Type': 'text/plain'} + url = self.counters_path(bucket.name, key, **options) + status, headers, body = self._request('POST', url, headers, + str(amount)) + if return_value and status == 200: + return long(body.strip()) + elif status == 204: + return True + else: + self.check_http_code(status, [200, 204]) + def check_http_code(self, status, expected_statuses): if not status in expected_statuses: raise Exception('Expected status %s, received %s' % diff --git a/riak/transports/pbc/messages.py b/riak/transports/pbc/messages.py index d4b867c8..487b611a 100644 --- a/riak/transports/pbc/messages.py +++ b/riak/transports/pbc/messages.py @@ -51,6 +51,10 @@ MSG_CODE_SEARCH_QUERY_RESP = 28 MSG_CODE_RESET_BUCKET_REQ = 29 MSG_CODE_RESET_BUCKET_RESP = 30 +MSG_CODE_COUNTER_UPDATE_REQ = 50 +MSG_CODE_COUNTER_UPDATE_RESP = 51 +MSG_CODE_COUNTER_GET_REQ = 52 +MSG_CODE_COUNTER_GET_RESP = 53 # These responses don't include messages EMPTY_RESPONSES = [ @@ -93,5 +97,10 @@ MSG_CODE_SEARCH_QUERY_REQ: riak_pb.RpbSearchQueryReq, MSG_CODE_SEARCH_QUERY_RESP: riak_pb.RpbSearchQueryResp, MSG_CODE_RESET_BUCKET_REQ: riak_pb.RpbResetBucketReq, - MSG_CODE_RESET_BUCKET_RESP: None + MSG_CODE_RESET_BUCKET_RESP: None, + MSG_CODE_COUNTER_UPDATE_REQ: riak_pb.RpbCounterUpdateReq, + MSG_CODE_COUNTER_UPDATE_RESP: riak_pb.RpbCounterUpdateResp, + MSG_CODE_COUNTER_GET_REQ: riak_pb.RpbCounterGetReq, + MSG_CODE_COUNTER_GET_RESP: riak_pb.RpbCounterGetResp + } diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 0b62689a..d4f4aeba 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -55,7 +55,11 @@ MSG_CODE_SEARCH_QUERY_REQ, MSG_CODE_SEARCH_QUERY_RESP, MSG_CODE_RESET_BUCKET_REQ, - MSG_CODE_RESET_BUCKET_RESP + MSG_CODE_RESET_BUCKET_RESP, + MSG_CODE_COUNTER_UPDATE_REQ, + MSG_CODE_COUNTER_UPDATE_RESP, + MSG_CODE_COUNTER_GET_REQ, + MSG_CODE_COUNTER_GET_RESP ) @@ -396,3 +400,50 @@ def search(self, index, query, **params): docs.append(resultdoc) result['docs'] = docs return result + + def get_counter(self, bucket, key, **params): + if not self.counters(): + raise NotImplementedError("Counters are not supported") + + req = riak_pb.RpbCounterGetReq() + req.bucket = bucket.name + req.key = key + if params.get('r') is not None: + req.r = self._encode_quorum(params['r']) + if params.get('pr') is not None: + req.pr = self._encode_quorum(params['pr']) + if params.get('basic_quorum') is not None: + req.basic_quorum = params['basic_quorum'] + if params.get('notfound_ok') is not None: + req.notfound_ok = params['notfound_ok'] + + msg_code, resp = self._request(MSG_CODE_COUNTER_GET_REQ, req, + MSG_CODE_COUNTER_GET_RESP) + if resp.HasField('value'): + return resp.value + else: + return None + + def update_counter(self, bucket, key, value, **params): + if not self.counters(): + raise NotImplementedError("Counters are not supported") + + req = riak_pb.RpbCounterUpdateReq() + req.bucket = bucket.name + req.key = key + req.amount = value + if params.get('w') is not None: + req.w = self._encode_quorum(params['w']) + if params.get('dw') is not None: + req.dw = self._encode_quorum(params['dw']) + if params.get('pw') is not None: + req.pw = self._encode_quorum(params['pw']) + if params.get('returnvalue') is not None: + req.returnvalue = params['returnvalue'] + + msg_code, resp = self._request(MSG_CODE_COUNTER_UPDATE_REQ, req, + MSG_CODE_COUNTER_UPDATE_RESP) + if resp.HasField('value'): + return resp.value + else: + return True From abab89256df45b62abf8ce0cb6d852b65a4c28b2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 11:38:03 -0500 Subject: [PATCH 054/672] Allow longs for update amounts, since transport returns longs. --- riak/client/operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index f1220c7a..4597fd06 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -316,7 +316,7 @@ def update_counter(self, bucket, key, value, w=None, dw=None, pw=None, :param returnvalue: whether to return the updated value of the counter :type returnvalue: bool """ - if type(value) is not int: + if type(value) not in (int, long): raise TypeError("Counter update amount must be an integer") if value == 0: raise ValueError("Cannot increment counter by 0") From 6b85a04fadee837d324f9104baf85ceb7800c16a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 14:16:57 -0500 Subject: [PATCH 055/672] Add surface API for streaming list-buckets. --- riak/client/operations.py | 21 +++++++++++++++++++++ riak/transports/transport.py | 7 ++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index c928128f..b6b8e877 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -35,9 +35,30 @@ def get_buckets(self, transport): Get the list of buckets as RiakBucket instances. NOTE: Do not use this in production, as it requires traversing through all keys stored in a cluster. + + :rtype list of RiakBucket instances """ return [self.bucket(name) for name in transport.get_buckets()] + def stream_buckets(self): + """ + Streams the list of buckets. This is a generator method that + should be iterated over. NOTE: Do not use this in production, + as it requires traversing through all keys stored in a + cluster. + + :rtype iterator + """ + with self._transport() as transport: + stream = transport.stream_buckets() + try: + for bucket_list in stream: + bucket_list = [self.bucket(name) for name in bucket_list] + if len(bucket_list) > 0: + yield bucket_list + finally: + stream.close() + @retryable def ping(self, transport): """ diff --git a/riak/transports/transport.py b/riak/transports/transport.py index c6d581c7..11185791 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -90,7 +90,12 @@ def delete(self, robj, rw=None): def get_buckets(self): """ Serialize get buckets request and deserialize response - @return dict() + """ + raise NotImplementedError + + def stream_buckets(self): + """ + Streams the list of buckets through an iterator """ raise NotImplementedError From a70ab9b6e1c6b1714c40922c713e861e7b04927d Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 15:00:51 -0500 Subject: [PATCH 056/672] Add implementation of streaming list-buckets. * Feature detection for streaming list-buckets was added. * HTTP bucket_list_path was modified to allow overriding the 'buckets' qs param. * Since streaming list-buckets and list-keys use roughly the same JSON format, the common bits were factored out. --- riak/transports/feature_detect.py | 7 +++++++ riak/transports/http/resources.py | 4 ++-- riak/transports/http/stream.py | 24 ++++++++++++++++++------ riak/transports/http/transport.py | 19 ++++++++++++++++++- riak/transports/pbc/stream.py | 19 ++++++++++++++++++- riak/transports/pbc/transport.py | 18 +++++++++++++++++- 6 files changed, 80 insertions(+), 11 deletions(-) diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 3712ec8b..5dac0180 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -107,6 +107,13 @@ def pb_all_bucket_props(self): """ return self.server_version >= versions[1.4] + def bucket_stream(self): + """ + Whether streaming bucket lists are supported. + :rtype bool + """ + return self.server_version >= versions[1.4] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index d46fb414..92c52032 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -38,8 +38,8 @@ def mapred_path(self, **options): return mkpath(self.riak_kv_wm_mapred, **options) def bucket_list_path(self, **options): - query = options.copy() - query.update(buckets=True) + query = {'buckets': True} + query.update(options) if self.riak_kv_wm_buckets: return mkpath(self.riak_kv_wm_buckets, **query) else: diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 2f1620f9..ec49eee4 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -51,10 +51,8 @@ def close(self): pass -class RiakHttpKeyStream(RiakHttpStream): - """ - Streaming iterator for list-keys over HTTP - """ +class RiakHttpJsonStream(RiakHttpStream): + _json_field = None def next(self): while '}' not in self.buffer and not self.response_done: @@ -64,12 +62,26 @@ def next(self): idx = string.index(self.buffer, '}') + 1 chunk = self.buffer[:idx] self.buffer = self.buffer[idx:] - keys = json.loads(chunk)[u'keys'] - return keys + field = json.loads(chunk)[self._json_field] + return field else: raise StopIteration +class RiakHttpKeyStream(RiakHttpJsonStream): + """ + Streaming iterator for list-keys over HTTP + """ + _json_field = u'keys' + + +class RiakHttpBucketStream(RiakHttpJsonStream): + """ + Streaming iterator for list-buckets over HTTP + """ + _json_field = u'buckets' + + class RiakHttpMultipartStream(RiakHttpStream): """ Streaming iterator for multipart messages over HTTP diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 632f37bf..91fd910d 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -33,7 +33,8 @@ from riak.transports.http.codec import RiakHttpCodec from riak.transports.http.stream import ( RiakHttpKeyStream, - RiakHttpMapReduceStream) + RiakHttpMapReduceStream, + RiakHttpBucketStream) from riak import RiakError @@ -192,6 +193,22 @@ def get_buckets(self): else: raise Exception('Error getting buckets.') + def stream_buckets(self): + """ + Stream list of buckets through an iterator + """ + if not self.bucket_stream(): + raise NotImplementedError('Streaming list-buckets is not ' + 'supported') + + url = self.bucket_list_path(buckets="stream") + status, headers, response = self._request('GET', url, stream=True) + + if status == 200: + return RiakHttpBucketStream(response) + else: + raise Exception('Error listing buckets.') + def get_bucket_props(self, bucket): """ Get properties for a bucket diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index 02c69918..b292f276 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -20,7 +20,8 @@ import json from riak.transports.pbc.messages import ( MSG_CODE_LIST_KEYS_RESP, - MSG_CODE_MAPRED_RESP + MSG_CODE_MAPRED_RESP, + MSG_CODE_LIST_BUCKETS_RESP ) @@ -96,3 +97,19 @@ def next(self): raise StopIteration return response.phase, json.loads(response.response) + + +class RiakPbcBucketStream(RiakPbcStream): + """ + Used internally by RiakPbcTransport to implement key-list streams. + """ + + _expect = MSG_CODE_LIST_BUCKETS_RESP + + def next(self): + response = super(RiakPbcBucketStream, self).next() + + if response.done and len(response.buckets) is 0: + raise StopIteration + + return response.buckets diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 0b62689a..c4ded703 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -24,7 +24,7 @@ from riak.transports.transport import RiakTransport from riak.riak_object import VClock from connection import RiakPbcConnection -from stream import RiakPbcKeyStream, RiakPbcMapredStream +from stream import RiakPbcKeyStream, RiakPbcMapredStream, RiakPbcBucketStream from codec import RiakPbcCodec from messages import ( @@ -259,6 +259,22 @@ def get_buckets(self): expect=MSG_CODE_LIST_BUCKETS_RESP) return resp.buckets + def stream_buckets(self): + """ + Stream list of buckets through an iterator + """ + + if not self.bucket_stream(): + raise NotImplementedError('Streaming list-buckets is not ' + 'supported') + + req = riak_pb.RpbListBucketsReq() + req.stream = True + + self._send_msg(MSG_CODE_LIST_BUCKETS_REQ, req) + + return RiakPbcBucketStream(self) + def get_bucket_props(self, bucket): """ Serialize bucket property request and deserialize response From 156d1e5445afcec1b6fcf497e5ade739289f9e4a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 15:50:25 -0500 Subject: [PATCH 057/672] Modify surface API for timeouts and update a bunch of docstrings. --- riak/bucket.py | 12 ++++++---- riak/client/operations.py | 45 ++++++++++++++++++++++++------------ riak/riak_object.py | 21 ++++++++++++----- riak/transports/transport.py | 37 ++++++++++++----------------- 4 files changed, 68 insertions(+), 47 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 966ebe68..d8ff0832 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -179,7 +179,7 @@ def new_binary(self, key=None, data=None, 'param instead of data') return self.new(key, encoded_data=data, content_type=content_type) - def get(self, key, r=None, pr=None): + def get(self, key, r=None, pr=None, timeout=None): """ Retrieve an object from Riak. @@ -189,12 +189,14 @@ def get(self, key, r=None, pr=None): :type r: integer :param pr: PR-Value of the request (defaults to bucket's PR) :type pr: integer + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: :class:`RiakObject ` """ obj = RiakObject(self._client, self, key) - return obj.reload(r=r, pr=pr) + return obj.reload(r=r, pr=pr, timeout=timeout) - def get_binary(self, key, r=None, pr=None): + def get_binary(self, key, r=None, pr=None, timeout=None): """ Retrieve a binary/string object from Riak. DEPRECATED @@ -204,11 +206,13 @@ def get_binary(self, key, r=None, pr=None): :type r: integer :param pr: PR-Value of the request (defaults to bucket's PR) :type pr: integer + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: :class:`RiakObject ` """ deprecated('RiakBucket.get_binary is deprecated, ' 'use RiakBucket.get') - return self.get(key, r=r, pr=pr) + return self.get(key, r=r, pr=pr, timeout=timeout) def _get_resolver(self): if callable(self._resolver): diff --git a/riak/client/operations.py b/riak/client/operations.py index b6b8e877..09295422 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -24,33 +24,37 @@ class RiakClientOperations(RiakClientTransport): Methods for RiakClient that result in requests sent to the Riak cluster. - Note that all of these methods have an implicit 'transport' + Note that many of these methods have an implicit 'transport' argument that will be prepended automatically as part of the retry logic, and does not need to be supplied by the user. """ @retryable - def get_buckets(self, transport): + def get_buckets(self, transport, timeout=None): """ Get the list of buckets as RiakBucket instances. NOTE: Do not use this in production, as it requires traversing through all keys stored in a cluster. + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype list of RiakBucket instances """ - return [self.bucket(name) for name in transport.get_buckets()] + return [self.bucket(name) for name in transport.get_buckets(timeout=timeout)] - def stream_buckets(self): + def stream_buckets(self, timeout=None): """ Streams the list of buckets. This is a generator method that should be iterated over. NOTE: Do not use this in production, as it requires traversing through all keys stored in a cluster. + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype iterator """ with self._transport() as transport: - stream = transport.stream_buckets() + stream = transport.stream_buckets(timeout=timeout) try: for bucket_list in stream: bucket_list = [self.bucket(name) for name in bucket_list] @@ -121,17 +125,19 @@ def clear_bucket_props(self, transport, bucket): return transport.clear_bucket_props(bucket) @retryable - def get_keys(self, transport, bucket): + def get_keys(self, transport, bucket, timeout=None): """ Lists all keys in a bucket. :param bucket: the bucket whose properties will be set :type bucket: RiakBucket + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: list """ - return transport.get_keys(bucket) + return transport.get_keys(bucket, timeout=timeout) - def stream_keys(self, bucket): + def stream_keys(self, bucket, timeout=None): """ Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. @@ -139,10 +145,12 @@ def stream_keys(self, bucket): :param bucket: the bucket whose properties will be set :type bucket: RiakBucket + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: iterator """ with self._transport() as transport: - stream = transport.stream_keys(bucket) + stream = transport.stream_keys(bucket, timeout=timeout) try: for keylist in stream: if len(keylist) > 0: @@ -152,7 +160,7 @@ def stream_keys(self, bucket): @retryable def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, - if_none_match=None): + if_none_match=None, timeout=None): """ Stores an object in the Riak cluster. @@ -170,13 +178,16 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, :param if_none_match: whether to fail the write if the object exists :type if_none_match: boolean + :param timeout: a timeout value in milliseconds + :type timeout: int """ return transport.put(robj, w=w, dw=dw, pw=pw, return_body=return_body, - if_none_match=if_none_match) + if_none_match=if_none_match, + timeout=timeout) @retryable - def get(self, transport, robj, r=None, pr=None): + def get(self, transport, robj, r=None, pr=None, timeout=None): """ Fetches the contents of a Riak object. @@ -186,16 +197,18 @@ def get(self, transport, robj, r=None, pr=None): :type r: integer, string, None :param pr: the primary read quorum :type pr: integer, string, None + :param timeout: a timeout value in milliseconds + :type timeout: int """ if not isinstance(robj.key, basestring): raise TypeError( 'key must be a string, instead got {0}'.format(repr(robj.key))) - return transport.get(robj, r=r, pr=pr) + return transport.get(robj, r=r, pr=pr, timeout=timeout) @retryable def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, - pr=None, pw=None): + pr=None, pw=None, timeout=None): """ Deletes an object from Riak. @@ -213,9 +226,11 @@ def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, :type dw: integer, string, None :param pw: the primary write quorum :type pw: integer, string, None + :param timeout: a timeout value in milliseconds + :type timeout: int """ return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, - pw=pw) + pw=pw, timeout=timeout) @retryable def mapred(self, transport, inputs, query, timeout): diff --git a/riak/riak_object.py b/riak/riak_object.py index f6a4aa8f..b74d61c7 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -242,7 +242,7 @@ def get_sibling(self, index): return self.siblings[index] def store(self, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): + if_none_match=False, timeout=None): """ Store the object in Riak. When this operation completes, the object could contain new metadata and possibly new data if Riak @@ -265,6 +265,8 @@ def store(self, w=None, dw=None, pw=None, return_body=True, :param if_none_match: Should the object be stored only if there is no key previously defined :type if_none_match: bool + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: RiakObject """ if len(self.siblings) != 1: raise ConflictError("Attempting to store an invalid object, " @@ -272,11 +274,12 @@ def store(self, w=None, dw=None, pw=None, return_body=True, self.client.put(self, w=w, dw=dw, pw=pw, return_body=return_body, - if_none_match=if_none_match) + if_none_match=if_none_match, + timeout=timeout) return self - def reload(self, r=None, pr=None): + def reload(self, r=None, pr=None, timeout=None): """ Reload the object from Riak. When this operation completes, the object could contain new metadata and a new value, if the object @@ -289,13 +292,16 @@ def reload(self, r=None, pr=None): be available before performing the read that precedes the put :type pr: integer + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: RiakObject """ - self.client.get(self, r=r, pr=pr) + self.client.get(self, r=r, pr=pr, timeout=timeout) return self - def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None, + timeout=None): """ Delete this object from Riak. @@ -319,10 +325,13 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): :param pw: PW-value, require this many primary partitions to be available before performing the put :type pw: integer + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: RiakObject """ - self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) + self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw, + timeout=timeout) self.clear() return self diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 11185791..e022d02a 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -61,39 +61,37 @@ def make_fixed_client_id(self): def ping(self): """ Ping the remote server - @return boolean """ raise NotImplementedError - def get(self, robj, r=None): + def get(self, robj, r=None, pr=None, timeout=None): """ - Serialize get request and deserialize response - @return (vclock=None, [(metadata, value)]=None) + Fetches an object. """ raise NotImplementedError - def put(self, robj, w=None, dw=None, return_body=True): + def put(self, robj, w=None, dw=None, pw=None, return_body=None, + if_none_match=None, timeout=None): """ - Serialize put request and deserialize response - if 'content' - is true, retrieve the updated metadata/content - @return (vclock=None, [(metadata, value)]=None) + Stores an object. """ raise NotImplementedError - def delete(self, robj, rw=None): + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, + pw=None, timeout=None): """ - Serialize delete request and deserialize response + Deletes an object. @return true """ raise NotImplementedError - def get_buckets(self): + def get_buckets(self, timeout=None): """ - Serialize get buckets request and deserialize response + Gets the list of buckets as strings. """ raise NotImplementedError - def stream_buckets(self): + def stream_buckets(self, timeout=None): """ Streams the list of buckets through an iterator """ @@ -101,34 +99,29 @@ def stream_buckets(self): def get_bucket_props(self, bucket): """ - Serialize get bucket property request and deserialize response - @return dict() + Fetches properties for the given bucket. """ raise NotImplementedError def set_bucket_props(self, bucket, props): """ - Serialize set bucket property request and deserialize response - bucket = bucket object - props = dictionary of properties - @return boolean + Sets properties on the given bucket. """ raise NotImplementedError def clear_bucket_props(self, bucket): """ Reset bucket properties to their defaults - bucket = bucket object """ raise NotImplementedError - def get_keys(self, bucket): + def get_keys(self, bucket, timeout=None): """ Lists all keys within the given bucket. """ raise NotImplementedError - def stream_keys(self, bucket): + def stream_keys(self, bucket, timeout=None): """ Streams the list of keys for the bucket through an iterator. """ From a501cd287171f63730a7f5fd9b6317e9487c39eb Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 16:49:13 -0500 Subject: [PATCH 058/672] Add assertions for counters feature detection. --- riak/tests/test_feature_detection.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py index 4048c0d5..fad2372e 100644 --- a/riak/tests/test_feature_detection.py +++ b/riak/tests/test_feature_detection.py @@ -56,6 +56,7 @@ def test_pre_10(self): self.assertFalse(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) def test_10(self): t = DummyTransport("1.0.3") @@ -68,6 +69,7 @@ def test_10(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) def test_11(self): t = DummyTransport("1.1.4") @@ -80,6 +82,7 @@ def test_11(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) def test_12(self): t = DummyTransport("1.2.0") @@ -92,6 +95,7 @@ def test_12(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) def test_12_loose(self): t = DummyTransport("1.2.1p3") @@ -104,6 +108,7 @@ def test_12_loose(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) def test_14(self): t = DummyTransport("1.4.0rc1") @@ -116,6 +121,7 @@ def test_14(self): self.assertTrue(t.pb_head()) self.assertTrue(t.pb_clear_bucket_props()) self.assertTrue(t.pb_all_bucket_props()) + self.assertTrue(t.counters()) if __name__ == '__main__': unittest.main() From 8249d288d908e1f4dc86527641b641055c51fb5b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 19:01:32 -0500 Subject: [PATCH 059/672] Validate timeout parameters. --- riak/client/operations.py | 22 +++++++++++++++++++++- riak/tests/test_all.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 09295422..638f7243 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -40,7 +40,9 @@ def get_buckets(self, transport, timeout=None): :type timeout: int :rtype list of RiakBucket instances """ - return [self.bucket(name) for name in transport.get_buckets(timeout=timeout)] + _validate_timeout(timeout) + return [self.bucket(name) for name in + transport.get_buckets(timeout=timeout)] def stream_buckets(self, timeout=None): """ @@ -53,6 +55,7 @@ def stream_buckets(self, timeout=None): :type timeout: int :rtype iterator """ + _validate_timeout(timeout) with self._transport() as transport: stream = transport.stream_buckets(timeout=timeout) try: @@ -135,6 +138,7 @@ def get_keys(self, transport, bucket, timeout=None): :type timeout: int :rtype: list """ + _validate_timeout(timeout) return transport.get_keys(bucket, timeout=timeout) def stream_keys(self, bucket, timeout=None): @@ -149,6 +153,7 @@ def stream_keys(self, bucket, timeout=None): :type timeout: int :rtype: iterator """ + _validate_timeout(timeout) with self._transport() as transport: stream = transport.stream_keys(bucket, timeout=timeout) try: @@ -181,6 +186,7 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, :param timeout: a timeout value in milliseconds :type timeout: int """ + _validate_timeout(timeout) return transport.put(robj, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match, @@ -200,6 +206,7 @@ def get(self, transport, robj, r=None, pr=None, timeout=None): :param timeout: a timeout value in milliseconds :type timeout: int """ + _validate_timeout(timeout) if not isinstance(robj.key, basestring): raise TypeError( 'key must be a string, instead got {0}'.format(repr(robj.key))) @@ -229,6 +236,7 @@ def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, :param timeout: a timeout value in milliseconds :type timeout: int """ + _validate_timeout(timeout) return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw, timeout=timeout) @@ -245,6 +253,7 @@ def mapred(self, transport, inputs, query, timeout): :type timeout: integer, None :rtype: mixed """ + _validate_timeout(timeout) return transport.mapred(inputs, query, timeout) def stream_mapred(self, inputs, query, timeout): @@ -260,6 +269,7 @@ def stream_mapred(self, inputs, query, timeout): :type timeout: integer, None :rtype: iterator """ + _validate_timeout(timeout) with self._transport() as transport: stream = transport.stream_mapred(inputs, query, timeout) try: @@ -307,3 +317,13 @@ def fulltext_delete(self, transport, index, docs=None, queries=None): :type queries: list """ transport.fulltext_delete(index, docs, queries) + + +def _validate_timeout(timeout): + """ + Raises an exception if the given timeout is an invalid value. + """ + if not (timeout is None or + (type(timeout) in (int, long) and + timeout > 0)): + raise ValueError("timeout must be a positive integer") diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 592ecdf1..89172760 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -128,6 +128,40 @@ def test_request_retries(self): # error. self.assertRaises(IOError, client.ping) + def test_timeout_validation(self): + bucket = self.client.bucket(self.bucket_name) + key = self.key_name + obj = bucket.new(key) + for bad in [0, -1, False, "foo"]: + with self.assertRaises(ValueError): + self.client.get_buckets(timeout=bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_buckets(timeout=bad): + pass + + with self.assertRaises(ValueError): + self.client.get_keys(bucket, timeout=bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_keys(bucket, timeout=bad): + pass + + with self.assertRaises(ValueError): + self.client.put(obj, timeout=bad) + + with self.assertRaises(ValueError): + self.client.get(obj, timeout=bad) + + with self.assertRaises(ValueError): + self.client.delete(obj, timeout=bad) + + with self.assertRaises(ValueError): + self.client.mapred([], [], bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_mapred([], [], bad): + pass class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, From f110966c4175e4a7a30bccc4e35408403fa9c6b5 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 19:02:17 -0500 Subject: [PATCH 060/672] Add timeouts to the backend and test streaming buckets. --- riak/tests/test_all.py | 1 + riak/tests/test_kv.py | 24 +++++++++++++++++++ riak/transports/feature_detect.py | 7 ++++++ riak/transports/http/resources.py | 2 +- riak/transports/http/transport.py | 31 +++++++++++++----------- riak/transports/pbc/messages.py | 2 +- riak/transports/pbc/transport.py | 40 ++++++++++++++++++++++--------- 7 files changed, 80 insertions(+), 27 deletions(-) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 89172760..c484e158 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -163,6 +163,7 @@ def test_timeout_validation(self): for i in self.client.stream_mapred([], [], bad): pass + class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, BucketPropsTest, diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index b78046e9..de0ea790 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -423,6 +423,30 @@ def test_list_buckets(self): buckets = self.client.get_buckets() self.assertTrue(self.bucket_name in [x.name for x in buckets]) + def test_stream_buckets(self): + bucket = self.client.bucket(self.bucket_name) + bucket.new(self.key_name, data={"foo": "one", + "bar": "baz"}).store() + buckets = [] + for bucket_list in self.client.stream_buckets(): + buckets.extend(bucket_list) + + self.assertTrue(self.bucket_name in [x.name for x in buckets]) + + def test_stream_buckets_abort(self): + bucket = self.client.bucket(self.bucket_name) + bucket.new(self.key_name, data={"foo": "one", + "bar": "baz"}).store() + try: + for bucket_list in self.client.stream_buckets(): + raise RuntimeError("abort") + except RuntimeError: + pass + + robj = bucket.get(self.key_name) + self.assertTrue(robj.exists) + self.assertEqual(len(robj.siblings), 1) + def generate_siblings(self, original, count=5, delay=None): vals = [] for i in range(count): diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 5dac0180..2ffd2652 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -114,6 +114,13 @@ def bucket_stream(self): """ return self.server_version >= versions[1.4] + def client_timeouts(self): + """ + Whether client-supplied timeouts are supported. + :rtype bool + """ + return self.server_version >= versions[1.4] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index 92c52032..28ac59f2 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -51,7 +51,7 @@ def bucket_properties_path(self, bucket, **options): "props", **options) else: query = options.copy() - query.update(props=True, keys=True) + query.update(props=True, keys=False) return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), **query) def key_list_path(self, bucket, **options): diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 91fd910d..43300fb5 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -106,25 +106,26 @@ def get_resources(self): else: return {} - def get(self, robj, r=None, pr=None): + def get(self, robj, r=None, pr=None, timeout=None): """ Get a bucket/key from the server """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'r': r, 'pr': pr} + params = {'r': r, 'pr': pr, 'timeout': timeout} url = self.object_path(robj.bucket.name, robj.key, **params) response = self._request('GET', url) return self._parse_body(robj, response, [200, 300, 404]) def put(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): + if_none_match=False, timeout=None): """ Puts a (possibly new) object. """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} + params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw, + 'timeout': timeout} url = self.object_path(robj.bucket.name, robj.key, **params) headers = self._build_put_headers(robj, if_none_match=if_none_match) content = bytearray(robj.encoded_data) @@ -143,13 +144,15 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, self.check_http_code(response[0], expect) return None - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None, + timeout=None): """ Delete an object. """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} + params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw, + 'timeout': timeout} headers = {} url = self.object_path(robj.bucket.name, robj.key, **params) if self.tombstone_vclocks() and robj.vclock is not None: @@ -158,11 +161,11 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): self.check_http_code(response[0], [204, 404]) return self - def get_keys(self, bucket): + def get_keys(self, bucket, timeout=None): """ Fetch a list of keys for the bucket """ - url = self.key_list_path(bucket.name) + url = self.key_list_path(bucket.name, timeout=timeout) status, _, body = self._request('GET', url) if status == 200: @@ -171,8 +174,8 @@ def get_keys(self, bucket): else: raise Exception('Error listing keys.') - def stream_keys(self, bucket): - url = self.key_list_path(bucket.name, keys='stream') + def stream_keys(self, bucket, timeout=None): + url = self.key_list_path(bucket.name, keys='stream', timeout=timeout) status, headers, response = self._request('GET', url, stream=True) if status == 200: @@ -180,11 +183,11 @@ def stream_keys(self, bucket): else: raise Exception('Error listing keys.') - def get_buckets(self): + def get_buckets(self, timeout=None): """ Fetch a list of all buckets """ - url = self.bucket_list_path() + url = self.bucket_list_path(timeout=timeout) status, headers, body = self._request('GET', url) if status == 200: @@ -193,7 +196,7 @@ def get_buckets(self): else: raise Exception('Error getting buckets.') - def stream_buckets(self): + def stream_buckets(self, timeout=None): """ Stream list of buckets through an iterator """ @@ -201,7 +204,7 @@ def stream_buckets(self): raise NotImplementedError('Streaming list-buckets is not ' 'supported') - url = self.bucket_list_path(buckets="stream") + url = self.bucket_list_path(buckets="stream", timeout=timeout) status, headers, response = self._request('GET', url, stream=True) if status == 200: diff --git a/riak/transports/pbc/messages.py b/riak/transports/pbc/messages.py index d4b867c8..58a2adb4 100644 --- a/riak/transports/pbc/messages.py +++ b/riak/transports/pbc/messages.py @@ -78,7 +78,7 @@ MSG_CODE_PUT_RESP: riak_pb.RpbPutResp, MSG_CODE_DEL_REQ: riak_pb.RpbDelReq, MSG_CODE_DEL_RESP: None, - MSG_CODE_LIST_BUCKETS_REQ: None, + MSG_CODE_LIST_BUCKETS_REQ: riak_pb.RpbListBucketsReq, MSG_CODE_LIST_BUCKETS_RESP: riak_pb.RpbListBucketsResp, MSG_CODE_LIST_KEYS_REQ: riak_pb.RpbListKeysReq, MSG_CODE_LIST_KEYS_RESP: riak_pb.RpbListKeysResp, diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index c4ded703..25bb70da 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -117,7 +117,7 @@ def _set_client_id(self, client_id): client_id = property(_get_client_id, _set_client_id, doc="""the client ID for this connection""") - def get(self, robj, r=None, pr=None): + def get(self, robj, r=None, pr=None, timeout=None): """ Serialize get request and deserialize response """ @@ -128,7 +128,8 @@ def get(self, robj, r=None, pr=None): req.r = self._encode_quorum(r) if self.quorum_controls() and pr: req.pr = self._encode_quorum(pr) - + if self.client_timeouts() and timeout: + req.timeout = timeout if self.tombstone_vclocks(): req.deletedvclock = 1 @@ -154,7 +155,7 @@ def get(self, robj, r=None, pr=None): return robj def put(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): + if_none_match=False, timeout=None): """ Serialize get request and deserialize response """ @@ -172,6 +173,8 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, req.return_body = 1 if if_none_match: req.if_none_match = 1 + if self.client_timeouts() and timeout: + req.timeout = timeout req.bucket = bucket.name if robj.key: @@ -196,7 +199,8 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, return robj - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None, + timeout=None): """ Serialize get request and deserialize response """ @@ -218,6 +222,9 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): if pw: req.pw = self._encode_quorum(pw) + if self.client_timeouts() and timeout: + req.timeout = timeout + if self.tombstone_vclocks() and robj.vclock: req.vclock = robj.vclock.encode('binary') @@ -228,38 +235,45 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): MSG_CODE_DEL_RESP) return self - def get_keys(self, bucket): + def get_keys(self, bucket, timeout=None): """ Lists all keys within a bucket. """ keys = [] - for keylist in self.stream_keys(bucket): + for keylist in self.stream_keys(bucket, timeout=timeout): for key in keylist: keys.append(key) return keys - def stream_keys(self, bucket): + def stream_keys(self, bucket, timeout=None): """ Streams keys from a bucket, returning an iterator that yields lists of keys. """ req = riak_pb.RpbListKeysReq() req.bucket = bucket.name + if self.client_timeouts() and timeout: + req.timeout = timeout self._send_msg(MSG_CODE_LIST_KEYS_REQ, req) return RiakPbcKeyStream(self) - def get_buckets(self): + def get_buckets(self, timeout=None): """ Serialize bucket listing request and deserialize response """ - msg_code, resp = self._request(MSG_CODE_LIST_BUCKETS_REQ, - expect=MSG_CODE_LIST_BUCKETS_RESP) + req = None + if self.client_timeouts() and timeout: + req = riak_pb.RpbListBucketsReq() + req.timeout = timeout + + msg_code, resp = self._request(MSG_CODE_LIST_BUCKETS_REQ, req, + MSG_CODE_LIST_BUCKETS_RESP) return resp.buckets - def stream_buckets(self): + def stream_buckets(self, timeout=None): """ Stream list of buckets through an iterator """ @@ -270,6 +284,10 @@ def stream_buckets(self): req = riak_pb.RpbListBucketsReq() req.stream = True + # Bucket streaming landed in the same release as timeouts, so + # we don't need to check the capability. + if timeout: + req.timeout = timeout self._send_msg(MSG_CODE_LIST_BUCKETS_REQ, req) From 3c5430bf393d6dd07caac11b36784f492c8ab978 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 19:39:50 -0500 Subject: [PATCH 061/672] Add failing test for streaming 2i. --- riak/tests/test_2i.py | 50 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index d303eb4b..60a97fb6 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -15,7 +15,7 @@ class TwoITests(object): def is_2i_supported(self): # Immediate test to see if 2i is even supported w/ the backend try: - self.client.index('foo', 'bar_bin', 'baz').run() + self.client.get_index('foo', 'bar_bin', 'baz') return True except Exception as e: if "indexes_not_supported" in str(e): @@ -25,7 +25,7 @@ def is_2i_supported(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_store(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") # Create a new object with indexes... bucket = self.client.bucket(self.bucket_name) @@ -106,7 +106,7 @@ def test_secondary_index_store(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_set_indexes(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) foo = bucket.new('foo', 1) @@ -124,7 +124,7 @@ def test_set_indexes(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_remove_indexes(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) bar = bucket.new('bar', 1).add_index('bar_int', 1)\ @@ -184,7 +184,7 @@ def test_remove_indexes(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_query(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) @@ -240,7 +240,7 @@ def test_secondary_index_query(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_invalid_name(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) @@ -250,7 +250,7 @@ def test_secondary_index_invalid_name(self): @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') def test_set_index(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) obj = bucket.new('bar', 1) @@ -264,3 +264,39 @@ def test_set_index(self): self.assertEqual(set((('bar_int', 3), ('bar2_int', 1))), obj.indexes) obj.set_index('bar2_int', 10) self.assertEqual(set((('bar_int', 3), ('bar2_int', 10))), obj.indexes) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_stream_index(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I not supported") + + bucket = self.client.bucket(self.bucket_name) + + o1 = bucket.\ + new(self.key_name, 'data1').\ + add_index('field1_bin', 'val1').\ + add_index('field2_int', 1001).\ + store() + o2 = bucket.\ + new(self.randname(), 'data1').\ + add_index('field1_bin', 'val2').\ + add_index('field2_int', 1002).\ + store() + o3 = bucket.\ + new(self.randname(), 'data1').\ + add_index('field1_bin', 'val3').\ + add_index('field2_int', 1003).\ + store() + o4 = bucket.\ + new(self.randname(), 'data1').\ + add_index('field1_bin', 'val4').\ + add_index('field2_int', 1004).\ + store() + + keys = [] + for entries in self.client.stream_index(bucket, 'field1_bin', + 'val1', 'val3'): + keys.append(entries) + + # Riak 1.4 ensures that entries come back in-order + self.assertEqual([o1.key, o2.key, o3.key], keys) From 46740ce5ee863bc9a84c9fd1ff9254d3e5ffe691 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 20:08:30 -0500 Subject: [PATCH 062/672] Add front-end to stream_index operation. --- riak/bucket.py | 7 +++++++ riak/client/operations.py | 23 +++++++++++++++++++++++ riak/tests/test_feature_detection.py | 6 ++++++ riak/transports/feature_detect.py | 7 +++++++ riak/transports/transport.py | 6 ++++++ 5 files changed, 49 insertions(+) diff --git a/riak/bucket.py b/riak/bucket.py index 966ebe68..4c54c6f0 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -411,6 +411,13 @@ def get_index(self, index, startkey, endkey=None): """ return self._client.get_index(self.name, index, startkey, endkey) + def stream_index(self, index, startkey, endkey=None): + """ + Queries a secondary index over objects in this bucket, + streaming keys via an iterator. + """ + return self._client.stream_index(self.name, index, startkey, endkey) + def delete(self, key, **kwargs): """Deletes an object from riak. diff --git a/riak/client/operations.py b/riak/client/operations.py index c928128f..d6004318 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -66,6 +66,29 @@ def get_index(self, transport, bucket, index, startkey, endkey=None): """ return transport.get_index(bucket, index, startkey, endkey) + def stream_index(self, bucket, index, startkey, endkey=None): + """ + Queries a secondary index, streaming matching keys through an + iterator. + + :param bucket: the bucket whose index will be queried + :type bucket: RiakBucket + :param index: the index to query + :type index: string + :param startkey: the sole key to query, or beginning of the query range + :type startkey: string, integer + :param endkey: the end of the query range (optional if equality) + :type endkey: string, integer + :rtype: iterable + """ + with self._transport() as transport: + stream = transport.stream_index(bucket, index, startkey, endkey) + try: + for item in stream: + yield item + finally: + stream.close() + @retryable def get_bucket_props(self, transport, bucket): """ diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py index 4048c0d5..a8f0fed2 100644 --- a/riak/tests/test_feature_detection.py +++ b/riak/tests/test_feature_detection.py @@ -56,6 +56,7 @@ def test_pre_10(self): self.assertFalse(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.stream_indexes()) def test_10(self): t = DummyTransport("1.0.3") @@ -68,6 +69,7 @@ def test_10(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.stream_indexes()) def test_11(self): t = DummyTransport("1.1.4") @@ -80,6 +82,7 @@ def test_11(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.stream_indexes()) def test_12(self): t = DummyTransport("1.2.0") @@ -92,6 +95,7 @@ def test_12(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.stream_indexes()) def test_12_loose(self): t = DummyTransport("1.2.1p3") @@ -104,6 +108,7 @@ def test_12_loose(self): self.assertTrue(t.pb_head()) self.assertFalse(t.pb_clear_bucket_props()) self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.stream_indexes()) def test_14(self): t = DummyTransport("1.4.0rc1") @@ -116,6 +121,7 @@ def test_14(self): self.assertTrue(t.pb_head()) self.assertTrue(t.pb_clear_bucket_props()) self.assertTrue(t.pb_all_bucket_props()) + self.assertTrue(t.stream_indexes()) if __name__ == '__main__': unittest.main() diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 3712ec8b..220b6446 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -107,6 +107,13 @@ def pb_all_bucket_props(self): """ return self.server_version >= versions[1.4] + def stream_indexes(self): + """ + Whether secondary indexes support streaming responses. + :rtype bool + """ + return self.server_version >= versions[1.4] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index c6d581c7..d373152e 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -167,6 +167,12 @@ def get_index(self, bucket, index, startkey, endkey=None): """ raise NotImplementedError + def stream_index(self, bucket, index, startkey, endkey=None): + """ + Streams a secondary index query. + """ + raise NotImplementedError + def fulltext_add(self, index, *docs): """ Adds documents to the full-text index. From e48ab19b1852258aa3ee37e55cfcc711c2238455 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 21:10:59 -0500 Subject: [PATCH 063/672] Fix bug in test. --- riak/tests/test_2i.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 60a97fb6..1ea5cc4b 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -294,9 +294,8 @@ def test_stream_index(self): store() keys = [] - for entries in self.client.stream_index(bucket, 'field1_bin', - 'val1', 'val3'): - keys.append(entries) + for entries in bucket.stream_index('field1_bin', 'val1', 'val3'): + keys.extend(entries) # Riak 1.4 ensures that entries come back in-order self.assertEqual([o1.key, o2.key, o3.key], keys) From a10fd6b607936ac8b43bbf67e1cc2c5cb9fffc2e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 1 Jul 2013 21:12:25 -0500 Subject: [PATCH 064/672] Implement basic streaming of secondary index responses. There are edge cases around the response format in the stream iterator classes that I hope to address in future commits. They are helpfully marked by "WAT". --- riak/transports/http/stream.py | 20 ++++++++++++++++++++ riak/transports/http/transport.py | 19 ++++++++++++++++++- riak/transports/pbc/codec.py | 23 +++++++++++++++++++++++ riak/transports/pbc/stream.py | 26 +++++++++++++++++++++++++- riak/transports/pbc/transport.py | 22 +++++++++++++--------- 5 files changed, 99 insertions(+), 11 deletions(-) diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 2f1620f9..668413c9 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -123,3 +123,23 @@ def next(self): message = super(RiakHttpMapReduceStream, self).next() payload = json.loads(message.get_payload()) return payload['phase'], payload['data'] + + +class RiakHttpIndexStream(RiakHttpMultipartStream): + """ + Streaming iterator for secondary indexes over HTTP + """ + + def next(self): + message = super(RiakHttpIndexStream, self).next() + payload = json.loads(message.get_payload()) + if u'keys' in payload: + return payload[u'keys'] + elif u'results' in payload: + structs = payload[u'results'] + # Format is {"results":[{"2ikey":"primarykey"}, ...]} + munged = [ d.items()[0] for d in structs ] + return munged + else: + # WAT + self.next() diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 632f37bf..ac19741c 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -33,7 +33,8 @@ from riak.transports.http.codec import RiakHttpCodec from riak.transports.http.stream import ( RiakHttpKeyStream, - RiakHttpMapReduceStream) + RiakHttpMapReduceStream, + RiakHttpIndexStream) from riak import RiakError @@ -285,6 +286,22 @@ def get_index(self, bucket, index, startkey, endkey=None): json_data = json.loads(body) return json_data[u'keys'][:] + def stream_index(self, bucket, index, startkey, endkey=None): + """ + Streams a secondary index query. + """ + if not self.stream_indexes(): + raise NotImplementedError("Secondary index streaming is not " + "supported") + + url = self.index_path(bucket, index, startkey, endkey, stream=True) + status, headers, response = self._request('GET', url, stream=True) + + if status == 200: + return RiakHttpIndexStream(response) + else: + raise Exception('Error streaming secondary index.') + def search(self, index, query, **params): """ Performs a search query. diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 250995aa..a2f3c916 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -373,3 +373,26 @@ def _encode_hook(self, hook, msg): else: self._encode_modfun(hook, msg.modfun) return msg + + def _encode_index_req(self, bucket, index, startkey, endkey=None): + """ + Encodes a secondary index request into the protobuf message. + + :param bucket: the bucket whose index to query + :type bucket: string + :param index: the index to query + :type index: string + :param startkey: the value or beginning of the range + :type startkey: integer, string + :param endkey: the end of the range + :type endkey: integer, string + """ + req = riak_pb.RpbIndexReq(bucket=bucket, index=index) + if endkey: + req.qtype = riak_pb.RpbIndexReq.range + req.range_min = str(startkey) + req.range_max = str(endkey) + else: + req.qtype = riak_pb.RpbIndexReq.eq + req.key = str(startkey) + return req diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index 02c69918..3224ee41 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -20,7 +20,8 @@ import json from riak.transports.pbc.messages import ( MSG_CODE_LIST_KEYS_RESP, - MSG_CODE_MAPRED_RESP + MSG_CODE_MAPRED_RESP, + MSG_CODE_INDEX_RESP ) @@ -96,3 +97,26 @@ def next(self): raise StopIteration return response.phase, json.loads(response.response) + + +class RiakPbcIndexStream(RiakPbcStream): + """ + Used internally by RiakPbcTransport to implement Secondary Index + streams. + """ + + _expect = MSG_CODE_INDEX_RESP + + def next(self): + response = super(RiakPbcIndexStream, self).next() + + if response.done and not (response.keys or response.results): + raise StopIteration + + if response.keys: + return response.keys + elif response.results: + return [(r.key, r.value) for r in response.results] + else: + # WAT + return self.next() diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 0b62689a..52fb27dd 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -24,7 +24,7 @@ from riak.transports.transport import RiakTransport from riak.riak_object import VClock from connection import RiakPbcConnection -from stream import RiakPbcKeyStream, RiakPbcMapredStream +from stream import RiakPbcKeyStream, RiakPbcMapredStream, RiakPbcIndexStream from codec import RiakPbcCodec from messages import ( @@ -338,19 +338,23 @@ def get_index(self, bucket, index, startkey, endkey=None): if not self.pb_indexes(): return self._get_index_mapred_emu(bucket, index, startkey, endkey) - req = riak_pb.RpbIndexReq(bucket=bucket, index=index) - if endkey: - req.qtype = riak_pb.RpbIndexReq.range - req.range_min = str(startkey) - req.range_max = str(endkey) - else: - req.qtype = riak_pb.RpbIndexReq.eq - req.key = str(startkey) + req = self._encode_index_req(bucket, index, startkey, endkey) msg_code, resp = self._request(MSG_CODE_INDEX_REQ, req, MSG_CODE_INDEX_RESP) return resp.keys + def stream_index(self, bucket, index, startkey, endkey=None): + if not self.stream_indexes(): + raise NotImplementedError("Secondary index streaming is not " + "supported") + req = self._encode_index_req(bucket, index, startkey, endkey) + req.stream = True + + self._send_msg(MSG_CODE_INDEX_REQ, req) + + return RiakPbcIndexStream(self) + def search(self, index, query, **params): if not self.pb_search(): return self._search_mapred_emu(index, query) From 5fcbea162247933a4584498ffe6d569b8a732f9f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 2 Jul 2013 12:44:27 -0500 Subject: [PATCH 065/672] Add return_terms option. --- riak/bucket.py | 10 ++-- riak/client/operations.py | 16 ++++-- riak/tests/test_2i.py | 81 ++++++++++++++++--------------- riak/transports/http/codec.py | 4 +- riak/transports/http/stream.py | 12 ++++- riak/transports/http/transport.py | 24 ++++++--- riak/transports/pbc/codec.py | 12 +++-- riak/transports/pbc/stream.py | 18 +++++-- riak/transports/pbc/transport.py | 22 ++++++--- riak/util.py | 7 +++ 10 files changed, 136 insertions(+), 70 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 4c54c6f0..d8a03db8 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -405,18 +405,20 @@ def search(self, query, **params): """ return self._client.solr.search(self.name, query, **params) - def get_index(self, index, startkey, endkey=None): + def get_index(self, index, startkey, endkey=None, return_terms=None): """ Queries a secondary index over objects in this bucket, returning keys. """ - return self._client.get_index(self.name, index, startkey, endkey) + return self._client.get_index(self.name, index, startkey, endkey, + return_terms=return_terms) - def stream_index(self, index, startkey, endkey=None): + def stream_index(self, index, startkey, endkey=None, return_terms=None): """ Queries a secondary index over objects in this bucket, streaming keys via an iterator. """ - return self._client.stream_index(self.name, index, startkey, endkey) + return self._client.stream_index(self.name, index, startkey, endkey, + return_terms=return_terms) def delete(self, key, **kwargs): """Deletes an object from riak. diff --git a/riak/client/operations.py b/riak/client/operations.py index d6004318..6fd4956c 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -50,7 +50,8 @@ def ping(self, transport): is_alive = ping @retryable - def get_index(self, transport, bucket, index, startkey, endkey=None): + def get_index(self, transport, bucket, index, startkey, endkey=None, + return_terms=None): """ Queries a secondary index, returning matching keys. @@ -62,11 +63,15 @@ def get_index(self, transport, bucket, index, startkey, endkey=None): :type startkey: string, integer :param endkey: the end of the query range (optional if equality) :type endkey: string, integer + :param return_terms: whether to include the secondary index value + :type return_terms: boolean :rtype: list """ - return transport.get_index(bucket, index, startkey, endkey) + return transport.get_index(bucket, index, startkey, endkey, + return_terms=return_terms) - def stream_index(self, bucket, index, startkey, endkey=None): + def stream_index(self, bucket, index, startkey, endkey=None, + return_terms=None): """ Queries a secondary index, streaming matching keys through an iterator. @@ -79,10 +84,13 @@ def stream_index(self, bucket, index, startkey, endkey=None): :type startkey: string, integer :param endkey: the end of the query range (optional if equality) :type endkey: string, integer + :param return_terms: whether to include the secondary index value + :type return_terms: boolean :rtype: iterable """ with self._transport() as transport: - stream = transport.stream_index(bucket, index, startkey, endkey) + stream = transport.stream_index(bucket, index, startkey, endkey, + return_terms=return_terms) try: for item in stream: yield item diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 1ea5cc4b..996741c6 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -186,56 +186,29 @@ def test_secondary_index_query(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") - bucket = self.client.bucket(self.bucket_name) - - bucket.\ - new('mykey1', 'data1').\ - add_index('field1_bin', 'val1').\ - add_index('field2_int', 1001).\ - store() - bucket.\ - new('mykey2', 'data1').\ - add_index('field1_bin', 'val2').\ - add_index('field2_int', 1002).\ - store() - bucket.\ - new('mykey3', 'data1').\ - add_index('field1_bin', 'val3').\ - add_index('field2_int', 1003).\ - store() - bucket.\ - new('mykey4', 'data1').\ - add_index('field1_bin', 'val4').\ - add_index('field2_int', 1004).\ - store() + bucket, o1, o2, o3, o4 = self._create_index_objects() # Test an equality query... results = bucket.get_index('field1_bin', 'val2') self.assertEquals(1, len(results)) - self.assertEquals('mykey2', str(results[0])) + self.assertEquals(o2.key, str(results[0])) # Test a range query... results = bucket.get_index('field1_bin', 'val2', 'val4') vals = set([str(key) for key in results]) self.assertEquals(3, len(results)) - self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) + self.assertEquals(set([o2.key, o3.key, o4.key]), vals) # Test an equality query... results = bucket.get_index('field2_int', 1002) self.assertEquals(1, len(results)) - self.assertEquals('mykey2', str(results[0])) + self.assertEquals(o2.key, str(results[0])) # Test a range query... results = bucket.get_index('field2_int', 1002, 1004) vals = set([str(key) for key in results]) self.assertEquals(3, len(results)) - self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) - - # Clean up... - bucket.get('mykey1').delete() - bucket.get('mykey2').delete() - bucket.get('mykey3').delete() - bucket.get('mykey4').delete() + self.assertEquals(set([o2.key, o3.key, o4.key]), vals) @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_secondary_index_invalid_name(self): @@ -270,6 +243,43 @@ def test_stream_index(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") + bucket, o1, o2, o3, o4 = self._create_index_objects() + + keys = [] + for entries in bucket.stream_index('field1_bin', 'val1', 'val3'): + keys.extend(entries) + + # Riak 1.4 ensures that entries come back in-order + self.assertEqual([o1.key, o2.key, o3.key], keys) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # Test synchronous index query + pairs = bucket.get_index('field1_bin', 'val2', 'val4', + return_terms=True) + + self.assertEqual([('val2', o2.key), + ('val3', o3.key), + ('val4', o4.key)], pairs) + + # Test streaming index query + spairs = [] + for chunk in bucket.stream_index('field2_int', 1002, 1004, + return_terms=True): + spairs.extend(chunk) + + self.assertEqual([(1002, o2.key), (1003, o3.key), (1004, o4.key)], + spairs) + + def _create_index_objects(self): + """ + Creates a number of index objects to be used in 2i test + """ bucket = self.client.bucket(self.bucket_name) o1 = bucket.\ @@ -293,9 +303,4 @@ def test_stream_index(self): add_index('field2_int', 1004).\ store() - keys = [] - for entries in bucket.stream_index('field1_bin', 'val1', 'val3'): - keys.extend(entries) - - # Riak 1.4 ensures that entries come back in-order - self.assertEqual([o1.key, o2.key, o3.key], keys) + return bucket, o1, o2, o3, o4 diff --git a/riak/transports/http/codec.py b/riak/transports/http/codec.py index 25bb7dab..7b9e2076 100644 --- a/riak/transports/http/codec.py +++ b/riak/transports/http/codec.py @@ -35,6 +35,7 @@ from riak.riak_object import VClock from riak.multidict import MultiDict from riak.transports.http.search import XMLSearchResult +from riak.util import decode_index_value class RiakHttpCodec(object): @@ -127,8 +128,7 @@ def _parse_sibling(self, sibling, headers, data): reader = csv.reader([value], skipinitialspace=True) for line in reader: for token in line: - if field.endswith("_int"): - token = int(token) + token = decode_index_value(field, token) sibling.add_index(field, token) elif header == 'x-riak-deleted': sibling.exists = False diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 668413c9..949d64d0 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -21,6 +21,7 @@ import re from cgi import parse_header from email import message_from_string +from riak.util import decode_index_value class RiakHttpStream(object): @@ -130,6 +131,11 @@ class RiakHttpIndexStream(RiakHttpMultipartStream): Streaming iterator for secondary indexes over HTTP """ + def __init__(self, response, index, return_terms): + super(RiakHttpIndexStream, self).__init__(response) + self.index = index + self.return_terms = return_terms + def next(self): message = super(RiakHttpIndexStream, self).next() payload = json.loads(message.get_payload()) @@ -138,8 +144,10 @@ def next(self): elif u'results' in payload: structs = payload[u'results'] # Format is {"results":[{"2ikey":"primarykey"}, ...]} - munged = [ d.items()[0] for d in structs ] - return munged + return [self._decode_pair(d.items()[0]) for d in structs] else: # WAT self.next() + + def _decode_pair(self, pair): + return (decode_index_value(self.index, pair[0]), pair[1]) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index ac19741c..76cc3a96 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -36,6 +36,7 @@ RiakHttpMapReduceStream, RiakHttpIndexStream) from riak import RiakError +from riak.util import decode_index_value class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakHttpCodec, @@ -276,17 +277,27 @@ def stream_mapred(self, inputs, query, timeout=None): 'Error running MapReduce operation. Headers: %s Body: %s' % (repr(headers), repr(response.read()))) - def get_index(self, bucket, index, startkey, endkey=None): + def get_index(self, bucket, index, startkey, endkey=None, + return_terms=None): """ Performs a secondary index query. """ - url = self.index_path(bucket, index, startkey, endkey) + params = {'return_terms': return_terms} + url = self.index_path(bucket, index, startkey, endkey, **params) status, headers, body = self._request('GET', url) self.check_http_code(status, [200]) json_data = json.loads(body) - return json_data[u'keys'][:] + if return_terms: + results = [] + for result in json_data[u'results'][:]: + term, key = result.items()[0] + results.append((decode_index_value(index, term), key),) + return results + else: + return json_data[u'keys'][:] - def stream_index(self, bucket, index, startkey, endkey=None): + def stream_index(self, bucket, index, startkey, endkey=None, + return_terms=None): """ Streams a secondary index query. """ @@ -294,11 +305,12 @@ def stream_index(self, bucket, index, startkey, endkey=None): raise NotImplementedError("Secondary index streaming is not " "supported") - url = self.index_path(bucket, index, startkey, endkey, stream=True) + params = {'return_terms': return_terms, 'stream': True} + url = self.index_path(bucket, index, startkey, endkey, **params) status, headers, response = self._request('GET', url, stream=True) if status == 200: - return RiakHttpIndexStream(response) + return RiakHttpIndexStream(response, index, return_terms) else: raise Exception('Error streaming secondary index.') diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index a2f3c916..ddf61a98 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -18,6 +18,7 @@ import riak_pb from riak import RiakError from riak.content import RiakContent +from riak.util import decode_index_value def _invert(d): @@ -147,8 +148,7 @@ def _decode_content(self, rpb_content, sibling): sibling.usermeta = dict([(usermd.key, usermd.value) for usermd in rpb_content.usermeta]) sibling.indexes = set([(index.key, - self._decode_index_value(index.key, - index.value)) + decode_index_value(index.key, index.value)) for index in rpb_content.indexes]) sibling.encoded_data = rpb_content.value @@ -374,7 +374,8 @@ def _encode_hook(self, hook, msg): self._encode_modfun(hook, msg.modfun) return msg - def _encode_index_req(self, bucket, index, startkey, endkey=None): + def _encode_index_req(self, bucket, index, startkey, endkey=None, + return_terms=None): """ Encodes a secondary index request into the protobuf message. @@ -386,6 +387,9 @@ def _encode_index_req(self, bucket, index, startkey, endkey=None): :type startkey: integer, string :param endkey: the end of the range :type endkey: integer, string + :param return_terms: whether to return the index term with the key + :type return_terms: bool + :rtype riak_pb.RpbIndexReq """ req = riak_pb.RpbIndexReq(bucket=bucket, index=index) if endkey: @@ -395,4 +399,6 @@ def _encode_index_req(self, bucket, index, startkey, endkey=None): else: req.qtype = riak_pb.RpbIndexReq.eq req.key = str(startkey) + if return_terms is not None: + req.return_terms = return_terms return req diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index 3224ee41..ff8345e6 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -107,16 +107,24 @@ class RiakPbcIndexStream(RiakPbcStream): _expect = MSG_CODE_INDEX_RESP + def __init__(self, transport, index, return_terms=False): + super(RiakPbcIndexStream, self).__init__(transport) + self.index = index + self.return_terms = return_terms + def next(self): response = super(RiakPbcIndexStream, self).next() if response.done and not (response.keys or response.results): raise StopIteration - if response.keys: + if self.return_terms and response.results: + return [(self._coerce(r.key), r.value) for r in response.results] + elif response.keys: return response.keys - elif response.results: - return [(r.key, r.value) for r in response.results] + + def _coerce(self, index_value): + if "_int" in self.index: + return long(index_value) else: - # WAT - return self.next() + return str(index_value) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 52fb27dd..1ef91560 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -23,6 +23,7 @@ from riak import RiakError from riak.transports.transport import RiakTransport from riak.riak_object import VClock +from riak.util import decode_index_value from connection import RiakPbcConnection from stream import RiakPbcKeyStream, RiakPbcMapredStream, RiakPbcIndexStream from codec import RiakPbcCodec @@ -334,26 +335,35 @@ def stream_mapred(self, inputs, query, timeout=None): return RiakPbcMapredStream(self) - def get_index(self, bucket, index, startkey, endkey=None): + def get_index(self, bucket, index, startkey, endkey=None, + return_terms=None): if not self.pb_indexes(): return self._get_index_mapred_emu(bucket, index, startkey, endkey) - req = self._encode_index_req(bucket, index, startkey, endkey) + req = self._encode_index_req(bucket, index, startkey, endkey, + return_terms=return_terms) msg_code, resp = self._request(MSG_CODE_INDEX_REQ, req, MSG_CODE_INDEX_RESP) - return resp.keys + if return_terms: + return [(decode_index_value(index, pair.key), pair.value) + for pair in resp.results] + else: + return resp.keys - def stream_index(self, bucket, index, startkey, endkey=None): + def stream_index(self, bucket, index, startkey, endkey=None, + return_terms=None): if not self.stream_indexes(): raise NotImplementedError("Secondary index streaming is not " "supported") - req = self._encode_index_req(bucket, index, startkey, endkey) + + req = self._encode_index_req(bucket, index, startkey, endkey, + return_terms=return_terms) req.stream = True self._send_msg(MSG_CODE_INDEX_REQ, req) - return RiakPbcIndexStream(self) + return RiakPbcIndexStream(self, index, return_terms) def search(self, index, query, **params): if not self.pb_search(): diff --git a/riak/util.py b/riak/util.py index 448ee8d7..f096caf7 100644 --- a/riak/util.py +++ b/riak/util.py @@ -123,3 +123,10 @@ def __get__(self, obj, cls): value = self.fget(obj) setattr(obj, self.func_name, value) return value + + +def decode_index_value(index, value): + if "_int" in index: + return long(value) + else: + return str(value) From 66cef408387a247946c889dfe8249ffcd229bab9 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 3 Jul 2013 15:00:45 -0500 Subject: [PATCH 066/672] Add pagination to secondary indexes, commit the first. This introduces the IndexPage class, which is necessary both to wrap the results into a sensible API, and to provide consistency between single-roundtrip and streaming options. The IndexPage class can appear as if it were a list, is iterable for streaming and regular iteration purposes. Regardless of whether pagination is used, an IndexPage will be returned (thanks to the quirkiness of Python's generators). This leaves equality/return-terms dangling, to be addressed in the next commit. Sending an equality query with the return-terms flag on does not result in pairs being returned, but simply keys. Obviously, the client knows what the index term is and so can inject it into the result (which is what we will do for consistency). --- riak/bucket.py | 14 +++- riak/client/index_page.py | 122 ++++++++++++++++++++++++++++++ riak/client/operations.py | 53 +++++++++---- riak/tests/test_2i.py | 122 ++++++++++++++++++++++++++++++ riak/transports/http/stream.py | 6 +- riak/transports/http/transport.py | 20 +++-- riak/transports/pbc/codec.py | 12 ++- riak/transports/pbc/stream.py | 17 +++-- riak/transports/pbc/transport.py | 22 ++++-- riak/transports/transport.py | 6 +- 10 files changed, 347 insertions(+), 47 deletions(-) create mode 100644 riak/client/index_page.py diff --git a/riak/bucket.py b/riak/bucket.py index d8a03db8..74aaa380 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -405,20 +405,26 @@ def search(self, query, **params): """ return self._client.solr.search(self.name, query, **params) - def get_index(self, index, startkey, endkey=None, return_terms=None): + def get_index(self, index, startkey, endkey=None, return_terms=None, + max_results=None, continuation=None): """ Queries a secondary index over objects in this bucket, returning keys. """ return self._client.get_index(self.name, index, startkey, endkey, - return_terms=return_terms) + return_terms=return_terms, + max_results=max_results, + continuation=continuation) - def stream_index(self, index, startkey, endkey=None, return_terms=None): + def stream_index(self, index, startkey, endkey=None, return_terms=None, + max_results=None, continuation=None): """ Queries a secondary index over objects in this bucket, streaming keys via an iterator. """ return self._client.stream_index(self.name, index, startkey, endkey, - return_terms=return_terms) + return_terms=return_terms, + max_results=max_results, + continuation=continuation) def delete(self, key, **kwargs): """Deletes an object from riak. diff --git a/riak/client/index_page.py b/riak/client/index_page.py new file mode 100644 index 00000000..9a8725cc --- /dev/null +++ b/riak/client/index_page.py @@ -0,0 +1,122 @@ +""" +Copyright 2013 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from collections import namedtuple, Sequence + + +CONTINUATION = namedtuple('Continuation', ['c']) + + +class IndexPage(Sequence, object): + """ + Encapsulates a single page of results from a secondary index + query, with the ability to iterate over results (if not streamed), + capture the page marker (continuation), and automatically fetch + the next page. + + While users will interact with this object, it will be created + automatically by the client and does not need to be instantiated + elsewhere. + """ + def __init__(self, client, bucket, index, startkey, endkey, return_terms, + max_results): + self.client = client + self.bucket = bucket + self.index = index + self.startkey = startkey + self.endkey = endkey + self.return_terms = return_terms + self.max_results = max_results + self.results = None + self.continuation = None + self.stream = False + + def __iter__(self): + if self.results: + try: + for result in self.results: + if self.stream and isinstance(result, CONTINUATION): + self.continuation = result.c + else: + yield result + finally: + if self.stream: + self.results.close() + else: + raise ValueError("No index results to iterate") + + def __len__(self): + if not self.stream and self.results is not None: + return len(self.results) + else: + raise ValueError("Streamed index page has no length") + + def __getitem__(self, index): + if not self.stream and self.results is not None: + return self.results[index] + else: + raise ValueError("Streamed index page has no entries") + + def __eq__(self, other): + if isinstance(other, list) and not (self.stream or + self.results is None): + return self.results == other + elif isinstance(other, IndexPage): + return other.__dict__ == self.__dict__ + else: + return False + + def __ne__(self, other): + return not self.__eq__(other) + + def has_next_page(self): + """ + Whether there is another page available, i.e. the response + included a continuation. + """ + return self.continuation is not None + + def next_page(self, stream=None): + """ + Fetches the next page using the same parameters as the + original query. + + Note that if streaming was used before, it will be used again + unless overridden. + + :param stream: whether to enable streaming. `True` enables, + `False` disables, `None` uses previous value. + :type stream: boolean + """ + if not self.continuation: + raise ValueError("Cannot get next index page, no continuation") + + if stream is not None: + self.stream = stream + + args = {'bucket': self.bucket, + 'index': self.index, + 'startkey': self.startkey, + 'endkey': self.endkey, + 'return_terms': self.return_terms, + 'max_results': self.max_results, + 'continuation': self.continuation} + if self.stream: + return self.client.stream_index(**args) + else: + return self.client.get_index(**args) diff --git a/riak/client/operations.py b/riak/client/operations.py index 6fd4956c..a49b24e1 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -17,6 +17,7 @@ """ from transport import RiakClientTransport, retryable, retryableHttpOnly +from index_page import IndexPage class RiakClientOperations(RiakClientTransport): @@ -51,7 +52,7 @@ def ping(self, transport): @retryable def get_index(self, transport, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, continuation=None): """ Queries a secondary index, returning matching keys. @@ -65,13 +66,29 @@ def get_index(self, transport, bucket, index, startkey, endkey=None, :type endkey: string, integer :param return_terms: whether to include the secondary index value :type return_terms: boolean - :rtype: list + :param max_results: the maximum number of results to return (page size) + :type max_results: integer + :param continuation: the opaque continuation returned from a + previous paginated request + :type continuation: string + :rtype: :class:`riak.client.index_page.IndexPage` """ - return transport.get_index(bucket, index, startkey, endkey, - return_terms=return_terms) + if return_terms and endkey is None: + raise ValueError("Cannot use return_terms with an equality query") + + page = IndexPage(self, bucket, index, startkey, endkey, + return_terms, max_results) + + results, continuation = transport.get_index( + bucket, index, startkey, endkey, return_terms=return_terms, + max_results=max_results, continuation=continuation) + + page.results = results + page.continuation = continuation + return page def stream_index(self, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, continuation=None): """ Queries a secondary index, streaming matching keys through an iterator. @@ -86,16 +103,24 @@ def stream_index(self, bucket, index, startkey, endkey=None, :type endkey: string, integer :param return_terms: whether to include the secondary index value :type return_terms: boolean - :rtype: iterable - """ + :param max_results: the maximum number of results to return (page size) + :type max_results: integer + :param continuation: the opaque continuation returned from a + previous paginated request + :type continuation: string + :rtype: :class:`riak.client.index_page.IndexPage` + """ + if return_terms and endkey is None: + raise ValueError("Cannot use return_terms with an equality query") + + page = IndexPage(self, bucket, index, startkey, endkey, + return_terms, max_results) with self._transport() as transport: - stream = transport.stream_index(bucket, index, startkey, endkey, - return_terms=return_terms) - try: - for item in stream: - yield item - finally: - stream.close() + page.stream = True + page.results = transport.stream_index( + bucket, index, startkey, endkey, return_terms=return_terms, + max_results=max_results, continuation=continuation) + return page @retryable def get_bucket_props(self, transport, bucket): diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 996741c6..aada2192 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -276,6 +276,128 @@ def test_index_return_terms(self): self.assertEqual([(1002, o2.key), (1003, o3.key), (1004, o4.key)], spairs) + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_pagination(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + results = bucket.get_index('field1_bin', 'val0', 'val5', + max_results=2) + # Number of results =< page size + self.assertLessEqual(2, len(results)) + # Results are in-order + self.assertEqual([o1.key, o2.key], results) + + # Continuation/next page present when page size smaller than + # total results size + self.assertIsNotNone(results.continuation) + self.assertTrue(results.has_next_page()) + + # Retrieving next page gets more results + page2 = results.next_page() + self.assertLessEqual(2, len(page2)) + self.assertEqual([o3.key, o4.key], page2) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_pagination_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # ========= Above steps work for return-terms ========== + results = bucket.get_index('field1_bin', 'val0', 'val5', + max_results=2, return_terms=True) + # Number of results =< page size + self.assertLessEqual(2, len(results)) + # Results are in-order + self.assertEqual([('val1', o1.key), ('val2', o2.key)], results) + + # Continuation/next page present when page size smaller than + # total results size + self.assertIsNotNone(results.continuation) + self.assertTrue(results.has_next_page()) + + # Retrieving next page gets more results + page2 = results.next_page() + self.assertLessEqual(2, len(results)) + self.assertEqual([('val3', o3.key), ('val4', o4.key)], page2) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_pagination_stream(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # ========= Above steps work for streaming ========== + stream = bucket.stream_index('field1_bin', 'val0', 'val5', + max_results=2) + results = [] + for result in stream: + results.extend(result) + + # Number of results =< page size + self.assertLessEqual(2, len(results)) + # Results are in-order + self.assertEqual([o1.key, o2.key], results) + + # Continuation/next page present when page size smaller than + # total results size + self.assertIsNotNone(stream.continuation) + self.assertTrue(stream.has_next_page()) + + # Retrieving next page gets more results + results = [] + for result in stream.next_page(): + results.extend(result) + self.assertLessEqual(2, len(results)) + self.assertEqual([o3.key, o4.key], results) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_pagination_stream_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # ========= Above steps work for streaming with return-terms ========== + + stream = bucket.stream_index('field1_bin', 'val0', 'val5', + max_results=2, return_terms=True) + results = [] + for result in stream: + results.extend(result) + + # Number of results =< page size + self.assertLessEqual(2, len(results)) + # Results are in-order + self.assertEqual([('val1', o1.key), ('val2', o2.key)], results) + + # Continuation/next page present when page size smaller than + # total results size + self.assertIsNotNone(stream.continuation) + self.assertTrue(stream.has_next_page()) + + # Retrieving next page gets more results + results = [] + for result in stream.next_page(): + results.extend(result) + self.assertLessEqual(2, len(results)) + self.assertEqual([('val3', o3.key), ('val4', o4.key)], results) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_eq_query_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + results = bucket.get_index('field2_int', 1001, return_terms=True) + self.assertEqual([(1001, o1.key)], results) + def _create_index_objects(self): """ Creates a number of index objects to be used in 2i test diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 949d64d0..d442cd3e 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -22,6 +22,7 @@ from cgi import parse_header from email import message_from_string from riak.util import decode_index_value +from riak.client.index_page import CONTINUATION class RiakHttpStream(object): @@ -145,9 +146,8 @@ def next(self): structs = payload[u'results'] # Format is {"results":[{"2ikey":"primarykey"}, ...]} return [self._decode_pair(d.items()[0]) for d in structs] - else: - # WAT - self.next() + elif u'continuation' in payload: + return CONTINUATION(payload[u'continuation']) def _decode_pair(self, pair): return (decode_index_value(self.index, pair[0]), pair[1]) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 76cc3a96..ffc70d5e 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -278,26 +278,31 @@ def stream_mapred(self, inputs, query, timeout=None): (repr(headers), repr(response.read()))) def get_index(self, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, continuation=None): """ Performs a secondary index query. """ - params = {'return_terms': return_terms} + params = {'return_terms': return_terms, 'max_results': max_results, + 'continuation': continuation} url = self.index_path(bucket, index, startkey, endkey, **params) status, headers, body = self._request('GET', url) self.check_http_code(status, [200]) json_data = json.loads(body) - if return_terms: + if return_terms and u'results' in json_data: results = [] for result in json_data[u'results'][:]: term, key = result.items()[0] results.append((decode_index_value(index, term), key),) - return results else: - return json_data[u'keys'][:] + results = json_data[u'keys'][:] + + if max_results and u'continuation' in json_data: + return (results, json_data[u'continuation']) + else: + return (results, None) def stream_index(self, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, continuation=None): """ Streams a secondary index query. """ @@ -305,7 +310,8 @@ def stream_index(self, bucket, index, startkey, endkey=None, raise NotImplementedError("Secondary index streaming is not " "supported") - params = {'return_terms': return_terms, 'stream': True} + params = {'return_terms': return_terms, 'stream': True, + 'max_results': max_results, 'continuation': continuation} url = self.index_path(bucket, index, startkey, endkey, **params) status, headers, response = self._request('GET', url, stream=True) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index ddf61a98..9a849665 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -375,7 +375,8 @@ def _encode_hook(self, hook, msg): return msg def _encode_index_req(self, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, + continuation=None): """ Encodes a secondary index request into the protobuf message. @@ -389,6 +390,11 @@ def _encode_index_req(self, bucket, index, startkey, endkey=None, :type endkey: integer, string :param return_terms: whether to return the index term with the key :type return_terms: bool + :param max_results: the maximum number of results to return (page size) + :type max_results: integer + :param continuation: the opaque continuation returned from a + previous paginated request + :type continuation: string :rtype riak_pb.RpbIndexReq """ req = riak_pb.RpbIndexReq(bucket=bucket, index=index) @@ -401,4 +407,8 @@ def _encode_index_req(self, bucket, index, startkey, endkey=None, req.key = str(startkey) if return_terms is not None: req.return_terms = return_terms + if max_results: + req.max_results = max_results + if continuation: + req.continuation = continuation return req diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index ff8345e6..b46b8227 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -23,6 +23,8 @@ MSG_CODE_MAPRED_RESP, MSG_CODE_INDEX_RESP ) +from riak.util import decode_index_value +from riak.client.index_page import CONTINUATION class RiakPbcStream(object): @@ -115,16 +117,15 @@ def __init__(self, transport, index, return_terms=False): def next(self): response = super(RiakPbcIndexStream, self).next() - if response.done and not (response.keys or response.results): + if response.done and not (response.keys or + response.results or + response.continuation): raise StopIteration if self.return_terms and response.results: - return [(self._coerce(r.key), r.value) for r in response.results] + return [(decode_index_value(self.index, r.key), r.value) + for r in response.results] elif response.keys: return response.keys - - def _coerce(self, index_value): - if "_int" in self.index: - return long(index_value) - else: - return str(index_value) + elif response.continuation: + return CONTINUATION(response.continuation) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 1ef91560..3ce914fe 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -336,29 +336,35 @@ def stream_mapred(self, inputs, query, timeout=None): return RiakPbcMapredStream(self) def get_index(self, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, continuation=None): if not self.pb_indexes(): return self._get_index_mapred_emu(bucket, index, startkey, endkey) req = self._encode_index_req(bucket, index, startkey, endkey, - return_terms=return_terms) + return_terms, max_results, continuation) msg_code, resp = self._request(MSG_CODE_INDEX_REQ, req, MSG_CODE_INDEX_RESP) - if return_terms: - return [(decode_index_value(index, pair.key), pair.value) - for pair in resp.results] + + if return_terms and resp.results: + results = [(decode_index_value(index, pair.key), pair.value) + for pair in resp.results] + else: + results = resp.keys + + if max_results: + return (results, resp.continuation) else: - return resp.keys + return (results, None) def stream_index(self, bucket, index, startkey, endkey=None, - return_terms=None): + return_terms=None, max_results=None, continuation=None): if not self.stream_indexes(): raise NotImplementedError("Secondary index streaming is not " "supported") req = self._encode_index_req(bucket, index, startkey, endkey, - return_terms=return_terms) + return_terms, max_results, continuation) req.stream = True self._send_msg(MSG_CODE_INDEX_REQ, req) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index d373152e..64605fa8 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -161,13 +161,15 @@ def search(self, index, query, **params): """ raise NotImplementedError - def get_index(self, bucket, index, startkey, endkey=None): + def get_index(self, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, continuation=None): """ Performs a secondary index query. """ raise NotImplementedError - def stream_index(self, bucket, index, startkey, endkey=None): + def stream_index(self, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, continuation=None): """ Streams a secondary index query. """ From 45d266fc54633259d6204376417ba0c2bf40d31b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 3 Jul 2013 16:22:06 -0500 Subject: [PATCH 067/672] Add pagination to secondary indexes, commit the second. This refactors a bit of the IndexPage class so as to support the equality/return-terms issue left dangling in the last commit. Along the way, I added a test for streaming with the eq/rt combo, and discovered an issue where the PBC interface returned the `keys` field from the response directly when it was expected to be a list. This wouldn't normally be an issue, but was necessitated by the eq/rt issue. This completes the work for #252. --- riak/client/index_page.py | 81 +++++++++++++++++++++++++------- riak/client/operations.py | 6 --- riak/tests/test_2i.py | 13 +++++ riak/transports/pbc/stream.py | 2 +- riak/transports/pbc/transport.py | 2 +- 5 files changed, 80 insertions(+), 24 deletions(-) diff --git a/riak/client/index_page.py b/riak/client/index_page.py index 9a8725cc..11416c18 100644 --- a/riak/client/index_page.py +++ b/riak/client/index_page.py @@ -47,41 +47,60 @@ def __init__(self, client, bucket, index, startkey, endkey, return_terms, self.stream = False def __iter__(self): - if self.results: - try: - for result in self.results: - if self.stream and isinstance(result, CONTINUATION): - self.continuation = result.c - else: - yield result - finally: - if self.stream: - self.results.close() - else: + """ + Emulates the iterator interface. When streaming, this means + delegating to the stream, otherwise iterating over the + existing result set. + """ + if self.results is None: raise ValueError("No index results to iterate") + try: + for result in self.results: + if self.stream and isinstance(result, CONTINUATION): + self.continuation = result.c + else: + yield self._inject_term(result) + finally: + if self.stream: + self.results.close() + def __len__(self): - if not self.stream and self.results is not None: + """ + Returns the length of the captured results. + """ + if self._has_results(): return len(self.results) else: raise ValueError("Streamed index page has no length") def __getitem__(self, index): - if not self.stream and self.results is not None: + """ + Fetches an item by index from the captured results. + """ + if self._has_results(): return self.results[index] else: raise ValueError("Streamed index page has no entries") def __eq__(self, other): - if isinstance(other, list) and not (self.stream or - self.results is None): - return self.results == other + """ + An IndexPage can pretend to be equal to a list when it has + captured results by simply comparing the internal results to + the passed list. Otherwise the other object needs to be an + equivalent IndexPage. + """ + if isinstance(other, list) and self._has_results(): + return self._inject_term(self.results) == other elif isinstance(other, IndexPage): return other.__dict__ == self.__dict__ else: return False def __ne__(self, other): + """ + Converse of __eq__. + """ return not self.__eq__(other) def has_next_page(self): @@ -120,3 +139,33 @@ def next_page(self, stream=None): return self.client.stream_index(**args) else: return self.client.get_index(**args) + + def _has_results(self): + """ + When not streaming, have results been assigned? + """ + return not (self.stream or self.results is None) + + def _should_inject_term(self, term): + """ + The index term should be injected when using an equality query + and the return terms option. If the term is already a tuple, + it can be skipped. + """ + return self.return_terms and not self.endkey + + def _inject_term(self, result): + """ + Upgrades a result (streamed or not) to include the index term + when an equality query is used with return_terms. + """ + if self._should_inject_term(result): + if type(result) is list: + return [ (self.startkey, r) for r in result ] + else: + return (self.startkey, result) + else: + return result + + def __repr__(self): + return "<{!s} {!r}>".format(self.__class__.__name__, self.__dict__) diff --git a/riak/client/operations.py b/riak/client/operations.py index a49b24e1..88b70df6 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -73,9 +73,6 @@ def get_index(self, transport, bucket, index, startkey, endkey=None, :type continuation: string :rtype: :class:`riak.client.index_page.IndexPage` """ - if return_terms and endkey is None: - raise ValueError("Cannot use return_terms with an equality query") - page = IndexPage(self, bucket, index, startkey, endkey, return_terms, max_results) @@ -110,9 +107,6 @@ def stream_index(self, bucket, index, startkey, endkey=None, :type continuation: string :rtype: :class:`riak.client.index_page.IndexPage` """ - if return_terms and endkey is None: - raise ValueError("Cannot use return_terms with an equality query") - page = IndexPage(self, bucket, index, startkey, endkey, return_terms, max_results) with self._transport() as transport: diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index aada2192..9a83eb49 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -398,6 +398,19 @@ def test_index_eq_query_return_terms(self): results = bucket.get_index('field2_int', 1001, return_terms=True) self.assertEqual([(1001, o1.key)], results) + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + def test_index_eq_query_stream_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + results = [] + for item in bucket.stream_index('field2_int', 1001, return_terms=True): + results.extend(item) + + self.assertEqual([(1001, o1.key)], results) + def _create_index_objects(self): """ Creates a number of index objects to be used in 2i test diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index b46b8227..fd9766ba 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -126,6 +126,6 @@ def next(self): return [(decode_index_value(self.index, r.key), r.value) for r in response.results] elif response.keys: - return response.keys + return response.keys[:] elif response.continuation: return CONTINUATION(response.continuation) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 3ce914fe..7eb5e51c 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -350,7 +350,7 @@ def get_index(self, bucket, index, startkey, endkey=None, results = [(decode_index_value(index, pair.key), pair.value) for pair in resp.results] else: - results = resp.keys + results = resp.keys[:] if max_results: return (results, resp.continuation) From 7c1a7c4aca9e9ea2d5453031747e7855b04f10a9 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 23 Jul 2013 14:34:42 -0500 Subject: [PATCH 068/672] Fix some pep8, pyflakes and merge bugs. --- riak/client/index_page.py | 2 +- riak/client/multiget.py | 4 ++-- riak/tests/test_all.py | 29 +++++++++++++++-------------- riak/transports/pbc/transport.py | 3 ++- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/riak/client/index_page.py b/riak/client/index_page.py index 11416c18..5314f4db 100644 --- a/riak/client/index_page.py +++ b/riak/client/index_page.py @@ -161,7 +161,7 @@ def _inject_term(self, result): """ if self._should_inject_term(result): if type(result) is list: - return [ (self.startkey, r) for r in result ] + return [(self.startkey, r) for r in result] else: return (self.startkey, result) else: diff --git a/riak/client/multiget.py b/riak/client/multiget.py index 9995133b..665eba77 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -158,7 +158,7 @@ def multiget(client, keys, **options): for _ in range(len(keys)): if RIAK_MULTIGET_POOL.stopped(): raise RuntimeError("Multi-get operation interrupted by pool " - "stopping!") + "stopping!") results.append(outq.get()) outq.task_done() @@ -169,7 +169,7 @@ def multiget(client, keys, **options): from riak import RiakClient import riak.benchmark as benchmark client = RiakClient(protocol='pbc') - bkeys = [ ('multiget', str(key)) for key in xrange(10000) ] + bkeys = [('multiget', str(key)) for key in xrange(10000)] data = open(__file__).read() diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 66cf5823..b8220189 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -136,20 +136,6 @@ def test_timeout_validation(self): for bad in [0, -1, False, "foo"]: with self.assertRaises(ValueError): self.client.get_buckets(timeout=bad) - def test_multiget_bucket(self): - """ - Multiget operations can be invoked on buckets. - """ - keys = [self.key_name, self.randname(), self.randname()] - for key in keys: - self.client.bucket(self.bucket_name)\ - .new(key, encoded_data=key, content_type="text/plain")\ - .store() - results = self.client.bucket(self.bucket_name).multiget(keys) - for obj in results: - self.assertIsInstance(obj, RiakObject) - self.assertTrue(obj.exists) - self.assertEqual(obj.key, obj.encoded_data) with self.assertRaises(ValueError): for i in self.client.stream_buckets(timeout=bad): @@ -178,6 +164,20 @@ def test_multiget_bucket(self): for i in self.client.stream_mapred([], [], bad): pass + def test_multiget_bucket(self): + """ + Multiget operations can be invoked on buckets. + """ + keys = [self.key_name, self.randname(), self.randname()] + for key in keys: + self.client.bucket(self.bucket_name)\ + .new(key, encoded_data=key, content_type="text/plain")\ + .store() + results = self.client.bucket(self.bucket_name).multiget(keys) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertTrue(obj.exists) + self.assertEqual(obj.key, obj.encoded_data) def test_multiget_errors(self): """ @@ -204,6 +204,7 @@ def test_multiget_notfounds(self): self.assertIsInstance(obj, RiakObject) self.assertFalse(obj.exists) + class RiakPbcTransportTestCase(BasicKVTests, KVFileTests, BucketPropsTest, diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 35468d70..9a760bbe 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -25,7 +25,8 @@ from riak.riak_object import VClock from riak.util import decode_index_value from connection import RiakPbcConnection -from stream import RiakPbcKeyStream, RiakPbcMapredStream, RiakPbcBucketStream, RiakPbcIndexStream +from stream import (RiakPbcKeyStream, RiakPbcMapredStream, RiakPbcBucketStream, + RiakPbcIndexStream) from codec import RiakPbcCodec from messages import ( From cf2ecf40d620974123f69f87e65c0b93e058392c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 23 Jul 2013 14:59:39 -0500 Subject: [PATCH 069/672] Update version to 2.0.0 in docs. --- docs/conf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d8e2e07d..c8bba0b5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,16 +41,16 @@ # General information about the project. project = u'Riak Python Client' -copyright = u'2010-2012, Basho Technologies' +copyright = u'2010-2013, Basho Technologies' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '1.4.1' +version = '2.0.0' # The full version, including alpha/beta/rc tags. -release = '1.4.1' +release = '2.0.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 461fc55cc5440873ce828215220083a6f0329e1f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 23 Jul 2013 15:00:05 -0500 Subject: [PATCH 070/672] Add autodoc for RiakContent class. --- docs/content.rst | 9 +++++++++ docs/index.rst | 1 + 2 files changed, 10 insertions(+) create mode 100644 docs/content.rst diff --git a/docs/content.rst b/docs/content.rst new file mode 100644 index 00000000..04880600 --- /dev/null +++ b/docs/content.rst @@ -0,0 +1,9 @@ +.. ref-content: + +=========== +RiakContent +=========== + +.. currentmodule:: riak.content + +.. autoclass:: riak.content.RiakContent diff --git a/docs/index.rst b/docs/index.rst index f8261594..d5b2e16a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,6 +26,7 @@ Contents: client bucket riak_object + content mapreduce Indices and tables From 01f6ef86cf5d7c49602bae03255423c6d6404e39 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 23 Jul 2013 15:00:32 -0500 Subject: [PATCH 071/672] Fix some typos in RiakBucket docs. --- docs/bucket.rst | 8 ++++---- riak/bucket.py | 15 ++++++--------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/bucket.rst b/docs/bucket.rst index 297d2b46..aceaa730 100644 --- a/docs/bucket.rst +++ b/docs/bucket.rst @@ -1,9 +1,9 @@ .. ref-bucket: -========== -RiakBucket -========== +=========================== +Bucket Objects (RiakBucket) +=========================== .. currentmodule:: riak.bucket -.. autoclass:: riak.bucket.RiakBucket +.. autoclass:: RiakBucket diff --git a/riak/bucket.py b/riak/bucket.py index 98690409..7770f3e6 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -224,7 +224,7 @@ def multiget(self, keys, r=None, pr=None): :type r: integer :param pr: PR-Value for the requests (defaults to bucket's PR) :type pr: integer - :rtype list of :class:`RiakObject ` + :rtype: list of :class:`RiakObject ` """ bkeys = [(self.name, key) for key in keys] return self._client.multiget(bkeys, r=r, pr=pr) @@ -253,13 +253,10 @@ def _set_resolver(self, value): N-value for this bucket, which is the number of replicas that will be written of each object in the bucket. - .. warning:: - - Set this once before you write any data to the bucket, and never - change it again, otherwise unpredictable things could happen. - This should only be used if you know what you are doing. - - :type nval: integer + .. warning:: Set this once before you write any data to the + bucket, and never change it again, otherwise unpredictable + things could happen. This should only be used if you know what + you are doing. """) allow_mult = bucket_property('allow_mult', doc=""" @@ -461,7 +458,7 @@ def get_counter(self, key, **kwargs): :param key: the key of the counter :type key: string - :rtype int + :rtype: int """ return self._client.get_counter(self, key, **kwargs) From 26ccf8f8047a00197e18d45697aff99b8b73117a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 08:59:06 -0500 Subject: [PATCH 072/672] Rewrite client docs page, breaking into sections. * Changes default autodoc options. * Improve documentation of some methods, adjusting the signature as necessary. * Ensure the retryable decorator maintains the docstring. --- docs/client.rst | 149 ++++++++++++++++++++++++++++++++++++-- docs/conf.py | 4 +- docs/index.rst | 2 +- riak/client/__init__.py | 44 ++++++++--- riak/client/operations.py | 110 +++++++++++++++++++++++++--- riak/client/transport.py | 3 + 6 files changed, 281 insertions(+), 31 deletions(-) diff --git a/docs/client.rst b/docs/client.rst index dff87c9f..ebeb664e 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -1,9 +1,148 @@ -.. ref-client: +==================== +Client & Connections +==================== -========== -RiakClient -========== +-------- +Overview +-------- + +To connect to a Riak cluster, you must create a +:py:class:`~riak.client.RiakClient` object. The default configuration +connects to a single Riak node on ``localhost`` with the default +ports. The below instantiation statements are all equivalent:: + + from riak import RiakClient, RiakNode + + RiakClient() + RiakClient(protocol='http', host='127.0.0.1', http_port=8098) + RiakClient(nodes=[{'host':'127.0.0.1','http_port':8098}]) + RiakClient(protocol='http', nodes=[RiakNode()]) + + +.. note:: Connections are not established until you attempt to perform + an operation. If the host or port are incorrect, you will not get + an error raised immediately. + +The client maintains a connection pool behind the scenes, one for each +protocol. Connections are opened as-needed; a random node is selected +when a new connection is requested. + +-------------- +RiakClient API +-------------- .. currentmodule:: riak.client +.. autoclass:: RiakClient + + .. autoattribute:: PROTOCOLS + .. autoattribute:: protocol + .. autoattribute:: client_id + .. attribute:: nodes + + The list of :class:`nodes ` that this + client will connect to. It is best not to modify this property + directly, as it is not thread-safe. + + .. attribute:: RETRY_COUNT + + The maximum number of times to retry requests where it is + permitted, default is 3. Retries will attempt to select nodes + with better error rates, excluding nodes where the request + failed. + +^^^^^ +Nodes +^^^^^ + +The ``nodes`` attribute of ``RiakClient`` objects is a list of +``RiakNode`` objects. If you include multiple host specifications in +the ``RiakClient`` constructor, they will be turned into this type. + +.. autoclass:: riak.node.RiakNode + :members: + +^^^^^^^^^^^^^^^^^^^^^^^ +Client-level Operations +^^^^^^^^^^^^^^^^^^^^^^^ + +Some operations are not scoped by buckets and can be performed on the +client directly: + +.. automethod:: RiakClient.ping +.. automethod:: RiakClient.get_buckets +.. automethod:: RiakClient.stream_buckets + +^^^^^^^^^^^^^^^^^ +Accessing Buckets +^^^^^^^^^^^^^^^^^ + +Most client operations are on :py:class:`bucket objects +` or keys within those buckets. Use the +``bucket`` method for creating buckets that will proxy operations to +the called client. + +.. automethod:: RiakClient.bucket + +^^^^^^^^^^^^^^^^^^^^^^^ +Bucket-level Operations +^^^^^^^^^^^^^^^^^^^^^^^ + +.. automethod:: RiakClient.get_bucket_props +.. automethod:: RiakClient.set_bucket_props +.. automethod:: RiakClient.clear_bucket_props +.. automethod:: RiakClient.get_keys +.. automethod:: RiakClient.stream_keys + +^^^^^^^^^^^^^^^^^^^^ +Key-level Operations +^^^^^^^^^^^^^^^^^^^^ + +.. automethod:: RiakClient.get +.. automethod:: RiakClient.put +.. automethod:: RiakClient.delete +.. automethod:: RiakClient.multiget +.. automethod:: RiakClient.get_counter +.. automethod:: RiakClient.update_counter + +^^^^^^^^^^^^^^^^ +Query Operations +^^^^^^^^^^^^^^^^ + +.. automethod:: RiakClient.mapred +.. automethod:: RiakClient.stream_mapred +.. automethod:: RiakClient.get_index +.. automethod:: RiakClient.stream_index +.. automethod:: RiakClient.fulltext_search +.. automethod:: RiakClient.fulltext_add +.. automethod:: RiakClient.fulltext_delete + +^^^^^^^^^^^^^ +Serialization +^^^^^^^^^^^^^ + +The client supports automatic transformation of Riak responses into +Python types if encoders and decoders are registered for the +media-types. Supported by default are ``application/json`` and +``text/plain``. + +.. autofunction:: default_encoder +.. automethod:: RiakClient.get_encoder +.. automethod:: RiakClient.set_encoder +.. automethod:: RiakClient.get_decoder +.. automethod:: RiakClient.set_decoder + + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Deprecated Methods and Properties +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. warning:: These methods exist solely for backwards-compatibility and should not + be used unless code is being ported from an older version. + +.. automethod:: RiakClient.get_transport +.. automethod:: RiakClient.get_client_id +.. automethod:: RiakClient.set_client_id +.. attribute:: RiakClient.solr -.. autoclass:: riak.client.RiakClient + Returns a RiakSearch object which can access search indexes. + DEPRECATED diff --git a/docs/conf.py b/docs/conf.py index c8bba0b5..bbc2064f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -256,6 +256,6 @@ #epub_tocdup = True # Autodoc settings -autodoc_default_flags = ['members', 'undoc-members'] -autodoc_member_order = 'bysource' +autodoc_default_flags = ['no-undoc-members'] +autodoc_member_order = 'groupwise' autoclass_content = 'both' diff --git a/docs/index.rst b/docs/index.rst index d5b2e16a..7ca823a0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,7 +14,7 @@ Installation .. _Pip: http://pip.openplans.org/ .. _easy_install: http://pypi.python.org/pypi/setuptools -.. _PyPI: http://pypi.python.org/pypi/riak/1.4.0 +.. _PyPI: http://pypi.python.org/pypi/riak/ Contents: diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 503109de..c3de41b8 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -55,6 +55,7 @@ class RiakClient(RiakMapReduceChain, RiakClientOperations): or by using the methods on related objects. """ + #: The supported protocols PROTOCOLS = ['http', 'https', 'pbc'] def __init__(self, protocol='http', transport_options={}, @@ -120,16 +121,18 @@ def _set_protocol(self, value): protocol = property(_get_protocol, _set_protocol, doc= """ - Which protocol to prefer, one of PROTOCOLS. - Please note that when one protocol is - selected, the other protocols MAY NOT attempt - to connect. Changing to another protocol will - cause a connection on the next request. - - Some requests are only valid over 'http' or - 'https', and will always be sent via those - transports, regardless of which protocol is - preferred. + Which protocol to prefer, one of + :attr:`PROTOCOLS + `. Please + note that when one protocol is selected, the + other protocols MAY NOT attempt to connect. + Changing to another protocol will cause a + connection on the next request. + + Some requests are only valid over ``'http'`` + or ``'https'``, and will always be sent via + those transports, regardless of which protocol + is preferred. """) def get_transport(self): @@ -181,6 +184,10 @@ def _set_client_id(self, client_id): def get_encoder(self, content_type): """ Get the encoding function for the provided content type. + + :param content_type: the requested media type + :type content_type: str + :rtype: function """ return self._encoders.get(content_type) @@ -188,7 +195,10 @@ def set_encoder(self, content_type, encoder): """ Set the encoding function for the provided content type. - :param encoder: + :param content_type: the requested media type + :type content_type: str + :param encoder: an encoding function, takes a single object + argument and returns a string :type encoder: function """ self._encoders[content_type] = encoder @@ -196,6 +206,10 @@ def set_encoder(self, content_type, encoder): def get_decoder(self, content_type): """ Get the decoding function for the provided content type. + + :param content_type: the requested media type + :type content_type: str + :rtype: function """ return self._decoders.get(content_type) @@ -203,7 +217,10 @@ def set_decoder(self, content_type, decoder): """ Set the decoding function for the provided content type. - :param decoder: + :param content_type: the requested media type + :type content_type: str + :param decoder: a decoding function, takes a string and + returns a Python type :type decoder: function """ self._decoders[content_type] = decoder @@ -226,7 +243,10 @@ def bucket(self, name): def solr(self): """ Returns a RiakSearch object which can access search indexes. + DEPRECATED """ + deprecated("``solr`` is deprecated, use ``fulltext_search``," + " ``fulltext_add`` and ``fulltext_delete`` directly") return RiakSearch(self) def _create_node(self, n): diff --git a/riak/client/operations.py b/riak/client/operations.py index a1906e01..22fc49b3 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -34,13 +34,20 @@ class RiakClientOperations(RiakClientTransport): @retryable def get_buckets(self, transport, timeout=None): """ - Get the list of buckets as RiakBucket instances. - NOTE: Do not use this in production, as it requires traversing through - all keys stored in a cluster. + get_buckets(timeout=None) + + Get the list of buckets as :class:`RiakBucket + ` instances. + + .. warning:: Do not use this in production, as it requires + traversing through all keys stored in a cluster. + + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. :param timeout: a timeout value in milliseconds :type timeout: int - :rtype list of RiakBucket instances + :rtype: list of :class:`RiakBucket ` instances """ _validate_timeout(timeout) return [self.bucket(name) for name in @@ -49,13 +56,15 @@ def get_buckets(self, transport, timeout=None): def stream_buckets(self, timeout=None): """ Streams the list of buckets. This is a generator method that - should be iterated over. NOTE: Do not use this in production, - as it requires traversing through all keys stored in a - cluster. + should be iterated over. + + .. warning:: Do not use this in production, as it requires + traversing through all keys stored in a cluster. :param timeout: a timeout value in milliseconds :type timeout: int - :rtype iterator + :rtype: iterator that yields lists of :class:`RiakBucket + ` instances """ _validate_timeout(timeout) with self._transport() as transport: @@ -71,8 +80,13 @@ def stream_buckets(self, timeout=None): @retryable def ping(self, transport): """ + ping() + Check if the Riak server for this ``RiakClient`` instance is alive. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :rtype: boolean """ return transport.ping() @@ -83,8 +97,14 @@ def ping(self, transport): def get_index(self, transport, bucket, index, startkey, endkey=None, return_terms=None, max_results=None, continuation=None): """ + get_index(bucket, index, startkey, endkey=None, return_terms=None,\ + max_results=None, continuation=None) + Queries a secondary index, returning matching keys. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param bucket: the bucket whose index will be queried :type bucket: RiakBucket :param index: the index to query @@ -148,8 +168,13 @@ def stream_index(self, bucket, index, startkey, endkey=None, @retryable def get_bucket_props(self, transport, bucket): """ + get_bucket_props(bucket) + Fetches bucket properties for the given bucket. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param bucket: the bucket whose properties will be fetched :type bucket: RiakBucket :rtype: dict @@ -159,8 +184,13 @@ def get_bucket_props(self, transport, bucket): @retryable def set_bucket_props(self, transport, bucket, props): """ + set_bucket_props(bucket, props) + Sets bucket properties for the given bucket. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param bucket: the bucket whose properties will be set :type bucket: RiakBucket :param props: the properties to set @@ -171,8 +201,13 @@ def set_bucket_props(self, transport, bucket, props): @retryable def clear_bucket_props(self, transport, bucket): """ + clear_bucket_props(bucket) + Resets bucket properties for the given bucket. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param bucket: the bucket whose properties will be set :type bucket: RiakBucket """ @@ -181,8 +216,13 @@ def clear_bucket_props(self, transport, bucket): @retryable def get_keys(self, transport, bucket, timeout=None): """ + get_keys(bucket, timeout=None) + Lists all keys in a bucket. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param bucket: the bucket whose properties will be set :type bucket: RiakBucket :param timeout: a timeout value in milliseconds @@ -197,7 +237,6 @@ def stream_keys(self, bucket, timeout=None): Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. - :param bucket: the bucket whose properties will be set :type bucket: RiakBucket :param timeout: a timeout value in milliseconds @@ -218,8 +257,14 @@ def stream_keys(self, bucket, timeout=None): def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=None, timeout=None): """ + put(robj, w=None, dw=None, pw=None, return_body=None,\ + if_none_match=None, timeout=None) + Stores an object in the Riak cluster. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param robj: the object to store :type robj: RiakObject :param w: the write quorum @@ -246,8 +291,13 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, @retryable def get(self, transport, robj, r=None, pr=None, timeout=None): """ + get(robj, r=None, pr=None, timeout=None) + Fetches the contents of a Riak object. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param robj: the object to fetch :type robj: RiakObject :param r: the read quorum @@ -268,8 +318,14 @@ def get(self, transport, robj, r=None, pr=None, timeout=None): def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None, timeout=None): """ + delete(robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None,\ + timeout=None) + Deletes an object from Riak. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param robj: the object to store :type robj: RiakObject :param rw: the read/write (delete) quorum @@ -294,8 +350,13 @@ def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, @retryable def mapred(self, transport, inputs, query, timeout): """ + mapred(inputs, query, timeout) + Executes a MapReduce query. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param inputs: the input list/structure :type inputs: list, dict :param query: the list of query phases @@ -332,8 +393,13 @@ def stream_mapred(self, inputs, query, timeout): @retryable def fulltext_search(self, transport, index, query, **params): """ + fulltext_search(index, query, **params) + Performs a full-text search query. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param index: the bucket/index to search over :type index: string :param query: the search query @@ -346,8 +412,14 @@ def fulltext_search(self, transport, index, query, **params): @retryableHttpOnly def fulltext_add(self, transport, index, docs): """ + fulltext_add(index, docs) + Adds documents to the full-text index. + .. note:: This request is automatically retried + :attr:`RETRY_COUNT` times if it fails due to network error. + Only HTTP will be used for this request. + :param index: the bucket/index in which to index these docs :type index: string :param docs: the list of documents @@ -358,8 +430,14 @@ def fulltext_add(self, transport, index, docs): @retryableHttpOnly def fulltext_delete(self, transport, index, docs=None, queries=None): """ + fulltext_delete(index, docs=None, queries=None) + Removes documents from the full-text index. + .. note:: This request is automatically retried + :attr:`RETRY_COUNT` times if it fails due to network error. + Only HTTP will be used for this request. + :param index: the bucket/index from which to delete :type index: string :param docs: a list of documents (with ids) @@ -377,7 +455,8 @@ def multiget(self, pairs, **params): :type pairs: list :param params: additional request flags, e.g. r, pr :type params: dict - :rtype list + :rtype: list of :class:`RiakObject ` + instances """ return multiget(self, pairs, **params) @@ -385,8 +464,14 @@ def multiget(self, pairs, **params): def get_counter(self, transport, bucket, key, r=None, pr=None, basic_quorum=None, notfound_ok=None): """ + get_counter(bucket, key, r=None, pr=None, basic_quorum=None,\ + notfound_ok=None) + Gets the value of a counter. + .. note:: This request is automatically retried :attr:`RETRY_COUNT` + times if it fails due to network error. + :param bucket: the bucket of the counter :type bucket: RiakBucket :param key: the key of the counter @@ -400,13 +485,16 @@ def get_counter(self, transport, bucket, key, r=None, pr=None, :type basic_quorum: bool :param notfound_ok: whether to treat not-found responses as successful :type notfound_ok: bool - :rtype integer + :rtype: integer """ return transport.get_counter(bucket, key, r=r, pr=pr) def update_counter(self, bucket, key, value, w=None, dw=None, pw=None, returnvalue=False): """ + update_counter(bucket, key, value, w=None, dw=None, pw=None,\ + returnvalue=False) + Updates a counter by the given value. This operation is not idempotent and so should not be retried automatically. diff --git a/riak/client/transport.py b/riak/client/transport.py index 38f4b272..eb2b7481 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -124,6 +124,9 @@ def thunk(transport): return self._with_retries(pool, thunk) + wrapper.__doc__ = fn.__doc__ + wrapper.__repr__ = fn.__repr__ + return wrapper From 1353f1231cfbc3bdcf7c8eed8f292919e625d987 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 10:11:00 -0500 Subject: [PATCH 073/672] Remove the tutorial. --- docs/index.rst | 2 - docs/tutorial.rst | 424 ---------------------------------------------- 2 files changed, 426 deletions(-) delete mode 100644 docs/tutorial.rst diff --git a/docs/index.rst b/docs/index.rst index 7ca823a0..9b1ed90a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,8 +21,6 @@ Contents: .. toctree:: :maxdepth: 2 - tutorial - client bucket riak_object diff --git a/docs/tutorial.rst b/docs/tutorial.rst deleted file mode 100644 index 2506d114..00000000 --- a/docs/tutorial.rst +++ /dev/null @@ -1,424 +0,0 @@ -.. ref-tutorial: - -======== -Tutorial -======== - -This tutorial assumes basic working knowledge of how Riak works & what it can -do. If you need a more comprehensive overview how to use Riak, please check out -the `Riak Fast Track`_. - -.. _`Riak Fast Track`: http://wiki.basho.com/The-Riak-Fast-Track.html - - -Quick Start -=========== - -For the impatient, simple usage of the official Python binding for Riak looks -like:: - - import riak - - # Connect to Riak. - client = riak.RiakClient() - - # Choose the bucket to store data in. - bucket = client.bucket('test') - - - # Supply a key to store data under. - # The ``data`` can be any data Python's ``json`` encoder can handle. - person = bucket.new('riak_developer_1', data={ - 'name': 'John Smith', - 'age': 28, - 'company': 'Mr. Startup!', - }) - # Save the object to Riak. - person.store() - - -Connecting To Riak -================== - -There are two supported ways to connect to Riak, the HTTP interface & the -`Protocol Buffers`_ interface. Both provide the same API & full access to -Riak. - -The HTTP interface is easier to setup & is well suited for development use. It -is the slower of the two interfaces, but if you are only making a handful of -requests, it is more than capable. - -The Protocol Buffers (also called ``protobuf``) is more difficult to setup but -is significantly faster (2-3x) and is more suitable for production use. This -interface is better suited to a higher number of requests. - -.. _`Protocol Buffers`: http://code.google.com/p/protobuf/ - -To use the HTTP interface and connecting to a local Riak on the default port, -no arguments are needed:: - - import riak - - client = riak.RiakClient() - -The constructor also configuration options such as ``host``, ``http_port``, -``pb_port`` & ``prefix``. Please refer to the :doc:`client` documentation -for full details. - -To use the Protocol Buffers interface:: - - import riak - - client = riak.RiakClient(pb_port=8087, protocol='pbc') - -.. warning: - - Riak's default port is 8098. However, when using the Protocol Buffers, the - Riak listens on port 8087. If you forget this, you will *NOT* get an - immediate error, but will instead receive an error when fetching or storing - data to the effect of ``RiakError: 'Socket returned short read 135 - - expected 8192'``. - -The ``protocol`` argument indicates to the client which backend to use. -We didn't need to specify it in the HTTP example because ``http`` is the -default class. Available options are: ``http``, ``https``, & ``pbc``. - - -Using Buckets -============= - -Buckets in Riak's terminology are segmented keyspaces. They are a way to -categorize different types of data and are roughly analogous to tables in an -RDBMS. - -Once you have a ``client``, selecting a bucket is simple. Provide a string of -the name of the bucket to use:: - - test_bucket = client.bucket('test') - -If the bucket does not exist, Riak will create it for you. You can also open -as many buckets as you need:: - - user_bucket = client.bucket('user') - profile_bucket = client.bucket('profile') - status_bucket = client.bucket('status') - -If needed, you can also manually instantiate a bucket like so:: - - user_bucket = riak.RiakBucket(client, 'user') - -The buckets themselves provide many different methods. The most commonly used -are: - -* ``get`` - Fetches a key's value (decoded from JSON). -* ``get_binary`` - Also fetches a key's raw value (plain text or binary). -* ``new`` - Creates a new key/value pair (encoded in JSON). -* ``new_binary`` - Creates a new key/raw value pair. - -See the full :doc:`bucket` documentation for the other methods. - - -Storing Keys/Values -=================== - -Once you've got a working client/bucket, the next task at hand is storing data. -Riak provides several ways to store your data, but the most common are a -JSON-encoded structure or a binary blob. - -To store JSON-encoded data, you'd do something like the following:: - - import riak - - client = riak.RiakClient() - user_bucket = client.bucket('user') - - # We're creating the user data & keying off their username. - new_user = user_bucket.new('johndoe', data={ - 'first_name': 'John', - 'last_name': 'Doe', - 'gender': 'm', - 'website': 'http://example.com/', - 'is_active': True, - }) - # Note that the user hasn't been stored in Riak yet. - new_user.store() - -Note that any data Python's ``json`` (or ``simplejson``) encoder can handle is -fair game. - -As mentioned, Riak can also handle binary data, such as images, audio files, -etc. Storing binary data looks almost identical:: - - import riak - - client = riak.RiakClient() - user_photo_bucket = client.bucket('user_photo') - - # For example purposes, we'll read a file off the filesystem, but you can get - # the data from anywhere. - the_photo_data = open('/tmp/johndoe_headshot.jpg', 'rb').read() - - # We're storing the photo in a different bucket but keyed off the same - # username. - new_user = user_photo_bucket.new_binary('johndoe', data=the_photo_data, content_type='image/jpeg') - new_user.store() - -You can also manually store data by using ``RiakObject``:: - - import riak - import time - import uuid - - client = riak.RiakClient() - status_bucket = client.bucket('status') - - # We use ``uuid.uuid1().hex`` here to create a unique identifier for the status. - post_uuid = uuid.uuid1().hex - new_status = riak.RiakObject(client, status_bucket, post_uuid) - - # Add in the data you want to store. - new_status.set_data({ - 'message': 'First post!', - 'created': time.time(), - 'is_public': True, - }) - - # Set the content type. - new_status.set_content_type('application/json') - - # We want to do JSON-encoding on the value. - new_status._encode_data = True - - # Again, make sure you save it. - new_status.store() - - -Getting Single Values Out -========================= - -Storing data is all well and good, but you'll need to get that data out at a -later date. - -Riak provides several ways to get data out, though fetching single key/value -pairs is the easiest. Just like storing the data, you can pull the data out -in either the JSON-decoded form or a binary blob. Getting the JSON-decoded -data out looks like:: - - import riak - - client = riak.RiakClient() - user_bucket = client.bucket('user') - - johndoe = user_bucket.get('johndoe') - - # You've now got a ``RiakObject``. To get at the values in a dictionary - # form, call: - johndoe_dict = johndoe.data - -Getting binary data out looks like:: - - import riak - - client = riak.RiakClient() - user_photo_bucket = client.bucket('user_photo') - - johndoe = user_photo_bucket.get_binary('johndoe') - - # You've now got a ``RiakObject``. To get at the binary data, call: - johndoe_headshot = johndoe.data - -Manually fetching data is also possible:: - - import riak - - client = riak.RiakClient() - status_bucket = client.bucket('status') - - # We're using the UUID generated from the above section. - first_post_status = riak.RiakObject(client, status_bucket, post_uuid) - first_post_status._encode_data = True - r = status_bucket.get_r() - - # Calling ``reload`` will cause the ``RiakObject`` instance to load fresh - # data/metadata from Riak. - first_post_status.reload(r) - - # Finally, pull out the data. - message = first_post_status.data['message'] - - -Fetching Data Via Map/Reduce -============================ - -When you need to work with larger sets of data, one of the tools at your -disposal is MapReduce_. This technique iterates over all of the data, returning -data from the map phase & combining all the different maps in the reduce -phase(s). - -.. _MapReduce: http://wiki.basho.com/MapReduce.html - -To perform a map operation, such as returning all active users, you can do -something like:: - - import riak - - client = riak.RiakClient() - # First, you need to ``add`` the bucket you want to MapReduce on. - query = client.add('user') - # Then, you supply a Javascript map function as the code to be executed. - query.map("function(v) { var data = JSON.parse(v.values[0].data); if(data.is_active == true) { return [[v.key, data]]; } return []; }") - - for result in query.run(): - # Print the key (``v.key``) and the value for that key (``data``). - print "%s - %s" % (result[0], result[1]) - - # Results in something like: - # - # mr_smith - {'first_name': 'Mister', 'last_name': 'Smith', 'is_active': True} - # johndoe - {'first_name': 'John', 'last_name': 'Doe', 'is_active': True} - # annabody - {'first_name': 'Anna', 'last_name': 'Body', 'is_active': True} - -You can also do this manually:: - - import riak - - client = riak.RiakClient() - query = riak.RiakMapReduce(client).add('user') - query.map("function(v) { var data = JSON.parse(v.values[0].data); if(data.is_active == true) { return [[v.key, data]]; } return []; }") - - for result in query.run(): - print "%s - %s" % (result[0], result[1]) - -Adding a reduce phase, say to sort by username (key), looks almost identical:: - - import riak - - client = riak.RiakClient() - query = client.add('user') - query.map("function(v) { var data = JSON.parse(v.values[0].data); if(data.is_active == true) { return [[v.key, data]]; } return []; }") - query.reduce("function(values) { return values.sort(); }") - - for result in query.run(): - # Print the key (``v.key``) and the value for that key (``data``). - print "%s - %s" % (result[0], result[1]) - - # Results in something like: - # - # annabody - {'first_name': 'Anna', 'last_name': 'Body', 'is_active': True} - # johndoe - {'first_name': 'John', 'last_name': 'Doe', 'is_active': True} - # mr_smith - {'first_name': 'Mister', 'last_name': 'Smith', 'is_active': True} - - -Working With Related Data Via Links -=================================== - -Links_ are powerful concept in Riak that allow, within the key/value pair's -metadata, relations between objects. - -.. _Links: http://wiki.basho.com/Links.html - -Adding them to your data is relatively trivial. For instance, we'll link a -user's statuses to their user data:: - - import riak - import uuid - - client = riak.RiakClient() - user_bucket = client.bucket('user') - status_bucket = client.bucket('status') - - johndoe = user_bucket.get('johndoe') - - new_status = status_bucket.new(uuid.uuid1().hex, data={ - 'message': 'First post!', - 'created': time.time(), - 'is_public': True, - }) - # Add one direction (from status to user)... - new_status.add_link(johndoe) - new_status.store() - - # ... Then add the other direction. - johndoe.add_link(new_status) - johndoe.store() - -Fetching the data is equally simple:: - - import riak - - client = riak.RiakClient() - user_bucket = client.bucket('user') - - johndoe = user_bucket.get('johndoe') - - for status_link in johndoe.get_links(): - # Since what we get back are lightweight ``RiakLink`` objects, we need to - # get the associated ``RiakObject`` to access its data. - status = status_link.get() - print status.data['message'] - - -Using Search -============ - -`Riak Search`_ is a new feature available as of Riak 0.13. It allows you to create -queries that filter on data in the values without writing a MapReduce. It takes -inspiration from Lucene_, a popular Java-based search library, and incorporates -a Solr-like interface into Riak. The setup of this is outside the realm of this -tutorial, but usage of this feature looks like:: - - import riak - - client = riak.RiakClient() - - # First parameter is the bucket we want to search within, the second - # is the query we want to perform. - search_query = client.search('user', 'first_name:[Anna TO John]') - - for result in search_query.run(): - # You get ``RiakLink`` objects back. - user = result.get() - user_data = user.data - print "%s %s" % (user_data['first_name'], user_data['last_name']) - - # Results in something like: - # - # John Doe - # Anna Body - -.. _`Riak Search`: http://wiki.basho.com/Riak-Search.html -.. _Lucene: http://lucene.apache.org/ - -Using Secondary Indexes -======================= - -Secondary Indexes is a new feature available as of Riak 1.0. It -allows you to tag an object with index metadata, and then later find -the object by querying the metadata, returning a list of matching keys. - -Your Riak cluster must have Secondary Indexes enabled. See the Riak -documentation for details. - -Usage of this feature looks like:: - - import riak - - client = riak.RiakClient() - bucket = client.bucket('mybucket') - - # Create and store the object with indexes... - obj = bucket.new('mykey1', 'mydata') - obj.add_index('field1_bin', 'val1') - obj.add_index('field2_int', 1001) - obj.store() - - # Query the indexes. The return value is a list of ``RiakLink`` objects. - results = client.index('mybucket', 'field1_bin', 'val1').run() - - # Query the indexes using a range... - results = client.index('mybucket', 'field1_bin', 'val1', 'val5').run() - - # Remove an index entry... - obj = bucket.get('mykey1') - obj.remove_index('field1_bin', 'val1') - obj.store() From 9263fcd6482d0584649abedd1860e6527696c7a0 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 11:34:39 -0500 Subject: [PATCH 074/672] Document more deprecated methods on client. --- docs/client.rst | 25 ++++++++++++++++++++++--- riak/util.py | 20 ++++++++++++++++++-- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/docs/client.rst b/docs/client.rst index ebeb664e..531cfc1d 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -37,6 +37,11 @@ RiakClient API .. autoattribute:: PROTOCOLS .. autoattribute:: protocol .. autoattribute:: client_id + .. attribute:: resolver + + The sibling-resolution function for this client. Defaults + to :func:`riak.resolver.default_resolver`. + .. attribute:: nodes The list of :class:`nodes ` that this @@ -136,8 +141,9 @@ media-types. Supported by default are ``application/json`` and Deprecated Methods and Properties ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. warning:: These methods exist solely for backwards-compatibility and should not - be used unless code is being ported from an older version. +.. warning:: These methods and attributes exist solely for + backwards-compatibility and should not be used unless code is being + ported from an older version. .. automethod:: RiakClient.get_transport .. automethod:: RiakClient.get_client_id @@ -145,4 +151,17 @@ Deprecated Methods and Properties .. attribute:: RiakClient.solr Returns a RiakSearch object which can access search indexes. - DEPRECATED + **DEPRECATED** + +.. automethod:: RiakClient.get_r +.. automethod:: RiakClient.set_r +.. automethod:: RiakClient.get_pr +.. automethod:: RiakClient.set_pr +.. automethod:: RiakClient.get_w +.. automethod:: RiakClient.set_w +.. automethod:: RiakClient.get_dw +.. automethod:: RiakClient.set_dw +.. automethod:: RiakClient.get_pw +.. automethod:: RiakClient.set_pw +.. automethod:: RiakClient.get_rw +.. automethod:: RiakClient.set_rw diff --git a/riak/util.py b/riak/util.py index f096caf7..9d371b92 100644 --- a/riak/util.py +++ b/riak/util.py @@ -80,7 +80,7 @@ def __deprecateQuorumAccessor(klass, parent, quorum): getter_name = "get_%s" % quorum setter_name = "set_%s" % quorum if not parent: - def direct_getter(self, val=None): + def direct_getter(self, value=None): deprecated(QDEPMESSAGE % klass.__name__) if val: return val @@ -88,7 +88,7 @@ def direct_getter(self, val=None): getter = direct_getter else: - def parent_getter(self, val=None): + def parent_getter(self, value=None): deprecated(QDEPMESSAGE % klass.__name__) if val: return val @@ -103,6 +103,22 @@ def setter(self, value): setattr(self, propname, value) return self + getter.__doc__ = """ + Gets the value used in requests for the {!r} quorum. + If not set, returns the passed value. **DEPRECATED** + + :param value: the value to use if not set + :type value: mixed + :rtype: mixed""".format(quorum) + + setter.__doc__ = """ + Sets the value used in requests for the {!r} quorum. + **DEPRECATED** + + :param value: the value to use if not set + :type value: mixed + """ + setattr(klass, getter_name, getter) setattr(klass, setter_name, setter) From 6cbbb96e05c5461f0c457280ddb8f7bac2e11d6b Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 12:13:00 -0500 Subject: [PATCH 075/672] Update bucket docs. --- docs/bucket.rst | 161 ++++++++++++++++++++++++++++++++++++++++++++++-- riak/bucket.py | 100 +++++++++++++++++++----------- 2 files changed, 219 insertions(+), 42 deletions(-) diff --git a/docs/bucket.rst b/docs/bucket.rst index aceaa730..9aa81df7 100644 --- a/docs/bucket.rst +++ b/docs/bucket.rst @@ -1,9 +1,160 @@ -.. ref-bucket: - -=========================== -Bucket Objects (RiakBucket) -=========================== +======= +Buckets +======= .. currentmodule:: riak.bucket +-------- +Overview +-------- + +Buckets are both namespaces for the key-value pairs you store in Riak, +and containers for properties that apply to that namespace. Buckets +should be created via the :meth:`bucket() +` method on the client object, like so:: + + import riak + + client = riak.RiakClient() + mybucket = client.bucket('mybucket') + +-------------- +RiakBucket API +-------------- + .. autoclass:: RiakBucket + + .. attribute:: name + + The name of the bucket, a string. + + .. autoattribute:: resolver + +^^^^^^^^^^^^^^^^^ +Bucket properties +^^^^^^^^^^^^^^^^^ + +Bucket properties are flags and defaults that apply to all keys in the +bucket. + +.. automethod:: RiakBucket.get_properties +.. automethod:: RiakBucket.set_properties +.. automethod:: RiakBucket.clear_properties +.. automethod:: RiakBucket.get_property +.. automethod:: RiakBucket.set_property + +""""""""""""""""""""""""""""""" +Shortcuts for common properties +""""""""""""""""""""""""""""""" + +Some of the most commonly-used bucket properties are exposed as object +properties as well. The getters and setters simply call +:meth:`RiakBucket.get_property` and :meth:`RiakBucket.set_property` +respectively. + +.. autoattribute:: RiakBucket.n_val +.. autoattribute:: RiakBucket.allow_mult +.. autoattribute:: RiakBucket.r +.. autoattribute:: RiakBucket.pr +.. autoattribute:: RiakBucket.w +.. autoattribute:: RiakBucket.dw +.. autoattribute:: RiakBucket.pw +.. autoattribute:: RiakBucket.rw + +"""""""""""""""""""" +Shortcuts for search +"""""""""""""""""""" + +When Riak Search is enabled on the server, you can toggle which +buckets have automatic indexing turned on using the ``search`` bucket +property (and on older versions, the ``precommit`` property). These +methods simplify interacting with that configuration. + +.. automethod:: RiakBucket.search_enabled +.. automethod:: RiakBucket.enable_search +.. automethod:: RiakBucket.disable_search + +^^^^^^^^^^^^^^^^^ +Working with keys +^^^^^^^^^^^^^^^^^ + +The primary purpose of buckets is to act as namespaces for keys. As +such, you can use the bucket object to create, fetch and delete +:class:`objects `. + +.. automethod:: RiakBucket.new +.. automethod:: RiakBucket.new_from_file +.. automethod:: RiakBucket.get +.. automethod:: RiakBucket.multiget +.. automethod:: RiakBucket.delete + +"""""""" +Counters +"""""""" + +Rather than returning objects, the counter operations new to Riak 1.4 +act directly on the value of the counter. + +.. automethod:: RiakBucket.get_counter +.. automethod:: RiakBucket.update_counter + +^^^^^^^^^^^^^^^^ +Query operations +^^^^^^^^^^^^^^^^ + +.. automethod:: RiakBucket.search +.. automethod:: RiakBucket.get_index +.. automethod:: RiakBucket.stream_index + + +^^^^^^^^^^^^^ +Serialization +^^^^^^^^^^^^^ + +Similar to :class:`RiakClient `, buckets can +register custom transformation functions for media-types. When +undefined on the bucket, :meth:`RiakBucket.get_encoder` and +:meth:`RiakBucket.get_decoder` will delegate to the client associated +with the bucket. + +.. automethod:: RiakBucket.get_encoder +.. automethod:: RiakBucket.set_encoder +.. automethod:: RiakBucket.get_decoder +.. automethod:: RiakBucket.set_decoder + + +^^^^^^^^^^^^ +Listing keys +^^^^^^^^^^^^ + +Shortcuts for :meth:`RiakClient.get_keys() +` and +:meth:`RiakClient.stream_keys() +` are exposed on the bucket +object. The same admonitions for these operations apply. + +.. automethod:: RiakBucket.get_keys +.. automethod:: RiakBucket.stream_keys + +^^^^^^^^^^^^^^^^^^ +Deprecated methods +^^^^^^^^^^^^^^^^^^ + +.. warning:: These methods exist solely for backwards-compatibility and should not + be used unless code is being ported from an older version. + +.. automethod:: RiakBucket.new_binary +.. automethod:: RiakBucket.new_binary_from_file +.. automethod:: RiakBucket.get_binary +.. automethod:: RiakBucket.get_r +.. automethod:: RiakBucket.set_r +.. automethod:: RiakBucket.get_pr +.. automethod:: RiakBucket.set_pr +.. automethod:: RiakBucket.get_w +.. automethod:: RiakBucket.set_w +.. automethod:: RiakBucket.get_dw +.. automethod:: RiakBucket.set_dw +.. automethod:: RiakBucket.get_pw +.. automethod:: RiakBucket.set_pw +.. automethod:: RiakBucket.get_rw +.. automethod:: RiakBucket.set_rw diff --git a/riak/bucket.py b/riak/bucket.py index 7770f3e6..22dc3cb9 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -43,8 +43,6 @@ class RiakBucket(object): objects within the bucket. """ - SEARCH_PRECOMMIT_HOOK = {"mod": "riak_search_kv_hook", "fun": "precommit"} - def __init__(self, client, name): """ Returns a new ``RiakBucket`` instance. @@ -86,6 +84,8 @@ def get_encoder(self, content_type): Get the encoding function for the provided content type for this bucket. + :param content_type: the requested media type + :type content_type: str :param content_type: Content type requested """ if content_type in self._encoders: @@ -98,9 +98,11 @@ def set_encoder(self, content_type, encoder): Set the encoding function for the provided content type for this bucket. - :param content_type: Content type for encoder - :param encoder: Function to encode with - will be called with - data as single argument. + :param content_type: the requested media type + :type content_type: str + :param encoder: an encoding function, takes a single object + argument and returns a string data as single argument. + :type encoder: function """ self._encoders[content_type] = encoder return self @@ -110,7 +112,9 @@ def get_decoder(self, content_type): Get the decoding function for the provided content type for this bucket. - :param content_type: Content type for decoder + :param content_type: the requested media type + :type content_type: str + :rtype: function """ if content_type in self._decoders: return self._decoders[content_type] @@ -122,9 +126,11 @@ def set_decoder(self, content_type, decoder): Set the decoding function for the provided content type for this bucket. - :param content_type: Content type for decoder - :param decoder: Function to decode with - will be called with - string + :param content_type: the requested media type + :type content_type: str + :param decoder: a decoding function, takes a string and + returns a Python type + :type decoder: function """ self._decoders[content_type] = decoder return self @@ -164,7 +170,7 @@ def new_binary(self, key=None, data=None, Create a new :class:`RiakObject ` that will be stored as plain text/binary. A shortcut for manually instantiating a :class:`RiakObject - `. + `. **DEPRECATED** :param key: Name of the key. :type key: string @@ -198,7 +204,7 @@ def get(self, key, r=None, pr=None, timeout=None): def get_binary(self, key, r=None, pr=None, timeout=None): """ - Retrieve a binary/string object from Riak. DEPRECATED + Retrieve a binary/string object from Riak. **DEPRECATED** :param key: Name of the key. :type key: string @@ -246,8 +252,7 @@ def _set_resolver(self, value): resolver = property(_get_resolver, _set_resolver, doc= """The sibling-resolution function for this bucket. If the resolver is not set, the - client's resolver will be used. :type - callable""") + client's resolver will be used.""") n_val = bucket_property('n_val', doc=""" N-value for this bucket, which is the number of replicas @@ -261,8 +266,7 @@ def _set_resolver(self, value): allow_mult = bucket_property('allow_mult', doc=""" If set to True, then writes with conflicting data will be stored - and returned to the client. This situation can be detected by - calling has_siblings() and get_siblings(). + and returned to the client. :type bool: boolean """) @@ -344,7 +348,6 @@ def get_properties(self): def clear_properties(self): """ Reset all bucket properties to their defaults. - """ return self._client.clear_bucket_props(self) @@ -352,9 +355,7 @@ def get_keys(self): """ Return all keys within the bucket. - .. warning:: - - At current, this is a very expensive operation. Use with caution. + :rtype: list of keys """ return self._client.get_keys(self) @@ -362,18 +363,21 @@ def stream_keys(self): """ Streams all keys within the bucket through an iterator. - .. warning:: - - At current, this is a very expensive operation. Use with caution. - :rtype: iterator """ return self._client.stream_keys(self) def new_from_file(self, key, filename): """ - Create a new Riak object in the bucket, using the content of - the specified file. + Create a new Riak object in the bucket, using the contents of + the specified file. This is a shortcut for :meth:`new`, where the + ``encoded_data`` and ``content_type`` are set for you. + + :param key: the key of the new object + :type key: string + :param filename: the file to read the contents from + :type filename: string + :rtype: :class:`RiakObject ` """ binary_data = open(filename, "rb").read() mimetype, encoding = mimetypes.guess_type(filename) @@ -386,21 +390,31 @@ def new_from_file(self, key, filename): return self.new(key, encoded_data=binary_data, content_type=mimetype) def new_binary_from_file(self, key, filename): + """ + Create a new Riak object in the bucket, using the contents of + the specified file. This is a shortcut for :meth:`new`, where the + ``encoded_data`` and ``content_type`` are set for you. **DEPRECATED** + + :param key: the key of the new object + :type key: string + :param filename: the file to read the contents from + :type filename: string + :rtype: :class:`RiakObject ` + """ deprecated('RiakBucket.new_binary_from_file is deprecated, use ' 'RiakBucket.new_from_file') return self.new_from_file(key, filename) def search_enabled(self): """ - Returns True if the search precommit hook is enabled for this + Returns True if search indexing is enabled for this bucket. """ return self.get_properties().get('search', False) def enable_search(self): """ - Enable search for this bucket by installing the precommit hook to - index objects in it. + Enable search indexing for this bucket. """ if not self.search_enabled(): self.set_property('search', True) @@ -408,8 +422,7 @@ def enable_search(self): def disable_search(self): """ - Disable search for this bucket by removing the precommit hook to - index objects in it. + Disable search indexing for this bucket. """ if self.search_enabled(): self.set_property('search', False) @@ -417,14 +430,19 @@ def disable_search(self): def search(self, query, **params): """ - Queries a search index over objects in this bucket/index. + Queries a search index over objects in this bucket/index. See + :meth:`RiakClient.fulltext_search() + ` for more details. """ return self._client.solr.search(self.name, query, **params) def get_index(self, index, startkey, endkey=None, return_terms=None, max_results=None, continuation=None): """ - Queries a secondary index over objects in this bucket, returning keys. + Queries a secondary index over objects in this bucket, + returning keys or index/key pairs. See + :meth:`RiakClient.get_index() + ` for more details. """ return self._client.get_index(self.name, index, startkey, endkey, return_terms=return_terms, @@ -435,7 +453,9 @@ def stream_index(self, index, startkey, endkey=None, return_terms=None, max_results=None, continuation=None): """ Queries a secondary index over objects in this bucket, - streaming keys via an iterator. + streaming keys or index/key pairs via an iterator. See + :meth:`RiakClient.stream_index() + ` for more details. """ return self._client.stream_index(self.name, index, startkey, endkey, return_terms=return_terms, @@ -443,9 +463,10 @@ def stream_index(self, index, startkey, endkey=None, return_terms=None, continuation=continuation) def delete(self, key, **kwargs): - """Deletes an object from riak. + """Deletes an object from riak. Short hand for + bucket.new(key).delete(). See :meth:`RiakClient.delete() + ` for options. - Short hand for bucket.new(key).delete() :param key: The key for the object :type key: string :rtype: RiakObject @@ -454,7 +475,9 @@ def delete(self, key, **kwargs): def get_counter(self, key, **kwargs): """ - Gets the value of a counter stored in this bucket. + Gets the value of a counter stored in this bucket. See + :meth:`RiakClient.get_counter() + ` for options. :param key: the key of the counter :type key: string @@ -465,7 +488,10 @@ def get_counter(self, key, **kwargs): def update_counter(self, key, value, **kwargs): """ Updates the value of a counter stored in this bucket. Positive - values increment the counter, negative values decrement. + values increment the counter, negative values decrement. See + :meth:`RiakClient.update_counter() + ` for options. + :param key: the key of the counter :type key: string From 6ec6f6a619d46345f43f447041c545e17b652c6f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 12:32:47 -0500 Subject: [PATCH 076/672] Use the bootstrap theme. This requires installing `sphinx-bootstrap-theme` from PyPi. --- docs/conf.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index bbc2064f..4ea6f92b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ # serve to show the default. import sys, os +import sphinx_bootstrap_theme # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -81,7 +82,7 @@ #show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = 'friendly' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] @@ -91,15 +92,20 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'bootstrap' +# html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +html_theme_options = { + 'navbar_site_name':"Documentation", + 'globaltoc_depth': 2, + 'bootswatch_theme': 'cerulean' +} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +html_theme_path = sphinx_bootstrap_theme.get_html_theme_path() # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". From 6fecbfb5b191d2fde303f33a185c1feed77104d6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 15:04:56 -0500 Subject: [PATCH 077/672] Move contents into a floating well, rename/remove some files. --- docs/content.rst | 9 --------- docs/index.rst | 30 +++++++++++++++++++--------- docs/{riak_object.rst => object.rst} | 0 3 files changed, 21 insertions(+), 18 deletions(-) delete mode 100644 docs/content.rst rename docs/{riak_object.rst => object.rst} (100%) diff --git a/docs/content.rst b/docs/content.rst deleted file mode 100644 index 04880600..00000000 --- a/docs/content.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. ref-content: - -=========== -RiakContent -=========== - -.. currentmodule:: riak.content - -.. autoclass:: riak.content.RiakContent diff --git a/docs/index.rst b/docs/index.rst index 9b1ed90a..5fb8c0dd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,5 +1,18 @@ Riak Python Client -===================== +================== + +.. cssclass:: well pull-right +.. compound:: + + **Contents** + + .. toctree:: + :maxdepth: 2 + + client + bucket + object + mapreduce Installation ------------ @@ -16,16 +29,15 @@ Installation .. _easy_install: http://pypi.python.org/pypi/setuptools .. _PyPI: http://pypi.python.org/pypi/riak/ -Contents: +Development +----------- + +All development is done on Github_. Use Issues_ to report +problems or submit contributions. -.. toctree:: - :maxdepth: 2 +.. _Github: https://github.com/basho/riak-python-client/ +.. _Issues: https://github.com/basho/riak-python-client/issues - client - bucket - riak_object - content - mapreduce Indices and tables ------------------ diff --git a/docs/riak_object.rst b/docs/object.rst similarity index 100% rename from docs/riak_object.rst rename to docs/object.rst From 51379b605f0fcf4cfd97eac45d665a17c08f63d6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 22:02:32 -0500 Subject: [PATCH 078/672] Tweak some headers formatting. Also, fix a bug where deprecated quorum accessor docs weren't being formatted properly. --- docs/bucket.rst | 44 +++++++++++++++++++++----------------------- docs/client.rst | 36 ++++++++++++++++-------------------- riak/util.py | 2 +- 3 files changed, 38 insertions(+), 44 deletions(-) diff --git a/docs/bucket.rst b/docs/bucket.rst index 9aa81df7..8f0ea833 100644 --- a/docs/bucket.rst +++ b/docs/bucket.rst @@ -4,10 +4,6 @@ Buckets .. currentmodule:: riak.bucket --------- -Overview --------- - Buckets are both namespaces for the key-value pairs you store in Riak, and containers for properties that apply to that namespace. Buckets should be created via the :meth:`bucket() @@ -19,7 +15,7 @@ should be created via the :meth:`bucket() mybucket = client.bucket('mybucket') -------------- -RiakBucket API +Bucket objects -------------- .. autoclass:: RiakBucket @@ -30,9 +26,9 @@ RiakBucket API .. autoattribute:: resolver -^^^^^^^^^^^^^^^^^ +----------------- Bucket properties -^^^^^^^^^^^^^^^^^ +----------------- Bucket properties are flags and defaults that apply to all keys in the bucket. @@ -43,9 +39,9 @@ bucket. .. automethod:: RiakBucket.get_property .. automethod:: RiakBucket.set_property -""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Shortcuts for common properties -""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Some of the most commonly-used bucket properties are exposed as object properties as well. The getters and setters simply call @@ -61,9 +57,9 @@ respectively. .. autoattribute:: RiakBucket.pw .. autoattribute:: RiakBucket.rw -"""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^ Shortcuts for search -"""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^ When Riak Search is enabled on the server, you can toggle which buckets have automatic indexing turned on using the ``search`` bucket @@ -74,9 +70,9 @@ methods simplify interacting with that configuration. .. automethod:: RiakBucket.enable_search .. automethod:: RiakBucket.disable_search -^^^^^^^^^^^^^^^^^ +----------------- Working with keys -^^^^^^^^^^^^^^^^^ +----------------- The primary purpose of buckets is to act as namespaces for keys. As such, you can use the bucket object to create, fetch and delete @@ -88,9 +84,11 @@ such, you can use the bucket object to create, fetch and delete .. automethod:: RiakBucket.multiget .. automethod:: RiakBucket.delete -"""""""" +.. _counters: + +^^^^^^^^ Counters -"""""""" +^^^^^^^^ Rather than returning objects, the counter operations new to Riak 1.4 act directly on the value of the counter. @@ -98,18 +96,18 @@ act directly on the value of the counter. .. automethod:: RiakBucket.get_counter .. automethod:: RiakBucket.update_counter -^^^^^^^^^^^^^^^^ +---------------- Query operations -^^^^^^^^^^^^^^^^ +---------------- .. automethod:: RiakBucket.search .. automethod:: RiakBucket.get_index .. automethod:: RiakBucket.stream_index -^^^^^^^^^^^^^ +------------- Serialization -^^^^^^^^^^^^^ +------------- Similar to :class:`RiakClient `, buckets can register custom transformation functions for media-types. When @@ -123,9 +121,9 @@ with the bucket. .. automethod:: RiakBucket.set_decoder -^^^^^^^^^^^^ +------------ Listing keys -^^^^^^^^^^^^ +------------ Shortcuts for :meth:`RiakClient.get_keys() ` and @@ -136,9 +134,9 @@ object. The same admonitions for these operations apply. .. automethod:: RiakBucket.get_keys .. automethod:: RiakBucket.stream_keys -^^^^^^^^^^^^^^^^^^ +------------------ Deprecated methods -^^^^^^^^^^^^^^^^^^ +------------------ .. warning:: These methods exist solely for backwards-compatibility and should not be used unless code is being ported from an older version. diff --git a/docs/client.rst b/docs/client.rst index 531cfc1d..053dcd78 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -2,10 +2,6 @@ Client & Connections ==================== --------- -Overview --------- - To connect to a Riak cluster, you must create a :py:class:`~riak.client.RiakClient` object. The default configuration connects to a single Riak node on ``localhost`` with the default @@ -28,7 +24,7 @@ protocol. Connections are opened as-needed; a random node is selected when a new connection is requested. -------------- -RiakClient API +Client objects -------------- .. currentmodule:: riak.client @@ -66,9 +62,9 @@ the ``RiakClient`` constructor, they will be turned into this type. .. autoclass:: riak.node.RiakNode :members: -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- Client-level Operations -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- Some operations are not scoped by buckets and can be performed on the client directly: @@ -77,9 +73,9 @@ client directly: .. automethod:: RiakClient.get_buckets .. automethod:: RiakClient.stream_buckets -^^^^^^^^^^^^^^^^^ +----------------- Accessing Buckets -^^^^^^^^^^^^^^^^^ +----------------- Most client operations are on :py:class:`bucket objects ` or keys within those buckets. Use the @@ -88,9 +84,9 @@ the called client. .. automethod:: RiakClient.bucket -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- Bucket-level Operations -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- .. automethod:: RiakClient.get_bucket_props .. automethod:: RiakClient.set_bucket_props @@ -98,9 +94,9 @@ Bucket-level Operations .. automethod:: RiakClient.get_keys .. automethod:: RiakClient.stream_keys -^^^^^^^^^^^^^^^^^^^^ +-------------------- Key-level Operations -^^^^^^^^^^^^^^^^^^^^ +-------------------- .. automethod:: RiakClient.get .. automethod:: RiakClient.put @@ -109,9 +105,9 @@ Key-level Operations .. automethod:: RiakClient.get_counter .. automethod:: RiakClient.update_counter -^^^^^^^^^^^^^^^^ +---------------- Query Operations -^^^^^^^^^^^^^^^^ +---------------- .. automethod:: RiakClient.mapred .. automethod:: RiakClient.stream_mapred @@ -121,9 +117,9 @@ Query Operations .. automethod:: RiakClient.fulltext_add .. automethod:: RiakClient.fulltext_delete -^^^^^^^^^^^^^ +------------- Serialization -^^^^^^^^^^^^^ +------------- The client supports automatic transformation of Riak responses into Python types if encoders and decoders are registered for the @@ -137,9 +133,9 @@ media-types. Supported by default are ``application/json`` and .. automethod:: RiakClient.set_decoder -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Deprecated Methods and Properties -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------ +Deprecated Methods +------------------ .. warning:: These methods and attributes exist solely for backwards-compatibility and should not be used unless code is being diff --git a/riak/util.py b/riak/util.py index 9d371b92..c8f79777 100644 --- a/riak/util.py +++ b/riak/util.py @@ -117,7 +117,7 @@ def setter(self, value): :param value: the value to use if not set :type value: mixed - """ + """.format(quorum) setattr(klass, getter_name, getter) setattr(klass, setter_name, setter) From 3fbd8bf2c69512a20bf52b09323908ab398b75a1 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 25 Jul 2013 22:03:18 -0500 Subject: [PATCH 079/672] WIP Object docs, with explanations of siblings and resolvers. --- docs/object.rst | 138 ++++++++++++++++++++++++++++++++++++++++++-- riak/content.py | 21 +++++-- riak/resolver.py | 12 ++-- riak/riak_object.py | 55 ++++++++++-------- 4 files changed, 186 insertions(+), 40 deletions(-) diff --git a/docs/object.rst b/docs/object.rst index b85fcbb1..4c9d015a 100644 --- a/docs/object.rst +++ b/docs/object.rst @@ -1,9 +1,137 @@ -.. ref-riak-object: +============== +Keys & Objects +============== -========== +.. currentmodule:: riak.riak_object + +Keys in Riak are namespaced into :class:`buckets +`, and their associated values are represented +by :class:`objects `, not to be confused with Python +"objects". A :class:`RiakObject` is a container for the key, the +:ref:`vclock`, the value(s) and any metadata associated with the +value(s). + +---------- RiakObject -========== +---------- -.. currentmodule:: riak.riak_object +.. autoclass:: RiakObject + + .. attribute:: key + + The key of this object, a string. If not present, the server + will generate a key the first time this object is stored. + + .. attribute:: bucket + + The :class:`bucket ` to which this + object belongs. + + .. autoattribute:: resolver + .. attribute:: vclock + + The :ref:`vclock` for this object. + + .. autoattribute:: exists + +.. _vclock: + +^^^^^^^^^^^^ +Vector clock +^^^^^^^^^^^^ + +Vector clocks are Riak's means of tracking the relationships between +writes to a key. It is best practice to fetch the latest version of a +key before attempting to modify or overwrite the value; if you do not, +you may create :ref:`siblings` or lose data! The content of a vector +clock is essentially opaque to the user. + +.. autoclass:: VClock + +----------- +Persistence +----------- + +Fetching, storing, and deleting keys are the bread-and-butter of Riak. + +.. automethod:: RiakObject.store +.. automethod:: RiakObject.reload +.. automethod:: RiakObject.delete + +.. _object_accessors: + +------------------ +Value and Metadata +------------------ + +Unless you have enabled :ref:`siblings` via the :attr:`allow_mult +` bucket property, you can +inspect and manipulate the value and metadata of an object directly using these +properties and methods: + +.. autoattribute:: RiakObject.data +.. autoattribute:: RiakObject.encoded_data +.. autoattribute:: RiakObject.content_type +.. autoattribute:: RiakObject.charset +.. autoattribute:: RiakObject.content_encoding +.. autoattribute:: RiakObject.last_modified +.. autoattribute:: RiakObject.etag +.. autoattribute:: RiakObject.usermeta +.. autoattribute:: RiakObject.links +.. autoattribute:: RiakObject.indexes +.. automethod:: RiakObject.add_index +.. automethod:: RiakObject.remove_index +.. automethod:: RiakObject.set_index +.. automethod:: RiakObject.add_link + +.. _siblings: + +-------- +Siblings +-------- + +Because Riak's consistency model is "eventual" (and not linearizable), +there is no way for it to disambiguate writes that happen +concurrently. The :ref:`vclock` helps establish a +"happens after" relationships so that concurrent writes can be +detected, but with the exception of :ref:`counters`, Riak has no way +to determine which write has the correct value. + +Instead, when :attr:`allow_mult ` +is ``True``, Riak keeps all writes that appear to be concurrent. Thus, +the contents of a key's value may, in fact, be multiple values, which +are called "siblings". Siblings are modeled in :class:`RiakContent +` objects, which contain all of the same +:ref:`object_accessors` methods and attributes as the parent object. + +.. autoattribute:: RiakObject.siblings + +.. autoclass:: riak.content.RiakContent + +You do not typically have to create :class:`RiakContent +` objects yourself, but they will be created +for you when :meth:`fetching ` objects from Riak. + +.. note:: The :ref:`object_accessors` accessors on :class:`RiakObject` + are actually proxied to the first sibling when the object has only + one. + + +^^^^^^^^^^^^^^^^^^^^^^^ +Conflicts and Resolvers +^^^^^^^^^^^^^^^^^^^^^^^ + +When an object is *not* in conflict, it has only one sibling. When it +is in conflict, you will have to resolve the conflict before it can be +written again. How you choose to resolve the conflict is up to you, +but you can automate the process using a :attr:`resolver +` function. + +.. autofunction:: riak.resolver.default_resolver +.. autofunction:: riak.resolver.last_written_resolver + +If you do not supply a resolver function, or your resolver leaves +multiple siblings present, accessing the :ref:`object_accessors` will +result in a :exc:`ConflictError ` being raised. -.. autoclass:: riak.riak_object.RiakObject +.. autoexception:: riak.ConflictError diff --git a/riak/content.py b/riak/content.py index dce93a4e..1768c219 100644 --- a/riak/content.py +++ b/riak/content.py @@ -109,6 +109,8 @@ def _deserialize(self, value): def add_index(self, field, value): """ + add_index(field, value) + Tag this object with the specified field/value pair for indexing. @@ -116,7 +118,7 @@ def add_index(self, field, value): :type field: string :param value: The index value. :type value: string or integer - :rtype: RiakObject + :rtype: :class:`RiakObject ` """ if field[-4:] not in ("_bin", "_int"): raise RiakError("Riak 2i fields must end with either '_bin'" @@ -128,6 +130,8 @@ def add_index(self, field, value): def remove_index(self, field=None, value=None): """ + remove_index(field=None, value=None) + Remove the specified field/value pair as an index on this object. @@ -135,7 +139,7 @@ def remove_index(self, field=None, value=None): :type field: string :param value: The index value. :type value: string or integer - :rtype: RiakObject + :rtype: :class:`RiakObject ` """ if not field and not value: self.indexes.clear() @@ -154,14 +158,17 @@ def remove_index(self, field=None, value=None): def set_index(self, field, value): """ - Works like add_index, but ensures that there is only one index - on given field. If other found, then removes it first. + set_index(field, value) + + Works like :meth:`add_index`, but ensures that there is only + one index on given field. If other found, then removes it + first. :param field: The index field. :type field: string :param value: The index value. :type value: string or integer - :rtype: RiakObject + :rtype: :class:`RiakObject ` """ to_rem = set((x for x in self.indexes if x[0] == field)) self.indexes.difference_update(to_rem) @@ -169,6 +176,8 @@ def set_index(self, field, value): def add_link(self, obj, tag=None): """ + add_link(obj, tag=None) + Add a link to a RiakObject. :param obj: Either a RiakObject or 3 item link tuple consisting @@ -177,7 +186,7 @@ def add_link(self, obj, tag=None): :param tag: Optional link tag. Defaults to bucket name. It is ignored if ``obj`` is a 3 item link tuple. :type tag: string - :rtype: RiakObject + :rtype: :class:`RiakObject ` """ if isinstance(obj, tuple): newlink = obj diff --git a/riak/resolver.py b/riak/resolver.py index 30bfcc73..c54779ca 100644 --- a/riak/resolver.py +++ b/riak/resolver.py @@ -20,12 +20,14 @@ def default_resolver(riak_object): """ The default conflict-resolution function, which does nothing. To - implement a resolver, define a function that sets the ``siblings`` - property on the passed ``RiakObject`` instance to a list - containing a single ``RiakContent`` object. + implement a resolver, define a function that sets the + :attr:`siblings ` property + on the passed :class:`RiakObject ` + instance to a list containing a single :class:`RiakContent + ` object. :param riak_object: an object-in-conflict that will be resolved - :type riak_object: RiakObject + :type riak_object: :class:`RiakObject ` """ pass @@ -36,7 +38,7 @@ def last_written_resolver(riak_object): recently-modified sibling by timestamp. :param riak_object: an object-in-conflict that will be resolved - :type riak_object: RiakObject + :type riak_object: :class:`RiakObject ` """ lm = lambda x: x.last_modified riak_object.siblings = [max(riak_object.siblings, key=lm), ] diff --git a/riak/riak_object.py b/riak/riak_object.py index b74d61c7..0c9573a9 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -58,6 +58,8 @@ def _delegate(self, *args, **kwargs): raise ConflictError() return getattr(self.siblings[0], name).__call__(*args, **kwargs) + _delegate.__doc__ = getattr(RiakContent, name).__doc__ + return _delegate @@ -125,6 +127,9 @@ def __init__(self, client, bucket, key=None): self.vclock = None self.siblings = [RiakContent(self)] + #: The list of sibling values contained in this object + siblings = [] + def __hash__(self): return hash((self.key, self.bucket, self.vclock)) @@ -146,7 +151,7 @@ def __ne__(self, other): this property will result in decoding the `encoded_data` property into Python values. The decoding is dependent on the `content_type` property and the bucket's registered decoders. - :type mixed """) + """) encoded_data = content_property('encoded_data', doc=""" The raw data stored in this object, essentially the encoded @@ -154,42 +159,42 @@ def __ne__(self, other): will result in encoding the `data` property into a string. The encoding is dependent on the `content_type` property and the bucket's registered encoders. - :type basestring""") + """) charset = content_property('charset', doc=""" - The character set of the encoded data - :type string""") + The character set of the encoded data as a string + """) content_type = content_property('content_type', doc=""" - The MIME media type of the encoded data - :type string""") + The MIME media type of the encoded data as a string + """) content_encoding = content_property('content_encoding', doc=""" The encoding (compression) of the encoded data. Valid values are identity, deflate, gzip - :type string""") + """) last_modified = content_property('last_modified', """ The UNIX timestamp of the modification time of this value. - :type float""") + """) etag = content_property('etag', """ A unique entity-tag for the value. - :type string""") + """) usermeta = content_property('usermeta', doc=""" - Arbitrary user-defined metadata, mapping strings to strings. - :type dict""") + Arbitrary user-defined metadata dict, mapping strings to strings. + """) links = content_property('links', doc=""" - A collection of bucket/key/tag 3-tuples representing links to - other keys. - :type set""") + A set of bucket/key/tag 3-tuples representing links to other + keys. + """) indexes = content_property('indexes', doc=""" The set of secondary index entries, consisting of index-name/value tuples - :type set""") + """) get_encoded_data = content_method('get_encoded_data') set_encoded_data = content_method('set_encoded_data') @@ -210,10 +215,9 @@ def _exists(self): return self.siblings[0].exists exists = property(_exists, None, doc=""" - Whether the object exists. This is only False when there are no - siblings (the object was not found), or the solitary sibling is - a tombstone. - :type bool + Whether the object exists. This is only ``False`` when there + are no siblings (the object was not found), or the solitary + sibling is a tombstone. """) def _get_resolver(self): @@ -233,8 +237,7 @@ def _set_resolver(self, value): resolver = property(_get_resolver, _set_resolver, doc= """The sibling-resolution function for this object. If the resolver is not set, the - bucket's resolver will be used. :type - callable""") + bucket's resolver will be used.""") def get_sibling(self, index): deprecated("RiakObject.get_sibling is deprecated, use the " @@ -267,7 +270,7 @@ def store(self, w=None, dw=None, pw=None, return_body=True, :type if_none_match: bool :param timeout: a timeout value in milliseconds :type timeout: int - :rtype: RiakObject """ + :rtype: :class:`RiakObject` """ if len(self.siblings) != 1: raise ConflictError("Attempting to store an invalid object, " "resolve the siblings first") @@ -285,6 +288,10 @@ def reload(self, r=None, pr=None, timeout=None): object could contain new metadata and a new value, if the object was updated in Riak since it was last retrieved. + .. note:: Even if the key is not found in Riak, this will + return a :class:`RiakObject`. Check the :attr:`exists` + property to see if the key was found. + :param r: R-Value, wait for this many partitions to respond before returning to client. :type r: integer @@ -294,7 +301,7 @@ def reload(self, r=None, pr=None, timeout=None): :type pr: integer :param timeout: a timeout value in milliseconds :type timeout: int - :rtype: RiakObject + :rtype: :class:`RiakObject` """ self.client.get(self, r=r, pr=pr, timeout=timeout) @@ -327,7 +334,7 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None, :type pw: integer :param timeout: a timeout value in milliseconds :type timeout: int - :rtype: RiakObject + :rtype: :class:`RiakObject` """ self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw, From 6dcc74de347a504a4598cdc97d74b04050539458 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 26 Jul 2013 11:05:28 -0500 Subject: [PATCH 080/672] Make some template tweaks. --- docs/_static/custom.css | 17 +++++++++++++ docs/_templates/layout.html | 49 +++++++++++++++++++++++++++++++++++++ docs/conf.py | 9 ++++--- 3 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 docs/_static/custom.css create mode 100644 docs/_templates/layout.html diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 00000000..d8a66dbe --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,17 @@ +div.admonition p { + margin: 0; +} + +p.admonition-title { + float: left; + margin-right: 0.5em ! important; +} + +p.admonition-title:after { + content: ":"; + font-weight: bold; +} + +div.alert-info a { + color: #555; +} \ No newline at end of file diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 00000000..91c92523 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,49 @@ +{% extends "!layout.html" %} + +{% block sidebarrel %}{% endblock %} + +{%- block footer %} +
+ +
+
+
+

+ Back to top + {% if theme_source_link_position == "footer" %} +
+ {% include "sourcelink.html" %} + {% endif %} +

+

+ {%- if show_copyright %} + {%- if hasdoc('copyright') %} + {% trans path=pathto('copyright'), copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %}
+ {%- else %} + {% trans copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %}
+ {%- endif %} + {%- endif %} + {%- if last_updated %} + {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %}
+ {%- endif %} + {%- if show_sphinx %} + {% trans sphinx_version=sphinx_version|e %}Created using Sphinx {{ sphinx_version }}.{% endtrans %}
+ {%- endif %} +

+
+
+{%- endblock %} + + +{% set css_files = css_files + ['_static/custom.css'] %} diff --git a/docs/conf.py b/docs/conf.py index 4ea6f92b..38b7abc2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -82,7 +82,7 @@ #show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'friendly' +pygments_style = 'tango' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] @@ -99,9 +99,10 @@ # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - 'navbar_site_name':"Documentation", + 'bootswatch_theme': 'cerulean', + 'navbar_site_name':"Docs", 'globaltoc_depth': 2, - 'bootswatch_theme': 'cerulean' + 'source_link_position':'footer' } # Add any paths that contain custom themes here, relative to this directory. @@ -126,7 +127,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -#html_static_path = ['_static'] +html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. From f31b4c3b586461f815404f67e006841aaa915db7 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 26 Jul 2013 15:45:27 -0500 Subject: [PATCH 081/672] Add "Advanced" page with all the miscellaneous and implementation details. --- docs/advanced.rst | 142 ++++++++++++++++++++++++++++++ docs/index.rst | 1 + riak/benchmark.py | 8 +- riak/client/multiget.py | 31 +++++-- riak/client/transport.py | 2 + riak/transports/feature_detect.py | 51 ++++++++--- riak/transports/pool.py | 26 +++--- riak/transports/transport.py | 3 +- riak/util.py | 8 +- 9 files changed, 233 insertions(+), 39 deletions(-) create mode 100644 docs/advanced.rst diff --git a/docs/advanced.rst b/docs/advanced.rst new file mode 100644 index 00000000..8bb3a9dc --- /dev/null +++ b/docs/advanced.rst @@ -0,0 +1,142 @@ +============================ + Advanced Usage & Internals +============================ + +This page contains documentation for aspects of library internals that +you will rarely need to interact with, but are important for +understanding how it works and development purposes. + +--------------- +Connection pool +--------------- + +.. currentmodule:: riak.transports.pool + +.. autoexception:: BadResource +.. autoclass:: Element + :members: +.. autoclass:: Pool + :members: + +.. autoclass:: PoolIterator + :members: + :special-members: + +----------- +Retry logic +----------- + +.. currentmodule:: riak.client.transport + +.. autoclass:: RiakClientTransport + :members: + :private-members: + +.. autofunction:: _is_retryable + +.. autofunction:: retryable + +.. autofunction:: retryableHttpOnly + +-------- +Multiget +-------- + +.. currentmodule:: riak.client.multiget + +.. autodata:: POOL_SIZE + +.. autoclass:: Task + +.. autoclass:: MultiGetPool + :members: + :private-members: + +.. autodata:: RIAK_MULTIGET_POOL + +.. autofunction:: multiget + +---------- +Transports +---------- + +.. currentmodule:: riak.transports.transport + +.. autoclass:: RiakTransport + :members: + :private-members: + +.. currentmodule:: riak.transports.feature_detect + +.. autoclass:: FeatureDetection + :members: + :private-members: + +^^^^^^^^^^^^^^ +HTTP Transport +^^^^^^^^^^^^^^ + +.. currentmodule:: riak.transports.http + +.. autoclass:: RiakHttpPool + +.. autofunction:: is_retryable + +.. autoclass:: RiakHttpTransport + :members: + +^^^^^^^^^^^^^^^^^^^^^^^^^^ +Protocol Buffers Transport +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. currentmodule:: riak.transports.pbc + +.. autoclass:: RiakPbcTransport + :members: + +--------- +Utilities +--------- + +^^^^^^^^^^^^^^^^^ +Multi-valued Dict +^^^^^^^^^^^^^^^^^ + +.. currentmodule:: riak.multidict + +.. autoclass:: MultiDict + + .. automethod:: add + .. automethod:: getall + .. automethod:: getone + .. automethod:: mixed + .. automethod:: dict_of_lists + +^^^^^^^^^^^^^^^^^^ +Micro-benchmarking +^^^^^^^^^^^^^^^^^^ + +.. currentmodule:: riak.benchmark + +.. autofunction:: measure + +.. autofunction:: measure_with_rehearsal + +.. autoclass:: Benchmark + :members: + +^^^^^^^^^^^^^ +Miscellaneous +^^^^^^^^^^^^^ + +.. currentmodule:: riak.util + +.. autofunction:: quacks_like_dict + +.. autofunction:: deep_merge + +.. autofunction:: deprecated + +.. autofunction:: deprecateQuorumAccessors + +.. autoclass:: lazy_property diff --git a/docs/index.rst b/docs/index.rst index 5fb8c0dd..43589f77 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,6 +13,7 @@ Riak Python Client bucket object mapreduce + advanced Installation ------------ diff --git a/riak/benchmark.py b/riak/benchmark.py index c3eade41..7372a68c 100644 --- a/riak/benchmark.py +++ b/riak/benchmark.py @@ -25,9 +25,9 @@ def measure_with_rehearsal(): """ Runs a benchmark when used as an iterator, injecting a garbage - collection between iterations. Example: + collection between iterations. Example:: - for b in benchmark.measure_with_rehearsal(): + for b in riak.benchmark.measure_with_rehearsal(): with b.report("pow"): for _ in range(10000): math.pow(2,10000) @@ -40,9 +40,9 @@ def measure_with_rehearsal(): def measure(): """ - Runs a benchmark once when used as a context manager. Example: + Runs a benchmark once when used as a context manager. Example:: - with benchmark.measure() as b: + with riak.benchmark.measure() as b: with b.report("pow"): for _ in range(10000): math.pow(2,10000) diff --git a/riak/client/multiget.py b/riak/client/multiget.py index 665eba77..1af248d4 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -25,15 +25,17 @@ try: + #: The default size of the worker pool, either based on the number + #: of CPUS or defaulting to 6 POOL_SIZE = cpu_count() except NotImplementedError: # Make an educated guess POOL_SIZE = 6 - +#: A :class:`namedtuple` for tasks that are fed to workers in the +#: multiget pool. Task = namedtuple('Task', ['client', 'outq', 'bucket', 'key', 'options']) - class MultiGetPool(object): """ Encapsulates a pool of fetcher threads. These threads can be used @@ -41,6 +43,11 @@ class MultiGetPool(object): """ def __init__(self, size=POOL_SIZE): + """ + :param size: the desired size of the worker pool + :type size: int + """ + self._inq = Queue() self._size = size self._started = Event() @@ -110,7 +117,10 @@ def __del__(self): def _fetcher(self): """ - The body of the multi-get worker. + The body of the multi-get worker. Loops until + :meth:`_should_quit` returns ``True``, taking tasks off the + input queue, fetching the object, and putting them on the + output queue. """ while not self._should_quit(): task = self._inq.get() @@ -131,21 +141,26 @@ def _should_quit(self): input queue is empty. Once the stop flag is set, new enqueues are disallowed, meaning that the workers can safely drain the queue before exiting. - :rtype boolean + :rtype: boolean """ return self.stopped() and self._inq.empty() +#: The default pool is automatically created and stored in this constant. RIAK_MULTIGET_POOL = MultiGetPool() def multiget(client, keys, **options): """ Executes a parallel-fetch across multiple threads. Returns a list - containing RiakObject instances, or 3-tuples of bucket, key, and - the exception raised. - - :rtype list + containing :class:`~riak.riak_object.RiakObject` instances, or + 3-tuples of bucket, key, and the exception raised. + + :param client: the client to use + :type client: :class:`~riak.client.RiakClient` + :param keys: the bucket/key pairs to fetch in parallel + :type keys: list of two-tuples -- bucket/key pairs + :rtype: list """ outq = Queue() diff --git a/riak/client/transport.py b/riak/client/transport.py index eb2b7481..ccef4813 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -37,6 +37,8 @@ class RiakClientTransport(object): @contextmanager def _transport(self): """ + _transport() + Yields a single transport to the caller from the default pool, without retries. """ diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index ab31d886..fcbc43b1 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -29,18 +29,30 @@ class FeatureDetection(object): + """ + Implements boolean methods that can be checked for the presence of + specific server-side features. Subclasses must implement the + :meth:`_server_version` method to use this functionality, which + should return the server's version as a string. + + :class:`FeatureDetection` is a parent class of + :class:`RiakTransport `. + """ + def _server_version(self): """ Gets the server version from the server. To be implemented by the individual transport class. - :rtype string + + :rtype: string """ raise NotImplementedError def phaseless_mapred(self): """ Whether MapReduce requests can be submitted without phases. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.1] @@ -49,14 +61,15 @@ def pb_indexes(self): Whether secondary index queries are supported over Protocol Buffers - :rtype bool + :rtype: bool """ return self.server_version >= versions[1.2] def pb_search(self): """ Whether search queries are supported over Protocol Buffers - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.2] @@ -64,7 +77,8 @@ def pb_conditionals(self): """ Whether conditional fetch/store semantics are supported over Protocol Buffers - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1] @@ -72,14 +86,16 @@ def quorum_controls(self): """ Whether additional quorums and FSM controls are available, e.g. primary quorums, basic_quorum, notfound_ok - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1] def tombstone_vclocks(self): """ Whether 'not found' responses might include vclocks - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1] @@ -87,7 +103,8 @@ def pb_head(self): """ Whether partial-fetches (vclock and metadata only) are supported over Protocol Buffers - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1] @@ -95,7 +112,8 @@ def pb_clear_bucket_props(self): """ Whether bucket properties can be cleared over Protocol Buffers. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.4] @@ -103,35 +121,40 @@ def pb_all_bucket_props(self): """ Whether all normal bucket properties are supported over Protocol Buffers. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.4] def counters(self): """ Whether CRDT counters are supported. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.4] def bucket_stream(self): """ Whether streaming bucket lists are supported. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.4] def client_timeouts(self): """ Whether client-supplied timeouts are supported. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.4] def stream_indexes(self): """ Whether secondary indexes support streaming responses. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.4] diff --git a/riak/transports/pool.py b/riak/transports/pool.py index 13d5cdf7..b150808a 100644 --- a/riak/transports/pool.py +++ b/riak/transports/pool.py @@ -23,29 +23,33 @@ # This file is a rough port of the Innertube Ruby library class BadResource(StandardError): """ - Users of a Pool should raise this error when the pool element - currently in-use is bad and should be removed from the pool. + Users of a :class:`Pool` should raise this error when the pool + element currently in-use is bad and should be removed from the + pool. """ pass class Element(object): """ - A member of the Pool, a container for the actual resource being - pooled and a marker for whether the resource is currently claimed. + A member of the :class:`Pool`, a container for the actual resource + being pooled and a marker for whether the resource is currently + claimed. """ def __init__(self, obj): """ Creates a new Element, wrapping the passed object as the pooled resource. + :param obj: the resource to wrap :type obj: object """ - """The wrapped pool resource.""" self.object = obj - """Whether the resource is currently in use.""" + """The wrapped pool resource.""" + self.claimed = False + """Whether the resource is currently in use.""" class Pool(object): @@ -59,7 +63,7 @@ class Pool(object): method also allows filtering of the pool and supplying a default value to be used as the resource if no elements are free. - Example: + Example:: from riak.Pool import Pool, BadResource class ListPool(Pool): @@ -80,7 +84,7 @@ def destroy_resource(self): def __init__(self): """ Creates a new Pool. This should be called manually if you - override the __init__ method in a subclass. + override the :meth:`__init__` method in a subclass. """ self.lock = threading.RLock() self.releaser = threading.Condition(self.lock) @@ -89,6 +93,8 @@ def __init__(self): @contextmanager def take(self, _filter=None, default=None): """ + take(_filter=None, default=None) + Claims a resource from the pool for use in a thread-safe, reentrant manner (as part of a with statement). Resources are created as needed when all members of the pool are claimed or @@ -98,7 +104,7 @@ def take(self, _filter=None, default=None): of the pool :type _filter: callable :param default: a value that will be used instead of calling - create_resource if a new resource needs to be created + :meth:`create_resource` if a new resource needs to be created """ if not _filter: def _filter(obj): @@ -151,7 +157,7 @@ def __iter__(self): def clear(self): """ - Removes all resources from the pool, calling delete_element + Removes all resources from the pool, calling :meth:`delete_element` with each one so that the resources are cleaned up. """ for element in self: diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 6a98f2bf..7feb087c 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -28,7 +28,8 @@ class RiakTransport(FeatureDetection): """ - Class to encapsulate transport details + Class to encapsulate transport details and methods. All protocol + transports are subclasses of this class. """ def _get_client_id(self): diff --git a/riak/util.py b/riak/util.py index c8f79777..4b69bfd7 100644 --- a/riak/util.py +++ b/riak/util.py @@ -55,6 +55,9 @@ def deep_merge(a, b): def deprecated(message, stacklevel=3): + """ + Prints a deprecation warning to the console. + """ warnings.warn(message, UserWarning, stacklevel=stacklevel) QUORUMS = ['r', 'pr', 'w', 'dw', 'pw', 'rw'] @@ -125,8 +128,9 @@ def setter(self, value): class lazy_property(object): ''' - meant to be used for lazy evaluation of an object attribute. - property should represent non-mutable data, as it replaces itself. + A method decorator meant to be used for lazy evaluation and + memoization of an object attribute. The property should represent + immutable data, as it replaces itself on first access. ''' def __init__(self, fget): From a6984188839578b7b7c3bcf2e89ebf22e0884800 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 29 Jul 2013 10:48:52 -0500 Subject: [PATCH 082/672] Add query page. * Remove mapreduce page, merge and expand into query page. * Move `RiakLink` to advanced page. * Fix some docstrings in mapreduce and index_page. --- docs/advanced.rst | 6 ++ docs/index.rst | 2 +- docs/mapreduce.rst | 15 --- docs/query.rst | 186 ++++++++++++++++++++++++++++++++++++++ riak/client/index_page.py | 9 +- riak/mapreduce.py | 44 +++++---- 6 files changed, 226 insertions(+), 36 deletions(-) delete mode 100644 docs/mapreduce.rst create mode 100644 docs/query.rst diff --git a/docs/advanced.rst b/docs/advanced.rst index 8bb3a9dc..5bfcfb34 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -98,6 +98,12 @@ Protocol Buffers Transport Utilities --------- +^^^^^^^^^^^^^^^^^^ +Link wrapper class +^^^^^^^^^^^^^^^^^^ + +.. autoclass:: riak.mapreduce.RiakLink + ^^^^^^^^^^^^^^^^^ Multi-valued Dict ^^^^^^^^^^^^^^^^^ diff --git a/docs/index.rst b/docs/index.rst index 43589f77..62ae5571 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,7 +12,7 @@ Riak Python Client client bucket object - mapreduce + query advanced Installation diff --git a/docs/mapreduce.rst b/docs/mapreduce.rst deleted file mode 100644 index 19f5bb57..00000000 --- a/docs/mapreduce.rst +++ /dev/null @@ -1,15 +0,0 @@ -.. ref-mapreduce: - -============= -RiakMapReduce -============= - -.. currentmodule:: riak.mapreduce - -.. autoclass:: riak.mapreduce.RiakMapReduce - -.. autoclass:: riak.mapreduce.RiakMapReducePhase - -.. autoclass:: riak.mapreduce.RiakLinkPhase - -.. autoclass:: riak.mapreduce.RiakLink diff --git a/docs/query.rst b/docs/query.rst new file mode 100644 index 00000000..22fad8e3 --- /dev/null +++ b/docs/query.rst @@ -0,0 +1,186 @@ +============= +Query Methods +============= + +Although most operations you will do involve directly interacting with +known buckets and keys, there are additional ways to get information +out of Riak. + +----------------- +Secondary Indexes +----------------- + +:ref:`Objects ` can be :meth:`tagged +` with :attr:`secondary index +entries `. Those entries can then +be queried over :meth:`the bucket ` +for equality or across ranges.:: + + bucket = client.bucket("index_test") + + # Tag an object with indexes and save + sean = bucket.new("seancribbs") + sean.add_index("fname_bin", "Sean") + sean.add_index("byear_int", 1979) + sean.store() + + # Performs an equality query + seans = bucket.get_index("fname_bin", "Sean") + + # Performs a range query + eighties = bucket.get_index("byear_int", 1980, 1989) + +Secondary indexes are also available via :meth:`MapReduce +`. + +^^^^^^^^^^^^^^^^^^ +Riak 1.4+ Features +^^^^^^^^^^^^^^^^^^ + +.. note:: The features below will raise ``NotImplementedError`` if + requested against a server that does not support them. + +Sometimes the number of results from such a query is too great to +process in one payload, so you can also :meth:`stream the results +`:: + + for keys in bucket.stream_index("bmonth_int", 1): + # keys is a list of matching keys + print keys + +Both the regular :meth:`~riak.bucket.RiakBucket.get_index` method and +the :meth:`~riak.bucket.RiakBucket.stream_index` method allow you to +return the index entry along with the matching key as tuples using the +``return_terms`` option:: + + bucket.get_index("byear_int", 1970, 1990, return_terms=True) + # => [(1979, 'seancribbs')] + +You can also limit the number of results using the ``max_results`` +option, which enables pagination:: + + results = bucket.get_index("fname_bin", "S", "T", max_results=20) + +All of these features are implemented using the +:class:`~riak.client.index_page.IndexPage` class, which emulates a +list but also supports streaming and capturing the +:attr:`~riak.client.index_page.IndexPage.continuation`, which is a +sort of pointer to the next page of results:: + + # Detect whether there are more results + if results.has_next_page(): + + # Fetch the next page of results manually + more = bucket.get_index("fname_bin", "S", "T", max_results=20, + continuation=results.continuation) + + # Fetch the next page of results automatically + more = results.next_page() + +.. currentmodule:: riak.client.index_page + +.. autoclass:: IndexPage + + .. autoattribute:: continuation + .. automethod:: has_next_page + .. automethod:: next_page + .. automethod:: __eq__ + .. automethod:: __iter__ + .. automethod:: __getitem__ + +--------------- +Fulltext Search +--------------- + +If Riak Search is enabled, you can query an index via the bucket's +:meth:`~riak.bucket.RiakBucket.search` method:: + + bucket.enable_search() + bucket.new("one", data={'value':'one'}, + content_type="application/json").store() + + bucket.search('value=one') + +To manually add and remove documents from an index (without an +associated key), use the :class:`~riak.client.RiakClient` +:meth:`~riak.client.RiakClient.fulltext_add` and +:meth:`~riak.client.RiakClient.fulltext_delete` methods directly. + +--------- +MapReduce +--------- + +.. currentmodule:: riak.mapreduce + +:class:`MapReduce` allows you to construct query-processing jobs that +are performed mostly in-parallel around the Riak cluster. You can +think of it as a pipeline, where inputs are fed in one end, they pass +through a number of ``map`` and ``reduce`` phases, and then are +returned to the client. + +^^^^^^^^^^^^^^^^^^^^^^ +Constructing the query +^^^^^^^^^^^^^^^^^^^^^^ + +.. autoclass:: RiakMapReduce + +^^^^^^ +Inputs +^^^^^^ + +The first step is to identify the inputs that should be processed. +They can be: + +#. An entire :meth:`bucket ` +#. An entire bucket, with the :meth:`keys filtered by criteria ` +#. A :meth:`list of bucket/key pairs ` or bucket/key/data triples +#. A :meth:`fulltext search query ` +#. A :meth:`secondary-index query ` + +Adding inputs always returns the ``RiakMapReduce`` object so that you +can chain the construction of the query job. + +.. automethod:: RiakMapReduce.add_bucket +.. automethod:: RiakMapReduce.add_key_filters +.. automethod:: RiakMapReduce.add_key_filter + +.. automethod:: RiakMapReduce.add +.. automethod:: RiakMapReduce.add_object +.. automethod:: RiakMapReduce.add_bucket_key_data + +.. automethod:: RiakMapReduce.search +.. automethod:: RiakMapReduce.index + +^^^^^^ +Phases +^^^^^^ + +The second step is to add processing phases to the query. ``map`` +phases load and process individual keys, returning one or more +results, while ``reduce`` phases operate over collections of results +from previous phases. ``link`` phases are a special type of ``map`` +phase that extract matching :attr:`~riak.riak_object.RiakObject.links` +from the object, usually so they can be used in a subsequent ``map`` +phase. + +Any number of phases can return results directly to the client by +passing ``keep=True``. + +.. automethod:: RiakMapReduce.map +.. automethod:: RiakMapReduce.reduce +.. automethod:: RiakMapReduce.link + +.. autoclass:: RiakMapReducePhase + +.. autoclass:: RiakLinkPhase + +^^^^^^^^^ +Execution +^^^^^^^^^ + +Query results can either be executed in one round-trip, or streamed +back to the client. The format of results will depend on the structure +of the ``map`` and ``reduce`` phases the query contains. + +.. automethod:: RiakMapReduce.run +.. automethod:: RiakMapReduce.stream diff --git a/riak/client/index_page.py b/riak/client/index_page.py index 5314f4db..b4c26e27 100644 --- a/riak/client/index_page.py +++ b/riak/client/index_page.py @@ -43,9 +43,16 @@ def __init__(self, client, bucket, index, startkey, endkey, return_terms, self.return_terms = return_terms self.max_results = max_results self.results = None - self.continuation = None self.stream = False + continuation = None + """ + The opaque page marker that is used when fetching the next chunk + of results. The user can simply call :meth:`next_page` to do so, + or pass this to the :meth:`~riak.client.RiakClient.get_index` + method using the ``continuation`` option. + """ + def __iter__(self): """ Emulates the iterator interface. When streaming, this means diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 5792aab7..17f3eab3 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -21,6 +21,8 @@ from collections import Iterable, namedtuple from riak import RiakError +#: Links are just bucket/key/tag tuples, this class provides a +#: backwards-compatible format: ``RiakLink(bucket, key, tag)`` RiakLink = namedtuple("RiakLink", ("bucket", "key", "tag")) @@ -34,8 +36,9 @@ class RiakMapReduce(object): def __init__(self, client): """ Construct a Map/Reduce object. - :param client: A RiakClient object. - :type client: RiakClient + + :param client: the client that will perform the query + :type client: :class:`~riak.client.RiakClient` """ self._client = client self._phases = [] @@ -57,7 +60,7 @@ def add(self, arg1, arg2=None, arg3=None): :type arg2: string, list, None :param arg3: key data for this input (must be convertible to JSON) :type arg3: string, list, dict, None - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if (arg2 is None) and (arg3 is None): if isinstance(arg1, RiakObject): @@ -73,7 +76,7 @@ def add_object(self, obj): :param obj: the object to add :type obj: RiakObject - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ return self.add_bucket_key_data(obj._bucket._name, obj._key, None) @@ -87,7 +90,7 @@ def add_bucket_key_data(self, bucket, key, data): :type key: string :param data: the key-specific data :type data: string, list, dict, None - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if self._input_mode == 'bucket': raise ValueError('Already added a bucket, can\'t add an object.') @@ -108,7 +111,7 @@ def add_bucket(self, bucket): :param bucket: the bucket :type bucket: string - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ self._input_mode = 'bucket' self._inputs = bucket @@ -120,7 +123,7 @@ def add_key_filters(self, key_filters): :param key_filters: a list of filters :type key_filters: list - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if self._input_mode == 'query': raise ValueError('Key filters are not supported in a query.') @@ -134,7 +137,7 @@ def add_key_filter(self, *args): :param args: a filter :type args: list - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if self._input_mode == 'query': raise ValueError('Key filters are not supported in a query.') @@ -151,7 +154,7 @@ def search(self, bucket, query): :type bucket: string :param query: The search query :type query: string - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ self._input_mode = 'query' self._inputs = {'module': 'riak_search', @@ -173,6 +176,7 @@ def index(self, bucket, index, startkey, endkey=None): :type startkey: string, integer :param endkey: The end key of index range (if doing a range query) :type endkey: string, integer, None + :rtype: :class:`RiakMapReduce` """ self._input_mode = 'query' @@ -192,7 +196,7 @@ def link(self, bucket='_', tag='_', keep=False): Add a link phase to the map/reduce operation. :param bucket: Bucket name (default '_', which means all - buckets) + buckets) :type bucket: string :param tag: Tag (default '_', which means any tag) :type tag: string @@ -200,7 +204,7 @@ def link(self, bucket='_', tag='_', keep=False): the map/reduce. (default False, unless this is the last step in the phase) :type keep: boolean - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ self._phases.append(RiakLinkPhase(bucket, tag, keep)) return self @@ -217,7 +221,7 @@ def map(self, function, options=None): :param options: phase options, containing 'language', 'keep' flag, and/or 'arg'. :type options: dict - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if options is None: options = dict() @@ -245,7 +249,7 @@ def reduce(self, function, options=None): :type function: string, list :param options: phase options, containing 'language', 'keep' flag, and/or 'arg'. - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if options is None: options = dict() @@ -265,8 +269,8 @@ def reduce(self, function, options=None): def run(self, timeout=None): """ Run the map/reduce operation synchronously. Returns a list of - results, or a list of links if the last phase is a - link phase. + results, or a list of links if the last phase is a link phase. + Shortcut for :meth:`riak.client.RiakClient.mapred`. :param timeout: Timeout in milliseconds :type timeout: integer, None @@ -309,11 +313,12 @@ def run(self, timeout=None): def stream(self, timeout=None): """ - Streams the MapReduce query (returns an iterator). + Streams the MapReduce query (returns an iterator). Shortcut + for :meth:`riak.client.RiakClient.stream_mapred`. :param timeout: Timeout in milliseconds :type timeout: integer - :rtype: iterator + :rtype: iterator that yields (phase_num, data) tuples """ query, lrf = self._normalize_query() return self._client.stream_mapred(self._inputs, query, timeout) @@ -569,13 +574,14 @@ class RiakLinkPhase(object): map/reduce operation. Normally you won't need to use this object directly, but instead - call ``link`` on RiakMapReduce objects to add instances to the - query. + call :meth:`RiakMapReduce.link` on RiakMapReduce objects to add + instances to the query. """ def __init__(self, bucket, tag, keep): """ Construct a RiakLinkPhase object. + :param bucket: - The bucket name :type bucket: string :param tag: The tag From 51169f598420e37ed0d691c5944b046f74d12087 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 29 Jul 2013 10:49:18 -0500 Subject: [PATCH 083/672] Tweak layout to put pagers at top and bottom. --- docs/_templates/layout.html | 41 +++++++++++++------------------------ 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 91c92523..31ae631c 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -2,7 +2,8 @@ {% block sidebarrel %}{% endblock %} -{%- block footer %} +{%- block content %} +{{ navBar() }}
    {%- if prev %} @@ -16,34 +17,20 @@ title="{{ _('next chapter') }}">{{ next.title }}{{"→"|safe}} {%- endif %}
-
-
-
-

- Back to top - {% if theme_source_link_position == "footer" %} -
- {% include "sourcelink.html" %} - {% endif %} -

-

- {%- if show_copyright %} - {%- if hasdoc('copyright') %} - {% trans path=pathto('copyright'), copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %}
- {%- else %} - {% trans copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %}
- {%- endif %} - {%- endif %} - {%- if last_updated %} - {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %}
+ {% block body %}{% endblock %} +

-
+ + {%- endblock %} - {% set css_files = css_files + ['_static/custom.css'] %} From 7507fb8f25e629fe6ad0c0723024a0c7628e3693 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 29 Jul 2013 10:50:56 -0500 Subject: [PATCH 084/672] Fix some PEP8 and pyflakes things. --- riak/client/multiget.py | 1 + riak/util.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/riak/client/multiget.py b/riak/client/multiget.py index 1af248d4..ac096189 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -36,6 +36,7 @@ #: multiget pool. Task = namedtuple('Task', ['client', 'outq', 'bucket', 'key', 'options']) + class MultiGetPool(object): """ Encapsulates a pool of fetcher threads. These threads can be used diff --git a/riak/util.py b/riak/util.py index 4b69bfd7..20f6d129 100644 --- a/riak/util.py +++ b/riak/util.py @@ -85,16 +85,16 @@ def __deprecateQuorumAccessor(klass, parent, quorum): if not parent: def direct_getter(self, value=None): deprecated(QDEPMESSAGE % klass.__name__) - if val: - return val + if value: + return value return getattr(self, propname, "default") getter = direct_getter else: def parent_getter(self, value=None): deprecated(QDEPMESSAGE % klass.__name__) - if val: - return val + if value: + return value parentInstance = getattr(self, parent) return getattr(self, propname, getattr(parentInstance, propname, "default")) From d59a5d760bb2cb148042a4c0fff099b4c8b2da29 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 29 Jul 2013 11:14:14 -0500 Subject: [PATCH 085/672] Document some of the mapred shortcut methods. --- docs/query.rst | 35 +++++++++++++++++++++++++++++++++++ riak/riak_object.py | 16 ++++++++-------- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/docs/query.rst b/docs/query.rst index 22fad8e3..40587b3c 100644 --- a/docs/query.rst +++ b/docs/query.rst @@ -151,6 +151,8 @@ can chain the construction of the query job. .. automethod:: RiakMapReduce.search .. automethod:: RiakMapReduce.index +.. autoclass:: RiakKeyFilter + ^^^^^^ Phases ^^^^^^ @@ -174,6 +176,24 @@ passing ``keep=True``. .. autoclass:: RiakLinkPhase +""""""""""""""" +Phase shortcuts +""""""""""""""" + +A number of commonly-used phases are also available as shortcut +methods: + +.. automethod:: RiakMapReduce.map_values +.. automethod:: RiakMapReduce.map_values_json +.. automethod:: RiakMapReduce.reduce_sum +.. automethod:: RiakMapReduce.reduce_min +.. automethod:: RiakMapReduce.reduce_max +.. automethod:: RiakMapReduce.reduce_sort +.. automethod:: RiakMapReduce.reduce_numeric_sort +.. automethod:: RiakMapReduce.reduce_limit +.. automethod:: RiakMapReduce.reduce_slice +.. automethod:: RiakMapReduce.filter_not_found + ^^^^^^^^^ Execution ^^^^^^^^^ @@ -184,3 +204,18 @@ of the ``map`` and ``reduce`` phases the query contains. .. automethod:: RiakMapReduce.run .. automethod:: RiakMapReduce.stream + +^^^^^^^^^^^^^^^^^^^^^ +Shortcut constructors +^^^^^^^^^^^^^^^^^^^^^ + +:class:`~riak.riak_object.RiakObject` contains some shortcut methods +that make it more convenient to begin constructing +:class:`RiakMapReduce` queries. + +.. currentmodule:: riak.riak_object + +.. automethod:: RiakObject.add +.. automethod:: RiakObject.link +.. automethod:: RiakObject.map +.. automethod:: RiakObject.reduce diff --git a/riak/riak_object.py b/riak/riak_object.py index 0c9573a9..7115a9c0 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -354,9 +354,9 @@ def clear(self): def add(self, *args): """ Start assembling a Map/Reduce operation. - A shortcut for :func:`RiakMapReduce.add`. + A shortcut for :meth:`~riak.mapreduce.RiakMapReduce.add`. - :rtype: RiakMapReduce + :rtype: :class:`~riak.mapreduce.RiakMapReduce` """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) @@ -365,9 +365,9 @@ def add(self, *args): def link(self, *args): """ Start assembling a Map/Reduce operation. - A shortcut for :func:`RiakMapReduce.link`. + A shortcut for :meth:`~riak.mapreduce.RiakMapReduce.link`. - :rtype: RiakMapReduce + :rtype: :class:`~riak.mapreduce.RiakMapReduce` """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) @@ -376,9 +376,9 @@ def link(self, *args): def map(self, *args): """ Start assembling a Map/Reduce operation. - A shortcut for :func:`RiakMapReduce.map`. + A shortcut for :meth:`~riak.mapreduce.RiakMapReduce.map`. - :rtype: RiakMapReduce + :rtype: :class:`~riak.mapreduce.RiakMapReduce` """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) @@ -387,9 +387,9 @@ def map(self, *args): def reduce(self, *args): """ Start assembling a Map/Reduce operation. - A shortcut for :func:`RiakMapReduce.reduce`. + A shortcut for :meth:`~riak.mapreduce.RiakMapReduce.reduce`. - :rtype: RiakMapReduce + :rtype: :class:`~riak.mapreduce.RiakMapReduce` """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) From f4d658d92c4491f80eef73085d60b12cb31a08a5 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 29 Jul 2013 11:21:58 -0500 Subject: [PATCH 086/672] Fixup the pager on the index page and point at the tutorial docs. --- docs/_templates/layout.html | 1 + docs/index.rst | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 31ae631c..61243358 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -18,6 +18,7 @@ {%- endif %} {% block body %}{% endblock %} +
 
    {%- if prev %}