From 29d624a1f2217144ff115ad86af8d6a798b4cec0 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 10:40:43 -0500 Subject: [PATCH 01/45] WIP transport refactor. Things are broken, do not use. * Connection manager and monitor are gone. * PBC transport is broken up into multiple files for clarity. * Outline of streaming operations (currently only PBC). * All requests go through the RiakClient object, which handles the connection pools. Accessing the transport directly is now forbidden. * The client can connect to multiple nodes, and each node has an associated error rate (to be used later when implementing error recovery). The node selected for new connections depends on the error rate. TODO: * Break up HTTP into smaller files and add resource detection (for URL generation). * Add streaming operations to HTTP. --- riak/bucket.py | 21 +- riak/client.py | 263 +++++------ riak/client/operations.py | 155 +++++++ riak/mapreduce.py | 135 ++++-- riak/node.py | 87 ++++ riak/riak_object.py | 33 +- riak/search.py | 71 +-- riak/tests/test_all.py | 38 +- riak/tests/test_mapreduce.py | 2 +- riak/transports/connection.py | 192 -------- riak/transports/http.py | 121 +++-- riak/transports/monitor.py | 47 -- riak/transports/pbc.py | 744 ------------------------------ riak/transports/pbc/__init__.py | 43 ++ riak/transports/pbc/codec.py | 143 ++++++ riak/transports/pbc/connection.py | 128 +++++ riak/transports/pbc/messages.py | 48 ++ riak/transports/pbc/stream.py | 86 ++++ riak/transports/pbc/transport.py | 401 ++++++++++++++++ riak/transports/transport.py | 25 +- 20 files changed, 1477 insertions(+), 1306 deletions(-) create mode 100644 riak/client/operations.py create mode 100644 riak/node.py delete mode 100644 riak/transports/connection.py delete mode 100644 riak/transports/monitor.py delete mode 100644 riak/transports/pbc.py create mode 100644 riak/transports/pbc/__init__.py create mode 100644 riak/transports/pbc/codec.py create mode 100644 riak/transports/pbc/connection.py create mode 100644 riak/transports/pbc/messages.py create mode 100644 riak/transports/pbc/stream.py create mode 100644 riak/transports/pbc/transport.py diff --git a/riak/bucket.py b/riak/bucket.py index 0782a2e1..eeed19fa 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -303,8 +303,7 @@ def set_properties(self, props): :param props: An associative array of key:value. :type props: array """ - t = self._client.get_transport() - t.set_bucket_props(self, props) + self._client.set_bucket_props(self, props) def get_properties(self): """ @@ -312,8 +311,7 @@ def get_properties(self): :rtype: array """ - t = self._client.get_transport() - return t.get_bucket_props(self) + return self._client.get_bucket_props(self) def get_keys(self): """ @@ -323,7 +321,17 @@ def get_keys(self): At current, this is a very expensive operation. Use with caution. """ - return self._client.get_transport().get_keys(self) + return self._client.get_keys(self) + + def stream_keys(self): + """ + Return all keys within the bucket. + + .. warning:: + + At current, this is a very expensive operation. Use with caution. + """ + return self._client.stream_keys(self) def new_binary_from_file(self, key, filename): """ @@ -377,5 +385,4 @@ def get_index(self, index, startkey, endkey=None): """ Queries a secondary index over objects in this bucket, returning keys. """ - return self._client._transport.get_index(self.name, index, startkey, - endkey) + return self._client.get_index(self.name, index, startkey, endkey) diff --git a/riak/client.py b/riak/client.py index 483445a0..26d2430b 100644 --- a/riak/client.py +++ b/riak/client.py @@ -1,4 +1,5 @@ """ +Copyright 2011 Basho Technologies, Inc. Copyright 2010 Rusty Klophaus Copyright 2010 Justin Sheehy Copyright 2009 Jay Baird @@ -17,120 +18,140 @@ specific language governing permissions and limitations under the License. """ -# Use json as first choice, simplejson as second choice. + try: import json except ImportError: import simplejson as json +from contextlib import contextmanager + +from riak.client.operations import RiakClientOperations +from riak.node import RiakNode from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduce +from riak.mapreduce import RiakMapReduceChain from riak.search import RiakSearch -from riak.transports import RiakHttpTransport +from riak.transports.http import RiakHttpPool +from riak.transports.pbc import RiakPbcPool from riak.util import deprecated from riak.util import deprecateQuorumAccessors +from riak.util import lazy_property @deprecateQuorumAccessors -class RiakClient(object): +class RiakClient(RiakMapReduceChain, RiakClientOperations): """ - The ``RiakClient`` object holds information necessary to connect to - Riak. The Riak API uses HTTP, so there is no persistent - connection, and the ``RiakClient`` object is extremely lightweight. + The ``RiakClient`` object holds information necessary to connect + to Riak. Requests can be made to Riak directly through the client + or by using the methods on related objects. """ - def __init__(self, host='127.0.0.1', port=8098, prefix='riak', - mapred_prefix='mapred', transport_class=None, - client_id=None, solr_transport_class=None, - transport_options=None): + + PROTOCOLS = ['http', 'https', 'pbc'] + + def __init__(self, protocol='http', transport_options={}, + nodes=None, **unused_args): """ Construct a new ``RiakClient`` object. - :param host: Hostname or IP address - :type host: string - :param port: Port number - :type port: integer - :param prefix: Interface prefix - :type prefix: string - :param mapred_prefix: MapReduce prefix - :type mapred_prefix: string - :param transport_class: transport class to use - :type transport_class: :class:`RiakTransport` - - :param solr_transport_class: HTTP-based transport class for - Solr interface queries - :type solr_transport_class: :class:`RiakHttpTransport` + :param protocol: the preferred protocol, defaults to 'http' + :type protocol: string + :param nodes: a list of node configurations, + where each configuration is a dict containing the keys + 'host', 'http_port', and 'pb_port' + :type nodes: list :param transport_options: Optional key-value args to pass to the transport constuctor :type transport_options: dict """ - if transport_class is None: - transport_class = RiakHttpTransport - - api = getattr(transport_class, 'api', 1) - if api >= 2: - hostports = [(host, port), ] - self._cm = transport_class.default_cm(hostports) - - # If no transport options are provided, then default to the - # empty dict, otherwise just pass through what we are provided. - if transport_options is None: - transport_options = {} - - self._transport = transport_class(self._cm, - prefix=prefix, - mapred_prefix=mapred_prefix, - client_id=client_id, - **transport_options) + if 'port' in unused_args: + deprecated("port option is deprecated, use http_port or pb_port," + + " or the nodes option") + + if 'transport_class' in unused_args: + deprecated( + "transport_class is deprecated, use the protocol option") + + if nodes is None: + self.nodes = [self._create_node(unused_args), ] else: - deprecated('please upgrade the transport to the new API') - self._cm = None - self._transport = transport_class(host, port, client_id=client_id) + self.nodes = [self._create_node(n) for n in nodes] + + self.protocol = protocol or 'http' + + self._http_pool = RiakHttpPool(self, **transport_options) + self._pb_pool = RiakPbcPool(self, **transport_options) self._encoders = {'application/json': json.dumps, 'text/json': json.dumps} self._decoders = {'application/json': json.loads, 'text/json': json.loads} - self._solr = None - self._host = host - self._port = port + self._buckets = {} + + @property + def protocol(self): + return self._protocol + + @property.setter + def protocol(self, value): + if protocol and protocol not in self.PROTOCOLS: + raise ValueError("protocol option is invalid, must be one of %s" % + repr(self.PROTOCOLS)) + self._protocol = value def get_transport(self): """ Get the transport instance the client is using for it's connection. """ - return self._transport + deprecated("get_transport is deprecated, use client, " + + "bucket, or object methods instead") + return None def get_client_id(self): """ Get the ``client_id`` for this ``RiakClient`` instance. + DEPRECATED :rtype: string """ - return self._transport.get_client_id() + deprecated( + "``get_client_id`` is deprecated, use the ``client_id`` property") + return self.client_id def set_client_id(self, client_id): """ Set the client_id for this ``RiakClient`` instance. - .. warning:: - - Refer to - http://wiki.basho.com/Client-Implementation-Guide.html#Client-IDs - for information on how to set the client_id. - :param client_id: The new client_id. :type client_id: string - :rtype: self """ - self._transport.set_client_id(client_id) + deprecated( + "``set_client_id`` is deprecated, use the ``client_id`` property") + self.client_id = client_id return self + @property + def client_id(self): + """ + The client ID for this client instance + + :rtype: string + """ + with self.transport() as transport: + return transport.get_client_id() + + @client_id.setter + def client_id(self, client_id): + for http in self._http_pool: + http.client_id = client_id + for pb in self._pb_pool: + pb.client_id = client_id + def get_encoder(self, content_type): """ Get the encoding function for the provided content type. """ - if content_type in self._encoders: - return self._encoders[content_type] + return self._encoders.get(content_type) def set_encoder(self, content_type, encoder): """ @@ -140,14 +161,12 @@ def set_encoder(self, content_type, encoder): :type encoder: function """ self._encoders[content_type] = encoder - return self def get_decoder(self, content_type): """ Get the decoding function for the provided content type. """ - if content_type in self._decoders: - return self._decoders[content_type] + return self._decoders.get(content_type) def set_decoder(self, content_type, decoder): """ @@ -157,15 +176,6 @@ def set_decoder(self, content_type, decoder): :type decoder: function """ self._decoders[content_type] = decoder - return self - - def get_buckets(self): - """ - Get the list of buckets. - NOTE: Do not use this in production, as it requires traversing through - all keys stored in a cluster. - """ - return self._transport.get_buckets() def bucket(self, name): """ @@ -174,83 +184,48 @@ def bucket(self, name): :rtype: :class:`RiakBucket ` """ - return RiakBucket(self, name) - - def is_alive(self): - """ - Check if the Riak server for this ``RiakClient`` instance is alive. - - :rtype: boolean - """ - return self._transport.ping() - - def add(self, *args): - """ - Start assembling a Map/Reduce operation. A shortcut for - :func:`RiakMapReduce.add`. - - :rtype: :class:`RiakMapReduce` - """ - mr = RiakMapReduce(self) - return apply(mr.add, args) - - def search(self, *args): - """ - Start assembling a Map/Reduce operation based on search - results. This command will return an error unless executed - against a Riak Search cluster. A shortcut for - :func:`RiakMapReduce.search`. - - :rtype: :class:`RiakMapReduce` - """ - mr = RiakMapReduce(self) - return apply(mr.search, args) - - def index(self, *args): - """ - Start assembling a Map/Reduce operation based on secondary - index query results. - - :rtype: :class:`RiakMapReduce` - """ - mr = RiakMapReduce(self) - return apply(mr.index, args) - - def link(self, *args): - """ - Start assembling a Map/Reduce operation. A shortcut for - :func:`RiakMapReduce.link`. + if name in self._buckets: + return self._buckets[name] + else: + bucket = RiakBucket(self, name) + self._buckets[name] = bucket + return bucket - :rtype: :class:`RiakMapReduce` + @lazy_property + def solr(self): """ - mr = RiakMapReduce(self) - return apply(mr.link, args) - - def map(self, *args): + Returns a RiakSearch object which can access search indexes. """ - Start assembling a Map/Reduce operation. A shortcut for - :func:`RiakMapReduce.map`. + return RiakSearch(self) - :rtype: :class:`RiakMapReduce` - """ - mr = RiakMapReduce(self) - return apply(mr.map, args) + def _create_node(self, n): + if isinstance(n, RiakNode): + return n + elif isinstance(n, tuple) and len(n) is 3: + host, http_port, pb_port = n + return RiakNode(host=host, + http_port=http_port, + pb_port=pb_port) + elif isinstance(n, dict): + return RiakNode(**n) + else: + raise TypeError("%s is not a valid node configuration" + % repr(n)) - def reduce(self, *args): + def _choose_node(self, nodes=self.nodes): """ - Start assembling a Map/Reduce operation. A shortcut for - :func:`RiakMapReduce.reduce`. - - :rtype: :class:`RiakMapReduce` + Chooses a random node from the list of nodes in the client, + taking into account each node's recent error rate. + :rtype RiakNode """ - mr = RiakMapReduce(self) - return apply(mr.reduce, args) - - def get_index(self, bucket, index, startkey, endkey=None): - return self._transport.get_index(bucket, index, startkey, endkey) + # Prefer nodes which have gone a reasonable time without + # errors + def _error_rate(node): + return node.error_rate.value() + good = [n for n in nodes if _error_rate(n) < 0.1] - def solr(self): - if self._solr is None: - self._solr = RiakSearch(self, host=self._host, port=self._port) - - return self._solr + if len(good) is 0: + # Fall back to a minimally broken node + return min(nodes, key=_error_rate) + else: + return random.choice(good) diff --git a/riak/client/operations.py b/riak/client/operations.py new file mode 100644 index 00000000..f7ede8a6 --- /dev/null +++ b/riak/client/operations.py @@ -0,0 +1,155 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from riak.bucket import RiakBucket + + +class RiakClientOperations(object): + """ + Methods for RiakClient that result in requests sent to the Riak + cluster. + """ + + def get_buckets(self): + """ + Get the list of buckets as RiakBucket instances. + NOTE: Do not use this in production, as it requires traversing through + all keys stored in a cluster. + """ + with self.transport() as transport: + return [RiakBucket(self, name) for name in transport.get_buckets()] + + def ping(self): + """ + Check if the Riak server for this ``RiakClient`` instance is alive. + + :rtype: boolean + """ + with self.transport() as transport: + return transport.ping() + + is_alive = ping + + def get_index(self, bucket, index, startkey, endkey=None): + """ + Queries a secondary index, returning matching keys. + """ + with self.transport() as transport: + return transport.get_index(bucket, index, startkey, endkey) + + def get_bucket_props(self, bucket): + """ + Fetches bucket properties for the given bucket. + """ + with self.transport() as transport: + return transport.get_bucket_props(bucket) + + def set_bucket_props(self, bucket, props): + """ + Sets bucket properties for the given bucket. + """ + with self.transport() as transport: + return transport.set_bucket_props(bucket, props) + + def get_keys(self, bucket): + """ + Lists all keys in a bucket. + """ + with self.transport() as transport: + return transport.get_keys(bucket) + + def stream_keys(self, bucket): + """ + Lists all keys in a bucket via a stream. This is a generator + method which should be iterated over. + """ + with self.transport() as transport: + for keylist in return transport.stream_keys(bucket): + yield keylist + + def put(self, robj, w=None, dw=None, pw=None, return_body=None, + if_none_match=None): + """ + Stores an object in the Riak cluster. + """ + with self.transport() as transport: + return transport.put(robj, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) + + def put_new(self, robj, w=None, dw=None, pw=None, return_body=None, + if_none_match=None): + """ + Stores an object in the Riak cluster with a generated key. + """ + with self.transport() as transport: + return transport.put_new(robj, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) + + def get(self, robj, r=None, pr=None, vtag=None): + """ + Fetches the contents of a Riak object. + """ + with self.transport() as transport: + return transport.get(robj, r=r, pr=pr, vtag=vtag) + + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + """ + Deletes an object from Riak. + """ + with self.transport() as transport: + return transport.delete(robj, rw=rw, r=r, w=w, dw=dw=, pr=pr, + pw=pw) + + def mapred(self, inputs, query, timeout): + """ + Executes a MapReduce query + """ + with self.transport() as transport: + return transport.mapred(inputs, query, timeout) + + def stream_mapred(self, inputs, query, timeout): + """ + Streams a MapReduce query as (phase, data) pairs. This is a + generator method which should be iterated over. + """ + with self.transport() as transport: + for phase, data in transport.stream_mapred(inputs, query, timeout): + yield phase, data + + def fulltext_search(self, index, query, **params): + """ + Performs a full-text search query. + """ + with self.transport() as transport: + return transport.search(index, query, **params) + + def fulltext_add(self, index, docs): + """ + Adds documents to the full-text index. + """ + with self._http_pool.take() as transport: + transport.fulltext_add(self, index, docs) + + def fulltext_delete(self, index, docs=None, queries=None): + """ + Removes documents from the full-text index. + """ + with self._http_pool.take() as transport: + transport.fulltext_delete(index, docs, queries) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 7279b0a0..6db573be 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -202,15 +202,45 @@ def run(self, timeout=None): @param integer timeout - Timeout in milliseconds. @return array() """ + query, link_results_flag = self._normalize_query() + + result = self.client.mapred(self._inputs, query, timeout) + + # If the last phase is NOT a link phase, then return the result. + if not (link_results_flag + or isinstance(self._phases[-1], RiakLinkPhase)): + return result + + # If there are no results, then return an empty list. + if result == None: + return [] + + # Otherwise, if the last phase IS a link phase, then convert the + # results to RiakLink objects. + a = [] + for r in result: + if (len(r) == 2): + link = RiakLink(r[0], r[1]) + elif (len(r) == 3): + link = RiakLink(r[0], r[1], r[2]) + link._client = self._client + a.append(link) + + return a + + def stream(self, timeout=None): + """ + Streams the MapReduce query (returns an iterator). + """ + query, lrf = self._normalize_query() + return self.client.stream_mapred(self._inputs, query, timeout) + + def _normalize_query(self): num_phases = len(self._phases) - # If there are no phases, then just echo the inputs back to the user. - if (num_phases == 0): - self.reduce(["riak_kv_mapreduce", "reduce_identity"]) - num_phases = 1 + # If there are no phases, return the keys as links + if num_phases is 0: link_results_flag = True - else: - link_results_flag = False # Convert all phases to associative arrays. Also, # if none of the phases are accumulating, then set the last one to @@ -236,30 +266,7 @@ def run(self, timeout=None): self._inputs = {'bucket': bucket_name, 'key_filters': self._key_filters} - t = self._client.get_transport() - result = t.mapred(self._inputs, query, timeout) - - # If the last phase is NOT a link phase, then return the result. - if not (link_results_flag - or isinstance(self._phases[-1], RiakLinkPhase)): - return result - - # If there are no results, then return an empty list. - if result == None: - return [] - - # Otherwise, if the last phase IS a link phase, then convert the - # results to RiakLink objects. - a = [] - for r in result: - if (len(r) == 2): - link = RiakLink(r[0], r[1]) - elif (len(r) == 3): - link = RiakLink(r[0], r[1], r[2]) - link._client = self._client - a.append(link) - - return a + return query, link_results_flag ## # Start Shortcuts to built-ins @@ -546,3 +553,71 @@ def function(*args): def __iter__(self): return iter(self._filters) + + +class RiakMapReduceChain(object): + """ + Mixin to add chaining from the client object directly into a + MapReduce operation. + """ + def add(self, *args): + """ + Start assembling a Map/Reduce operation. A shortcut for + :func:`RiakMapReduce.add`. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.add, args) + + def search(self, *args): + """ + Start assembling a Map/Reduce operation based on search + results. This command will return an error unless executed + against a Riak Search cluster. A shortcut for + :func:`RiakMapReduce.search`. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.search, args) + + def index(self, *args): + """ + Start assembling a Map/Reduce operation based on secondary + index query results. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.index, args) + + def link(self, *args): + """ + Start assembling a Map/Reduce operation. A shortcut for + :func:`RiakMapReduce.link`. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.link, args) + + def map(self, *args): + """ + Start assembling a Map/Reduce operation. A shortcut for + :func:`RiakMapReduce.map`. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.map, args) + + def reduce(self, *args): + """ + Start assembling a Map/Reduce operation. A shortcut for + :func:`RiakMapReduce.reduce`. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.reduce, args) diff --git a/riak/node.py b/riak/node.py new file mode 100644 index 00000000..e9307f6a --- /dev/null +++ b/riak/node.py @@ -0,0 +1,87 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" +import math +import time +from threading import RLock +from riak.util import deprecated + + +class Decaying(object): + """ + A float value which decays exponentially toward 0 over time. This + is used internally to select nodes for new connections that have + had the least errors within the recent period. + """ + + def __init__(self, p=0.0, e=math.e, r=None): + """ + Creates a new decaying error counter. + + :param p: the initial value (defaults to 0.0) + :type p: float + :param e: the exponent base (defaults to math.e) + :type e: float + :param r: timescale factor (defaults to decaying 50% over 10 + seconds, i.e. log(0.5) / 10) + :type r: float + """ + self.p = p + self.e = e + self.r = r or (math.log(0.5) / 10) + self.lock = RLock() + self.t0 = time.time() + + def incr(self, d): + """ + Increases the value by the argument. + """ + with self.lock: + self.p = self.value() + d + + def value(self): + """ + Returns the current value (adjusted for the time decay) + """ + with self.lock: + now = time.time() + dt = now - self.t0 + self.t0 = now + self.p = self.p * (math.pow(self.e, self.r * dt)) + return self.p + + +class RiakNode(object): + """ + The internal representation of a Riak node to which the client can + connect. Encapsulates both the configuration for the node and + error tracking used for node-selection. + """ + + def __init__(self, host='127.0.0.1', http_port=8098, pb_port=8087, + **unused_args): + """ + Creates a node. + """ + + if 'port' in unused_args: + deprecated("port option is deprecated, use http_port or pb_port") + + self.host = host + self.http_port = http_port + self.pb_port = pb_port + self.error_rate = Decaying() diff --git a/riak/riak_object.py b/riak/riak_object.py index 4964ae49..78cf21bb 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -18,7 +18,8 @@ under the License. """ import copy -from metadata import * +from riak.metadata import * +from riak.mapreduce import * from riak import RiakError from riak.riak_index_entry import RiakIndexEntry @@ -125,8 +126,8 @@ def _get_usermeta(self): def _set_usermeta(self, usermeta): self.metadata[MD_USERMETA] = usermeta return self - - usermeta = property(_get_usermeta, _set_usermeta, + + usermeta = property(_get_usermeta, _set_usermeta, doc=""" The custom user metadata on this object. This doesn't include things like content type and links, but only @@ -148,7 +149,8 @@ def add_index(self, field, value): :rtype: self """ if field[-4:] not in ("_bin", "_int"): - raise RiakError("Riak 2i fields must end with either '_bin' or '_int'.") + raise RiakError( + "Riak 2i fields must end with either '_bin' or '_int'.") rie = RiakIndexEntry(field, value) if not rie in self.metadata[MD_INDEX]: @@ -367,19 +369,21 @@ def store(self, w=None, dw=None, pw=None, return_body=True, "store one of the siblings instead") # Issue the put over our transport - t = self.client.get_transport() + # t = self.client.get_transport() if self.key is None: - key, vclock, metadata = t.put_new(self, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) + key, vclock, metadata = self.client.put_new( + self, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) self.exists = True self.key = key self.vclock = vclock self.metadata = metadata else: - result = t.put(self, w=w, dw=dw, pw=pw, return_body=return_body, - if_none_match=if_none_match) + result = self.client.put(self, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) if result is not None and result != ('', []): self._populate(result) @@ -397,8 +401,7 @@ def reload(self, r=None, pr=None, vtag=None): :rtype: self """ - t = self.client.get_transport() - result = t.get(self, r=r, pr=pr, vtag=vtag) + result = self.client.get(self, r=r, pr=pr, vtag=vtag) self.clear() if result is not None and result != ('', []): @@ -432,8 +435,8 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): :type pw: integer :rtype: self """ - t = self.client.get_transport() - result = t.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) + + result = self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) self.clear() return self @@ -576,5 +579,3 @@ def reduce(self, params): mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) return apply(mr.reduce, params) - -from mapreduce import * diff --git a/riak/search.py b/riak/search.py index 8a3e9a91..1d8fe914 100644 --- a/riak/search.py +++ b/riak/search.py @@ -1,85 +1,22 @@ from riak.transports import RiakHttpTransport from xml.etree import ElementTree -from xml.dom.minidom import Document class RiakSearch(object): - def __init__(self, client, transport_class=None, - host="127.0.0.1", port=8098): - if transport_class is None: - transport_class = RiakHttpTransport - - api = getattr(transport_class, 'api', 1) - if api >= 2: - hostports = [(host, port), ] - self._cm = transport_class.default_cm(hostports) - self._transport = transport_class(self._cm, prefix="/solr") - else: - # The old code which attempted to use api==1 would actually - # throw a NameError, so it was obviously never used. We will - # simply raise an error here, intead of a gentle warning. - raise DeprecationWarning('please upgrade the transport to the ' - 'new API') - + def __init__(self, client, **unused_args): self._client = client - self._decoders = {"text/xml": ElementTree.fromstring} - - def get_decoder(self, content_type): - decoder = (self._client.get_decoder(content_type) - or self._decoders[content_type]) - if not decoder: - decoder = self.decode - - return decoder - - def decode(self, data): - return data def add(self, index, *docs): - xml = Document() - root = xml.createElement('add') - for doc in docs: - doc_element = xml.createElement('doc') - for key, value in doc.iteritems(): - field = xml.createElement('field') - field.setAttribute("name", key) - text = xml.createTextNode(value) - field.appendChild(text) - doc_element.appendChild(field) - root.appendChild(doc_element) - xml.appendChild(root) - - url = "/solr/%s/update" % index - self._transport.post_request(uri=url, body=xml.toxml(), - content_type="text/xml") + self._client.fulltext_add(index, docs) index = add def delete(self, index, docs=None, queries=None): - xml = Document() - root = xml.createElement('delete') - if docs: - for doc in docs: - doc_element = xml.createElement('id') - text = xml.createTextNode(doc) - doc_element.appendChild(text) - root.appendChild(doc_element) - if queries: - for query in queries: - query_element = xml.createElement('query') - text = xml.createTextNode(query) - query_element.appendChild(text) - root.appendChild(query_element) - - xml.appendChild(root) - - url = "/solr/%s/update" % index - self._transport.post_request(uri=url, body=xml.toxml(), - content_type="text/xml") + self._client.fulltext_delete(index, docs, queries) remove = delete def search(self, index, query, **params): - return self._client._transport.search(index, query, **params) + return self._client.fulltext_search(index, query, **params) select = search diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 5693d797..fe21c6af 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -56,16 +56,24 @@ class BaseTestCase(object): + host = None + pb_port = None + http_port = None + @staticmethod def randint(): return random.randint(1, 999999) - def create_client(self, host=None, port=None, transport_class=None): - host = host or self.host - port = port or self.port - transport_class = transport_class or self.transport_class - return RiakClient(self.host, self.port, - transport_class=self.transport_class) + def create_client(self, host=None, http_port=None, pb_port=None, + protocol=None, **client_args): + host = host or self.host or HOST + http_port = http_port or self.http_port or HTTP_PORT + pb_port = pb_port or self.pb_port or PB_PORT + protocol = protocol or self.protocol + return RiakClient(protocol=protocol, + host=host, + http_port=http_port, + pb_port=pb_port, **client_args) def setUp(self): self.client = self.create_client() @@ -94,21 +102,18 @@ def setUp(self): if not HAVE_PROTO: self.skipTest('protobuf is unavailable') self.host = PB_HOST - self.port = PB_PORT - self.transport_class = RiakPbcTransport + self.pb_port = PB_PORT + self.protocol = 'pbc' super(RiakPbcTransportTestCase, self).setUp() def test_uses_client_id_if_given(self): - self.host = PB_HOST - self.port = PB_PORT zero_client_id = "\0\0\0\0" - c = RiakClient(PB_HOST, PB_PORT, - transport_class=RiakPbcTransport, - client_id=zero_client_id) + c = self.create_client(client_id=zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) def test_close_underlying_socket_fails(self): - c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport) + self.skipTest("TODO: No longer using connection manager, replace") + c = self.create_client() bucket = c.bucket('bucket_test_close') rand = self.randint() @@ -130,6 +135,7 @@ def test_close_underlying_socket_fails(self): self.assertRaises(socket.error, bucket.get, 'foo') def test_close_underlying_socket_retry(self): + self.skipTest("TODO: No longer using bare transport, replace") c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport, transport_options={"max_attempts": 2}) @@ -183,8 +189,8 @@ class RiakHttpTransportTestCase(BasicKVTests, def setUp(self): self.host = HTTP_HOST - self.port = HTTP_PORT - self.transport_class = RiakHttpTransport + self.http_port = HTTP_PORT + self.protocol = 'http' super(RiakHttpTransportTestCase, self).setUp() def test_no_returnbody(self): diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 23df62e2..f8293f8c 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -103,7 +103,7 @@ def test_client_exceptional_paths(self): with self.assertRaises(ValueError): mr = self.client.search('bucket', 'fleh') mr.add_key_filter("tokenize", "-", 1) - + class JSMapReduceTests(object): def test_javascript_source_map(self): diff --git a/riak/transports/connection.py b/riak/transports/connection.py deleted file mode 100644 index d26b85f4..00000000 --- a/riak/transports/connection.py +++ /dev/null @@ -1,192 +0,0 @@ -""" -Copyright 2011 Greg Stein - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -import httplib -import socket -import contextlib -import functools - - -class ConnectionManager(object): - - # Must be constructable with: connection_class(host, port) - # Must have two attribute: host and port - # Must have a close() method - connection_class = None - - def __init__(self, hostports=[]): - # We want a private copy of this list: either to detach the argument - # default, or to detach from the caller's list. - self.hostports = hostports[:] - - # Open a connection to each specified host/port. On single-threaded - # systems, this will create a round-robin across all specified servers. - # When multi-threaded, this will give us an initial set for all the - # threads to work with (and more will be created, according to demand). - self.conns = [self.connection_class(host, port) - for host, port in hostports] - - def add_hostport(self, host, port): - self.hostports.append((host, port)) - - # Open an initial connection. For single-threaded, this adds to the - # round-robin pool. On multi-threaded, it simply gives us an extra - # connectiong for the load-balancing across the servers. - self.conns.append(self.connection_class(host, port)) - - def remove_host(self, host, port=None): - if port is None: - self.hostports = [(h, p) for h, p in self.hostports - if h != host] - else: - self.hostports.remove((host, port)) - - # Now that the host/port pair has been removed from self.hostports, - # no connections on this pair will be added in .giveback(). Thus, the - # existing connections are all that may exist at this time. We'll - # snapshot the list, and look for offending connections, then try and - # remove them, being wary that race conditions may remove them before - # we can remove it. - for conn in self.conns[:]: - if conn.host == host and (port is None or conn.port == port): - try: - self.conns.remove(conn) - except ValueError: - # Another thread removed the connection. It won't - # be coming back, so we have nothing to do here. - pass - else: - # If the connection was still present (no - # ValueError), then we should go ahead and close - # it down. - conn.close() - - # Just in case somebody uses a host/port combo and typos... - remove_hostport = remove_host - - def take(self): - if len(self.conns) == 0: - # RACE: in a multi-threaded environment, a conn might arrive in - # self.conns, but... no biggy. If we're bouncing up against - # needing a new connection, then we'll just create one. - return self._new_connection() - - # RACE: self.conns might empty out right now, so we need to protect - # our access to it. - try: - # round-robin: take from the front, we'll append when it comes back - return self.conns.pop(0) - except IndexError: - return self._new_connection() - - def giveback(self, conn): - # Connections using a host/port pair that is NOT in self.hostports - # should be ignored. Likely, remove_host() was called while this - # connection was borrowed for some work. - if (conn.host, conn.port) in self.hostports: - self.conns.append(conn) - else: - # Proactively close the connection. The caller won't know whether - # we put it into our list, or left the connection for the caller - # to deal with (and close). - conn.close() - - @contextlib.contextmanager - def withconn(self): - conn = self.take() - try: - yield conn - finally: - self.giveback(conn) - - def _new_connection(self): - if len(self.hostports) == 0: - raise NoHostsDefined() - - # Grab the first host/port combo. We'll put this at the end, so that - # we do a round-robin on the host/port pairs. - host, port = self.hostports[0] - conn = self.connection_class(host, port) - - if len(self.hostports) == 1: - # No rotation needed. - return conn - - # Be careful about rotating. We want to append before - # removing, so that we never hit a len==0 race condition - # (which could prevent the creation of needed connections). - self.hostports.append((host, port)) - - # RACE: another thread may have appended the same host/port - # pair. We will add another pair. Each thread will remove - # one (either [0], or one that had been appened), resulting - # in a correct state of a single pair in the list. - # RACE: another thread may get the host/port pair from - # hostports[0] before we have a chance to remove it. We - # don't need precision round-robin behavior; just something - # close. - # RACE: another thread may have removed hostports[0] (which we - # are also trying to remove), but it will have placed - # another copy at the end before doing so. We have also - # added a host/port pair, and will remove one, leaving the - # list in a correct state. - self.hostports.remove((host, port)) - - return conn - - -class Socket(object): - - def __init__(self, host, port): - self.host = host - self.port = port - - self.sock = None - - def maybe_connect(self): - if self.sock is None: - self.sock = s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - try: - s.connect((self.host, self.port)) - except: - self.close() - raise - - def close(self): - if self.sock is not None: - self.sock.close() - self.sock = None - - -class FactoryConnectionManager(ConnectionManager): - - def __init__(self, connection_class, hostports=[]): - self.connection_class = connection_class - ConnectionManager.__init__(self, hostports) - - -def cm_using(connection_class): - return functools.partial(FactoryConnectionManager, connection_class) - -HTTPConnectionManager = cm_using(httplib.HTTPConnection) -SocketConnectionManager = cm_using(Socket) - - -class NoHostsDefined(Exception): - pass diff --git a/riak/transports/http.py b/riak/transports/http.py index 58ac88b4..b8be3d27 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -17,7 +17,6 @@ specific language governing permissions and limitations under the License. """ -from __future__ import with_statement import urllib import re @@ -31,68 +30,71 @@ except ImportError: import simplejson as json -from transport import RiakTransport +from riak.transports.transport import RiakTransport +from riak.transports.pool import Pool, BadResource from riak.metadata import * from riak.mapreduce import RiakLink from riak import RiakError from riak.riak_index_entry import RiakIndexEntry from riak.multidict import MultiDict -from connection import HTTPConnectionManager import riak.util from xml.etree import ElementTree +from xml.dom.minidom import Document + # subtract length of "Link: " header string and newline MAX_LINK_HEADER_SIZE = 8192 - 8 -class RiakHttpTransport(RiakTransport): +class RiakHttpPool(Pool): """ - The RiakHttpTransport object holds information necessary to - connect to Riak. The Riak API uses HTTP, so there is no persistent - connection, and the RiakClient object is extremely lightweight. + A pool of HTTP(S) transport connections. """ + def __init__(self, client, **options): + self.client = client + self.transport_options = options + if client.protocol is 'https': + self.connection_class = httplib.HTTPConnection + else: + self.connection_class = httplib.HTTPSConnection + super(RiakHttpPool, self).__init__() - # We're using the new RiakTransport API - api = 2 + def create_resource(self): + node = self.client.choose_node() + return RiakHttpTransport(node=node, + client=self.client, + connection_class=self.connection_class, + **self.options) - # The ConnectionManager class that this transport prefers. - default_cm = HTTPConnectionManager + def destroy_resource(self, transport): + transport.close() + + +class RiakHttpTransport(RiakTransport): + """ + The RiakHttpTransport object holds information necessary to + connect to Riak via HTTP. + """ - # How many times to retry a request - RETRY_COUNT = 3 + api = 3 - def __init__(self, cm, - prefix='riak', mapred_prefix='mapred', client_id=None, + def __init__(self, node=None, + client=None, + connection_class=httplib.HTTPConnection + client_id=None, **unused_options): """ - Construct a new RiakClient object. - @param string host - Hostname or IP address (default '127.0.0.1') - @param int port - Port number (default 8098) - @param string prefix - Interface prefix (default 'riak') - @param string mapred_prefix - MapReduce prefix (default 'mapred') - @param string client_id - client id to use for vector clocks + Construct a new HTTP connection to Riak. """ super(RiakHttpTransport, self).__init__() - self._conns = cm - self._prefix = prefix - self._mapred_prefix = mapred_prefix + + self._client = client + self._node = node + self._connection_class = connection_class self._client_id = client_id if not self._client_id: self._client_id = self.make_random_client_id() - def __copy__(self): - ### not implemented right now - raise Exception('not implemented') - ### we don't have _host and _port. will fix after some refactoring... - return RiakHttpTransport(self._host, self._port, self._prefix, - self._mapred_prefix) - - def set_client_id(self, client_id): - self._client_id = client_id - - def get_client_id(self): - return self._client_id - def ping(self): """ Check server is alive over HTTP @@ -362,6 +364,51 @@ def search(self, index, query, **params): else: raise ValueError("Could not decode search response") + def fulltext_add(self, index, docs): + """ + Adds documents to the search index. + """ + xml = Document() + root = xml.createElement('add') + for doc in docs: + doc_element = xml.createElement('doc') + for key in doc: + value = doc[key] + field = xml.createElement('field') + field.setAttribute("name", key) + text = xml.createTextNode(value) + field.appendChild(text) + doc_element.appendChild(field) + root.appendChild(doc_element) + xml.appendChild(root) + + url = "/solr/%s/update" % index + self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") + + def fulltext_delete(self, index, docs=None, queries=None): + """ + Removes documents from the full-text index. + """ + xml = Document() + root = xml.createElement('delete') + if docs: + for doc in docs: + doc_element = xml.createElement('id') + text = xml.createTextNode(doc) + doc_element.appendChild(text) + root.appendChild(doc_element) + if queries: + for query in queries: + query_element = xml.createElement('query') + text = xml.createTextNode(query) + query_element.appendChild(text) + root.appendChild(query_element) + + xml.appendChild(root) + + url = "/solr/%s/update" % index + self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") + def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] if not status in expected_statuses: diff --git a/riak/transports/monitor.py b/riak/transports/monitor.py deleted file mode 100644 index c6223518..00000000 --- a/riak/transports/monitor.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Copyright 2011 Greg Stein - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -import threading -import time - - -class Monitor(object): - - def __init__(self, cm, transport): - self._cm = cm - self._transport = transport - - self._stop_loop = False - self._thread = threading.Thread(target=self._run) - - self._periodic = 0.050 # 50 msec - - def start(self): - self._thread.start() - - def terminate(self): - self._stop_loop = True - self._thread.join() - - def _run(self): - while not self._stop_loop: - ### look for changes in the ring servers - - ### see if some offline servers came back - - time.sleep(self._periodic) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py deleted file mode 100644 index cfef37dd..00000000 --- a/riak/transports/pbc.py +++ /dev/null @@ -1,744 +0,0 @@ -""" -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" -from __future__ import with_statement - -import errno -import socket -import struct -try: - import json -except ImportError: - import simplejson as json - -from riak import RiakError -from riak.mapreduce import RiakLink -from riak.metadata import ( - MD_CHARSET, - MD_CTYPE, - MD_ENCODING, - MD_INDEX, - MD_LASTMOD, - MD_LASTMOD_USECS, - MD_LINKS, - MD_USERMETA, - MD_VTAG, - ) -from riak.riak_index_entry import RiakIndexEntry -from riak.transports import connection -from riak.transports.transport import RiakTransport -import riak.util - -try: - import riak_pb -except ImportError: - riak_pb = None - -## Protocol codes -MSG_CODE_ERROR_RESP = 0 -MSG_CODE_PING_REQ = 1 -MSG_CODE_PING_RESP = 2 -MSG_CODE_GET_CLIENT_ID_REQ = 3 -MSG_CODE_GET_CLIENT_ID_RESP = 4 -MSG_CODE_SET_CLIENT_ID_REQ = 5 -MSG_CODE_SET_CLIENT_ID_RESP = 6 -MSG_CODE_GET_SERVER_INFO_REQ = 7 -MSG_CODE_GET_SERVER_INFO_RESP = 8 -MSG_CODE_GET_REQ = 9 -MSG_CODE_GET_RESP = 10 -MSG_CODE_PUT_REQ = 11 -MSG_CODE_PUT_RESP = 12 -MSG_CODE_DEL_REQ = 13 -MSG_CODE_DEL_RESP = 14 -MSG_CODE_LIST_BUCKETS_REQ = 15 -MSG_CODE_LIST_BUCKETS_RESP = 16 -MSG_CODE_LIST_KEYS_REQ = 17 -MSG_CODE_LIST_KEYS_RESP = 18 -MSG_CODE_GET_BUCKET_REQ = 19 -MSG_CODE_GET_BUCKET_RESP = 20 -MSG_CODE_SET_BUCKET_REQ = 21 -MSG_CODE_SET_BUCKET_RESP = 22 -MSG_CODE_MAPRED_REQ = 23 -MSG_CODE_MAPRED_RESP = 24 -MSG_CODE_INDEX_REQ = 25 -MSG_CODE_INDEX_RESP = 26 -MSG_CODE_SEARCH_QUERY_REQ = 27 -MSG_CODE_SEARCH_QUERY_RESP = 28 - -RIAKC_RW_ONE = 4294967294 -RIAKC_RW_QUORUM = 4294967293 -RIAKC_RW_ALL = 4294967292 -RIAKC_RW_DEFAULT = 4294967291 - -# These are a specific set of socket errors -# that could be raised on send/recv that indicate -# that the socket is closed or reset, and is not -# usable. On seeing any of these errors, the socket -# should be closed, and the connection re-established. -CONN_CLOSED_ERRORS = ( - errno.EHOSTUNREACH, - errno.ECONNRESET, - errno.EBADF, - errno.EPIPE - ) - - -class SocketWithId(connection.Socket): - def __init__(self, host, port): - super(SocketWithId, self).__init__(host, port) - self.last_client_id = None - - def maybe_connect(self): - # If we're going to establish a new connection, then reset the last - # client_id used on this connection. - if self.sock is None: - self.last_client_id = None - super(SocketWithId, self).maybe_connect() - - def send(self, pkt): - try: - self.sock.sendall(pkt) - except socket.error, e: - # If the socket is in a bad state, close it and allow it - # to re-connect on the next try - if e[0] in CONN_CLOSED_ERRORS: - self.close() - raise - - def recv(self, want_len): - try: - res = self.sock.recv(want_len) - - # Assume the socket is closed if no data is - # returned on a blocking read. - if len(res) == 0 and want_len > 0: - self.close() - - return res - - except socket.error, e: - # If the socket is in a bad state, close it and allow it - # to re-connect on the next try - if e[0] in CONN_CLOSED_ERRORS: - self.close() - raise - - -class RiakPbcTransport(RiakTransport): - """ - The RiakPbcTransport object holds a connection to the protocol - buffers interface on the riak server. - """ - - # We're using the new RiakTransport API - api = 2 - - rw_names = { - 'default': RIAKC_RW_DEFAULT, - 'all': RIAKC_RW_ALL, - 'quorum': RIAKC_RW_QUORUM, - 'one': RIAKC_RW_ONE - } - - # The ConnectionManager class that this transport prefers. - default_cm = connection.cm_using(SocketWithId) - - def __init__(self, cm, client_id=None, max_attempts=1, **unused_options): - """ - Construct a new RiakPbcTransport object. - """ - if riak_pb is None: - raise RiakError("this transport is not available (no protobuf)") - - super(RiakPbcTransport, self).__init__() - - self._cm = cm - self._client_id = client_id - self._max_attempts = max_attempts - - # FeatureDetection API - def _server_version(self): - return self.get_server_info()['server_version'] - - def translate_rw_val(self, rw): - val = self.rw_names.get(rw) - if val is None: - return rw - return val - - def __copy__(self): - return RiakPbcTransport(self._cm, self._client_id) - - def ping(self): - """ - Ping the remote server - @return boolean - """ - # An expected response code of None implies "any response is valid". - msg_code, msg = self.send_msg_code(MSG_CODE_PING_REQ, None) - if msg_code == MSG_CODE_PING_RESP: - return 1 - else: - return 0 - - def get_server_info(self): - """ - Get information about the server - """ - msg_code, resp = self.send_msg_code(MSG_CODE_GET_SERVER_INFO_REQ, - MSG_CODE_GET_SERVER_INFO_RESP) - return {'node': resp.node, 'server_version': resp.server_version} - - def get_client_id(self): - """ - Get the client id used by this connection - """ - msg_code, resp = self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ, - MSG_CODE_GET_CLIENT_ID_RESP) - return resp.client_id - - def set_client_id(self, client_id): - """ - Set the client id used by this connection - """ - req = riak_pb.RpbSetClientIdReq() - req.client_id = client_id - - msg_code, resp = self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req, - MSG_CODE_SET_CLIENT_ID_RESP) - - # Using different client_id values across connections is a bad idea - # since you never know which connection you might use for a given - # API call. Setting the client_id manually (rather than as part of - # the transport construction) can be error-prone since the connection - # could drop and be reinstated using self._client_id. - # - # To minimize the potential impact of variant client_id values across - # connections, we'll store this new client_id and use it for all - # future connections. - self._client_id = client_id - - return True - - def get(self, robj, r=None, pr=None, vtag=None): - """ - Serialize get request and deserialize response - """ - if vtag is not None: - raise RiakError("PB transport does not support vtags") - - bucket = robj.bucket - - req = riak_pb.RpbGetReq() - if r: - req.r = self.translate_rw_val(r) - if self.quorum_controls() and pr: - req.pr = self.translate_rw_val(pr) - - if self.tombstone_vclocks(): - req.deletedvclock = 1 - - req.bucket = bucket.name - req.key = robj.key - - # An expected response code of None implies "any response is valid". - msg_code, resp = self.send_msg(MSG_CODE_GET_REQ, req, None) - if msg_code == MSG_CODE_GET_RESP: - contents = [] - for c in resp.content: - contents.append(self.decode_content(c)) - return resp.vclock, contents - else: - return None - - def put(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """ - Serialize get request and deserialize response - """ - bucket = robj.bucket - - req = riak_pb.RpbPutReq() - if w: - req.w = self.translate_rw_val(w) - if dw: - req.dw = self.translate_rw_val(dw) - if self.quorum_controls() and pw: - req.pw = self.translate_rw_val(pw) - - if return_body: - req.return_body = 1 - if if_none_match: - req.if_none_match = 1 - - req.bucket = bucket.name - req.key = robj.key - vclock = robj.vclock - if vclock: - req.vclock = vclock - - self.pbify_content(robj.metadata, - robj.get_encoded_data(), - req.content) - - msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req, - MSG_CODE_PUT_RESP) - if resp is not None: - contents = [] - for c in resp.content: - contents.append(self.decode_content(c)) - return resp.vclock, contents - - def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """Put a new object into the Riak store, returning its (new) key. - - If return_meta is False, then the vlock and metadata return values - will be None. - - @return (key, vclock, metadata) - """ - # Note that this won't work on 0.14 nodes. - bucket = robj.bucket - - req = riak_pb.RpbPutReq() - if w: - req.w = self.translate_rw_val(w) - if dw: - req.dw = self.translate_rw_val(dw) - if self.quorum_controls() and pw: - req.pw = self.translate_rw_val(pw) - - if return_body: - req.return_body = 1 - if if_none_match: - req.if_none_match = 1 - - req.bucket = bucket.name - - self.pbify_content(robj.metadata, - robj.get_encoded_data(), - req.content) - - msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req, - MSG_CODE_PUT_RESP) - if not resp: - raise RiakError("missing response object") - if len(resp.content) != 1: - raise RiakError("siblings were returned from object creation") - - metadata, content = self.decode_content(resp.content[0]) - return resp.key, resp.vclock, metadata - - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): - """ - Serialize get request and deserialize response - """ - bucket = robj.bucket - - req = riak_pb.RpbDelReq() - if rw: - req.rw = self.translate_rw_val(rw) - if r: - req.r = self.translate_rw_val(r) - if w: - req.w = self.translate_rw_val(w) - if dw: - req.dw = self.translate_rw_val(dw) - - if self.quorum_controls(): - if pr: - req.pr = self.translate_rw_val(pr) - if pw: - req.pw = self.translate_rw_val(pw) - - if self.tombstone_vclocks() and robj.vclock: - req.vclock = robj.vclock - - req.bucket = bucket.name - req.key = robj.key - - msg_code, resp = self.send_msg(MSG_CODE_DEL_REQ, req, - MSG_CODE_DEL_RESP) - return self - - def get_keys(self, bucket): - """ - Lists all keys within a bucket. - """ - req = riak_pb.RpbListKeysReq() - req.bucket = bucket.name - - keys = [] - - def _handle_response(resp): - for key in resp.keys: - keys.append(key) - self.send_msg_multi(MSG_CODE_LIST_KEYS_REQ, req, - MSG_CODE_LIST_KEYS_RESP, _handle_response) - - return keys - - def get_buckets(self): - """ - Serialize bucket listing request and deserialize response - """ - msg_code, resp = self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ, - MSG_CODE_LIST_BUCKETS_RESP) - return resp.buckets - - def get_bucket_props(self, bucket): - """ - Serialize bucket property request and deserialize response - """ - req = riak_pb.RpbGetBucketReq() - req.bucket = bucket.name - - msg_code, resp = self.send_msg(MSG_CODE_GET_BUCKET_REQ, req, - MSG_CODE_GET_BUCKET_RESP) - props = {} - if resp.props.HasField('n_val'): - props['n_val'] = resp.props.n_val - if resp.props.HasField('allow_mult'): - props['allow_mult'] = resp.props.allow_mult - - return props - - def set_bucket_props(self, bucket, props): - """ - Serialize set bucket property request and deserialize response - """ - req = riak_pb.RpbSetBucketReq() - req.bucket = bucket.name - for key in props: - if key not in ['n_val', 'allow_mult']: - raise NotImplementedError - - if 'n_val' in props: - req.props.n_val = props['n_val'] - if 'allow_mult' in props: - req.props.allow_mult = props['allow_mult'] - - msg_code, resp = self.send_msg(MSG_CODE_SET_BUCKET_REQ, req, - MSG_CODE_SET_BUCKET_RESP) - return self - - def mapred(self, inputs, query, timeout=None): - # Construct the job, optionally set the timeout... - job = {'inputs': inputs, 'query': query} - if timeout is not None: - job['timeout'] = timeout - - content = json.dumps(job) - - req = riak_pb.RpbMapRedReq() - req.request = content - req.content_type = "application/json" - - # dictionary of phase results - each content should be an encoded array - # which is appended to the result for that phase. - result = {} - - def _handle_response(resp): - if resp.HasField("phase") and resp.HasField("response"): - content = json.loads(resp.response) - if resp.phase in result: - result[resp.phase] += content - else: - result[resp.phase] = content - self.send_msg_multi(MSG_CODE_MAPRED_REQ, req, MSG_CODE_MAPRED_RESP, - _handle_response) - - # If a single result - return the same as the HTTP interface does - # otherwise return all the phase information - if not len(result): - return None - elif len(result) == 1: - return result[max(result.keys())] - else: - return result - - def get_index(self, bucket, index, startkey, endkey=None): - if not self.pb_indexes(): - return self._get_index_mapred_emu(bucket, index, startkey, endkey) - - req = riak_pb.RpbIndexReq(bucket=bucket, index=index) - if endkey: - req.qtype = riak_pb.RpbIndexReq.range - req.range_min = str(startkey) - req.range_max = str(endkey) - else: - req.qtype = riak_pb.RpbIndexReq.eq - req.key = str(startkey) - - msg_code, resp = self.send_msg(MSG_CODE_INDEX_REQ, req, - MSG_CODE_INDEX_RESP) - return resp.keys - - def search(self, index, query, **params): - if not self.pb_search(): - return self._search_mapred_emu(index, query) - - req = riak_pb.RpbSearchQueryReq(index=index, q=query) - if 'rows' in params: - req.rows = params['rows'] - if 'start' in params: - req.start = params['start'] - if 'sort' in params: - req.sort = params['sort'] - if 'filter' in params: - req.filter = params['filter'] - if 'df' in params: - req.df = params['df'] - if 'op' in params: - req.op = params['op'] - if 'q.op' in params: - req.op = params['q.op'] - if 'fl' in params: - if isinstance(params['fl'], list): - req.fl.extend(params['fl']) - else: - req.fl.append(params['fl']) - if 'presort' in params: - req.presort = params['presort'] - - msg_code, resp = self.send_msg(MSG_CODE_SEARCH_QUERY_REQ, req, - MSG_CODE_SEARCH_QUERY_RESP) - - result = {} - if resp.HasField('max_score'): - result['max_score'] = resp.max_score - if resp.HasField('num_found'): - result['num_found'] = resp.num_found - docs = [] - for doc in resp.docs: - resultdoc = {} - for pair in doc.fields: - resultdoc[pair.key] = pair.value - docs.append(resultdoc) - result['docs'] = docs - return result - - def send_msg_code(self, msg_code, expect): - with self._cm.withconn() as conn: - self.send_pkt(conn, struct.pack("!iB", 1, msg_code)) - return self.recv_msg(conn, expect) - - def encode_msg(self, msg_code, msg): - str = msg.SerializeToString() - slen = len(str) - hdr = struct.pack("!iB", 1 + slen, msg_code) - return hdr + str - - def send_msg(self, msg_code, msg, expect): - with self._cm.withconn() as conn: - self.send_pkt(conn, self.encode_msg(msg_code, msg)) - if msg_code == MSG_CODE_SET_CLIENT_ID_REQ: - conn.last_client_id = self._client_id - return self.recv_msg(conn, expect) - - def send_msg_multi(self, msg_code, msg, expect, handler): - with self._cm.withconn() as conn: - self.send_pkt(conn, self.encode_msg(msg_code, msg)) - while True: - msg_code, resp = self.recv_msg(conn, expect) - handler(resp) - if resp.HasField("done") and resp.done: - break - - def send_pkt(self, conn, pkt): - attempt = 0 - e = None - for attempt in xrange(self._max_attempts): - e = None - try: - conn.maybe_connect() - - # If the last client_id used on this connection is - # different than our client_id, then set a new ID on - # the connection. - if conn.last_client_id != self._client_id: - req = riak_pb.RpbSetClientIdReq() - req.client_id = self._client_id - conn.send(self.encode_msg(MSG_CODE_SET_CLIENT_ID_REQ, req)) - conn.last_client_id = self._client_id - self.recv_msg(conn, MSG_CODE_SET_CLIENT_ID_RESP) - - conn.send(pkt) - break - except socket.error, e: - # If this is some unknown socket error bail out - # instead of retrying - if e[0] not in CONN_CLOSED_ERRORS: - raise - - # Max attempts reached, raise whatever exception we are getting - if attempt + 1 == self._max_attempts and e is not None: - raise e - - def recv_msg(self, conn, expect): - self.recv_pkt(conn) - msg_code, = struct.unpack("B", self._inbuf[:1]) - if msg_code == MSG_CODE_ERROR_RESP: - msg = riak_pb.RpbErrorResp() - msg.ParseFromString(self._inbuf[1:]) - raise Exception(msg.errmsg) - elif msg_code == MSG_CODE_PING_RESP: - msg = None - elif msg_code == MSG_CODE_GET_SERVER_INFO_RESP: - msg = riak_pb.RpbGetServerInfoResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_GET_CLIENT_ID_RESP: - msg = riak_pb.RpbGetClientIdResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_SET_CLIENT_ID_RESP: - msg = None - elif msg_code == MSG_CODE_GET_RESP: - msg = riak_pb.RpbGetResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_PUT_RESP: - msg = riak_pb.RpbPutResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_DEL_RESP: - msg = None - elif msg_code == MSG_CODE_LIST_KEYS_RESP: - msg = riak_pb.RpbListKeysResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_LIST_BUCKETS_RESP: - msg = riak_pb.RpbListBucketsResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_GET_BUCKET_RESP: - msg = riak_pb.RpbGetBucketResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_SET_BUCKET_RESP: - msg = None - elif msg_code == MSG_CODE_MAPRED_RESP: - msg = riak_pb.RpbMapRedResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_INDEX_RESP: - msg = riak_pb.RpbIndexResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_SEARCH_QUERY_RESP: - msg = riak_pb.RpbSearchQueryResp() - msg.ParseFromString(self._inbuf[1:]) - else: - raise Exception("unknown msg code %s" % msg_code) - if expect and msg_code != expect: - raise RiakError("unexpected protocol buffer message code: %d" - % msg_code) - return msg_code, msg - - def recv_pkt(self, conn): - nmsglen = conn.recv(4) - if len(nmsglen) != 4: - raise RiakError( - "Socket returned short packet length %d - expected 4" - % len(nmsglen)) - msglen, = struct.unpack('!i', nmsglen) - self._inbuf_len = msglen - self._inbuf = '' - while len(self._inbuf) < msglen: - want_len = min(8192, msglen - len(self._inbuf)) - recv_buf = conn.recv(want_len) - if not recv_buf: - break - self._inbuf += recv_buf - if len(self._inbuf) != self._inbuf_len: - raise RiakError("Socket returned short packet %d - expected %d" - % (len(self._inbuf), self._inbuf_len)) - - def decode_contents(self, rpb_contents): - contents = [] - for rpb_c in rpb_contents: - contents.append(self.decode_content(rpb_c)) - return contents - - def decode_content(self, rpb_content): - metadata = {} - if rpb_content.HasField("deleted"): - metadata[MD_DELETED] = True - if rpb_content.HasField("content_type"): - metadata[MD_CTYPE] = rpb_content.content_type - if rpb_content.HasField("charset"): - metadata[MD_CHARSET] = rpb_content.charset - if rpb_content.HasField("content_encoding"): - metadata[MD_ENCODING] = rpb_content.content_encoding - if rpb_content.HasField("vtag"): - metadata[MD_VTAG] = rpb_content.vtag - links = [] - for link in rpb_content.links: - if link.HasField("bucket"): - bucket = link.bucket - else: - bucket = None - if link.HasField("key"): - key = link.key - else: - key = None - if link.HasField("tag"): - tag = link.tag - else: - tag = None - links.append(RiakLink(bucket, key, tag)) - if links: - metadata[MD_LINKS] = links - if rpb_content.HasField("last_mod"): - metadata[MD_LASTMOD] = rpb_content.last_mod - if rpb_content.HasField("last_mod_usecs"): - metadata[MD_LASTMOD_USECS] = rpb_content.last_mod_usecs - usermeta = {} - for usermd in rpb_content.usermeta: - usermeta[usermd.key] = usermd.value - if len(usermeta) > 0: - metadata[MD_USERMETA] = usermeta - indexes = [] - for index in rpb_content.indexes: - rie = RiakIndexEntry(index.key, index.value) - indexes.append(rie) - if len(indexes) > 0: - metadata[MD_INDEX] = indexes - return metadata, rpb_content.value - - def pbify_content(self, metadata, data, rpb_content): - # Convert the broken out fields, building up - # pbmetadata for any unknown ones - for k, v in metadata.iteritems(): - if k == MD_CTYPE: - rpb_content.content_type = v - elif k == MD_CHARSET: - rpb_content.charset = v - elif k == MD_ENCODING: - rpb_content.charset = v - elif k == MD_USERMETA: - for uk, uv in v.iteritems(): - pair = rpb_content.usermeta.add() - pair.key = uk - pair.value = uv - elif k == MD_INDEX: - for rie in v: - pair = rpb_content.indexes.add() - pair.key = rie.get_field() - pair.value = rie.get_value() - elif k == MD_LINKS: - for link in v: - pb_link = rpb_content.links.add() - pb_link.bucket = link.get_bucket() - pb_link.key = link.get_key() - pb_link.tag = link.get_tag() - rpb_content.value = data diff --git a/riak/transports/pbc/__init__.py b/riak/transports/pbc/__init__.py new file mode 100644 index 00000000..d3620738 --- /dev/null +++ b/riak/transports/pbc/__init__.py @@ -0,0 +1,43 @@ +""" +Copyright 2012 Basho Technologies, Inc. +Copyright 2010 Rusty Klophaus +Copyright 2010 Justin Sheehy +Copyright 2009 Jay Baird + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + + +from riak.transports.pool import Pool +from riak.transports.pbc.transport import RiakPbcTransport + + +class RiakPbcPool(Pool): + """ + A resource pool of PBC transports. + """ + def __init__(self, client, **options): + super(RiakPbcPool, self).__init__() + self._client = client + self._options = options + + def create_resource(self): + node = self._client._choose_node() + return RiakPbcTransport(node=node, + client=self._client, + **self._options) + + def destroy_resource(self, pbc): + pbc.close() diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py new file mode 100644 index 00000000..dca7d2f0 --- /dev/null +++ b/riak/transports/pbc/codec.py @@ -0,0 +1,143 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" +from riak.metadata import ( + MD_CHARSET, + MD_CTYPE, + MD_ENCODING, + MD_INDEX, + MD_LASTMOD, + MD_LASTMOD_USECS, + MD_LINKS, + MD_USERMETA, + MD_VTAG, + ) + +try: + import riak_pb +except ImportError: + riak_pb = None + +RIAKC_RW_ONE = 4294967294 +RIAKC_RW_QUORUM = 4294967293 +RIAKC_RW_ALL = 4294967292 +RIAKC_RW_DEFAULT = 4294967291 + + +class RiakPbcCodec(object): + """ + Protobuffs Encoding and decoding methods for RiakPbcTransport. + """ + + rw_names = { + 'default': RIAKC_RW_DEFAULT, + 'all': RIAKC_RW_ALL, + 'quorum': RIAKC_RW_QUORUM, + 'one': RIAKC_RW_ONE + } + + def __init__(self, **unused_args): + if riak_pb is None: + raise NotImplementedError("this transport is not available") + super(RiakPbcCodec, self).__init__(**unused_args) + + def translate_rw_val(self, rw): + val = self.rw_names.get(rw) + if val is None: + return rw + elif type(rw) is int and rw >= 0: + return val + else: + return None + + def decode_contents(self, rpb_contents): + return [self.decode_content(rpb_c) for rpb_c in rpb_contents] + + def decode_content(self, rpb_content): + metadata = {} + if rpb_content.HasField("deleted"): + metadata[MD_DELETED] = True + if rpb_content.HasField("content_type"): + metadata[MD_CTYPE] = rpb_content.content_type + if rpb_content.HasField("charset"): + metadata[MD_CHARSET] = rpb_content.charset + if rpb_content.HasField("content_encoding"): + metadata[MD_ENCODING] = rpb_content.content_encoding + if rpb_content.HasField("vtag"): + metadata[MD_VTAG] = rpb_content.vtag + links = [] + for link in rpb_content.links: + if link.HasField("bucket"): + bucket = link.bucket + else: + bucket = None + if link.HasField("key"): + key = link.key + else: + key = None + if link.HasField("tag"): + tag = link.tag + else: + tag = None + links.append(RiakLink(bucket, key, tag)) + if links: + metadata[MD_LINKS] = links + if rpb_content.HasField("last_mod"): + metadata[MD_LASTMOD] = rpb_content.last_mod + if rpb_content.HasField("last_mod_usecs"): + metadata[MD_LASTMOD_USECS] = rpb_content.last_mod_usecs + usermeta = {} + for usermd in rpb_content.usermeta: + usermeta[usermd.key] = usermd.value + if len(usermeta) > 0: + metadata[MD_USERMETA] = usermeta + indexes = [] + for index in rpb_content.indexes: + rie = RiakIndexEntry(index.key, index.value) + indexes.append(rie) + if len(indexes) > 0: + metadata[MD_INDEX] = indexes + return metadata, rpb_content.value + + def encode_content(self, metadata, data, rpb_content): + # Convert the broken out fields, building up + # pbmetadata for any unknown ones + for k in metadata: + v = metadata[k] + if k == MD_CTYPE: + rpb_content.content_type = v + elif k == MD_CHARSET: + rpb_content.charset = v + elif k == MD_ENCODING: + rpb_content.charset = v + elif k == MD_USERMETA: + for uk in v: + pair = rpb_content.usermeta.add() + pair.key = uk + pair.value = v[uk] + elif k == MD_INDEX: + for rie in v: + pair = rpb_content.indexes.add() + pair.key = rie.get_field() + pair.value = rie.get_value() + elif k == MD_LINKS: + for link in v: + pb_link = rpb_content.links.add() + pb_link.bucket = link.get_bucket() + pb_link.key = link.get_key() + pb_link.tag = link.get_tag() + rpb_content.value = data diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py new file mode 100644 index 00000000..b08b46ba --- /dev/null +++ b/riak/transports/pbc/connection.py @@ -0,0 +1,128 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import errno +import socket +import struct +from contextlib import contextmanager + +# These are a specific set of socket errors +# that could be raised on send/recv that indicate +# that the socket is closed or reset, and is not +# usable. On seeing any of these errors, the socket +# should be closed, and the connection re-established. +CONN_CLOSED_ERRORS = ( + errno.EHOSTUNREACH, + errno.ECONNRESET, + errno.EBADF, + errno.EPIPE + ) + + +class RiakPbcConnection(object): + """ + Connection-related methods for RiakPbcTransport. + """ + def _encode_msg(self, msg_code, msg=None): + if msg is None: + return struct.pack("!iB", 1, msg_code) + msgstr = msg.SerializeToString() + slen = len(msgstr) + hdr = struct.pack("!iB", 1 + slen, msg_code) + return hdr + msgstr + + def _request(self, msg_code, msg=None, expect=None): + self._send_msg(msg_code, msg) + return self._recv_msg(expect) + + def _send_msg(self, msg_code, msg): + self._socket.send(self.encode_msg(msg_code, msg)) + + def _recv_msg(self, expect=None): + self._recv_pkt() + msg_code, = struct.unpack("B", self._inbuf[:1]) + if msg_code == MSG_CODE_ERROR_RESP: + msg = riak_pb.RpbErrorResp() + msg.ParseFromString(self._inbuf[1:]) + raise Exception(msg.errmsg) + elif msg_code == MSG_CODE_PING_RESP: + msg = None + elif msg_code == MSG_CODE_GET_SERVER_INFO_RESP: + msg = riak_pb.RpbGetServerInfoResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_GET_CLIENT_ID_RESP: + msg = riak_pb.RpbGetClientIdResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_SET_CLIENT_ID_RESP: + msg = None + elif msg_code == MSG_CODE_GET_RESP: + msg = riak_pb.RpbGetResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_PUT_RESP: + msg = riak_pb.RpbPutResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_DEL_RESP: + msg = None + elif msg_code == MSG_CODE_LIST_KEYS_RESP: + msg = riak_pb.RpbListKeysResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_LIST_BUCKETS_RESP: + msg = riak_pb.RpbListBucketsResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_GET_BUCKET_RESP: + msg = riak_pb.RpbGetBucketResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_SET_BUCKET_RESP: + msg = None + elif msg_code == MSG_CODE_MAPRED_RESP: + msg = riak_pb.RpbMapRedResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_INDEX_RESP: + msg = riak_pb.RpbIndexResp() + msg.ParseFromString(self._inbuf[1:]) + elif msg_code == MSG_CODE_SEARCH_QUERY_RESP: + msg = riak_pb.RpbSearchQueryResp() + msg.ParseFromString(self._inbuf[1:]) + else: + raise Exception("unknown msg code %s" % msg_code) + if expect and msg_code != expect: + raise RiakError("unexpected protocol buffer message code: %d" + % msg_code) + return msg_code, msg + + def _recv_pkt(self): + nmsglen = self._socket.recv(4) + if len(nmsglen) != 4: + raise RiakError( + "Socket returned short packet length %d - expected 4" + % len(nmsglen)) + msglen, = struct.unpack('!i', nmsglen) + self._inbuf_len = msglen + self._inbuf = '' + while len(self._inbuf) < msglen: + want_len = min(8192, msglen - len(self._inbuf)) + recv_buf = conn.recv(want_len) + if not recv_buf: + break + self._inbuf += recv_buf + if len(self._inbuf) != self._inbuf_len: + raise RiakError("Socket returned short packet %d - expected %d" + % (len(self._inbuf), self._inbuf_len)) + + def close(self): + self._socket.shutdown(socket.SHUT_RDWR) diff --git a/riak/transports/pbc/messages.py b/riak/transports/pbc/messages.py new file mode 100644 index 00000000..c8175867 --- /dev/null +++ b/riak/transports/pbc/messages.py @@ -0,0 +1,48 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +## Protocol codes +MSG_CODE_ERROR_RESP = 0 +MSG_CODE_PING_REQ = 1 +MSG_CODE_PING_RESP = 2 +MSG_CODE_GET_CLIENT_ID_REQ = 3 +MSG_CODE_GET_CLIENT_ID_RESP = 4 +MSG_CODE_SET_CLIENT_ID_REQ = 5 +MSG_CODE_SET_CLIENT_ID_RESP = 6 +MSG_CODE_GET_SERVER_INFO_REQ = 7 +MSG_CODE_GET_SERVER_INFO_RESP = 8 +MSG_CODE_GET_REQ = 9 +MSG_CODE_GET_RESP = 10 +MSG_CODE_PUT_REQ = 11 +MSG_CODE_PUT_RESP = 12 +MSG_CODE_DEL_REQ = 13 +MSG_CODE_DEL_RESP = 14 +MSG_CODE_LIST_BUCKETS_REQ = 15 +MSG_CODE_LIST_BUCKETS_RESP = 16 +MSG_CODE_LIST_KEYS_REQ = 17 +MSG_CODE_LIST_KEYS_RESP = 18 +MSG_CODE_GET_BUCKET_REQ = 19 +MSG_CODE_GET_BUCKET_RESP = 20 +MSG_CODE_SET_BUCKET_REQ = 21 +MSG_CODE_SET_BUCKET_RESP = 22 +MSG_CODE_MAPRED_REQ = 23 +MSG_CODE_MAPRED_RESP = 24 +MSG_CODE_INDEX_REQ = 25 +MSG_CODE_INDEX_RESP = 26 +MSG_CODE_SEARCH_QUERY_REQ = 27 +MSG_CODE_SEARCH_QUERY_RESP = 28 diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py new file mode 100644 index 00000000..a71f6c20 --- /dev/null +++ b/riak/transports/pbc/stream.py @@ -0,0 +1,86 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +try: + import json +except ImportError: + import simplejson as json + +from riak.transports.pbc.messages import MSG_CODE_LIST_KEYS_RESP +from riak.transports.pbc.messages import MSG_CODE_MAPRED_RESP + + +class RiakPbcStream(class): + """ + Used internally by RiakPbcTransport to implement streaming + operations. Implements the iterator interface. + """ + def __init__(self, transport): + self.transport = transport + + def __iter__(self): + return self + + def next(self): + expect = self._expect + try: + resp = self.transport._recv_msg(expect) + if(self._is_done(resp)): + raise StopIteration + else: + return resp + except StopIteration: + pass + except: + # TODO: which exceptions do we expect to be generated? + # Should we raise BadResource? + raise StopIteration + + def _is_done(self, response): + raise NotImplementedError + + +class RiakPbcKeyStream(RiakPbcStream): + """ + Used internally by RiakPbcTransport to implement key-list streams. + """ + + _expect = MSG_CODE_LIST_KEYS_RESP + + def next(self): + response = super(RiakPbcKeyStream, self).__next__() + return response.keys + + def _is_done(self, response): + return response.done + + +class RiakPbcMapredStream(RiakPbcStream): + """ + Used internally by RiakPbcTransport to implement MapReduce + streams. + """ + + _expect = MSG_CODE_MAPRED_RESP + + def next(self): + response = super(RiakPbcMapredStream, self).next() + return (response.phase, json.loads(response.response)) + + def _is_done(self, response): + return response.done diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py new file mode 100644 index 00000000..90a00b40 --- /dev/null +++ b/riak/transports/pbc/transport.py @@ -0,0 +1,401 @@ +""" +Copyright 2012 Basho Technologies, Inc. +Copyright 2010 Rusty Klophaus +Copyright 2010 Justin Sheehy +Copyright 2009 Jay Baird + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +try: + import json +except ImportError: + import simplejson as json + +from riak import RiakError +from riak.mapreduce import RiakLink +from riak.riak_index_entry import RiakIndexEntry +from riak.transports.transport import RiakTransport +from riak.transports.pbc.connection import RiakPbcConnection +from riak.transports.pbc.stream import RiakPbcKeyStream, RiakPbcMapredStream +from riak.transports.pbc.codec import RiakPbcCodec +from riak.transports.pbc.messages import * +import riak.util + + +class RiakPbcTransport(RiakTransport, RiakPbcConnection, RiakPbcCodec): + """ + The RiakPbcTransport object holds a connection to the protocol + buffers interface on the riak server. + """ + + api = 3 + + def __init__(self, node=None, client=None, connect_timeout=None, + request_timeout=None, **unused_options): + """ + Construct a new RiakPbcTransport object. + """ + super(RiakPbcTransport, self).__init__() + + self._client = client + self._node = node + self._address = (node.host, node.pb_port) + self._timeouts = {'connect': connect_timeout, + 'request': request_timeout} + + # FeatureDetection API + def _server_version(self): + return self.get_server_info()['server_version'] + + def ping(self): + """ + Ping the remote server + @return boolean + """ + + msg_code, msg = self._request(MSG_CODE_PING_REQ) + if msg_code == MSG_CODE_PING_RESP: + return 1 + else: + return 0 + + def get_server_info(self): + """ + Get information about the server + """ + msg_code, resp = self._request(MSG_CODE_GET_SERVER_INFO_REQ, + expect=MSG_CODE_GET_SERVER_INFO_RESP) + return {'node': resp.node, 'server_version': resp.server_version} + + @property + def client_id(self): + """the client ID for this connection""" + msg_code, resp = self._request(MSG_CODE_GET_CLIENT_ID_REQ, + expect=MSG_CODE_GET_CLIENT_ID_RESP) + return resp.client_id + + @client_id.setter + def client_id(self, client_id): + req = riak_pb.RpbSetClientIdReq() + req.client_id = client_id + + msg_code, resp = self._request(MSG_CODE_SET_CLIENT_ID_REQ, req, + MSG_CODE_SET_CLIENT_ID_RESP) + + self._client_id = client_id + + def get(self, robj, r=None, pr=None, vtag=None): + """ + Serialize get request and deserialize response + """ + if vtag is not None: + raise RiakError("PB transport does not support vtags") + + bucket = robj.bucket + + req = riak_pb.RpbGetReq() + if r: + req.r = self.translate_rw_val(r) + if self.quorum_controls() and pr: + req.pr = self.translate_rw_val(pr) + + if self.tombstone_vclocks(): + req.deletedvclock = 1 + + req.bucket = bucket.name + req.key = robj.key + + msg_code, resp = self._request(MSG_CODE_GET_REQ, req) + if msg_code == MSG_CODE_GET_RESP: + contents = [] + for c in resp.content: + contents.append(self.decode_content(c)) + return resp.vclock, contents + else: + return None + + def put(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): + """ + Serialize get request and deserialize response + """ + bucket = robj.bucket + + req = riak_pb.RpbPutReq() + if w: + req.w = self.translate_rw_val(w) + if dw: + req.dw = self.translate_rw_val(dw) + if self.quorum_controls() and pw: + req.pw = self.translate_rw_val(pw) + + if return_body: + req.return_body = 1 + if if_none_match: + req.if_none_match = 1 + + req.bucket = bucket.name + req.key = robj.key + vclock = robj.vclock() + if vclock: + req.vclock = vclock + + self.pbify_content(robj.metadata, + robj.get_encoded_data(), + req.content) + + msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, + MSG_CODE_PUT_RESP) + if resp is not None: + contents = [] + for c in resp.content: + contents.append(self.decode_content(c)) + return resp.vclock, contents + + def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): + """Put a new object into the Riak store, returning its (new) key. + + If return_meta is False, then the vlock and metadata return values + will be None. + + @return (key, vclock, metadata) + """ + # Note that this won't work on 0.14 nodes. + bucket = robj.bucket + + req = riak_pb.RpbPutReq() + if w: + req.w = self.translate_rw_val(w) + if dw: + req.dw = self.translate_rw_val(dw) + if self.quorum_controls() and pw: + req.pw = self.translate_rw_val(pw) + + if return_body: + req.return_body = 1 + if if_none_match: + req.if_none_match = 1 + + req.bucket = bucket.name + + self.pbify_content(robj.metadata, + robj.get_encoded_data(), + req.content) + + msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, + MSG_CODE_PUT_RESP) + if not resp: + raise RiakError("missing response object") + if len(resp.content) != 1: + raise RiakError("siblings were returned from object creation") + + metadata, content = self.decode_content(resp.content[0]) + return resp.key, resp.vclock, metadata + + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + """ + Serialize get request and deserialize response + """ + bucket = robj.bucket + + req = riak_pb.RpbDelReq() + if rw: + req.rw = self.translate_rw_val(rw) + if r: + req.r = self.translate_rw_val(r) + if w: + req.w = self.translate_rw_val(w) + if dw: + req.dw = self.translate_rw_val(dw) + + if self.quorum_controls(): + if pr: + req.pr = self.translate_rw_val(pr) + if pw: + req.pw = self.translate_rw_val(pw) + + if self.tombstone_vclocks() and robj.vclock(): + req.vclock = robj.vclock() + + req.bucket = bucket.name + req.key = robj.key + + msg_code, resp = self._request(MSG_CODE_DEL_REQ, req, + MSG_CODE_DEL_RESP) + return self + + def get_keys(self, bucket): + """ + Lists all keys within a bucket. + """ + keys = [] + for keylist in self.stream_keys(bucket): + keys = keys + keylist + + return keys + + def stream_keys(self, bucket): + """ + Streams keys from a bucket, returning an iterator that yields + lists of keys. + """ + req = riak_pb.RpbListKeysReq() + req.bucket = bucket.name + + self._send_msg(MSG_CODE_LIST_KEYS_REQ, req) + + return RiakPbcKeyStream(self) + + def get_buckets(self): + """ + Serialize bucket listing request and deserialize response + """ + msg_code, resp = self._request(MSG_CODE_LIST_BUCKETS_REQ, + MSG_CODE_LIST_BUCKETS_RESP) + return resp.buckets + + def get_bucket_props(self, bucket): + """ + Serialize bucket property request and deserialize response + """ + req = riak_pb.RpbGetBucketReq() + req.bucket = bucket.name + + msg_code, resp = self._request(MSG_CODE_GET_BUCKET_REQ, req, + MSG_CODE_GET_BUCKET_RESP) + props = {} + if resp.props.HasField('n_val'): + props['n_val'] = resp.props.n_val + if resp.props.HasField('allow_mult'): + props['allow_mult'] = resp.props.allow_mult + + return props + + def set_bucket_props(self, bucket, props): + """ + Serialize set bucket property request and deserialize response + """ + req = riak_pb.RpbSetBucketReq() + req.bucket = bucket.name + for key in props: + if key not in ['n_val', 'allow_mult']: + raise NotImplementedError + + if 'n_val' in props: + req.props.n_val = props['n_val'] + if 'allow_mult' in props: + req.props.allow_mult = props['allow_mult'] + + msg_code, resp = self._request(MSG_CODE_SET_BUCKET_REQ, req, + MSG_CODE_SET_BUCKET_RESP) + return self + + def mapred(self, inputs, query, timeout=None): + # dictionary of phase results - each content should be an encoded array + # which is appended to the result for that phase. + result = {} + for phase, content in self.stream_mapred(inputs, query, timeout): + if phase in result: + result[phase] += content + else: + result[phase] = content + + # If a single result - return the same as the HTTP interface does + # otherwise return all the phase information + if not len(result): + return None + elif len(result) == 1: + return result[max(result.keys())] + else: + return result + + def stream_mapred(self, inputs, query, timeout=None): + # Construct the job, optionally set the timeout... + job = {'inputs': inputs, 'query': query} + if timeout is not None: + job['timeout'] = timeout + + content = json.dumps(job) + + req = riak_pb.RpbMapRedReq() + req.request = content + req.content_type = "application/json" + + self._send_msg(MSG_CODE_MAPRED_REQ, req) + + return RiakPbcMapredStream(self) + + def get_index(self, bucket, index, startkey, endkey=None): + if not self.pb_indexes(): + return self._get_index_mapred_emu(bucket, index, startkey, endkey) + + req = riak_pb.RpbIndexReq(bucket=bucket, index=index) + if endkey: + req.qtype = riak_pb.RpbIndexReq.range + req.range_min = str(startkey) + req.range_max = str(endkey) + else: + req.qtype = riak_pb.RpbIndexReq.eq + req.key = str(startkey) + + msg_code, resp = self._request(MSG_CODE_INDEX_REQ, req, + MSG_CODE_INDEX_RESP) + return resp.keys + + def search(self, index, query, **params): + if not self.pb_search(): + return self._search_mapred_emu(index, query) + + req = riak_pb.RpbSearchQueryReq(index=index, q=query) + if 'rows' in params: + req.rows = params['rows'] + if 'start' in params: + req.start = params['start'] + if 'sort' in params: + req.sort = params['sort'] + if 'filter' in params: + req.filter = params['filter'] + if 'df' in params: + req.df = params['df'] + if 'op' in params: + req.op = params['op'] + if 'q.op' in params: + req.op = params['q.op'] + if 'fl' in params: + if isinstance(params['fl'], list): + req.fl.extend(params['fl']) + else: + req.fl.append(params['fl']) + if 'presort' in params: + req.presort = params['presort'] + + msg_code, resp = self._request(MSG_CODE_SEARCH_QUERY_REQ, req, + MSG_CODE_SEARCH_QUERY_RESP) + + result = {} + if resp.HasField('max_score'): + result['max_score'] = resp.max_score + if resp.HasField('num_found'): + result['num_found'] = resp.num_found + docs = [] + for doc in resp.docs: + resultdoc = {} + for pair in doc.fields: + resultdoc[pair.key] = pair.value + docs.append(resultdoc) + result['docs'] = docs + return result diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 0b0ff32c..975d1dac 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -31,11 +31,14 @@ class RiakTransport(FeatureDetection): Class to encapsulate transport details """ - # Subclasses should specify their API level. - # * missing or 1: the API used up and through 1.3.x. - # * 2: the API introduced with 1.4.x - # - # api = 2 + @property + def client_id(self): + """the client ID for this connection""" + return self._client_id + + @client_id.setter + def client_id(self, value): + self._client_id = value @classmethod def make_random_client_id(self): @@ -149,6 +152,18 @@ def get_index(self, bucket, index, startkey, endkey=None): """ raise NotImplementedError + def fulltext_add(self, index, *docs): + """ + Adds documents to the full-text index. + """ + raise NotImplementedError + + def fulltext_delete(self, index, docs=None, queries=None): + """ + Removes documents from the full-text index. + """ + raise NotImplementedError + def _search_mapred_emu(self, index, query): """ Emulates a search request via MapReduce. Used in the case From 486c5a7056877b6b6102ec104041fc8d0e6b6390 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 12:22:13 -0500 Subject: [PATCH 02/45] Actually connect to PBC by connecting the socket. --- riak/transports/pbc/connection.py | 4 ++++ riak/transports/pbc/transport.py | 1 + 2 files changed, 5 insertions(+) diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index b08b46ba..b4fc5853 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -124,5 +124,9 @@ def _recv_pkt(self): raise RiakError("Socket returned short packet %d - expected %d" % (len(self._inbuf), self._inbuf_len)) + def _connect(self): + self._socket = socket.create_connection(self._address, + self._timeouts.connect) + def close(self): self._socket.shutdown(socket.SHUT_RDWR) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 90a00b40..23730cf5 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -55,6 +55,7 @@ def __init__(self, node=None, client=None, connect_timeout=None, self._address = (node.host, node.pb_port) self._timeouts = {'connect': connect_timeout, 'request': request_timeout} + self._connect() # FeatureDetection API def _server_version(self): From 29cd619d00c596e4724c67544e502c453e837d4e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 12:41:05 -0500 Subject: [PATCH 03/45] Add transport method to client. --- riak/client.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/riak/client.py b/riak/client.py index 26d2430b..b2f4ff47 100644 --- a/riak/client.py +++ b/riak/client.py @@ -229,3 +229,16 @@ def _error_rate(node): return min(nodes, key=_error_rate) else: return random.choice(good) + + @contextmanager + def transport(self, protocol=self.protocol): + if protocol in ['http', 'https']: + pool = self._http_pool + elif protocol is 'pbc': + pool = self._pb_pool + else: + raise ValueError("invalid protocol %s" % protocol) + + # Replace with recovery logic later + with pool.take() as transport: + yield transport From 32081343794299cb506f49890be30ba3feb0f9b2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 13:06:56 -0500 Subject: [PATCH 04/45] Fix some typos and comments from @ultimatebuster. --- riak/client.py | 6 +++--- riak/transports/http.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/riak/client.py b/riak/client.py index b2f4ff47..4aecb1f4 100644 --- a/riak/client.py +++ b/riak/client.py @@ -25,7 +25,7 @@ import simplejson as json from contextlib import contextmanager - +from weakref import WeakValueDictionary from riak.client.operations import RiakClientOperations from riak.node import RiakNode from riak.bucket import RiakBucket @@ -86,7 +86,7 @@ def __init__(self, protocol='http', transport_options={}, 'text/json': json.dumps} self._decoders = {'application/json': json.loads, 'text/json': json.loads} - self._buckets = {} + self._buckets = WeakValueDictionary() @property def protocol(self): @@ -94,7 +94,7 @@ def protocol(self): @property.setter def protocol(self, value): - if protocol and protocol not in self.PROTOCOLS: + if value not in self.PROTOCOLS: raise ValueError("protocol option is invalid, must be one of %s" % repr(self.PROTOCOLS)) self._protocol = value diff --git a/riak/transports/http.py b/riak/transports/http.py index b8be3d27..43f0b7e4 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -52,7 +52,7 @@ class RiakHttpPool(Pool): """ def __init__(self, client, **options): self.client = client - self.transport_options = options + self.options = options if client.protocol is 'https': self.connection_class = httplib.HTTPConnection else: From f60700bc0c341e91edc9afadf68fdaec5286b6c1 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 13:07:46 -0500 Subject: [PATCH 05/45] Move client -> client/__init__. --- riak/{client.py => client/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename riak/{client.py => client/__init__.py} (100%) diff --git a/riak/client.py b/riak/client/__init__.py similarity index 100% rename from riak/client.py rename to riak/client/__init__.py From f188f8e7904c631ee1d072fbe846c5a6253e91d5 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 15:08:02 -0500 Subject: [PATCH 06/45] Move connection-claiming into a new module, add retry logic. * RiakClient.transport is now RiakClient._transport (via RiakClientTransport mixin). * HTTP and PBC transports define their own version of what are retryable exceptions. --- riak/client/__init__.py | 17 ++------ riak/client/operations.py | 32 +++++++------- riak/client/transport.py | 69 +++++++++++++++++++++++++++++++ riak/transports/http.py | 19 +++++++++ riak/transports/pbc/__init__.py | 30 +++++++++++++- riak/transports/pbc/connection.py | 13 +----- 6 files changed, 137 insertions(+), 43 deletions(-) create mode 100644 riak/client/transport.py diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 4aecb1f4..c57fc9f6 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -27,6 +27,7 @@ from contextlib import contextmanager from weakref import WeakValueDictionary from riak.client.operations import RiakClientOperations +from riak.client.transport import RiakClientTransport from riak.node import RiakNode from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduce @@ -40,7 +41,8 @@ @deprecateQuorumAccessors -class RiakClient(RiakMapReduceChain, RiakClientOperations): +class RiakClient(RiakMapReduceChain, RiakClientOperations, + RiakClientTransport): """ The ``RiakClient`` object holds information necessary to connect to Riak. Requests can be made to Riak directly through the client @@ -229,16 +231,3 @@ def _error_rate(node): return min(nodes, key=_error_rate) else: return random.choice(good) - - @contextmanager - def transport(self, protocol=self.protocol): - if protocol in ['http', 'https']: - pool = self._http_pool - elif protocol is 'pbc': - pool = self._pb_pool - else: - raise ValueError("invalid protocol %s" % protocol) - - # Replace with recovery logic later - with pool.take() as transport: - yield transport diff --git a/riak/client/operations.py b/riak/client/operations.py index f7ede8a6..90928b49 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -31,7 +31,7 @@ def get_buckets(self): NOTE: Do not use this in production, as it requires traversing through all keys stored in a cluster. """ - with self.transport() as transport: + with self._transport() as transport: return [RiakBucket(self, name) for name in transport.get_buckets()] def ping(self): @@ -40,7 +40,7 @@ def ping(self): :rtype: boolean """ - with self.transport() as transport: + with self._transport() as transport: return transport.ping() is_alive = ping @@ -49,28 +49,28 @@ def get_index(self, bucket, index, startkey, endkey=None): """ Queries a secondary index, returning matching keys. """ - with self.transport() as transport: + with self._transport() as transport: return transport.get_index(bucket, index, startkey, endkey) def get_bucket_props(self, bucket): """ Fetches bucket properties for the given bucket. """ - with self.transport() as transport: + with self._transport() as transport: return transport.get_bucket_props(bucket) def set_bucket_props(self, bucket, props): """ Sets bucket properties for the given bucket. """ - with self.transport() as transport: + with self._transport() as transport: return transport.set_bucket_props(bucket, props) def get_keys(self, bucket): """ Lists all keys in a bucket. """ - with self.transport() as transport: + with self._transport() as transport: return transport.get_keys(bucket) def stream_keys(self, bucket): @@ -78,7 +78,7 @@ def stream_keys(self, bucket): Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. """ - with self.transport() as transport: + with self._transport() as transport: for keylist in return transport.stream_keys(bucket): yield keylist @@ -87,7 +87,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=None, """ Stores an object in the Riak cluster. """ - with self.transport() as transport: + with self._transport() as transport: return transport.put(robj, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) @@ -97,7 +97,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=None, """ Stores an object in the Riak cluster with a generated key. """ - with self.transport() as transport: + with self._transport() as transport: return transport.put_new(robj, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) @@ -106,14 +106,14 @@ def get(self, robj, r=None, pr=None, vtag=None): """ Fetches the contents of a Riak object. """ - with self.transport() as transport: + with self._transport() as transport: return transport.get(robj, r=r, pr=pr, vtag=vtag) def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Deletes an object from Riak. """ - with self.transport() as transport: + with self._transport() as transport: return transport.delete(robj, rw=rw, r=r, w=w, dw=dw=, pr=pr, pw=pw) @@ -121,7 +121,7 @@ def mapred(self, inputs, query, timeout): """ Executes a MapReduce query """ - with self.transport() as transport: + with self._transport() as transport: return transport.mapred(inputs, query, timeout) def stream_mapred(self, inputs, query, timeout): @@ -129,7 +129,7 @@ def stream_mapred(self, inputs, query, timeout): Streams a MapReduce query as (phase, data) pairs. This is a generator method which should be iterated over. """ - with self.transport() as transport: + with self._transport() as transport: for phase, data in transport.stream_mapred(inputs, query, timeout): yield phase, data @@ -137,19 +137,19 @@ def fulltext_search(self, index, query, **params): """ Performs a full-text search query. """ - with self.transport() as transport: + with self._transport() as transport: return transport.search(index, query, **params) def fulltext_add(self, index, docs): """ Adds documents to the full-text index. """ - with self._http_pool.take() as transport: + with self._transport(protocol='http') as transport: transport.fulltext_add(self, index, docs) def fulltext_delete(self, index, docs=None, queries=None): """ Removes documents from the full-text index. """ - with self._http_pool.take() as transport: + with self._transport(protocol='http') as transport: transport.fulltext_delete(index, docs, queries) diff --git a/riak/client/transport.py b/riak/client/transport.py new file mode 100644 index 00000000..a1a6cb36 --- /dev/null +++ b/riak/client/transport.py @@ -0,0 +1,69 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +from contextlib import contextmanager +from riak.transports.pool import BadResource +from riak.transports.pbc import is_retryable as is_pbc_retryable +from riak.transports.http import is_retryable as is_http_retryable +import httplib + + +class RiakClientTransport(class): + """ + Methods for RiakClient related to transport selection and retries. + """ + + @contextmanager + def _transport(self, protocol=self.protocol): + if protocol in ['http', 'https']: + pool = self._http_pool + elif protocol is 'pbc': + pool = self._pb_pool + else: + raise ValueError("invalid protocol %s" % protocol) + + with self._retryable(pool) as transport: + yield transport + + @contextmanager + def _retryable(self, pool): + skip_nodes = [] + # TODO: Make a property? + retries = 3 + + def _skip_bad_nodes(transport): + return transport.node not in skip_nodes + + while retries > 0: + try: + with pool.take(_filter=_skip_bad_nodes) as transport: + try: + yield transport + except (IOError, httplib.HTTPException) as e: + if is_pbc_retryable(e) or is_http_retryable(e): + retries -= 1 + transport.node.error_rate.incr(1) + skip_nodes.append(transport.node) + raise BadResource(e) + else: + raise e + except BadResource as br: + if retries > 0: + continue + else: + raise br.args[0] diff --git a/riak/transports/http.py b/riak/transports/http.py index 43f0b7e4..aaa8ec1c 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -755,3 +755,22 @@ def close(self): return {'num_found': self.num_found, 'max_score': self.max_score, 'docs': self.docs} + + +CONN_CLOSED_ERRORS = ( + httplib.NotConnected, + httplib.IncompleteRead, + httplib.ImproperConnectionState + ) + + +def is_retryable(err): + """ + Determines if the given exception is something that is + network/socket-related and should thus cause the HTTP connection + to close and the operation retried on another node. + """ + for errtype in CONN_CLOSED_ERRORS: + if isinstance(err, errtype): + return True + return False diff --git a/riak/transports/pbc/__init__.py b/riak/transports/pbc/__init__.py index d3620738..ef772777 100644 --- a/riak/transports/pbc/__init__.py +++ b/riak/transports/pbc/__init__.py @@ -19,7 +19,7 @@ under the License. """ - +import errno from riak.transports.pool import Pool from riak.transports.pbc.transport import RiakPbcTransport @@ -41,3 +41,31 @@ def create_resource(self): def destroy_resource(self, pbc): pbc.close() + +# These are a specific set of socket errors +# that could be raised on send/recv that indicate +# that the socket is closed or reset, and is not +# usable. On seeing any of these errors, the socket +# should be closed, and the connection re-established. +CONN_CLOSED_ERRORS = ( + errno.EHOSTUNREACH, + errno.ECONNRESET, + errno.ECONNREFUSED, + errno.ECONNABORTED, + errno.ETIMEDOUT, + errno.EBADF, + errno.EPIPE + ) + + +def is_retryable(err): + """ + Determines if the given exception is something that is + network/socket-related and should thus cause the PBC connection to + close and the operation retried on another node. + """ + if isinstance(err, socket.error): + code = err.args[0] + return code in CONN_CLOSED_ERRORS + else: + return False diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index b4fc5853..c8ba785f 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -20,18 +20,7 @@ import socket import struct from contextlib import contextmanager - -# These are a specific set of socket errors -# that could be raised on send/recv that indicate -# that the socket is closed or reset, and is not -# usable. On seeing any of these errors, the socket -# should be closed, and the connection re-established. -CONN_CLOSED_ERRORS = ( - errno.EHOSTUNREACH, - errno.ECONNRESET, - errno.EBADF, - errno.EPIPE - ) +from riak import RiakError class RiakPbcConnection(object): From 12f596c0a2edc1d44eb7d2f60fec3c5096d09fd3 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 15:13:07 -0500 Subject: [PATCH 07/45] The node is a private attribute on the transport. --- riak/client/transport.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/riak/client/transport.py b/riak/client/transport.py index a1a6cb36..924d00f3 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -47,7 +47,7 @@ def _retryable(self, pool): retries = 3 def _skip_bad_nodes(transport): - return transport.node not in skip_nodes + return transport._node not in skip_nodes while retries > 0: try: @@ -57,8 +57,8 @@ def _skip_bad_nodes(transport): except (IOError, httplib.HTTPException) as e: if is_pbc_retryable(e) or is_http_retryable(e): retries -= 1 - transport.node.error_rate.incr(1) - skip_nodes.append(transport.node) + transport._node.error_rate.incr(1) + skip_nodes.append(transport._node) raise BadResource(e) else: raise e From 618f75a57a83de2dc4d22d689cd4e889ea6c14c6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 15:42:38 -0500 Subject: [PATCH 08/45] Move http.py in preparation for refactoring. --- riak/transports/{http.py => http/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename riak/transports/{http.py => http/__init__.py} (100%) diff --git a/riak/transports/http.py b/riak/transports/http/__init__.py similarity index 100% rename from riak/transports/http.py rename to riak/transports/http/__init__.py From f924239c381354c5214631901f864a6ca23830c4 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 17:16:36 -0500 Subject: [PATCH 09/45] Start breaking out HTTP into multiple files. --- riak/transports/http/__init__.py | 711 +----------------------------- riak/transports/http/search.py | 48 ++ riak/transports/http/transport.py | 678 ++++++++++++++++++++++++++++ riak/transports/pbc/transport.py | 2 - 4 files changed, 728 insertions(+), 711 deletions(-) create mode 100644 riak/transports/http/search.py create mode 100644 riak/transports/http/transport.py diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index aaa8ec1c..478ebf5c 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -18,28 +18,8 @@ under the License. """ -import urllib -import re -import csv -from cStringIO import StringIO -import httplib -import socket -import errno -try: - import json -except ImportError: - import simplejson as json - -from riak.transports.transport import RiakTransport -from riak.transports.pool import Pool, BadResource -from riak.metadata import * -from riak.mapreduce import RiakLink -from riak import RiakError -from riak.riak_index_entry import RiakIndexEntry -from riak.multidict import MultiDict -import riak.util -from xml.etree import ElementTree -from xml.dom.minidom import Document +from riak.transports.pool import Pool +from riak.transports.http.transport import RiakHttpTransport # subtract length of "Link: " header string and newline @@ -70,693 +50,6 @@ def destroy_resource(self, transport): transport.close() -class RiakHttpTransport(RiakTransport): - """ - The RiakHttpTransport object holds information necessary to - connect to Riak via HTTP. - """ - - api = 3 - - def __init__(self, node=None, - client=None, - connection_class=httplib.HTTPConnection - client_id=None, - **unused_options): - """ - Construct a new HTTP connection to Riak. - """ - super(RiakHttpTransport, self).__init__() - - self._client = client - self._node = node - self._connection_class = connection_class - self._client_id = client_id - if not self._client_id: - self._client_id = self.make_random_client_id() - - def ping(self): - """ - Check server is alive over HTTP - """ - response = self.http_request('GET', '/ping') - return(response is not None) and (response[1] == 'OK') - - def stats(self): - """ - Gets performance statistics and server information - """ - # TODO: use resource detection - response = self.http_request('GET', '/stats', - {'Accept': 'application/json'}) - if response[0]['http_code'] is 200: - return json.loads(response[1]) - else: - return None - - # FeatureDetection API - private - def _server_version(self): - stats = self.stats() - if stats is not None: - return stats['riak_kv_version'] - # If stats is disabled, we can't assume the Riak version - # is >= 1.1. However, we can assume the new URL scheme is - # at least version 1.0 - elif 'riak_kv_wm_buckets' in self.get_resources(): - return "1.0.0" - else: - return "0.14.0" - - def get_resources(self): - """ - Gets a JSON mapping of server-side resource names to paths - :rtype dict - """ - response = self.http_request('GET', '/', - {'Accept': 'application/json'}) - if response[0]['http_code'] is 200: - return json.loads(response[1]) - else: - return {} - - def get(self, robj, r=None, pr=None, vtag=None): - """ - Get a bucket/key from the server - """ - # We could detect quorum_controls here but HTTP ignores - # unknown flags/params. - params = {'r': r, 'pr': pr} - if vtag is not None: - params['vtag'] = vtag - url = self.build_rest_path(robj.bucket, robj.key, - params=params) - response = self.http_request('GET', url) - return self.parse_body(response, [200, 300, 404]) - - def put(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """ - Serialize put request and deserialize response - """ - # We could detect quorum_controls here but HTTP ignores - # unknown flags/params. - params = {'returnbody': str(return_body).lower(), - 'w': w, 'dw': dw, 'pw': pw} - url = self.build_rest_path(bucket=robj.bucket, - key=robj.key, - params=params) - headers = self.build_put_headers(robj) - # TODO: use a more general 'prevent_stale_writes' semantics, - # which is a superset of the if_none_match semantics. - if if_none_match: - headers["If-None-Match"] = "*" - content = robj.get_encoded_data() - return self.do_put(url, headers, content, return_body, - key=robj.key) - - def do_put(self, url, headers, content, return_body=False, key=None): - if key is None: - response = self.http_request('POST', url, headers, content) - else: - response = self.http_request('PUT', url, headers, content) - - if return_body: - return self.parse_body(response, [200, 201, 300]) - else: - self.check_http_code(response, [204]) - return None - - def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """Put a new object into the Riak store, returning its (new) key.""" - # We could detect quorum_controls here but HTTP ignores - # unknown flags/params. - params = {'returnbody': str(return_body).lower(), 'w': w, 'dw': dw, - 'pw': pw} - url = self.build_rest_path(bucket=robj.bucket, params=params) - headers = self.build_put_headers(robj) - # TODO: use a more general 'prevent_stale_writes' semantics, - # which is a superset of the if_none_match semantics. - if if_none_match: - headers["If-None-Match"] = "*" - content = robj.get_encoded_data() - response = self.http_request('POST', url, headers, content) - location = response[0]['location'] - idx = location.rindex('/') - key = location[(idx + 1):] - if return_body: - vclock, [(metadata, data)] = self.parse_body(response, [201]) - return key, vclock, metadata - else: - self.check_http_code(response, [201]) - return key, None, None - - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): - """ - Delete an object. - """ - # We could detect quorum_controls here but HTTP ignores - # unknown flags/params. - params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} - headers = {} - url = self.build_rest_path(robj.bucket, robj.key, - params=params) - if self.tombstone_vclocks() and robj.vclock is not None: - headers['X-Riak-Vclock'] = robj.vclock - response = self.http_request('DELETE', url, headers) - self.check_http_code(response, [204, 404]) - return self - - def get_keys(self, bucket): - """ - Fetch a list of keys for the bucket - """ - params = {'props': 'True', 'keys': 'true'} - url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', url) - - headers, encoded_props = response[0:2] - if headers['http_code'] == 200: - props = json.loads(encoded_props) - return props['keys'] - else: - raise Exception('Error getting bucket properties.') - - def get_buckets(self): - """ - Fetch a list of all buckets - """ - params = {'buckets': 'true'} - url = self.build_rest_path(None, params=params) - response = self.http_request('GET', url) - - headers, encoded_props = response[0:2] - if headers['http_code'] == 200: - props = json.loads(encoded_props) - return props['buckets'] - else: - raise Exception('Error getting buckets.') - - def get_bucket_props(self, bucket): - """ - Get properties for a bucket - """ - # Run the request... - params = {'props': 'true', 'keys': 'false'} - url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', url) - - headers = response[0] - encoded_props = response[1] - if headers['http_code'] == 200: - props = json.loads(encoded_props) - return props['props'] - else: - raise Exception('Error getting bucket properties.') - - def set_bucket_props(self, bucket, props): - """ - Set the properties on the bucket object given - """ - url = self.build_rest_path(bucket) - headers = {'Content-Type': 'application/json'} - content = json.dumps({'props': props}) - - # Run the request... - response = self.http_request('PUT', url, headers, content) - - # Handle the response... - if response is None: - raise Exception('Error setting bucket properties.') - - # Check the response value... - status = response[0]['http_code'] - if status != 204: - raise Exception('Error setting bucket properties.') - return True - - def mapred(self, inputs, query, timeout=None): - """ - Run a MapReduce query. - """ - if not self.phaseless_mapred() and (query is None or len(query) is 0): - raise Exception( - 'Phase-less MapReduce is not supported by Riak node') - - # Construct the job, optionally set the timeout... - job = {'inputs': inputs, 'query': query} - if timeout is not None: - job['timeout'] = timeout - - content = json.dumps(job) - - # Do the request... - url = "/" + self._mapred_prefix - headers = {'Content-Type': 'application/json'} - response = self.http_request('POST', url, headers, content) - - # Make sure the expected status code came back... - status = response[0]['http_code'] - if status != 200: - raise Exception( - 'Error running MapReduce operation. Headers: %s Body: %s' % - (repr(response[0]), repr(response[1]))) - - result = json.loads(response[1]) - return result - - def get_index(self, bucket, index, startkey, endkey=None): - """ - Performs a secondary index query. - """ - # TODO: use resource detection - segments = ["buckets", bucket, "index", index, str(startkey)] - if endkey: - segments.append(str(endkey)) - uri = '/%s' % ('/'.join(segments)) - headers, data = response = self.get_request(uri) - self.check_http_code(response, [200]) - jsonData = json.loads(data) - return jsonData[u'keys'][:] - - def search(self, index, query, **params): - """ - Performs a search query. - """ - if index is None: - index = 'search' - - options = {'q': query, 'wt': 'json'} - if 'op' in params: - op = params.pop('op') - options['q.op'] = op - - options.update(params) - # TODO: use resource detection - uri = "/solr/%s/select" % index - headers, data = response = self.get_request(uri, options) - self.check_http_code(response, [200]) - if 'json' in headers['content-type']: - results = json.loads(data) - return self._normalize_json_search_response(results) - elif 'xml' in headers['content-type']: - return self._normalize_xml_search_response(data) - else: - raise ValueError("Could not decode search response") - - def fulltext_add(self, index, docs): - """ - Adds documents to the search index. - """ - xml = Document() - root = xml.createElement('add') - for doc in docs: - doc_element = xml.createElement('doc') - for key in doc: - value = doc[key] - field = xml.createElement('field') - field.setAttribute("name", key) - text = xml.createTextNode(value) - field.appendChild(text) - doc_element.appendChild(field) - root.appendChild(doc_element) - xml.appendChild(root) - - url = "/solr/%s/update" % index - self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") - - def fulltext_delete(self, index, docs=None, queries=None): - """ - Removes documents from the full-text index. - """ - xml = Document() - root = xml.createElement('delete') - if docs: - for doc in docs: - doc_element = xml.createElement('id') - text = xml.createTextNode(doc) - doc_element.appendChild(text) - root.appendChild(doc_element) - if queries: - for query in queries: - query_element = xml.createElement('query') - text = xml.createTextNode(query) - query_element.appendChild(text) - root.appendChild(query_element) - - xml.appendChild(root) - - url = "/solr/%s/update" % index - self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") - - def check_http_code(self, response, expected_statuses): - status = response[0]['http_code'] - if not status in expected_statuses: - raise Exception('Expected status %s, received %s : %s' % - (expected_statuses, status, response[1])) - - def parse_body(self, response, expected_statuses): - """ - Given the output of RiakUtils.http_request and a list of - statuses, populate the object. Only for use by the Riak client - library. - @return self - """ - # If no response given, then return. - if response is None: - return self - - # Make sure expected code came back - self.check_http_code(response, expected_statuses) - - # Update the object... - headers = response[0] - data = response[1] - status = headers['http_code'] - - # Check if the server is down(status==0) - if not status: - ### we need the host/port that was used. - m = 'Could not contact Riak Server: http://$HOST:$PORT !' - raise RiakError(m) - - # If 404(Not Found), then clear the object. - if status == 404: - return None - - # If 300(Siblings), then return the list of siblings - elif status == 300: - # Parse and get rid of 'Siblings:' string in element 0 - siblings = data.strip().split('\n') - siblings.pop(0) - return siblings - - # Parse the headers... - vclock = None - metadata = {MD_USERMETA: {}, MD_INDEX: []} - links = [] - for header, value in headers.iteritems(): - if header == 'content-type': - metadata[MD_CTYPE] = value - elif header == 'charset': - metadata[MD_CHARSET] = value - elif header == 'content-encoding': - metadata[MD_CTYPE] = value - elif header == 'etag': - metadata[MD_VTAG] = value - elif header == 'link': - self.parse_links(links, headers['link']) - elif header == 'last-modified': - metadata[MD_LASTMOD] = value - elif header.startswith('x-riak-meta-'): - metakey = header.replace('x-riak-meta-', '') - metadata[MD_USERMETA][metakey] = value - elif header.startswith('x-riak-index-'): - field = header.replace('x-riak-index-', '') - reader = csv.reader([value], skipinitialspace=True) - for line in reader: - for token in line: - rie = RiakIndexEntry(field, token) - metadata[MD_INDEX].append(rie) - elif header == 'x-riak-vclock': - vclock = value - elif header == 'x-riak-deleted': - metadata[MD_DELETED] = True - if links: - metadata[MD_LINKS] = links - - return vclock, [(metadata, data)] - - def to_link_header(self, link): - """ - Convert this RiakLink object to a link header string. Used internally. - """ - header = '' - header += '; riaktag="' - header += urllib.quote_plus(link.get_tag()) + '"' - return header - - def parse_links(self, links, linkHeaders): - """ - Private. - @return self - """ - oldform = "; ?riaktag=\"([^\']+)\"" - newform = "; ?riaktag=\"([^\']+)\"" - for linkHeader in linkHeaders.strip().split(','): - linkHeader = linkHeader.strip() - matches = (re.match(oldform, linkHeader) or - re.match(newform, linkHeader)) - if matches is not None: - link = RiakLink(urllib.unquote_plus(matches.group(2)), - urllib.unquote_plus(matches.group(3)), - urllib.unquote_plus(matches.group(4))) - links.append(link) - return self - - def add_links_for_riak_object(self, robject, headers): - links = robject.get_links() - if links: - current_header = '' - for link in links: - header = self.to_link_header(link) - if len(current_header + header) > MAX_LINK_HEADER_SIZE: - headers.add('Link', current_header) - current_header = '' - - if current_header != '': - header = ', ' + header - current_header += header - - headers.add('Link', current_header) - - return headers - - def get_request(self, uri=None, params=None): - url = self.build_rest_path(bucket=None, params=params, prefix=uri) - return self.http_request('GET', url) - - def post_request(self, uri=None, body=None, params=None, - content_type="application/json"): - uri = self.build_rest_path(prefix=uri, params=params) - return self.http_request('POST', uri, {'Content-Type': content_type}, - body) - - # Utility functions used by Riak library. - - def build_rest_path(self, bucket=None, key=None, params=None, prefix=None): - """ - Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, - construct and return a URL. - """ - # Build 'http://hostname:port/prefix/bucket' - path = '' - path += '/' + (prefix or self._prefix) - - # Add '.../bucket' - if bucket is not None: - path += '/' + urllib.quote_plus(bucket.name) - - # Add '.../key' - if key is not None: - path += '/' + urllib.quote_plus(key) - - # Add query parameters. - if params is not None: - s = '' - for key in params.keys(): - if params[key] is not None: - if s != '': - s += '&' - s += (urllib.quote_plus(key) + '=' + - urllib.quote_plus(str(params[key]))) - path += '?' + s - - # Return. - return path - - def build_put_headers(self, robj): - """Build the headers for a POST/PUT request.""" - - # Construct the headers... - headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', - 'Content-Type': robj.content_type, - 'X-Riak-ClientId': self._client_id}) - - # Add the vclock if it exists... - if robj.vclock is not None: - headers['X-Riak-Vclock'] = robj.vclock - - # Create the header from metadata - links = self.add_links_for_riak_object(robj, headers) - - for key, value in robj.usermeta.iteritems(): - headers['X-Riak-Meta-%s' % key] = value - - for rie in robj.get_indexes(): - key = 'X-Riak-Index-%s' % rie.get_field() - if key in headers: - headers[key] += ", " + rie.get_value() - else: - headers[key] = rie.get_value() - - return headers - - def http_request(self, method, uri, headers=None, body=''): - """ - Given a Method, URL, Headers, and Body, perform and HTTP request, - and return a 2-tuple containing a dictionary of response headers - and the response body. - """ - if headers is None: - headers = {} - # Run the request... - for retry in range(self.RETRY_COUNT): - with self._conns.withconn() as conn: - ### should probably build this try/except into a custom - ### contextmanager for the connection. - try: - conn.request(method, uri, body, headers) - response = conn.getresponse() - - try: - # Get the response headers... - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value - - # Get the body... - response_body = response.read() - finally: - response.close() - - return response_headers, response_body - except socket.error, e: - conn.close() - if e[0] == errno.ECONNRESET: - # Grab another connection and try again. - continue - # Don't know how to handle this. - raise - except httplib.HTTPException: - # Just close the connection and try again. - conn.close() - continue - - # No luck, even with retrying. - raise RiakError("could not get a response") - - def _normalize_json_search_response(self, json): - """ - Normalizes a JSON search response so that PB and HTTP have the - same return value - """ - result = {} - if u'response' in json: - result['num_found'] = json[u'response'][u'numFound'] - result['max_score'] = float(json[u'response'][u'maxScore']) - docs = [] - for doc in json[u'response'][u'docs']: - resdoc = {u'id': doc[u'id']} - if u'fields' in doc: - for k, v in doc[u'fields'].iteritems(): - resdoc[k] = v - docs.append(resdoc) - result['docs'] = docs - return result - - def _normalize_xml_search_response(self, xml): - """ - Normalizes an XML search response so that PB and HTTP have the - same return value - """ - target = XMLSearchResult() - parser = ElementTree.XMLParser(target=target) - parser.feed(xml) - return parser.close() - - @classmethod - def build_headers(cls, headers): - return ['%s: %s' % (header, value) - for header, value in headers.iteritems()] - - @classmethod - def parse_http_headers(cls, headers): - """ - Parse an HTTP Header string into an asssociative array of - response headers. - """ - retVal = {} - fields = headers.split("\n") - for field in fields: - matches = re.match("([^:]+):(.+)", field) - if matches is None: - continue - key = matches.group(1).lower() - value = matches.group(2).strip() - if key in retVal.keys(): - if isinstance(retVal[key], list): - retVal[key].append(value) - else: - retVal[key] = [retVal[key]].append(value) - else: - retVal[key] = value - return retVal - - -class XMLSearchResult(object): - # Match tags that are document fields - fieldtags = ['str', 'int', 'date'] - - def __init__(self): - # Results - self.num_found = 0 - self.max_score = 0.0 - self.docs = [] - - # Parser state - self.currdoc = None - self.currfield = None - self.currvalue = None - - def start(self, tag, attrib): - if tag == 'result': - self.num_found = int(attrib['numFound']) - self.max_score = float(attrib['maxScore']) - elif tag == 'doc': - self.currdoc = {} - elif tag in self.fieldtags and self.currdoc is not None: - self.currfield = attrib['name'] - - def end(self, tag): - if tag == 'doc' and self.currdoc is not None: - self.docs.append(self.currdoc) - self.currdoc = None - elif tag in self.fieldtags and self.currdoc is not None: - if tag == 'int': - self.currvalue = int(self.currvalue) - self.currdoc[self.currfield] = self.currvalue - self.currfield = None - self.currvalue = None - - def data(self, data): - if self.currfield: - # riak_solr_output adds NL + 6 spaces - data = data.rstrip() - if self.currvalue: - self.currvalue += data - else: - self.currvalue = data - - def close(self): - return {'num_found': self.num_found, - 'max_score': self.max_score, - 'docs': self.docs} - - CONN_CLOSED_ERRORS = ( httplib.NotConnected, httplib.IncompleteRead, diff --git a/riak/transports/http/search.py b/riak/transports/http/search.py new file mode 100644 index 00000000..4e6c69e6 --- /dev/null +++ b/riak/transports/http/search.py @@ -0,0 +1,48 @@ +class XMLSearchResult(object): + # Match tags that are document fields + fieldtags = ['str', 'int', 'date'] + + def __init__(self): + # Results + self.num_found = 0 + self.max_score = 0.0 + self.docs = [] + + # Parser state + self.currdoc = None + self.currfield = None + self.currvalue = None + + def start(self, tag, attrib): + if tag == 'result': + self.num_found = int(attrib['numFound']) + self.max_score = float(attrib['maxScore']) + elif tag == 'doc': + self.currdoc = {} + elif tag in self.fieldtags and self.currdoc is not None: + self.currfield = attrib['name'] + + def end(self, tag): + if tag == 'doc' and self.currdoc is not None: + self.docs.append(self.currdoc) + self.currdoc = None + elif tag in self.fieldtags and self.currdoc is not None: + if tag == 'int': + self.currvalue = int(self.currvalue) + self.currdoc[self.currfield] = self.currvalue + self.currfield = None + self.currvalue = None + + def data(self, data): + if self.currfield: + # riak_solr_output adds NL + 6 spaces + data = data.rstrip() + if self.currvalue: + self.currvalue += data + else: + self.currvalue = data + + def close(self): + return {'num_found': self.num_found, + 'max_score': self.max_score, + 'docs': self.docs} diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py new file mode 100644 index 00000000..0104e510 --- /dev/null +++ b/riak/transports/http/transport.py @@ -0,0 +1,678 @@ +""" +Copyright 2012 Basho Technologies, Inc. +Copyright 2010 Rusty Klophaus +Copyright 2010 Justin Sheehy +Copyright 2009 Jay Baird + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +try: + import json +except ImportError: + import simplejson as json + +import urllib +import re +import csv +from cStringIO import StringIO +import httplib +import socket +import errno +from riak.transports.transport import RiakTransport +from riak.transports.http.search import XMLSearchResult +from riak.metadata import * +from riak.mapreduce import RiakLink +from riak import RiakError +from riak.riak_index_entry import RiakIndexEntry +from riak.multidict import MultiDict +import riak.util +from xml.etree import ElementTree +from xml.dom.minidom import Document + + +class RiakHttpTransport(RiakTransport): + """ + The RiakHttpTransport object holds information necessary to + connect to Riak via HTTP. + """ + + def __init__(self, node=None, + client=None, + connection_class=httplib.HTTPConnection + client_id=None, + **unused_options): + """ + Construct a new HTTP connection to Riak. + """ + super(RiakHttpTransport, self).__init__() + + self._client = client + self._node = node + self._connection_class = connection_class + self._client_id = client_id + if not self._client_id: + self._client_id = self.make_random_client_id() + + def ping(self): + """ + Check server is alive over HTTP + """ + response = self.http_request('GET', '/ping') + return(response is not None) and (response[1] == 'OK') + + def stats(self): + """ + Gets performance statistics and server information + """ + # TODO: use resource detection + response = self.http_request('GET', '/stats', + {'Accept': 'application/json'}) + if response[0]['http_code'] is 200: + return json.loads(response[1]) + else: + return None + + # FeatureDetection API - private + def _server_version(self): + stats = self.stats() + if stats is not None: + return stats['riak_kv_version'] + # If stats is disabled, we can't assume the Riak version + # is >= 1.1. However, we can assume the new URL scheme is + # at least version 1.0 + elif 'riak_kv_wm_buckets' in self.get_resources(): + return "1.0.0" + else: + return "0.14.0" + + def get_resources(self): + """ + Gets a JSON mapping of server-side resource names to paths + :rtype dict + """ + response = self.http_request('GET', '/', + {'Accept': 'application/json'}) + if response[0]['http_code'] is 200: + return json.loads(response[1]) + else: + return {} + + def get(self, robj, r=None, pr=None, vtag=None): + """ + Get a bucket/key from the server + """ + # We could detect quorum_controls here but HTTP ignores + # unknown flags/params. + params = {'r': r, 'pr': pr} + if vtag is not None: + params['vtag'] = vtag + url = self.build_rest_path(robj.bucket, robj.key, + params=params) + response = self.http_request('GET', url) + return self.parse_body(response, [200, 300, 404]) + + def put(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): + """ + Serialize put request and deserialize response + """ + # We could detect quorum_controls here but HTTP ignores + # unknown flags/params. + params = {'returnbody': str(return_body).lower(), + 'w': w, 'dw': dw, 'pw': pw} + url = self.build_rest_path(bucket=robj.bucket, + key=robj.key, + params=params) + headers = self.build_put_headers(robj) + # TODO: use a more general 'prevent_stale_writes' semantics, + # which is a superset of the if_none_match semantics. + if if_none_match: + headers["If-None-Match"] = "*" + content = robj.get_encoded_data() + return self.do_put(url, headers, content, return_body, + key=robj.key) + + def do_put(self, url, headers, content, return_body=False, key=None): + if key is None: + response = self.http_request('POST', url, headers, content) + else: + response = self.http_request('PUT', url, headers, content) + + if return_body: + return self.parse_body(response, [200, 201, 300]) + else: + self.check_http_code(response, [204]) + return None + + def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False): + """Put a new object into the Riak store, returning its (new) key.""" + # We could detect quorum_controls here but HTTP ignores + # unknown flags/params. + params = {'returnbody': str(return_body).lower(), 'w': w, 'dw': dw, + 'pw': pw} + url = self.build_rest_path(bucket=robj.bucket, params=params) + headers = self.build_put_headers(robj) + # TODO: use a more general 'prevent_stale_writes' semantics, + # which is a superset of the if_none_match semantics. + if if_none_match: + headers["If-None-Match"] = "*" + content = robj.get_encoded_data() + response = self.http_request('POST', url, headers, content) + location = response[0]['location'] + idx = location.rindex('/') + key = location[(idx + 1):] + if return_body: + vclock, [(metadata, data)] = self.parse_body(response, [201]) + return key, vclock, metadata + else: + self.check_http_code(response, [201]) + return key, None, None + + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + """ + Delete an object. + """ + # We could detect quorum_controls here but HTTP ignores + # unknown flags/params. + params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} + headers = {} + url = self.build_rest_path(robj.bucket, robj.key, + params=params) + if self.tombstone_vclocks() and robj.vclock is not None: + headers['X-Riak-Vclock'] = robj.vclock + response = self.http_request('DELETE', url, headers) + self.check_http_code(response, [204, 404]) + return self + + def get_keys(self, bucket): + """ + Fetch a list of keys for the bucket + """ + params = {'props': 'True', 'keys': 'true'} + url = self.build_rest_path(bucket, params=params) + response = self.http_request('GET', url) + + headers, encoded_props = response[0:2] + if headers['http_code'] == 200: + props = json.loads(encoded_props) + return props['keys'] + else: + raise Exception('Error getting bucket properties.') + + def get_buckets(self): + """ + Fetch a list of all buckets + """ + params = {'buckets': 'true'} + url = self.build_rest_path(None, params=params) + response = self.http_request('GET', url) + + headers, encoded_props = response[0:2] + if headers['http_code'] == 200: + props = json.loads(encoded_props) + return props['buckets'] + else: + raise Exception('Error getting buckets.') + + def get_bucket_props(self, bucket): + """ + Get properties for a bucket + """ + # Run the request... + params = {'props': 'true', 'keys': 'false'} + url = self.build_rest_path(bucket, params=params) + response = self.http_request('GET', url) + + headers = response[0] + encoded_props = response[1] + if headers['http_code'] == 200: + props = json.loads(encoded_props) + return props['props'] + else: + raise Exception('Error getting bucket properties.') + + def set_bucket_props(self, bucket, props): + """ + Set the properties on the bucket object given + """ + url = self.build_rest_path(bucket) + headers = {'Content-Type': 'application/json'} + content = json.dumps({'props': props}) + + # Run the request... + response = self.http_request('PUT', url, headers, content) + + # Handle the response... + if response is None: + raise Exception('Error setting bucket properties.') + + # Check the response value... + status = response[0]['http_code'] + if status != 204: + raise Exception('Error setting bucket properties.') + return True + + def mapred(self, inputs, query, timeout=None): + """ + Run a MapReduce query. + """ + if not self.phaseless_mapred() and (query is None or len(query) is 0): + raise Exception( + 'Phase-less MapReduce is not supported by Riak node') + + # Construct the job, optionally set the timeout... + job = {'inputs': inputs, 'query': query} + if timeout is not None: + job['timeout'] = timeout + + content = json.dumps(job) + + # Do the request... + url = "/" + self._mapred_prefix + headers = {'Content-Type': 'application/json'} + response = self.http_request('POST', url, headers, content) + + # Make sure the expected status code came back... + status = response[0]['http_code'] + if status != 200: + raise Exception( + 'Error running MapReduce operation. Headers: %s Body: %s' % + (repr(response[0]), repr(response[1]))) + + result = json.loads(response[1]) + return result + + def get_index(self, bucket, index, startkey, endkey=None): + """ + Performs a secondary index query. + """ + # TODO: use resource detection + segments = ["buckets", bucket, "index", index, str(startkey)] + if endkey: + segments.append(str(endkey)) + uri = '/%s' % ('/'.join(segments)) + headers, data = response = self.get_request(uri) + self.check_http_code(response, [200]) + jsonData = json.loads(data) + return jsonData[u'keys'][:] + + def search(self, index, query, **params): + """ + Performs a search query. + """ + if index is None: + index = 'search' + + options = {'q': query, 'wt': 'json'} + if 'op' in params: + op = params.pop('op') + options['q.op'] = op + + options.update(params) + # TODO: use resource detection + uri = "/solr/%s/select" % index + headers, data = response = self.get_request(uri, options) + self.check_http_code(response, [200]) + if 'json' in headers['content-type']: + results = json.loads(data) + return self._normalize_json_search_response(results) + elif 'xml' in headers['content-type']: + return self._normalize_xml_search_response(data) + else: + raise ValueError("Could not decode search response") + + def fulltext_add(self, index, docs): + """ + Adds documents to the search index. + """ + xml = Document() + root = xml.createElement('add') + for doc in docs: + doc_element = xml.createElement('doc') + for key in doc: + value = doc[key] + field = xml.createElement('field') + field.setAttribute("name", key) + text = xml.createTextNode(value) + field.appendChild(text) + doc_element.appendChild(field) + root.appendChild(doc_element) + xml.appendChild(root) + + url = "/solr/%s/update" % index + self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") + + def fulltext_delete(self, index, docs=None, queries=None): + """ + Removes documents from the full-text index. + """ + xml = Document() + root = xml.createElement('delete') + if docs: + for doc in docs: + doc_element = xml.createElement('id') + text = xml.createTextNode(doc) + doc_element.appendChild(text) + root.appendChild(doc_element) + if queries: + for query in queries: + query_element = xml.createElement('query') + text = xml.createTextNode(query) + query_element.appendChild(text) + root.appendChild(query_element) + + xml.appendChild(root) + + url = "/solr/%s/update" % index + self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") + + def check_http_code(self, response, expected_statuses): + status = response[0]['http_code'] + if not status in expected_statuses: + raise Exception('Expected status %s, received %s : %s' % + (expected_statuses, status, response[1])) + + def parse_body(self, response, expected_statuses): + """ + Given the output of RiakUtils.http_request and a list of + statuses, populate the object. Only for use by the Riak client + library. + @return self + """ + # If no response given, then return. + if response is None: + return self + + # Make sure expected code came back + self.check_http_code(response, expected_statuses) + + # Update the object... + headers = response[0] + data = response[1] + status = headers['http_code'] + + # Check if the server is down(status==0) + if not status: + ### we need the host/port that was used. + m = 'Could not contact Riak Server: http://$HOST:$PORT !' + raise RiakError(m) + + # If 404(Not Found), then clear the object. + if status == 404: + return None + + # If 300(Siblings), then return the list of siblings + elif status == 300: + # Parse and get rid of 'Siblings:' string in element 0 + siblings = data.strip().split('\n') + siblings.pop(0) + return siblings + + # Parse the headers... + vclock = None + metadata = {MD_USERMETA: {}, MD_INDEX: []} + links = [] + for header, value in headers.iteritems(): + if header == 'content-type': + metadata[MD_CTYPE] = value + elif header == 'charset': + metadata[MD_CHARSET] = value + elif header == 'content-encoding': + metadata[MD_CTYPE] = value + elif header == 'etag': + metadata[MD_VTAG] = value + elif header == 'link': + self.parse_links(links, headers['link']) + elif header == 'last-modified': + metadata[MD_LASTMOD] = value + elif header.startswith('x-riak-meta-'): + metakey = header.replace('x-riak-meta-', '') + metadata[MD_USERMETA][metakey] = value + elif header.startswith('x-riak-index-'): + field = header.replace('x-riak-index-', '') + reader = csv.reader([value], skipinitialspace=True) + for line in reader: + for token in line: + rie = RiakIndexEntry(field, token) + metadata[MD_INDEX].append(rie) + elif header == 'x-riak-vclock': + vclock = value + elif header == 'x-riak-deleted': + metadata[MD_DELETED] = True + if links: + metadata[MD_LINKS] = links + + return vclock, [(metadata, data)] + + def to_link_header(self, link): + """ + Convert this RiakLink object to a link header string. Used internally. + """ + header = '' + header += '; riaktag="' + header += urllib.quote_plus(link.get_tag()) + '"' + return header + + def parse_links(self, links, linkHeaders): + """ + Private. + @return self + """ + oldform = "; ?riaktag=\"([^\']+)\"" + newform = "; ?riaktag=\"([^\']+)\"" + for linkHeader in linkHeaders.strip().split(','): + linkHeader = linkHeader.strip() + matches = (re.match(oldform, linkHeader) or + re.match(newform, linkHeader)) + if matches is not None: + link = RiakLink(urllib.unquote_plus(matches.group(2)), + urllib.unquote_plus(matches.group(3)), + urllib.unquote_plus(matches.group(4))) + links.append(link) + return self + + def add_links_for_riak_object(self, robject, headers): + links = robject.get_links() + if links: + current_header = '' + for link in links: + header = self.to_link_header(link) + if len(current_header + header) > MAX_LINK_HEADER_SIZE: + headers.add('Link', current_header) + current_header = '' + + if current_header != '': + header = ', ' + header + current_header += header + + headers.add('Link', current_header) + + return headers + + def get_request(self, uri=None, params=None): + url = self.build_rest_path(bucket=None, params=params, prefix=uri) + return self.http_request('GET', url) + + def post_request(self, uri=None, body=None, params=None, + content_type="application/json"): + uri = self.build_rest_path(prefix=uri, params=params) + return self.http_request('POST', uri, {'Content-Type': content_type}, + body) + + # Utility functions used by Riak library. + + def build_rest_path(self, bucket=None, key=None, params=None, prefix=None): + """ + Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, + construct and return a URL. + """ + # Build 'http://hostname:port/prefix/bucket' + path = '' + path += '/' + (prefix or self._prefix) + + # Add '.../bucket' + if bucket is not None: + path += '/' + urllib.quote_plus(bucket.name) + + # Add '.../key' + if key is not None: + path += '/' + urllib.quote_plus(key) + + # Add query parameters. + if params is not None: + s = '' + for key in params.keys(): + if params[key] is not None: + if s != '': + s += '&' + s += (urllib.quote_plus(key) + '=' + + urllib.quote_plus(str(params[key]))) + path += '?' + s + + # Return. + return path + + def build_put_headers(self, robj): + """Build the headers for a POST/PUT request.""" + + # Construct the headers... + headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', + 'Content-Type': robj.content_type, + 'X-Riak-ClientId': self._client_id}) + + # Add the vclock if it exists... + if robj.vclock is not None: + headers['X-Riak-Vclock'] = robj.vclock + + # Create the header from metadata + links = self.add_links_for_riak_object(robj, headers) + + for key, value in robj.usermeta.iteritems(): + headers['X-Riak-Meta-%s' % key] = value + + for rie in robj.get_indexes(): + key = 'X-Riak-Index-%s' % rie.get_field() + if key in headers: + headers[key] += ", " + rie.get_value() + else: + headers[key] = rie.get_value() + + return headers + + def http_request(self, method, uri, headers=None, body=''): + """ + Given a Method, URL, Headers, and Body, perform and HTTP request, + and return a 2-tuple containing a dictionary of response headers + and the response body. + """ + if headers is None: + headers = {} + # Run the request... + for retry in range(self.RETRY_COUNT): + with self._conns.withconn() as conn: + ### should probably build this try/except into a custom + ### contextmanager for the connection. + try: + conn.request(method, uri, body, headers) + response = conn.getresponse() + + try: + # Get the response headers... + response_headers = {'http_code': response.status} + for (key, value) in response.getheaders(): + response_headers[key.lower()] = value + + # Get the body... + response_body = response.read() + finally: + response.close() + + return response_headers, response_body + except socket.error, e: + conn.close() + if e[0] == errno.ECONNRESET: + # Grab another connection and try again. + continue + # Don't know how to handle this. + raise + except httplib.HTTPException: + # Just close the connection and try again. + conn.close() + continue + + # No luck, even with retrying. + raise RiakError("could not get a response") + + def _normalize_json_search_response(self, json): + """ + Normalizes a JSON search response so that PB and HTTP have the + same return value + """ + result = {} + if u'response' in json: + result['num_found'] = json[u'response'][u'numFound'] + result['max_score'] = float(json[u'response'][u'maxScore']) + docs = [] + for doc in json[u'response'][u'docs']: + resdoc = {u'id': doc[u'id']} + if u'fields' in doc: + for k, v in doc[u'fields'].iteritems(): + resdoc[k] = v + docs.append(resdoc) + result['docs'] = docs + return result + + def _normalize_xml_search_response(self, xml): + """ + Normalizes an XML search response so that PB and HTTP have the + same return value + """ + target = XMLSearchResult() + parser = ElementTree.XMLParser(target=target) + parser.feed(xml) + return parser.close() + + @classmethod + def build_headers(cls, headers): + return ['%s: %s' % (header, value) + for header, value in headers.iteritems()] + + @classmethod + def parse_http_headers(cls, headers): + """ + Parse an HTTP Header string into an asssociative array of + response headers. + """ + retVal = {} + fields = headers.split("\n") + for field in fields: + matches = re.match("([^:]+):(.+)", field) + if matches is None: + continue + key = matches.group(1).lower() + value = matches.group(2).strip() + if key in retVal.keys(): + if isinstance(retVal[key], list): + retVal[key].append(value) + else: + retVal[key] = [retVal[key]].append(value) + else: + retVal[key] = value + return retVal diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 23730cf5..6cd96e38 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -41,8 +41,6 @@ class RiakPbcTransport(RiakTransport, RiakPbcConnection, RiakPbcCodec): buffers interface on the riak server. """ - api = 3 - def __init__(self, node=None, client=None, connect_timeout=None, request_timeout=None, **unused_options): """ From 47cc10d1dd2dc5ef2ad8809ea404786b9dbbcdb2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 27 Dec 2012 17:17:21 -0500 Subject: [PATCH 10/45] Add resource URL path generator methods, not wired up. --- riak/transports/http/resources.py | 168 ++++++++++++++++++++++++++++++ riak/transports/http/transport.py | 3 +- 2 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 riak/transports/http/resources.py diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py new file mode 100644 index 00000000..ccbfe896 --- /dev/null +++ b/riak/transports/http/resources.py @@ -0,0 +1,168 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import re +from urllib import quote_plus +from riak import RiakError +from riak.util import lazy_property + +class RiakHttpResources(object): + """ + Methods for RiakHttpTransport related to URL generation, i.e. + creating the proper paths. + """ + + def ping_path(self): + return mkpath(self.riak_kv_wm_ping) + + def stats_path(self): + return mkpath(self.riak_kv_wm_stats) + + def mapred_path(self, **options): + return mkpath(self.kv_wm_mapred, **options) + + def bucket_list_path(self, **options): + query = options.copy() + query.update(buckets=True) + if self.riak_kv_wm_buckets: + return mkpath(self.riak_kv_wm_buckets, **query) + else: + return mkpath(self.riak_kv_wm_raw, **query) + + def bucket_properties_path(self, bucket, **options): + if self.riak_kv_wm_buckets: + return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), + "props", **options) + else: + query = options.copy() + query.update(props=True, keys=True) + return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), **query) + + def key_list_path(self, bucket, **options): + query = options.copy() + query.update(keys=True, props=False) + if self.riak_kv_wm_buckets: + return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), "keys", + **query) + else: + return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), **query) + + def object_path(self, bucket, key=None, **options): + if key: + key = quote_plus(key) + if self.riak_kv_wm_buckets: + return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), "keys", + key, **options) + else: + return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), key, + **options) + + # TODO: link_walk_path is undefined here because there is no path + # to it in the client without using MapReduce. + + def index_path(self, bucket, index, start, finish=None, **options): + if not self.riak_kv_wm_buckets: + raise RiakError("Indexes are unsupported by this Riak node") + if finish: + finish = quote_plus(str(finish)) + return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), + "index", quote_plus(str(start)), finish, **options) + + def solr_select_path(self, index, query, **options): + if not self.riak_solr_searcher_wm: + raise RiakError("Riak Search is unsupported by this Riak node") + qs = {'q': query, 'wt': 'json'} + qs.update(options) + if index: + index = quote_plus(index) + return mkpath(self.riak_solr_searcher_wm, index, "select", **qs) + + def solr_update_path(self, index): + if not self.riak_solr_searcher_wm: + raise RiakError("Riak Search is unsupported by this Riak node") + if index: + index = quote_plus(index) + return mkpath(self.riak_solr_indexer_wm, index, "update") + + def luwak_path(self, key=None): + if not self.luwak_wm_file: + raise RiakError("Luwak is unsupported by this Riak node") + if key: + key = quote_plus(key) + return mkpath(self.luwak_wm_file, key) + + @lazy_property + def riak_kv_wm_buckets(self): + return self.resources.get('riak_kv_wm_buckets') + + @lazy_property + def riak_kv_wm_raw(self): + return self.resources.get('riak_kv_wm_raw') or "/riak" + + @lazy_property + def riak_kv_wm_link_walker(self): + return self.resources.get('riak_kv_wm_linkwalker') or "/riak" + + @lazy_property + def riak_kv_wm_mapred(self): + return self.resources.get('riak_kv_wm_mapred') or "/mapred" + + @lazy_property + def riak_kv_wm_ping(self): + return self.resources.get('riak_kv_wm_ping') or "/ping" + + @lazy_property + def riak_kv_wm_stats(self): + return self.resources.get('riak_kv_wm_stats') or "/stats" + + @lazy_property + def riak_solr_searcher_wm(self): + return self.resources.get('riak_solr_searcher_wm') + + @lazy_property + def riak_solr_indexer_wm(self): + return self.resources.get('riak_solr_indexer_wm') + + @lazy_property + def luwak_wm_file(self): + return self.resources.get('luwak_wm_file') + + @lazy_property + def resources(self): + return self.get_resources() + + +def mkpath(*segments, **query): + """ + Constructs the path & query portion of a URI from path segments + and a dict. + """ + # Remove empty segments (e.g. no key specified) + segments = [s for s in segments if s is not None] + # Join the segments into a path + pathstring = '/'.join(segments) + # Remove extra slashes + pathstring = re.sub('/+', '/', pathstring) + # Add the query string if it exists + if len(query) > 0: + pathstring += "?" + urllib.urlencode(query).lower() + + if not pathstring.startswith('/'): + pathstring = '/' + pathstring + + return pathstring diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 0104e510..8af2e940 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -32,6 +32,7 @@ import socket import errno from riak.transports.transport import RiakTransport +from riak.transports.http.resources import RiakHttpResources from riak.transports.http.search import XMLSearchResult from riak.metadata import * from riak.mapreduce import RiakLink @@ -43,7 +44,7 @@ from xml.dom.minidom import Document -class RiakHttpTransport(RiakTransport): +class RiakHttpTransport(RiakHttpResources, RiakTransport): """ The RiakHttpTransport object holds information necessary to connect to Riak via HTTP. From 34078d26c15912301f356db571bfed37e5a6187f Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 28 Dec 2012 12:27:28 -0500 Subject: [PATCH 11/45] Move HTTP request methods into a new file and wire up URL generators. --- riak/transports/http/connection.py | 71 +++++++++++ riak/transports/http/resources.py | 4 + riak/transports/http/transport.py | 192 +++++++---------------------- 3 files changed, 120 insertions(+), 147 deletions(-) create mode 100644 riak/transports/http/connection.py diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py new file mode 100644 index 00000000..68b922fb --- /dev/null +++ b/riak/transports/http/connection.py @@ -0,0 +1,71 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import httplib + + +class RiakHttpConnection(object): + """ + Connection and low-level request methods for RiakHttpTransport. + """ + + def GET(self, uri, headers={}): + return self._request('GET', uri, headers, '') + + def POST(self, uri, headers={}, body=''): + return self._request('POST', uri, headers, body) + + def PUT(self, uri, headers={}, body=''): + return self._request('PUT', uri, headers, body) + + def DELETE(self, uri, headers={}, body=''): + return self.request('DELETE', uri, headers, body) + + def HEAD(self, uri, headers={}): + return self.request('HEAD', uri, headers, '') + + def _request(self, method, uri, headers={}, body=''): + """ + Given a Method, URL, Headers, and Body, perform and HTTP request, + and return a 2-tuple containing a dictionary of response headers + and the response body. + """ + try: + self._connection.request(method, uri, body, headers) + response = self._connection.getresponse() + + response_headers = {'http_code': response.status} + for (key, value) in response.getheaders(): + response_headers[key.lower()] = value + + # TODO: Support streaming responses + response_body = response.read() + finally: + response.close() + + return response_headers, response_body + + def _connect(self): + self._connection = self._connection_class(self._node.host, + self._node.http_port) + + def close(self): + try: + self._connection.close() + except httplib.NotConnected: + pass diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index ccbfe896..b0a60f22 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -21,6 +21,7 @@ from riak import RiakError from riak.util import lazy_property + class RiakHttpResources(object): """ Methods for RiakHttpTransport related to URL generation, i.e. @@ -160,6 +161,9 @@ def mkpath(*segments, **query): pathstring = re.sub('/+', '/', pathstring) # Add the query string if it exists if len(query) > 0: + for key in query.keys(): + if query[key] is None: + query.pop(key) pathstring += "?" + urllib.urlencode(query).lower() if not pathstring.startswith('/'): diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 8af2e940..e7b4a725 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -33,6 +33,7 @@ import errno from riak.transports.transport import RiakTransport from riak.transports.http.resources import RiakHttpResources +from riak.transports.http.connection import RiakHttpConnection from riak.transports.http.search import XMLSearchResult from riak.metadata import * from riak.mapreduce import RiakLink @@ -44,7 +45,7 @@ from xml.dom.minidom import Document -class RiakHttpTransport(RiakHttpResources, RiakTransport): +class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakTransport): """ The RiakHttpTransport object holds information necessary to connect to Riak via HTTP. @@ -66,21 +67,20 @@ def __init__(self, node=None, self._client_id = client_id if not self._client_id: self._client_id = self.make_random_client_id() + self._connect() def ping(self): """ Check server is alive over HTTP """ - response = self.http_request('GET', '/ping') + response = self.GET(self.ping_path()) return(response is not None) and (response[1] == 'OK') def stats(self): """ Gets performance statistics and server information """ - # TODO: use resource detection - response = self.http_request('GET', '/stats', - {'Accept': 'application/json'}) + response = self.GET(self.stats_path(), {'Accept': 'application/json'}) if response[0]['http_code'] is 200: return json.loads(response[1]) else: @@ -94,7 +94,7 @@ def _server_version(self): # If stats is disabled, we can't assume the Riak version # is >= 1.1. However, we can assume the new URL scheme is # at least version 1.0 - elif 'riak_kv_wm_buckets' in self.get_resources(): + elif self.riak_kv_wm_buckets: return "1.0.0" else: return "0.14.0" @@ -104,8 +104,7 @@ def get_resources(self): Gets a JSON mapping of server-side resource names to paths :rtype dict """ - response = self.http_request('GET', '/', - {'Accept': 'application/json'}) + response = self.GET('/', {'Accept': 'application/json'}) if response[0]['http_code'] is 200: return json.loads(response[1]) else: @@ -117,12 +116,9 @@ def get(self, robj, r=None, pr=None, vtag=None): """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'r': r, 'pr': pr} - if vtag is not None: - params['vtag'] = vtag - url = self.build_rest_path(robj.bucket, robj.key, - params=params) - response = self.http_request('GET', url) + params = {'r': r, 'pr': pr, 'vtag': vtag} + url = self.object_path(robj.bucket.name, robj.key, **params) + response = self.GET(url) return self.parse_body(response, [200, 300, 404]) def put(self, robj, w=None, dw=None, pw=None, return_body=True, @@ -132,12 +128,10 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'returnbody': str(return_body).lower(), - 'w': w, 'dw': dw, 'pw': pw} - url = self.build_rest_path(bucket=robj.bucket, - key=robj.key, - params=params) + params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} + url = self.object_path(robj.bucket.name, robj.key, **params) headers = self.build_put_headers(robj) + # TODO: use a more general 'prevent_stale_writes' semantics, # which is a superset of the if_none_match semantics. if if_none_match: @@ -148,9 +142,9 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, def do_put(self, url, headers, content, return_body=False, key=None): if key is None: - response = self.http_request('POST', url, headers, content) + response = self.POST(url, headers, content) else: - response = self.http_request('PUT', url, headers, content) + response = self.PUT(url, headers, content) if return_body: return self.parse_body(response, [200, 201, 300]) @@ -163,16 +157,15 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, """Put a new object into the Riak store, returning its (new) key.""" # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'returnbody': str(return_body).lower(), 'w': w, 'dw': dw, - 'pw': pw} - url = self.build_rest_path(bucket=robj.bucket, params=params) + params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} + url = self.object_path(robj.bucket.name, **params) headers = self.build_put_headers(robj) # TODO: use a more general 'prevent_stale_writes' semantics, # which is a superset of the if_none_match semantics. if if_none_match: headers["If-None-Match"] = "*" content = robj.get_encoded_data() - response = self.http_request('POST', url, headers, content) + response = self.POST(url, headers, content) location = response[0]['location'] idx = location.rindex('/') key = location[(idx + 1):] @@ -191,11 +184,10 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): # unknown flags/params. params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} headers = {} - url = self.build_rest_path(robj.bucket, robj.key, - params=params) + url = self.object_path(robj.bucket.name, robj.key, **params) if self.tombstone_vclocks() and robj.vclock is not None: headers['X-Riak-Vclock'] = robj.vclock - response = self.http_request('DELETE', url, headers) + response = self.DELETE(url, headers) self.check_http_code(response, [204, 404]) return self @@ -203,24 +195,22 @@ def get_keys(self, bucket): """ Fetch a list of keys for the bucket """ - params = {'props': 'True', 'keys': 'true'} - url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', url) + url = self.key_list_path(bucket.name) + response = self.GET(url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: props = json.loads(encoded_props) return props['keys'] else: - raise Exception('Error getting bucket properties.') + raise Exception('Error listing keys.') def get_buckets(self): """ Fetch a list of all buckets """ - params = {'buckets': 'true'} - url = self.build_rest_path(None, params=params) - response = self.http_request('GET', url) + url = self.bucket_list_path() + response = self.GET(url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: @@ -234,9 +224,8 @@ def get_bucket_props(self, bucket): Get properties for a bucket """ # Run the request... - params = {'props': 'true', 'keys': 'false'} - url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', url) + url = self.bucket_properties_path(bucket.name) + response = self.GET(url) headers = response[0] encoded_props = response[1] @@ -250,12 +239,12 @@ def set_bucket_props(self, bucket, props): """ Set the properties on the bucket object given """ - url = self.build_rest_path(bucket) + url = self.bucket_properties_path(bucket.name) headers = {'Content-Type': 'application/json'} content = json.dumps({'props': props}) # Run the request... - response = self.http_request('PUT', url, headers, content) + response = self.PUT(url, headers, content) # Handle the response... if response is None: @@ -283,9 +272,9 @@ def mapred(self, inputs, query, timeout=None): content = json.dumps(job) # Do the request... - url = "/" + self._mapred_prefix + url = self.mapred_path() headers = {'Content-Type': 'application/json'} - response = self.http_request('POST', url, headers, content) + response = self.POST(url, headers, content) # Make sure the expected status code came back... status = response[0]['http_code'] @@ -301,12 +290,8 @@ def get_index(self, bucket, index, startkey, endkey=None): """ Performs a secondary index query. """ - # TODO: use resource detection - segments = ["buckets", bucket, "index", index, str(startkey)] - if endkey: - segments.append(str(endkey)) - uri = '/%s' % ('/'.join(segments)) - headers, data = response = self.get_request(uri) + response = self.GET(self.index_path(bucket, index, startkey, endkey)) + headers, data = response self.check_http_code(response, [200]) jsonData = json.loads(data) return jsonData[u'keys'][:] @@ -318,15 +303,14 @@ def search(self, index, query, **params): if index is None: index = 'search' - options = {'q': query, 'wt': 'json'} + options = {} if 'op' in params: op = params.pop('op') options['q.op'] = op options.update(params) - # TODO: use resource detection - uri = "/solr/%s/select" % index - headers, data = response = self.get_request(uri, options) + response = self.GET(self.solr_select_path(index, query, **options)) + headers, data = response self.check_http_code(response, [200]) if 'json' in headers['content-type']: results = json.loads(data) @@ -354,8 +338,9 @@ def fulltext_add(self, index, docs): root.appendChild(doc_element) xml.appendChild(root) - url = "/solr/%s/update" % index - self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") + self.POST(self.solr_update_path(index), + {'Content-Type': 'text/xml'}, + xml.toxml()) def fulltext_delete(self, index, docs=None, queries=None): """ @@ -378,8 +363,9 @@ def fulltext_delete(self, index, docs=None, queries=None): xml.appendChild(root) - url = "/solr/%s/update" % index - self.post_request(uri=url, body=xml.toxml(), content_type="text/xml") + self.POST(self.solr_update_path(index), + {'Content-Type': 'text/xml'}, + xml.toxml()) def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] @@ -389,10 +375,7 @@ def check_http_code(self, response, expected_statuses): def parse_body(self, response, expected_statuses): """ - Given the output of RiakUtils.http_request and a list of - statuses, populate the object. Only for use by the Riak client - library. - @return self + Parse the body of an object response and populate the object. """ # If no response given, then return. if response is None: @@ -476,8 +459,8 @@ def parse_links(self, links, linkHeaders): Private. @return self """ - oldform = "; ?riaktag=\"([^\']+)\"" - newform = "; ?riaktag=\"([^\']+)\"" + oldform = "; ?riaktag=\"([^\"]+)\"" + newform = "; ?riaktag=\"([^\"]+)\"" for linkHeader in linkHeaders.strip().split(','): linkHeader = linkHeader.strip() matches = (re.match(oldform, linkHeader) or @@ -507,49 +490,8 @@ def add_links_for_riak_object(self, robject, headers): return headers - def get_request(self, uri=None, params=None): - url = self.build_rest_path(bucket=None, params=params, prefix=uri) - return self.http_request('GET', url) - - def post_request(self, uri=None, body=None, params=None, - content_type="application/json"): - uri = self.build_rest_path(prefix=uri, params=params) - return self.http_request('POST', uri, {'Content-Type': content_type}, - body) - # Utility functions used by Riak library. - def build_rest_path(self, bucket=None, key=None, params=None, prefix=None): - """ - Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, - construct and return a URL. - """ - # Build 'http://hostname:port/prefix/bucket' - path = '' - path += '/' + (prefix or self._prefix) - - # Add '.../bucket' - if bucket is not None: - path += '/' + urllib.quote_plus(bucket.name) - - # Add '.../key' - if key is not None: - path += '/' + urllib.quote_plus(key) - - # Add query parameters. - if params is not None: - s = '' - for key in params.keys(): - if params[key] is not None: - if s != '': - s += '&' - s += (urllib.quote_plus(key) + '=' + - urllib.quote_plus(str(params[key]))) - path += '?' + s - - # Return. - return path - def build_put_headers(self, robj): """Build the headers for a POST/PUT request.""" @@ -577,50 +519,6 @@ def build_put_headers(self, robj): return headers - def http_request(self, method, uri, headers=None, body=''): - """ - Given a Method, URL, Headers, and Body, perform and HTTP request, - and return a 2-tuple containing a dictionary of response headers - and the response body. - """ - if headers is None: - headers = {} - # Run the request... - for retry in range(self.RETRY_COUNT): - with self._conns.withconn() as conn: - ### should probably build this try/except into a custom - ### contextmanager for the connection. - try: - conn.request(method, uri, body, headers) - response = conn.getresponse() - - try: - # Get the response headers... - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value - - # Get the body... - response_body = response.read() - finally: - response.close() - - return response_headers, response_body - except socket.error, e: - conn.close() - if e[0] == errno.ECONNRESET: - # Grab another connection and try again. - continue - # Don't know how to handle this. - raise - except httplib.HTTPException: - # Just close the connection and try again. - conn.close() - continue - - # No luck, even with retrying. - raise RiakError("could not get a response") - def _normalize_json_search_response(self, json): """ Normalizes a JSON search response so that PB and HTTP have the From 21c1c387d66f4973f165759a181ee97be414c941 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 28 Dec 2012 12:43:28 -0500 Subject: [PATCH 12/45] Make RETRY_COUNT a property and use a for loop over a range instead of while. --- riak/client/transport.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/riak/client/transport.py b/riak/client/transport.py index 924d00f3..f7a4fd3b 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -28,6 +28,8 @@ class RiakClientTransport(class): Methods for RiakClient related to transport selection and retries. """ + RETRY_COUNT = 3 + @contextmanager def _transport(self, protocol=self.protocol): if protocol in ['http', 'https']: @@ -43,27 +45,24 @@ def _transport(self, protocol=self.protocol): @contextmanager def _retryable(self, pool): skip_nodes = [] - # TODO: Make a property? - retries = 3 def _skip_bad_nodes(transport): return transport._node not in skip_nodes - while retries > 0: + for retry in range(self.RETRY_COUNT): try: with pool.take(_filter=_skip_bad_nodes) as transport: try: yield transport except (IOError, httplib.HTTPException) as e: if is_pbc_retryable(e) or is_http_retryable(e): - retries -= 1 transport._node.error_rate.incr(1) skip_nodes.append(transport._node) raise BadResource(e) else: raise e except BadResource as br: - if retries > 0: + if retry < (self.RETRY_COUNT - 1): continue else: raise br.args[0] From 6db4661993789525b9640b3fed628e143295c258 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 28 Dec 2012 16:32:41 -0500 Subject: [PATCH 13/45] Fix gobs of pyflakes, pylint, pep8 problems. --- riak/__init__.py | 8 +-- riak/client/__init__.py | 44 +++++------- riak/client/operations.py | 7 +- riak/client/transport.py | 11 ++- riak/mapreduce.py | 6 +- riak/riak_object.py | 24 +++++-- riak/search.py | 19 ++++- riak/test_server.py | 5 ++ riak/tests/suite.py | 1 - riak/tests/test_all.py | 109 ++++++++++++++--------------- riak/tests/test_kv.py | 20 ++---- riak/tests/test_mapreduce.py | 4 +- riak/tests/test_pool.py | 16 ++--- riak/transports/__init__.py | 2 - riak/transports/http/__init__.py | 5 +- riak/transports/http/connection.py | 8 ++- riak/transports/http/resources.py | 6 +- riak/transports/http/transport.py | 39 ++++++----- riak/transports/pbc/__init__.py | 1 + riak/transports/pbc/codec.py | 8 +-- riak/transports/pbc/connection.py | 50 ++++++++++--- riak/transports/pbc/stream.py | 12 ++-- riak/transports/pbc/transport.py | 73 +++++++++++++------ riak/transports/transport.py | 11 ++- riak/util.py | 25 +++++-- 25 files changed, 304 insertions(+), 210 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index 1e116161..0d8ab38a 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -38,13 +38,7 @@ def __init__(self, value): def __str__(self): return repr(self.value) -from riak_object import RiakObject -from bucket import RiakBucket -from client import RiakClient -from mapreduce import RiakMapReduce, RiakMapReducePhase, RiakLinkPhase,\ - RiakKeyFilter -from transports.pbc import RiakPbcTransport -from transports.http import RiakHttpTransport +from mapreduce import RiakKeyFilter ONE = "one" ALL = "all" diff --git a/riak/client/__init__.py b/riak/client/__init__.py index c57fc9f6..c5619532 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -19,18 +19,12 @@ under the License. """ -try: - import json -except ImportError: - import simplejson as json - -from contextlib import contextmanager +import json +import random from weakref import WeakValueDictionary from riak.client.operations import RiakClientOperations -from riak.client.transport import RiakClientTransport from riak.node import RiakNode from riak.bucket import RiakBucket -from riak.mapreduce import RiakMapReduce from riak.mapreduce import RiakMapReduceChain from riak.search import RiakSearch from riak.transports.http import RiakHttpPool @@ -41,8 +35,7 @@ @deprecateQuorumAccessors -class RiakClient(RiakMapReduceChain, RiakClientOperations, - RiakClientTransport): +class RiakClient(RiakMapReduceChain, RiakClientOperations): """ The ``RiakClient`` object holds information necessary to connect to Riak. Requests can be made to Riak directly through the client @@ -90,17 +83,18 @@ def __init__(self, protocol='http', transport_options={}, 'text/json': json.loads} self._buckets = WeakValueDictionary() - @property - def protocol(self): + def _get_protocol(self): return self._protocol - @property.setter - def protocol(self, value): + def _set_protocol(self, value): if value not in self.PROTOCOLS: raise ValueError("protocol option is invalid, must be one of %s" % repr(self.PROTOCOLS)) self._protocol = value + protocol = property(_get_protocol, _set_protocol, + doc="""which protocol to prefer""") + def get_transport(self): """ Get the transport instance the client is using for it's connection. @@ -132,23 +126,19 @@ def set_client_id(self, client_id): self.client_id = client_id return self - @property - def client_id(self): - """ - The client ID for this client instance - - :rtype: string - """ - with self.transport() as transport: + def _get_client_id(self): + with self._transport() as transport: return transport.get_client_id() - @client_id.setter - def client_id(self, client_id): + def _set_client_id(self, client_id): for http in self._http_pool: http.client_id = client_id for pb in self._pb_pool: pb.client_id = client_id + client_id = property(_get_client_id, _set_client_id, + doc="""The client ID for this client instance""") + def get_encoder(self, content_type): """ Get the encoding function for the provided content type. @@ -214,16 +204,20 @@ def _create_node(self, n): raise TypeError("%s is not a valid node configuration" % repr(n)) - def _choose_node(self, nodes=self.nodes): + def _choose_node(self, nodes=None): """ Chooses a random node from the list of nodes in the client, taking into account each node's recent error rate. :rtype RiakNode """ + if not nodes: + nodes = self.nodes + # Prefer nodes which have gone a reasonable time without # errors def _error_rate(node): return node.error_rate.value() + good = [n for n in nodes if _error_rate(n) < 0.1] if len(good) is 0: diff --git a/riak/client/operations.py b/riak/client/operations.py index 90928b49..49fa25e4 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -17,9 +17,10 @@ """ from riak.bucket import RiakBucket +from transport import RiakClientTransport -class RiakClientOperations(object): +class RiakClientOperations(RiakClientTransport): """ Methods for RiakClient that result in requests sent to the Riak cluster. @@ -79,7 +80,7 @@ def stream_keys(self, bucket): method which should be iterated over. """ with self._transport() as transport: - for keylist in return transport.stream_keys(bucket): + for keylist in transport.stream_keys(bucket): yield keylist def put(self, robj, w=None, dw=None, pw=None, return_body=None, @@ -114,7 +115,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): Deletes an object from Riak. """ with self._transport() as transport: - return transport.delete(robj, rw=rw, r=r, w=w, dw=dw=, pr=pr, + return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) def mapred(self, inputs, query, timeout): diff --git a/riak/client/transport.py b/riak/client/transport.py index f7a4fd3b..4f449994 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -23,7 +23,7 @@ import httplib -class RiakClientTransport(class): +class RiakClientTransport(object): """ Methods for RiakClient related to transport selection and retries. """ @@ -31,7 +31,9 @@ class RiakClientTransport(class): RETRY_COUNT = 3 @contextmanager - def _transport(self, protocol=self.protocol): + def _transport(self, protocol=None): + if not protocol: + protocol = self.protocol if protocol in ['http', 'https']: pool = self._http_pool elif protocol is 'pbc': @@ -66,3 +68,8 @@ def _skip_bad_nodes(transport): continue else: raise br.args[0] + + # These will be set or redefined by the RiakClient initializer + protocol = 'http' + _http_pool = None + _pb_pool = None diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 6db573be..d64108cc 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -204,7 +204,7 @@ def run(self, timeout=None): """ query, link_results_flag = self._normalize_query() - result = self.client.mapred(self._inputs, query, timeout) + result = self._client.mapred(self._inputs, query, timeout) # If the last phase is NOT a link phase, then return the result. if not (link_results_flag @@ -233,7 +233,7 @@ def stream(self, timeout=None): Streams the MapReduce query (returns an iterator). """ query, lrf = self._normalize_query() - return self.client.stream_mapred(self._inputs, query, timeout) + return self._client.stream_mapred(self._inputs, query, timeout) def _normalize_query(self): num_phases = len(self._phases) @@ -260,7 +260,7 @@ def _normalize_query(self): if (type(self._inputs) == str): bucket_name = self._inputs elif (type(self._inputs) == RiakBucket): - bucket_name = self._inputs.get_name() + bucket_name = self._inputs.name if (bucket_name is not None): self._inputs = {'bucket': bucket_name, diff --git a/riak/riak_object.py b/riak/riak_object.py index 78cf21bb..b37e4ebd 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -18,10 +18,24 @@ under the License. """ import copy -from riak.metadata import * -from riak.mapreduce import * +from riak.metadata import ( + # MD_CHARSET, + MD_CTYPE, + # MD_ENCODING, + MD_INDEX, + # MD_LASTMOD, + # MD_LASTMOD_USECS, + MD_LINKS, + MD_USERMETA + # MD_VTAG, + # MD_DELETED + ) +from mapreduce import ( + RiakMapReduce, + RiakLink + ) from riak import RiakError -from riak.riak_index_entry import RiakIndexEntry +from riak_index_entry import RiakIndexEntry class RiakObject(object): @@ -436,7 +450,7 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): :rtype: self """ - result = self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) + self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) self.clear() return self @@ -488,7 +502,7 @@ def _populate(self, result): for sibling in siblings: sibling._set_siblings(siblings) else: - raise RiakError("do not know how to handle type %s" % type(Result)) + raise RiakError("do not know how to handle type %s" % type(result)) def get_sibling(self, i, r=None, pr=None): """ diff --git a/riak/search.py b/riak/search.py index 1d8fe914..6a25c9eb 100644 --- a/riak/search.py +++ b/riak/search.py @@ -1,5 +1,20 @@ -from riak.transports import RiakHttpTransport -from xml.etree import ElementTree +""" +Copyright 2010 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" class RiakSearch(object): diff --git a/riak/test_server.py b/riak/test_server.py index 7c6348af..4010fb76 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -92,6 +92,11 @@ class TestServer(object): }, } + _temp_bin = None + _temp_etc = None + _temp_log = None + _temp_pipe = None + def __init__(self, tmp_dir="/tmp/riak/test_server", bin_dir=os.path.expanduser("~/.riak/install/riak-0.14.2/bin"), vm_args=None, **options): diff --git a/riak/tests/suite.py b/riak/tests/suite.py index af74197a..6f96c84d 100644 --- a/riak/tests/suite.py +++ b/riak/tests/suite.py @@ -1,4 +1,3 @@ -import riak.tests.test_server_test import os.path import platform diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index fe21c6af..7d9611a8 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -3,19 +3,14 @@ import os import random -import socket import platform if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest -import uuid -import time from riak import RiakClient -from riak import RiakPbcTransport -from riak import RiakHttpTransport from riak.mapreduce import RiakLink from riak import RiakKeyFilter, key_filter @@ -30,7 +25,7 @@ from riak.tests.test_2i import TwoITests try: - import riak_pb + __import__('riak_pb') HAVE_PROTO = True except ImportError: HAVE_PROTO = False @@ -111,61 +106,61 @@ def test_uses_client_id_if_given(self): c = self.create_client(client_id=zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) - def test_close_underlying_socket_fails(self): - self.skipTest("TODO: No longer using connection manager, replace") - c = self.create_client() - - bucket = c.bucket('bucket_test_close') - rand = self.randint() - obj = bucket.new('foo', rand) - obj.store() - obj = bucket.get('foo') - self.assertTrue(obj.exists) - self.assertEqual(obj.bucket.name, 'bucket_test_close') - self.assertEqual(obj.key, 'foo') - self.assertEqual(obj.data, rand) - - # Close the underlying socket. This gets a bit sketchy, - # since we are reaching into the internals, but there is - # no other way to get at the socket - conns = c._cm.conns - conns[0].sock.close() - - # This shoud fail with a socket error now - self.assertRaises(socket.error, bucket.get, 'foo') - - def test_close_underlying_socket_retry(self): - self.skipTest("TODO: No longer using bare transport, replace") - c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport, - transport_options={"max_attempts": 2}) - - bucket = c.bucket('bucket_test_close') - rand = self.randint() - obj = bucket.new('barbaz', rand) - obj.store() - obj = bucket.get('barbaz') - self.assertTrue(obj.exists) - self.assertEqual(obj.bucket.name, 'bucket_test_close') - self.assertEqual(obj.key, 'barbaz') - self.assertEqual(obj.data, rand) - - # Close the underlying socket. This gets a bit sketchy, - # since we are reaching into the internals, but there is - # no other way to get at the socket - conns = c._cm.conns - conns[0].sock.close() - - # This should work, since we have a retry - obj = bucket.get('barbaz') - self.assertTrue(obj.exists) - self.assertEqual(obj.bucket.name, 'bucket_test_close') - self.assertEqual(obj.key, 'barbaz') - self.assertEqual(obj.data, rand) + # def test_close_underlying_socket_fails(self): + # self.skipTest("TODO: No longer using connection manager, replace") + # c = self.create_client() + + # bucket = c.bucket('bucket_test_close') + # rand = self.randint() + # obj = bucket.new('foo', rand) + # obj.store() + # obj = bucket.get('foo') + # self.assertTrue(obj.exists) + # self.assertEqual(obj.bucket.name, 'bucket_test_close') + # self.assertEqual(obj.key, 'foo') + # self.assertEqual(obj.data, rand) + + # # Close the underlying socket. This gets a bit sketchy, + # # since we are reaching into the internals, but there is + # # no other way to get at the socket + # conns = c._cm.conns + # conns[0].sock.close() + + # # This shoud fail with a socket error now + # self.assertRaises(socket.error, bucket.get, 'foo') + + # def test_close_underlying_socket_retry(self): + # self.skipTest("TODO: No longer using bare transport, replace") + # c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport, + # transport_options={"max_attempts": 2}) + + # bucket = c.bucket('bucket_test_close') + # rand = self.randint() + # obj = bucket.new('barbaz', rand) + # obj.store() + # obj = bucket.get('barbaz') + # self.assertTrue(obj.exists) + # self.assertEqual(obj.bucket.name, 'bucket_test_close') + # self.assertEqual(obj.key, 'barbaz') + # self.assertEqual(obj.data, rand) + + # # Close the underlying socket. This gets a bit sketchy, + # # since we are reaching into the internals, but there is + # # no other way to get at the socket + # conns = c._cm.conns + # conns[0].sock.close() + + # # This should work, since we have a retry + # obj = bucket.get('barbaz') + # self.assertTrue(obj.exists) + # self.assertEqual(obj.bucket.name, 'bucket_test_close') + # self.assertEqual(obj.key, 'barbaz') + # self.assertEqual(obj.data, rand) def test_bucket_search_enabled(self): with self.assertRaises(NotImplementedError): bucket = self.client.bucket("unsearch_bucket") - test = bucket.search_enabled() + bucket.search_enabled() def test_enable_search_commit_hook(self): with self.assertRaises(NotImplementedError): diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index b477f944..08a16c04 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -2,10 +2,7 @@ import os import cPickle import copy -try: - import json -except ImportError: - import simplejson as json +import json class NotJsonSerializable(object): @@ -288,13 +285,13 @@ class PbcBucketPropsTest(object): def test_rw_settings(self): bucket = self.client.bucket('rwsettings') with self.assertRaises(NotImplementedError): - test = bucket.r + bucket.r with self.assertRaises(NotImplementedError): - test = bucket.w + bucket.w with self.assertRaises(NotImplementedError): - test = bucket.dw + bucket.dw with self.assertRaises(NotImplementedError): - test = bucket.rw + bucket.rw with self.assertRaises(NotImplementedError): bucket.r = 2 @@ -308,9 +305,9 @@ def test_rw_settings(self): def test_primary_quora(self): bucket = self.client.bucket('primary_quora') with self.assertRaises(NotImplementedError): - test = bucket.pr + bucket.pr with self.assertRaises(NotImplementedError): - test = bucket.pw + bucket.pw with self.assertRaises(NotImplementedError): bucket.pr = 2 @@ -321,7 +318,6 @@ def test_primary_quora(self): class KVFileTests(object): def test_store_binary_object_from_file(self): bucket = self.client.bucket('bucket') - rand = str(self.randint()) filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') obj = bucket.new_binary_from_file('foo_from_file', filepath) obj.store() @@ -331,7 +327,6 @@ def test_store_binary_object_from_file(self): def test_store_binary_object_from_file_should_use_default_mimetype(self): bucket = self.client.bucket('bucket') - rand = str(self.randint()) filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir, 'THANKS') obj = bucket.new_binary_from_file('foo_from_file', filepath) @@ -341,7 +336,6 @@ def test_store_binary_object_from_file_should_use_default_mimetype(self): def test_store_binary_object_from_file_should_fail_if_file_not_found(self): bucket = self.client.bucket('bucket') - rand = str(self.randint()) self.assertRaises(IOError, bucket.new_binary_from_file, 'not_found_from_file', 'FILE_NOT_FOUND') obj = bucket.get_binary('not_found_from_file') diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index f8293f8c..249d7590 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from riak.mapreduce import RiakLink -from riak import RiakKeyFilter, key_filter +from riak import key_filter class LinkTests(object): @@ -44,7 +44,7 @@ def test_set_links(self): def test_set_links_all_links(self): bucket = self.client.bucket("bucket") foo1 = bucket.new("foo", 1) - foo2 = bucket.new("foo2", 2).store() + bucket.new("foo2", 2).store() links = [RiakLink("bucket", "foo2")] foo1.set_links(links, True) links = foo1.get_links() diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index f57a2cd3..d552bcaa 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -69,7 +69,7 @@ def test_yields_same_object_in_serial_access(self): with pool.take() as element2: self.assertEqual(1, len(pool.elements)) - self.assertEqual([1, 2], element) + self.assertEqual([1, 2], element2) self.assertEqual(1, len(pool.elements)) @@ -93,8 +93,8 @@ def test_unlocks_when_exception_raised(self): """ pool = SimplePool() try: - with pool.take() as x: - with pool.take() as y: + with pool.take(): + with pool.take(): raise RuntimeError except: self.assertEqual(2, len(pool.elements)) @@ -111,7 +111,7 @@ def test_removes_bad_resource(self): self.assertEqual([1], element) element.append(2) try: - with pool.take() as baddie: + with pool.take(): raise BadResource except BadResource: self.assertEqual(0, len(pool.elements)) @@ -127,8 +127,8 @@ def filtereven(numlist): return numlist[0] % 2 == 0 pool = SimplePool() - with pool.take() as x: - with pool.take() as y: + with pool.take(): + with pool.take(): pass with pool.take(_filter=filtereven) as f: @@ -143,7 +143,7 @@ def test_requires_filter_to_be_callable(self): pool = SimplePool() with self.assertRaises(TypeError): - with pool.take(_filter=badfilter) as resource: + with pool.take(_filter=badfilter): pass def test_yields_default_when_empty(self): @@ -245,7 +245,7 @@ def test_clear(self): pool = SimplePool() def worker_run(): - with pool.take() as a: + with pool.take(): startq.put(1) startq.join() sleep(rand.uniform(0, 0.5)) diff --git a/riak/transports/__init__.py b/riak/transports/__init__.py index 9c59adb6..e69de29b 100644 --- a/riak/transports/__init__.py +++ b/riak/transports/__init__.py @@ -1,2 +0,0 @@ -from http import RiakHttpTransport -from pbc import RiakPbcTransport diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index 478ebf5c..f3461e2e 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -18,14 +18,11 @@ under the License. """ +import httplib from riak.transports.pool import Pool from riak.transports.http.transport import RiakHttpTransport -# subtract length of "Link: " header string and newline -MAX_LINK_HEADER_SIZE = 8192 - 8 - - class RiakHttpPool(Pool): """ A pool of HTTP(S) transport connections. diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py index 68b922fb..0eeb5425 100644 --- a/riak/transports/http/connection.py +++ b/riak/transports/http/connection.py @@ -34,10 +34,10 @@ def PUT(self, uri, headers={}, body=''): return self._request('PUT', uri, headers, body) def DELETE(self, uri, headers={}, body=''): - return self.request('DELETE', uri, headers, body) + return self._request('DELETE', uri, headers, body) def HEAD(self, uri, headers={}): - return self.request('HEAD', uri, headers, '') + return self._request('HEAD', uri, headers, '') def _request(self, method, uri, headers={}, body=''): """ @@ -69,3 +69,7 @@ def close(self): self._connection.close() except httplib.NotConnected: pass + + # These are set by the RiakHttpTransport initializer + _connection_class = httplib.HTTPConnection + _node = None diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index b0a60f22..44d1b7ff 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -17,7 +17,7 @@ """ import re -from urllib import quote_plus +from urllib import quote_plus, urlencode from riak import RiakError from riak.util import lazy_property @@ -35,7 +35,7 @@ def stats_path(self): return mkpath(self.riak_kv_wm_stats) def mapred_path(self, **options): - return mkpath(self.kv_wm_mapred, **options) + return mkpath(self.riak_kv_wm_mapred, **options) def bucket_list_path(self, **options): query = options.copy() @@ -164,7 +164,7 @@ def mkpath(*segments, **query): for key in query.keys(): if query[key] is None: query.pop(key) - pathstring += "?" + urllib.urlencode(query).lower() + pathstring += "?" + urlencode(query).lower() if not pathstring.startswith('/'): pathstring = '/' + pathstring diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index e7b4a725..342188bd 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -19,32 +19,39 @@ under the License. """ -try: - import json -except ImportError: - import simplejson as json - +import json import urllib import re import csv -from cStringIO import StringIO import httplib -import socket -import errno from riak.transports.transport import RiakTransport from riak.transports.http.resources import RiakHttpResources from riak.transports.http.connection import RiakHttpConnection from riak.transports.http.search import XMLSearchResult -from riak.metadata import * +from riak.metadata import ( + MD_CHARSET, + MD_CTYPE, + # MD_ENCODING, + MD_INDEX, + MD_LASTMOD, + # MD_LASTMOD_USECS, + MD_LINKS, + MD_USERMETA, + MD_VTAG, + MD_DELETED + ) from riak.mapreduce import RiakLink from riak import RiakError from riak.riak_index_entry import RiakIndexEntry from riak.multidict import MultiDict -import riak.util from xml.etree import ElementTree from xml.dom.minidom import Document +# subtract length of "Link: " header string and newline +MAX_LINK_HEADER_SIZE = 8192 - 8 + + class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakTransport): """ The RiakHttpTransport object holds information necessary to @@ -53,7 +60,7 @@ class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakTransport): def __init__(self, node=None, client=None, - connection_class=httplib.HTTPConnection + connection_class=httplib.HTTPConnection, client_id=None, **unused_options): """ @@ -446,12 +453,8 @@ def to_link_header(self, link): """ Convert this RiakLink object to a link header string. Used internally. """ - header = '' - header += '; riaktag="' - header += urllib.quote_plus(link.get_tag()) + '"' + url = self.object_path(link.get_bucket(), link.get_key()) + header = '<%s>; riaktag="%s"' % (url, link.get_tag()) return header def parse_links(self, links, linkHeaders): @@ -505,7 +508,7 @@ def build_put_headers(self, robj): headers['X-Riak-Vclock'] = robj.vclock # Create the header from metadata - links = self.add_links_for_riak_object(robj, headers) + self.add_links_for_riak_object(robj, headers) for key, value in robj.usermeta.iteritems(): headers['X-Riak-Meta-%s' % key] = value diff --git a/riak/transports/pbc/__init__.py b/riak/transports/pbc/__init__.py index ef772777..14705673 100644 --- a/riak/transports/pbc/__init__.py +++ b/riak/transports/pbc/__init__.py @@ -20,6 +20,7 @@ """ import errno +import socket from riak.transports.pool import Pool from riak.transports.pbc.transport import RiakPbcTransport diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index dca7d2f0..ae9a67b9 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -25,12 +25,12 @@ MD_LINKS, MD_USERMETA, MD_VTAG, + MD_DELETED ) -try: - import riak_pb -except ImportError: - riak_pb = None +import riak_pb +from riak.riak_index_entry import RiakIndexEntry +from riak.mapreduce import RiakLink RIAKC_RW_ONE = 4294967294 RIAKC_RW_QUORUM = 4294967293 diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index c8ba785f..1b55b843 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -16,11 +16,41 @@ under the License. """ -import errno import socket import struct -from contextlib import contextmanager +import riak_pb from riak import RiakError +from messages import ( + MSG_CODE_ERROR_RESP, + # MSG_CODE_PING_REQ, + MSG_CODE_PING_RESP, + # MSG_CODE_GET_CLIENT_ID_REQ, + MSG_CODE_GET_CLIENT_ID_RESP, + # MSG_CODE_SET_CLIENT_ID_REQ, + MSG_CODE_SET_CLIENT_ID_RESP, + # MSG_CODE_GET_SERVER_INFO_REQ, + MSG_CODE_GET_SERVER_INFO_RESP, + # MSG_CODE_GET_REQ, + MSG_CODE_GET_RESP, + # MSG_CODE_PUT_REQ, + MSG_CODE_PUT_RESP, + # MSG_CODE_DEL_REQ, + MSG_CODE_DEL_RESP, + # MSG_CODE_LIST_BUCKETS_REQ, + MSG_CODE_LIST_BUCKETS_RESP, + # MSG_CODE_LIST_KEYS_REQ, + MSG_CODE_LIST_KEYS_RESP, + # MSG_CODE_GET_BUCKET_REQ, + MSG_CODE_GET_BUCKET_RESP, + # MSG_CODE_SET_BUCKET_REQ, + MSG_CODE_SET_BUCKET_RESP, + # MSG_CODE_MAPRED_REQ, + MSG_CODE_MAPRED_RESP, + # MSG_CODE_INDEX_REQ, + MSG_CODE_INDEX_RESP, + # MSG_CODE_SEARCH_QUERY_REQ, + MSG_CODE_SEARCH_QUERY_RESP + ) class RiakPbcConnection(object): @@ -40,15 +70,15 @@ def _request(self, msg_code, msg=None, expect=None): return self._recv_msg(expect) def _send_msg(self, msg_code, msg): - self._socket.send(self.encode_msg(msg_code, msg)) + self._socket.send(self._encode_msg(msg_code, msg)) def _recv_msg(self, expect=None): self._recv_pkt() msg_code, = struct.unpack("B", self._inbuf[:1]) if msg_code == MSG_CODE_ERROR_RESP: - msg = riak_pb.RpbErrorResp() - msg.ParseFromString(self._inbuf[1:]) - raise Exception(msg.errmsg) + err = riak_pb.RpbErrorResp() + err.ParseFromString(self._inbuf[1:]) + raise RiakError(err.errmsg) elif msg_code == MSG_CODE_PING_RESP: msg = None elif msg_code == MSG_CODE_GET_SERVER_INFO_RESP: @@ -105,7 +135,7 @@ def _recv_pkt(self): self._inbuf = '' while len(self._inbuf) < msglen: want_len = min(8192, msglen - len(self._inbuf)) - recv_buf = conn.recv(want_len) + recv_buf = self._socket.recv(want_len) if not recv_buf: break self._inbuf += recv_buf @@ -115,7 +145,11 @@ def _recv_pkt(self): def _connect(self): self._socket = socket.create_connection(self._address, - self._timeouts.connect) + self._timeouts['connect']) def close(self): self._socket.shutdown(socket.SHUT_RDWR) + + # These are set in the RiakPbcTransport initializer + _address = None + _timeouts = {} diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index a71f6c20..f63fb645 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -16,20 +16,20 @@ under the License. """ -try: - import json -except ImportError: - import simplejson as json +import json from riak.transports.pbc.messages import MSG_CODE_LIST_KEYS_RESP from riak.transports.pbc.messages import MSG_CODE_MAPRED_RESP -class RiakPbcStream(class): +class RiakPbcStream(object): """ Used internally by RiakPbcTransport to implement streaming operations. Implements the iterator interface. """ + + _expect = None + def __init__(self, transport): self.transport = transport @@ -80,7 +80,7 @@ class RiakPbcMapredStream(RiakPbcStream): def next(self): response = super(RiakPbcMapredStream, self).next() - return (response.phase, json.loads(response.response)) + return response.phase, json.loads(response.response) def _is_done(self, response): return response.done diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 6cd96e38..ebe37685 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -19,20 +19,47 @@ under the License. """ -try: - import json -except ImportError: - import simplejson as json - +import json +import riak_pb from riak import RiakError -from riak.mapreduce import RiakLink -from riak.riak_index_entry import RiakIndexEntry from riak.transports.transport import RiakTransport -from riak.transports.pbc.connection import RiakPbcConnection -from riak.transports.pbc.stream import RiakPbcKeyStream, RiakPbcMapredStream -from riak.transports.pbc.codec import RiakPbcCodec -from riak.transports.pbc.messages import * -import riak.util +from connection import RiakPbcConnection +from stream import RiakPbcKeyStream, RiakPbcMapredStream +from codec import RiakPbcCodec +from messages import ( + # MSG_CODE_ERROR_RESP, + MSG_CODE_PING_REQ, + MSG_CODE_PING_RESP, + MSG_CODE_GET_CLIENT_ID_REQ, + MSG_CODE_GET_CLIENT_ID_RESP, + MSG_CODE_SET_CLIENT_ID_REQ, + MSG_CODE_SET_CLIENT_ID_RESP, + MSG_CODE_GET_SERVER_INFO_REQ, + MSG_CODE_GET_SERVER_INFO_RESP, + MSG_CODE_GET_REQ, + MSG_CODE_GET_RESP, + MSG_CODE_PUT_REQ, + MSG_CODE_PUT_RESP, + MSG_CODE_DEL_REQ, + MSG_CODE_DEL_RESP, + MSG_CODE_LIST_BUCKETS_REQ, + MSG_CODE_LIST_BUCKETS_RESP, + MSG_CODE_LIST_KEYS_REQ, + # MSG_CODE_LIST_KEYS_RESP, + MSG_CODE_GET_BUCKET_REQ, + MSG_CODE_GET_BUCKET_RESP, + MSG_CODE_SET_BUCKET_REQ, + MSG_CODE_SET_BUCKET_RESP, + MSG_CODE_MAPRED_REQ, + # MSG_CODE_MAPRED_RESP, + MSG_CODE_INDEX_REQ, + MSG_CODE_INDEX_RESP, + MSG_CODE_SEARCH_QUERY_REQ, + MSG_CODE_SEARCH_QUERY_RESP + ) + + +# from messages import * class RiakPbcTransport(RiakTransport, RiakPbcConnection, RiakPbcCodec): @@ -79,15 +106,12 @@ def get_server_info(self): expect=MSG_CODE_GET_SERVER_INFO_RESP) return {'node': resp.node, 'server_version': resp.server_version} - @property - def client_id(self): - """the client ID for this connection""" + def _get_client_id(self): msg_code, resp = self._request(MSG_CODE_GET_CLIENT_ID_REQ, expect=MSG_CODE_GET_CLIENT_ID_RESP) return resp.client_id - @client_id.setter - def client_id(self, client_id): + def _set_client_id(self, client_id): req = riak_pb.RpbSetClientIdReq() req.client_id = client_id @@ -96,6 +120,9 @@ def client_id(self, client_id): self._client_id = client_id + client_id = property(_get_client_id, _set_client_id, + doc="""the client ID for this connection""") + def get(self, robj, r=None, pr=None, vtag=None): """ Serialize get request and deserialize response @@ -152,9 +179,9 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, if vclock: req.vclock = vclock - self.pbify_content(robj.metadata, - robj.get_encoded_data(), - req.content) + self.encode_content(robj.metadata, + robj.get_encoded_data(), + req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, MSG_CODE_PUT_RESP) @@ -191,9 +218,9 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, req.bucket = bucket.name - self.pbify_content(robj.metadata, - robj.get_encoded_data(), - req.content) + self.encode_content(robj.metadata, + robj.get_encoded_data(), + req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, MSG_CODE_PUT_RESP) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 975d1dac..25e2cba1 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -17,7 +17,6 @@ specific language governing permissions and limitations under the License. """ -from riak import RiakError import base64 import random import threading @@ -31,15 +30,15 @@ class RiakTransport(FeatureDetection): Class to encapsulate transport details """ - @property - def client_id(self): - """the client ID for this connection""" + def _get_client_id(self): return self._client_id - @client_id.setter - def client_id(self, value): + def _set_client_id(self, value): self._client_id = value + client_id = property(_get_client_id, _set_client_id, + doc="""the client ID for this connection""") + @classmethod def make_random_client_id(self): """ diff --git a/riak/util.py b/riak/util.py index 6d58dcd8..c590db53 100644 --- a/riak/util.py +++ b/riak/util.py @@ -1,10 +1,23 @@ -import warnings +""" +Copyright 2010 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 -try: - from collections import Mapping -except ImportError: - # compatibility with Python 2.5 - Mapping = dict +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import warnings +from collections import Mapping def quacks_like_dict(object): From 991df2b368cd53c19948bd765a08e1fcc0cc0fec Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Fri, 28 Dec 2012 18:21:03 -0500 Subject: [PATCH 14/45] Fixed a bunch of test failures and stupid typos/bugs. --- riak/bucket.py | 2 +- riak/client/__init__.py | 2 +- riak/client/operations.py | 2 +- riak/client/transport.py | 13 +++++---- riak/mapreduce.py | 7 +++-- riak/riak_object.py | 9 +++--- riak/search.py | 4 +-- riak/tests/test_all.py | 8 +++--- riak/tests/test_search.py | 44 +++++++++++++----------------- riak/transports/http/__init__.py | 6 ++-- riak/transports/http/connection.py | 4 ++- riak/transports/http/resources.py | 15 ++++++---- riak/transports/pbc/connection.py | 4 +-- riak/transports/pbc/stream.py | 2 +- riak/transports/pbc/transport.py | 10 +++---- 15 files changed, 67 insertions(+), 65 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index eeed19fa..9560026f 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -379,7 +379,7 @@ def search(self, query, **params): """ Queries a search index over objects in this bucket/index. """ - return self._client.solr().search(self.name, query, **params) + return self._client.solr.search(self.name, query, **params) def get_index(self, index, startkey, endkey=None): """ diff --git a/riak/client/__init__.py b/riak/client/__init__.py index c5619532..4b816551 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -128,7 +128,7 @@ def set_client_id(self, client_id): def _get_client_id(self): with self._transport() as transport: - return transport.get_client_id() + return transport.client_id def _set_client_id(self, client_id): for http in self._http_pool: diff --git a/riak/client/operations.py b/riak/client/operations.py index 49fa25e4..5a5614ab 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -146,7 +146,7 @@ def fulltext_add(self, index, docs): Adds documents to the full-text index. """ with self._transport(protocol='http') as transport: - transport.fulltext_add(self, index, docs) + transport.fulltext_add(index, docs) def fulltext_delete(self, index, docs=None, queries=None): """ diff --git a/riak/client/transport.py b/riak/client/transport.py index 4f449994..c6db8823 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -41,7 +41,7 @@ def _transport(self, protocol=None): else: raise ValueError("invalid protocol %s" % protocol) - with self._retryable(pool) as transport: + with pool.take() as transport: yield transport @contextmanager @@ -57,19 +57,20 @@ def _skip_bad_nodes(transport): try: yield transport except (IOError, httplib.HTTPException) as e: - if is_pbc_retryable(e) or is_http_retryable(e): + if is_retryable(e): transport._node.error_rate.incr(1) skip_nodes.append(transport._node) raise BadResource(e) else: raise e except BadResource as br: - if retry < (self.RETRY_COUNT - 1): - continue - else: - raise br.args[0] + continue # These will be set or redefined by the RiakClient initializer protocol = 'http' _http_pool = None _pb_pool = None + + +def is_retryable(error): + return is_pbc_retryable(error) or is_http_retryable(error) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index d64108cc..4bd7712c 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -18,8 +18,6 @@ under the License. """ import urllib -from riak_object import RiakObject -from bucket import RiakBucket from collections import Iterable @@ -241,6 +239,8 @@ def _normalize_query(self): # If there are no phases, return the keys as links if num_phases is 0: link_results_flag = True + else: + link_results_flag = False # Convert all phases to associative arrays. Also, # if none of the phases are accumulating, then set the last one to @@ -621,3 +621,6 @@ def reduce(self, *args): """ mr = RiakMapReduce(self) return apply(mr.reduce, args) + +from riak.riak_object import RiakObject +from riak.bucket import RiakBucket diff --git a/riak/riak_object.py b/riak/riak_object.py index b37e4ebd..3bad9519 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -30,12 +30,12 @@ # MD_VTAG, # MD_DELETED ) -from mapreduce import ( +from riak.mapreduce import ( RiakMapReduce, RiakLink ) from riak import RiakError -from riak_index_entry import RiakIndexEntry +from riak.riak_index_entry import RiakIndexEntry class RiakObject(object): @@ -81,6 +81,8 @@ def _set_data(self, data, content_type=None): self.content_type = "application/json" else: self.content_type = "application/octet-stream" + if content_type: + self.content_type = content_type self._data = data return self @@ -382,9 +384,6 @@ def store(self, w=None, dw=None, pw=None, return_body=True, raise RiakError("Attempting to store an invalid object," "store one of the siblings instead") - # Issue the put over our transport - # t = self.client.get_transport() - if self.key is None: key, vclock, metadata = self.client.put_new( self, w=w, dw=dw, pw=pw, diff --git a/riak/search.py b/riak/search.py index 6a25c9eb..b340ba53 100644 --- a/riak/search.py +++ b/riak/search.py @@ -22,12 +22,12 @@ def __init__(self, client, **unused_args): self._client = client def add(self, index, *docs): - self._client.fulltext_add(index, docs) + self._client.fulltext_add(index, docs=docs) index = add def delete(self, index, docs=None, queries=None): - self._client.fulltext_delete(index, docs, queries) + self._client.fulltext_delete(index, docs=docs, queries=queries) remove = delete diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 7d9611a8..2d5c5d20 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -10,9 +10,9 @@ else: import unittest -from riak import RiakClient -from riak.mapreduce import RiakLink -from riak import RiakKeyFilter, key_filter +from riak.client import RiakClient +from riak.mapreduce import RiakLink, RiakKeyFilter +from riak import key_filter from riak.test_server import TestServer @@ -104,7 +104,7 @@ def setUp(self): def test_uses_client_id_if_given(self): zero_client_id = "\0\0\0\0" c = self.create_client(client_id=zero_client_id) - self.assertEqual(zero_client_id, c.get_client_id()) + self.assertEqual(zero_client_id, c.client_id) # def test_close_underlying_socket_fails(self): # self.skipTest("TODO: No longer using connection manager, replace") diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index f3223da0..a56ec63c 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -35,61 +35,55 @@ def test_disable_search_commit_hook(self): class SolrSearchTests(object): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_document_to_index(self): - self.client.solr().add("searchbucket", + self.client.solr.add("searchbucket", {"id": "doc", "username": "tony"}) - results = self.client.solr().search("searchbucket", "username:tony") + results = self.client.solr.search("searchbucket", "username:tony") self.assertEquals("tony", results['docs'][0]['username']) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_multiple_documents_to_index(self): - self.client.solr().add("searchbucket", + self.client.solr.add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - results = self.client.solr()\ + results = self.client.solr\ .search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(2, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_id(self): - self.client.solr().add("searchbucket", + self.client.solr.add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - self.client.solr().delete("searchbucket", docs=["dizzy"]) - results = self.client.solr()\ + self.client.solr.delete("searchbucket", docs=["dizzy"]) + results = self.client.solr\ .search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): - self.client.solr().add("searchbucket", + self.client.solr.add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - self.client.solr()\ + self.client.solr\ .delete("searchbucket", queries=["username:dizzy", "username:russell"]) - results = self.client.solr()\ + results = self.client.solr\ .search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query_and_id(self): - self.client.solr().add("searchbucket", + self.client.solr.add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) - self.client.solr().delete("searchbucket", + self.client.solr.delete("searchbucket", docs=["dizzy"], queries=["username:russell"]) - results = self.client.solr()\ + results = self.client.solr\ .search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results['docs'])) - def test_build_rest_path_excludes_empty_query_params(self): - self.assertEquals( - self.client.get_transport().build_rest_path( - bucket=self.client.bucket("foo"), - key="bar", params={'r': None}), "/riak/foo/bar?") - class SearchTests(object): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') @@ -110,16 +104,16 @@ def test_solr_search_with_params_from_bucket(self): def test_solr_search_with_params(self): bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr().search("searchbucket", - "username:roidrage", wt="xml") + results = self.client.solr.search("searchbucket", + "username:roidrage", wt="xml") self.assertEquals(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search(self): bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr().search("searchbucket", - "username:roidrage") + results = self.client.solr.search("searchbucket", + "username:roidrage") self.assertEquals(1, len(results["docs"])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') @@ -133,7 +127,7 @@ def test_search_integration(self): bucket.new("five", {"foo": "five", "bar": "yellow"}).store() # Run some operations... - results = self.client.solr().search("searchbucket", + results = self.client.solr.search("searchbucket", "foo:one OR foo:two") if (len(results) == 0): print "\n\nNot running test \"testSearchIntegration()\".\n" @@ -144,5 +138,5 @@ def test_search_integration(self): self.assertEqual(len(results['docs']), 2) query = "(foo:one OR foo:two OR foo:three OR foo:four) AND\ (NOT bar:green)" - results = self.client.solr().search("searchbucket", query) + results = self.client.solr.search("searchbucket", query) self.assertEqual(len(results['docs']), 3) diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index f3461e2e..d6069444 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -31,13 +31,13 @@ def __init__(self, client, **options): self.client = client self.options = options if client.protocol is 'https': - self.connection_class = httplib.HTTPConnection - else: self.connection_class = httplib.HTTPSConnection + else: + self.connection_class = httplib.HTTPConnection super(RiakHttpPool, self).__init__() def create_resource(self): - node = self.client.choose_node() + node = self.client._choose_node() return RiakHttpTransport(node=node, client=self.client, connection_class=self.connection_class, diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py index 0eeb5425..09550d21 100644 --- a/riak/transports/http/connection.py +++ b/riak/transports/http/connection.py @@ -45,6 +45,7 @@ def _request(self, method, uri, headers={}, body=''): and return a 2-tuple containing a dictionary of response headers and the response body. """ + response = None try: self._connection.request(method, uri, body, headers) response = self._connection.getresponse() @@ -56,7 +57,8 @@ def _request(self, method, uri, headers={}, body=''): # TODO: Support streaming responses response_body = response.read() finally: - response.close() + if response: + response.close() return response_headers, response_body diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index 44d1b7ff..538a4126 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -109,7 +109,7 @@ def luwak_path(self, key=None): @lazy_property def riak_kv_wm_buckets(self): - return self.resources.get('riak_kv_wm_buckets') + return self.resources.get('riak_kv_wm_index') @lazy_property def riak_kv_wm_raw(self): @@ -159,12 +159,15 @@ def mkpath(*segments, **query): pathstring = '/'.join(segments) # Remove extra slashes pathstring = re.sub('/+', '/', pathstring) + # Add the query string if it exists - if len(query) > 0: - for key in query.keys(): - if query[key] is None: - query.pop(key) - pathstring += "?" + urlencode(query).lower() + _query = {} + for key in query: + if query[key] is not None: + _query[key] = query[key] + + if len(_query) > 0: + pathstring += "?" + urlencode(_query).lower() if not pathstring.startswith('/'): pathstring = '/' + pathstring diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index 1b55b843..06010fd9 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -120,8 +120,8 @@ def _recv_msg(self, expect=None): else: raise Exception("unknown msg code %s" % msg_code) if expect and msg_code != expect: - raise RiakError("unexpected protocol buffer message code: %d" - % msg_code) + raise RiakError("unexpected protocol buffer message code: %d, %s" + % (msg_code, repr(msg))) return msg_code, msg def _recv_pkt(self): diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index f63fb645..f8d85636 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -63,7 +63,7 @@ class RiakPbcKeyStream(RiakPbcStream): _expect = MSG_CODE_LIST_KEYS_RESP def next(self): - response = super(RiakPbcKeyStream, self).__next__() + response = super(RiakPbcKeyStream, self).next() return response.keys def _is_done(self, response): diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index ebe37685..1c3727a0 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -175,7 +175,7 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, req.bucket = bucket.name req.key = robj.key - vclock = robj.vclock() + vclock = robj.vclock if vclock: req.vclock = vclock @@ -223,7 +223,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, req.content) msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, - MSG_CODE_PUT_RESP) + MSG_CODE_PUT_RESP) if not resp: raise RiakError("missing response object") if len(resp.content) != 1: @@ -254,8 +254,8 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): if pw: req.pw = self.translate_rw_val(pw) - if self.tombstone_vclocks() and robj.vclock(): - req.vclock = robj.vclock() + if self.tombstone_vclocks() and robj.vclock: + req.vclock = robj.vclock req.bucket = bucket.name req.key = robj.key @@ -291,7 +291,7 @@ def get_buckets(self): Serialize bucket listing request and deserialize response """ msg_code, resp = self._request(MSG_CODE_LIST_BUCKETS_REQ, - MSG_CODE_LIST_BUCKETS_RESP) + expect=MSG_CODE_LIST_BUCKETS_RESP) return resp.buckets def get_bucket_props(self, bucket): From 75c2e5413e3ebd23ff23729ced42d8996e24f179 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 11:18:23 -0500 Subject: [PATCH 15/45] Fix the iteration logic in PBC streaming. --- riak/transports/pbc/stream.py | 38 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index f8d85636..b481f811 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -31,28 +31,26 @@ class RiakPbcStream(object): _expect = None def __init__(self, transport): + self.finished = False self.transport = transport def __iter__(self): return self def next(self): - expect = self._expect - try: - resp = self.transport._recv_msg(expect) - if(self._is_done(resp)): - raise StopIteration - else: - return resp - except StopIteration: - pass - except: - # TODO: which exceptions do we expect to be generated? - # Should we raise BadResource? + if self.finished: raise StopIteration + msg_code, resp = self.transport._recv_msg(self._expect) + if(self._is_done(resp)): + self.finished = True + + return resp + def _is_done(self, response): - raise NotImplementedError + # This could break if new messages don't name the field the + # same thing. + return response.done class RiakPbcKeyStream(RiakPbcStream): @@ -64,10 +62,11 @@ class RiakPbcKeyStream(RiakPbcStream): def next(self): response = super(RiakPbcKeyStream, self).next() - return response.keys - def _is_done(self, response): - return response.done + if response.done and not response.HasField('keys'): + raise StopIteration + + return response.keys class RiakPbcMapredStream(RiakPbcStream): @@ -80,7 +79,8 @@ class RiakPbcMapredStream(RiakPbcStream): def next(self): response = super(RiakPbcMapredStream, self).next() - return response.phase, json.loads(response.response) - def _is_done(self, response): - return response.done + if response.done and not response.HasField('response'): + raise StopIteration + + return response.phase, json.loads(response.response) From f263f2f35dcb92d4498bc64f991ba92e9f8c6c9a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 11:31:34 -0500 Subject: [PATCH 16/45] Fix get_buckets logic. --- riak/client/operations.py | 2 +- riak/tests/test_kv.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 5a5614ab..448802c4 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -33,7 +33,7 @@ def get_buckets(self): all keys stored in a cluster. """ with self._transport() as transport: - return [RiakBucket(self, name) for name in transport.get_buckets()] + return [self.bucket(name) for name in transport.get_buckets()] def ping(self): """ diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 08a16c04..c18d621e 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -239,7 +239,7 @@ def test_list_buckets(self): bucket = self.client.bucket("list_bucket") bucket.new("one", {"foo": "one", "bar": "red"}).store() buckets = self.client.get_buckets() - self.assertTrue("list_bucket" in buckets) + self.assertTrue(bucket in buckets) class HTTPBucketPropsTest(object): From 9adb48b3b4a1e806e4e02cead4d5095dfe6b5fa2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 11:36:06 -0500 Subject: [PATCH 17/45] Add a print format for RiakBucket. --- riak/bucket.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/riak/bucket.py b/riak/bucket.py index 9560026f..1749e546 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -386,3 +386,6 @@ def get_index(self, index, startkey, endkey=None): Queries a secondary index over objects in this bucket, returning keys. """ return self._client.get_index(self.name, index, startkey, endkey) + + def __str__(self): + return ''.format(self.name) From 3ebede28c150e2a05440eb428f89157c2f5f3b86 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 11:39:16 -0500 Subject: [PATCH 18/45] Fix PBC list-keys. --- riak/transports/pbc/stream.py | 2 +- riak/transports/pbc/transport.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index b481f811..69594228 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -63,7 +63,7 @@ class RiakPbcKeyStream(RiakPbcStream): def next(self): response = super(RiakPbcKeyStream, self).next() - if response.done and not response.HasField('keys'): + if response.done and len(response.keys) is 0: raise StopIteration return response.keys diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 1c3727a0..8969b338 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -270,7 +270,8 @@ def get_keys(self, bucket): """ keys = [] for keylist in self.stream_keys(bucket): - keys = keys + keylist + for key in keylist: + keys.append(key) return keys From 87a00bde6c6cbd95fc54bc72636e2515a7e76927 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 12:00:34 -0500 Subject: [PATCH 19/45] Fix search bug: shouldn't lowercase AND/OR/NOT in query string --- riak/transports/http/resources.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index 538a4126..4adcceea 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -163,11 +163,13 @@ def mkpath(*segments, **query): # Add the query string if it exists _query = {} for key in query: - if query[key] is not None: + if query[key] in [False, True]: + _query[key] = str(query[key]).lower() + elif query[key] is not None: _query[key] = query[key] if len(_query) > 0: - pathstring += "?" + urlencode(_query).lower() + pathstring += "?" + urlencode(_query) if not pathstring.startswith('/'): pathstring = '/' + pathstring From 8443fa089de5954802868bcf62c3f2221fab7cdf Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 12:03:21 -0500 Subject: [PATCH 20/45] Fix bug in index_path. --- riak/transports/http/resources.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index 4adcceea..3b930d2f 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -82,7 +82,8 @@ def index_path(self, bucket, index, start, finish=None, **options): if finish: finish = quote_plus(str(finish)) return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), - "index", quote_plus(str(start)), finish, **options) + "index", quote_plus(index), quote_plus(str(start)), + finish, **options) def solr_select_path(self, index, query, **options): if not self.riak_solr_searcher_wm: From d3bc5e7221017a80ea979923ce483442d6e6c7a6 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 12:20:50 -0500 Subject: [PATCH 21/45] Fix encoding differences for binary objects between HTTP/PBC, 2.6/2.7. ALL TESTS GREEN! --- riak/bucket.py | 4 ++++ riak/transports/pbc/codec.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/riak/bucket.py b/riak/bucket.py index 1749e546..2fa2d088 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -340,6 +340,10 @@ def new_binary_from_file(self, key, filename): """ binary_data = open(filename, "rb").read() mimetype, encoding = mimetypes.guess_type(filename) + if encoding: + binary_data = bytearray(binary_data, encoding) + else: + binary_data = bytearray(binary_data) if not mimetype: mimetype = 'application/octet-stream' return self.new_binary(key, binary_data, mimetype) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index ae9a67b9..b7df3eda 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -140,4 +140,4 @@ def encode_content(self, metadata, data, rpb_content): pb_link.bucket = link.get_bucket() pb_link.key = link.get_key() pb_link.tag = link.get_tag() - rpb_content.value = data + rpb_content.value = str(data) From 117f00cd04818212912f797562e771c5b47299a3 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Sun, 30 Dec 2012 12:25:51 -0500 Subject: [PATCH 22/45] Fix a few pyflakes and pep8 issues. --- riak/client/operations.py | 1 - riak/client/transport.py | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 448802c4..036f53af 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -16,7 +16,6 @@ under the License. """ -from riak.bucket import RiakBucket from transport import RiakClientTransport diff --git a/riak/client/transport.py b/riak/client/transport.py index c6db8823..39d9aa33 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -30,6 +30,11 @@ class RiakClientTransport(object): RETRY_COUNT = 3 + # These will be set or redefined by the RiakClient initializer + protocol = 'http' + _http_pool = None + _pb_pool = None + @contextmanager def _transport(self, protocol=None): if not protocol: @@ -63,14 +68,9 @@ def _skip_bad_nodes(transport): raise BadResource(e) else: raise e - except BadResource as br: + except BadResource: continue - # These will be set or redefined by the RiakClient initializer - protocol = 'http' - _http_pool = None - _pb_pool = None - def is_retryable(error): return is_pbc_retryable(error) or is_http_retryable(error) From 6c3d67338847ab55fb31828efdfb75f10344ceca Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 08:16:25 -0500 Subject: [PATCH 23/45] Remove capitalized methods from RiakHttpConnection. --- riak/transports/http/connection.py | 15 ---------- riak/transports/http/transport.py | 45 ++++++++++++++++-------------- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py index 09550d21..5f0a143c 100644 --- a/riak/transports/http/connection.py +++ b/riak/transports/http/connection.py @@ -24,21 +24,6 @@ class RiakHttpConnection(object): Connection and low-level request methods for RiakHttpTransport. """ - def GET(self, uri, headers={}): - return self._request('GET', uri, headers, '') - - def POST(self, uri, headers={}, body=''): - return self._request('POST', uri, headers, body) - - def PUT(self, uri, headers={}, body=''): - return self._request('PUT', uri, headers, body) - - def DELETE(self, uri, headers={}, body=''): - return self._request('DELETE', uri, headers, body) - - def HEAD(self, uri, headers={}): - return self._request('HEAD', uri, headers, '') - def _request(self, method, uri, headers={}, body=''): """ Given a Method, URL, Headers, and Body, perform and HTTP request, diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 342188bd..183315dc 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -80,14 +80,15 @@ def ping(self): """ Check server is alive over HTTP """ - response = self.GET(self.ping_path()) + response = self._request('GET', self.ping_path()) return(response is not None) and (response[1] == 'OK') def stats(self): """ Gets performance statistics and server information """ - response = self.GET(self.stats_path(), {'Accept': 'application/json'}) + response = self._request('GET', self.stats_path(), + {'Accept': 'application/json'}) if response[0]['http_code'] is 200: return json.loads(response[1]) else: @@ -111,7 +112,7 @@ def get_resources(self): Gets a JSON mapping of server-side resource names to paths :rtype dict """ - response = self.GET('/', {'Accept': 'application/json'}) + response = self._request('GET', '/', {'Accept': 'application/json'}) if response[0]['http_code'] is 200: return json.loads(response[1]) else: @@ -125,7 +126,7 @@ def get(self, robj, r=None, pr=None, vtag=None): # unknown flags/params. params = {'r': r, 'pr': pr, 'vtag': vtag} url = self.object_path(robj.bucket.name, robj.key, **params) - response = self.GET(url) + response = self._request('GET', url) return self.parse_body(response, [200, 300, 404]) def put(self, robj, w=None, dw=None, pw=None, return_body=True, @@ -149,9 +150,9 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, def do_put(self, url, headers, content, return_body=False, key=None): if key is None: - response = self.POST(url, headers, content) + response = self._request('POST', url, headers, content) else: - response = self.PUT(url, headers, content) + response = self._request('PUT', url, headers, content) if return_body: return self.parse_body(response, [200, 201, 300]) @@ -172,7 +173,7 @@ def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if if_none_match: headers["If-None-Match"] = "*" content = robj.get_encoded_data() - response = self.POST(url, headers, content) + response = self._request('POST', url, headers, content) location = response[0]['location'] idx = location.rindex('/') key = location[(idx + 1):] @@ -194,7 +195,7 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): url = self.object_path(robj.bucket.name, robj.key, **params) if self.tombstone_vclocks() and robj.vclock is not None: headers['X-Riak-Vclock'] = robj.vclock - response = self.DELETE(url, headers) + response = self._request('DELETE', url, headers) self.check_http_code(response, [204, 404]) return self @@ -203,7 +204,7 @@ def get_keys(self, bucket): Fetch a list of keys for the bucket """ url = self.key_list_path(bucket.name) - response = self.GET(url) + response = self._request('GET', url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: @@ -217,7 +218,7 @@ def get_buckets(self): Fetch a list of all buckets """ url = self.bucket_list_path() - response = self.GET(url) + response = self._request('GET', url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: @@ -232,7 +233,7 @@ def get_bucket_props(self, bucket): """ # Run the request... url = self.bucket_properties_path(bucket.name) - response = self.GET(url) + response = self._request('GET', url) headers = response[0] encoded_props = response[1] @@ -251,7 +252,7 @@ def set_bucket_props(self, bucket, props): content = json.dumps({'props': props}) # Run the request... - response = self.PUT(url, headers, content) + response = self._request('PUT', url, headers, content) # Handle the response... if response is None: @@ -281,7 +282,7 @@ def mapred(self, inputs, query, timeout=None): # Do the request... url = self.mapred_path() headers = {'Content-Type': 'application/json'} - response = self.POST(url, headers, content) + response = self._request('POST', url, headers, content) # Make sure the expected status code came back... status = response[0]['http_code'] @@ -297,7 +298,8 @@ def get_index(self, bucket, index, startkey, endkey=None): """ Performs a secondary index query. """ - response = self.GET(self.index_path(bucket, index, startkey, endkey)) + url = self.index_path(bucket, index, startkey, endkey) + response = self._request('GET', url) headers, data = response self.check_http_code(response, [200]) jsonData = json.loads(data) @@ -316,7 +318,8 @@ def search(self, index, query, **params): options['q.op'] = op options.update(params) - response = self.GET(self.solr_select_path(index, query, **options)) + url = self.solr_select_path(index, query, **options) + response = self._request('GET', url) headers, data = response self.check_http_code(response, [200]) if 'json' in headers['content-type']: @@ -345,9 +348,9 @@ def fulltext_add(self, index, docs): root.appendChild(doc_element) xml.appendChild(root) - self.POST(self.solr_update_path(index), - {'Content-Type': 'text/xml'}, - xml.toxml()) + self._request('POST', self.solr_update_path(index), + {'Content-Type': 'text/xml'}, + xml.toxml()) def fulltext_delete(self, index, docs=None, queries=None): """ @@ -370,9 +373,9 @@ def fulltext_delete(self, index, docs=None, queries=None): xml.appendChild(root) - self.POST(self.solr_update_path(index), - {'Content-Type': 'text/xml'}, - xml.toxml()) + self._request('POST', self.solr_update_path(index), + {'Content-Type': 'text/xml'}, + xml.toxml()) def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] From 88f691cff147eab47a24159a2d3fe25b3417790c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 08:28:00 -0500 Subject: [PATCH 24/45] Spec out missing methods. --- riak/transports/transport.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 25e2cba1..b8a82f62 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -119,9 +119,27 @@ def set_bucket_props(self, bucket, props): """ raise NotImplementedError + def get_keys(self, bucket): + """ + Lists all keys within the given bucket. + """ + raise NotImplementedError + + def stream_keys(self, bucket): + """ + Streams the list of keys for the bucket through an iterator. + """ + raise NotImplementedError + def mapred(self, inputs, query, timeout=None): """ - Serialize map/reduce request + Sends a MapReduce request synchronously. + """ + raise NotImplementedError + + def stream_mapred(self, inputs, query, timeout=None): + """ + Streams the results of a MapReduce request through an iterator. """ raise NotImplementedError From af2facaef8ed4771e23f3c50f25231b970007648 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 08:57:52 -0500 Subject: [PATCH 25/45] Use a dict to match message code with protobuf message, simplifying decode logic. --- riak/transports/pbc/connection.py | 97 ++++++++----------------------- riak/transports/pbc/messages.py | 46 ++++++++++++++- 2 files changed, 69 insertions(+), 74 deletions(-) diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index 06010fd9..1f3966de 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -18,38 +18,10 @@ import socket import struct -import riak_pb from riak import RiakError from messages import ( - MSG_CODE_ERROR_RESP, - # MSG_CODE_PING_REQ, - MSG_CODE_PING_RESP, - # MSG_CODE_GET_CLIENT_ID_REQ, - MSG_CODE_GET_CLIENT_ID_RESP, - # MSG_CODE_SET_CLIENT_ID_REQ, - MSG_CODE_SET_CLIENT_ID_RESP, - # MSG_CODE_GET_SERVER_INFO_REQ, - MSG_CODE_GET_SERVER_INFO_RESP, - # MSG_CODE_GET_REQ, - MSG_CODE_GET_RESP, - # MSG_CODE_PUT_REQ, - MSG_CODE_PUT_RESP, - # MSG_CODE_DEL_REQ, - MSG_CODE_DEL_RESP, - # MSG_CODE_LIST_BUCKETS_REQ, - MSG_CODE_LIST_BUCKETS_RESP, - # MSG_CODE_LIST_KEYS_REQ, - MSG_CODE_LIST_KEYS_RESP, - # MSG_CODE_GET_BUCKET_REQ, - MSG_CODE_GET_BUCKET_RESP, - # MSG_CODE_SET_BUCKET_REQ, - MSG_CODE_SET_BUCKET_RESP, - # MSG_CODE_MAPRED_REQ, - MSG_CODE_MAPRED_RESP, - # MSG_CODE_INDEX_REQ, - MSG_CODE_INDEX_RESP, - # MSG_CODE_SEARCH_QUERY_REQ, - MSG_CODE_SEARCH_QUERY_RESP + MESSAGE_CLASSES, + MSG_CODE_ERROR_RESP ) @@ -57,6 +29,7 @@ class RiakPbcConnection(object): """ Connection-related methods for RiakPbcTransport. """ + def _encode_msg(self, msg_code, msg=None): if msg is None: return struct.pack("!iB", 1, msg_code) @@ -75,53 +48,18 @@ def _send_msg(self, msg_code, msg): def _recv_msg(self, expect=None): self._recv_pkt() msg_code, = struct.unpack("B", self._inbuf[:1]) - if msg_code == MSG_CODE_ERROR_RESP: - err = riak_pb.RpbErrorResp() - err.ParseFromString(self._inbuf[1:]) + + if msg_code is MSG_CODE_ERROR_RESP: + err = self._parse_msg(msg_code, self._inbuf[1:]) raise RiakError(err.errmsg) - elif msg_code == MSG_CODE_PING_RESP: - msg = None - elif msg_code == MSG_CODE_GET_SERVER_INFO_RESP: - msg = riak_pb.RpbGetServerInfoResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_GET_CLIENT_ID_RESP: - msg = riak_pb.RpbGetClientIdResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_SET_CLIENT_ID_RESP: - msg = None - elif msg_code == MSG_CODE_GET_RESP: - msg = riak_pb.RpbGetResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_PUT_RESP: - msg = riak_pb.RpbPutResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_DEL_RESP: - msg = None - elif msg_code == MSG_CODE_LIST_KEYS_RESP: - msg = riak_pb.RpbListKeysResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_LIST_BUCKETS_RESP: - msg = riak_pb.RpbListBucketsResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_GET_BUCKET_RESP: - msg = riak_pb.RpbGetBucketResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_SET_BUCKET_RESP: - msg = None - elif msg_code == MSG_CODE_MAPRED_RESP: - msg = riak_pb.RpbMapRedResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_INDEX_RESP: - msg = riak_pb.RpbIndexResp() - msg.ParseFromString(self._inbuf[1:]) - elif msg_code == MSG_CODE_SEARCH_QUERY_RESP: - msg = riak_pb.RpbSearchQueryResp() - msg.ParseFromString(self._inbuf[1:]) + elif msg_code in MESSAGE_CLASSES: + msg = self._parse_msg(msg_code, self._inbuf[1:]) else: raise Exception("unknown msg code %s" % msg_code) + if expect and msg_code != expect: - raise RiakError("unexpected protocol buffer message code: %d, %s" - % (msg_code, repr(msg))) + raise RiakError("unexpected protocol buffer message code: %d, %r" + % (msg_code, msg)) return msg_code, msg def _recv_pkt(self): @@ -150,6 +88,19 @@ def _connect(self): def close(self): self._socket.shutdown(socket.SHUT_RDWR) + def _parse_msg(self, code, packet): + try: + pbclass = MESSAGE_CLASSES[code] + except KeyError: + pbclass = None + + if pbclass is None: + return None + + pbo = pbclass() + pbo.ParseFromString(packet) + return pbo + # These are set in the RiakPbcTransport initializer _address = None _timeouts = {} diff --git a/riak/transports/pbc/messages.py b/riak/transports/pbc/messages.py index c8175867..fb9dba5c 100644 --- a/riak/transports/pbc/messages.py +++ b/riak/transports/pbc/messages.py @@ -16,7 +16,10 @@ under the License. """ -## Protocol codes +import riak_pb + + +# Protocol codes MSG_CODE_ERROR_RESP = 0 MSG_CODE_PING_REQ = 1 MSG_CODE_PING_RESP = 2 @@ -46,3 +49,44 @@ MSG_CODE_INDEX_RESP = 26 MSG_CODE_SEARCH_QUERY_REQ = 27 MSG_CODE_SEARCH_QUERY_RESP = 28 + +# These responses don't include messages +EMPTY_RESPONSES = [ + MSG_CODE_PING_RESP, + MSG_CODE_SET_CLIENT_ID_RESP, + MSG_CODE_DEL_RESP, + MSG_CODE_SET_BUCKET_RESP +] + +# Mapping from code to protobuf class +MESSAGE_CLASSES = { + MSG_CODE_ERROR_RESP: riak_pb.RpbErrorResp, + MSG_CODE_PING_REQ: None, + MSG_CODE_PING_RESP: None, + MSG_CODE_GET_CLIENT_ID_REQ: None, + MSG_CODE_GET_CLIENT_ID_RESP: riak_pb.RpbGetClientIdResp, + MSG_CODE_SET_CLIENT_ID_REQ: riak_pb.RpbSetClientIdReq, + MSG_CODE_SET_CLIENT_ID_RESP: None, + MSG_CODE_GET_SERVER_INFO_REQ: None, + MSG_CODE_GET_SERVER_INFO_RESP: riak_pb.RpbGetServerInfoResp, + MSG_CODE_GET_REQ: riak_pb.RpbGetReq, + MSG_CODE_GET_RESP: riak_pb.RpbGetResp, + MSG_CODE_PUT_REQ: riak_pb.RpbPutReq, + MSG_CODE_PUT_RESP: riak_pb.RpbPutResp, + MSG_CODE_DEL_REQ: riak_pb.RpbDelReq, + MSG_CODE_DEL_RESP: None, + MSG_CODE_LIST_BUCKETS_REQ: None, + MSG_CODE_LIST_BUCKETS_RESP: riak_pb.RpbListBucketsResp, + MSG_CODE_LIST_KEYS_REQ: riak_pb.RpbListKeysReq, + MSG_CODE_LIST_KEYS_RESP: riak_pb.RpbListKeysResp, + MSG_CODE_GET_BUCKET_REQ: riak_pb.RpbGetBucketReq, + MSG_CODE_GET_BUCKET_RESP: riak_pb.RpbGetBucketResp, + MSG_CODE_SET_BUCKET_REQ: riak_pb.RpbSetBucketReq, + MSG_CODE_SET_BUCKET_RESP: None, + MSG_CODE_MAPRED_REQ: riak_pb.RpbMapRedReq, + MSG_CODE_MAPRED_RESP: riak_pb.RpbMapRedResp, + MSG_CODE_INDEX_REQ: riak_pb.RpbIndexReq, + MSG_CODE_INDEX_RESP: riak_pb.RpbIndexResp, + MSG_CODE_SEARCH_QUERY_REQ: riak_pb.RpbSearchQueryReq, + MSG_CODE_SEARCH_QUERY_RESP: riak_pb.RpbSearchQueryResp +} From 80476eeb8693b683c743c427335968d2bcb13e4c Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 09:00:05 -0500 Subject: [PATCH 26/45] Fix protobuf build dependency. --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3e63c8c1..a47837e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,9 @@ language: python python: - "2.6" - "2.7" -install: ./setup.py develop +install: + - ./setup.py develop + - ./setup.py easy_install protobuf script: ./setup.py test before_script: sudo /usr/sbin/search-cmd install searchbucket notifications: From 2adb47d8814a77e8c0d2a7a861c5d18f37ba69d3 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 12:18:02 -0500 Subject: [PATCH 27/45] Add key streaming to HTTP. --- riak/transports/http/connection.py | 12 ++++-- riak/transports/http/resources.py | 4 +- riak/transports/http/stream.py | 66 ++++++++++++++++++++++++++++++ riak/transports/http/transport.py | 13 +++++- 4 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 riak/transports/http/stream.py diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py index 5f0a143c..1a8dca10 100644 --- a/riak/transports/http/connection.py +++ b/riak/transports/http/connection.py @@ -24,7 +24,7 @@ class RiakHttpConnection(object): Connection and low-level request methods for RiakHttpTransport. """ - def _request(self, method, uri, headers={}, body=''): + def _request(self, method, uri, headers={}, body='', stream=False): """ Given a Method, URL, Headers, and Body, perform and HTTP request, and return a 2-tuple containing a dictionary of response headers @@ -39,10 +39,14 @@ def _request(self, method, uri, headers={}, body=''): for (key, value) in response.getheaders(): response_headers[key.lower()] = value - # TODO: Support streaming responses - response_body = response.read() + if stream: + # The caller is responsible for fully reading the + # response and closing it when streaming. + response_body = response + else: + response_body = response.read() finally: - if response: + if response and not stream: response.close() return response_headers, response_body diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index 3b930d2f..b551cbae 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -55,8 +55,8 @@ def bucket_properties_path(self, bucket, **options): return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), **query) def key_list_path(self, bucket, **options): - query = options.copy() - query.update(keys=True, props=False) + query = {'keys': True, 'props': False} + query.update(options) if self.riak_kv_wm_buckets: return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), "keys", **query) diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py new file mode 100644 index 00000000..46d57cac --- /dev/null +++ b/riak/transports/http/stream.py @@ -0,0 +1,66 @@ +""" +Copyright 2012 Basho Technologies, Inc. + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import json +import string + +class RiakHttpStream(object): + """ + Base class for HTTP streaming iterators. + """ + + BLOCK_SIZE = 2048 + + def __init__(self, response): + self.response = response + self.buffer = '' + self.response_done = False + + def __iter__(self): + return self + + def read(self): + chunk = self.response.read(self.BLOCK_SIZE) + if chunk is '': + self.response_done = True + self.buffer += chunk + + def next(self): + raise NotImplementedError + +class RiakHttpKeyStream(RiakHttpStream): + """ + Streaming iterator for list-keys over HTTP + """ + + def next(self): + while True: + while '}' not in self.buffer and not self.response_done: + self.read() + + if '}' in self.buffer: + idx = string.index(self.buffer, '}') + 1 + chunk = self.buffer[:idx] + self.buffer = self.buffer[idx:] + keys = json.loads(chunk)[u'keys'] + if len(keys) is 0: + continue + else: + return keys + else: + raise StopIteration diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 183315dc..ead726bd 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -28,6 +28,7 @@ from riak.transports.http.resources import RiakHttpResources from riak.transports.http.connection import RiakHttpConnection from riak.transports.http.search import XMLSearchResult +from riak.transports.http.stream import RiakHttpKeyStream from riak.metadata import ( MD_CHARSET, MD_CTYPE, @@ -207,12 +208,22 @@ def get_keys(self, bucket): response = self._request('GET', url) headers, encoded_props = response[0:2] - if headers['http_code'] == 200: + if headers['http_code'] is 200: props = json.loads(encoded_props) return props['keys'] else: raise Exception('Error listing keys.') + def stream_keys(self, bucket): + url = self.key_list_path(bucket.name, keys='stream') + headers, response = self._request('GET', url, stream=True) + + if headers['http_code'] is 200: + return RiakHttpKeyStream(response) + else: + stream.close() + raise Exception('Error listing keys.') + def get_buckets(self): """ Fetch a list of all buckets From 44a25e598bf4d8b0a34d99689a5d0ddaf84332cd Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 13:10:22 -0500 Subject: [PATCH 28/45] Simplify HTTP key streaming by pushing empty-result culling into the client. --- riak/client/operations.py | 3 ++- riak/transports/http/stream.py | 26 +++++++++++--------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 036f53af..e9fd8ee1 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -80,7 +80,8 @@ def stream_keys(self, bucket): """ with self._transport() as transport: for keylist in transport.stream_keys(bucket): - yield keylist + if len(keylist) > 0: + yield keylist def put(self, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=None): diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 46d57cac..77a84f6f 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -49,18 +49,14 @@ class RiakHttpKeyStream(RiakHttpStream): """ def next(self): - while True: - while '}' not in self.buffer and not self.response_done: - self.read() - - if '}' in self.buffer: - idx = string.index(self.buffer, '}') + 1 - chunk = self.buffer[:idx] - self.buffer = self.buffer[idx:] - keys = json.loads(chunk)[u'keys'] - if len(keys) is 0: - continue - else: - return keys - else: - raise StopIteration + while '}' not in self.buffer and not self.response_done: + self.read() + + if '}' in self.buffer: + idx = string.index(self.buffer, '}') + 1 + chunk = self.buffer[:idx] + self.buffer = self.buffer[idx:] + keys = json.loads(chunk)[u'keys'] + return keys + else: + raise StopIteration From 30046643b5f9fdc051e4c51c64fdf852a2c0ab5a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 13:52:56 -0500 Subject: [PATCH 29/45] Add unit test for key streaming. --- riak/tests/test_kv.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index c18d621e..5b4a5e90 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -72,6 +72,18 @@ def test_generate_key(self): self.assertNotIn(o.key, existing_keys) self.assertEqual(len(bucket.get_keys()), len(existing_keys) + 1) + def test_stream_keys(self): + bucket = self.client.bucket('random_key_bucket') + regular_keys = bucket.get_keys() + self.assertNotEqual(len(regular_keys), 0) + streamed_keys = [] + for keylist in bucket.stream_keys(): + self.assertNotEqual([], keylist) + for key in keylist: + self.assertIsInstance(key, basestring) + streamed_keys += keylist + self.assertEqual(sorted(regular_keys), sorted(streamed_keys)) + def test_binary_store_and_get(self): bucket = self.client.bucket('bucket') # Store as binary, retrieve as binary, then compare... From 87ba4422403ca39ebb0d42157fd3d94c2c46cb5a Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Mon, 31 Dec 2012 17:17:40 -0500 Subject: [PATCH 30/45] Add streaming MapReduce to HTTP. --- riak/transports/http/stream.py | 57 +++++++++++++++++++++++++++++++ riak/transports/http/transport.py | 30 ++++++++++------ riak/transports/pbc/transport.py | 6 +--- riak/transports/transport.py | 13 +++++++ 4 files changed, 91 insertions(+), 15 deletions(-) diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 77a84f6f..960fd3c3 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -18,6 +18,9 @@ import json import string +import re +from cgi import parse_header +from email import message_from_string class RiakHttpStream(object): """ @@ -60,3 +63,57 @@ def next(self): return keys else: raise StopIteration + +class RiakHttpMultipartStream(RiakHttpStream): + """ + Streaming iterator for multipart messages over HTTP + """ + def __init__(self, response): + super(RiakHttpMultipartStream, self).__init__(response) + ctypehdr = response.getheader('content-type') + _, params = parse_header(ctypehdr) + self.boundary_re = re.compile('\r?\n--%s(?:--)?\r?\n' % + re.escape(params['boundary'])) + self.next_boundary = None + self.seen_first = False + + def next(self): + # multipart/mixed starts with a boundary, then the first part. + if not self.seen_first: + self.read_until_boundary() + self.advance_buffer() + self.seen_first = True + + self.read_until_boundary() + + if self.next_boundary: + part = self.advance_buffer() + message = message_from_string(part) + return message + else: + raise StopIteration + + def try_match(self): + self.next_boundary = self.boundary_re.search(self.buffer) + return self.next_boundary + + def advance_buffer(self): + part = self.buffer[:self.next_boundary.start()] + self.buffer = self.buffer[self.next_boundary.end():] + self.next_boundary = None + return part + + def read_until_boundary(self): + while not self.try_match() and not self.response_done: + self.read() + + +class RiakHttpMapReduceStream(RiakHttpMultipartStream): + """ + Streaming iterator for MapReduce over HTTP + """ + + def next(self): + message = super(RiakHttpMapReduceStream, self).next() + payload = json.loads(message.get_payload()) + return payload['phase'], payload['data'] diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index ead726bd..8cbbcb7c 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -28,7 +28,10 @@ from riak.transports.http.resources import RiakHttpResources from riak.transports.http.connection import RiakHttpConnection from riak.transports.http.search import XMLSearchResult -from riak.transports.http.stream import RiakHttpKeyStream +from riak.transports.http.stream import ( + RiakHttpKeyStream, + RiakHttpMapReduceStream + ) from riak.metadata import ( MD_CHARSET, MD_CTYPE, @@ -279,16 +282,8 @@ def mapred(self, inputs, query, timeout=None): """ Run a MapReduce query. """ - if not self.phaseless_mapred() and (query is None or len(query) is 0): - raise Exception( - 'Phase-less MapReduce is not supported by Riak node') - # Construct the job, optionally set the timeout... - job = {'inputs': inputs, 'query': query} - if timeout is not None: - job['timeout'] = timeout - - content = json.dumps(job) + content = self._construct_mapred_json(inputs, query, timeout) # Do the request... url = self.mapred_path() @@ -305,6 +300,21 @@ def mapred(self, inputs, query, timeout=None): result = json.loads(response[1]) return result + def stream_mapred(self, inputs, query, timeout=None): + content = self._construct_mapred_json(inputs, query, timeout) + + url = self.mapred_path(chunked=True) + reqheaders = {'Content-Type': 'application/json'} + headers, response = self._request('POST', url, reqheaders, + content, stream=True) + + if headers['http_code'] is 200: + return RiakHttpMapReduceStream(response) + else: + raise Exception( + 'Error running MapReduce operation. Headers: %s Body: %s' % + (repr(headers), repr(response.read()))) + def get_index(self, bucket, index, startkey, endkey=None): """ Performs a secondary index query. diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 8969b338..bc60cc21 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -352,11 +352,7 @@ def mapred(self, inputs, query, timeout=None): def stream_mapred(self, inputs, query, timeout=None): # Construct the job, optionally set the timeout... - job = {'inputs': inputs, 'query': query} - if timeout is not None: - job['timeout'] = timeout - - content = json.dumps(job) + content = self._construct_mapred_json(inputs, query, timeout) req = riak_pb.RpbMapRedReq() req.request = content diff --git a/riak/transports/transport.py b/riak/transports/transport.py index b8a82f62..27d1fc52 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -22,6 +22,7 @@ import threading import platform import os +import json from feature_detect import FeatureDetection @@ -230,3 +231,15 @@ def _get_index_mapred_emu(self, bucket, index, startkey, endkey=None): 'key': startkey}, phases) return [key for bucket, key in result] + + def _construct_mapred_json(self, inputs, query, timeout=None): + if not self.phaseless_mapred() and (query is None or len(query) is 0): + raise Exception( + 'Phase-less MapReduce is not supported by Riak node') + + job = {'inputs': inputs, 'query': query} + if timeout is not None: + job['timeout'] = timeout + + content = json.dumps(job) + return content From 997d6e955d2426047003d537fdd278c4402badde Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 10 Jan 2013 12:29:40 -0600 Subject: [PATCH 31/45] Fix some 'is' problems. --- riak/client/transport.py | 2 +- riak/transports/http/__init__.py | 2 +- riak/transports/http/stream.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/riak/client/transport.py b/riak/client/transport.py index 39d9aa33..9a000e5e 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -41,7 +41,7 @@ def _transport(self, protocol=None): protocol = self.protocol if protocol in ['http', 'https']: pool = self._http_pool - elif protocol is 'pbc': + elif protocol == 'pbc': pool = self._pb_pool else: raise ValueError("invalid protocol %s" % protocol) diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index d6069444..e7cd660e 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -30,7 +30,7 @@ class RiakHttpPool(Pool): def __init__(self, client, **options): self.client = client self.options = options - if client.protocol is 'https': + if client.protocol == 'https': self.connection_class = httplib.HTTPSConnection else: self.connection_class = httplib.HTTPConnection diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 960fd3c3..5130a0dd 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -39,7 +39,7 @@ def __iter__(self): def read(self): chunk = self.response.read(self.BLOCK_SIZE) - if chunk is '': + if chunk == '': self.response_done = True self.buffer += chunk From 4b8636497a7365598de3a6604650364cbdd652ac Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 29 Jan 2013 11:10:54 -0600 Subject: [PATCH 32/45] Rethink the automatic retries stuff. --- riak/client/operations.py | 120 ++++++++++++++++++++------------------ riak/client/transport.py | 44 ++++++++------ 2 files changed, 88 insertions(+), 76 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index e9fd8ee1..efba5128 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -16,141 +16,145 @@ under the License. """ -from transport import RiakClientTransport - +from transport import RiakClientTransport, retryable, retryableHttpOnly class RiakClientOperations(RiakClientTransport): """ Methods for RiakClient that result in requests sent to the Riak cluster. + + Note that all of these methods have an implicit 'transport' + argument that will be prepended automatically as part of the retry + logic, and does not need to be supplied by the user. """ - def get_buckets(self): + @retryable + def get_buckets(self, transport): """ Get the list of buckets as RiakBucket instances. NOTE: Do not use this in production, as it requires traversing through all keys stored in a cluster. """ - with self._transport() as transport: - return [self.bucket(name) for name in transport.get_buckets()] + return [self.bucket(name) for name in transport.get_buckets()] - def ping(self): + @retryable + def ping(self, transport): """ Check if the Riak server for this ``RiakClient`` instance is alive. :rtype: boolean """ - with self._transport() as transport: - return transport.ping() + return transport.ping() is_alive = ping - def get_index(self, bucket, index, startkey, endkey=None): + @retryable + def get_index(self, transport, bucket, index, startkey, endkey=None): """ Queries a secondary index, returning matching keys. """ - with self._transport() as transport: - return transport.get_index(bucket, index, startkey, endkey) + return transport.get_index(bucket, index, startkey, endkey) - def get_bucket_props(self, bucket): + @retryable + def get_bucket_props(self, transport, bucket): """ Fetches bucket properties for the given bucket. """ - with self._transport() as transport: - return transport.get_bucket_props(bucket) + return transport.get_bucket_props(bucket) - def set_bucket_props(self, bucket, props): + @retryable + def set_bucket_props(self, transport, bucket, props): """ Sets bucket properties for the given bucket. """ - with self._transport() as transport: - return transport.set_bucket_props(bucket, props) + return transport.set_bucket_props(bucket, props) - def get_keys(self, bucket): + @retryable + def get_keys(self, transport, bucket): """ Lists all keys in a bucket. """ - with self._transport() as transport: - return transport.get_keys(bucket) + return transport.get_keys(bucket) - def stream_keys(self, bucket): + @retryable + def stream_keys(self, transport, bucket): """ Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. """ - with self._transport() as transport: - for keylist in transport.stream_keys(bucket): - if len(keylist) > 0: - yield keylist + for keylist in transport.stream_keys(bucket): + if len(keylist) > 0: + yield keylist - def put(self, robj, w=None, dw=None, pw=None, return_body=None, + @retryable + def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=None): """ Stores an object in the Riak cluster. """ - with self._transport() as transport: - return transport.put(robj, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) + return transport.put(robj, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) - def put_new(self, robj, w=None, dw=None, pw=None, return_body=None, + @retryable + def put_new(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=None): """ Stores an object in the Riak cluster with a generated key. """ - with self._transport() as transport: - return transport.put_new(robj, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) + return transport.put_new(robj, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match) - def get(self, robj, r=None, pr=None, vtag=None): + @retryable + def get(self, transport, robj, r=None, pr=None, vtag=None): """ Fetches the contents of a Riak object. """ - with self._transport() as transport: - return transport.get(robj, r=r, pr=pr, vtag=vtag) + return transport.get(robj, r=r, pr=pr, vtag=vtag) - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + @retryable + def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, pr=None, + pw=None): """ Deletes an object from Riak. """ - with self._transport() as transport: - return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, - pw=pw) + return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, + pw=pw) - def mapred(self, inputs, query, timeout): + @retryable + def mapred(self, transport, inputs, query, timeout): """ Executes a MapReduce query """ - with self._transport() as transport: - return transport.mapred(inputs, query, timeout) + return transport.mapred(inputs, query, timeout) - def stream_mapred(self, inputs, query, timeout): + @retryable + def stream_mapred(self, transport, inputs, query, timeout): """ Streams a MapReduce query as (phase, data) pairs. This is a generator method which should be iterated over. """ - with self._transport() as transport: - for phase, data in transport.stream_mapred(inputs, query, timeout): - yield phase, data + for phase, data in transport.stream_mapred(inputs, query, timeout): + yield phase, data - def fulltext_search(self, index, query, **params): + @retryableHttpOnly + def fulltext_search(self, transport, index, query, **params): """ Performs a full-text search query. """ - with self._transport() as transport: - return transport.search(index, query, **params) + return transport.search(index, query, **params) - def fulltext_add(self, index, docs): + @retryableHttpOnly + def fulltext_add(self, transport, index, docs): """ Adds documents to the full-text index. """ - with self._transport(protocol='http') as transport: - transport.fulltext_add(index, docs) + transport.fulltext_add(index, docs) - def fulltext_delete(self, index, docs=None, queries=None): + @retryableHttpOnly + def fulltext_delete(self, transport, index, docs=None, queries=None): """ Removes documents from the full-text index. """ - with self._transport(protocol='http') as transport: - transport.fulltext_delete(index, docs, queries) + transport.fulltext_delete(index, docs, queries) diff --git a/riak/client/transport.py b/riak/client/transport.py index 9a000e5e..fc46abb5 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -16,7 +16,6 @@ under the License. """ -from contextlib import contextmanager from riak.transports.pool import BadResource from riak.transports.pbc import is_retryable as is_pbc_retryable from riak.transports.http import is_retryable as is_http_retryable @@ -35,22 +34,7 @@ class RiakClientTransport(object): _http_pool = None _pb_pool = None - @contextmanager - def _transport(self, protocol=None): - if not protocol: - protocol = self.protocol - if protocol in ['http', 'https']: - pool = self._http_pool - elif protocol == 'pbc': - pool = self._pb_pool - else: - raise ValueError("invalid protocol %s" % protocol) - - with pool.take() as transport: - yield transport - - @contextmanager - def _retryable(self, pool): + def _with_retries(self, pool, fn): skip_nodes = [] def _skip_bad_nodes(transport): @@ -60,7 +44,7 @@ def _skip_bad_nodes(transport): try: with pool.take(_filter=_skip_bad_nodes) as transport: try: - yield transport + return fn(transport) except (IOError, httplib.HTTPException) as e: if is_retryable(e): transport._node.error_rate.incr(1) @@ -71,6 +55,30 @@ def _skip_bad_nodes(transport): except BadResource: continue + def _choose_pool(self, protocol=None): + if not protocol: + protocol = self.protocol + if protocol in ['http', 'https']: + pool = self._http_pool + elif protocol == 'pbc': + pool = self._pb_pool + else: + raise ValueError("invalid protocol %s" % protocol) + return pool + def is_retryable(error): return is_pbc_retryable(error) or is_http_retryable(error) + + +def retryable(fn, protocol=None): + def wrapper(self, *args, **kwargs): + pool = self._choose_pool(protocol) + def thunk(transport): + return fn(self, transport, *args, **kwargs) + return self._with_retries(pool, thunk) + return wrapper + + +def retryableHttpOnly(fn): + return retryable(fn, protocol='http') From 054a5905cad5fc60b65046b1cbe75589e042e807 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 29 Jan 2013 11:25:22 -0600 Subject: [PATCH 33/45] Restore _transport(), which is used when getting the client id. --- riak/client/transport.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/riak/client/transport.py b/riak/client/transport.py index fc46abb5..54e26de6 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -15,7 +15,7 @@ specific language governing permissions and limitations under the License. """ - +from contextlib import contextmanager from riak.transports.pool import BadResource from riak.transports.pbc import is_retryable as is_pbc_retryable from riak.transports.http import is_retryable as is_http_retryable @@ -34,6 +34,12 @@ class RiakClientTransport(object): _http_pool = None _pb_pool = None + @contextmanager + def _transport(self): + pool = self._choose_pool() + with pool.take() as transport: + yield transport + def _with_retries(self, pool, fn): skip_nodes = [] From add0760ca01399694e991a2caa37ad5e7f187a09 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 29 Jan 2013 12:10:43 -0600 Subject: [PATCH 34/45] Ensure that streams drain the socket and cleanup when there is an exception in user code. --- riak/client/operations.py | 18 +++++++++++++----- riak/tests/test_all.py | 4 +++- riak/tests/test_mapreduce.py | 33 ++++++++++++++++++++++++++++++++- riak/transports/http/stream.py | 5 +++++ riak/transports/pbc/stream.py | 14 ++++++++++++-- 5 files changed, 65 insertions(+), 9 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index efba5128..00366022 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -82,9 +82,13 @@ def stream_keys(self, transport, bucket): Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. """ - for keylist in transport.stream_keys(bucket): - if len(keylist) > 0: - yield keylist + stream = transport.stream_keys(bucket) + try: + for keylist in stream: + if len(keylist) > 0: + yield keylist + finally: + stream.close() @retryable def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, @@ -135,8 +139,12 @@ def stream_mapred(self, transport, inputs, query, timeout): Streams a MapReduce query as (phase, data) pairs. This is a generator method which should be iterated over. """ - for phase, data in transport.stream_mapred(inputs, query, timeout): - yield phase, data + stream = transport.stream_mapred(inputs, query, timeout) + try: + for phase, data in stream: + yield phase, data + finally: + stream.close() @retryableHttpOnly def fulltext_search(self, transport, index, query, **params): diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 2d5c5d20..98eeba65 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -19,7 +19,7 @@ from riak.tests.test_search import SearchTests, \ EnableSearchTests, SolrSearchTests from riak.tests.test_mapreduce import MapReduceAliasTests, \ - ErlangMapReduceTests, JSMapReduceTests, LinkTests + ErlangMapReduceTests, JSMapReduceTests, LinkTests, MapReduceStreamTests from riak.tests.test_kv import BasicKVTests, KVFileTests, \ HTTPBucketPropsTest, PbcBucketPropsTest from riak.tests.test_2i import TwoITests @@ -89,6 +89,7 @@ class RiakPbcTransportTestCase(BasicKVTests, ErlangMapReduceTests, JSMapReduceTests, MapReduceAliasTests, + MapReduceStreamTests, SearchTests, BaseTestCase, unittest.TestCase): @@ -176,6 +177,7 @@ class RiakHttpTransportTestCase(BasicKVTests, ErlangMapReduceTests, JSMapReduceTests, MapReduceAliasTests, + MapReduceStreamTests, EnableSearchTests, SolrSearchTests, SearchTests, diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 249d7590..13bb1920 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from riak.mapreduce import RiakLink +from riak.mapreduce import RiakLink, RiakMapReduce from riak import key_filter @@ -499,3 +499,34 @@ def test_filter_not_found(self): .run() self.assertEqual(sorted(result), [1, 2]) + +class MapReduceStreamTests(object): + def test_stream_results(self): + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + mr = RiakMapReduce(self.client).add('bucket', 'one').add('bucket', 'two') + mr.map_values_json() + results = [] + for phase, data in mr.stream(): + results.extend(data) + + self.assertEqual(sorted(results), [1,2]) + + def test_stream_cleanup(self): + bucket = self.client.bucket('bucket') + bucket.new('one', data=1).store() + bucket.new('two', data=2).store() + + mr = RiakMapReduce(self.client).add('bucket', 'one').add('bucket', 'two') + mr.map_values_json() + try: + for phase, data in mr.stream(): + raise RuntimeError("woops") + except RuntimeError: + pass + + # This should not raise an exception + obj = bucket.get('one') + self.assertEqual(1, obj.data) diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 5130a0dd..585c7489 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -46,6 +46,10 @@ def read(self): def next(self): raise NotImplementedError + def close(self): + pass + + class RiakHttpKeyStream(RiakHttpStream): """ Streaming iterator for list-keys over HTTP @@ -64,6 +68,7 @@ def next(self): else: raise StopIteration + class RiakHttpMultipartStream(RiakHttpStream): """ Streaming iterator for multipart messages over HTTP diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index 69594228..99a0d866 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -18,8 +18,9 @@ import json -from riak.transports.pbc.messages import MSG_CODE_LIST_KEYS_RESP -from riak.transports.pbc.messages import MSG_CODE_MAPRED_RESP +from riak.transports.pbc.messages import ( + MSG_CODE_LIST_KEYS_RESP, MSG_CODE_MAPRED_RESP + ) class RiakPbcStream(object): @@ -52,6 +53,15 @@ def _is_done(self, response): # same thing. return response.done + def close(self): + # We have to drain the socket to make sure that we don't get + # weird responses when the some other request comes after a + # failed/prematurely-terminated one. + try: + while self.next(): + pass + except StopIteration: + pass class RiakPbcKeyStream(RiakPbcStream): """ From efc091866da617b6c97b37afe1c8ecaacd4d4c73 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 29 Jan 2013 12:15:41 -0600 Subject: [PATCH 35/45] PEP8 cleanup. --- riak/client/operations.py | 9 +++++---- riak/client/transport.py | 3 +++ riak/tests/test_mapreduce.py | 11 +++++++---- riak/transports/http/stream.py | 1 + riak/transports/pbc/stream.py | 1 + 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 00366022..09747049 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -18,6 +18,7 @@ from transport import RiakClientTransport, retryable, retryableHttpOnly + class RiakClientOperations(RiakClientTransport): """ Methods for RiakClient that result in requests sent to the Riak @@ -101,8 +102,8 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=if_none_match) @retryable - def put_new(self, transport, robj, w=None, dw=None, pw=None, return_body=None, - if_none_match=None): + def put_new(self, transport, robj, w=None, dw=None, pw=None, + return_body=None, if_none_match=None): """ Stores an object in the Riak cluster with a generated key. """ @@ -118,8 +119,8 @@ def get(self, transport, robj, r=None, pr=None, vtag=None): return transport.get(robj, r=r, pr=pr, vtag=vtag) @retryable - def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, pr=None, - pw=None): + def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, + pr=None, pw=None): """ Deletes an object from Riak. """ diff --git a/riak/client/transport.py b/riak/client/transport.py index 54e26de6..ae7f749c 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -80,9 +80,12 @@ def is_retryable(error): def retryable(fn, protocol=None): def wrapper(self, *args, **kwargs): pool = self._choose_pool(protocol) + def thunk(transport): return fn(self, transport, *args, **kwargs) + return self._with_retries(pool, thunk) + return wrapper diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 13bb1920..b6e2d1b2 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -500,26 +500,29 @@ def test_filter_not_found(self): self.assertEqual(sorted(result), [1, 2]) + class MapReduceStreamTests(object): def test_stream_results(self): bucket = self.client.bucket('bucket') bucket.new('one', data=1).store() bucket.new('two', data=2).store() - mr = RiakMapReduce(self.client).add('bucket', 'one').add('bucket', 'two') + mr = RiakMapReduce(self.client).add('bucket', 'one')\ + .add('bucket', 'two') mr.map_values_json() results = [] for phase, data in mr.stream(): results.extend(data) - self.assertEqual(sorted(results), [1,2]) + self.assertEqual(sorted(results), [1, 2]) - def test_stream_cleanup(self): + def test_stream_cleanoperationsup(self): bucket = self.client.bucket('bucket') bucket.new('one', data=1).store() bucket.new('two', data=2).store() - mr = RiakMapReduce(self.client).add('bucket', 'one').add('bucket', 'two') + mr = RiakMapReduce(self.client).add('bucket', 'one')\ + .add('bucket', 'two') mr.map_values_json() try: for phase, data in mr.stream(): diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 585c7489..4e09f3fd 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -22,6 +22,7 @@ from cgi import parse_header from email import message_from_string + class RiakHttpStream(object): """ Base class for HTTP streaming iterators. diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index 99a0d866..689a26d1 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -63,6 +63,7 @@ def close(self): except StopIteration: pass + class RiakPbcKeyStream(RiakPbcStream): """ Used internally by RiakPbcTransport to implement key-list streams. From 41df3a7135a58bba1c41103d326e73e0228f80c2 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 29 Jan 2013 12:18:05 -0600 Subject: [PATCH 36/45] Cleanup some pyflakes errors. --- riak/transports/http/transport.py | 1 - riak/transports/pbc/transport.py | 1 - 2 files changed, 2 deletions(-) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 8cbbcb7c..7f172e5e 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -224,7 +224,6 @@ def stream_keys(self, bucket): if headers['http_code'] is 200: return RiakHttpKeyStream(response) else: - stream.close() raise Exception('Error listing keys.') def get_buckets(self): diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index bc60cc21..426619b3 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -19,7 +19,6 @@ under the License. """ -import json import riak_pb from riak import RiakError from riak.transports.transport import RiakTransport From 76edabe9abeea837f0102d557cde8ad346251950 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Tue, 29 Jan 2013 13:18:11 -0600 Subject: [PATCH 37/45] Test aborting a key stream. --- riak/tests/test_kv.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 5b4a5e90..65862410 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -84,6 +84,20 @@ def test_stream_keys(self): streamed_keys += keylist self.assertEqual(sorted(regular_keys), sorted(streamed_keys)) + def test_stream_keys_abort(self): + bucket = self.client.bucket('random_key_bucket') + regular_keys = bucket.get_keys() + self.assertNotEqual(len(regular_keys), 0) + try: + for keylist in bucket.stream_keys(): + raise RuntimeError("abort") + except RuntimeError: + pass + + # If the stream was closed correctly, this will not error + robj = bucket.get(regular_keys[0]) + self.assertEqual(True, robj.exists) + def test_binary_store_and_get(self): bucket = self.client.bucket('bucket') # Store as binary, retrieve as binary, then compare... From 7060c30df61b17efbc659d6ccc02b038486846c7 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Wed, 30 Jan 2013 16:11:13 -0600 Subject: [PATCH 38/45] Document all the things --- riak/__init__.py | 3 + riak/bucket.py | 48 ++-- riak/client/__init__.py | 6 +- riak/client/operations.py | 119 +++++++++- riak/client/transport.py | 40 +++- riak/mapreduce.py | 346 +++++++++++++++++++++++------ riak/node.py | 12 + riak/riak_object.py | 24 +- riak/search.py | 17 ++ riak/tests/test_all.py | 51 ----- riak/transports/http/__init__.py | 2 + riak/transports/http/connection.py | 3 + riak/transports/pbc/__init__.py | 2 + riak/transports/pbc/codec.py | 22 ++ riak/transports/pbc/connection.py | 3 + riak/transports/pbc/stream.py | 2 +- riak/transports/pbc/transport.py | 8 +- 17 files changed, 547 insertions(+), 161 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index 0d8ab38a..2296f2a1 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -32,6 +32,9 @@ class RiakError(Exception): + """ + Base class for exceptions generated in the Riak API. + """ def __init__(self, value): self.value = value diff --git a/riak/bucket.py b/riak/bucket.py index 2fa2d088..0dae731a 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -222,7 +222,11 @@ def _set_r(self, val): def _get_r(self): return self.get_property('r') - r = property(_get_r, _set_r) + r = property(_get_r, _set_r, doc=""" + The default 'read' quorum for this bucket (how many replicas must + reply for a successful read). This should be an integer less than + the 'n_val' property, or a string of 'one', 'quorum', 'all', or + 'default'""") def _set_pr(self, val): return self.set_property('pr', val) @@ -230,7 +234,11 @@ def _set_pr(self, val): def _get_pr(self): return self.get_property('pr') - pr = property(_get_pr, _set_pr) + pr = property(_get_pr, _set_pr, doc=""" + The default 'primary read' quorum for this bucket (how many + primary replicas are required for a successful read). This should + be an integer less than the 'n_val' property, or a string of + 'one', 'quorum', 'all', or 'default'""") def _set_rw(self, val): return self.set_property('rw', val) @@ -238,7 +246,11 @@ def _set_rw(self, val): def _get_rw(self): return self.get_property('rw') - rw = property(_get_rw, _set_rw) + rw = property(_get_rw, _set_rw, doc=""" + The default 'read' and 'write' quorum for this bucket (equivalent + to 'r' and 'w' but for deletes). This should be an integer less + than the 'n_val' property, or a string of 'one', 'quorum', 'all', + or 'default'""") def _set_w(self, val): return self.set_property('w', val) @@ -246,7 +258,11 @@ def _set_w(self, val): def _get_w(self): return self.get_property('w') - w = property(_get_w, _set_w) + w = property(_get_w, _set_w, doc=""" + The default 'write' quorum for this bucket (how many replicas must + acknowledge receipt of a write). This should be an integer less + than the 'n_val' property, or a string of 'one', 'quorum', 'all', + or 'default'""") def _set_dw(self, val): return self.set_property('dw', val) @@ -254,7 +270,11 @@ def _set_dw(self, val): def _get_dw(self): return self.get_property('dw') - dw = property(_get_dw, _set_dw) + dw = property(_get_dw, _set_dw, doc=""" + The default 'durable write' quorum for this bucket (how many + replicas must commit the write). This should be an integer less + than the 'n_val' property, or a string of 'one', 'quorum', 'all', + or 'default'""") def _set_pw(self, val): return self.set_property('pw', val) @@ -262,16 +282,16 @@ def _set_pw(self, val): def _get_pw(self): return self.get_property('pw') - pw = property(_get_pw, _set_pw) + pw = property(_get_pw, _set_pw, doc=""" + The default 'primary write' quorum for this bucket (how many + primary replicas are required for a successful write). This should + be an integer less than the 'n_val' property, or a string of + 'one', 'quorum', 'all', or 'default'""") def set_property(self, key, value): """ Set a bucket property. - .. warning:: - - This should only be used if you know what you are doing. - :param key: Property to set. :type key: string :param value: Property value. @@ -296,12 +316,8 @@ def set_properties(self, props): """ Set multiple bucket properties in one call. - .. warning:: - - This should only be used if you know what you are doing. - - :param props: An associative array of key:value. - :type props: array + :param props: A dictionary of properties + :type props: dict """ self._client.set_bucket_props(self, props) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 4b816551..4fc285cf 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -93,11 +93,12 @@ def _set_protocol(self, value): self._protocol = value protocol = property(_get_protocol, _set_protocol, - doc="""which protocol to prefer""") + doc="""Which protocol to prefer, one of PROTOCOLS""") def get_transport(self): """ - Get the transport instance the client is using for it's connection. + Get the transport instance the client is using for it's + connection. DEPRECATED """ deprecated("get_transport is deprecated, use client, " + "bucket, or object methods instead") @@ -117,6 +118,7 @@ def get_client_id(self): def set_client_id(self, client_id): """ Set the client_id for this ``RiakClient`` instance. + DEPRECATED :param client_id: The new client_id. :type client_id: string diff --git a/riak/client/operations.py b/riak/client/operations.py index 09747049..d9c3f33f 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -53,6 +53,16 @@ def ping(self, transport): def get_index(self, transport, bucket, index, startkey, endkey=None): """ Queries a secondary index, returning matching keys. + + :param bucket: the bucket whose index will be queried + :type bucket: RiakBucket + :param index: the index to query + :type index: string + :param startkey: the sole key to query, or beginning of the query range + :type startkey: string, integer + :param endkey: the end of the query range (optional if equality) + :type endkey: string, integer + :rtype: list """ return transport.get_index(bucket, index, startkey, endkey) @@ -60,6 +70,10 @@ def get_index(self, transport, bucket, index, startkey, endkey=None): def get_bucket_props(self, transport, bucket): """ Fetches bucket properties for the given bucket. + + :param bucket: the bucket whose properties will be fetched + :type bucket: RiakBucket + :rtype: dict """ return transport.get_bucket_props(bucket) @@ -67,6 +81,11 @@ def get_bucket_props(self, transport, bucket): def set_bucket_props(self, transport, bucket, props): """ Sets bucket properties for the given bucket. + + :param bucket: the bucket whose properties will be set + :type bucket: RiakBucket + :param props: the properties to set + :type props: dict """ return transport.set_bucket_props(bucket, props) @@ -74,6 +93,10 @@ def set_bucket_props(self, transport, bucket, props): def get_keys(self, transport, bucket): """ Lists all keys in a bucket. + + :param bucket: the bucket whose properties will be set + :type bucket: RiakBucket + :rtype: list """ return transport.get_keys(bucket) @@ -82,6 +105,11 @@ def stream_keys(self, transport, bucket): """ Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. + + + :param bucket: the bucket whose properties will be set + :type bucket: RiakBucket + :rtype: iterator """ stream = transport.stream_keys(bucket) try: @@ -96,6 +124,21 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=None): """ Stores an object in the Riak cluster. + + :param robj: the object to store + :type robj: RiakObject + :param w: the write quorum + :type w: integer, string, None + :param dw: the durable write quorum + :type dw: integer, string, None + :param pw: the primary write quorum + :type pw: integer, string, None + :param return_body: whether to return the resulting object + after the write + :type return_body: boolean + :param if_none_match: whether to fail the write if the object + exists + :type if_none_match: boolean """ return transport.put(robj, w=w, dw=dw, pw=pw, return_body=return_body, @@ -106,6 +149,21 @@ def put_new(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=None): """ Stores an object in the Riak cluster with a generated key. + + :param robj: the object to store + :type robj: RiakObject + :param w: the write quorum + :type w: integer, string, None + :param dw: the durable write quorum + :type dw: integer, string, None + :param pw: the primary write quorum + :type pw: integer, string, None + :param return_body: whether to return the resulting object + after the write + :type return_body: boolean + :param if_none_match: whether to fail the write if the object + exists + :type if_none_match: boolean """ return transport.put_new(robj, w=w, dw=dw, pw=pw, return_body=return_body, @@ -115,6 +173,15 @@ def put_new(self, transport, robj, w=None, dw=None, pw=None, def get(self, transport, robj, r=None, pr=None, vtag=None): """ Fetches the contents of a Riak object. + + :param robj: the object to fetch + :type robj: RiakObject + :param r: the read quorum + :type r: integer, string, None + :param pr: the primary read quorum + :type pr: integer, string, None + :param vtag: the specific sibling to fetch + :type vtag: string """ return transport.get(robj, r=r, pr=pr, vtag=vtag) @@ -123,6 +190,21 @@ def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Deletes an object from Riak. + + :param robj: the object to store + :type robj: RiakObject + :param rw: the read/write (delete) quorum + :type rw: integer, string, None + :param r: the read quorum + :type r: integer, string, None + :param pr: the primary read quorum + :type pr: integer, string, None + :param w: the write quorum + :type w: integer, string, None + :param dw: the durable write quorum + :type dw: integer, string, None + :param pw: the primary write quorum + :type pw: integer, string, None """ return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) @@ -130,7 +212,15 @@ def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, @retryable def mapred(self, transport, inputs, query, timeout): """ - Executes a MapReduce query + Executes a MapReduce query. + + :param inputs: the input list/structure + :type inputs: list, dict + :param query: the list of query phases + :type query: list + :param timeout: the query timeout + :type timeout: integer, None + :rtype: mixed """ return transport.mapred(inputs, query, timeout) @@ -139,6 +229,14 @@ def stream_mapred(self, transport, inputs, query, timeout): """ Streams a MapReduce query as (phase, data) pairs. This is a generator method which should be iterated over. + + :param inputs: the input list/structure + :type inputs: list, dict + :param query: the list of query phases + :type query: list + :param timeout: the query timeout + :type timeout: integer, None + :rtype: iterator """ stream = transport.stream_mapred(inputs, query, timeout) try: @@ -151,6 +249,13 @@ def stream_mapred(self, transport, inputs, query, timeout): def fulltext_search(self, transport, index, query, **params): """ Performs a full-text search query. + + :param index: the bucket/index to search over + :type index: string + :param query: the search query + :type query: string + :param params: additional query flags + :type params: dict """ return transport.search(index, query, **params) @@ -158,6 +263,11 @@ def fulltext_search(self, transport, index, query, **params): def fulltext_add(self, transport, index, docs): """ Adds documents to the full-text index. + + :param index: the bucket/index in which to index these docs + :type index: string + :param docs: the list of documents + :type docs: list """ transport.fulltext_add(index, docs) @@ -165,5 +275,12 @@ def fulltext_add(self, transport, index, docs): def fulltext_delete(self, transport, index, docs=None, queries=None): """ Removes documents from the full-text index. + + :param index: the bucket/index from which to delete + :type index: string + :param docs: a list of documents (with ids) + :type docs: list + :param queries: a list of queries to match and delete + :type queries: list """ transport.fulltext_delete(index, docs, queries) diff --git a/riak/client/transport.py b/riak/client/transport.py index ae7f749c..5c42c55c 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -36,11 +36,23 @@ class RiakClientTransport(object): @contextmanager def _transport(self): + """ + Yields a single transport to the caller from the default pool, + without retries. + """ pool = self._choose_pool() with pool.take() as transport: yield transport def _with_retries(self, pool, fn): + """ + Performs the passed function with retries against the given pool. + + :param pool: the connection pool to use + :type pool: Pool + :param fn: the function to pass a transport + :type fn: function + """ skip_nodes = [] def _skip_bad_nodes(transport): @@ -52,7 +64,7 @@ def _skip_bad_nodes(transport): try: return fn(transport) except (IOError, httplib.HTTPException) as e: - if is_retryable(e): + if _is_retryable(e): transport._node.error_rate.incr(1) skip_nodes.append(transport._node) raise BadResource(e) @@ -62,6 +74,14 @@ def _skip_bad_nodes(transport): continue def _choose_pool(self, protocol=None): + """ + Selects a connection pool according to the default protocol + and the passed one. + + :param protocol: the protocol to use + :type protocol: string + :rtype: Pool + """ if not protocol: protocol = self.protocol if protocol in ['http', 'https']: @@ -73,11 +93,23 @@ def _choose_pool(self, protocol=None): return pool -def is_retryable(error): +def _is_retryable(error): + """ + Determines whether a given error is retryable according to the + exceptions allowed to be retried by each transport. + + :param error: the error to check + :type error: Exception + :rtype: boolean + """ return is_pbc_retryable(error) or is_http_retryable(error) def retryable(fn, protocol=None): + """ + Wraps a client operation that can be retried according to the set + RETRY_COUNT. Used internally. + """ def wrapper(self, *args, **kwargs): pool = self._choose_pool(protocol) @@ -90,4 +122,8 @@ def thunk(transport): def retryableHttpOnly(fn): + """ + Wraps a retryable client operation that is only valid over HTTP. + Used internally. + """ return retryable(fn, protocol='http') diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 4bd7712c..ebbdedff 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -24,13 +24,15 @@ class RiakMapReduce(object): """ The RiakMapReduce object allows you to build up and run a - map/reduce operation on Riak. + map/reduce operation on Riak. Most methods return the object on + which it was called, modified with new information, so you can + chain calls together to build the job. """ def __init__(self, client): """ Construct a Map/Reduce object. - @param RiakClient client - A RiakClient object. - @return RiakMapReduce + :param client: A RiakClient object. + :type client: RiakClient """ self._client = client self._phases = [] @@ -44,10 +46,14 @@ def add(self, arg1, arg2=None, arg3=None): different forms, depending on the provided inputs. You can specify either a RiakObject, a string bucket name, or a bucket, key, and additional arg. - @param mixed arg1 - RiakObject or Bucket - @param mixed arg2 - Key or List or blank - @param mixed arg3 - Arg or blank - @return RiakMapReduce + + :param arg1: the object or bucket to add + :type arg1: RiakObject, string + :param arg2: a key or list of keys to add (if a bucket is given in arg1) + :type arg2: string, list, None + :param arg3: key data for this input (must be convertible to JSON) + :type arg3: string, list, dict, None + :rtype: RiakMapReduce """ if (arg2 is None) and (arg3 is None): if isinstance(arg1, RiakObject): @@ -58,9 +64,27 @@ def add(self, arg1, arg2=None, arg3=None): return self.add_bucket_key_data(arg1, arg2, arg3) def add_object(self, obj): + """ + Adds a RiakObject to the inputs. + + :param obj: the object to add + :type obj: RiakObject + :rtype: RiakMapReduce + """ return self.add_bucket_key_data(obj._bucket._name, obj._key, None) def add_bucket_key_data(self, bucket, key, data): + """ + Adds a bucket/key/keydata triple to the inputs. + + :param bucket: the bucket + :type bucket: string + :param key: the key or list of keys + :type key: string + :param data: the key-specific data + :type data: string, list, dict, None + :rtype: RiakMapReduce + """ if self._input_mode == 'bucket': raise ValueError('Already added a bucket, can\'t add an object.') elif self._input_mode == 'query': @@ -75,11 +99,25 @@ def add_bucket_key_data(self, bucket, key, data): return self def add_bucket(self, bucket): + """ + Adds all keys in a bucket to the inputs. + + :param bucket: the bucket + :type bucket: string + :rtype: RiakMapReduce + """ self._input_mode = 'bucket' self._inputs = bucket return self def add_key_filters(self, key_filters): + """ + Adds key filters to the inputs. + + :param key_filters: a list of filters + :type key_filters: list + :rtype: RiakMapReduce + """ if self._input_mode == 'query': raise ValueError('Key filters are not supported in a query.') @@ -87,6 +125,13 @@ def add_key_filters(self, key_filters): return self def add_key_filter(self, *args): + """ + Add a single key filter to the inputs. + + :param args: a filter + :type args: list + :rtype: RiakMapReduce + """ if self._input_mode == 'query': raise ValueError('Key filters are not supported in a query.') @@ -97,8 +142,12 @@ def search(self, bucket, query): """ Begin a map/reduce operation using a Search. This command will return an error unless executed against a Riak Search cluster. - @param bucket - The bucket over which to perform the search. - @param query - The search query. + + :param bucket: The bucket over which to perform the search + :type bucket: string + :param query: The search query + :type query: string + :rtype: RiakMapReduce """ self._input_mode = 'query' self._inputs = {'module': 'riak_search', @@ -110,10 +159,16 @@ def index(self, bucket, index, startkey, endkey=None): """ Begin a map/reduce operation using a Secondary Index query. - @param bucket - The bucket over which to perform the search. - @param index - The index to use for query - @param startkey - The start key of index range - @param endkey - The end key of index range or blank + + :param bucket: The bucket over which to perform the query + :type bucket: string + :param index: The index to use for query + :type index: string + :param startkey: The start key of index range, or the + value which all entries must equal + :type startkey: string, integer + :param endkey: The end key of index range (if doing a range query) + :type endkey: string, integer, None """ self._input_mode = 'query' @@ -131,13 +186,17 @@ def index(self, bucket, index, startkey, endkey=None): def link(self, bucket='_', tag='_', keep=False): """ Add a link phase to the map/reduce operation. - @param string bucket - Bucket name (default '_', which means all + + :param bucket: Bucket name (default '_', which means all buckets) - @param string tag - Tag (default '_', which means any tag) - @param boolean keep - Flag whether to keep results from this - stage in the map/reduce. (default False, unless this is the last - step in the phase) - @return self + :type bucket: string + :param tag: Tag (default '_', which means any tag) + :type tag: string + :param keep: Flag whether to keep results from this stage in + the map/reduce. (default False, unless this is the last step + in the phase) + :type keep: boolean + :rtype: RiakMapReduce """ self._phases.append(RiakLinkPhase(bucket, tag, keep)) return self @@ -145,13 +204,16 @@ def link(self, bucket='_', tag='_', keep=False): def map(self, function, options=None): """ Add a map phase to the map/reduce operation. - @param mixed function - Either a named Javascript function (ie: - 'Riak.mapValues'), or an anonymous javascript function (ie: - 'function(...) ... ' or an array ['erlang_module', - 'function']. - @param array() options - An optional associative array - containing 'language', 'keep' flag, and/or 'arg'. - @return self + + :param function: Either a named Javascript function (ie: + 'Riak.mapValues'), or an anonymous javascript function (ie: + 'function(...) ... ' or an array ['erlang_module', + 'function']. + :type function: string, list + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + :rtype: RiakMapReduce """ if options is None: options = dict() @@ -171,12 +233,15 @@ def map(self, function, options=None): def reduce(self, function, options=None): """ Add a reduce phase to the map/reduce operation. - @param mixed function - Either a named Javascript function (ie. - 'Riak.mapValues'), or an anonymous javascript function(ie: - 'function(...) { ... }' or an array ['erlang_module', 'function']. - @param array() options - An optional associative array - containing 'language', 'keep' flag, and/or 'arg'. - @return self + + :param function: Either a named Javascript function (ie. + 'Riak.reduceSum'), or an anonymous javascript function(ie: + 'function(...) { ... }' or an array ['erlang_module', + 'function']. + :type function: string, list + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :rtype: RiakMapReduce """ if options is None: options = dict() @@ -195,10 +260,13 @@ def reduce(self, function, options=None): def run(self, timeout=None): """ - Run the map/reduce operation. Returns an array of results, or an - array of RiakLink objects if the last phase is a link phase. - @param integer timeout - Timeout in milliseconds. - @return array() + Run the map/reduce operation synchronously. Returns a list of + results, or a list of RiakLink objects if the last phase is a + link phase. + + :param timeout: Timeout in milliseconds + :type timeout: integer, None + :rtype: list """ query, link_results_flag = self._normalize_query() @@ -229,6 +297,10 @@ def run(self, timeout=None): def stream(self, timeout=None): """ Streams the MapReduce query (returns an iterator). + + :param timeout: Timeout in milliseconds + :type timeout: integer + :rtype: iterator """ query, lrf = self._normalize_query() return self._client.stream_mapred(self._inputs, query, timeout) @@ -272,21 +344,72 @@ def _normalize_query(self): # Start Shortcuts to built-ins ## def map_values(self, options=None): + """ + Adds the Javascript built-in ``Riak.mapValues`` to the query + as a map phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.map("Riak.mapValues", options=options) def map_values_json(self, options=None): + """ + Adds the Javascript built-in ``Riak.mapValuesJson`` to the + query as a map phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.map("Riak.mapValuesJson", options=options) def reduce_sum(self, options=None): + """ + Adds the Javascript built-in ``Riak.reduceSum`` to the query + as a reduce phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.reduce("Riak.reduceSum", options=options) def reduce_min(self, options=None): + """ + Adds the Javascript built-in ``Riak.reduceMin`` to the query + as a reduce phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.reduce("Riak.reduceMin", options=options) def reduce_max(self, options=None): + """ + Adds the Javascript built-in ``Riak.reduceMax`` to the query + as a reduce phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.reduce("Riak.reduceMax", options=options) def reduce_sort(self, js_cmp=None, options=None): + """ + Adds the Javascript built-in ``Riak.reduceSort`` to the query + as a reduce phase. + + :param js_cmp: A Javascript comparator function as specified by + Array.sort() + :type js_cmp: string + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ if options is None: options = dict() @@ -296,9 +419,27 @@ def reduce_sort(self, js_cmp=None, options=None): return self.reduce("Riak.reduceSort", options=options) def reduce_numeric_sort(self, options=None): + """ + Adds the Javascript built-in ``Riak.reduceNumericSort`` to the + query as a reduce phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.reduce("Riak.reduceNumericSort", options=options) def reduce_limit(self, limit, options=None): + """ + Adds the Javascript built-in ``Riak.reduceLimit`` to the query + as a reduce phase. + + :param limit: the maximum number of results to return + :type limit: integer + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ if options is None: options = dict() @@ -310,6 +451,18 @@ def reduce_limit(self, limit, options=None): return self.reduce(code, options=options) def reduce_slice(self, start, end, options=None): + """ + Adds the Javascript built-in ``Riak.reduceSlice`` to the + query as a reduce phase. + + :param start: the beginning of the slice + :type start: integer + :param end: the end of the slice + :type end: integer + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ if options is None: options = dict() @@ -317,25 +470,42 @@ def reduce_slice(self, start, end, options=None): return self.reduce("Riak.reduceSlice", options=options) def filter_not_found(self, options=None): + """ + Adds the Javascript built-in ``Riak.filterNotFound`` to the query + as a reduce phase. + + :param options: phase options, containing 'language', 'keep' + flag, and/or 'arg'. + :type options: dict + """ return self.reduce("Riak.filterNotFound", options=options) class RiakMapReducePhase(object): """ - The RiakMapReducePhase holds information about a Map phase or - Reduce phase in a RiakMapReduce operation. + The RiakMapReducePhase holds information about a Map or Reduce + phase in a RiakMapReduce operation. + + Normally you won't need to use this object directly, but instead + call methods on RiakMapReduce objects to add instances to the + query. """ def __init__(self, type, function, language, keep, arg): """ Construct a RiakMapReducePhase object. - @param string type - 'map'placeholder149'reduce' - @param mixed function - string or array(): - @param string language - 'javascript'placeholder149'erlang' - @param boolean keep - True to return the output of this phase in - the results. - @param mixed arg - Additional value to pass into the map or - reduce function. + + :param type: the phase type - 'map', 'reduce', 'link' + :type type: string + :param function: the function to execute + :type function: string, list + :param language: 'javascript' or 'erlang' + :type language: string + :param keep: whether to return the output of this phase in the results. + :type keep: boolean + :param arg: Additional static value to pass into the map or + reduce function. + :type arg: string, dict, list """ try: if isinstance(function, basestring): @@ -351,8 +521,10 @@ def __init__(self, type, function, language, keep, arg): def to_array(self): """ - Convert the RiakMapReducePhase to an associative array. Used - internally. + Convert the RiakMapReducePhase to a format that can be output + into JSON. Used internally. + + :rtype: dict """ stepdef = {'keep': self._keep, 'language': self._language, @@ -379,14 +551,21 @@ class RiakLinkPhase(object): """ The RiakLinkPhase object holds information about a Link phase in a map/reduce operation. + + Normally you won't need to use this object directly, but instead + call ``link`` on RiakMapReduce objects to add instances to the + query. """ def __init__(self, bucket, tag, keep): """ Construct a RiakLinkPhase object. - @param string bucket - The bucket name. - @param string tag - The tag. - @param boolean keep - True to return results of this phase. + :param bucket: - The bucket name + :type bucket: string + :param tag: The tag + :type tag: string + :param keep: whether to return results of this phase. + :type keep: boolean """ self._bucket = bucket self._tag = tag @@ -394,8 +573,8 @@ def __init__(self, bucket, tag, keep): def to_array(self): """ - Convert the RiakLinkPhase to an associative array. Used - internally. + Convert the RiakLinkPhase to a format that can be output into + JSON. Used internally. """ stepdef = {'bucket': self._bucket, 'tag': self._tag, @@ -412,9 +591,13 @@ class RiakLink(object): def __init__(self, bucket, key, tag=None): """ Construct a RiakLink object. - @param string bucket - The bucket name. - @param string key - The key. - @param string tag - The tag. + + :param bucket: the bucket name + :type bucket: string + :param key: the key + :type key: string + :param tag: the tag + :type tag: string """ self._bucket = bucket self._key = key @@ -424,31 +607,38 @@ def __init__(self, bucket, key, tag=None): def get(self, r=None): """ Retrieve the RiakObject to which this link points. - @param integer r - The R-value to use. - @return RiakObject + + :param r: the read quorum to use + :type r: string, integer + :rtype: RiakObject """ return self._client.bucket(self._bucket).get(self._key, r) def get_binary(self, r=None): """ Retrieve the RiakObject to which this link points, as a binary. - @param integer r - The R-value to use. - @return RiakObject + + :param r: the read quorum to use + :type r: string, integer + :rtype: RiakObject """ return self._client.bucket(self._bucket).get_binary(self._key, r) def get_bucket(self): """ Get the bucket name of this link. - @return string + + :rtype: string """ return self._bucket def set_bucket(self, name): """ Set the bucket name of this link. - @param string name - The bucket name. - @return self + + :param name: the bucket name + :type name: string + :rtype: RiakLink """ self._bucket = name return self @@ -456,15 +646,18 @@ def set_bucket(self, name): def get_key(self): """ Get the key of this link. - @return string + + :rtype: string """ return self._key def set_key(self, key): """ Set the key of this link. - @param string key - The key. - @return self + + :param key: the key + :type key: string + :rtype: RiakLink """ self._key = key return self @@ -472,7 +665,8 @@ def set_key(self, key): def get_tag(self): """ Get the tag of this link. - @return string + + :rtype: string """ if (self._tag is None): return self._bucket @@ -482,15 +676,20 @@ def get_tag(self): def set_tag(self, tag): """ Set the tag of this link. - @param string tag - The tag. - @return self + + :param tag: the tag + :type tag: string + :rtype: RiakLink """ self._tag = tag return self def to_link_header(self, client): """ - Convert this RiakLink object to a link header string. Used internally. + Convert this RiakLink object to a link header string. Used + internally. + + :rtype: string """ link = '' link += ' Date: Wed, 30 Jan 2013 16:19:05 -0600 Subject: [PATCH 39/45] One last fix. --- riak/bucket.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/riak/bucket.py b/riak/bucket.py index 0dae731a..781aa4f3 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -323,9 +323,9 @@ def set_properties(self, props): def get_properties(self): """ - Retrieve an associative array of all bucket properties. + Retrieve a dict of all bucket properties. - :rtype: array + :rtype: dict """ return self._client.get_bucket_props(self) @@ -341,11 +341,13 @@ def get_keys(self): def stream_keys(self): """ - Return all keys within the bucket. + Streams all keys within the bucket through an iterator. .. warning:: At current, this is a very expensive operation. Use with caution. + + :rtype: iterator """ return self._client.stream_keys(self) From 0abef5ff0d0f564a045331b3712d6b2fec9d82e3 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 09:07:58 -0600 Subject: [PATCH 40/45] Remove some more 'is' in favor of '=='. --- riak/transports/http/transport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 7f172e5e..959b313a 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -211,7 +211,7 @@ def get_keys(self, bucket): response = self._request('GET', url) headers, encoded_props = response[0:2] - if headers['http_code'] is 200: + if headers['http_code'] == 200: props = json.loads(encoded_props) return props['keys'] else: @@ -221,7 +221,7 @@ def stream_keys(self, bucket): url = self.key_list_path(bucket.name, keys='stream') headers, response = self._request('GET', url, stream=True) - if headers['http_code'] is 200: + if headers['http_code'] == 200: return RiakHttpKeyStream(response) else: raise Exception('Error listing keys.') From fb4e2dd5960f52713a335526d657556ea63bef04 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 09:11:36 -0600 Subject: [PATCH 41/45] Remove commented imports. --- riak/riak_object.py | 6 ------ riak/transports/http/transport.py | 2 -- 2 files changed, 8 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index fbcf4a17..cdc0359e 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -19,16 +19,10 @@ """ import copy from riak.metadata import ( - # MD_CHARSET, MD_CTYPE, - # MD_ENCODING, MD_INDEX, - # MD_LASTMOD, - # MD_LASTMOD_USECS, MD_LINKS, MD_USERMETA - # MD_VTAG, - # MD_DELETED ) from riak.mapreduce import ( RiakMapReduce, diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 959b313a..9d8269d8 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -35,10 +35,8 @@ from riak.metadata import ( MD_CHARSET, MD_CTYPE, - # MD_ENCODING, MD_INDEX, MD_LASTMOD, - # MD_LASTMOD_USECS, MD_LINKS, MD_USERMETA, MD_VTAG, From 880615036d3da7ce1ae38b83d2d6b475d207be2e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 09:12:35 -0600 Subject: [PATCH 42/45] Stylistic fix. --- riak/riak_object.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/riak/riak_object.py b/riak/riak_object.py index cdc0359e..7e1e6573 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -165,8 +165,8 @@ def add_index(self, field, value): :rtype: RiakObject """ if field[-4:] not in ("_bin", "_int"): - raise RiakError( - "Riak 2i fields must end with either '_bin' or '_int'.") + raise RiakError("Riak 2i fields must end with either '_bin'" + " or '_int'.") rie = RiakIndexEntry(field, value) if not rie in self.metadata[MD_INDEX]: From 93019a8f5b57df6e6e20427f07dc83c8d7e30e62 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 09:12:53 -0600 Subject: [PATCH 43/45] Make read() private on HTTP stream iterators. --- riak/transports/http/stream.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 4e09f3fd..2f1620f9 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -38,7 +38,7 @@ def __init__(self, response): def __iter__(self): return self - def read(self): + def _read(self): chunk = self.response.read(self.BLOCK_SIZE) if chunk == '': self.response_done = True @@ -58,7 +58,7 @@ class RiakHttpKeyStream(RiakHttpStream): def next(self): while '}' not in self.buffer and not self.response_done: - self.read() + self._read() if '}' in self.buffer: idx = string.index(self.buffer, '}') + 1 @@ -111,7 +111,7 @@ def advance_buffer(self): def read_until_boundary(self): while not self.try_match() and not self.response_done: - self.read() + self._read() class RiakHttpMapReduceStream(RiakHttpMultipartStream): From fdb1aa42d2da3ab7753cc436acfd5e735252921e Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 09:14:18 -0600 Subject: [PATCH 44/45] PEP8 line-length fix. --- riak/mapreduce.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index ebbdedff..6031eeb2 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -49,7 +49,8 @@ def add(self, arg1, arg2=None, arg3=None): :param arg1: the object or bucket to add :type arg1: RiakObject, string - :param arg2: a key or list of keys to add (if a bucket is given in arg1) + :param arg2: a key or list of keys to add (if a bucket is + given in arg1) :type arg2: string, list, None :param arg3: key data for this input (must be convertible to JSON) :type arg3: string, list, dict, None From 535d0bcd14dfe729fe378e244d3f2a974a15a477 Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 31 Jan 2013 14:50:49 -0600 Subject: [PATCH 45/45] Remove retries from streaming per peer-review discussion. --- riak/client/operations.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index d9c3f33f..e28c78e3 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -100,8 +100,7 @@ def get_keys(self, transport, bucket): """ return transport.get_keys(bucket) - @retryable - def stream_keys(self, transport, bucket): + def stream_keys(self, bucket): """ Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. @@ -111,13 +110,14 @@ def stream_keys(self, transport, bucket): :type bucket: RiakBucket :rtype: iterator """ - stream = transport.stream_keys(bucket) - try: - for keylist in stream: - if len(keylist) > 0: - yield keylist - finally: - stream.close() + with self._transport() as transport: + stream = transport.stream_keys(bucket) + try: + for keylist in stream: + if len(keylist) > 0: + yield keylist + finally: + stream.close() @retryable def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, @@ -224,8 +224,7 @@ def mapred(self, transport, inputs, query, timeout): """ return transport.mapred(inputs, query, timeout) - @retryable - def stream_mapred(self, transport, inputs, query, timeout): + def stream_mapred(self, inputs, query, timeout): """ Streams a MapReduce query as (phase, data) pairs. This is a generator method which should be iterated over. @@ -238,12 +237,13 @@ def stream_mapred(self, transport, inputs, query, timeout): :type timeout: integer, None :rtype: iterator """ - stream = transport.stream_mapred(inputs, query, timeout) - try: - for phase, data in stream: - yield phase, data - finally: - stream.close() + with self._transport() as transport: + stream = transport.stream_mapred(inputs, query, timeout) + try: + for phase, data in stream: + yield phase, data + finally: + stream.close() @retryableHttpOnly def fulltext_search(self, transport, index, query, **params):