diff --git a/.gitignore b/.gitignore index a8d3d3a0..ef74d171 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ docs/_build build/ dist/ riak.egg-info/ +*.egg diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..d4c4ec76 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: python +python: + - "2.6" + - "2.7" +install: ./setup.py develop +script: ./setup.py test +# TODO: get search enabled on Travis' Riak install, or fix the test server +env: "SKIP_LUWAK=1 SKIP_SEARCH=1" +notifications: + email: clients@basho.com diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..e691aa45 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include docs/* +include erl_src/* +include THANKS +include README.rst +include LICENSE +include RELEASE_NOTES.md diff --git a/README.rst b/README.rst index f322d5db..62cf297e 100644 --- a/README.rst +++ b/README.rst @@ -2,6 +2,8 @@ Python Client for Riak ======================== +.. image:: https://secure.travis-ci.org/basho/riak-python-client.png?branch=master + :target: http://travis-ci.org/basho/riak-python-client Documentation ============== @@ -11,21 +13,28 @@ The documentation source is found in `docs/ subdirectory `_ and can be built with `Sphinx `_. -Documentation for Riak is available at http://wiki.basho.com/How-Things-Work.html +Documentation for Riak is available at http://wiki.basho.com/Riak.html Install ======= +The recommended version of Python for use with this client is Python 2.7. + You must have `Protocol Buffers`_ installed before you can install the Riak Client. From the Riak Python Client root directory, execute:: python setup.py install +There is an additional dependency on the Python package `setuptools`. Please install `setuptools` first, e.g. ``port install py27-setuptools`` for OS X and MacPorts. + Unit Test =========== -To run the unit tests, execute:: +To run the unit tests against a Riak server (with default TCP port configuration) on localhost, execute:: python setup.py test +If you don't have `Luwak `_ or `Riak Search `_ enabled you can set the ``SKIP_LUWAK`` and ``SKIP_SEARCH`` environment variables to skip those tests. + +If your Riak server isn't running on localhost, use the environment variables ``RIAK_TEST_HOST`` and ``RIAK_TEST_HTTP_PORT`` and ``RIAK_TEST_PB_PORT=8087`` to specify where to find the Riak server. ======== Tutorial diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 3dc10669..fc181517 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,50 @@ # Riak Python Client Release Notes +## 1.4.0 Feature Release - 2012-03-30 + +Release 1.4.0 is a feature release comprising over 117 individual +commits. + +Noteworthy features: + +* Python 2.6 and 2.7 are supported. On 2.6, the unittest2 package is + required to run the test suite. +* Google's official protobuf package (2.4.1 or later) is now a + dependency. The package from downloads.basho.com/support is no + longer necessary. +* Travis-CI is enabled on the client. Go to + http://travis-ci.org/basho/riak-python-client for build status. +* Riak 1.0+ features, namely secondary indexes and primary quora + (PR/PW), are supported. +* `if_none_match` is a valid request option when storing objects, and + will prevent the write when set to `True` if the key already exists. +* Links can be set wholesale using the `set_links()` method. +* Transport-specific options can be passed through when creating a + `Client` object. +* A connection manager was added that will (when manipulated manually) + allow connections to multiple Riak nodes. This will be fully + integrated in a future release. + +Noteworthy bugfixes: + +* Links now use the proper URL-encoding in HTTP headers, preventing + problems with explosion from multiple encoding passes. +* Many fixes were applied to make the Protocol Buffers transport more + stable. +* `RiakObject.get_content_type()` will behave properly when content + type is not set. +* Deprecated transport classes were removed since their functionality + had folded into the primary transports. +* A temporary fix was made for unicode bucket/key names which raises + an error when they are used and cannot be coerced to ASCII. +* The Erlang sources/beams for the TestServer are now included in the + package. +* MapReduce failures will now produce a more useful error message and + be handled properly when no results are returned. + +There are lots of other great fixes from our wonderful +community. [Check them out!](https://github.com/basho/riak-python-client/compare/1.3.0...1.4.0) + ## 1.3.0 Feature Release - 2011-08-04 Release 1.3.0 is a feature release bringing a slew of updates. diff --git a/THANKS b/THANKS index f734f6c9..16927ec4 100644 --- a/THANKS +++ b/THANKS @@ -1,14 +1,44 @@ The following people have contributed to the Riak Python client: +Andrew Thompson Andy Gross -Justin Sheehy -Rusty Klophaus +Armon Dadgar +Brett Hoerner +Brian Roach +Bryan Fink +Daniel Lindsley +Daniel Néri +Daniel Reverri +David Koblas +Dmitry Rozhkov +Eric Florenzano +Eric Moritz +Filip de Waard +Gilles Devaux +Greg Nelson +Greg Stein +Gregory Burd +Ian Plosker Jayson Baird +Jeffrey Massung Jon Meredith -Eric Florenzano -Silas Sewell -Matt Heitzenroder +Josip Lisec +Justin Sheehy +Kevin Smith Mark Erdmann -Greg Nelson +Mark Phillips +Mathias Meyer +Matt Heitzenroder Mikhail Sobolev -Eric Moritz +Reid Draper +Russell Brown +Rusty Klophaus +Scott Lystig Fritchie +Sean Cribbs +Shuhao Wu +Silas Sewell +Socrates Lee +Soren Hansen +Sreejith Kesavan +Timothée Peignier +William Kral diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 825e9ac3..f1fe1994 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -387,3 +387,37 @@ tutorial, but usage of this feature looks like:: .. _`Riak Search`: http://wiki.basho.com/Riak-Search.html .. _Lucene: http://lucene.apache.org/ + +Using Secondary Indexes +======================= + +Secondary Indexes is a new feature available as of Riak 1.0. It +allows you to tag an object with index metadata, and then later find +the object by querying the metadata, returning a list of matching keys. + +Your Riak cluster must have Secondary Indexes enabled. See the Riak +documentation for details. + +Usage of this feature looks like:: + + import riak + + client = riak.RiakClient() + bucket = client.bucket('mybucket') + + # Create and store the object with indexes... + obj = bucket.new('mykey1', 'mydata') + obj.add_index('field1_bin', 'val1') + obj.add_index('field2_int', 1001) + obj.store() + + # Query the indexes. The return value is a list of ``RiakLink`` objects. + results = client.index('mybucket', 'field1_bin', 'val1').run() + + # Query the indexes using a range... + results = client.index('mybucket', 'field1_bin', 'val1', 'val5').run() + + # Remove an index entry... + obj = bucket.get('mykey1') + obj.remove_index('field1_bin', 'val1') + obj.store() diff --git a/erl_src/riak_kv_test_backend.beam b/erl_src/riak_kv_test_backend.beam deleted file mode 100644 index f5a61ce9..00000000 Binary files a/erl_src/riak_kv_test_backend.beam and /dev/null differ diff --git a/erl_src/riak_kv_test_backend.erl b/erl_src/riak_kv_test_backend.erl deleted file mode 100644 index 29a7513b..00000000 --- a/erl_src/riak_kv_test_backend.erl +++ /dev/null @@ -1,174 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% riak_kv_test_backend: storage engine based on ETS tables -%% -%% -%% ------------------------------------------------------------------- - -% @doc riak_kv_test_backend is a Riak storage backend using ets that -% exposes a reset function for efficiently clearing stored data. - --module(riak_kv_test_backend). --behavior(riak_kv_backend). --behavior(gen_server). --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --endif. --export([start/2,stop/1,get/2,put/3,list/1,list_bucket/2,delete/2, - is_empty/1, drop/1, fold/3, callback/3, reset/0]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - - -% @type state() = term(). --record(state, {t, p}). - -% @spec start(Partition :: integer(), Config :: proplist()) -> -% {ok, state()} | {{error, Reason :: term()}, state()} -start(Partition, _Config) -> - gen_server:start_link(?MODULE, [Partition], []). - -% @spec reset() -> ok | {error, timeout} -reset() -> - Pids = lists:foldl(fun(Item, Acc) -> - case lists:prefix("test_backend", atom_to_list(Item)) of - true -> [whereis(Item)|Acc]; - _ -> Acc - end - end, [], registered()), - [gen_server:cast(Pid,{reset, self()})|| Pid <- Pids], - receive_reset(Pids). - -receive_reset([]) -> ok; -receive_reset(Pids) -> - receive - {reset, Pid} -> - receive_reset(lists:delete(Pid, Pids)) - after 1000 -> - {error, timeout} - end. - -%% @private -init([Partition]) -> - PName = list_to_atom("test_backend" ++ integer_to_list(Partition)), - P = list_to_atom(integer_to_list(Partition)), - register(PName, self()), - {ok, #state{t=ets:new(P,[]), p=P}}. - -%% @private -handle_cast({reset,From}, State) -> - ets:delete_all_objects(State#state.t), - From ! {reset, self()}, - {noreply, State}; -handle_cast(_, State) -> {noreply, State}. - -%% @private -handle_call(stop,_From,State) -> {reply, srv_stop(State), State}; -handle_call({get,BKey},_From,State) -> {reply, srv_get(State,BKey), State}; -handle_call({put,BKey,Val},_From,State) -> - {reply, srv_put(State,BKey,Val),State}; -handle_call({delete,BKey},_From,State) -> {reply, srv_delete(State,BKey),State}; -handle_call(list,_From,State) -> {reply, srv_list(State), State}; -handle_call({list_bucket,Bucket},_From,State) -> - {reply, srv_list_bucket(State, Bucket), State}; -handle_call(is_empty, _From, State) -> - {reply, ets:info(State#state.t, size) =:= 0, State}; -handle_call(drop, _From, State) -> - ets:delete(State#state.t), - {reply, ok, State}; -handle_call({fold, Fun0, Acc}, _From, State) -> - Fun = fun({{B,K}, V}, AccIn) -> Fun0({B,K}, V, AccIn) end, - Reply = ets:foldl(Fun, Acc, State#state.t), - {reply, Reply, State}. - -% @spec stop(state()) -> ok | {error, Reason :: term()} -stop(SrvRef) -> gen_server:call(SrvRef,stop). -srv_stop(State) -> - true = ets:delete(State#state.t), - ok. - -% get(state(), riak_object:bkey()) -> -% {ok, Val :: binary()} | {error, Reason :: term()} -% key must be 160b -get(SrvRef, BKey) -> gen_server:call(SrvRef,{get,BKey}). -srv_get(State, BKey) -> - case ets:lookup(State#state.t,BKey) of - [] -> {error, notfound}; - [{BKey,Val}] -> {ok, Val}; - Err -> {error, Err} - end. - -% put(state(), riak_object:bkey(), Val :: binary()) -> -% ok | {error, Reason :: term()} -% key must be 160b -put(SrvRef, BKey, Val) -> gen_server:call(SrvRef,{put,BKey,Val}). -srv_put(State,BKey,Val) -> - true = ets:insert(State#state.t, {BKey,Val}), - ok. - -% delete(state(), riak_object:bkey()) -> -% ok | {error, Reason :: term()} -% key must be 160b -delete(SrvRef, BKey) -> gen_server:call(SrvRef,{delete,BKey}). -srv_delete(State, BKey) -> - true = ets:delete(State#state.t, BKey), - ok. - -% list(state()) -> [riak_object:bkey()] -list(SrvRef) -> gen_server:call(SrvRef,list). -srv_list(State) -> - MList = ets:match(State#state.t,{'$1','_'}), - list(MList,[]). -list([],Acc) -> Acc; -list([[K]|Rest],Acc) -> list(Rest,[K|Acc]). - -% list_bucket(term(), Bucket :: riak_object:bucket()) -> [Key :: binary()] -list_bucket(SrvRef, Bucket) -> - gen_server:call(SrvRef,{list_bucket, Bucket}). -srv_list_bucket(State, {filter, Bucket, Fun}) -> - MList = lists:filter(Fun, ets:match(State#state.t,{{Bucket,'$1'},'_'})), - list(MList,[]); -srv_list_bucket(State, Bucket) -> - case Bucket of - '_' -> MatchSpec = {{'$1','_'},'_'}; - _ -> MatchSpec = {{Bucket,'$1'},'_'} - end, - MList = ets:match(State#state.t,MatchSpec), - list(MList,[]). - -is_empty(SrvRef) -> gen_server:call(SrvRef, is_empty). - -drop(SrvRef) -> gen_server:call(SrvRef, drop). - -fold(SrvRef, Fun, Acc0) -> gen_server:call(SrvRef, {fold, Fun, Acc0}, infinity). - -%% Ignore callbacks for other backends so multi backend works -callback(_State, _Ref, _Msg) -> - ok. - -%% @private -handle_info(_Msg, State) -> {noreply, State}. - -%% @private -terminate(_Reason, _State) -> ok. - -%% @private -code_change(_OldVsn, State, _Extra) -> {ok, State}. - -%% -%% Test -%% --ifdef(TEST). - -% @private -simple_test() -> - riak_kv_backend:standard_test(?MODULE, []). - --ifdef(EQC). -%% @private -eqc_test() -> - ?assertEqual(true, backend_eqc:test(?MODULE, true)). - --endif. % EQC --endif. % TEST diff --git a/erl_src/riak_search_test_backend.beam b/erl_src/riak_search_test_backend.beam deleted file mode 100644 index ddef8d24..00000000 Binary files a/erl_src/riak_search_test_backend.beam and /dev/null differ diff --git a/riak/__init__.py b/riak/__init__.py index d8577d1d..8a334910 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -41,8 +41,8 @@ def __str__(self): from client import RiakClient from mapreduce import RiakMapReduce, RiakMapReducePhase, RiakLinkPhase,\ RiakKeyFilter -from transports.pbc import RiakPbcTransport, RiakPbcCachedTransport -from transports.http import RiakHttpTransport, RiakHttpReuseTransport, RiakHttpPoolTransport +from transports.pbc import RiakPbcTransport +from transports.http import RiakHttpTransport ONE = "one" ALL = "all" diff --git a/riak/bucket.py b/riak/bucket.py index cef43271..64c492c0 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -38,7 +38,10 @@ def __init__(self, client, name): :param name: The bucket name :type name: string """ - if isinstance(name, unicode): + try: + if isinstance(name, basestring): + name = name.encode('ascii') + except UnicodeError: raise TypeError('Unicode bucket names are not supported.') self._client = client @@ -47,6 +50,8 @@ def __init__(self, client, name): self._w = None self._dw = None self._rw = None + self._pr = None + self._pw = None self._encoders = {} self._decoders = {} @@ -156,6 +161,57 @@ def set_rw(self, rw): self._rw = rw return self + def get_pr(self, pr=None): + """ + Get the PR-value for this bucket, if it is set, otherwise return + the PR-value for the client. + + :rtype: integer + """ + if (pr is not None): + return pr + if (self._pr is not None): + return self._pr + return self._client.get_pr() + + def set_pr(self, pr): + """ + Set the PR-value for this bucket. See :func:`set_r` for more + information. + + :param pr: The new PR-value + :type pr: integer + :rtype: self + """ + self._pr = pr + return self + + + def get_pw(self, pw=None): + """ + Get the PW-value for this bucket, if it is set, otherwise return + the PW-value for the client. + + :rtype: integer + """ + if (pw is not None): + return pw + if (self._pw is not None): + return self._pw + return self._client.get_pw() + + def set_pw(self, pw): + """ + Set the PW-value for this bucket. See :func:`set_r` for more + information. + + :param pw: The new PR-value + :type pw: integer + :rtype: self + """ + self._pw = pw + return self + def get_encoder(self, content_type): """ Get the encoding function for the provided content type for this bucket. @@ -210,7 +266,10 @@ def new(self, key, data=None, content_type='application/json'): :type data: object :rtype: :class:`RiakObject ` """ - if isinstance(data, unicode): + try: + if isinstance(data, basestring): + data = data.encode('ascii') + except UnicodeError: raise TypeError('Unicode data values are not supported.') obj = RiakObject(self._client, self, key) @@ -238,7 +297,7 @@ def new_binary(self, key, data, content_type='application/octet-stream'): obj._encode_data = False return obj - def get(self, key, r=None): + def get(self, key, r=None, pr=None): """ Retrieve a JSON-encoded object from Riak. @@ -246,14 +305,17 @@ def get(self, key, r=None): :type key: string :param r: R-Value of the request (defaults to bucket's R) :type r: integer + :param pr: PR-Value of the request (defaults to bucket's PR) + :type pr: integer :rtype: :class:`RiakObject ` """ obj = RiakObject(self._client, self, key) obj._encode_data = True r = self.get_r(r) - return obj.reload(r) + pr = self.get_pr(pr) + return obj.reload(r=r, pr=pr) - def get_binary(self, key, r=None): + def get_binary(self, key, r=None, pr=None): """ Retrieve a binary/string object from Riak. @@ -261,12 +323,15 @@ def get_binary(self, key, r=None): :type key: string :param r: R-Value of the request (defaults to bucket's R) :type r: integer + :param pr: PR-Value of the request (defaults to bucket's PR) + :type pr: integer :rtype: :class:`RiakObject ` """ obj = RiakObject(self._client, self, key) obj._encode_data = False r = self.get_r(r) - return obj.reload(r) + pr = self.get_pr(pr) + return obj.reload(r=r, pr=pr) def set_n_val(self, nval): """ @@ -422,7 +487,7 @@ def new_binary_from_file(self, key, filename): if not mimetype: mimetype = 'application/octet-stream' return self.new_binary(key, binary_data, mimetype) - + def search_enabled(self): """ Returns True if the search precommit hook is enabled for this bucket. diff --git a/riak/client.py b/riak/client.py index 867cd2e4..e53f2347 100644 --- a/riak/client.py +++ b/riak/client.py @@ -23,10 +23,12 @@ except ImportError: import simplejson as json -from riak.transports import RiakHttpTransport from riak.bucket import RiakBucket from riak.mapreduce import RiakMapReduce from riak.search import RiakSearch +from riak.transports import RiakHttpTransport +from riak.util import deprecated + class RiakClient(object): """ @@ -36,7 +38,8 @@ class RiakClient(object): """ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', mapred_prefix='mapred', transport_class=None, - client_id=None, solr_transport_class=None): + client_id=None, solr_transport_class=None, + transport_options=None): """ Construct a new ``RiakClient`` object. @@ -52,24 +55,42 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', :type transport_class: :class:`RiakTransport` :param solr_transport_class: HTTP-based transport class for Solr interface queries :type transport_class: :class:`RiakHttpTransport` - """ - if not transport_class: - self._transport = RiakHttpTransport(host, - port, - prefix, - mapred_prefix, - client_id) + :param transport_options: Optional key-value args to pass to the transport constuctor + :type transport_options: dict + """ + if transport_class is None: + transport_class = RiakHttpTransport + + api = getattr(transport_class, 'api', 1) + if api >= 2: + hostports = [ (host, port), ] + self._cm = transport_class.default_cm(hostports) + + # If no transport options are provided, then default to the + # empty dict, otherwise just pass through what we are provided. + if transport_options is None: + transport_options = {} + + self._transport = transport_class(self._cm, + prefix=prefix, + mapred_prefix=mapred_prefix, + client_id=client_id, + **transport_options) else: + deprecated('please upgrade the transport to the new API') + self._cm = None self._transport = transport_class(host, port, client_id=client_id) self._r = "default" self._w = "default" self._dw = "default" self._rw = "default" - self._encoders = {'application/json':json.dumps, - 'text/json':json.dumps} - self._decoders = {'application/json':json.loads, - 'text/json':json.loads} + self._pr = "default" + self._pw = "default" + self._encoders = {'application/json': json.dumps, + 'text/json': json.dumps} + self._decoders = {'application/json': json.loads, + 'text/json': json.loads} self._solr = None self._host = host self._port = port @@ -163,6 +184,46 @@ def set_rw(self, rw): self._rw = rw return self + def get_pr(self): + """ + Get the PR-value setting for this ``RiakClient``. (default 0) + + :rtype: integer + """ + return self._pr + + def set_pr(self, pr): + """ + Set the PR-value for this ``RiakClient`` instance. See :func:`set_r` for a + description of how these values are used. + + :param pr: The PR value. + :type pr: integer + :rtype: self + """ + self._pr = pr + return self + + def get_pw(self): + """ + Get the PW-value setting for this ``RiakClient``. (default 0) + + :rtype: integer + """ + return self._pr + + def set_pw(self, pw): + """ + Set the PW-value for this ``RiakClient`` instance. See :func:`set_r` for a + description of how these values are used. + + :param pw: The W value. + :type pw: integer + :rtype: self + """ + self._pr = pr + return self + def get_client_id(self): """ Get the ``client_id`` for this ``RiakClient`` instance. @@ -265,6 +326,16 @@ def search(self, *args): mr = RiakMapReduce(self) return apply(mr.search, args) + def index(self, *args): + """ + Start assembling a Map/Reduce operation based on secondary + index query results. + + :rtype: :class:`RiakMapReduce` + """ + mr = RiakMapReduce(self) + return apply(mr.index, args) + def link(self, *args): """ Start assembling a Map/Reduce operation. A shortcut for :func:`RiakMapReduce.link`. diff --git a/riak/erl_src/riak_kv_test_backend.beam b/riak/erl_src/riak_kv_test_backend.beam new file mode 100644 index 00000000..485d6b75 Binary files /dev/null and b/riak/erl_src/riak_kv_test_backend.beam differ diff --git a/riak/erl_src/riak_kv_test_backend.erl b/riak/erl_src/riak_kv_test_backend.erl new file mode 100644 index 00000000..4ddf7455 --- /dev/null +++ b/riak/erl_src/riak_kv_test_backend.erl @@ -0,0 +1,522 @@ +%% ------------------------------------------------------------------- +%% +%% riak_memory_backend: storage engine using ETS tables +%% +%% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +%% @doc riak_kv_memory_backend is a Riak storage backend that uses ets +%% tables to store all data in memory. +%% +%% === Configuration Options === +%% +%% The following configuration options are available for the memory backend. +%% The options should be specified in the `memory_backend' section of your +%% app.config file. +%% +%%
    +%%
  • `ttl' - The time in seconds that an object should live before being expired.
    • +%%
  • `max_memory' - The amount of memory in megabytes to limit the backend to.
    • +%%
+%% + +-module(riak_kv_test_backend). +-behavior(riak_kv_backend). + +%% KV Backend API +-export([api_version/0, + start/2, + stop/1, + get/3, + put/5, + delete/4, + drop/1, + fold_buckets/4, + fold_keys/4, + fold_objects/4, + is_empty/1, + status/1, + callback/3, + reset/0]). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +-define(API_VERSION, 1). +-define(CAPABILITIES, [async_fold]). + +-record(state, {data_ref :: integer() | atom(), + time_ref :: integer() | atom(), + max_memory :: undefined | integer(), + used_memory=0 :: integer(), + ttl :: integer()}). + +-type state() :: #state{}. +-type config() :: []. + +%% =================================================================== +%% Public API +%% =================================================================== + +%% TestServer reset + +-spec reset() -> ok | {error, timeout}. +reset() -> + {ok, Ring} = riak_core_ring_manager:get_my_ring(), + [ ets:delete_all_objects(list_to_atom("kv" ++ integer_to_list(P))) || + P <- riak_core_ring:my_indices(Ring) ], + ok. + +%% KV Backend API + +%% @doc Return the major version of the +%% current API and a capabilities list. +-spec api_version() -> {integer(), [atom()]}. +api_version() -> + {?API_VERSION, ?CAPABILITIES}. + +%% @doc Start the memory backend +-spec start(integer(), config()) -> {ok, state()}. +start(Partition, Config) -> + TTL = config_value(ttl, Config), + MemoryMB = config_value(max_memory, Config), + case MemoryMB of + undefined -> + MaxMemory = undefined, + TimeRef = undefined; + _ -> + MaxMemory = MemoryMB * 1024 * 1024, + TimeRef = ets:new(list_to_atom(integer_to_list(Partition)), [ordered_set]) + end, + DataRef = ets:new(list_to_atom("kv" ++ integer_to_list(Partition)), [named_table, public]), + {ok, #state{data_ref=DataRef, + max_memory=MaxMemory, + time_ref=TimeRef, + ttl=TTL}}. + +%% @doc Stop the memory backend +-spec stop(state()) -> ok. +stop(#state{data_ref=DataRef, + max_memory=MaxMemory, + time_ref=TimeRef}) -> + catch ets:delete(DataRef), + case MaxMemory of + undefined -> + ok; + _ -> + catch ets:delete(TimeRef) + end, + ok. + +%% @doc Retrieve an object from the memory backend +-spec get(riak_object:bucket(), riak_object:key(), state()) -> + {ok, any(), state()} | + {ok, not_found, state()} | + {error, term(), state()}. +get(Bucket, Key, State=#state{data_ref=DataRef, + ttl=TTL}) -> + case ets:lookup(DataRef, {Bucket, Key}) of + [] -> {error, not_found, State}; + [{{Bucket, Key}, {{ts, Timestamp}, Val}}] -> + case exceeds_ttl(Timestamp, TTL) of + true -> + delete(Bucket, Key, undefined, State), + {error, not_found, State}; + false -> + {ok, Val, State} + end; + [{{Bucket, Key}, Val}] -> + {ok, Val, State}; + Error -> + {error, Error, State} + end. + +%% @doc Insert an object into the memory backend. +%% NOTE: The memory backend does not currently +%% support secondary indexing and the _IndexSpecs +%% parameter is ignored. +-type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. +-spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> + {ok, state()} | + {error, term(), state()}. +put(Bucket, PrimaryKey, _IndexSpecs, Val, State=#state{data_ref=DataRef, + max_memory=MaxMemory, + time_ref=TimeRef, + ttl=TTL, + used_memory=UsedMemory}) -> + Now = now(), + case TTL of + undefined -> + Val1 = Val; + _ -> + Val1 = {{ts, Now}, Val} + end, + case do_put(Bucket, PrimaryKey, Val1, DataRef) of + {ok, Size} -> + %% If the memory is capped update timestamp table + %% and check if the memory usage is over the cap. + case MaxMemory of + undefined -> + UsedMemory1 = UsedMemory; + _ -> + time_entry(Bucket, PrimaryKey, Now, TimeRef), + Freed = trim_data_table(MaxMemory, + UsedMemory + Size, + DataRef, + TimeRef, + 0), + UsedMemory1 = UsedMemory + Size - Freed + end, + {ok, State#state{used_memory=UsedMemory1}}; + {error, Reason} -> + {error, Reason, State} + end. + +%% @doc Delete an object from the memory backend +%% NOTE: The memory backend does not currently +%% support secondary indexing and the _IndexSpecs +%% parameter is ignored. +-spec delete(riak_object:bucket(), riak_object:key(), [index_spec()], state()) -> + {ok, state()}. +delete(Bucket, Key, _IndexSpecs, State=#state{data_ref=DataRef, + time_ref=TimeRef, + used_memory=UsedMemory}) -> + case TimeRef of + undefined -> + UsedMemory1 = UsedMemory; + _ -> + %% Lookup the object so we can delete its + %% entry from the time table and account + %% for the memory used. + [Object] = ets:lookup(DataRef, {Bucket, Key}), + case Object of + {_, {{ts, Timestamp}, _}} -> + ets:delete(TimeRef, Timestamp), + UsedMemory1 = UsedMemory - object_size(Object); + _ -> + UsedMemory1 = UsedMemory + end + end, + ets:delete(DataRef, {Bucket, Key}), + {ok, State#state{used_memory=UsedMemory1}}. + +%% @doc Fold over all the buckets. +-spec fold_buckets(riak_kv_backend:fold_buckets_fun(), + any(), + [], + state()) -> {ok, any()}. +fold_buckets(FoldBucketsFun, Acc, Opts, #state{data_ref=DataRef}) -> + FoldFun = fold_buckets_fun(FoldBucketsFun), + case lists:member(async_fold, Opts) of + true -> + BucketFolder = + fun() -> + {Acc0, _} = ets:foldl(FoldFun, {Acc, sets:new()}, DataRef), + Acc0 + end, + {async, BucketFolder}; + false -> + {Acc0, _} = ets:foldl(FoldFun, {Acc, sets:new()}, DataRef), + {ok, Acc0} + end. + +%% @doc Fold over all the keys for one or all buckets. +-spec fold_keys(riak_kv_backend:fold_keys_fun(), + any(), + [{atom(), term()}], + state()) -> {ok, term()} | {async, fun()}. +fold_keys(FoldKeysFun, Acc, Opts, #state{data_ref=DataRef}) -> + Bucket = proplists:get_value(bucket, Opts), + FoldFun = fold_keys_fun(FoldKeysFun, Bucket), + case lists:member(async_fold, Opts) of + true -> + {async, get_folder(FoldFun, Acc, DataRef)}; + false -> + Acc0 = ets:foldl(FoldFun, Acc, DataRef), + {ok, Acc0} + end. + +%% @doc Fold over all the objects for one or all buckets. +-spec fold_objects(riak_kv_backend:fold_objects_fun(), + any(), + [{atom(), term()}], + state()) -> {ok, any()} | {async, fun()}. +fold_objects(FoldObjectsFun, Acc, Opts, #state{data_ref=DataRef}) -> + Bucket = proplists:get_value(bucket, Opts), + FoldFun = fold_objects_fun(FoldObjectsFun, Bucket), + case lists:member(async_fold, Opts) of + true -> + {async, get_folder(FoldFun, Acc, DataRef)}; + false -> + Acc0 = ets:foldl(FoldFun, Acc, DataRef), + {ok, Acc0} + end. + +%% @doc Delete all objects from this memory backend +-spec drop(state()) -> {ok, state()}. +drop(State=#state{data_ref=DataRef, + time_ref=TimeRef}) -> + ets:delete_all_objects(DataRef), + case TimeRef of + undefined -> + ok; + _ -> + ets:delete_all_objects(TimeRef) + end, + {ok, State}. + +%% @doc Returns true if this memory backend contains any +%% non-tombstone values; otherwise returns false. +-spec is_empty(state()) -> boolean(). +is_empty(#state{data_ref=DataRef}) -> + ets:info(DataRef, size) =:= 0. + +%% @doc Get the status information for this memory backend +-spec status(state()) -> [{atom(), term()}]. +status(#state{data_ref=DataRef, + time_ref=TimeRef}) -> + DataStatus = ets:info(DataRef), + case TimeRef of + undefined -> + [{data_table_status, DataStatus}]; + _ -> + TimeStatus = ets:info(TimeRef), + [{data_table_status, DataStatus}, + {time_table_status, TimeStatus}] + end. + +%% @doc Register an asynchronous callback +-spec callback(reference(), any(), state()) -> {ok, state()}. +callback(_Ref, _Msg, State) -> + {ok, State}. + +%% =================================================================== +%% Internal functions +%% =================================================================== + +%% @TODO Some of these implementations may be suboptimal. +%% Need to do some measuring and testing to refine the +%% implementations. + +%% @private +%% Return a function to fold over the buckets on this backend +fold_buckets_fun(FoldBucketsFun) -> + fun({{Bucket, _}, _}, {Acc, BucketSet}) -> + case sets:is_element(Bucket, BucketSet) of + true -> + {Acc, BucketSet}; + false -> + {FoldBucketsFun(Bucket, Acc), + sets:add_element(Bucket, BucketSet)} + end + end. + +%% @private +%% Return a function to fold over keys on this backend +fold_keys_fun(FoldKeysFun, undefined) -> + fun({{Bucket, Key}, _}, Acc) -> + FoldKeysFun(Bucket, Key, Acc) + end; +fold_keys_fun(FoldKeysFun, Bucket) -> + fun({{B, Key}, _}, Acc) -> + case B =:= Bucket of + true -> + FoldKeysFun(Bucket, Key, Acc); + false -> + Acc + end + end. + +%% @private +%% Return a function to fold over keys on this backend +fold_objects_fun(FoldObjectsFun, undefined) -> + fun({{Bucket, Key}, Value}, Acc) -> + FoldObjectsFun(Bucket, Key, Value, Acc) + end; +fold_objects_fun(FoldObjectsFun, Bucket) -> + fun({{B, Key}, Value}, Acc) -> + case B =:= Bucket of + true -> + FoldObjectsFun(Bucket, Key, Value, Acc); + false -> + Acc + end + end. + +%% @private +get_folder(FoldFun, Acc, DataRef) -> + fun() -> + ets:foldl(FoldFun, Acc, DataRef) + end. + +%% @private +do_put(Bucket, Key, Val, Ref) -> + Object = {{Bucket, Key}, Val}, + true = ets:insert(Ref, Object), + {ok, object_size(Object)}. + +%% @private +config_value(Key, Config) -> + config_value(Key, Config, undefined). + +%% @private +config_value(Key, Config, Default) -> + case proplists:get_value(Key, Config) of + undefined -> + app_helper:get_env(memory_backend, Key, Default); + Value -> + Value + end. + +%% Check if this timestamp is past the ttl setting. +exceeds_ttl(Timestamp, TTL) -> + Diff = (timer:now_diff(now(), Timestamp) / 1000 / 1000), + Diff > TTL. + +%% @private +time_entry(Bucket, Key, Now, TimeRef) -> + ets:insert(TimeRef, {Now, {Bucket, Key}}). + +%% @private +%% @doc Dump some entries if the max memory size has +%% been breached. +trim_data_table(MaxMemory, UsedMemory, _, _, Freed) when + (UsedMemory - Freed) =< MaxMemory -> + Freed; +trim_data_table(MaxMemory, UsedMemory, DataRef, TimeRef, Freed) -> + %% Delete the oldest object + OldestSize = delete_oldest(DataRef, TimeRef), + trim_data_table(MaxMemory, + UsedMemory, + DataRef, + TimeRef, + Freed + OldestSize). + +%% @private +delete_oldest(DataRef, TimeRef) -> + OldestTime = ets:first(TimeRef), + case OldestTime of + '$end_of_table' -> + 0; + _ -> + OldestKey = ets:lookup_element(TimeRef, OldestTime, 2), + ets:delete(TimeRef, OldestTime), + case ets:lookup(DataRef, OldestKey) of + [] -> + delete_oldest(DataRef, TimeRef); + [Object] -> + ets:delete(DataRef, OldestKey), + object_size(Object) + end + end. + +%% @private +object_size(Object) -> + case Object of + {{Bucket, Key}, {{ts, _}, Val}} -> + ok; + {{Bucket, Key}, Val} -> + ok + end, + size(Bucket) + size(Key) + size(Val). + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +simple_test_() -> + riak_kv_backend:standard_test(?MODULE, []). + +ttl_test_() -> + Config = [{ttl, 15}], + {ok, State} = start(42, Config), + + Bucket = <<"Bucket">>, + Key = <<"Key">>, + Value = <<"Value">>, + + [ + %% Put an object + ?_assertEqual({ok, State}, put(Bucket, Key, [], Value, State)), + %% Wait 1 second to access it + ?_assertEqual(ok, timer:sleep(1000)), + ?_assertEqual({ok, Value, State}, get(Bucket, Key, State)), + %% Wait 3 seconds and access it again + ?_assertEqual(ok, timer:sleep(3000)), + ?_assertEqual({ok, Value, State}, get(Bucket, Key, State)), + %% Wait 15 seconds and it should expire + {timeout, 30000, ?_assertEqual(ok, timer:sleep(15000))}, + %% This time it should be gone + ?_assertEqual({error, not_found, State}, get(Bucket, Key, State)) + ]. + +%% @private +max_memory_test_() -> + %% Set max size to 1.5kb + Config = [{max_memory, 1.5 * (1 / 1024)}], + {ok, State} = start(42, Config), + + Bucket = <<"Bucket">>, + Key1 = <<"Key1">>, + Value1 = list_to_binary(string:copies("1", 1024)), + Key2 = <<"Key2">>, + Value2 = list_to_binary(string:copies("2", 1024)), + + %% Write Key1 to the datastore + {ok, State1} = put(Bucket, Key1, [], Value1, State), + timer:sleep(timer:seconds(1)), + %% Write Key2 to the datastore + {ok, State2} = put(Bucket, Key2, [], Value2, State1), + + [ + %% Key1 should be kicked out + ?_assertEqual({error, not_found, State2}, get(Bucket, Key1, State2)), + %% Key2 should still be present + ?_assertEqual({ok, Value2, State2}, get(Bucket, Key2, State2)) + ]. + +-ifdef(EQC). + +eqc_test_() -> + {spawn, + [{inorder, + [{setup, + fun setup/0, + fun cleanup/1, + [ + {timeout, 60000, + [?_assertEqual(true, + backend_eqc:test(?MODULE, true))]} + ]}]}]}. + +setup() -> + application:load(sasl), + application:set_env(sasl, sasl_error_logger, {file, "riak_kv_memory_backend_eqc_sasl.log"}), + error_logger:tty(false), + error_logger:logfile({open, "riak_kv_memory_backend_eqc.log"}), + ok. + +cleanup(_) -> + ok. + +-endif. % EQC + +-endif. % TEST diff --git a/riak/erl_src/riak_search_test_backend.beam b/riak/erl_src/riak_search_test_backend.beam new file mode 100644 index 00000000..b20a76c9 Binary files /dev/null and b/riak/erl_src/riak_search_test_backend.beam differ diff --git a/erl_src/riak_search_test_backend.erl b/riak/erl_src/riak_search_test_backend.erl similarity index 96% rename from erl_src/riak_search_test_backend.erl rename to riak/erl_src/riak_search_test_backend.erl index 79b9da99..477903eb 100644 --- a/erl_src/riak_search_test_backend.erl +++ b/riak/erl_src/riak_search_test_backend.erl @@ -24,19 +24,19 @@ stream_results/3 ]). --include("riak_search.hrl"). +-include_lib("riak_search/include/riak_search.hrl"). -record(state, {partition, table}). reset() -> {ok, Ring} = riak_core_ring_manager:get_my_ring(), - [ ets:delete_all_objects(list_to_atom(integer_to_list(P))) || + [ ets:delete_all_objects(list_to_atom("rs" ++ integer_to_list(P))) || P <- riak_core_ring:my_indices(Ring) ], riak_search_config:clear(), ok. start(Partition, _Config) -> - Table = ets:new(list_to_atom(integer_to_list(Partition)), + Table = ets:new(list_to_atom("rs" ++ integer_to_list(Partition)), [named_table, public, ordered_set]), {ok, #state{partition=Partition, table=Table}}. diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 01020c97..fa7d4f2b 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -63,8 +63,8 @@ def add_object(self, obj): def add_bucket_key_data(self, bucket, key, data) : if self._input_mode == 'bucket': raise Exception('Already added a bucket, can\'t add an object.') - elif self._input_mode == 'search': - raise Exception('Already added a search query, can\'t add an object.') + elif self._input_mode == 'query': + raise Exception('Already added a query, can\'t add an object.') else: self._inputs.append([bucket, key, data]) return self @@ -75,15 +75,15 @@ def add_bucket(self, bucket) : return self def add_key_filters(self, key_filters) : - if self._input_mode == 'search': - raise Exception('Key filters are not supported in search query.') + if self._input_mode == 'query': + raise Exception('Key filters are not supported in a query.') self._key_filters.extend(key_filters) return self def add_key_filter(self, *args) : - if self._input_mode == 'search': - raise Exception('Key filters are not supported in search query.') + if self._input_mode == 'query': + raise Exception('Key filters are not supported in a query.') self._key_filters.append(args) return self @@ -95,12 +95,31 @@ def search(self, bucket, query): @param bucket - The bucket over which to perform the search. @param query - The search query. """ - self._input_mode = 'search' + self._input_mode = 'query' self._inputs = {'module':'riak_search', 'function':'mapred_search', 'arg':[bucket, query]} return self + def index(self, bucket, index, startkey, endkey = None): + """ + Begin a map/reduce operation using a Secondary Index + query. + @param bucket - The bucket over which to perform the search. + @param query - The search query. + """ + self._input_mode = 'query' + + if endkey == None: + self._inputs = {'bucket': bucket, + 'index':index, + 'key':startkey } + else: + self._inputs = {'bucket':bucket, + 'index':index, + 'start':startkey, + 'end':endkey } + return self def link(self, bucket='_', tag='_', keep=False): """ @@ -215,6 +234,10 @@ def run(self, timeout=None): if not link_results_flag: return result + # If there are no results, then return an empty list. + if result == None: + return [] + # Otherwise, if the last phase IS a link phase, then convert the # results to RiakLink objects. a = [] @@ -297,7 +320,10 @@ def __init__(self, type, function, language, keep, arg): @param mixed arg - Additional value to pass into the map or reduce function. """ - if isinstance(function, unicode): + try: + if isinstance(function, basestring): + function = function.encode('ascii') + except UnicodeError: raise TypeError('Unicode encoded functions are not supported.') self._type = type diff --git a/riak/metadata.py b/riak/metadata.py index 5efad289..37c81b2a 100644 --- a/riak/metadata.py +++ b/riak/metadata.py @@ -25,3 +25,4 @@ MD_LASTMOD = "lastmod" MD_LASTMOD_USECS = "lastmod-usecs" MD_USERMETA = "usermeta" +MD_INDEX = "index" diff --git a/riak/riak_index_entry.py b/riak/riak_index_entry.py new file mode 100644 index 00000000..0a26bb13 --- /dev/null +++ b/riak/riak_index_entry.py @@ -0,0 +1,62 @@ +""" +Copyright 2010 Rusty Klophaus +Copyright 2010 Justin Sheehy +Copyright 2009 Jay Baird + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +class RiakIndexEntry: + def __init__(self, field, value): + self._field = field + self._value = str(value) + + def get_field(self): + return self._field + + def get_value(self): + return self._value + + def __str__(self): + return "RiakIndexEntry(field = '%s', value='%s')" % (self._field, self._value) + + def __eq__(self, other): + if not isinstance(other, RiakIndexEntry): + return False + + return \ + self.get_field() == other.get_field() and \ + self.get_value() == other.get_value() + + def __cmp__(self, other): + if other == None: + raise TypeError("RiakIndexEntry cannot be compared to None") + + if not isinstance(other, RiakIndexEntry): + raise TypeError("RiakIndexEntry cannot be compared to %s" % other.__class__.__name__) + + if self.get_field() < other.get_field(): + return -1 + + if self.get_field() > other.get_field(): + return 1 + + if self.get_value() < other.get_value(): + return -1 + + if self.get_value() > other.get_value(): + return 1 + + return 0 diff --git a/riak/riak_object.py b/riak/riak_object.py index 3d552900..6a36a905 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -20,6 +20,7 @@ import types, copy, re from metadata import * from riak import RiakError +from riak.riak_index_entry import RiakIndexEntry class RiakObject(object): """ @@ -38,7 +39,10 @@ def __init__(self, client, bucket, key=None): is generated by the server when :func:`store` is called. :type key: string """ - if isinstance(key, unicode): + try: + if isinstance(key, basestring): + key = key.encode('ascii') + except UnicodeError: raise TypeError('Unicode keys are not supported.') self._client = client @@ -47,7 +51,7 @@ def __init__(self, client, bucket, key=None): self._encode_data = True self._vclock = None self._data = None - self._metadata = {MD_USERMETA: {}} + self._metadata = {MD_USERMETA: {}, MD_INDEX: []} self._links = [] self._siblings = [] self._exists = False @@ -177,6 +181,77 @@ def set_usermeta(self, usermeta): self._metadata[MD_USERMETA] = usermeta return self + def add_index(self, field, value): + """ + Tag this object with the specified field/value pair for indexing. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: self + """ + rie = RiakIndexEntry(field, value) + if not rie in self._metadata[MD_INDEX]: + self._metadata[MD_INDEX].append(rie) + + return self + + def remove_index(self, field=None, value=None): + """ + Remove the specified field/value pair as an index on this object. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: self + """ + if not field and not value: + ries = self._metadata[MD_INDEX][:] + elif field and not value: + ries = [x for x in self._metadata[MD_INDEX] if x.get_field() == field] + elif field and value: + ries = [RiakIndexEntry(field, value)] + else: + raise Exception("Cannot pass value without a field name while removing index") + + for rie in ries: + if rie in self._metadata[MD_INDEX]: + self._metadata[MD_INDEX].remove(rie) + return self + + remove_indexes = remove_index + + def set_indexes(self, indexes): + """ + Replaces all indexes on a Riak object. Currenly supports an iterable of 2 item tuples, + (field, value) + + :param indexes: iterable of 2 item tuples consisting the field and value. + :rtype: self + """ + new_indexes = [] + for field, value in indexes: + rie = RiakIndexEntry(field, value) + new_indexes.append(rie) + self._metadata[MD_INDEX] = new_indexes + + return self + + def get_indexes(self, field = None): + """ + Get a list of the index entries for this object. If a field is provided, returns a list + + :param field: The index field. + :type field: string or None + :rtype: (array of RiakIndexEntry) or (array of string or integer) + """ + if field == None: + return self._metadata[MD_INDEX] + else: + return [x.get_value() for x in self._metadata[MD_INDEX] if x.get_field() == field] + def exists(self): """ Return True if the object exists, False otherwise. Allows you to @@ -196,7 +271,13 @@ def get_content_type(self): :rtype: string """ - return self._metadata[MD_CTYPE] + try: + return self._metadata[MD_CTYPE] + except KeyError: + if self._encode_data: + return "application/json" + else: + return "application/octet-stream" def set_content_type(self, content_type): """ @@ -209,6 +290,33 @@ def set_content_type(self, content_type): self._metadata[MD_CTYPE] = content_type return self + def set_links(self, links, all_link=False): + """ + Replaces all links to a RiakObject + + :param links: An iterable of 2-item tuples, consisting of (RiakObject, tag). This could also be an iterable of + just a RiakObject, instead of the tuple, then a tag of None would be used. Lastly, it could also be an + iterable of RiakLink. They have tags built-in. + :param all_link: A boolean indicates if links are all RiakLink objects + This speeds up the operation. + """ + if all_link: + self._metadata[MD_LINKS] = links + return self + + new_links = [] + for item in links: + if isinstance(item, RiakLink): + link = item + elif isinstance(item, RiakObject): + link = RiakLink(item._bucket._name, item._key, None) + else: + link = RiakLink(item[0]._bucket._name, item[0]._key, item[1]) + new_links.append(link) + + self._metadata[MD_LINKS] = new_links + return self + def add_link(self, obj, tag=None): """ Add a link to a RiakObject. @@ -270,7 +378,7 @@ def get_links(self): else: return [] - def store(self, w=None, dw=None, return_body=True): + def store(self, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """ Store the object in Riak. When this operation completes, the object could contain new metadata and possibly new data if Riak @@ -283,24 +391,39 @@ def store(self, w=None, dw=None, return_body=True): :param dw: DW-value, wait for this many partitions to confirm the write before returning to client. :type dw: integer + :param pw: PW-value, require this many primary partitions to be available + before performing the put + :type pw: integer :param return_body: if the newly stored object should be retrieved :type return_body: bool + :param if_none_match: Should the object be stored only if there is no + key previously defined + :type if_none_match: bool :rtype: self """ # Use defaults if not specified... w = self._bucket.get_w(w) - dw = self._bucket.get_dw(w) + dw = self._bucket.get_dw(dw) + pw = self._bucket.get_pw(pw) - # Issue the get over our transport + # Issue the put over our transport t = self._client.get_transport() - Result = t.put(self, w, dw, return_body) - if Result is not None: - self.populate(Result) + + if self._key is None: + key, vclock, metadata = t.put_new(self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) + self._exists = True + self._key = key + self._vclock = vclock + self.set_metadata(metadata) + else: + Result = t.put(self, w=w, dw=dw, pw=pw, return_body=return_body, if_none_match=if_none_match) + if Result is not None: + self.populate(Result) return self - def reload(self, r=None, vtag=None): + def reload(self, r=None, pr=None, vtag=None): """ Reload the object from Riak. When this operation completes, the object could contain new metadata and a new value, if the object @@ -313,8 +436,9 @@ def reload(self, r=None, vtag=None): """ # Do the request... r = self._bucket.get_r(r) + pr = self._bucket.get_pr(pr) t = self._client.get_transport() - Result = t.get(self, r, vtag) + Result = t.get(self, r=r, pr=pr, vtag=vtag) self.clear() if Result is not None: @@ -323,19 +447,39 @@ def reload(self, r=None, vtag=None): return self - def delete(self, rw=None): + def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Delete this object from Riak. :param rw: RW-value. Wait until this many partitions have - deleted the object before responding. + deleted the object before responding. (deprecated in Riak 1.0+, use R/W/DW) :type rw: integer + :param r: R-value, wait for this many partitions to read object + before performing the put + :type r: integer + :param w: W-value, wait for this many partitions to respond + before returning to client. + :type w: integer + :param dw: DW-value, wait for this many partitions to + confirm the write before returning to client. + :type dw: integer + :param pr: PR-value, require this many primary partitions to be available + before performing the read that precedes the put + :type pr: integer + :param pr: PW-value, require this many primary partitions to be available + before performing the put + :type pw: integer :rtype: self """ # Use defaults if not specified... rw = self._bucket.get_rw(rw) + r = self._bucket.get_r(r) + w = self._bucket.get_w(w) + dw = self._bucket.get_dw(dw) + pr = self._bucket.get_pr(pr) + pw = self._bucket.get_pw(pw) t = self._client.get_transport() - Result = t.delete(self, rw) + Result = t.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) self.clear() return self @@ -381,6 +525,8 @@ def populate(self, Result) : if len(contents) > 0: (metadata, data) = contents.pop(0) self._exists = True + if not metadata.has_key(MD_INDEX): + metadata[MD_INDEX] = [] self.set_metadata(metadata) self.set_encoded_data(data) # Create objects for all siblings @@ -395,20 +541,6 @@ def populate(self, Result) : else: raise RiakError("do not know how to handle type " + str(type(Result))) - def populate_links(self, linkHeaders) : - """ - Private. - - :rtype: self - """ - for linkHeader in linkHeaders.strip().split(','): - linkHeader = linkHeader.strip() - matches = re.match("\<\/([^\/]+)\/([^\/]+)\/([^\/]+)\>; ?riaktag=\"([^\']+)\"", linkHeader) - if (matches is not None): - link = RiakLink(matches.group(2), matches.group(3), matches.group(4)) - self._links.append(link) - return self - def has_siblings(self): """ Return True if this object has siblings. @@ -425,7 +557,7 @@ def get_sibling_count(self): """ return len(self._siblings) - def get_sibling(self, i, r=None): + def get_sibling(self, i, r=None, pr=None): """ Retrieve a sibling by sibling number. @@ -441,11 +573,12 @@ def get_sibling(self, i, r=None): else: # Use defaults if not specified. r = self._bucket.get_r(r) + pr = self._bucket.get_pr(pr) # Run the request... vtag = self._siblings[i] obj = RiakObject(self._client, self._bucket, self._key) - obj.reload(r, vtag) + obj.reload(r=r, pr=pr, vtag=vtag) # And make sure it knows who it's siblings are self._siblings[i] = obj @@ -461,7 +594,7 @@ def get_siblings(self, r=None): :type r: integer :rtype: array of RiakObject """ - a = [self] + a = [] for i in range(self.get_sibling_count()): a.append(self.get_sibling(i, r)) return a diff --git a/riak/search.py b/riak/search.py index c01f9d3d..7fe2ab73 100644 --- a/riak/search.py +++ b/riak/search.py @@ -5,12 +5,20 @@ class RiakSearch: def __init__(self, client, transport_class=None, host="127.0.0.1", port=8098): - if not transport_class: - self._transport = RiakHttpTransport(host, - port, - "/solr") + if transport_class is None: + transport_class = RiakHttpTransport + + api = getattr(transport_class, 'api', 1) + if api >= 2: + hostports = [ (host, port), ] + self._cm = transport_class.default_cm(hostports) + self._transport = transport_class(self._cm, prefix="/solr") else: - self._transport = transport_class(host, port, client_id=client_id) + # The old code which attempted to use api==1 would actually + # throw a NameError, so it was obviously never used. We will + # simply raise an error here, intead of a gentle warning. + raise DeprecationWarning('please upgrade the transport to the ' + 'new API') self._client = client self._decoders = {"text/xml": ElementTree.fromstring} diff --git a/riak/test_server.py b/riak/test_server.py index 0262aaa4..56715fbe 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -1,3 +1,5 @@ +from __future__ import with_statement + import os.path import threading import string @@ -8,6 +10,27 @@ from subprocess import Popen, PIPE from riak.util import deep_merge +try: + bytes +except NameError: + bytes = str + +class Atom(object): + def __init__(self, s): + self.str = s + + def __str__(self): + return str(self.str) + + def __repr__(self): + return repr(self.str) + + def __eq__(self, other): + return self.str == other + + def __cmp__(self, other): + return cmp(self.str, other) + def erlang_config(hash, depth=1): def printable(item): k, v = item @@ -37,7 +60,7 @@ class TestServer: "-smp": "enable", "-env ERL_MAX_PORTS": 4096, "-env ERL_FULLSWEEP_AFTER": 10, - "-pa": os.path.abspath(os.path.join(os.path.dirname(__file__), "../erl_src")) + "-pa": os.path.abspath(os.path.join(os.path.dirname(__file__), "erl_src")) } APP_CONFIG_DEFAULTS = { @@ -48,7 +71,7 @@ class TestServer: "ring_creation_size": 64 }, "riak_kv": { - "storage_backend": bytearray("riak_kv_test_backend"), + "storage_backend": Atom("riak_kv_test_backend"), "pb_ip": "127.0.0.1", "pb_port": 9002, "js_vm_count": 8, @@ -60,7 +83,7 @@ class TestServer: }, "riak_search": { "enabled": True, - "search_backend": bytearray("riak_search_test_backend") + "search_backend": Atom("riak_search_test_backend") }, "luwak": { "enabled": True @@ -71,7 +94,7 @@ def __init__(self, tmp_dir="/tmp/riak/test_server", bin_dir=os.path.expanduser("~/.riak/install/riak-0.14.2/bin"), vm_args=None, **options): self._lock = threading.Lock() - self.temp_dir = "/tmp/riak/test_server" + self.temp_dir = tmp_dir self.bin_dir = bin_dir self._prepared = False self._started = False @@ -85,6 +108,7 @@ def __init__(self, tmp_dir="/tmp/riak/test_server", self.app_config[key] = deep_merge(self.app_config[key], value) self.app_config["riak_core"]["ring_state_dir"] = os.path.join(self.temp_dir, "data", "ring") + self.app_config["riak_core"]["platform_data_dir"] = self.temp_dir def prepare(self): if not self._prepared: @@ -176,6 +200,7 @@ def write_riak_script(self): line = re.sub("(RUNNER_USER=)(.*)", r'\1', line) line = re.sub("(RUNNER_LOG_DIR=)(.*)", r'\1%s' % self._temp_log, line) line = re.sub("(PIPE_DIR=)(.*)", r'\1%s' % self._temp_pipe, line) + line = re.sub("(PLATFORM_DATA_DIR=)(.*)", r'\1%s' % self.temp_dir, line) if string.strip(line) == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}": line = "RUNNER_BASE_DIR=%s\n" % os.path.normpath(os.path.join(self.bin_dir, "..")) diff --git a/riak/tests/suite.py b/riak/tests/suite.py index bddfd45b..2e4735b7 100644 --- a/riak/tests/suite.py +++ b/riak/tests/suite.py @@ -1,6 +1,11 @@ -import unittest import riak.tests.test_server_test import os.path +import platform + +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest def additional_tests(): top_level = os.path.join(os.path.dirname(__file__), "../../") diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 7b5f5bdb..3ae08a4e 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import with_statement import copy import cPickle @@ -8,23 +9,38 @@ import simplejson as json import os import random -import unittest +import socket +import platform + +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest import uuid import time from riak import RiakClient -from riak import RiakPbcTransport, RiakPbcCachedTransport -from riak import RiakHttpTransport, RiakHttpPoolTransport, RiakHttpReuseTransport +from riak import RiakPbcTransport +from riak import RiakHttpTransport from riak import RiakKeyFilter, key_filter +from riak.riak_index_entry import RiakIndexEntry from riak.mapreduce import RiakLink from riak.test_server import TestServer +try: + import riak.transports.riakclient_pb2 + HAVE_PROTO = True +except ImportError: + HAVE_PROTO = False + HOST = os.environ.get('RIAK_TEST_HOST', 'localhost') HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) PB_HOST = os.environ.get('RIAK_TEST_PB_HOST', HOST) HTTP_PORT = int(os.environ.get('RIAK_TEST_HTTP_PORT', '8098')) PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '0')) +SKIP_LUWAK = int(os.environ.get('SKIP_LUWAK', '0')) +SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '0')) USE_TEST_SERVER = int(os.environ.get('USE_TEST_SERVER', '0')) if USE_TEST_SERVER: @@ -35,7 +51,6 @@ test_server.prepare() test_server.start() -SKIP_LUWAK = int(os.environ.get('SKIP_LUWAK', '0')) class NotJsonSerializable(object): @@ -75,6 +90,13 @@ def create_client(self, host=None, port=None, transport_class=None): def setUp(self): self.client = self.create_client() + # make sure these are not left over from a previous, failed run + bucket = self.client.bucket('bucket') + o = bucket.get('nonexistent_key_json') + o.delete() + o = bucket.get('nonexistent_key_binary') + o.delete() + def test_is_alive(self): self.assertTrue(self.client.is_alive()) @@ -89,12 +111,33 @@ def test_store_and_get(self): self.assertEqual(obj.get_key(), 'foo') self.assertEqual(obj.get_data(), rand) - #unicode input should raise a TypeError, - #to avoid issues further down the line - self.assertRaises(TypeError, self.client.bucket, u'bucket') + # unicode objects are fine, as long as they don't + # contain any non-ASCII chars + self.client.bucket(u'bucket') + self.assertRaises(TypeError, self.client.bucket, u'búcket') + self.assertRaises(TypeError, self.client.bucket, 'búcket') + + bucket.get(u'foo') + self.assertRaises(TypeError, bucket.get, u'føø') + self.assertRaises(TypeError, bucket.get, 'føø') + + self.assertRaises(TypeError, bucket.new, u'foo', 'éå') self.assertRaises(TypeError, bucket.new, u'foo', 'éå') self.assertRaises(TypeError, bucket.new, 'foo', u'éå') - self.assertRaises(TypeError, bucket.get, u'foo') + self.assertRaises(TypeError, bucket.new, 'foo', u'éå') + + def test_generate_key(self): + # Ensure that Riak generates a random key when + # the key passed to bucket.new() is None. + bucket = self.client.bucket('random_key_bucket') + existing_keys = bucket.get_keys() + o = bucket.new(None, data={}) + self.assertIsNone(o.get_key()) + o.store() + self.assertIsNotNone(o.get_key()) + self.assertNotIn('/', o.get_key()) + self.assertNotIn(o.get_key(), existing_keys) + self.assertEqual(len(bucket.get_keys()), len(existing_keys) + 1) def test_binary_store_and_get(self): bucket = self.client.bucket('bucket') @@ -192,16 +235,40 @@ def test_rw_settings(self): bucket.set_rw("one") self.assertEqual(bucket.get_rw(), "one") - def test_siblings(self): - # Set up the bucket, clear any existing object... - bucket = self.client.bucket('multiBucket') - bucket.set_allow_multiples(True) - obj = bucket.get('foo') + def test_primary_quora(self): + bucket = self.client.bucket('primary_quora') + self.assertEqual(bucket.get_pr(), "default") + self.assertEqual(bucket.get_pw(), "default") + + bucket.set_pr(1) + self.assertEqual(bucket.get_pr(), 1) + + bucket.set_pw("quorum") + self.assertEqual(bucket.get_pw(), "quorum") + + def test_if_none_match(self): + bucket = self.client.bucket('if_none_match_test') + obj = bucket.get('obj') obj.delete() obj.reload() self.assertFalse(obj.exists()) - self.assertEqual(obj.get_data(), None) + obj.set_data(["first store"]) + obj.store() + + obj.set_data(["second store"]) + with self.assertRaises(Exception): + obj.store(if_none_match=True) + + def test_siblings(self): + # Set up the bucket, clear any existing object... + bucket = self.client.bucket('multiBucket') + bucket.set_allow_multiples(True) + obj = bucket.get_binary('foo') + # Even if it previously existed, let's store a base resolved version + # from which we can diverge by sending a stale vclock. + obj.set_data('start') + obj.store() # Store the same object five times... vals = set() @@ -213,9 +280,10 @@ def test_siblings(self): if randval not in vals: break - other_obj = other_bucket.new('foo', randval) + other_obj = other_bucket.new_binary('foo', str(randval)) + other_obj._vclock = obj._vclock other_obj.store() - vals.add(randval) + vals.add(str(randval)) # Make sure the object has itself plus four siblings... obj.reload() @@ -236,9 +304,6 @@ def test_siblings(self): self.assertEqual(obj.get_sibling_count(), 0) self.assertEqual(obj.get_data(), obj3.get_data()) - # Clean up for next test... - obj.delete() - def test_javascript_source_map(self): # Create the object... bucket = self.client.bucket("bucket") @@ -249,9 +314,16 @@ def test_javascript_source_map(self): "function (v) { return [JSON.parse(v.values[0].data)]; }").run() self.assertEqual(result, [2]) - #test unicode function + # test ASCII-encodable unicode is accepted + mr.map(u"function (v) { return [JSON.parse(v.values[0].data)]; }") + + # test non-ASCII-encodable unicode is rejected + self.assertRaises(TypeError, mr.map, + u"function (v) { /* æ */ return [JSON.parse(v.values[0].data)]; }") + + # test non-ASCII-encodable string is rejected self.assertRaises(TypeError, mr.map, - u"function (v) { return [JSON.parse(v.values[0].data)]; }") + "function (v) { /* æ */ return [JSON.parse(v.values[0].data)]; }") def test_javascript_named_map(self): # Create the object... @@ -396,14 +468,48 @@ def test_map_reduce_from_object(self): def test_store_and_get_links(self): # Create the object... bucket = self.client.bucket("bucket") - bucket.new("foo", 2) \ + bucket.new_binary("test_store_and_get_links", '2') \ .add_link(bucket.new("foo1")) \ .add_link(bucket.new("foo2"), "tag") \ .add_link(bucket.new("foo3"), "tag2!@#%^&*)") \ .store() - obj = bucket.get("foo") + obj = bucket.get("test_store_and_get_links") links = obj.get_links() self.assertEqual(len(links), 3) + for l in links: + if (l.get_key() == "foo1"): + self.assertEqual(l.get_tag(), "bucket") + elif (l.get_key() == "foo2"): + self.assertEqual(l.get_tag(), "tag") + elif (l.get_key() == "foo3"): + self.assertEqual(l.get_tag(), "tag2!@#%^&*)") + else: + self.assertEqual("unknown key", l.get_key()) + + def test_set_links(self): + # Create the object + bucket = self.client.bucket("bucket") + bucket.new("foo", 2).set_links([bucket.new("foo1"), + (bucket.new("foo2"), "tag"), + RiakLink("bucket", "foo2", "tag2")]).store() + obj = bucket.get("foo") + links = sorted(obj.get_links(), key=lambda x: x.get_key()) + self.assertEqual(len(links), 3) + self.assertEqual(links[0].get_key(), "foo1") + self.assertEqual(links[1].get_key(), "foo2") + self.assertEqual(links[1].get_tag(), "tag") + self.assertEqual(links[2].get_key(), "foo2") + self.assertEqual(links[2].get_tag(), "tag2") + + def test_set_links_all_links(self): + bucket = self.client.bucket("bucket") + foo1 = bucket.new("foo", 1) + foo2 = bucket.new("foo2", 2).store() + links = [RiakLink("bucket", "foo2")] + foo1.set_links(links, True) + links = foo1.get_links() + self.assertEqual(len(links), 1) + self.assertEqual(links[0].get_key(), "foo2") def test_link_walking(self): # Create the object... @@ -438,10 +544,8 @@ def test_store_of_missing_object(self): self.assertEqual(o.get_content_type(), "application/octet-stream") o.delete() - + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_search_integration(self): - if SKIP_SEARCH: - return True # Create some objects to search across... bucket = self.client.bucket("searchbucket") bucket.new("one", {"foo":"one", "bar":"red"}).store() @@ -463,7 +567,8 @@ def test_search_integration(self): def test_store_binary_object_from_file(self): bucket = self.client.bucket('bucket') rand = str(self.randint()) - obj = bucket.new_binary_from_file('foo_from_file', os.path.dirname(__file__) + "/test_all.py") + filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') + obj = bucket.new_binary_from_file('foo_from_file', filepath) obj.store() obj = bucket.get_binary('foo_from_file') self.assertNotEqual(obj.get_data(), None) @@ -472,7 +577,9 @@ def test_store_binary_object_from_file(self): def test_store_binary_object_from_file_should_use_default_mimetype(self): bucket = self.client.bucket('bucket') rand = str(self.randint()) - obj = bucket.new_binary_from_file('foo_from_file', os.path.dirname(__file__) + '/../../THANKS') + filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), + os.pardir, os.pardir, 'THANKS') + obj = bucket.new_binary_from_file('foo_from_file', filepath) obj.store() obj = bucket.get_binary('foo_from_file') self.assertEqual(obj.get_content_type(), 'application/octet-stream') @@ -499,6 +606,216 @@ def test_list_buckets(self): buckets = self.client.get_buckets() self.assertTrue("list_bucket" in buckets) + def is_2i_supported(self): + # Immediate test to see if 2i is even supported w/ the backend + try: + self.client.index('foo','bar_bin','baz').run() + return True + except Exception as e: + if "indexes_not_supported" in str(e): + return False + return True # it failed, but is supported! + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_secondary_index_store(self): + if not self.is_2i_supported(): + return True + + # Create a new object with indexes... + bucket = self.client.bucket('indexbucket') + rand = self.randint() + obj = bucket.new('mykey1', rand) + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_int', 1011) + obj.store() + + # Retrieve the object, check that the correct indexes exist... + obj = bucket.get('mykey1') + self.assertEqual(['val1a'], sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1011'], sorted(obj.get_indexes('field1_int'))) + + # Add more indexes and save... + obj.add_index('field1_bin', 'val1b') + obj.add_index('field1_int', 1012) + obj.store() + + # Retrieve the object, check that the correct indexes exist... + obj = bucket.get('mykey1') + self.assertEqual(['val1a', 'val1b'], sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1011', '1012'], sorted(obj.get_indexes('field1_int'))) + + # Check the get_indexes() function... + self.assertEqual([ + RiakIndexEntry('field1_bin', 'val1a'), + RiakIndexEntry('field1_bin', 'val1b'), + RiakIndexEntry('field1_int', 1011), + RiakIndexEntry('field1_int', 1012) + ], sorted(obj.get_indexes())) + + # Delete an index... + obj.remove_index('field1_bin', 'val1a') + obj.remove_index('field1_int', 1011) + obj.store() + + # Retrieve the object, check that the correct indexes exist... + obj = bucket.get('mykey1') + self.assertEqual(['val1b'], sorted(obj.get_indexes('field1_bin'))) + self.assertEqual(['1012'], sorted(obj.get_indexes('field1_int'))) + + # Check duplicate entries... + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_bin', 'val1a') + obj.add_index('field1_int', 1011) + obj.add_index('field1_int', 1011) + obj.add_index('field1_int', 1011) + + self.assertEqual([ + RiakIndexEntry('field1_bin', 'val1a'), + RiakIndexEntry('field1_bin', 'val1b'), + RiakIndexEntry('field1_int', 1011), + RiakIndexEntry('field1_int', 1012) + ], sorted(obj.get_indexes())) + + obj.store() + obj = bucket.get('mykey1') + + self.assertEqual([ + RiakIndexEntry('field1_bin', 'val1a'), + RiakIndexEntry('field1_bin', 'val1b'), + RiakIndexEntry('field1_int', 1011), + RiakIndexEntry('field1_int', 1012) + ], sorted(obj.get_indexes())) + + # Clean up... + bucket.get('mykey1').delete() + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_set_indexes(self): + if not self.is_2i_supported(): + return True + + bucket = self.client.bucket('indexbucket') + foo = bucket.new('foo', 1) + foo.set_indexes((('field1_bin', 'test'), ('field2_int', 1337))).store() + result = self.client.index('indexbucket', 'field2_int', 1337).run() + self.assertEqual(1, len(result)) + self.assertEqual('foo', result[0].get_key()) + + result = self.client.index('indexbucket', 'field1_bin', 'test').run() + self.assertEqual(1, len(result)) + self.assertEqual('foo', result[0].get_key()) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_remove_indexes(self): + if not self.is_2i_supported(): + return True + + bucket = self.client.bucket('indexbucket') + bar = bucket.new('bar', 1).add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + result = self.client.index('indexbucket', 'bar_int', 1).run() + self.assertEqual(1, len(result)) + self.assertEqual(3, len(bar.get_indexes())) + self.assertEqual(2, len(bar.get_indexes('bar_int'))) + + # remove all indexes + bar = bar.remove_indexes().store() + result = self.client.index('indexbucket', 'bar_int', 1).run() + self.assertEqual(0, len(result)) + result = self.client.index('indexbucket', 'baz_bin', 'baz').run() + self.assertEqual(0, len(result)) + self.assertEqual(0, len(bar.get_indexes())) + self.assertEqual(0, len(bar.get_indexes('bar_int'))) + self.assertEqual(0, len(bar.get_indexes('baz_bin'))) + + # add index again + bar = bar.add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + # remove all index with field='bar_int' + bar = bar.remove_index(field='bar_int').store() + result = self.client.index('indexbucket', 'bar_int', 1).run() + self.assertEqual(0, len(result)) + result = self.client.index('indexbucket', 'bar_int', 2).run() + self.assertEqual(0, len(result)) + result = self.client.index('indexbucket', 'baz_bin', 'baz').run() + self.assertEqual(1, len(result)) + self.assertEqual(1, len(bar.get_indexes())) + self.assertEqual(0, len(bar.get_indexes('bar_int'))) + self.assertEqual(1, len(bar.get_indexes('baz_bin'))) + + # add index again + bar = bar.add_index('bar_int', 1).add_index('bar_int', 2).add_index('baz_bin', 'baz').store() + # remove an index field value pair + bar = bar.remove_index(field='bar_int', value=2).store() + result = self.client.index('indexbucket', 'bar_int', 1).run() + self.assertEqual(1, len(result)) + result = self.client.index('indexbucket', 'bar_int', 2).run() + self.assertEqual(0, len(result)) + result = self.client.index('indexbucket', 'baz_bin', 'baz').run() + self.assertEqual(1, len(result)) + self.assertEqual(2, len(bar.get_indexes())) + self.assertEqual(1, len(bar.get_indexes('bar_int'))) + self.assertEqual(1, len(bar.get_indexes('baz_bin'))) + + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + def test_secondary_index_query(self): + if not self.is_2i_supported(): + return True + + bucket = self.client.bucket('indexbucket') + + bucket.\ + new('mykey1', 'data1').\ + add_index('field1_bin', 'val1').\ + add_index('field2_int', 1001).\ + store() + bucket.\ + new('mykey2', 'data1').\ + add_index('field1_bin', 'val2').\ + add_index('field2_int', 1002).\ + store() + bucket.\ + new('mykey3', 'data1').\ + add_index('field1_bin', 'val3').\ + add_index('field2_int', 1003).\ + store() + bucket.\ + new('mykey4', 'data1').\ + add_index('field1_bin', 'val4').\ + add_index('field2_int', 1004).\ + store() + + # Test an equality query... + results = self.client.index('indexbucket', 'field1_bin', 'val2').run() + self.assertEquals(1, len(results)) + self.assertEquals('mykey2', results[0].get_key()) + + # Test a range query... + results = self.client.index('indexbucket', 'field1_bin', 'val2', 'val4').run() + vals = set() + for i in results: + vals.add(i.get_key()) + self.assertEquals(3, len(results)) + self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) + + # Test an equality query... + results = self.client.index('indexbucket', 'field2_int', 1002).run() + self.assertEquals(1, len(results)) + self.assertEquals('mykey2', results[0].get_key()) + + # Test a range query... + results = self.client.index('indexbucket', 'field2_int', 1002, 1004).run() + vals = set() + for i in results: + vals.add(i.get_key()) + self.assertEquals(3, len(results)) + self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) + + # Clean up... + bucket.get('mykey1').delete() + bucket.get('mykey2').delete() + bucket.get('mykey3').delete() + bucket.get('mykey4').delete() + class MapReduceAliasTestMixIn(object): """This tests the map reduce aliases""" @@ -693,6 +1010,8 @@ class RiakPbcTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): def setUp(self): + if not HAVE_PROTO: + self.skipTest('protobuf is unavailable') self.host = PB_HOST self.port = PB_PORT self.transport_class = RiakPbcTransport @@ -707,22 +1026,54 @@ def test_uses_client_id_if_given(self): client_id = zero_client_id) self.assertEqual(zero_client_id, c.get_client_id()) # -class RiakPbcCachedTransportCase(BaseTestCase, MapReduceAliasTestMixIn, - unittest.TestCase): - def setUp(self): - self.host = PB_HOST - self.port = PB_PORT - self.transport_class = RiakPbcCachedTransport - super(RiakPbcCachedTransportCase, self).setUp() + def test_close_underlying_socket_fails(self): + c = RiakClient(PB_HOST, PB_PORT, transport_class = RiakPbcTransport) - def test_uses_client_id_if_given(self): - self.host = PB_HOST - self.port = PB_PORT - zero_client_id = "\0\0\0\0" - c = RiakClient(PB_HOST, PB_PORT, - transport_class = RiakPbcCachedTransport, - client_id = zero_client_id) - self.assertEqual(zero_client_id, c.get_client_id()) # + bucket = c.bucket('bucket_test_close') + rand = self.randint() + obj = bucket.new('foo', rand) + obj.store() + obj = bucket.get('foo') + self.assertTrue(obj.exists()) + self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') + self.assertEqual(obj.get_key(), 'foo') + self.assertEqual(obj.get_data(), rand) + + # Close the underlying socket. This gets a bit sketchy, + # since we are reaching into the internals, but there is + # no other way to get at the socket + conns = c._cm.conns + conns[0].sock.close() + + # This shoud fail with a socket error now + self.assertRaises(socket.error, bucket.get, 'foo') + + def test_close_underlying_socket_retry(self): + c = RiakClient(PB_HOST, PB_PORT, transport_class=RiakPbcTransport, + transport_options={"max_attempts": 2}) + + bucket = c.bucket('bucket_test_close') + rand = self.randint() + obj = bucket.new('barbaz', rand) + obj.store() + obj = bucket.get('barbaz') + self.assertTrue(obj.exists()) + self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') + self.assertEqual(obj.get_key(), 'barbaz') + self.assertEqual(obj.get_data(), rand) + + # Close the underlying socket. This gets a bit sketchy, + # since we are reaching into the internals, but there is + # no other way to get at the socket + conns = c._cm.conns + conns[0].sock.close() + + # This should work, since we have a retry + obj = bucket.get('barbaz') + self.assertTrue(obj.exists()) + self.assertEqual(obj.get_bucket().get_name(), 'bucket_test_close') + self.assertEqual(obj.get_key(), 'barbaz') + self.assertEqual(obj.get_data(), rand) class RiakHttpTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): @@ -738,15 +1089,6 @@ def test_no_returnbody(self): o = bucket.new("foo", "bar").store(return_body=False) self.assertEqual(o.vclock(), None) - def test_generate_key(self): - # Ensure that Riak generates a random key when - # the key passed to bucket.new() is None. - bucket = self.client.bucket('random_key_bucket') - for key in bucket.get_keys(): - bucket.get(str(key)).delete() - bucket.new(None, data={}).store() - self.assertEqual(len(bucket.get_keys()), 1) - def test_too_many_link_headers_shouldnt_break_http(self): bucket = self.client.bucket("bucket") o = bucket.new("lots_of_links", "My god, it's full of links!") @@ -774,22 +1116,18 @@ def test_disable_search_commit_hook(self): bucket.disable_search() self.assertFalse(self.client.bucket("no_search_bucket").search_enabled()) + @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_store_file_with_luwak(self): - if SKIP_LUWAK: - return True - - file = os.path.dirname(__file__) + "/test_all.py" + file = os.path.join(os.path.dirname(__file__), "test_all.py") with open(file, "r") as input_file: data = input_file.read() key = uuid.uuid1().hex self.client.store_file(key, data) + @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_store_get_file_with_luwak(self): - if SKIP_LUWAK: - return True - - file = os.path.dirname(__file__) + "/test_all.py" + file = os.path.join(os.path.dirname(__file__), "test_all.py") with open(file, "r") as input_file: data = input_file.read() @@ -799,11 +1137,9 @@ def test_store_get_file_with_luwak(self): file = self.client.get_file(key) self.assertEquals(data, file) + @unittest.skipIf(SKIP_LUWAK, 'SKIP_LUWAK is defined') def test_delete_file_with_luwak(self): - if SKIP_LUWAK: - return True - - file = os.path.dirname(__file__) + "/test_all.py" + file = os.path.join(os.path.dirname(__file__), "test_all.py") with open(file, "r") as input_file: data = input_file.read() @@ -815,125 +1151,73 @@ def test_delete_file_with_luwak(self): file = self.client.get_file(key) self.assertIsNone(file) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_from_bucket(self): - if SKIP_SEARCH: - return True bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage") self.assertEquals(1, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params_from_bucket(self): - if SKIP_SEARCH: - return True bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage", wt="xml") - self.assertEquals(1, len(list(results.find("result").iter("doc")))) + result = results.find("result") + if not hasattr(result, "iter"): setattr(result, "iter", result.getiterator) + self.assertEquals(1, len(list(result.iter("doc")))) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params(self): - if SKIP_SEARCH: - return True bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = self.client.solr().search("searchbucket", "username:roidrage", wt="xml") - self.assertEquals(1, len(list(results.find("result").iter("doc")))) + result = results.find("result") + if not hasattr(result, "iter"): setattr(result, "iter", result.getiterator) + self.assertEquals(1, len(list(result.iter("doc")))) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search(self): - if SKIP_SEARCH: - return True bucket = self.client.bucket('searchbucket') bucket.new("user", {"username": "roidrage"}).store() results = self.client.solr().search("searchbucket", "username:roidrage") self.assertEquals(1, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_document_to_index(self): - if SKIP_SEARCH: - return True - self.client.solr().add("searchbucket", {"id": "doc", "username": "tony"}) results = self.client.solr().search("searchbucket", "username:tony") self.assertEquals("tony", results["response"]["docs"][0]["fields"]["username"]) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_multiple_documents_to_index(self): - if SKIP_SEARCH: - return True self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(2, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_id(self): - if SKIP_SEARCH: - return True self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", docs=["dizzy"]) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(1, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): - if SKIP_SEARCH: - return True self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", queries=["username:dizzy", "username:russell"]) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results["response"]["docs"])) + @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query_and_id(self): - if SKIP_SEARCH: - return True self.client.solr().add("searchbucket", {"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}) self.client.solr().delete("searchbucket", docs=["dizzy"], queries=["username:russell"]) results = self.client.solr().search("searchbucket", "username:russell OR username:dizzy") self.assertEquals(0, len(results["response"]["docs"])) -class RiakHttpPoolTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): - - def setUp(self): - self.host = HTTP_HOST - self.port = HTTP_PORT - self.transport_class = RiakHttpPoolTransport - super(RiakHttpPoolTransportTestCase, self).setUp() - - def test_no_returnbody(self): - bucket = self.client.bucket("bucket") - o = bucket.new("foo", "bar").store(return_body=False) - self.assertEqual(o.vclock(), None) - - def test_generate_key(self): - # Ensure that Riak generates a random key when - # the key passed to bucket.new() is None. - bucket = self.client.bucket('random_key_bucket') - for key in bucket.get_keys(): - bucket.get(str(key)).delete() - bucket.new(None, data={}).store() - self.assertEqual(len(bucket.get_keys()), 1) - - def test_set_client_id(self): - self.client.set_client_id("Client") - self.assertEqual(self.client.get_transport().get_client_id(), "Client") - -class RiakHttpReuseTransportTestCase(BaseTestCase, MapReduceAliasTestMixIn, unittest.TestCase): - - def setUp(self): - self.host = HTTP_HOST - self.port = HTTP_PORT - self.transport_class = RiakHttpReuseTransport - super(RiakHttpReuseTransportTestCase, self).setUp() - - def test_no_returnbody(self): - bucket = self.client.bucket("bucket") - o = bucket.new("foo", "bar").store(return_body=False) - self.assertEqual(o.vclock(), None) - - def test_generate_key(self): - # Ensure that Riak generates a random key when - # the key passed to bucket.new() is None. - bucket = self.client.bucket('random_key_bucket') - for key in bucket.get_keys(): - bucket.get(str(key)).delete() - bucket.new(None, data={}).store() - self.assertEqual(len(bucket.get_keys()), 1) - + def test_build_rest_path_excludes_empty_query_params(self): + self.assertEquals(self.client.get_transport().build_rest_path(bucket=self.client.bucket("foo"), key="bar", params={'r': None}), "/riak/foo/bar?") class RiakTestFilter(unittest.TestCase): def test_simple(self): diff --git a/riak/tests/test_server_test.py b/riak/tests/test_server_test.py index 6a2437b8..1c1307cb 100644 --- a/riak/tests/test_server_test.py +++ b/riak/tests/test_server_test.py @@ -40,6 +40,11 @@ def test_set_ring_state_dir(self): def test_set_default_tmp_dir(self): self.assertEquals(self.test_server.temp_dir, "/tmp/riak/test_server") + def test_set_non_default_tmp_dir(self): + tmp_dir = '/not/the/default/dir' + server = TestServer(tmp_dir=tmp_dir) + self.assertEquals(server.temp_dir, tmp_dir) + def suite(): suite = unittest.TestSuite() suite.addTest(TestServerTestCase()) diff --git a/riak/transports/__init__.py b/riak/transports/__init__.py index f970fed9..55b7da3f 100644 --- a/riak/transports/__init__.py +++ b/riak/transports/__init__.py @@ -1,4 +1,4 @@ -from http import RiakHttpTransport, RiakHttpReuseTransport, RiakHttpPoolTransport -from pbc import RiakPbcTransport, RiakPbcCachedTransport +from http import RiakHttpTransport +from pbc import RiakPbcTransport diff --git a/riak/transports/connection.py b/riak/transports/connection.py new file mode 100644 index 00000000..67b16b46 --- /dev/null +++ b/riak/transports/connection.py @@ -0,0 +1,189 @@ +""" +Copyright 2011 Greg Stein + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import httplib +import socket +import contextlib +import functools + + +class ConnectionManager(object): + + # Must be constructable with: connection_class(host, port) + # Must have two attribute: host and port + # Must have a close() method + connection_class = None + + def __init__(self, hostports=[]): + # We want a private copy of this list: either to detach the argument + # default, or to detach from the caller's list. + self.hostports = hostports[:] + + # Open a connection to each specified host/port. On single-threaded + # systems, this will create a round-robin across all specified servers. + # When multi-threaded, this will give us an initial set for all the + # threads to work with (and more will be created, according to demand). + self.conns = [self.connection_class(host, port) + for host, port in hostports] + + def add_hostport(self, host, port): + self.hostports.append((host, port)) + + # Open an initial connection. For single-threaded, this adds to the + # round-robin pool. On multi-threaded, it simply gives us an extra + # connectiong for the load-balancing across the servers. + self.conns.append(self.connection_class(host, port)) + + def remove_host(self, host, port=None): + if port is None: + self.hostports = [(h, p) for h, p in self.hostports + if h != host] + else: + self.hostports.remove((host, port)) + + # Now that the host/port pair has been removed from self.hostports, + # no connections on this pair will be added in .giveback(). Thus, the + # existing connections are all that may exist at this time. We'll + # snapshot the list, and look for offending connections, then try and + # remove them, being wary that race conditions may remove them before + # we can remove it. + for conn in self.conns[:]: + if conn.host == host and (port is None or conn.port == port): + try: + self.conns.remove(conn) + except ValueError: + # Another thread removed the connection. It won't be coming back, + # so we have nothing to do here. + pass + else: + # If the connection was still present (no ValueError), then we + # should go ahead and close it down. + conn.close() + + # Just in case somebody uses a host/port combo and typos... + remove_hostport = remove_host + + def take(self): + if len(self.conns) == 0: + # RACE: in a multi-threaded environment, a conn might arrive in + # self.conns, but... no biggy. If we're bouncing up against + # needing a new connection, then we'll just create one. + return self._new_connection() + + # RACE: self.conns might empty out right now, so we need to protect + # our access to it. + try: + # round-robin: take from the front, we'll append when it comes back + return self.conns.pop(0) + except IndexError: + return self._new_connection() + + def giveback(self, conn): + # Connections using a host/port pair that is NOT in self.hostports + # should be ignored. Likely, remove_host() was called while this + # connection was borrowed for some work. + if (conn.host, conn.port) in self.hostports: + self.conns.append(conn) + else: + # Proactively close the connection. The caller won't know whether + # we put it into our list, or left the connection for the caller + # to deal with (and close). + conn.close() + + @contextlib.contextmanager + def withconn(self): + conn = self.take() + try: + yield conn + finally: + self.giveback(conn) + + def _new_connection(self): + if len(self.hostports) == 0: + raise NoHostsDefined() + + # Grab the first host/port combo. We'll put this at the end, so that + # we do a round-robin on the host/port pairs. + host, port = self.hostports[0] + conn = self.connection_class(host, port) + + if len(self.hostports) == 1: + # No rotation needed. + return conn + + # Be careful about rotating. We want to append before removing, so that + # we never hit a len==0 race condition (which could prevent the creation + # of needed connections). + self.hostports.append((host, port)) + + # RACE: another thread may have appended the same host/port pair. We + # will add another pair. Each thread will remove one (either [0], or + # one that had been appened), resulting in a correct state of a single + # pair in the list. + # RACE: another thread may get the host/port pair from hostports[0] + # before we have a chance to remove it. We don't need precision + # round-robin behavior; just something close. + # RACE: another thread may have removed hostports[0] (which we are + # also trying to remove), but it will have placed another copy at + # the end before doing so. We have also added a host/port pair, and + # will remove one, leaving the list in a correct state. + self.hostports.remove((host, port)) + + return conn + + +class Socket(object): + + def __init__(self, host, port): + self.host = host + self.port = port + + self.sock = None + + def maybe_connect(self): + if self.sock is None: + self.sock = s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + try: + s.connect((self.host, self.port)) + except: + self.close() + raise + + def close(self): + if self.sock is not None: + self.sock.close() + self.sock = None + + +class FactoryConnectionManager(ConnectionManager): + + def __init__(self, connection_class, hostports=[]): + self.connection_class = connection_class + ConnectionManager.__init__(self, hostports) + + +def cm_using(connection_class): + return functools.partial(FactoryConnectionManager, connection_class) + +HTTPConnectionManager = cm_using(httplib.HTTPConnection) +SocketConnectionManager = cm_using(Socket) + + +class NoHostsDefined(Exception): + pass diff --git a/riak/transports/http.py b/riak/transports/http.py index c719b8d1..c77955b6 100644 --- a/riak/transports/http.py +++ b/riak/transports/http.py @@ -17,15 +17,13 @@ specific language governing permissions and limitations under the License. """ -import urllib, re +from __future__ import with_statement + +import urllib, re, csv from cStringIO import StringIO -# Use pycurl as first choice, httplib as second choice. -try: - import pycurl - HAS_PYCURL = True -except ImportError: - import httplib - HAS_PYCURL = False +import httplib +import socket +import errno try: import json except ImportError: @@ -35,7 +33,10 @@ from riak.metadata import * from riak.mapreduce import RiakLink from riak import RiakError +from riak.riak_index_entry import RiakIndexEntry from riak.multidict import MultiDict +from connection import HTTPConnectionManager +import riak.util MAX_LINK_HEADER_SIZE = 8192 - 8 # substract length of "Link: " header string and newline @@ -45,9 +46,19 @@ class RiakHttpTransport(RiakTransport) : Riak. The Riak API uses HTTP, so there is no persistent connection, and the RiakClient object is extremely lightweight. """ - def __init__(self, host='127.0.0.1', port=8098, prefix='riak', - mapred_prefix='mapred', - client_id = None): + + # We're using the new RiakTransport API + api = 2 + + # The ConnectionManager class that this transport prefers. + default_cm = HTTPConnectionManager + + # How many times to retry a request + RETRY_COUNT = 3 + + def __init__(self, cm, + prefix='riak', mapred_prefix='mapred', client_id=None, + **unused_options): """ Construct a new RiakClient object. @param string host - Hostname or IP address (default '127.0.0.1') @@ -57,8 +68,7 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', @param string client_id - client id to use for vector clocks """ super(RiakHttpTransport, self).__init__() - self._host = host - self._port = port + self._conns = cm self._prefix = prefix self._mapred_prefix = mapred_prefix self._client_id = client_id @@ -66,6 +76,9 @@ def __init__(self, host='127.0.0.1', port=8098, prefix='riak', self._client_id = self.make_random_client_id() def __copy__(self): + ### not implemented right now + raise Exception('not implemented') + ### we don't have _host and _port. will fix after some refactoring... return RiakHttpTransport(self._host, self._port, self._prefix, self._mapred_prefix) @@ -79,54 +92,41 @@ def ping(self) : """ Check server is alive over HTTP """ - response = self.http_request('GET', self._host, self._port, '/ping') + response = self.http_request('GET', '/ping') return(response is not None) and (response[1] == 'OK') - def get(self, robj, r, vtag = None) : + def get(self, robj, r = None, pr = None, vtag = None) : """ Get a bucket/key from the server """ - params = {'r' : r} + params = {'r' : r, 'pr': pr} if vtag is not None: params['vtag'] = vtag - host, port, url = self.build_rest_path(robj.get_bucket(), robj.get_key(), - params=params) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(robj.get_bucket(), robj.get_key(), + params=params) + response = self.http_request('GET', url) return self.parse_body(response, [200, 300, 404]) - def put(self, robj, w = None, dw = None, return_body = True): + def put(self, robj, w = None, dw = None, pw = None, return_body = True, if_none_match=False): """ Serialize put request and deserialize response """ # Construct the URL... - params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw} - host, port, url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), - params=params) - - # Construct the headers... - headers = MultiDict({'Accept' : 'text/plain, */*; q=0.5', - 'Content-Type' : robj.get_content_type(), - 'X-Riak-ClientId' : self._client_id}) - - # Add the vclock if it exists... - if robj.vclock() is not None: - headers['X-Riak-Vclock'] = robj.vclock() - - # Create the header from metadata - links = self.add_links_for_riak_object(robj, headers) - - for key, value in robj.get_usermeta().iteritems(): - headers['X-Riak-Meta-%s' % key] = value - + params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw, 'pw' : pw } + url = self.build_rest_path(bucket=robj.get_bucket(), key=robj.get_key(), + params=params) + headers = self.build_put_headers(robj) + if if_none_match: + headers["If-None-Match"] = "*" content = robj.get_encoded_data() - return self.do_put(host, port, url, headers, content, return_body, key=robj.get_key()) + return self.do_put(url, headers, content, return_body, key=robj.get_key()) - def do_put(self, host, port, url, headers, content, return_body=False, key=None): + def do_put(self, url, headers, content, return_body=False, key=None): if key is None: - response = self.http_request('POST', host, port, url, headers, content) + response = self.http_request('POST', url, headers, content) else: - response = self.http_request('PUT', host, port, url, headers, content) + response = self.http_request('PUT', url, headers, content) if return_body: return self.parse_body(response, [200, 201, 300]) @@ -134,21 +134,42 @@ def do_put(self, host, port, url, headers, content, return_body=False, key=None) self.check_http_code(response, [204]) return None - def delete(self, robj, rw): + def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): + """Put a new object into the Riak store, returning its (new) key.""" + # Construct the URL... + params = {'returnbody' : str(return_body).lower(), 'w' : w, 'dw' : dw, 'pw' : pw} + url = self.build_rest_path(bucket=robj.get_bucket(), params=params) + headers = self.build_put_headers(robj) + if if_none_match: + headers["If-None-Match"] = "*" + content = robj.get_encoded_data() + response = self.http_request('POST', url, headers, content) + location = response[0]['location'] + idx = location.rindex('/') + key = location[idx+1:] + if return_body: + vclock, [(metadata, data)] = self.parse_body(response, [201]) + return key, vclock, metadata + else: + self.check_http_code(response, [201]) + return key, None, None + + def delete(self, robj, rw=None, r = None, w = None, dw = None, pr = None, pw = None): # Construct the URL... - params = {'rw' : rw} - host, port, url = self.build_rest_path(robj.get_bucket(), robj.get_key(), - params=params) + params = {'rw' : rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} + url = self.build_rest_path(robj.get_bucket(), robj.get_key(), + params=params) + # TODO: Send vclock of robj if it exists # Run the operation.. - response = self.http_request('DELETE', host, port, url) + response = self.http_request('DELETE', url) self.check_http_code(response, [204, 404]) return self def get_keys(self, bucket): params = {'props' : 'True', 'keys' : 'true'} - host, port, url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(bucket, params=params) + response = self.http_request('GET', url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: @@ -159,8 +180,8 @@ def get_keys(self, bucket): def get_buckets(self): params = {'buckets': 'true'} - host, port, url = self.build_rest_path(None, params=params) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(None, params=params) + response = self.http_request('GET', url) headers, encoded_props = response[0:2] if headers['http_code'] == 200: @@ -172,8 +193,8 @@ def get_buckets(self): def get_bucket_props(self, bucket): # Run the request... params = {'props' : 'True', 'keys' : 'False'} - host, port, url = self.build_rest_path(bucket, params=params) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(bucket, params=params) + response = self.http_request('GET', url) headers = response[0] encoded_props = response[1] @@ -188,12 +209,12 @@ def set_bucket_props(self, bucket, props): """ Set the properties on the bucket object given """ - host, port, url = self.build_rest_path(bucket) + url = self.build_rest_path(bucket) headers = {'Content-Type' : 'application/json'} content = json.dumps({'props' : props}) # Run the request... - response = self.http_request('PUT', host, port, url, headers, content) + response = self.http_request('PUT', url, headers, content) # Handle the response... if response is None: @@ -214,17 +235,21 @@ def mapred(self, inputs, query, timeout=None): content = json.dumps(job) # Do the request... - host = self._host - port = self._port url = "/" + self._mapred_prefix - response = self.http_request('POST', host, port, url, {}, content) + response = self.http_request('POST', url, {}, content) + + # Make sure the expected status code came back... + status = response[0]['http_code'] + if status != 200: + raise Exception('Error running MapReduce operation. Status: ' + str(status) + ' : ' + response[1]) + result = json.loads(response[1]) return result def check_http_code(self, response, expected_statuses): status = response[0]['http_code'] if not status in expected_statuses: - m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) + m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) + ' : ' + response[1] raise Exception(m) def parse_body(self, response, expected_statuses): @@ -248,12 +273,8 @@ def parse_body(self, response, expected_statuses): # Check if the server is down(status==0) if not status: - m = 'Could not contact Riak Server: http://' + self._host + ':' + str(self._port) + '!' - raise RiakError(m) - - # Verify that we got one of the expected statuses. Otherwise, raise an exception. - if not status in expected_statuses: - m = 'Expected status ' + str(expected_statuses) + ', received ' + str(status) + ### we need the host/port that was used. + m = 'Could not contact Riak Server: http://$HOST:$PORT !' raise RiakError(m) # If 404(Not Found), then clear the object. @@ -269,7 +290,7 @@ def parse_body(self, response, expected_statuses): # Parse the headers... vclock = None - metadata = {MD_USERMETA: {}} + metadata = {MD_USERMETA: {}, MD_INDEX: []} links = [] for header, value in headers.iteritems(): if header == 'content-type': @@ -286,6 +307,14 @@ def parse_body(self, response, expected_statuses): metadata[MD_LASTMOD] = value elif header.startswith('x-riak-meta-'): metadata[MD_USERMETA][header.replace('x-riak-meta-', '')] = value + elif header.startswith('x-riak-index-'): + field = header.replace('x-riak-index-', '') + reader = csv.reader([value], skipinitialspace=True) + for line in reader: + for token in line: + rie = RiakIndexEntry(field, token) + metadata[MD_INDEX].append(rie) + elif header == 'x-riak-vclock': vclock = value if links: @@ -312,9 +341,12 @@ def parse_links(self, links, linkHeaders): """ for linkHeader in linkHeaders.strip().split(','): linkHeader = linkHeader.strip() - matches = re.match("; ?riaktag=\"([^\']+)\"", linkHeader) + matches = re.match("; ?riaktag=\"([^\']+)\"", linkHeader) or \ + re.match("; ?riaktag=\"([^\']+)\"", linkHeader) if matches is not None: - link = RiakLink(matches.group(2), matches.group(3), matches.group(4)) + link = RiakLink(urllib.unquote_plus(matches.group(2)), + urllib.unquote_plus(matches.group(3)), + urllib.unquote_plus(matches.group(4))) links.append(link) return self @@ -325,7 +357,7 @@ def add_links_for_riak_object(self, robject, headers): for link in links: header = self.to_link_header(link) if len(current_header + header) > MAX_LINK_HEADER_SIZE: - headers.setdefault('Link', []).append(current_header) + headers.add('Link', current_header) current_header = '' if current_header != '': header = ', ' + header @@ -336,19 +368,19 @@ def add_links_for_riak_object(self, robject, headers): return headers def get_request(self, uri=None, params=None): - host, port, url = self.build_rest_path(bucket=None, params=params, prefix=uri) - return self.http_request('GET', host, port, url) + url = self.build_rest_path(bucket=None, params=params, prefix=uri) + return self.http_request('GET', url) def store_file(self, key, content_type="application/octet-stream", content=None): - host, port, url = self.build_rest_path(prefix='luwak', key=key) + url = self.build_rest_path(prefix='luwak', key=key) headers = {'Content-Type' : content_type, 'X-Riak-ClientId' : self._client_id} - return self.do_put(host, port, url, headers, content, key=key) + return self.do_put(url, headers, content, key=key) def get_file(self, key): - host, port, url = self.build_rest_path(prefix='luwak', key=key) - response = self.http_request('GET', host, port, url) + url = self.build_rest_path(prefix='luwak', key=key) + response = self.http_request('GET', url) result = self.parse_body(response, [200, 300, 404]) if result is not None: (vclock, data) = result @@ -356,23 +388,16 @@ def get_file(self, key): return body def delete_file(self, key): - host, port, url = self.build_rest_path(prefix='luwak', key=key) - response = self.http_request('DELETE', host, port, url) + url = self.build_rest_path(prefix='luwak', key=key) + response = self.http_request('DELETE', url) self.parse_body(response, [204, 404]) def post_request(self, uri=None, body=None, params=None, content_type="application/json"): - host, port, uri = self.build_rest_path(prefix=uri, params=params) - return self.http_request('POST', self._host, self._port, uri, {'Content-Type': content_type}, body) + uri = self.build_rest_path(prefix=uri, params=params) + return self.http_request('POST', uri, {'Content-Type': content_type}, body) # Utility functions used by Riak library. - @classmethod - def get_value(cls, key, array, defaultValue) : - if key in array: - return array[key] - else: - return defaultValue - def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : """ Given a RiakClient, RiakBucket, Key, LinkSpec, and Params, @@ -394,101 +419,84 @@ def build_rest_path(self, bucket=None, key=None, params=None, prefix=None) : if params is not None: s = '' for key in params.keys(): - if s != '': s += '&' - s += urllib.quote_plus(key) + '=' + urllib.quote_plus(str(params[key])) + if params[key] is not None: + if s != '': s += '&' + s += urllib.quote_plus(key) + '=' + urllib.quote_plus(str(params[key])) path += '?' + s # Return. - return self._host, self._port, path + return path - @classmethod - def http_request(cls, method, host, port, url, headers = None, obj = '') : - """ - Given a Method, URL, Headers, and Body, perform and HTTP request, - and return an array of arity 2 containing an associative array of - response headers and the response body. - """ - if not headers: - headers = {} - if HAS_PYCURL: - return cls.pycurl_request(method, host, port, url, headers, obj) - else: - return cls.httplib_request(method, host, port, url, headers, obj) + def build_put_headers(self, robj): + """Build the headers for a POST/PUT request.""" + # Construct the headers... + headers = MultiDict({'Accept' : 'text/plain, */*; q=0.5', + 'Content-Type' : robj.get_content_type(), + 'X-Riak-ClientId' : self._client_id}) - @classmethod - def httplib_request(cls, method, host, port, uri, headers = None, body=''): - if not headers: - headers = {} - # Run the request... - client = None - response = None - try: - client = httplib.HTTPConnection(host, port) - client.request(method, uri, body, headers) - response = client.getresponse() + # Add the vclock if it exists... + if robj.vclock() is not None: + headers['X-Riak-Vclock'] = robj.vclock() - # Get the response headers... - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value + # Create the header from metadata + links = self.add_links_for_riak_object(robj, headers) - # Get the body... - response_body = response.read() - response.close() + for key, value in robj.get_usermeta().iteritems(): + headers['X-Riak-Meta-%s' % key] = value - return response_headers, response_body - except: - if client is not None: client.close() - if response is not None: response.close() - raise + for rie in robj.get_indexes(): + key = 'X-Riak-Index-%s' % rie.get_field() + if key in headers: + headers[key] += ", " + rie.get_value() + else: + headers[key] = rie.get_value() + return headers - @classmethod - def pycurl_request(cls, method, host, port, uri, headers, body=''): - if not headers: + def http_request(self, method, uri, headers=None, body='') : + """ + Given a Method, URL, Headers, and Body, perform and HTTP request, + and return a 2-tuple containing a dictionary of response headers + and the response body. + """ + if headers is None: headers = {} - url = "http://" + host + ":" + str(port) + uri - # Set up Curl... - client = pycurl.Curl() - client.setopt(pycurl.URL, url) - client.setopt(pycurl.HTTPHEADER, cls.build_headers(headers)) - if method == 'GET': - client.setopt(pycurl.HTTPGET, 1) - elif method == 'POST': - client.setopt(pycurl.POST, 1) - client.setopt(pycurl.POSTFIELDS, body) - elif method == 'PUT': - client.setopt(pycurl.CUSTOMREQUEST, method) - client.setopt(pycurl.POSTFIELDS, body) - elif method == 'DELETE': - client.setopt(pycurl.CUSTOMREQUEST, method) - - # Capture the response headers... - response_headers_io = StringIO() - client.setopt(pycurl.HEADERFUNCTION, response_headers_io.write) - - # Capture the response body... - response_body_io = StringIO() - client.setopt(pycurl.WRITEFUNCTION, response_body_io.write) - - try: - # Run the request. - client.perform() - http_code = client.getinfo(pycurl.HTTP_CODE) - client.close() - - # Get the headers... - response_headers = cls.parse_http_headers(response_headers_io.getvalue()) - response_headers['http_code'] = http_code - - # Get the body... - response_body = response_body_io.getvalue() - - return response_headers, response_body - except: - if client is not None: client.close() - raise + # Run the request... + for retry in range(self.RETRY_COUNT): + with self._conns.withconn() as conn: + ### should probably build this try/except into a custom + ### contextmanager for the connection. + try: + conn.request(method, uri, body, headers) + response = conn.getresponse() + + try: + # Get the response headers... + response_headers = {'http_code': response.status} + for (key, value) in response.getheaders(): + response_headers[key.lower()] = value + + # Get the body... + response_body = response.read() + finally: + response.close() + + return response_headers, response_body + except socket.error, e: + conn.close() + if e[0] == errno.ECONNRESET: + # Grab another connection and try again. + continue + # Don't know how to handle this. + raise + except httplib.HTTPException: + # Just close the connection and try again. + conn.close() + continue + + # No luck, even with retrying. + raise RiakError("could not get a response") @classmethod def build_headers(cls, headers): @@ -515,101 +523,3 @@ def parse_http_headers(cls, headers) : else: retVal[key] = value return retVal - -import socket - -class RiakHttpReuseTransport(RiakHttpTransport): - """ - Reuse sockets - """ - - def __init__(self, host='127.0.0.1', port=8098, prefix='riak', - mapred_prefix='mapred', - client_id=None): - super(RiakHttpReuseTransport, self).__init__(host=host, - port=port, - prefix=prefix, - mapred_prefix= - mapred_prefix, - client_id=client_id) - - def __copy__(self): - return RiakHttpReuseTransport(self._host, self._port, self._prefix, - self._mapred_prefix) - - @classmethod - def httplib_request(cls, method, host, port, uri, headers, body=''): - # Run the request... - client = None - response = None - try: - client = httplib.HTTPConnection(host, port) - - #handle the connection myself, try to reuse sockets - client.auto_open = 0 - client.connect() - client.sock.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, - client.sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) | 1) - - client.request(method, uri, body, headers) - response = client.getresponse() - - # Get the response headers... - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value - - # Get the body... - response_body = response.read() - response.close() - - #close, this does not make any difference - client.close() - - return response_headers, response_body - except: - if client is not None: client.close() - if response is not None: response.close() - raise - -import urllib3 - -class RiakHttpPoolTransport(RiakHttpTransport): - """ - Use HTTP pool - """ - - http_pool = None - - def __init__(self, host='127.0.0.1', port=8098, prefix='riak', - mapred_prefix='mapred', - client_id=None): - super(RiakHttpPoolTransport, self).__init__(host=host, - port=port, - prefix=prefix, - mapred_prefix= - mapred_prefix, - client_id=client_id) - - def __copy__(self): - return RiakHttpPoolTransport(self._host, self._port, self._prefix, - self._mapred_prefix) - - @classmethod - def httplib_request(cls, method, host, port, uri, headers, body=''): - try: - if cls.http_pool is None: - cls.http_pool = urllib3.connection_from_url('http://%s:%d' % (host, port), maxsize=10) - - response = cls.http_pool.urlopen(method, uri, body, headers) - - response_headers = {'http_code': response.status} - for key, value in response.getheaders().iteritems(): - response_headers[key.lower()] = value - - response_body = response.data - - return response_headers, response_body - except: - raise diff --git a/riak/transports/monitor.py b/riak/transports/monitor.py new file mode 100644 index 00000000..c6223518 --- /dev/null +++ b/riak/transports/monitor.py @@ -0,0 +1,47 @@ +""" +Copyright 2011 Greg Stein + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +import threading +import time + + +class Monitor(object): + + def __init__(self, cm, transport): + self._cm = cm + self._transport = transport + + self._stop_loop = False + self._thread = threading.Thread(target=self._run) + + self._periodic = 0.050 # 50 msec + + def start(self): + self._thread.start() + + def terminate(self): + self._stop_loop = True + self._thread.join() + + def _run(self): + while not self._stop_loop: + ### look for changes in the ring servers + + ### see if some offline servers came back + + time.sleep(self._periodic) diff --git a/riak/transports/pbc.py b/riak/transports/pbc.py index ae71c764..0538c4f0 100644 --- a/riak/transports/pbc.py +++ b/riak/transports/pbc.py @@ -17,18 +17,39 @@ specific language governing permissions and limitations under the License. """ -import socket, struct +from __future__ import with_statement + +import errno +import socket +import struct try: import json except ImportError: import simplejson as json -from transport import RiakTransport -from riak.metadata import * -from riak.mapreduce import RiakMapReduce, RiakLink from riak import RiakError -import riakclient_pb2 +from riak.mapreduce import RiakLink +from riak.metadata import ( + MD_CHARSET, + MD_CTYPE, + MD_ENCODING, + MD_INDEX, + MD_LASTMOD, + MD_LASTMOD_USECS, + MD_LINKS, + MD_USERMETA, + MD_VTAG, + ) +from riak.riak_index_entry import RiakIndexEntry +from riak.transports import connection +from riak.transports.transport import RiakTransport +import riak.util + +try: + import riakclient_pb2 +except ImportError: + riakclient_pb2 = None ## Protocol codes MSG_CODE_ERROR_RESP = 0 @@ -62,7 +83,58 @@ RIAKC_RW_ALL = 4294967292 RIAKC_RW_DEFAULT = 4294967291 +# These are a specific set of socket errors +# that could be raised on send/recv that indicate +# that the socket is closed or reset, and is not +# usable. On seeing any of these errors, the socket +# should be closed, and the connection re-established. +CONN_CLOSED_ERRORS = ( + errno.EHOSTUNREACH, + errno.ECONNRESET, + errno.EBADF, + errno.EPIPE + ) + + +class SocketWithId(connection.Socket): + def __init__(self, host, port): + super(SocketWithId, self).__init__(host, port) + self.last_client_id = None + + def maybe_connect(self): + # If we're going to establish a new connection, then reset the last + # client_id used on this connection. + if self.sock is None: + self.last_client_id = None + super(SocketWithId, self).maybe_connect() + + def send(self, pkt): + try: + self.sock.sendall(pkt) + except socket.error, e: + # If the socket is in a bad state, close it and allow it + # to re-connect on the next try + if e[0] in CONN_CLOSED_ERRORS: + self.close() + raise + + def recv(self, want_len): + try: + res = self.sock.recv(want_len) + # Assume the socket is closed if no data is + # returned on a blocking read. + if len(res) == 0 and want_len > 0: + self.close() + + return res + + except socket.error, e: + # If the socket is in a bad state, close it and allow it + # to re-connect on the next try + if e[0] in CONN_CLOSED_ERRORS: + self.close() + raise class RiakPbcTransport(RiakTransport): @@ -70,23 +142,32 @@ class RiakPbcTransport(RiakTransport): The RiakPbcTransport object holds a connection to the protocol buffers interface on the riak server. """ + + # We're using the new RiakTransport API + api = 2 + rw_names = { - 'default' : RIAKC_RW_DEFAULT, - 'all' : RIAKC_RW_ALL, - 'quorum' : RIAKC_RW_QUORUM, - 'one' : RIAKC_RW_ONE + 'default': RIAKC_RW_DEFAULT, + 'all': RIAKC_RW_ALL, + 'quorum': RIAKC_RW_QUORUM, + 'one': RIAKC_RW_ONE } - def __init__(self, host='127.0.0.1', port=8087, client_id=None): + + # The ConnectionManager class that this transport prefers. + default_cm = connection.cm_using(SocketWithId) + + def __init__(self, cm, client_id=None, max_attempts=1, **unused_options): """ Construct a new RiakPbcTransport object. - @param string host - Hostname or IP address (default '127.0.0.1') - @param int port - Port number (default 8087) """ + if riakclient_pb2 is None: + raise RiakError("this transport is not available (no protobuf)") + super(RiakPbcTransport, self).__init__() - self._host = host - self._port = port + + self._cm = cm self._client_id = client_id - self._sock = None + self._max_attempts = max_attempts def translate_rw_val(self, rw): val = self.rw_names.get(rw) @@ -95,16 +176,15 @@ def translate_rw_val(self, rw): return val def __copy__(self): - return RiakPbcTransport(self._host, self._port) + return RiakPbcTransport(self._cm, self._client_id) def ping(self): """ Ping the remote server @return boolean """ - self.maybe_connect() - self.send_msg_code(MSG_CODE_PING_REQ) - msg_code, msg = self.recv_msg() + # An expected response code of None implies "any response is valid". + msg_code, msg = self.send_msg_code(MSG_CODE_PING_REQ, None) if msg_code == MSG_CODE_PING_RESP: return 1 else: @@ -114,13 +194,9 @@ def get_client_id(self): """ Get the client id used by this connection """ - self.maybe_connect() - self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ) - msg_code, resp = self.recv_msg() - if msg_code == MSG_CODE_GET_CLIENT_ID_RESP: - return resp.client_id - else: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg_code(MSG_CODE_GET_CLIENT_ID_REQ, + MSG_CODE_GET_CLIENT_ID_RESP) + return resp.client_id def set_client_id(self, client_id): """ @@ -129,15 +205,23 @@ def set_client_id(self, client_id): req = riakclient_pb2.RpbSetClientIdReq() req.client_id = client_id - self.maybe_connect() - self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req) - msg_code, resp = self.recv_msg() - if msg_code == MSG_CODE_SET_CLIENT_ID_RESP: - return True - else: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_SET_CLIENT_ID_REQ, req, + MSG_CODE_SET_CLIENT_ID_RESP) + + # Using different client_id values across connections is a bad idea + # since you never know which connection you might use for a given + # API call. Setting the client_id manually (rather than as part of + # the transport construction) can be error-prone since the connection + # could drop and be reinstated using self._client_id. + # + # To minimize the potential impact of variant client_id values across + # connections, we'll store this new client_id and use it for all + # future connections. + self._client_id = client_id + + return True - def get(self, robj, r = None, vtag = None): + def get(self, robj, r=None, pr=None, vtag=None): """ Serialize get request and deserialize response """ @@ -148,22 +232,22 @@ def get(self, robj, r = None, vtag = None): req = riakclient_pb2.RpbGetReq() req.r = self.translate_rw_val(r) + req.pr = self.translate_rw_val(pr) req.bucket = bucket.get_name() req.key = robj.get_key() - self.maybe_connect() - self.send_msg(MSG_CODE_GET_REQ, req) - msg_code, resp = self.recv_msg() + # An expected response code of None implies "any response is valid". + msg_code, resp = self.send_msg(MSG_CODE_GET_REQ, req, None) if msg_code == MSG_CODE_GET_RESP: contents = [] for c in resp.content: contents.append(self.decode_content(c)) return resp.vclock, contents else: - return 0 + return None - def put(self, robj, w = None, dw = None, return_body = True): + def put(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): """ Serialize get request and deserialize response """ @@ -172,8 +256,12 @@ def put(self, robj, w = None, dw = None, return_body = True): req = riakclient_pb2.RpbPutReq() req.w = self.translate_rw_val(w) req.dw = self.translate_rw_val(dw) + req.pw = self.translate_rw_val(pw) + if return_body: req.return_body = 1 + if if_none_match: + req.if_none_match = 1 req.bucket = bucket.get_name() req.key = robj.get_key() @@ -183,18 +271,49 @@ def put(self, robj, w = None, dw = None, return_body = True): self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) - self.maybe_connect() - self.send_msg(MSG_CODE_PUT_REQ, req) - msg_code, resp = self.recv_msg() - if msg_code != MSG_CODE_PUT_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req, + MSG_CODE_PUT_RESP) if resp is not None: contents = [] for c in resp.content: contents.append(self.decode_content(c)) return resp.vclock, contents - def delete(self, robj, rw = None): + def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, if_none_match=False): + """Put a new object into the Riak store, returning its (new) key. + + If return_meta is False, then the vlock and metadata return values + will be None. + + @return (key, vclock, metadata) + """ + bucket = robj.get_bucket() + + req = riakclient_pb2.RpbPutReq() + req.w = self.translate_rw_val(w) + req.dw = self.translate_rw_val(dw) + req.pw = self.translate_rw_val(pw) + + if return_body: + req.return_body = 1 + if if_none_match: + req.if_none_match = 1 + + req.bucket = bucket.get_name() + + self.pbify_content(robj.get_metadata(), robj.get_encoded_data(), req.content) + + msg_code, resp = self.send_msg(MSG_CODE_PUT_REQ, req, + MSG_CODE_PUT_RESP) + if not resp: + raise RiakError("missing response object") + if len(resp.content) != 1: + raise RiakError("siblings were returned from object creation") + + metadata, content = self.decode_content(resp.content[0]) + return resp.key, resp.vclock, metadata + + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): """ Serialize get request and deserialize response """ @@ -202,15 +321,19 @@ def delete(self, robj, rw = None): req = riakclient_pb2.RpbDelReq() req.rw = self.translate_rw_val(rw) + req.r = self.translate_rw_val(r) + req.w = self.translate_rw_val(w) + req.dw = self.translate_rw_val(dw) + req.pr = self.translate_rw_val(pr) + req.pw = self.translate_rw_val(pw) + + # TODO: Set the vclock if present req.bucket = bucket.get_name() req.key = robj.get_key() - self.maybe_connect() - self.send_msg(MSG_CODE_DEL_REQ, req) - msg_code, resp = self.recv_msg() - if msg_code != MSG_CODE_DEL_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_DEL_REQ, req, + MSG_CODE_DEL_RESP) return self def get_keys(self, bucket): @@ -220,19 +343,12 @@ def get_keys(self, bucket): req = riakclient_pb2.RpbListKeysReq() req.bucket = bucket.get_name() - self.maybe_connect() - self.send_msg(MSG_CODE_LIST_KEYS_REQ, req) keys = [] - while True: - msg_code, resp = self.recv_msg() - if msg_code != MSG_CODE_LIST_KEYS_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) - + def _handle_response(resp): for key in resp.keys: keys.append(key) - - if resp.HasField("done") and resp.done: - break + self.send_msg_multi(MSG_CODE_LIST_KEYS_REQ, req, + MSG_CODE_LIST_KEYS_RESP, _handle_response) return keys @@ -240,11 +356,8 @@ def get_buckets(self): """ Serialize bucket listing request and deserialize response """ - self.maybe_connect() - self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ) - msg_code, resp = self.recv_msg() - if msg_code != MSG_CODE_LIST_BUCKETS_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg_code(MSG_CODE_LIST_BUCKETS_REQ, + MSG_CODE_LIST_BUCKETS_RESP) return resp.buckets def get_bucket_props(self, bucket): @@ -254,11 +367,8 @@ def get_bucket_props(self, bucket): req = riakclient_pb2.RpbGetBucketReq() req.bucket = bucket.get_name() - self.maybe_connect() - self.send_msg(MSG_CODE_GET_BUCKET_REQ, req) - msg_code, resp = self.recv_msg() - if msg_code != MSG_CODE_GET_BUCKET_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + msg_code, resp = self.send_msg(MSG_CODE_GET_BUCKET_REQ, req, + MSG_CODE_GET_BUCKET_RESP) props = {} if resp.props.HasField('n_val'): props['n_val'] = resp.props.n_val @@ -267,31 +377,27 @@ def get_bucket_props(self, bucket): return props - def set_bucket_props(self, bucket, props): """ Serialize set bucket property request and deserialize response """ req = riakclient_pb2.RpbSetBucketReq() req.bucket = bucket.get_name() - if not 'n_val' in props and not 'allow_mult' in props: return self + if not 'n_val' in props and not 'allow_mult' in props: + return self if 'n_val' in props: req.props.n_val = props['n_val'] if 'allow_mult' in props: req.props.allow_mult = props['allow_mult'] - self.maybe_connect() - self.send_msg(MSG_CODE_SET_BUCKET_REQ, req) - msg_code, resp = self.recv_msg() - if msg_code != MSG_CODE_SET_BUCKET_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) - + msg_code, resp = self.send_msg(MSG_CODE_SET_BUCKET_REQ, req, + MSG_CODE_SET_BUCKET_RESP) return self def mapred(self, inputs, query, timeout=None): # Construct the job, optionally set the timeout... - job = {'inputs':inputs, 'query':query} + job = {'inputs': inputs, 'query': query} if timeout is not None: job['timeout'] = timeout @@ -301,25 +407,18 @@ def mapred(self, inputs, query, timeout=None): req.request = content req.content_type = "application/json" - self.maybe_connect() - self.send_msg(MSG_CODE_MAPRED_REQ, req) - # dictionary of phase results - each content should be an encoded array # which is appended to the result for that phase. result = {} - while True: - msg_code, resp = self.recv_msg() - if msg_code != MSG_CODE_MAPRED_RESP: - raise RiakError("unexpected protocol buffer message code: %d"%msg_code) + def _handle_response(resp): if resp.HasField("phase") and resp.HasField("response"): content = json.loads(resp.response) if resp.phase in result: result[resp.phase] += content else: result[resp.phase] = content - - if resp.HasField("done") and resp.done: - break; + self.send_msg_multi(MSG_CODE_MAPRED_REQ, req, MSG_CODE_MAPRED_RESP, + _handle_response) # If a single result - return the same as the HTTP interface does # otherwise return all the phase information @@ -330,23 +429,10 @@ def mapred(self, inputs, query, timeout=None): else: return result - - def maybe_connect(self): - if self._sock is None: - self._sock = s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - try: - s.connect((self._host, self._port)) - except: - self._sock = None - raise - - if self._client_id: - self.set_client_id(self._client_id) - - def send_msg_code(self, msg_code): - pkt = struct.pack("!iB", 1, msg_code) - self._sock.send(pkt) + def send_msg_code(self, msg_code, expect): + with self._cm.withconn() as conn: + self.send_pkt(conn, struct.pack("!iB", 1, msg_code)) + return self.recv_msg(conn, expect) def encode_msg(self, msg_code, msg): str = msg.SerializeToString() @@ -354,15 +440,53 @@ def encode_msg(self, msg_code, msg): hdr = struct.pack("!iB", 1 + slen, msg_code) return hdr + str - def send_msg(self, msg_code, msg): - pkt = self.encode_msg(msg_code, msg) - sent_len = self._sock.send(pkt) - if sent_len != len(pkt): - raise RiakError("PB socket returned short write {0} - expected {1}". - format(sent_len, len(pkt))) - - def recv_msg(self): - self.recv_pkt() + def send_msg(self, msg_code, msg, expect): + with self._cm.withconn() as conn: + self.send_pkt(conn, self.encode_msg(msg_code, msg)) + if msg_code == MSG_CODE_SET_CLIENT_ID_REQ: + conn.last_client_id = self._client_id + return self.recv_msg(conn, expect) + + def send_msg_multi(self, msg_code, msg, expect, handler): + with self._cm.withconn() as conn: + self.send_pkt(conn, self.encode_msg(msg_code, msg)) + while True: + msg_code, resp = self.recv_msg(conn, expect) + handler(resp) + if resp.HasField("done") and resp.done: + break + + def send_pkt(self, conn, pkt): + attempt = 0 + e = None + for attempt in xrange(self._max_attempts): + e = None + try: + conn.maybe_connect() + + # If the last client_id used on this connection is different than our + # client_id, then set a new ID on the connection. + if conn.last_client_id != self._client_id: + req = riakclient_pb2.RpbSetClientIdReq() + req.client_id = self._client_id + conn.send(self.encode_msg(MSG_CODE_SET_CLIENT_ID_REQ, req)) + conn.last_client_id = self._client_id + self.recv_msg(conn, MSG_CODE_SET_CLIENT_ID_RESP) + + conn.send(pkt) + break + except socket.error, e: + # If this is some unknown socket error bail out + # instead of retrying + if e[0] not in CONN_CLOSED_ERRORS: + raise + + # Max attempts reached, raise whatever exception we are getting + if attempt + 1 == self._max_attempts and e is not None: + raise e + + def recv_msg(self, conn, expect): + self.recv_pkt(conn) msg_code, = struct.unpack("B", self._inbuf[:1]) if msg_code == MSG_CODE_ERROR_RESP: msg = riakclient_pb2.RpbErrorResp() @@ -398,27 +522,29 @@ def recv_msg(self): msg = riakclient_pb2.RpbMapRedResp() msg.ParseFromString(self._inbuf[1:]) else: - raise Exception("unknown msg code {0}".format(msg_code)) + raise Exception("unknown msg code %s" % msg_code) + if expect and msg_code != expect: + raise RiakError("unexpected protocol buffer message code: %d" + % msg_code) return msg_code, msg - - def recv_pkt(self): - nmsglen = self._sock.recv(4) + def recv_pkt(self, conn): + nmsglen = conn.recv(4) if len(nmsglen) != 4: - self._sock = None - raise RiakError("Socket returned short packet length {0} - expected 4". - format(nmsglen)) + raise RiakError("Socket returned short packet length %d - expected 4" + % len(nmsglen)) msglen, = struct.unpack('!i', nmsglen) self._inbuf_len = msglen self._inbuf = '' while len(self._inbuf) < msglen: want_len = min(8192, msglen - len(self._inbuf)) - recv_buf = self._sock.recv(want_len) - if not recv_buf: break + recv_buf = conn.recv(want_len) + if not recv_buf: + break self._inbuf += recv_buf if len(self._inbuf) != self._inbuf_len: - raise RiakError("Socket returned short packet {0} - expected {1}". - format(len(self._inbuf), self._inbuf_len)) + raise RiakError("Socket returned short packet %d - expected %d" + % (len(self._inbuf), self._inbuf_len)) def decode_contents(self, rpb_contents): contents = [] @@ -462,12 +588,18 @@ def decode_content(self, rpb_content): usermeta[usermd.key] = usermd.value if len(usermeta) > 0: metadata[MD_USERMETA] = usermeta + indexes = [] + for index in rpb_content.indexes: + rie = RiakIndexEntry(index.key, index.value) + indexes.append(rie) + if len(indexes) > 0: + metadata[MD_INDEX] = indexes return metadata, rpb_content.value - def pbify_content(self, metadata, data, rpb_content) : + def pbify_content(self, metadata, data, rpb_content): # Convert the broken out fields, building up # pbmetadata for any unknown ones - for k,v in metadata.iteritems(): + for k, v in metadata.iteritems(): if k == MD_CTYPE: rpb_content.content_type = v elif k == MD_CHARSET: @@ -479,6 +611,11 @@ def pbify_content(self, metadata, data, rpb_content) : pair = rpb_content.usermeta.add() pair.key = uk pair.value = uv + elif k == MD_INDEX: + for rie in v: + pair = rpb_content.indexes.add() + pair.key = rie.get_field() + pair.value = rie.get_value() elif k == MD_LINKS: for link in v: pb_link = rpb_content.links.add() @@ -487,119 +624,3 @@ def pbify_content(self, metadata, data, rpb_content) : pb_link.tag = link.get_tag() rpb_content.value = data -from Queue import Empty, Full, Queue -import contextlib -class RiakPbcCachedTransport(RiakTransport): - """Threadsafe pool of PBC connections, based on urllib3's pool [aka Queue]""" - def __init__(self, host='127.0.0.1', port=8087, client_id=None, maxsize=0, block=False, timeout=None): - self.host = host - self.port = port - self.client_id = client_id - self.block = block - self.timeout = timeout - - self.pool = Queue(maxsize) - # Fill the queue up so that doing get() on it will block properly (check Queue#get) - [self.pool.put(None) for _ in xrange(maxsize)] - - def _new_connection(self): - """New PBC connection""" - return RiakPbcTransport(self.host, self.port, self.client_id) - - def _get_connection(self): - connection = None - try: - connection = self.pool.get(block=self.block, timeout=self.timeout) - except Empty: - pass - return connection or self._new_connection() - - def _put_connection(self, connection): - try: - self.pool.put(connection, block=False) - except Full: - pass - - @contextlib.contextmanager - def _get_connection_from_pool(self): - """checkout conn, try operation, put conn back in pool""" - connection = self._get_connection() - try: - yield connection - finally: - self._put_connection(connection) - - def ping(self): - """ - Ping the remote server - @return boolean - """ - with self._get_connection_from_pool() as connection: - return connection.ping() - - def get(self, robj, r = None, vtag = None): - """ - Serialize get request and deserialize response - @return (vclock=None, [(metadata, value)]=None) - """ - with self._get_connection_from_pool() as connection: - return connection.get(robj, r, vtag) - - def put(self, robj, w = None, dw = None, return_body = True): - """ - Serialize put request and deserialize response - if 'content' - is true, retrieve the updated metadata/content - @return (vclock=None, [(metadata, value)]=None) - """ - with self._get_connection_from_pool() as connection: - return connection.put(robj, w, dw, return_body) - - def delete(self, robj, rw = None): - """ - Serialize delete request and deserialize response - @return true - """ - with self._get_connection_from_pool() as connection: - return connection.delete(robj, rw) - - def get_buckets(self): - """ - Serialize bucket listing request and deserialize response - """ - with self._get_connection_from_pool() as connection: - return connection.get_buckets() - - def get_bucket_props(self, bucket) : - """ - Serialize get bucket property request and deserialize response - @return dict() - """ - with self._get_connection_from_pool() as connection: - return connection.get_bucket_props(bucket) - - def set_bucket_props(self, bucket, props) : - """ - Serialize set bucket property request and deserialize response - bucket = bucket object - props = dictionary of properties - @return boolean - """ - with self._get_connection_from_pool() as connection: - return connection.set_bucket_props(bucket, props) - - def mapred(self, inputs, query, timeout = None) : - """ - Serialize map/reduce request - """ - with self._get_connection_from_pool() as connection: - return connection.mapred(inputs, query, timeout) - - def set_client_id(self, client_id): - """Mmm, this can turn ugly if you use different id for different objects in the pool""" - with self._get_connection_from_pool() as connection: - return connection.set_client_id(client_id) - - def get_client_id(self): - """see set_client_id notes, you can do wrong with this""" - with self._get_connection_from_pool() as connection: - return connection.get_client_id() diff --git a/riak/transports/riakclient.proto b/riak/transports/riakclient.proto index 6acd0842..ac82cb10 100644 --- a/riak/transports/riakclient.proto +++ b/riak/transports/riakclient.proto @@ -120,12 +120,19 @@ message RpbGetReq { required bytes bucket = 1; required bytes key = 2; optional uint32 r = 3; + optional uint32 pr = 4; + optional bool basic_quorum = 5; + optional bool notfound_ok = 6; + optional bytes if_modified = 7; // fail if the supplied vclock does not match + optional bool head = 8; // return everything but the value + optional bool deletedvclock = 9; // return the tombstone's vclock, if applicable } // Get Response - if the record was not found there will be no content/vclock message RpbGetResp { repeated RpbContent content = 1; optional bytes vclock = 2; // the opaque vector clock for the object + optional bool unchanged = 3; } @@ -133,18 +140,23 @@ message RpbGetResp { // the key will be returned. message RpbPutReq { required bytes bucket = 1; - required bytes key = 2; + optional bytes key = 2; optional bytes vclock = 3; required RpbContent content = 4; optional uint32 w = 5; optional uint32 dw = 6; optional bool return_body = 7; + optional uint32 pw = 8; + optional bool if_not_modified = 9; + optional bool if_none_match = 10; + optional bool return_head = 11; } -// Put response - same as get response +// Put response - same as get response with optional key if one was generated message RpbPutResp { repeated RpbContent content = 1; optional bytes vclock = 2; // the opaque vector clock for the object + optional bytes key = 3; // the key generated, if any } @@ -153,6 +165,12 @@ message RpbDelReq { required bytes bucket = 1; required bytes key = 2; optional uint32 rw = 3; + optional bytes vclock = 4; + optional uint32 r = 5; + optional uint32 w = 6; + optional uint32 pr = 7; + optional uint32 pw = 8; + optional uint32 dw = 9; } // Delete response - not defined, will return a RpbDelResp on success or RpbErrorResp on failure @@ -224,6 +242,7 @@ message RpbContent { optional uint32 last_mod = 7; optional uint32 last_mod_usecs = 8; repeated RpbPair usermeta = 9; // user metadata stored with the object + repeated RpbPair indexes = 10; // user metadata stored with the object } // Key/value pair - used for user metadata diff --git a/riak/transports/riakclient_pb2.py b/riak/transports/riakclient_pb2.py index 68f9b2e0..e607fe72 100644 --- a/riak/transports/riakclient_pb2.py +++ b/riak/transports/riakclient_pb2.py @@ -7,10 +7,11 @@ # @@protoc_insertion_point(imports) + DESCRIPTOR = descriptor.FileDescriptor( name='riakclient.proto', package='', - serialized_pb='\n\x10riakclient.proto\"/\n\x0cRpbErrorResp\x12\x0e\n\x06\x65rrmsg\x18\x01 \x02(\x0c\x12\x0f\n\x07\x65rrcode\x18\x02 \x02(\r\"\'\n\x12RpbGetClientIdResp\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"&\n\x11RpbSetClientIdReq\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"<\n\x14RpbGetServerInfoResp\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12\x16\n\x0eserver_version\x18\x02 \x01(\x0c\"3\n\tRpbGetReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\t\n\x01r\x18\x03 \x01(\r\":\n\nRpbGetResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\"\x82\x01\n\tRpbPutReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\x0e\n\x06vclock\x18\x03 \x01(\x0c\x12\x1c\n\x07\x63ontent\x18\x04 \x02(\x0b\x32\x0b.RpbContent\x12\t\n\x01w\x18\x05 \x01(\r\x12\n\n\x02\x64w\x18\x06 \x01(\r\x12\x13\n\x0breturn_body\x18\x07 \x01(\x08\":\n\nRpbPutResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\"4\n\tRpbDelReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\n\n\x02rw\x18\x03 \x01(\r\"%\n\x12RpbListBucketsResp\x12\x0f\n\x07\x62uckets\x18\x01 \x03(\x0c\" \n\x0eRpbListKeysReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"-\n\x0fRpbListKeysResp\x12\x0c\n\x04keys\x18\x01 \x03(\x0c\x12\x0c\n\x04\x64one\x18\x02 \x01(\x08\"!\n\x0fRpbGetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"2\n\x10RpbGetBucketResp\x12\x1e\n\x05props\x18\x01 \x02(\x0b\x32\x0f.RpbBucketProps\"A\n\x0fRpbSetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x1e\n\x05props\x18\x02 \x02(\x0b\x32\x0f.RpbBucketProps\"5\n\x0cRpbMapRedReq\x12\x0f\n\x07request\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x02(\x0c\">\n\rRpbMapRedResp\x12\r\n\x05phase\x18\x01 \x01(\r\x12\x10\n\x08response\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\"\xc9\x01\n\nRpbContent\x12\r\n\x05value\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x01(\x0c\x12\x0f\n\x07\x63harset\x18\x03 \x01(\x0c\x12\x18\n\x10\x63ontent_encoding\x18\x04 \x01(\x0c\x12\x0c\n\x04vtag\x18\x05 \x01(\x0c\x12\x17\n\x05links\x18\x06 \x03(\x0b\x32\x08.RpbLink\x12\x10\n\x08last_mod\x18\x07 \x01(\r\x12\x16\n\x0elast_mod_usecs\x18\x08 \x01(\r\x12\x1a\n\x08usermeta\x18\t \x03(\x0b\x32\x08.RpbPair\"%\n\x07RpbPair\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c\"3\n\x07RpbLink\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0b\n\x03tag\x18\x03 \x01(\x0c\"3\n\x0eRpbBucketProps\x12\r\n\x05n_val\x18\x01 \x01(\r\x12\x12\n\nallow_mult\x18\x02 \x01(\x08') + serialized_pb='\n\x10riakclient.proto\"/\n\x0cRpbErrorResp\x12\x0e\n\x06\x65rrmsg\x18\x01 \x02(\x0c\x12\x0f\n\x07\x65rrcode\x18\x02 \x02(\r\"\'\n\x12RpbGetClientIdResp\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"&\n\x11RpbSetClientIdReq\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"<\n\x14RpbGetServerInfoResp\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12\x16\n\x0eserver_version\x18\x02 \x01(\x0c\"\xa4\x01\n\tRpbGetReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\t\n\x01r\x18\x03 \x01(\r\x12\n\n\x02pr\x18\x04 \x01(\r\x12\x14\n\x0c\x62\x61sic_quorum\x18\x05 \x01(\x08\x12\x13\n\x0bnotfound_ok\x18\x06 \x01(\x08\x12\x13\n\x0bif_modified\x18\x07 \x01(\x0c\x12\x0c\n\x04head\x18\x08 \x01(\x08\x12\x15\n\rdeletedvclock\x18\t \x01(\x08\"M\n\nRpbGetResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\x12\x11\n\tunchanged\x18\x03 \x01(\x08\"\xd3\x01\n\tRpbPutReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0e\n\x06vclock\x18\x03 \x01(\x0c\x12\x1c\n\x07\x63ontent\x18\x04 \x02(\x0b\x32\x0b.RpbContent\x12\t\n\x01w\x18\x05 \x01(\r\x12\n\n\x02\x64w\x18\x06 \x01(\r\x12\x13\n\x0breturn_body\x18\x07 \x01(\x08\x12\n\n\x02pw\x18\x08 \x01(\r\x12\x17\n\x0fif_not_modified\x18\t \x01(\x08\x12\x15\n\rif_none_match\x18\n \x01(\x08\x12\x13\n\x0breturn_head\x18\x0b \x01(\x08\"G\n\nRpbPutResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\x12\x0b\n\x03key\x18\x03 \x01(\x0c\"~\n\tRpbDelReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\n\n\x02rw\x18\x03 \x01(\r\x12\x0e\n\x06vclock\x18\x04 \x01(\x0c\x12\t\n\x01r\x18\x05 \x01(\r\x12\t\n\x01w\x18\x06 \x01(\r\x12\n\n\x02pr\x18\x07 \x01(\r\x12\n\n\x02pw\x18\x08 \x01(\r\x12\n\n\x02\x64w\x18\t \x01(\r\"%\n\x12RpbListBucketsResp\x12\x0f\n\x07\x62uckets\x18\x01 \x03(\x0c\" \n\x0eRpbListKeysReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"-\n\x0fRpbListKeysResp\x12\x0c\n\x04keys\x18\x01 \x03(\x0c\x12\x0c\n\x04\x64one\x18\x02 \x01(\x08\"!\n\x0fRpbGetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\"2\n\x10RpbGetBucketResp\x12\x1e\n\x05props\x18\x01 \x02(\x0b\x32\x0f.RpbBucketProps\"A\n\x0fRpbSetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x1e\n\x05props\x18\x02 \x02(\x0b\x32\x0f.RpbBucketProps\"5\n\x0cRpbMapRedReq\x12\x0f\n\x07request\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x02(\x0c\">\n\rRpbMapRedResp\x12\r\n\x05phase\x18\x01 \x01(\r\x12\x10\n\x08response\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\"\xe4\x01\n\nRpbContent\x12\r\n\x05value\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x01(\x0c\x12\x0f\n\x07\x63harset\x18\x03 \x01(\x0c\x12\x18\n\x10\x63ontent_encoding\x18\x04 \x01(\x0c\x12\x0c\n\x04vtag\x18\x05 \x01(\x0c\x12\x17\n\x05links\x18\x06 \x03(\x0b\x32\x08.RpbLink\x12\x10\n\x08last_mod\x18\x07 \x01(\r\x12\x16\n\x0elast_mod_usecs\x18\x08 \x01(\r\x12\x1a\n\x08usermeta\x18\t \x03(\x0b\x32\x08.RpbPair\x12\x19\n\x07indexes\x18\n \x03(\x0b\x32\x08.RpbPair\"%\n\x07RpbPair\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c\"3\n\x07RpbLink\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0b\n\x03tag\x18\x03 \x01(\x0c\"3\n\x0eRpbBucketProps\x12\r\n\x05n_val\x18\x01 \x01(\r\x12\x12\n\nallow_mult\x18\x02 \x01(\x08') @@ -169,6 +170,48 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='pr', full_name='RpbGetReq.pr', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='basic_quorum', full_name='RpbGetReq.basic_quorum', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='notfound_ok', full_name='RpbGetReq.notfound_ok', index=5, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='if_modified', full_name='RpbGetReq.if_modified', index=6, + number=7, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='head', full_name='RpbGetReq.head', index=7, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='deletedvclock', full_name='RpbGetReq.deletedvclock', index=8, + number=9, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -178,8 +221,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=212, - serialized_end=263, + serialized_start=213, + serialized_end=377, ) @@ -204,6 +247,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='unchanged', full_name='RpbGetResp.unchanged', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -213,8 +263,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=265, - serialized_end=323, + serialized_start=379, + serialized_end=456, ) @@ -234,7 +284,7 @@ options=None), descriptor.FieldDescriptor( name='key', full_name='RpbPutReq.key', index=1, - number=2, type=12, cpp_type=9, label=2, + number=2, type=12, cpp_type=9, label=1, has_default_value=False, default_value="", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -274,6 +324,34 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='pw', full_name='RpbPutReq.pw', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='if_not_modified', full_name='RpbPutReq.if_not_modified', index=8, + number=9, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='if_none_match', full_name='RpbPutReq.if_none_match', index=9, + number=10, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='return_head', full_name='RpbPutReq.return_head', index=10, + number=11, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -283,8 +361,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=326, - serialized_end=456, + serialized_start=459, + serialized_end=670, ) @@ -309,6 +387,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='key', full_name='RpbPutResp.key', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -318,8 +403,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=458, - serialized_end=516, + serialized_start=672, + serialized_end=743, ) @@ -351,6 +436,48 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='vclock', full_name='RpbDelReq.vclock', index=3, + number=4, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='r', full_name='RpbDelReq.r', index=4, + number=5, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='w', full_name='RpbDelReq.w', index=5, + number=6, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='pr', full_name='RpbDelReq.pr', index=6, + number=7, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='pw', full_name='RpbDelReq.pw', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='dw', full_name='RpbDelReq.dw', index=8, + number=9, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -360,8 +487,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=518, - serialized_end=570, + serialized_start=745, + serialized_end=871, ) @@ -388,8 +515,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=572, - serialized_end=609, + serialized_start=873, + serialized_end=910, ) @@ -416,8 +543,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=611, - serialized_end=643, + serialized_start=912, + serialized_end=944, ) @@ -451,8 +578,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=645, - serialized_end=690, + serialized_start=946, + serialized_end=991, ) @@ -479,8 +606,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=692, - serialized_end=725, + serialized_start=993, + serialized_end=1026, ) @@ -507,8 +634,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=727, - serialized_end=777, + serialized_start=1028, + serialized_end=1078, ) @@ -542,8 +669,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=779, - serialized_end=844, + serialized_start=1080, + serialized_end=1145, ) @@ -577,8 +704,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=846, - serialized_end=899, + serialized_start=1147, + serialized_end=1200, ) @@ -619,8 +746,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=901, - serialized_end=963, + serialized_start=1202, + serialized_end=1264, ) @@ -694,6 +821,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='indexes', full_name='RpbContent.indexes', index=9, + number=10, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -703,8 +837,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=966, - serialized_end=1167, + serialized_start=1267, + serialized_end=1495, ) @@ -738,8 +872,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=1169, - serialized_end=1206, + serialized_start=1497, + serialized_end=1534, ) @@ -780,8 +914,8 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=1208, - serialized_end=1259, + serialized_start=1536, + serialized_end=1587, ) @@ -815,11 +949,10 @@ options=None, is_extendable=False, extension_ranges=[], - serialized_start=1261, - serialized_end=1312, + serialized_start=1589, + serialized_end=1640, ) - _RPBGETRESP.fields_by_name['content'].message_type = _RPBCONTENT _RPBPUTREQ.fields_by_name['content'].message_type = _RPBCONTENT _RPBPUTRESP.fields_by_name['content'].message_type = _RPBCONTENT @@ -827,6 +960,28 @@ _RPBSETBUCKETREQ.fields_by_name['props'].message_type = _RPBBUCKETPROPS _RPBCONTENT.fields_by_name['links'].message_type = _RPBLINK _RPBCONTENT.fields_by_name['usermeta'].message_type = _RPBPAIR +_RPBCONTENT.fields_by_name['indexes'].message_type = _RPBPAIR +DESCRIPTOR.message_types_by_name['RpbErrorResp'] = _RPBERRORRESP +DESCRIPTOR.message_types_by_name['RpbGetClientIdResp'] = _RPBGETCLIENTIDRESP +DESCRIPTOR.message_types_by_name['RpbSetClientIdReq'] = _RPBSETCLIENTIDREQ +DESCRIPTOR.message_types_by_name['RpbGetServerInfoResp'] = _RPBGETSERVERINFORESP +DESCRIPTOR.message_types_by_name['RpbGetReq'] = _RPBGETREQ +DESCRIPTOR.message_types_by_name['RpbGetResp'] = _RPBGETRESP +DESCRIPTOR.message_types_by_name['RpbPutReq'] = _RPBPUTREQ +DESCRIPTOR.message_types_by_name['RpbPutResp'] = _RPBPUTRESP +DESCRIPTOR.message_types_by_name['RpbDelReq'] = _RPBDELREQ +DESCRIPTOR.message_types_by_name['RpbListBucketsResp'] = _RPBLISTBUCKETSRESP +DESCRIPTOR.message_types_by_name['RpbListKeysReq'] = _RPBLISTKEYSREQ +DESCRIPTOR.message_types_by_name['RpbListKeysResp'] = _RPBLISTKEYSRESP +DESCRIPTOR.message_types_by_name['RpbGetBucketReq'] = _RPBGETBUCKETREQ +DESCRIPTOR.message_types_by_name['RpbGetBucketResp'] = _RPBGETBUCKETRESP +DESCRIPTOR.message_types_by_name['RpbSetBucketReq'] = _RPBSETBUCKETREQ +DESCRIPTOR.message_types_by_name['RpbMapRedReq'] = _RPBMAPREDREQ +DESCRIPTOR.message_types_by_name['RpbMapRedResp'] = _RPBMAPREDRESP +DESCRIPTOR.message_types_by_name['RpbContent'] = _RPBCONTENT +DESCRIPTOR.message_types_by_name['RpbPair'] = _RPBPAIR +DESCRIPTOR.message_types_by_name['RpbLink'] = _RPBLINK +DESCRIPTOR.message_types_by_name['RpbBucketProps'] = _RPBBUCKETPROPS class RpbErrorResp(message.Message): __metaclass__ = reflection.GeneratedProtocolMessageType diff --git a/riak/transports/transport.py b/riak/transports/transport.py index b8245f0e..4454f48d 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -29,13 +29,19 @@ class RiakTransport(object): Class to encapsulate transport details """ + # Subclasses should specify their API level. + # * missing or 1: the API used up and through 1.3.x. + # * 2: the API introduced with 1.4.x + # + # api = 2 + @classmethod def make_random_client_id(self): """ Returns a random client identifier """ return 'py_%s' % base64.b64encode( - str(random.randint(1, 1073741824))) + str(random.randint(1, 0x40000000))) @classmethod def make_fixed_client_id(self): @@ -69,6 +75,16 @@ def put(self, robj, w = None, dw = None, return_body = True): """ raise RiakError("not implemented") + def put_new(self, robj, w=None, dw=None, return_meta=True): + """Put a new object into the Riak store, returning its (new) key. + + If return_meta is False, then the vlock and metadata return values + will be None. + + @return (key, vclock, metadata) + """ + raise RiakError("not implemented") + def delete(self, robj, rw = None): """ Serialize delete request and deserialize response @@ -134,9 +150,9 @@ def get_file(self, key): """ raise RiakError("luwak not supported by this transport.") + def delete_file(self, key): """ Delete an object in luwak. key = the object's key """ - def delete_file(self, key): raise RiakError("luwak not supported by this transport.") diff --git a/riak/util.py b/riak/util.py index c2e154a0..f0f8958d 100644 --- a/riak/util.py +++ b/riak/util.py @@ -1,8 +1,14 @@ -import collections +import warnings + +try: + from collections import Mapping +except ImportError: + # compatibility with Python 2.5 + Mapping = dict def quacks_like_dict(object): """Check if object is dict-like""" - return isinstance(object, collections.Mapping) + return isinstance(object, Mapping) def deep_merge(a, b): """Merge two deep dicts non-destructively @@ -31,3 +37,6 @@ def deep_merge(a, b): current_dst[key] = current_src[key] return dst + +def deprecated(message, stacklevel=3): + warnings.warn(message, DeprecationWarning, stacklevel=stacklevel) diff --git a/setup.py b/setup.py index 7814cde2..50980edc 100755 --- a/setup.py +++ b/setup.py @@ -2,9 +2,9 @@ import glob import os import subprocess +import platform from setuptools import setup, find_packages - def make_docs(): if not os.path.exists('docs'): os.mkdir('docs') @@ -16,21 +16,32 @@ def make_pb(): subprocess.call(['protoc', '--python_out=.', './riak/transports/riakclient.proto']) if __name__ == "__main__": + install_requires = {'protobuf': ['>= 2.4.0', '< 2.5.0'] } + tests_require = [] + if platform.python_version() < '2.7': + tests_require.append("unittest2") + setup( name='riak', - version='1.3.0', + version='1.4.0', packages = find_packages(), - install_requires = ['protobuf>=2.3.0', 'urllib3>=0.4.0'], - dependency_links = ["http://downloads.basho.com/support"], + install_requires = install_requires, + tests_require = tests_require, package_data = { - '' : ['*.proto'] + '' : ['*.proto'], + 'riak' : ['erl_src/*'] }, description='Python client for Riak', zip_safe=True, + include_package_data=True, license='Apache 2', platforms='Platform Independent', author='Basho Technologies', - author_email='riak@basho.com', + author_email='clients@basho.com', test_suite='riak.tests.suite', - url='https://github.com/basho/riak-python-client' + url='https://github.com/basho/riak-python-client', + classifiers = ['License :: OSI Approved :: Apache Software License', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Topic :: Database'] )