diff --git a/.gitignore b/.gitignore index 34e7a5bb..8d2441d7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,20 @@ -*.pyc - -docs/_build - -.*.swp -.coverage - +*~ +#*# +_build/ build/ +.coverage dist/ -riak.egg-info/ +docsrc/doctrees/ *.egg - -#*# -*~ +.eggs/ +envs/ +.idea/ +py-build/ +*.pyc +__pycache__/ +.python-version +README.rst +riak-*/ +riak.egg-info/ +.*.swp +.tox/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..510fba6e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,10 @@ +[submodule "riak_pb"] + path = riak_pb + url = git://github.com/basho/riak_pb.git +[submodule "tools"] + path = tools + url = git://github.com/basho/riak-client-tools.git +[submodule "docs"] + path = docs + url = https://github.com/basho/riak-python-client.git + branch = gh-pages diff --git a/.runner b/.runner new file mode 100755 index 00000000..91b20b5c --- /dev/null +++ b/.runner @@ -0,0 +1,162 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset + +have_tox='false' +if hash tox 2>/dev/null +then + echo '[INFO] tox command present, will use that to run tests' + have_tox='true' +fi + +have_py2='false' +if hash python2 2>/dev/null +then + have_py2='true' +fi + +have_py3='false' +if hash python3 2>/dev/null +then + have_py3='true' +fi + +have_riak_admin='false' +if hash riak-admin 2>/dev/null +then + have_riak_admin='true' + $riak_admin='riak-admin' +else + set +o nounset + + if [[ -x $RIAK_ADMIN ]] + then + have_riak_admin='true' + riak_admin="$RIAK_ADMIN" + elif [[ -x $RIAK_DIR/bin/riak-admin ]] + then + have_riak_admin='true' + riak_admin="$RIAK_DIR/bin/riak-admin" + fi + + set -o nounset +fi + +function lint +{ + if ! hash flake8 2>/dev/null + then + pip install --upgrade flake8 + fi + flake8 --exclude=riak/pb riak *.py +} + +function run_tests +{ + local protocol="${1:-pbc}" + export RIAK_TEST_PROTOCOL="$protocol" + if [[ $have_tox == 'true' ]] + then + tox + else + if [[ $have_py2 == 'true' ]] + then + python2 setup.py test + fi + if [[ $have_py3 == 'true' ]] + then + python3 setup.py test + fi + fi +} + +function run_tests_each_protocol +{ + for protocol in pbc http + do + run_tests "$protocol" + done +} + +function export_host_environment_vars +{ + local riak_test_host="${RIAK_TEST_HOST:-localhost}" + local -i riak_test_pb_port="${RIAK_TEST_PB_PORT:-8087}" + local -i riak_test_http_port="${RIAK_TEST_HTTP_PORT:-8098}" + export RIAK_TEST_HOST="$riak_test_host" + export RIAK_TEST_PB_PORT="$riak_test_pb_port" + export RIAK_TEST_HTTP_PORT="$riak_test_http_port" +} + +function export_test_environment_vars +{ + export RUN_BTYPES=1 + export RUN_CLIENT=1 + export RUN_DATATYPES=1 + export RUN_INDEXES=1 + export RUN_KV=1 + export RUN_MAPREDUCE=1 + export RUN_RESOLVE=1 + export RUN_TIMESERIES=1 + export RUN_YZ=1 +} + +function unexport_test_environment_vars +{ + export RUN_BTYPES=0 + export RUN_CLIENT=0 + export RUN_DATATYPES=0 + export RUN_INDEXES=0 + export RUN_KV=0 + export RUN_MAPREDUCE=0 + export RUN_RESOLVE=0 + export RUN_TIMESERIES=0 + export RUN_YZ=0 +} + +function security_test +{ + if [[ $have_riak_admin == 'true' ]] + then + export_host_environment_vars + unexport_test_environment_vars + export RUN_SECURITY=1 + $riak_admin security enable + run_tests 'pbc' + else + echo '[ERROR] riak-admin must be in PATH, RIAK_ADMIN var set to path, or RIAK_DIR set.' 1>&2 + exit 1 + fi +} + +function integration_test +{ + export_host_environment_vars + export_test_environment_vars + run_tests_each_protocol +} + +function timeseries_test +{ + unexport_test_environment_vars + export RUN_TIMESERIES=1 + run_tests_each_protocol +} + +arg="${1:-lint}" +case "$arg" in + 'lint') + lint;; + 'unit-test') + run_tests;; + 'integration-test') + integration_test;; + 'security-test') + security_test;; + 'timeseries-test') + timeseries_test;; + *) + echo "[ERROR] unknown argument: '$arg'" 1>&2 + exit 1;; +esac diff --git a/.travis.sh b/.travis.sh new file mode 100755 index 00000000..739c66cd --- /dev/null +++ b/.travis.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -o errexit + +flake8 --ignore E123,E126,E226,E722,E741 --exclude=riak/pb riak *.py + +sudo riak-admin security disable + +python setup.py test + +sudo riak-admin security enable + +if [[ $RIAK_TEST_PROTOCOL == 'pbc' ]] +then + export RUN_SECURITY=1 + python setup.py test --test-suite riak.tests.test_security +else + echo '[INFO]: security tests run on PB protocol only' +fi diff --git a/.travis.yml b/.travis.yml index a47837e0..7c46a5cd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,39 @@ +sudo: required +dist: trusty language: python python: - - "2.6" - - "2.7" + - '2.7' + - '3.6' + - nightly +addons: + hosts: + - riak-test install: - - ./setup.py develop - - ./setup.py easy_install protobuf -script: ./setup.py test -before_script: sudo /usr/sbin/search-cmd install searchbucket + - pip install --upgrade pip setuptools flake8 +before_script: + - jdk_switcher use oraclejdk8 + - sudo ./tools/travis-ci/riak-install -d "$RIAK_DOWNLOAD_URL" + - sudo ./tools/setup-riak -s +env: + matrix: + - RIAK_TEST_PROTOCOL=pbc RIAK_DOWNLOAD_URL=http://s3.amazonaws.com/downloads.basho.com/riak/2.0/2.0.7/ubuntu/trusty/riak_2.0.7-1_amd64.deb + - RIAK_TEST_PROTOCOL=http RIAK_DOWNLOAD_URL=http://s3.amazonaws.com/downloads.basho.com/riak/2.0/2.0.7/ubuntu/trusty/riak_2.0.7-1_amd64.deb + - RIAK_TEST_PROTOCOL=pbc RIAK_DOWNLOAD_URL=http://s3.amazonaws.com/downloads.basho.com/riak/2.2/2.2.0/ubuntu/trusty/riak_2.2.0-1_amd64.deb + - RIAK_TEST_PROTOCOL=http RIAK_DOWNLOAD_URL=http://s3.amazonaws.com/downloads.basho.com/riak/2.2/2.2.0/ubuntu/trusty/riak_2.2.0-1_amd64.deb + global: + - RIAK_TEST_PB_PORT=8087 + - RIAK_TEST_HTTP_PORT=8098 + - RUN_BTYPES=1 + - RUN_CLIENT=1 + - RUN_MAPREDUCE=1 + - RUN_KV=1 + - RUN_RESOLVE=1 + - RUN_YZ=1 + - RUN_DATATYPES=1 + - RUN_INDEXES=1 + - RUN_SECURITY=0 +script: + - ./.travis.sh notifications: - email: clients@basho.com -services: - - riak + slack: + secure: kU1XcvTAliCWKuYpMWEMbD4qkbmlnWGLAIKbBQjtIh5ZRzISgjdUFzGcC31eHoQFv12LQdp5KAFj0Y1FyEvLxi0W8VeWKpsBGc06ntuECaN9MNHRBzKKclrTMGTfpBWZ5IO17XSUu2lKaNz6GDGRkiZA+sxYAVPfZSXY3u86IuY= diff --git a/MANIFEST.in b/MANIFEST.in index 6f864d48..ddf59c00 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,8 @@ include docs/* include riak/erl_src/* -include THANKS +include README.md include README.rst include LICENSE -include RELEASE_NOTES.md +include RELNOTES.md +include version.py +include commands.py diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..166e4007 --- /dev/null +++ b/Makefile @@ -0,0 +1,130 @@ +unexport LANG +unexport LC_ADDRESS +unexport LC_COLLATE +unexport LC_CTYPE +unexport LC_IDENTIFICATION +unexport LC_MEASUREMENT +unexport LC_MESSAGES +unexport LC_MONETARY +unexport LC_NAME +unexport LC_NUMERIC +unexport LC_PAPER +unexport LC_TELEPHONE +unexport LC_TIME + +PANDOC_VERSION := $(shell pandoc --version) +PROTOC_VERSION := $(shell protoc --version) + +PROJDIR := $(realpath $(CURDIR)) +DOCSRC := $(PROJDIR)/docsrc +DOCTREES := $(DOCSRC)/doctrees +DOCSDIR := $(PROJDIR)/docs + +PYPI_REPOSITORY ?= pypi + +all: lint test + +.PHONY: lint +lint: + $(PROJDIR)/.runner lint + +.PHONY: docs +docs: + sphinx-build -b html -d $(DOCTREES) $(DOCSRC) $(DOCSDIR) + @echo "The HTML pages are in $(DOCSDIR)" + +.PHONY: pb_clean +pb_clean: + @echo "==> Python (clean)" + @rm -rf riak/pb/*_pb2.py riak/pb/*.pyc riak/pb/__pycache__ __pycache__ py-build + +.PHONY: pb_compile +pb_compile: pb_clean +ifeq ($(PROTOC_VERSION),) + $(error The protoc command is required to parse proto files) +endif +ifneq ($(PROTOC_VERSION),libprotoc 2.5.0) + $(error protoc must be version 2.5.0) +endif + @echo "==> Python (compile)" + @protoc -Iriak_pb/src --python_out=riak/pb riak_pb/src/*.proto + @python setup.py build_messages + +.PHONY: test_sdist +test_sdist: + @python setup.py sdist + +.PHONY: release_sdist +release_sdist: +ifeq ($(VERSION),) + $(error VERSION must be set to build a release and deploy this package) +endif +ifeq ($(PANDOC_VERSION),) + $(error The pandoc command is required to correctly convert README.md to rst format) +endif +ifeq ($(RELEASE_GPG_KEYNAME),) + $(error RELEASE_GPG_KEYNAME must be set to build a release and deploy this package) +endif +ifeq ("$(wildcard $(PROJDIR)/.python-version)","") + $(error expected $(PROJDIR)/.python-version to exist. Run $(PROJDIR)/build/pyenv-setup) +endif + @python -c 'import pypandoc' + @echo "==> Python tagging version $(VERSION)" + @$(PROJDIR)/build/publish $(VERSION) validate + @git tag --sign -a "$(VERSION)" -m "riak-python-client $(VERSION)" --local-user "$(RELEASE_GPG_KEYNAME)" + @git push --tags + @echo "==> pypi repository: $(PYPI_REPOSITORY)" + @echo "==> Python (sdist)" + @python setup.py sdist upload --repository $(PYPI_REPOSITORY) --show-response --sign --identity $(RELEASE_GPG_KEYNAME) + @$(PROJDIR)/build/publish $(VERSION) + +.PHONY: release +release: release_sdist +ifeq ($(RELEASE_GPG_KEYNAME),) + $(error RELEASE_GPG_KEYNAME must be set to build a release and deploy this package) +endif +ifeq ("$(wildcard $(PROJDIR)/.python-version)","") + $(error expected $(PROJDIR)/.python-version to exist. Run $(PROJDIR)/build/pyenv-setup) +endif + @echo "==> pypi repository: $(PYPI_REPOSITORY)" + @echo "==> Python 2.7 (bdist_egg)" + @python2.7 setup.py build --build-base=py-build/2.7 bdist_egg upload --repository $(PYPI_REPOSITORY) --show-response --sign --identity $(RELEASE_GPG_KEYNAME) + @echo "==> Python 3.3 (bdist_egg)" + @python3.3 setup.py build --build-base=py-build/3.3 bdist_egg upload --repository $(PYPI_REPOSITORY) --show-response --sign --identity $(RELEASE_GPG_KEYNAME) + @echo "==> Python 3.4 (bdist_egg)" + @python3.4 setup.py build --build-base=py-build/3.4 bdist_egg upload --repository $(PYPI_REPOSITORY) --show-response --sign --identity $(RELEASE_GPG_KEYNAME) + @echo "==> Python 3.5 (bdist_egg)" + @python3.5 setup.py build --build-base=py-build/3.5 bdist_egg upload --repository $(PYPI_REPOSITORY) --show-response --sign --identity $(RELEASE_GPG_KEYNAME) + +.PHONY: unit-test +unit-test: + @$(PROJDIR)/.runner unit-test + +.PHONY: integration-test +integration-test: + @$(PROJDIR)/.runner integration-test + +.PHONY: security-test +security-test: + @$(PROJDIR)/.runner security-test + +.PHONY: timeseries-test +timeseries-test: + @$(PROJDIR)/.runner timeseries-test + +.PHONY: test +test: integration-test + +.PHONY: help +help: + @echo '' + @echo ' Targets: + @echo ' ------------------------------------------------------------' + @echo ' lint - Run linter (flake8) ' + @echo ' test - Run all tests ' + @echo ' unit-test - Run unit tests ' + @echo ' integration-test - Run integration tests ' + @echo ' security-test - Run integration tests (security enabled) ' + @echo ' timeseries-test - Run timeseries integration tests ' + @echo ' ------------------------------------------------------------' + @echo '' diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..37c556ce --- /dev/null +++ b/NOTICE @@ -0,0 +1,2 @@ +Riak Python Client +Copyright 2010-present Basho Technologies, Inc. diff --git a/README.md b/README.md new file mode 100644 index 00000000..9a15864d --- /dev/null +++ b/README.md @@ -0,0 +1,149 @@ +# Python Client for Riak + +## Build Status + +[![Build Status](https://travis-ci.org/basho/riak-python-client.svg?branch=master)](https://travis-ci.org/basho/riak-python-client) + +## Documentation + +[Documentation for the Riak Python Client Library](http://basho.github.io/riak-python-client/index.html) is available [here](http://basho.github.io/riak-python-client/index.html). + +Documentation for Riak is available [here](http://docs.basho.com/riak/latest). + +## Repository Cloning + +*NOTE*: please clone this repository using the `--recursive` argument to `git clone` or follow the clone with `git submodule update --init`. This repository uses two submodules. + +# Installation + +The recommended versions of Python for use with this client are Python `2.7.8` (or greater, `2.7.11` as of `2016-06-21`), `3.3.x`, `3.4.x` and `3.5.x`. The latest version from each series should be preferred. Older versions of the Python `2.7.X` and `3.X` series should be used with caution as they are not covered by integration tests. + +## Riak TS (Timeseries) + +You must use version `2.7.11`, `3.4.4` or `3.5.1` (or greater within a version series). Otherwise you will be affected by [this Python bug](https://bugs.python.org/issue23517). + +## From Source + +```sh +python setup.py install +``` + +There are additional dependencies on Python packages `setuptools` and `protobuf`. + +## From PyPI + +Official packages are signed and published to [PyPI](https://pypi.python.org/pypi/riak). + +To install from [PyPI](https://pypi.python.org/pypi/riak) directly you can use `pip`. + +```sh +pip install riak +``` + +# Testing + +## Unit Tests + +Unit tests will be executed via `tox` if it is in your `PATH`, otherwise by the `python2` and (if available), `python3` executables: + +```sh +make unit-test +``` + +## Integration Tests + +You have two options to run Riak locally - either build from source, or use a pre-installed Riak package. + +### Source + +To setup the default test configuration, build a Riak node from a clone of `github.com/basho/riak`: + +```sh +# check out latest release tag +git checkout riak-2.1.4 +make locked-deps +make rel +``` + +[Source build documentation](http://docs.basho.com/riak/kv/latest/setup/installing/source/). + +When building from source, the protocol buffers port will be `8087` and HTTP will be `8098`. + +### Package + +Install using your platform's package manager ([docs](http://docs.basho.com/riak/kv/latest/setup/installing/)) + +When installing from a package, the protocol buffers port will be `8087` and HTTP will be `8098`. + +### Running Integration Tests + +* Ensure you've initialized this repo's submodules: + +```sh +git submodule update --init +``` + +* Run the following: + +```sh +./tools/setup-riak +make integration-test +``` + + +Contributors +-------------------------- + +* Andrew Thompson +* Andy Gross +* Armon Dadgar +* Brett Hazen +* Brett Hoerner +* Brian Roach +* Bryan Fink +* Daniel Lindsley +* Daniel Néri +* Daniel Reverri +* [Dan Root](https://github.com/daroot) +* [David Basden](https://github.com/dbasden) +* [David Delassus](https://github.com/linkdd) +* David Koblas +* Dmitry Rozhkov +* Eric Florenzano +* Eric Moritz +* Filip de Waard +* Gilles Devaux +* Greg Nelson +* Gregory Burd +* Greg Stein +* Ian Plosker +* Jayson Baird +* Jeffrey Massung +* Jon Meredith +* Josip Lisec +* Justin Sheehy +* Kevin Smith +* [Luke Bakken](https://github.com/lukebakken) +* Mark Erdmann +* Mark Phillips +* Mathias Meyer +* Matt Heitzenroder +* [Matt Lohier](https://github.com/aquam8) +* Mikhail Sobolev +* Reid Draper +* Russell Brown +* Rusty Klophaus +* Rusty Klophaus +* Scott Lystig Fritchie +* Sean Cribbs +* Shuhao Wu +* Silas Sewell +* Socrates Lee +* Soren Hansen +* Sreejith Kesavan +* Timothée Peignier +* [`tobixx`](https://github.com/tobixx) +* [Tin Tvrtković](https://github.com/Tinche) +* [Vitaly Shestovskiy](https://github.com/lamp0chka) +* William Kral +* [Yasser Souri](https://github.com/yassersouri) diff --git a/README.rst b/README.rst deleted file mode 100644 index c93a5937..00000000 --- a/README.rst +++ /dev/null @@ -1,588 +0,0 @@ -======================== -Python Client for Riak -======================== - -.. image:: https://secure.travis-ci.org/basho/riak-python-client.png?branch=master - :target: http://travis-ci.org/basho/riak-python-client - -Documentation -============== - -`Documentation for the Riak Python Client Library `_ is available here. -The documentation source is found in `docs/ subdirectory -`_ and can be -built with `Sphinx `_. - -Documentation for Riak is available at http://wiki.basho.com/Riak.html - -Install -======= - -The recommended version of Python for use with this client is Python 2.7. - -You must have `Protocol Buffers`_ installed before you can install the Riak Client. From the Riak Python Client root directory, execute:: - - python setup.py install - -There is an additional dependency on the Python package `setuptools`. Please install `setuptools` first, e.g. ``port install py27-setuptools`` for OS X and MacPorts. - -Unit Test -=========== -To run the unit tests against a Riak server (with default TCP port configuration) on localhost, execute:: - - python setup.py test - -If you don't have `Riak Search `_ enabled you can set the ``SKIP_SEARCH`` environment variable to skip that tests. - -If your Riak server isn't running on localhost, use the environment variables ``RIAK_TEST_HOST`` and ``RIAK_TEST_HTTP_PORT`` and ``RIAK_TEST_PB_PORT=8087`` to specify where to find the Riak server. - -======== -Tutorial -======== - -This tutorial assumes basic working knowledge of how Riak works & what it can -do. If you need a more comprehensive overview how to use Riak, please check out -the `Riak Fast Track`_. - -.. _`Riak Fast Track`: http://wiki.basho.com/The-Riak-Fast-Track.html - - -Quick Start -=========== - -For the impatient, simple usage of the official Python binding for Riak looks -like:: - - import riak - - # Connect to Riak. - client = riak.RiakClient() - - # Choose the bucket to store data in. - bucket = client.bucket('test') - - - # Supply a key to store data under. - # The ``data`` can be any data Python's ``json`` encoder can handle. - person = bucket.new('riak_developer_1', data={ - 'name': 'John Smith', - 'age': 28, - 'company': 'Mr. Startup!', - }) - # Save the object to Riak. - person.store() - - -Connecting To Riak -================== - -There are two supported ways to connect to Riak, the HTTP interface & the -`Protocol Buffers`_ interface. Both provide the same API & full access to -Riak. - -The HTTP interface is easier to setup & is well suited for development use. It -is the slower of the two interfaces, but if you are only making a handful of -requests, it is more than capable. - -The Protocol Buffers (also called ``protobuf``) is more difficult to setup but -is significantly faster (2-3x) and is more suitable for production use. This -interface is better suited to a higher number of requests. - -.. _`Protocol Buffers`: http://code.google.com/p/protobuf/ - -To use the HTTP interface and connecting to a local Riak on the default port, -no arguments are needed:: - - import riak - - client = riak.RiakClient() - -The constructor also configuration options such as ``host``, ``http_port``, -``pb_port`` & ``prefix``. Please refer to the :doc:`client` documentation -for full details. - -To use the Protocol Buffers interface:: - - import riak - - client = riak.RiakClient(pb_port=8087, protocol='pbc') - -.. warning: - - Riak's default port is 8098. However, when using the Protocol Buffers, the - Riak listens on port 8087. If you forget this, you will *NOT* get an - immediate error, but will instead receive an error when fetching or storing - data to the effect of ``RiakError: 'Socket returned short read 135 - - expected 8192'``. - -The ``protocol`` argument indicates to the client which backend to use. -We didn't need to specify it in the HTTP example because ``http`` is the -default class. Available options are: ``http``, ``https``, & ``pbc``. - - -Using Buckets -============= - -Buckets in Riak's terminology are segmented keyspaces. They are a way to -categorize different types of data and are roughly analogous to tables in an -RDBMS. - -Once you have a ``client``, selecting a bucket is simple. Provide a string of -the name of the bucket to use:: - - test_bucket = client.bucket('test') - -If the bucket does not exist, Riak will create it for you. You can also open -as many buckets as you need:: - - user_bucket = client.bucket('user') - profile_bucket = client.bucket('profile') - status_bucket = client.bucket('status') - -If needed, you can also manually instantiate a bucket like so:: - - user_bucket = riak.RiakBucket(client, 'user') - -The buckets themselves provide many different methods. The most commonly used -are: - -* ``get`` - Fetches a key's value (decoded from JSON). -* ``get_binary`` - Also fetches a key's raw value (plain text or binary). -* ``new`` - Creates a new key/value pair (encoded in JSON). -* ``new_binary`` - Creates a new key/raw value pair. - -See the full :doc:`bucket` documentation for the other methods. - - -Storing Keys/Values -=================== - -Once you've got a working client/bucket, the next task at hand is storing data. -Riak provides several ways to store your data, but the most common are a -JSON-encoded structure or a binary blob. - -To store JSON-encoded data, you'd do something like the following:: - - import riak - - client = riak.RiakClient() - user_bucket = client.bucket('user') - - # We're creating the user data & keying off their username. - new_user = user_bucket.new('johndoe', data={ - 'first_name': 'John', - 'last_name': 'Doe', - 'gender': 'm', - 'website': 'http://example.com/', - 'is_active': True, - }) - # Note that the user hasn't been stored in Riak yet. - new_user.store() - -Note that any data Python's ``json`` (or ``simplejson``) encoder can handle is -fair game. - -As mentioned, Riak can also handle binary data, such as images, audio files, -etc. Storing binary data looks almost identical:: - - import riak - - client = riak.RiakClient() - user_photo_bucket = client.bucket('user_photo') - - # For example purposes, we'll read a file off the filesystem, but you can get - # the data from anywhere. - the_photo_data = open('/tmp/johndoe_headshot.jpg', 'rb').read() - - # We're storing the photo in a different bucket but keyed off the same - # username. - new_user = user_photo_bucket.new_binary('johndoe', data=the_photo_data, content_type='image/jpeg') - new_user.store() - -You can also manually store data by using ``RiakObject``:: - - import riak - import time - import uuid - - client = riak.RiakClient() - status_bucket = client.bucket('status') - - # We use ``uuid.uuid1().hex`` here to create a unique identifier for the status. - post_uuid = uuid.uuid1().hex - new_status = riak.RiakObject(client, status_bucket, post_uuid) - - # Add in the data you want to store. - new_status.set_data({ - 'message': 'First post!', - 'created': time.time(), - 'is_public': True, - }) - - # Set the content type. - new_status.set_content_type('application/json') - - # We want to do JSON-encoding on the value. - new_status._encode_data = True - - # Again, make sure you save it. - new_status.store() - - -Getting Single Values Out -========================= - -Storing data is all well and good, but you'll need to get that data out at a -later date. - -Riak provides several ways to get data out, though fetching single key/value -pairs is the easiest. Just like storing the data, you can pull the data out -in either the JSON-decoded form or a binary blob. Getting the JSON-decoded -data out looks like:: - - import riak - - client = riak.RiakClient() - user_bucket = client.bucket('user') - - johndoe = user_bucket.get('johndoe') - - # You've now got a ``RiakObject``. To get at the values in a dictionary - # form, call: - johndoe_dict = johndoe.data - -Getting binary data out looks like:: - - import riak - - client = riak.RiakClient() - user_photo_bucket = client.bucket('user_photo') - - johndoe = user_photo_bucket.get_binary('johndoe') - - # You've now got a ``RiakObject``. To get at the binary data, call: - johndoe_headshot = johndoe.data - -Manually fetching data is also possible:: - - import riak - - client = riak.RiakClient() - status_bucket = client.bucket('status') - - # We're using the UUID generated from the above section. - first_post_status = riak.RiakObject(client, status_bucket, post_uuid) - first_post_status._encode_data = True - r = status_bucket.get_r() - - # Calling ``reload`` will cause the ``RiakObject`` instance to load fresh - # data/metadata from Riak. - first_post_status.reload(r) - - # Finally, pull out the data. - message = first_post_status.data['message'] - - -Fetching Data Via Map/Reduce -============================ - -When you need to work with larger sets of data, one of the tools at your -disposal is MapReduce_. This technique iterates over all of the data, returning -data from the map phase & combining all the different maps in the reduce -phase(s). - -.. _MapReduce: http://wiki.basho.com/MapReduce.html - -To perform a map operation, such as returning all active users, you can do -something like:: - - import riak - - client = riak.RiakClient() - # First, you need to ``add`` the bucket you want to MapReduce on. - query = client.add('user') - # Then, you supply a Javascript map function as the code to be executed. - query.map("function(v) { var data = JSON.parse(v.values[0].data); if(data.is_active == true) { return [[v.key, data]]; } return []; }") - - for result in query.run(): - # Print the key (``v.key``) and the value for that key (``data``). - print "%s - %s" % (result[0], result[1]) - - # Results in something like: - # - # mr_smith - {'first_name': 'Mister', 'last_name': 'Smith', 'is_active': True} - # johndoe - {'first_name': 'John', 'last_name': 'Doe', 'is_active': True} - # annabody - {'first_name': 'Anna', 'last_name': 'Body', 'is_active': True} - -You can also do this manually:: - - import riak - - client = riak.RiakClient() - query = riak.RiakMapReduce(client).add('user') - query.map("function(v) { var data = JSON.parse(v.values[0].data); if(data.is_active == true) { return [[v.key, data]]; } return []; }") - - for result in query.run(): - print "%s - %s" % (result[0], result[1]) - -Adding a reduce phase, say to sort by username (key), looks almost identical:: - - import riak - - client = riak.RiakClient() - query = client.add('user') - query.map("function(v) { var data = JSON.parse(v.values[0].data); if(data.is_active == true) { return [[v.key, data]]; } return []; }") - query.reduce("function(values) { return values.sort(); }") - - for result in query.run(): - # Print the key (``v.key``) and the value for that key (``data``). - print "%s - %s" % (result[0], result[1]) - - # Results in something like: - # - # annabody - {'first_name': 'Anna', 'last_name': 'Body', 'is_active': True} - # johndoe - {'first_name': 'John', 'last_name': 'Doe', 'is_active': True} - # mr_smith - {'first_name': 'Mister', 'last_name': 'Smith', 'is_active': True} - - -Working With Related Data Via Links -=================================== - -Links_ are powerful concept in Riak that allow, within the key/value pair's -metadata, relations between objects. - -.. _Links: http://wiki.basho.com/Links.html - -Adding them to your data is relatively trivial. For instance, we'll link a -user's statuses to their user data:: - - import riak - import uuid - - client = riak.RiakClient() - user_bucket = client.bucket('user') - status_bucket = client.bucket('status') - - johndoe = user_bucket.get('johndoe') - - new_status = status_bucket.new(uuid.uuid1().hex, data={ - 'message': 'First post!', - 'created': time.time(), - 'is_public': True, - }) - # Add one direction (from status to user)... - new_status.add_link(johndoe) - new_status.store() - - # ... Then add the other direction. - johndoe.add_link(new_status) - johndoe.store() - -Fetching the data is equally simple:: - - import riak - - client = riak.RiakClient() - user_bucket = client.bucket('user') - - johndoe = user_bucket.get('johndoe') - - for status_link in johndoe.get_links(): - # Since what we get back are lightweight ``RiakLink`` objects, we need to - # get the associated ``RiakObject`` to access its data. - status = status_link.get() - print status.data['message'] - - -Using Search -============ - -`Riak Search`_ is a new feature available as of Riak 0.13. It allows you to create -queries that filter on data in the values without writing a MapReduce. It takes -inspiration from Lucene_, a popular Java-based search library, and incorporates -a Solr-like interface into Riak. The setup of this is outside the realm of this -tutorial, but usage of this feature looks like:: - - import riak - - client = riak.RiakClient() - - # First parameter is the bucket we want to search within, the second - # is the query we want to perform. - search_query = client.search('user', 'first_name:[Anna TO John]') - - for result in search_query.run(): - # You get ``RiakLink`` objects back. - user = result.get() - user_data = user.data - print "%s %s" % (user_data['first_name'], user_data['last_name']) - - # Results in something like: - # - # John Doe - # Anna Body - -You can enable and disable search for specific buckets through convenience -methods that install/remove the precommit hook - - bucket = client.bucket('search') - - if bucket.search_enabled(): - bucket.disable_search() - else: - bucket.enable_search() - -Search using the Solr Interface -------------------------------- - -The search as outlined above goes through Riak's MapReduce facilities to find -and fetch objects. Sometimes you either want to go through the Solr-like -interface Riak Search offers, e.g. to index and search documents without storing -them in Riak KV and relying on the pre-commit hook to index. - -Using the Solr interface also allows you to specify sort and limit parameters, -which, using the search based on MapReduce, you'd have to do that with reduce -functions. - -You can index documents into search indexes as simple Python dicts, which need -to have an attribute named "id":: - - client = riak.RiakClient() - client.solr().add("user", {"id": "anna", "first_name": "Anna"}) - -To search for documents, specify the index and a query string:: - - client = riak.RiakClient() - client.solr().search("user", "first_name:Anna") - -Additionally you can specify all the parameters supported by the Solr -interface:: - - client.solr().search("user", "Anna", wt="json", df="first_name") - -The search interface supports both XML and JSON, parsing both result formats -into dicts. - -You can also remove documents from the index again, using either a list of -document ids or queries:: - - client.solr().delete("user", docs=["anna"], queries=["first_name:Anna"]) - -.. _`Riak Search`: http://wiki.basho.com/Riak-Search.html -.. _Lucene: http://lucene.apache.org/ -.. _`Riak Search - Querying via the Solr Interface`: http://wiki.basho.com/Riak-Search---Querying.html#Querying-via-the-Solr-Interface - -Using Key Filters -================== - -`Key filters`_ are a new feature available as of Riak 0.14. They are -a way to pre-process MapReduce inputs from a full bucket query simply -by examining the key — without loading the object first. This is -especially useful if your keys are composed of domain-specific -information that can be analyzed at query-time. - -To illustrate this, let’s contrive an example. Let’s say we’re storing -customer invoices with a key constructed from the customer name and -the date, in a bucket called “invoices”. Here are some sample keys:: - - basho-20101215 - google-20110103 - yahoo-20090613 - -To query all invoices for a given customer:: - - import riak - - client = riak.RiakClient() - - query = client.add("invoices") - query.add_key_filter("tokenize", "-", 1) - query.add_key_filter("eq", "google") - - query.map("""function(v) { - var data = JSON.parse(v.values[0].data); - return [[v.key, data]]; - }""") - - -Alternatively, you can use riak.key_filter to build key filters:: - - query.add_key_filters(key_filter.tokenize("-", 1).eq("google")) - -Boolean operators can be used with riak.f instances:: - - # Query basho's orders for 2010 - filters = key_filter.tokenize("-", 1).eq("basho")\ - & key_filter.tokenize("-", 2).starts_with("2010") - -Filters can be combined using the + operator to produce very complex -filters:: - - # Query invoices for basho or google - filters = key_filter.tokenize("-", 1) + (key_filter.eq("basho") | key_filter.eq("google")) - - # This is the same as the following key filters - [['tokenize', '-', 1], ['or', [['eq', 'google']], [['eq', 'yahoo']]]] - - -.. _`Key filters`: http://wiki.basho.com/Key-Filters.html - -Test Server -=========== - -The client includes a Riak test server that can be used to start a Riak instance -on demand for testing purposes in your application. It uses in-memory storage -backends for both Riak KV and Riak Search and is therefore reasonably fast for a -testing setup. The in-memory setups also make it easier to wipe all data in the -instance without having to list and delete all keys manually. The original code -comes from Ripple_, as do the file system implementations. - -The server needs a local Riak installation, of which it uses only the installed -Erlang libraries and the configuration files to generate and run a temporary -server in a different directory. Make sure you run the most recent stable -version of Riak, and not a development snapshot, where your mileage may vary. - -By default, the HTTP port is set to 9000 and the Protocol Buffers interface -listens on port 9001. - -To use it, simply point it to your local Riak installation, and the rest is done -automagically:: - - from riak.test_server import TestServer - - server = TestServer(bin_dir="/usr/local/riak/0.14.2/bin") - server.prepare() - server.start() - -The server is started as an external process, with communication going through -the Erlang console. That allows it to easily wipe the in-memory backends used by -Riak and Riak Search. You can use the recycle() method to clean up the server:: - - server.recycle() - -To change the default configuration, you can specify additional arguments for -the Erlang VM. Let's raise the maximum number of processes to 1000000, just for -fun:: - - server = TestServer(vm_args={"+P": "1000000"}) - -You can also change the default configuration used to generate the app.config -file for the Riak instance. The format of the attributes follows the convention -of the app.config file itself, using a dict with keys for every section in the -configuration file, so "riak_core", "riak_kv", and so on. These in turn are also -dicts, following the same key-value format of the app.config file. - -So to change the default HTTP port to 8080, you can do the following:: - - server = TestServer(riak_core={"web_port": 8080}) - -The server should shut down properly when you stop the Python process, but if -you only need it for a subset of your tests, just stop the server:: - - server.stop() - -If you plan on repeatedly running the test server, either in multiple test -suites or in subsequent test runs, be sure to call cleanup() before starting or -after stopping it. - -.. _Ripple: https://github.com/seancribbs/ripple diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md deleted file mode 100644 index fb5a56d5..00000000 --- a/RELEASE_NOTES.md +++ /dev/null @@ -1,143 +0,0 @@ -# Riak Python Client Release Notes - -## 1.5.2 Patch Release - -* Added optional `timeout` parameter to `transport_options` dictionary - when creating a RiakClient object with Protocol Buffers. - -## 1.5.1 Patch Release - 2012-10-24 - -Release 1.5.1 fixes one bug and some documentation errors. - -* Fix bug where `http_status` is used instead of `http_code`. -* Fix documentation of `RiakMapReduce.index` method. -* Fix documentation of `RiakClient.__init__` method. - -## 1.5.0 Feature Release - 2012-08-29 - -Release 1.5.0 is a feature release that supports Riak 1.2. - -Noteworthy features: - -* Riak 1.2 features are now supported, including Search and 2I queries - over Protocol Buffers transport. The Protocol Buffers message - definitions now exist as a separate package, available on - [PyPi](http://pypi.python.org/pypi/riak_pb/1.2.0). - - **NOTE:** The return value of search queries over HTTP and MapReduce - were changed to be compatible with the results returned from the - Protocol Buffers interface. -* The client will use a version-based feature detection scheme to - enable or disable various features, including the new Riak 1.2 - features. This enables compatibility with older nodes during a - rolling upgrade, or usage of the newer client with older clusters. - -Noteworthy bugfixes: - -* The code formatting and style was adjusted to fit PEP8 standards. -* All classes in the package are now "new-style". -* The PW accessor methods on RiakClient now get and set the right - instance variable. -* Various fixes were made to the TestServer and it will throw an - exception when it fails to start. - -## 1.4.1 Patch Release - 2012-06-19 - -Noteworthy features: - -* New Riak objects support Riak-created random keys - -Noteworthy bugfixes: - -* Map Reduce queries now use "application/json" as the Content-Type - -## 1.4.0 Feature Release - 2012-03-30 - -Release 1.4.0 is a feature release comprising over 117 individual -commits. - -Noteworthy features: - -* Python 2.6 and 2.7 are supported. On 2.6, the unittest2 package is - required to run the test suite. -* Google's official protobuf package (2.4.1 or later) is now a - dependency. The package from downloads.basho.com/support is no - longer necessary. -* Travis-CI is enabled on the client. Go to - http://travis-ci.org/basho/riak-python-client for build status. -* Riak 1.0+ features, namely secondary indexes and primary quora - (PR/PW), are supported. -* `if_none_match` is a valid request option when storing objects, and - will prevent the write when set to `True` if the key already exists. -* Links can be set wholesale using the `set_links()` method. -* Transport-specific options can be passed through when creating a - `Client` object. -* A connection manager was added that will (when manipulated manually) - allow connections to multiple Riak nodes. This will be fully - integrated in a future release. - -Noteworthy bugfixes: - -* Links now use the proper URL-encoding in HTTP headers, preventing - problems with explosion from multiple encoding passes. -* Many fixes were applied to make the Protocol Buffers transport more - stable. -* `RiakObject.get_content_type()` will behave properly when content - type is not set. -* Deprecated transport classes were removed since their functionality - had folded into the primary transports. -* A temporary fix was made for unicode bucket/key names which raises - an error when they are used and cannot be coerced to ASCII. -* The Erlang sources/beams for the TestServer are now included in the - package. -* MapReduce failures will now produce a more useful error message and - be handled properly when no results are returned. - -There are lots of other great fixes from our wonderful -community. [Check them out!](https://github.com/basho/riak-python-client/compare/1.3.0...1.4.0) - -## 1.3.0 Feature Release - 2011-08-04 - -Release 1.3.0 is a feature release bringing a slew of updates. - -Noteworthy features: - -* #37: Support for the Riak Search HTTP Interface (Mathias Meyer) -* #36: Support to store large files in Luwak (Mathias Meyer) -* #35: Convenience methods to enable, disable and check search indexing - on Riak buckets (Mathias Meyer) -* #34: Port of Ripple's test server to Python, allows faster testing - thanks to an in-memory Riak instance (Mathias Meyer) -* #31: New transports: A Protocol Buffers connection cache - (riak.transports.pbc.RiakPbcCacheTransport), a transport to reuse the - underlying TCP connections by setting SO_REUSEADDR on the socket - (riak.transports.http.RiakHttpReuseTransport), and one that tries to - reuse connections to the same host (riak.transports.http.RiakHttpPoolTransport) - (Gilles Devaux) - -Fixes: - -* #33: Respect maximum link header size when using HTTP. Link header is now - split up into multiple headers when it exceeds the maximum size of 8192 bytes. - (Mathias Meyer) -* #41: Connections potentially not returned to the protocol buffers connection - pool. (Reid Draper) -* #42: Reset protocol buffer connection up on connection error (Brett Hoerner) - -## 1.2.2 Patch Release - 2011-06-22 - -Release 1.2.2 is a minor patch release. - -Noteworthy fixes and improvements: - -* #29: Add an nicer API for using key filters with MapReduce (Eric Moritz) -* #13 and #24: Let Riak generate a key when none is specified (Mark Erdmann) -* #28: Function aliases for the Riak built-in MapReduce functions (Eric Moritz) -* #20: Add a convenience method to create Riak object directly from file (Ana Nelson) -* #16: Support return\_body parameter when creating a new object (Stefan Praszalowicz, Andy Gross) -* #17: Storing an object fails when it doesn't exist in Riak (Eric Moritz, Andy Gross) -* #18: Ensure that a default content type is set when none specified (Andy Gross) -* #22: Fix user meta data support (Mathias Meyer) -* #23: Fix links to the wiki (Mikhail Sobolev) -* #25: Enable support for code coverage when running tests (Mikhail Sobolev) -* #26: Debian packaging (Dmitry Rozhkov) diff --git a/RELNOTES.md b/RELNOTES.md new file mode 100644 index 00000000..6722c3ec --- /dev/null +++ b/RELNOTES.md @@ -0,0 +1,347 @@ +# Riak Python Client Release Notes + +## [`3.0.0` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-3.0.0) + +* [Running expensive operations *now raise exceptions*](https://github.com/basho/riak-python-client/pull/518). You can disable these exceptions for development purposes but should not do so in production. + +## [`2.7.0` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.7.0) + * Riak TS 1.5 support + * Support for `head` parameter + +## [`2.6.1` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.6.0) + * NOTE: Due to pypi upload errors, `2.6.1` takes the place of `2.6.0`. + +## [`2.6.0` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.6.0) + * NOTE: Due to pypi upload errors, `2.6.1` takes the place of `2.6.0`. + +## [`2.5.5` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.5.5) + + * [Stop all pools when client shuts down](https://github.com/basho/riak-python-client/pull/488) + * [Calling `close` on client closes pools, remove global multi pools](https://github.com/basho/riak-python-client/pull/490). *NOTE*: if you use the multi get or put features of the client, you *MUST* call `close()` on your `RiakClient` instance to correctly clean up the thread pools used for these multi-operations. + +## [`2.5.4` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.5.4) + + * [When converting `datetime` objects to send to Riak TS, `tzinfo` will be used if present](https://github.com/basho/riak-python-client/pull/486) + * [Workaround for incorrect version returned by Riak TS OSS](https://github.com/basho/riak-python-client/pull/472) + +## [`2.5.3` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.5.3) + + * [Bug fix for raising `BadResource`](https://github.com/basho/riak-python-client/pull/481) + +## [`2.5.2` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.5.2) + +* *NOTE*: for Riak TS data, automatic conversion from epoch values *to* Python `datetime` objects has been removed. If you would like to have automatic conversion, use `RiakClient(transport_options={'ts_convert_timestamp': True})` +* Miscellaneous fixes for term-to-binary encoding of messages for Riak TS. +* [Ensure `six` is not required during installation](https://github.com/basho/riak-python-client/pull/459) + +## [`2.5.0` Release - Deprecated](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.5.0) + +* *NOTE*: due to the `basho-erlastic` dependency, this version will not install correctly. Please use ``2.5.2``. +* *NOTE*: for Riak TS data, automatic conversion from epoch values *to* Python `datetime` objects has been removed. If you would like to have automatic conversion, use `RiakClient(transport_options={'ts_convert_timestamp': True})` +* [Socket Enhancements](https://github.com/basho/riak-python-client/pull/453) - Resolves [#399](https://github.com/basho/riak-python-client/issues/399) +* [Add multi-put](https://github.com/basho/riak-python-client/pull/452) +* [Add support for term-to-binary encoding](https://github.com/basho/riak-python-client/pull/448) *Note:* This requires at least version ``1.3.0`` of Riak TS. + +## `2.4.2` Patch Release - 2016-02-20 + +* [Fix SSL host name](https://github.com/basho/riak-python-client/pull/436) +* [Use `riak-client-tools`](https://github.com/basho/riak-python-client/issues/434) + +## `2.4.1` Patch Release - 2016-02-03 + +* [Riak TS: Millisecond precision](https://github.com/basho/riak-python-client/issues/430) +* [Fix release process](https://github.com/basho/riak-python-client/issues/429) + +## `2.4.0` Feature Release - 2016-01-13 + +This release enhances Riak Time Series functionality. + +* [Encapsulate table description](https://github.com/basho/riak-python-client/pull/422) + +## `2.3.0` Feature Release - 2015-12-14 + +Release `2.3.0` features support for new +[time series](https://github.com/basho/riak-python-client/pull/416) +functionality. + +This is release retires support for Python 2.6.x but adds support for +Python 3.5.x. + +There are also many bugfixes and new enhancements: + +* [The `riak_pb` module is now integrated into the Python Client] + (https://github.com/basho/riak-python-client/pull/418) +* [Support for Preflists and Write-Once bucket types] + (https://github.com/basho/riak-python-client/pull/414) +* [Support Riak `2.1.1`] + (https://github.com/basho/riak-python-client/pull/407) +* [Native SSL support for Python `2.7.9`+] + (https://github.com/basho/riak-python-client/pull/397) + + +## `2.2.0` Feature Release - 2014-12-18 + +Release `2.2.0` features support for +[Python 3](https://github.com/basho/riak-python-client/pull/379), +specifically 3.3 and 3.4. This version uses the native SSL security instead +of [pyOpenSSL](http://pypi.python.org/pypi/pyOpenSSL) which is required +for the Python 2 series. + +This release also includes many bugfixes and enhancements, most +notably: + +* [Fixed an issue with the implementation of `Mapping.__iter__`] + (https://github.com/basho/riak-python-client/pull/367) +* [Test client certificate generation updated] + (https://github.com/basho/riak-python-client/pull/373) +* [Protocol Buffers had a socket.send issue] + (https://github.com/basho/riak-python-client/pull/382) +* [Support for bucket types in Map/Reduce jobs added] + (https://github.com/basho/riak-python-client/pull/385) +* [Race condition in `RiakBucket` creation fixed] + (https://github.com/basho/riak-python-client/pull/386) +* [Data Types can now be deleted] + (https://github.com/basho/riak-python-client/pull/387) +* [2i Range Queries with a zero end index now work] + (https://github.com/basho/riak-python-client/pull/388) + + +## `2.1.0` Feature Release - 2014-09-03 + +Release `2.1.0` features support for Riak 2.0 capabilities including: + +* Bucket Types +* Riak Data Types (CRDTs) +* Search 2.0 (codename Yokozuna) +* Security: SSL/TLS, Authentication, and Authorization + +As a result of the new security features, the package now depends on +[pyOpenSSL](http://pypi.python.org/pypi/pyOpenSSL) and will warn if +your version of OpenSSL is too old. + +This release also includes many bugfixes and enhancements, most +notably: + +* The default protocol is now 'pbc', not 'http'. +* When used correctly, streaming requests no longer result in leaks + from the connection pool. +* The size of the multiget worker pool can be set when initializing + the client. +* Secondary index queries can now iterate over all pages in a query. +* The number of times a request is retried after network failure is + now configurable. +* The additional request options `basic_quorum` and `notfound_ok` are + now supported. + +## `2.0.3` Patch Release - 2014-03-06 + +Release `2.0.3` includes support for 1.4.4's 2I regexp feature and fixes +a few bugs: + +* Docs generation now uses the version from the top-level package. +* Some internal uses of the deprecated RiakClient.solr were removed. +* More errors will be caught and propagated properly from multiget + requests, preventing deadlocks on the caller side. + +## `2.0.2` Patch release - 2013-11-18 + +Release `2.0.2` includes support for the 1.4.1+ "timeout" option on +secondary index queries. + +## `2.0.1` Patch release - 2013-08-28 + +Release `2.0.1` includes a minor compatibility fix for Python 2.6 and an +updated README. + +## `2.0.0` Feature Release - 2013-07-30 + +Release 2.0 is the culmination of many months of rearchitecting the +client. Highlights: + +* Automatic connection to multiple nodes, with request retries, + through a thread-safe connection pool. +* All Riak 1.3 and 1.4 features, including bucket properties, + paginating and streaming secondary indexes, CRDT counters, + client-specified timeouts, and more. +* Cleaner, more Pythonic access to RiakObject and RiakBucket + attributes, favoring properties over methods where possible. +* Simpler representations of links (3-tuples) and index entries + (2-tuples). +* Streaming requests (keys, buckets, MapReduce, 2i) are now exposed as + iterators. +* Feature detection prevents sending requests to hosts that can't + handle them. +* Better handling of siblings -- you don't have to request them + individually anymore -- and registrable resolver functions. +* A new `multiget` operation that fetches a collection of keys using + a pool background threads. +* A more resilient, repeatable test suite that generates buckets and + key names that are essentially random. +* Last but not least, a brand new, more detailed documentation site! + +Other features: + +* Added an encoder/decoder pair to support `text/plain`. +* The Travis CI build will now install the latest Riak to run the + suite against. + +Other bugfixes: + +* The `charset` metadata can now be received via the `Content-Type` + header on HTTP. +* Objects with empty keys and buckets with empty names cannot be + created or accessed, as they are unaddressable over HTTP. +* Performance and compatibility of `TestServer` was improved. +* Non-ASCII request bodies are better supported on HTTP. +* Enabling and disabling search indexing on a bucket now uses the + `search` bucket property. + +## `1.5.2` Patch Release - 2013-01-31 + +Release `1.5.2` fixes some bugs and adds HTTPS/SSL support. + +* Added support for HTTPS. +* Fixed writing of the `app.config` for the `TestServer`. +* Reorganized the tests into multiple files and cases. +* Some methods on `RiakObject` were made private where appropriate. +* The version comparison used in feature detection was loosened to + support pre-release versions of Riak. +* Prevent fetching the `protobuf` package from Google Code. +* Prefer `simplejson` over `json` when present. + +## `1.5.1` Patch Release - 2012-10-24 + +Release `1.5.1` fixes one bug and some documentation errors. + +* Fix bug where `http_status` is used instead of `http_code`. +* Fix documentation of `RiakMapReduce.index` method. +* Fix documentation of `RiakClient.__init__` method. + +## `1.5.0` Feature Release - 2012-08-29 + +Release `1.5.0` is a feature release that supports Riak 1.2. + +Noteworthy features: + +* Riak 1.2 features are now supported, including Search and 2I queries + over Protocol Buffers transport. The Protocol Buffers message + definitions now exist as a separate package, available on + [PyPi](http://pypi.python.org/pypi/riak_pb/`1.2.0`). + + **NOTE:** The return value of search queries over HTTP and MapReduce + were changed to be compatible with the results returned from the + Protocol Buffers interface. +* The client will use a version-based feature detection scheme to + enable or disable various features, including the new Riak 1.2 + features. This enables compatibility with older nodes during a + rolling upgrade, or usage of the newer client with older clusters. + +Noteworthy bugfixes: + +* The code formatting and style was adjusted to fit PEP8 standards. +* All classes in the package are now "new-style". +* The PW accessor methods on RiakClient now get and set the right + instance variable. +* Various fixes were made to the TestServer and it will throw an + exception when it fails to start. + +## `1.4.1` Patch Release - 2012-06-19 + +Noteworthy features: + +* New Riak objects support Riak-created random keys + +Noteworthy bugfixes: + +* Map Reduce queries now use "application/json" as the Content-Type + +## `1.4.0` Feature Release - 2012-03-30 + +Release `1.4.0` is a feature release comprising over 117 individual +commits. + +Noteworthy features: + +* Python 2.6 and 2.7 are supported. On 2.6, the unittest2 package is + required to run the test suite. +* Google's official protobuf package (`2.4.1` or later) is now a + dependency. The package from downloads.basho.com/support is no + longer necessary. +* Travis-CI is enabled on the client. Go to + http://travis-ci.org/basho/riak-python-client for build status. +* Riak 1.0+ features, namely secondary indexes and primary quora + (PR/PW), are supported. +* `if_none_match` is a valid request option when storing objects, and + will prevent the write when set to `True` if the key already exists. +* Links can be set wholesale using the `set_links()` method. +* Transport-specific options can be passed through when creating a + `Client` object. +* A connection manager was added that will (when manipulated manually) + allow connections to multiple Riak nodes. This will be fully + integrated in a future release. + +Noteworthy bugfixes: + +* Links now use the proper URL-encoding in HTTP headers, preventing + problems with explosion from multiple encoding passes. +* Many fixes were applied to make the Protocol Buffers transport more + stable. +* `RiakObject.get_content_type()` will behave properly when content + type is not set. +* Deprecated transport classes were removed since their functionality + had folded into the primary transports. +* A temporary fix was made for unicode bucket/key names which raises + an error when they are used and cannot be coerced to ASCII. +* The Erlang sources/beams for the TestServer are now included in the + package. +* MapReduce failures will now produce a more useful error message and + be handled properly when no results are returned. + +There are lots of other great fixes from our wonderful +community. [Check them out!](https://github.com/basho/riak-python-client/compare/`1.3.0`...1.4.0) + +## `1.3.0` Feature Release - 2011-08-04 + +Release `1.3.0` is a feature release bringing a slew of updates. + +Noteworthy features: + +* #37: Support for the Riak Search HTTP Interface (Mathias Meyer) +* #36: Support to store large files in Luwak (Mathias Meyer) +* #35: Convenience methods to enable, disable and check search indexing + on Riak buckets (Mathias Meyer) +* #34: Port of Ripple's test server to Python, allows faster testing + thanks to an in-memory Riak instance (Mathias Meyer) +* #31: New transports: A Protocol Buffers connection cache + (riak.transports.pbc.RiakPbcCacheTransport), a transport to reuse the + underlying TCP connections by setting SO_REUSEADDR on the socket + (riak.transports.http.RiakHttpReuseTransport), and one that tries to + reuse connections to the same host (riak.transports.http.RiakHttpPoolTransport) + (Gilles Devaux) + +Fixes: + +* #33: Respect maximum link header size when using HTTP. Link header is now + split up into multiple headers when it exceeds the maximum size of 8192 bytes. + (Mathias Meyer) +* #41: Connections potentially not returned to the protocol buffers connection + pool. (Reid Draper) +* #42: Reset protocol buffer connection up on connection error (Brett Hoerner) + +## `1.2.2` Patch Release - 2011-06-22 + +Release `1.2.2` is a minor patch release. + +Noteworthy fixes and improvements: + +* #29: Add an nicer API for using key filters with MapReduce (Eric Moritz) +* #13 and #24: Let Riak generate a key when none is specified (Mark Erdmann) +* #28: Function aliases for the Riak built-in MapReduce functions (Eric Moritz) +* #20: Add a convenience method to create Riak object directly from file (Ana Nelson) +* #16: Support return\_body parameter when creating a new object (Stefan Praszalowicz, Andy Gross) +* #17: Storing an object fails when it doesn't exist in Riak (Eric Moritz, Andy Gross) +* #18: Ensure that a default content type is set when none specified (Andy Gross) +* #22: Fix user meta data support (Mathias Meyer) +* #23: Fix links to the wiki (Mikhail Sobolev) +* #25: Enable support for code coverage when running tests (Mikhail Sobolev) +* #26: Debian packaging (Dmitry Rozhkov) diff --git a/THANKS b/THANKS deleted file mode 100644 index 16927ec4..00000000 --- a/THANKS +++ /dev/null @@ -1,44 +0,0 @@ -The following people have contributed to the Riak Python client: - -Andrew Thompson -Andy Gross -Armon Dadgar -Brett Hoerner -Brian Roach -Bryan Fink -Daniel Lindsley -Daniel Néri -Daniel Reverri -David Koblas -Dmitry Rozhkov -Eric Florenzano -Eric Moritz -Filip de Waard -Gilles Devaux -Greg Nelson -Greg Stein -Gregory Burd -Ian Plosker -Jayson Baird -Jeffrey Massung -Jon Meredith -Josip Lisec -Justin Sheehy -Kevin Smith -Mark Erdmann -Mark Phillips -Mathias Meyer -Matt Heitzenroder -Mikhail Sobolev -Reid Draper -Russell Brown -Rusty Klophaus -Scott Lystig Fritchie -Sean Cribbs -Shuhao Wu -Silas Sewell -Socrates Lee -Soren Hansen -Sreejith Kesavan -Timothée Peignier -William Kral diff --git a/build/publish b/build/publish new file mode 100755 index 00000000..13268ac3 --- /dev/null +++ b/build/publish @@ -0,0 +1,189 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset + +declare -r debug='false' +declare -r tmpfile_file="/tmp/publish.$$.tmpfiles" + +function make_temp_file +{ + local template="${1:-publish.$$.XXXXXX}" + if [[ $template != *XXXXXX ]] + then + template="$template.XXXXXX" + fi + local tmp=$(mktemp -t "$template") + echo "$tmp" >> "$tmpfile_file" + echo "$tmp" +} + +function now +{ + date '+%Y-%m-%d %H:%M:%S' +} + +function pwarn +{ + echo "$(now) [warning]: $@" 1>&2 +} + +function perr +{ + echo "$(now) [error]: $@" 1>&2 +} + +function pinfo +{ + echo "$(now) [info]: $@" +} + +function pdebug +{ + if [[ $debug == 'true' ]] + then + echo "$(now) [debug]: $@" + fi +} + +function errexit +{ + perr "$@" + exit 1 +} + +function onexit +{ + if [[ -f $tmpfile_file ]] + then + for tmpfile in $(< $tmpfile_file) + do + pdebug "removing temp file $tmpfile" + rm -f $tmpfile + done + rm -f $tmpfile_file + fi +} + +function gh_publish { + if [[ -z $version_string ]] + then + errexit 'gh_publish: version_string required' + fi + + # NB: no 'v' here at start of version_string + local -r package_name="riak-$version_string.tar.gz" + local -r package="./dist/riak-$version_string.tar.gz" + if [[ ! -s $package ]] + then + errexit "gh_publish: expected to find $package in dist/" + fi + + # NB: we use a X.Y.Z tag + local -r release_json="{ + \"tag_name\" : \"$version_string\", + \"name\" : \"Riak Python Client $version_string\", + \"body\" : \"riak-python-client $version_string\nhttps://github.com/basho/riak-python-client/blob/master/RELNOTES.md\", + \"draft\" : false, + \"prerelease\" : $is_prerelease + }" + + pdebug "Release JSON: $release_json" + + local curl_content_file="$(make_temp_file)" + local curl_stdout_file="$(make_temp_file)" + local curl_stderr_file="$(make_temp_file)" + + curl -4so $curl_content_file -w '%{http_code}' -XPOST \ + -H "Authorization: token $(< $github_api_key_file)" -H 'Content-type: application/json' \ + 'https://api.github.com/repos/basho/riak-python-client/releases' -d "$release_json" 1> "$curl_stdout_file" 2> "$curl_stderr_file" + if [[ $? != 0 ]] + then + errexit "curl error exited with code: '$?' see '$curl_stderr_file'" + fi + + local -i curl_rslt="$(< $curl_stdout_file)" + if (( curl_rslt == 422 )) + then + pwarn "Release in GitHub already exists! (http code: '$curl_rslt')" + curl -4so $curl_content_file -w '%{http_code}' -XGET \ + -H "Authorization: token $(< $github_api_key_file)" -H 'Content-type: application/json' \ + "https://api.github.com/repos/basho/riak-python-client/releases/tags/$version_string" 1> "$curl_stdout_file" 2> "$curl_stderr_file" + if [[ $? != 0 ]] + then + errexit "curl error exited with code: '$?' see '$curl_stderr_file'" + fi + elif (( curl_rslt != 201 )) + then + errexit "Creating release in GitHub failed with http code '$curl_rslt'" + fi + + if [[ ! -s $curl_content_file ]] + then + errexit 'no release info to parse for asset uploads' + fi + + # "upload_url": "https://uploads.github.com/repos/basho/riak-python-client/releases/1115734/assets{?name,label}" + # https://uploads.github.com/repos/basho/riak-python-client/releases/1115734/assets{?name,label} + local -r upload_url_with_name=$(perl -ne 'print qq($1\n) and exit if /"upload_url"[ :]+"(https:\/\/[^"]+)"/' "$curl_content_file") + local -r upload_url="${upload_url_with_name/\{?name,label\}/?name=$package_name}" + + local curl_content_file="$(make_temp_file)" + local curl_stdout_file="$(make_temp_file)" + local curl_stderr_file="$(make_temp_file)" + + curl -4so $curl_content_file -w '%{http_code}' -XPOST \ + -H "Authorization: token $(< $github_api_key_file)" -H 'Content-type: application/x-compressed, application/x-tar' \ + "$upload_url" --data-binary "@$package" 1> "$curl_stdout_file" 2> "$curl_stderr_file" + if [[ $? != 0 ]] + then + errexit "curl error exited with code: '$?' see '$curl_stderr_file'" + fi + + curl_rslt="$(< $curl_stdout_file)" + if (( curl_rslt != 201 )) + then + errexit "Uploading release assets to GitHub failed with http code '$curl_rslt'" + fi +} + +trap onexit EXIT + +declare -r version_string="${1:-unknown}" + +# https://www.python.org/dev/peps/pep-0440/ +if [[ ! $version_string =~ ^[0-9].[0-9].[0-9]([abcr]+[0-9]+)?$ ]] +then + errexit 'first argument must be valid version string in X.Y.Z, X.Y.ZaN, X.Y.ZbN or X.Y.ZrcN format' +fi + +is_prerelease='false' +if [[ $version_string =~ ^[0-9].[0-9].[0-9][abcr]+[0-9]+$ ]] +then + pinfo "publishing pre-release version: $version_string" + is_prerelease='true' +else + pinfo "publishing version $version_string" +fi + +declare -r current_branch="$(git rev-parse --abbrev-ref HEAD)" + +declare -r github_api_key_file="$HOME/.ghapi" +if [[ ! -s $github_api_key_file ]] +then + errexit "please save your GitHub API token in $github_api_key_file" +fi + +# Validate commands +if ! hash curl 2>/dev/null +then + errexit "'curl' must be in your PATH" +fi + +validate=${2:-''} +if [[ $validate == 'validate' ]] +then + exit 0 +fi + +gh_publish diff --git a/build/pyenv-setup b/build/pyenv-setup new file mode 100755 index 00000000..7759d45a --- /dev/null +++ b/build/pyenv-setup @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +unset PYENV_VERSION + +if [[ ! -d $PYENV_ROOT ]] +then + export PYENV_ROOT="$HOME/.pyenv" +fi + +declare -r PROJDIR="$PWD" +if [[ ! -s $PROJDIR/riak/__init__.py ]] +then + echo "[ERROR] script must be run from the clone of github.com/basho/riak-python-client" 1>&2 + exit 1 +fi + +rm -f $PROJDIR/.python-version + +# Install pyenv if it's missing +if [[ ! -d $PYENV_ROOT ]] +then + git clone 'https://github.com/yyuu/pyenv.git' $PYENV_ROOT +else + (cd $PYENV_ROOT && git fetch --all) +fi + +(cd $PYENV_ROOT && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))) + +declare -r pyenv_alias_dir="$PYENV_ROOT/plugins/pyenv-alias" +if [[ ! -d $pyenv_alias_dir ]] +then + git clone 'https://github.com/s1341/pyenv-alias.git' $pyenv_alias_dir +else + (cd $pyenv_alias_dir && git pull origin master) +fi + +# Add pyenv root to PATH +# and initialize pyenv +if [[ $PATH != */.pyenv* ]] +then + echo "[INFO] adding $PYENV_ROOT/bin to PATH" + export PATH="$PYENV_ROOT/bin:$PATH" +fi + +if [[ $(type -t pyenv) != 'function' ]] +then + echo "[INFO] init pyenv" + eval "$(pyenv init -)" +fi + +do_pip_upgrades='false' + +# NB: 2.7.8 is special-cased +for pyver in 2.7 3.3 3.4 3.5 3.6 +do + riak_py_alias="riak_$pyver" + if ! pyenv versions | fgrep -v 'riak_2.7.8' | fgrep -q "$riak_py_alias" + then + # Need to install it + do_pip_upgrades='true' + + declare -i pymaj="${pyver%.*}" + declare -i pymin="${pyver#*.}" + pyver_latest="$(pyenv install --list | grep -E "^[[:space:]]+$pymaj\\.$pymin\\.[[:digit:]]+\$" | tail -n1 | sed -e 's/[[:space:]]//g')" + + echo "[INFO] installing Python $pyver_latest" + VERSION_ALIAS="$riak_py_alias" pyenv install "$pyver_latest" + fi +done + +if ! pyenv versions | fgrep -q 'riak_2.7.8' +then + # Need to install it + do_pip_upgrades='true' + + echo "[INFO] installing Python 2.7.8" + VERSION_ALIAS='riak_2.7.8' pyenv install '2.7.8' +fi + +pushd $PROJDIR +pyenv local 'riak_3.6' 'riak_3.5' 'riak_3.4' 'riak_3.3' 'riak_2.7' 'riak_2.7.8' + +pyenv rehash + +if [[ $do_pip_upgrades == 'true' ]] +then + for PY in $(pyenv versions --bare --skip-aliases | grep '^riak_') + do + echo "[INFO] $PY - upgrading pip / setuptools" + PYENV_VERSION="$PY" pip install --upgrade pip setuptools + done +fi + +python_version="$(python --version)" +if [[ $python_version == Python\ 3* ]] +then + pip install --ignore-installed tox + if ! pip show --quiet tox + then + echo "[ERROR] install of 'tox' failed" 1>&2 + popd + exit 1 + fi + pyenv rehash +else + echo "[ERROR] expected Python 3 to be 'python' at this point" 1>&2 + popd + exit 1 +fi + +popd diff --git a/commands.py b/commands.py new file mode 100644 index 00000000..a20557ab --- /dev/null +++ b/commands.py @@ -0,0 +1,405 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import os +import os.path +import re + +from distutils.core import Command +from distutils.errors import DistutilsOptionError +from distutils.file_util import write_file +from distutils import log +from subprocess import Popen, PIPE + + +__all__ = ['build_messages', 'setup_timeseries'] + + +# Exception classes used by this module. +class CalledProcessError(Exception): + """This exception is raised when a process run by check_call() or + check_output() returns a non-zero exit status. + The exit status will be stored in the returncode attribute; + check_output() will also store the output in the output attribute. + """ + def __init__(self, returncode, cmd, output=None): + self.returncode = returncode + self.cmd = cmd + self.output = output + + def __str__(self): + return "Command '%s' returned non-zero exit status %d" % (self.cmd, + self + .returncode) + + +def check_output(*popenargs, **kwargs): + """Run command with arguments and return its output as a byte string. + + If the exit code was non-zero it raises a CalledProcessError. The + CalledProcessError object will have the return code in the returncode + attribute and output in the output attribute. + + The arguments are the same as for the Popen constructor. Example: + + >>> check_output(["ls", "-l", "/dev/null"]) + 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' + + The stdout argument is not allowed as it is used internally. + To capture standard error in the result, use stderr=STDOUT. + + >>> import sys + >>> check_output(["/bin/sh", "-c", + ... "ls -l non_existent_file ; exit 0"], + ... stderr=sys.stdout) + 'ls: non_existent_file: No such file or directory\n' + """ + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be ' + 'overridden.') + process = Popen(stdout=PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise CalledProcessError(retcode, cmd, output=output) + return output + + +try: + import simplejson as json +except ImportError: + import json + + +class bucket_type_commands: + def initialize_options(self): + self.riak_admin = None + + def finalize_options(self): + if self.riak_admin is None: + raise DistutilsOptionError("riak-admin option not set") + + def run(self): + if self._check_available(): + for name in self._props: + self._create_and_activate_type(name, self._props[name]) + + def check_output(self, *args, **kwargs): + if self.dry_run: + log.info(' '.join(args)) + return bytearray() + else: + return check_output(*args, **kwargs) + + def _check_available(self): + try: + self.check_btype_command("list") + return True + except CalledProcessError: + log.error("Bucket types are not supported on this Riak node!") + return False + + def _create_and_activate_type(self, name, props): + # Check status of bucket-type + exists = False + active = False + try: + status = self.check_btype_command('status', name) + except CalledProcessError as e: + status = e.output + + exists = ('not an existing bucket type' not in status.decode('ascii')) + active = ('is active' in status.decode('ascii')) + + if exists or active: + log.info("Updating {0} bucket-type with props {1}" + .format(repr(name), repr(props))) + self.check_btype_command("update", name, + json.dumps({'props': props}, + separators=(',', ':'))) + else: + log.info("Creating {0} bucket-type with props {1}" + .format(repr(name), repr(props))) + self.check_btype_command("create", name, + json.dumps({'props': props}, + separators=(',', ':'))) + + if not active: + log.info('Activating {0} bucket-type'.format(repr(name))) + self.check_btype_command("activate", name) + + def check_btype_command(self, *args): + cmd = self._btype_command(*args) + return self.check_output(cmd) + + def run_btype_command(self, *args): + self.spawn(self._btype_command(*args)) + + def _btype_command(self, *args): + cmd = [self.riak_admin, "bucket-type"] + cmd.extend(args) + return cmd + + +class setup_timeseries(bucket_type_commands, Command): + """ + Creates bucket-types appropriate for timeseries. + """ + + description = "create bucket-types used in timeseries tests" + + user_options = [ + ('riak-admin=', None, 'path to the riak-admin script') + ] + + _props = { + 'GeoCheckin': { + 'n_val': 3, + 'table_def': ''' + CREATE TABLE GeoCheckin ( + geohash varchar not null, + user varchar not null, + time timestamp not null, + weather varchar not null, + temperature double, + PRIMARY KEY( + (geohash, user, quantum(time, 15, m)), + geohash, user, time + ) + )''' + } + } + + +class ComparableMixin(object): + def _compare(self, other, method): + try: + return method(self._cmpkey(), other._cmpkey()) + except (AttributeError, TypeError): + # _cmpkey not implemented, or return different type, + # so I can't compare with "other". + return NotImplemented + + def __lt__(self, other): + return self._compare(other, lambda s, o: s < o) + + def __le__(self, other): + return self._compare(other, lambda s, o: s <= o) + + def __eq__(self, other): + return self._compare(other, lambda s, o: s == o) + + def __ge__(self, other): + return self._compare(other, lambda s, o: s >= o) + + def __gt__(self, other): + return self._compare(other, lambda s, o: s > o) + + def __ne__(self, other): + return self._compare(other, lambda s, o: s != o) + + +class MessageCodeMapping(ComparableMixin): + def __init__(self, code, message, proto): + self.code = int(code) + self.message = message + self.proto = proto + self.message_code_name = self._message_code_name() + self.module_name = 'riak.pb.{0}_pb2'.format(self.proto) + self.message_class = self._message_class() + + def _cmpkey(self): + return self.code + + def __hash__(self): + return self.code + + def _message_code_name(self): + strip_rpb = re.sub(r"^Rpb", "", self.message) + word = re.sub(r"([A-Z]+)([A-Z][a-z])", r'\1_\2', strip_rpb) + word = re.sub(r"([a-z\d])([A-Z])", r'\1_\2', word) + word = word.replace("-", "_") + return "MSG_CODE_" + word.upper() + + def _message_class(self): + try: + pbmod = __import__(self.module_name, globals(), locals(), + [self.message]) + klass = pbmod.__dict__[self.message] + return klass + except KeyError: + log.warn("Did not find '%s' message class in module '%s'", + self.message, self.module_name) + except ImportError as e: + log.error("Could not import module '%s', exception: %s", + self.module_name, e) + raise + return None + + +# NOTE: TO RUN THIS SUCCESSFULLY, YOU NEED TO HAVE THESE +# PACKAGES INSTALLED: +# protobuf or python3_protobuf +# six +# +# Run the following command to install them: +# python setup.py install +# +# TO DEBUG: Set DISTUTILS_DEBUG=1 in the environment or run as +# 'python setup.py -vv build_messages' +class build_messages(Command): + """ + Generates message code mappings. Add to the build process using:: + + setup(cmd_class={'build_messages': build_messages}) + """ + + description = "generate protocol message code mappings" + + user_options = [ + ('source=', None, 'source CSV file containing message code mappings'), + ('destination=', None, 'destination Python source file') + ] + + # Used in loading and generating + _pb_imports = set() + _messages = set() + _linesep = os.linesep + _indented_item_sep = ',{0} '.format(_linesep) + + _docstring = [ + '' + '# This is a generated file. DO NOT EDIT.', + '', + '"""', + 'Constants and mappings between Riak protocol codes and messages.', + '"""', + '' + ] + + def initialize_options(self): + self.source = None + self.destination = None + self.update_import = None + + def finalize_options(self): + if self.source is None: + self.source = 'riak_pb/src/riak_pb_messages.csv' + if self.destination is None: + self.destination = 'riak/pb/messages.py' + + def run(self): + self.force = True + self.make_file(self.source, self.destination, + self._load_and_generate, []) + + def _load_and_generate(self): + self._format_python2_or_3() + self._load() + self._generate() + + def _load(self): + with open(self.source, 'r', buffering=1) as csvfile: + reader = csv.reader(csvfile) + for row in reader: + message = MessageCodeMapping(*row) + self._messages.add(message) + self._pb_imports.add(message.module_name) + + def _generate(self): + self._contents = [] + self._generate_doc() + self._generate_imports() + self._generate_codes() + self._generate_classes() + write_file(self.destination, self._contents) + + def _generate_doc(self): + # Write the license and docstring header + self._contents.extend(self._docstring) + + def _generate_imports(self): + # Write imports + for im in sorted(self._pb_imports): + self._contents.append("import {0}".format(im)) + + def _generate_codes(self): + # Write protocol code constants + self._contents.extend(['', "# Protocol codes"]) + for message in sorted(self._messages): + self._contents.append("{0} = {1}".format(message.message_code_name, + message.code)) + + def _generate_classes(self): + # Write message classes + classes = [self._generate_mapping(message) + for message in sorted(self._messages)] + + classes = self._indented_item_sep.join(classes) + self._contents.extend(['', + "# Mapping from code to protobuf class", + 'MESSAGE_CLASSES = {', + ' ' + classes, + '}']) + + def _generate_mapping(self, m): + if m.message_class is not None: + klass = "{0}.{1}".format(m.module_name, + m.message_class.__name__) + else: + klass = "None" + pair = "{0}: {1}".format(m.message_code_name, klass) + if len(pair) > 76: + # Try to satisfy PEP8, lulz + pair = (self._linesep + ' ').join(pair.split(' ')) + return pair + + def _format_python2_or_3(self): + """ + Change the PB files to use full pathnames for Python 3.x + and modify the metaclasses to be version agnostic + """ + pb_files = set() + with open(self.source, 'r', buffering=1) as csvfile: + reader = csv.reader(csvfile) + for row in reader: + _, _, proto = row + pb_files.add('riak/pb/{0}_pb2.py'.format(proto)) + + for im in sorted(pb_files): + with open(im, 'r', buffering=1) as pbfile: + contents = 'from six import *\n' + pbfile.read() + contents = re.sub(r'riak_pb2', + r'riak.pb.riak_pb2', + contents) + # Look for this pattern in the protoc-generated file: + # + # class RpbCounterGetResp(_message.Message): + # __metaclass__ = _reflection.GeneratedProtocolMessageType + # + # and convert it to: + # + # @add_metaclass(_reflection.GeneratedProtocolMessageType) + # class RpbCounterGetResp(_message.Message): + contents = re.sub( + r'class\s+(\S+)\((\S+)\):\s*\n' + '\s+__metaclass__\s+=\s+(\S+)\s*\n', + r'@add_metaclass(\3)\nclass \1(\2):\n', contents) + + with open(im, 'w', buffering=1) as pbfile: + pbfile.write(contents) diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index 6af855c1..00000000 --- a/debian/changelog +++ /dev/null @@ -1,6 +0,0 @@ -python-riak (1.2.1) unstable; urgency=low - - * Initial packaging - - -- Dmitry Rozhkov Wed, 27 Apr 2011 20:34:29 +0200 - diff --git a/debian/compat b/debian/compat deleted file mode 100644 index 7f8f011e..00000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -7 diff --git a/debian/control b/debian/control deleted file mode 100644 index bf01c5b2..00000000 --- a/debian/control +++ /dev/null @@ -1,15 +0,0 @@ -Source: python-riak -Section: python -Priority: optional -Maintainer: Basho Technologies -Build-Depends: debhelper (>= 7.0.50~), python-support -Standards-Version: 3.9.1 - -Package: python-riak -Architecture: all -Depends: ${python:Depends}, - ${misc:Depends}, - python-protobuf (>= 2.3.0) -Description: Python client for Riak - Python client for Riak - diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index 202260ca..00000000 --- a/debian/copyright +++ /dev/null @@ -1,17 +0,0 @@ -This package was debianized by Dmitry Rozhkov on -Wed, 27 Apr 2011 20:43:00 +0200. - -Upstream Authors: - Basho Technologies. Inc. - http://wiki.basho.com - -Copyright: - -Licensed to the Apache Software Foundation (ASF) under one or more contributor -license agreements. The ASF licenses this work to You under the Apache License, -Version 2.0 (the "License"); you may not use this work except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -On a Debian system, the license can be found at -/usr/share/common-licenses/Apache-2.0 . diff --git a/debian/rules b/debian/rules deleted file mode 100755 index 6fcad954..00000000 --- a/debian/rules +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/make -f -%: - dh $@ - -override_dh_clean: - dh_clean - rm -rf *.egg-info diff --git a/docs b/docs new file mode 160000 index 00000000..f8f1ae3b --- /dev/null +++ b/docs @@ -0,0 +1 @@ +Subproject commit f8f1ae3b2b8258ed494dec9530683fe29b381cf9 diff --git a/docs/bucket.rst b/docs/bucket.rst deleted file mode 100644 index 297d2b46..00000000 --- a/docs/bucket.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. ref-bucket: - -========== -RiakBucket -========== - -.. currentmodule:: riak.bucket - -.. autoclass:: riak.bucket.RiakBucket diff --git a/docs/client.rst b/docs/client.rst deleted file mode 100644 index dff87c9f..00000000 --- a/docs/client.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. ref-client: - -========== -RiakClient -========== - -.. currentmodule:: riak.client - -.. autoclass:: riak.client.RiakClient diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index f8261594..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,36 +0,0 @@ -Riak Python Client -===================== - -Installation ------------- - -#. Ensure Riak installed & running. (``riak ping``) -#. Install the Python client: - - #. If you use Pip_, ``pip install riak``. - #. If you use easy_install_, run ``easy_install riak``. - #. You can download the package off PyPI_, extract it and run - ``python setup.py install``. - -.. _Pip: http://pip.openplans.org/ -.. _easy_install: http://pypi.python.org/pypi/setuptools -.. _PyPI: http://pypi.python.org/pypi/riak/1.4.0 - -Contents: - -.. toctree:: - :maxdepth: 2 - - tutorial - - client - bucket - riak_object - mapreduce - -Indices and tables ------------------- - -* :ref:`genindex` -* :ref:`search` - diff --git a/docs/mapreduce.rst b/docs/mapreduce.rst deleted file mode 100644 index 19f5bb57..00000000 --- a/docs/mapreduce.rst +++ /dev/null @@ -1,15 +0,0 @@ -.. ref-mapreduce: - -============= -RiakMapReduce -============= - -.. currentmodule:: riak.mapreduce - -.. autoclass:: riak.mapreduce.RiakMapReduce - -.. autoclass:: riak.mapreduce.RiakMapReducePhase - -.. autoclass:: riak.mapreduce.RiakLinkPhase - -.. autoclass:: riak.mapreduce.RiakLink diff --git a/docs/riak_object.rst b/docs/riak_object.rst deleted file mode 100644 index b85fcbb1..00000000 --- a/docs/riak_object.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. ref-riak-object: - -========== -RiakObject -========== - -.. currentmodule:: riak.riak_object - -.. autoclass:: riak.riak_object.RiakObject diff --git a/docs/tutorial.rst b/docs/tutorial.rst deleted file mode 100644 index 2506d114..00000000 --- a/docs/tutorial.rst +++ /dev/null @@ -1,424 +0,0 @@ -.. ref-tutorial: - -======== -Tutorial -======== - -This tutorial assumes basic working knowledge of how Riak works & what it can -do. If you need a more comprehensive overview how to use Riak, please check out -the `Riak Fast Track`_. - -.. _`Riak Fast Track`: http://wiki.basho.com/The-Riak-Fast-Track.html - - -Quick Start -=========== - -For the impatient, simple usage of the official Python binding for Riak looks -like:: - - import riak - - # Connect to Riak. - client = riak.RiakClient() - - # Choose the bucket to store data in. - bucket = client.bucket('test') - - - # Supply a key to store data under. - # The ``data`` can be any data Python's ``json`` encoder can handle. - person = bucket.new('riak_developer_1', data={ - 'name': 'John Smith', - 'age': 28, - 'company': 'Mr. Startup!', - }) - # Save the object to Riak. - person.store() - - -Connecting To Riak -================== - -There are two supported ways to connect to Riak, the HTTP interface & the -`Protocol Buffers`_ interface. Both provide the same API & full access to -Riak. - -The HTTP interface is easier to setup & is well suited for development use. It -is the slower of the two interfaces, but if you are only making a handful of -requests, it is more than capable. - -The Protocol Buffers (also called ``protobuf``) is more difficult to setup but -is significantly faster (2-3x) and is more suitable for production use. This -interface is better suited to a higher number of requests. - -.. _`Protocol Buffers`: http://code.google.com/p/protobuf/ - -To use the HTTP interface and connecting to a local Riak on the default port, -no arguments are needed:: - - import riak - - client = riak.RiakClient() - -The constructor also configuration options such as ``host``, ``http_port``, -``pb_port`` & ``prefix``. Please refer to the :doc:`client` documentation -for full details. - -To use the Protocol Buffers interface:: - - import riak - - client = riak.RiakClient(pb_port=8087, protocol='pbc') - -.. warning: - - Riak's default port is 8098. However, when using the Protocol Buffers, the - Riak listens on port 8087. If you forget this, you will *NOT* get an - immediate error, but will instead receive an error when fetching or storing - data to the effect of ``RiakError: 'Socket returned short read 135 - - expected 8192'``. - -The ``protocol`` argument indicates to the client which backend to use. -We didn't need to specify it in the HTTP example because ``http`` is the -default class. Available options are: ``http``, ``https``, & ``pbc``. - - -Using Buckets -============= - -Buckets in Riak's terminology are segmented keyspaces. They are a way to -categorize different types of data and are roughly analogous to tables in an -RDBMS. - -Once you have a ``client``, selecting a bucket is simple. Provide a string of -the name of the bucket to use:: - - test_bucket = client.bucket('test') - -If the bucket does not exist, Riak will create it for you. You can also open -as many buckets as you need:: - - user_bucket = client.bucket('user') - profile_bucket = client.bucket('profile') - status_bucket = client.bucket('status') - -If needed, you can also manually instantiate a bucket like so:: - - user_bucket = riak.RiakBucket(client, 'user') - -The buckets themselves provide many different methods. The most commonly used -are: - -* ``get`` - Fetches a key's value (decoded from JSON). -* ``get_binary`` - Also fetches a key's raw value (plain text or binary). -* ``new`` - Creates a new key/value pair (encoded in JSON). -* ``new_binary`` - Creates a new key/raw value pair. - -See the full :doc:`bucket` documentation for the other methods. - - -Storing Keys/Values -=================== - -Once you've got a working client/bucket, the next task at hand is storing data. -Riak provides several ways to store your data, but the most common are a -JSON-encoded structure or a binary blob. - -To store JSON-encoded data, you'd do something like the following:: - - import riak - - client = riak.RiakClient() - user_bucket = client.bucket('user') - - # We're creating the user data & keying off their username. - new_user = user_bucket.new('johndoe', data={ - 'first_name': 'John', - 'last_name': 'Doe', - 'gender': 'm', - 'website': 'http://example.com/', - 'is_active': True, - }) - # Note that the user hasn't been stored in Riak yet. - new_user.store() - -Note that any data Python's ``json`` (or ``simplejson``) encoder can handle is -fair game. - -As mentioned, Riak can also handle binary data, such as images, audio files, -etc. Storing binary data looks almost identical:: - - import riak - - client = riak.RiakClient() - user_photo_bucket = client.bucket('user_photo') - - # For example purposes, we'll read a file off the filesystem, but you can get - # the data from anywhere. - the_photo_data = open('/tmp/johndoe_headshot.jpg', 'rb').read() - - # We're storing the photo in a different bucket but keyed off the same - # username. - new_user = user_photo_bucket.new_binary('johndoe', data=the_photo_data, content_type='image/jpeg') - new_user.store() - -You can also manually store data by using ``RiakObject``:: - - import riak - import time - import uuid - - client = riak.RiakClient() - status_bucket = client.bucket('status') - - # We use ``uuid.uuid1().hex`` here to create a unique identifier for the status. - post_uuid = uuid.uuid1().hex - new_status = riak.RiakObject(client, status_bucket, post_uuid) - - # Add in the data you want to store. - new_status.set_data({ - 'message': 'First post!', - 'created': time.time(), - 'is_public': True, - }) - - # Set the content type. - new_status.set_content_type('application/json') - - # We want to do JSON-encoding on the value. - new_status._encode_data = True - - # Again, make sure you save it. - new_status.store() - - -Getting Single Values Out -========================= - -Storing data is all well and good, but you'll need to get that data out at a -later date. - -Riak provides several ways to get data out, though fetching single key/value -pairs is the easiest. Just like storing the data, you can pull the data out -in either the JSON-decoded form or a binary blob. Getting the JSON-decoded -data out looks like:: - - import riak - - client = riak.RiakClient() - user_bucket = client.bucket('user') - - johndoe = user_bucket.get('johndoe') - - # You've now got a ``RiakObject``. To get at the values in a dictionary - # form, call: - johndoe_dict = johndoe.data - -Getting binary data out looks like:: - - import riak - - client = riak.RiakClient() - user_photo_bucket = client.bucket('user_photo') - - johndoe = user_photo_bucket.get_binary('johndoe') - - # You've now got a ``RiakObject``. To get at the binary data, call: - johndoe_headshot = johndoe.data - -Manually fetching data is also possible:: - - import riak - - client = riak.RiakClient() - status_bucket = client.bucket('status') - - # We're using the UUID generated from the above section. - first_post_status = riak.RiakObject(client, status_bucket, post_uuid) - first_post_status._encode_data = True - r = status_bucket.get_r() - - # Calling ``reload`` will cause the ``RiakObject`` instance to load fresh - # data/metadata from Riak. - first_post_status.reload(r) - - # Finally, pull out the data. - message = first_post_status.data['message'] - - -Fetching Data Via Map/Reduce -============================ - -When you need to work with larger sets of data, one of the tools at your -disposal is MapReduce_. This technique iterates over all of the data, returning -data from the map phase & combining all the different maps in the reduce -phase(s). - -.. _MapReduce: http://wiki.basho.com/MapReduce.html - -To perform a map operation, such as returning all active users, you can do -something like:: - - import riak - - client = riak.RiakClient() - # First, you need to ``add`` the bucket you want to MapReduce on. - query = client.add('user') - # Then, you supply a Javascript map function as the code to be executed. - query.map("function(v) { var data = JSON.parse(v.values[0].data); if(data.is_active == true) { return [[v.key, data]]; } return []; }") - - for result in query.run(): - # Print the key (``v.key``) and the value for that key (``data``). - print "%s - %s" % (result[0], result[1]) - - # Results in something like: - # - # mr_smith - {'first_name': 'Mister', 'last_name': 'Smith', 'is_active': True} - # johndoe - {'first_name': 'John', 'last_name': 'Doe', 'is_active': True} - # annabody - {'first_name': 'Anna', 'last_name': 'Body', 'is_active': True} - -You can also do this manually:: - - import riak - - client = riak.RiakClient() - query = riak.RiakMapReduce(client).add('user') - query.map("function(v) { var data = JSON.parse(v.values[0].data); if(data.is_active == true) { return [[v.key, data]]; } return []; }") - - for result in query.run(): - print "%s - %s" % (result[0], result[1]) - -Adding a reduce phase, say to sort by username (key), looks almost identical:: - - import riak - - client = riak.RiakClient() - query = client.add('user') - query.map("function(v) { var data = JSON.parse(v.values[0].data); if(data.is_active == true) { return [[v.key, data]]; } return []; }") - query.reduce("function(values) { return values.sort(); }") - - for result in query.run(): - # Print the key (``v.key``) and the value for that key (``data``). - print "%s - %s" % (result[0], result[1]) - - # Results in something like: - # - # annabody - {'first_name': 'Anna', 'last_name': 'Body', 'is_active': True} - # johndoe - {'first_name': 'John', 'last_name': 'Doe', 'is_active': True} - # mr_smith - {'first_name': 'Mister', 'last_name': 'Smith', 'is_active': True} - - -Working With Related Data Via Links -=================================== - -Links_ are powerful concept in Riak that allow, within the key/value pair's -metadata, relations between objects. - -.. _Links: http://wiki.basho.com/Links.html - -Adding them to your data is relatively trivial. For instance, we'll link a -user's statuses to their user data:: - - import riak - import uuid - - client = riak.RiakClient() - user_bucket = client.bucket('user') - status_bucket = client.bucket('status') - - johndoe = user_bucket.get('johndoe') - - new_status = status_bucket.new(uuid.uuid1().hex, data={ - 'message': 'First post!', - 'created': time.time(), - 'is_public': True, - }) - # Add one direction (from status to user)... - new_status.add_link(johndoe) - new_status.store() - - # ... Then add the other direction. - johndoe.add_link(new_status) - johndoe.store() - -Fetching the data is equally simple:: - - import riak - - client = riak.RiakClient() - user_bucket = client.bucket('user') - - johndoe = user_bucket.get('johndoe') - - for status_link in johndoe.get_links(): - # Since what we get back are lightweight ``RiakLink`` objects, we need to - # get the associated ``RiakObject`` to access its data. - status = status_link.get() - print status.data['message'] - - -Using Search -============ - -`Riak Search`_ is a new feature available as of Riak 0.13. It allows you to create -queries that filter on data in the values without writing a MapReduce. It takes -inspiration from Lucene_, a popular Java-based search library, and incorporates -a Solr-like interface into Riak. The setup of this is outside the realm of this -tutorial, but usage of this feature looks like:: - - import riak - - client = riak.RiakClient() - - # First parameter is the bucket we want to search within, the second - # is the query we want to perform. - search_query = client.search('user', 'first_name:[Anna TO John]') - - for result in search_query.run(): - # You get ``RiakLink`` objects back. - user = result.get() - user_data = user.data - print "%s %s" % (user_data['first_name'], user_data['last_name']) - - # Results in something like: - # - # John Doe - # Anna Body - -.. _`Riak Search`: http://wiki.basho.com/Riak-Search.html -.. _Lucene: http://lucene.apache.org/ - -Using Secondary Indexes -======================= - -Secondary Indexes is a new feature available as of Riak 1.0. It -allows you to tag an object with index metadata, and then later find -the object by querying the metadata, returning a list of matching keys. - -Your Riak cluster must have Secondary Indexes enabled. See the Riak -documentation for details. - -Usage of this feature looks like:: - - import riak - - client = riak.RiakClient() - bucket = client.bucket('mybucket') - - # Create and store the object with indexes... - obj = bucket.new('mykey1', 'mydata') - obj.add_index('field1_bin', 'val1') - obj.add_index('field2_int', 1001) - obj.store() - - # Query the indexes. The return value is a list of ``RiakLink`` objects. - results = client.index('mybucket', 'field1_bin', 'val1').run() - - # Query the indexes using a range... - results = client.index('mybucket', 'field1_bin', 'val1', 'val5').run() - - # Remove an index entry... - obj = bucket.get('mykey1') - obj.remove_index('field1_bin', 'val1') - obj.store() diff --git a/docs/Makefile b/docsrc/Makefile similarity index 100% rename from docs/Makefile rename to docsrc/Makefile diff --git a/docsrc/_templates/layout.html b/docsrc/_templates/layout.html new file mode 100644 index 00000000..61243358 --- /dev/null +++ b/docsrc/_templates/layout.html @@ -0,0 +1,37 @@ +{% extends "!layout.html" %} + +{% block sidebarrel %}{% endblock %} + +{%- block content %} +{{ navBar() }} +
+ + {% block body %}{% endblock %} +
 
+ +
+{%- endblock %} + +{% set css_files = css_files + ['_static/custom.css'] %} diff --git a/docsrc/advanced.rst b/docsrc/advanced.rst new file mode 100644 index 00000000..fe355f88 --- /dev/null +++ b/docsrc/advanced.rst @@ -0,0 +1,218 @@ +========================== +Advanced Usage & Internals +========================== + +This page contains documentation for aspects of library internals that +you will rarely need to interact with, but are important for +understanding how it works and development purposes. + +--------------- +Connection pool +--------------- + +.. currentmodule:: riak.transports.pool + +.. autoclass:: Resource + :members: + +.. autoclass:: Pool + :members: + +.. autoclass:: PoolIterator + +.. autoexception:: BadResource +.. autoexception:: ConnectionClosed + +----------- +Retry logic +----------- + +.. currentmodule:: riak.client.transport + +.. autoclass:: RiakClientTransport + :members: + :private-members: + +.. autofunction:: _is_retryable + +.. autofunction:: retryable + +.. autofunction:: retryableHttpOnly + +------------------- +Multiget / Multiput +------------------- + +.. currentmodule:: riak.client.multi + +.. autodata:: POOL_SIZE + +.. autoclass:: Task +.. autoclass:: PutTask + +.. autoclass:: MultiGetPool + :members: + :private-members: + +.. autofunction:: multiget + +.. autoclass:: MultiPutPool + :members: + :private-members: + +.. autofunction:: multiput + +--------- +Datatypes +--------- + +.. currentmodule:: riak.datatypes + +^^^^^^^^^^^^^^^^^^ +Datatype internals +^^^^^^^^^^^^^^^^^^ + +.. automethod:: Datatype.to_op +.. automethod:: Datatype._check_type +.. automethod:: Datatype._coerce_value +.. automethod:: Datatype._default_value +.. automethod:: Datatype._post_init +.. automethod:: Datatype._require_context +.. autoattribute:: Datatype.type_name +.. autoattribute:: Datatype._type_error_msg + +^^^^^^^^^^^^ +TypedMapView +^^^^^^^^^^^^ + +.. autoclass:: riak.datatypes.map.TypedMapView + :members: + :special-members: + +^^^^^^^^^^^^^^ +TYPES constant +^^^^^^^^^^^^^^ + +.. autodata:: TYPES + +---------- +Transports +---------- + +.. currentmodule:: riak.transports.transport + +.. autoclass:: Transport + :members: + :private-members: + +.. currentmodule:: riak.transports.feature_detect + +.. autoclass:: FeatureDetection + :members: + :private-members: + +^^^^^^^^^^^^^^^^ +Security helpers +^^^^^^^^^^^^^^^^ + +.. currentmodule:: riak.transports.security + +.. autofunction:: verify_cb +.. autofunction:: configure_context + +.. autoclass:: RiakWrappedSocket +.. autoclass:: fileobject + +.. automethod:: riak.security.SecurityCreds._check_revoked_cert +.. automethod:: riak.security.SecurityCreds._has_credential + +^^^^^^^^^^^^^^ +HTTP Transport +^^^^^^^^^^^^^^ + +.. currentmodule:: riak.transports.http + +.. autoclass:: HttpPool + +.. autofunction:: is_retryable + +.. autoclass:: HttpTransport + :members: + +^^^^^^^^^^^^^ +TCP Transport +^^^^^^^^^^^^^ + +.. currentmodule:: riak.transports.tcp + +.. autoclass:: TcpPool + +.. autofunction:: is_retryable + +.. autoclass:: TcpTransport + :members: + +--------- +Utilities +--------- + +^^^^^^^^^^^^^^^^^^ +Link wrapper class +^^^^^^^^^^^^^^^^^^ + +.. autoclass:: riak.mapreduce.RiakLink + +^^^^^^^^^^^^^^^^^ +Multi-valued Dict +^^^^^^^^^^^^^^^^^ + +.. currentmodule:: riak.multidict + +.. autoclass:: MultiDict + + .. automethod:: add + .. automethod:: getall + .. automethod:: getone + .. automethod:: mixed + .. automethod:: dict_of_lists + +^^^^^^^^^^^^^^^^^^ +Micro-benchmarking +^^^^^^^^^^^^^^^^^^ + +.. currentmodule:: riak.benchmark + +.. autofunction:: measure + +.. autofunction:: measure_with_rehearsal + +.. autoclass:: Benchmark + :members: + +^^^^^^^^^^^^^ +Miscellaneous +^^^^^^^^^^^^^ + +.. currentmodule:: riak.util + +.. autofunction:: quacks_like_dict + +.. autofunction:: deep_merge + +.. autofunction:: deprecated + +.. autoclass:: lazy_property + +------------------ +distutils commands +------------------ + +.. automodule:: commands + :members: + :undoc-members: + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Version extraction (``version`` module) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: version diff --git a/docsrc/bucket.rst b/docsrc/bucket.rst new file mode 100644 index 00000000..0c15036f --- /dev/null +++ b/docsrc/bucket.rst @@ -0,0 +1,225 @@ +.. _bucket_types: + +====================== +Buckets & Bucket Types +====================== + +.. currentmodule:: riak.bucket + +**Buckets** are both namespaces for the key-value pairs you store in +Riak, and containers for properties that apply to that namespace. In +older versions of Riak, this was the only logical organization +available. Now a higher-level collection called a **Bucket Type** can +group buckets together. They allow for efficiently setting properties +on a group of buckets at the same time. + +Unlike buckets, Bucket Types must be `explicitly created +`_ +and activated before being used:: + + riak-admin bucket-type create n_equals_1 '{"props":{"n_val":1}}' + riak-admin bucket-type activate n_equals_1 + +Bucket Type creation and activation is only supported via the +``riak-admin bucket-type`` command-line tool. Riak 2.0 does not +include an API to perform these actions, but the Python client *can* +:meth:`retrieve ` and :meth:`set +` bucket-type properties. + +If Bucket Types are not specified, the *default* bucket +type is used. These buckets should be created via the :meth:`bucket() +` method on the client object, like so:: + + import riak + + client = riak.RiakClient() + mybucket = client.bucket('mybucket') + +Buckets with a user-specified Bucket Type can also be created via the same +:meth:`bucket()` method with +an additional parameter or explicitly via +:meth:`bucket_type()`:: + + othertype = client.bucket_type('othertype') + otherbucket = othertype.bucket('otherbucket') + + # Alternate way to get a bucket within a bucket-type + mybucket = client.bucket('mybucket', bucket_type='mybuckettype') + +For more detailed discussion, see `Using Bucket Types +`_. + +-------------- +Bucket objects +-------------- + +.. autoclass:: RiakBucket + + .. attribute:: name + + The name of the bucket, a string. + + .. attribute:: bucket_type + + The parent :class:`BucketType` for the bucket. + + .. autoattribute:: resolver + +----------------- +Bucket properties +----------------- + +Bucket properties are flags and defaults that apply to all keys in the +bucket. + +.. automethod:: RiakBucket.get_properties +.. automethod:: RiakBucket.set_properties +.. automethod:: RiakBucket.clear_properties +.. automethod:: RiakBucket.get_property +.. automethod:: RiakBucket.set_property + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Shortcuts for common properties +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Some of the most commonly-used bucket properties are exposed as object +properties as well. The getters and setters simply call +:meth:`RiakBucket.get_property` and :meth:`RiakBucket.set_property` +respectively. + +.. autoattribute:: RiakBucket.n_val +.. autoattribute:: RiakBucket.allow_mult +.. autoattribute:: RiakBucket.r +.. autoattribute:: RiakBucket.pr +.. autoattribute:: RiakBucket.w +.. autoattribute:: RiakBucket.dw +.. autoattribute:: RiakBucket.pw +.. autoattribute:: RiakBucket.rw + +----------------- +Working with keys +----------------- + +The primary purpose of buckets is to act as namespaces for keys. As +such, you can use the bucket object to create, fetch and delete +:class:`objects `. + +.. automethod:: RiakBucket.new +.. automethod:: RiakBucket.new_from_file +.. automethod:: RiakBucket.get +.. automethod:: RiakBucket.multiget +.. automethod:: RiakBucket.delete + + +---------------- +Query operations +---------------- + +.. automethod:: RiakBucket.search +.. automethod:: RiakBucket.get_index +.. automethod:: RiakBucket.stream_index +.. automethod:: RiakBucket.paginate_index +.. automethod:: RiakBucket.paginate_stream_index + + +------------- +Serialization +------------- + +Similar to :class:`RiakClient `, buckets can +register custom transformation functions for media-types. When +undefined on the bucket, :meth:`RiakBucket.get_encoder` and +:meth:`RiakBucket.get_decoder` will delegate to the client associated +with the bucket. + +.. automethod:: RiakBucket.get_encoder +.. automethod:: RiakBucket.set_encoder +.. automethod:: RiakBucket.get_decoder +.. automethod:: RiakBucket.set_decoder + +------------ +Listing keys +------------ + +Shortcuts for :meth:`RiakClient.get_keys() +` and +:meth:`RiakClient.stream_keys() +` are exposed on the bucket +object. The same admonitions for these operations apply. + +.. automethod:: RiakBucket.get_keys +.. automethod:: RiakBucket.stream_keys + +------------------- +Bucket Type objects +------------------- + +.. autoclass:: BucketType + + .. attribute:: name + + The name of the Bucket Type, a string. + +.. automethod:: BucketType.is_default + +.. automethod:: BucketType.bucket + +---------------------- +Bucket Type properties +---------------------- + +Bucket Type properties are flags and defaults that apply to all buckets in the +Bucket Type. + +.. automethod:: BucketType.get_properties +.. automethod:: BucketType.set_properties +.. automethod:: BucketType.get_property +.. automethod:: BucketType.set_property +.. attribute:: BucketType.datatype + + The assigned datatype for this bucket type, if present. + + :rtype: None or str + +--------------- +Listing buckets +--------------- + +Shortcuts for :meth:`RiakClient.get_buckets() +` and +:meth:`RiakClient.stream_buckets() +` are exposed on the bucket +type object. This is similar to `Listing keys`_ on buckets. + +.. automethod:: BucketType.get_buckets +.. automethod:: BucketType.stream_buckets + +------------------- +Deprecated Features +------------------- + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Shortcuts for Riak Search 1.0 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When Riak Search 1.0 is enabled on the server, you can toggle which +buckets have automatic indexing turned on using the ``search`` bucket +property (and on older versions, the ``precommit`` property). These +methods simplify interacting with that configuration. + +.. automethod:: RiakBucket.search_enabled +.. automethod:: RiakBucket.enable_search +.. automethod:: RiakBucket.disable_search + +^^^^^^^^^^^^^^^ +Legacy Counters +^^^^^^^^^^^^^^^ + +The :meth:`~RiakBucket.get_counter` and +:meth:`~RiakBucket.update_counter`. See :ref:`legacy_counters` for +more details. + +.. warning:: Legacy counters are incompatible with Bucket Types. + +.. automethod:: RiakBucket.get_counter +.. automethod:: RiakBucket.update_counter diff --git a/docsrc/client.rst b/docsrc/client.rst new file mode 100644 index 00000000..f014afd9 --- /dev/null +++ b/docsrc/client.rst @@ -0,0 +1,218 @@ +==================== +Client & Connections +==================== + +To connect to a Riak cluster, you must create a +:py:class:`~riak.client.RiakClient` object. The default configuration +connects to a single Riak node on ``localhost`` with the default +ports. The below instantiation statements are all equivalent:: + + from riak import RiakClient, RiakNode + + RiakClient() + RiakClient(protocol='http', host='127.0.0.1', http_port=8098) + RiakClient(nodes=[{'host':'127.0.0.1','http_port':8098}]) + RiakClient(protocol='http', nodes=[RiakNode()]) + + +.. note:: Connections are not established until you attempt to perform + an operation. If the host or port are incorrect, you will not get + an error raised immediately. + +The client maintains a connection pool behind the scenes, one for each +protocol. Connections are opened as-needed; a random node is selected +when a new connection is requested. + +-------------- +Client objects +-------------- + +.. currentmodule:: riak.client +.. autoclass:: RiakClient + + .. autoattribute:: PROTOCOLS + + Prior to Riak 2.0 the ``'https'`` protocol was also an option, but now + secure connections are handled by the :ref:`security-label` feature. + + .. autoattribute:: protocol + .. autoattribute:: client_id + .. autoattribute:: resolver + .. attribute:: nodes + + The list of :class:`nodes ` that this + client will connect to. It is best not to modify this property + directly, as it is not thread-safe. + +^^^^^ +Nodes +^^^^^ + +The :attr:`nodes ` attribute of ``RiakClient`` objects is +a list of ``RiakNode`` objects. If you include multiple host +specifications in the ``RiakClient`` constructor, they will be turned +into this type. + +.. autoclass:: riak.node.RiakNode + :members: + +^^^^^^^^^^^ +Retry logic +^^^^^^^^^^^ + +Some operations that fail because of network errors or Riak node +failure may be safely retried on another node, and the client will do +so automatically. The items below can be used to configure this +behavior. + +.. autoattribute:: RiakClient.retries + +.. automethod:: RiakClient.retry_count + +.. autodata:: riak.client.transport.DEFAULT_RETRY_COUNT + +----------------------- +Client-level Operations +----------------------- + +Some operations are not scoped by buckets or bucket types and can be +performed on the client directly: + +.. automethod:: RiakClient.ping +.. automethod:: RiakClient.get_buckets +.. automethod:: RiakClient.stream_buckets + +---------------------------------- +Accessing Bucket Types and Buckets +---------------------------------- + +Most client operations are on :py:class:`bucket type objects +`, the :py:class:`bucket objects +` they contain or keys within those buckets. Use the +``bucket_type`` or ``bucket`` methods for creating bucket types and buckets +that will proxy operations to the called client. + +.. automethod:: RiakClient.bucket_type +.. automethod:: RiakClient.bucket + +---------------------- +Bucket Type Operations +---------------------- + +.. automethod:: RiakClient.get_bucket_type_props +.. automethod:: RiakClient.set_bucket_type_props + +----------------- +Bucket Operations +----------------- + +.. automethod:: RiakClient.get_bucket_props +.. automethod:: RiakClient.set_bucket_props +.. automethod:: RiakClient.clear_bucket_props +.. automethod:: RiakClient.get_keys +.. automethod:: RiakClient.stream_keys + +-------------------- +Key-level Operations +-------------------- + +.. automethod:: RiakClient.get +.. automethod:: RiakClient.put +.. automethod:: RiakClient.delete +.. automethod:: RiakClient.multiget +.. automethod:: RiakClient.fetch_datatype +.. automethod:: RiakClient.update_datatype + +-------------------- +Timeseries Operations +-------------------- + +.. automethod:: RiakClient.ts_describe +.. automethod:: RiakClient.ts_get +.. automethod:: RiakClient.ts_put +.. automethod:: RiakClient.ts_delete +.. automethod:: RiakClient.ts_query +.. automethod:: RiakClient.ts_stream_keys + +---------------- +Query Operations +---------------- + +.. automethod:: RiakClient.mapred +.. automethod:: RiakClient.stream_mapred +.. automethod:: RiakClient.get_index +.. automethod:: RiakClient.stream_index +.. automethod:: RiakClient.fulltext_search +.. automethod:: RiakClient.paginate_index +.. automethod:: RiakClient.paginate_stream_index + +----------------------------- +Search Maintenance Operations +----------------------------- + +.. automethod:: RiakClient.create_search_schema +.. automethod:: RiakClient.get_search_schema +.. automethod:: RiakClient.create_search_index +.. automethod:: RiakClient.get_search_index +.. automethod:: RiakClient.delete_search_index +.. automethod:: RiakClient.list_search_indexes + +------------- +Serialization +------------- + +The client supports automatic transformation of Riak responses into +Python types if encoders and decoders are registered for the +media-types. Supported by default are ``application/json`` and +``text/plain``. + +.. autofunction:: default_encoder +.. automethod:: RiakClient.get_encoder +.. automethod:: RiakClient.set_encoder +.. automethod:: RiakClient.get_decoder +.. automethod:: RiakClient.set_decoder + +------------------- +Deprecated Features +------------------- + +^^^^^^^^^^^^^^^^ +Full-text search +^^^^^^^^^^^^^^^^ + +The original version of Riak Search has been replaced by :ref:`yz-label`, +which is full-blown Solr integration with Riak. + +If Riak Search 1.0 is enabled, you can query an index via the bucket's +:meth:`~riak.bucket.RiakBucket.search` method:: + + bucket.enable_search() + bucket.new("one", data={'value':'one'}, + content_type="application/json").store() + + bucket.search('value=one') + +To manually add and remove documents from an index (without an +associated key), use the :class:`~riak.client.RiakClient` +:meth:`~riak.client.RiakClient.fulltext_add` and +:meth:`~riak.client.RiakClient.fulltext_delete` methods directly. + +.. automethod:: RiakClient.fulltext_add +.. automethod:: RiakClient.fulltext_delete + +.. _legacy_counters: + +^^^^^^^^^^^^^^^ +Legacy Counters +^^^^^^^^^^^^^^^ + +The first Data Type introduced in Riak 1.4 were `counters`. These pre-date +:ref:`Bucket Types ` and the current implementation. +Rather than returning objects, the counter operations +act directly on the value of the counter. Legacy counters are deprecated +as of Riak 2.0. Please use :py:class:`~riak.datatypes.Counter` instead. + +.. warning:: Legacy counters are incompatible with Bucket Types. + +.. automethod:: RiakClient.get_counter +.. automethod:: RiakClient.update_counter diff --git a/docs/conf.py b/docsrc/conf.py similarity index 84% rename from docs/conf.py rename to docsrc/conf.py index d8e2e07d..7b5cb000 100644 --- a/docs/conf.py +++ b/docsrc/conf.py @@ -1,9 +1,24 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # -*- coding: utf-8 -*- # # Riak (Python binding) documentation build configuration file, created by # sphinx-quickstart on Sun Nov 21 11:23:53 2010. # -# This file is execfile()d with the current directory set to its containing dir. +# This file is execfile()d with the current directory set to its +# containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. @@ -11,20 +26,29 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os + +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('..')) -# -- General configuration ----------------------------------------------------- +from version import get_version +# -- General configuration ---------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. extensions = ['sphinx.ext.autodoc'] # Add any paths that contain templates here, relative to this directory. @@ -41,16 +65,17 @@ # General information about the project. project = u'Riak Python Client' -copyright = u'2010-2012, Basho Technologies' +copyright = u'2010-2014, Basho Technologies' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # -# The short X.Y version. -version = '1.4.1' # The full version, including alpha/beta/rc tags. -release = '1.4.1' +release = get_version() + +# The short X.Y version. +version = '.'.join(release.split('.')[0:3]) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -66,8 +91,9 @@ # directories to ignore when looking for source files. exclude_patterns = ['_build'] -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True @@ -81,25 +107,24 @@ #show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = 'tango' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] -# -- Options for HTML output --------------------------------------------------- +# -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +# html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# Add any paths that contain custom themes here, relative to this +# directory. # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". @@ -120,7 +145,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -#html_static_path = ['_static'] +# html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. @@ -167,7 +192,7 @@ htmlhelp_basename = 'RiakPythonbindingdoc' -# -- Options for LaTeX output -------------------------------------------------- +# -- Options for LaTeX output ------------------------------------------------ # The paper size ('letter' or 'a4'). #latex_paper_size = 'letter' @@ -176,7 +201,8 @@ #latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). +# (source start file, target name, title, author, +# documentclass [howto/manual]). # latex_documents = [ # ('index', 'RiakPythonbinding.tex', u'Riak (Python binding) Documentation', # u'Daniel Lindsley', 'manual'), @@ -206,7 +232,7 @@ #latex_domain_indices = True -# -- Options for manual page output -------------------------------------------- +# -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). @@ -216,7 +242,7 @@ # ] -# -- Options for Epub output --------------------------------------------------- +# -- Options for Epub output ------------------------------------------------- # Bibliographic Dublin Core info. # epub_title = u'Riak (Python binding)' @@ -256,6 +282,7 @@ #epub_tocdup = True # Autodoc settings -autodoc_default_flags = ['members', 'undoc-members'] -autodoc_member_order = 'bysource' +autodoc_default_flags = ['no-undoc-members'] +autodoc_member_order = 'groupwise' autoclass_content = 'both' +primary_domain = 'py' \ No newline at end of file diff --git a/docsrc/datatypes.rst b/docsrc/datatypes.rst new file mode 100644 index 00000000..2d29a402 --- /dev/null +++ b/docsrc/datatypes.rst @@ -0,0 +1,237 @@ +.. _datatypes: +.. currentmodule:: riak.datatypes + +========== +Data Types +========== + +Traditionally all data stored in Riak was an opaque binary type. Then +in version 1.4 came the introduction of a :ref:`counter +`, the first Convergent Data Type supported in Riak. +In Riak 2.0, several additional Data Types were introduced. Riak +"knows" about these data types, and conflicting writes to them will +converge automatically without presenting :ref:`sibling values +` to the user. + +Here is the list of current Data Types: + + * :py:class:`~riak.datatypes.Counter` increments or decrements + integer values + * :py:class:`~riak.datatypes.Set` allows you to store multiple + distinct opaque binary values against a key + * :py:class:`~riak.datatypes.Map` is a nested, recursive + struct, or associative array. Think of it as a container for + composing ad hoc data structures from multiple Data Types. + Inside a map you may store sets, counters, flags, + registers, and even other maps + * :py:class:`~riak.datatypes.Register` stores binaries + accoring to last-write-wins logic within + :py:class:`~riak.datatypes.Map` + * :py:class:`~riak.datatypes.Flag` is similar to a boolean + and also must be within :py:class:`~riak.datatypes.Map` + +All Data Types must be stored in buckets bearing a +:class:`~riak.bucket.BucketType` that sets the +:attr:`~riak.bucket.BucketType.datatype` property to one of +``"counter"``, ``"set"``, or ``"map"``. Note that the bucket must have +the ``allow_mult`` property set to ``true``. + +These Data Types are stored just like :class:`RiakObjects +`, so size constraints that apply to +normal Riak values apply to Riak Data Types too. + +An in-depth discussion of Data Types, also known as CRDTs, +can be found at `Data Types +`_. + +Examples of using Data Types can be found at +`Using Data Types +`_. + +------------------ +Sending Operations +------------------ + +Riak Data Types provide a further departure from Riak's usual operation, +in that the API is operation-based. Rather than fetching the data structure, +reconciling conflicts, mutating the result, and writing it back, you instead +tell Riak what operations to perform on the Data Type. Here are some example +operations: + + * increment a :class:`Counter` by ``10`` + * add ``'joe'`` to a :class:`Set` + * remove the :class:`Set` field called ``'friends'`` from a :class:`Map` + * enable the prepay :class:`Flag` in a :class:`Map` + +Datatypes can be fetched and created just like +:class:`~riak.riak_object.RiakObject` instances, using +:meth:`RiakBucket.get ` and +:meth:`RiakBucket.new `, except that the +bucket must belong to a bucket-type that has a valid datatype +property. If we have a bucket-type named "social-graph" that has the +datatype `"set"`, we would fetch a :class:`Set` like so:: + + graph = client.bucket_type('social-graph') + graph.datatype # => 'set' + myfollowers = graph.bucket('followers').get('seancribbs') + # => a Set datatype + +Once we have a datatype, we can stage operations against it and then +send those operations to Riak:: + + myfollowers.add('javajolt') + myfollowers.discard('roach') + myfollowers.update() + +While this looks in code very similar to manipulating +:class:`~riak.riak_object.RiakObject` instances, only mutations are +enqueued locally, not the new value. + +--------------------------- +Context and Observed-Remove +--------------------------- + +In order for Riak Data Types to behave well, you must have an opaque +context received from a read when you: + + * :meth:`disable ` a :class:`Flag` + (set it to ``false``) + * remove a field from a :class:`Map` + * :meth:`remove ` an element from a :py:class:`Set` + +The basic rule is "you cannot remove something you haven't seen", and +the context tells Riak what you've actually seen, similar to the +:ref:`vclock` on :class:`~riak.riak_object.RiakObject`. The Python +client handles opaque contexts for you transparently as long as you +fetch before performing one of these actions. + +------------------------ +Datatype abstract class +------------------------ + +.. autoclass:: Datatype + + .. autoattribute:: value + .. autoattribute:: context + .. autoattribute:: modified + +^^^^^^^^^^^^^^^^^^^ +Persistence methods +^^^^^^^^^^^^^^^^^^^ + +.. automethod:: Datatype.reload +.. automethod:: Datatype.update +.. function:: Datatype.store(**params) + + This is an alias for :meth:`~riak.datatypes.Datatype.update`. + +.. automethod:: Datatype.delete +.. automethod:: Datatype.clear + +------- +Counter +------- + +.. autoclass:: Counter + +.. attribute:: Counter.value + + The current value of the counter. + + :rtype: int + +.. automethod:: Counter.increment +.. automethod:: Counter.decrement + +--- +Set +--- + +.. autoclass:: Set + +.. attribute:: Set.value + + An immutable copy of the current value of the set. + + :rtype: frozenset + +.. automethod:: Set.add +.. automethod:: Set.discard + +--- +Map +--- + +.. autoclass:: Map + +.. autoattribute:: Map.value + +.. attribute:: Map.counters + + Filters keys in the map to only those of counter types. Example:: + + map.counters['views'].increment() + del map.counters['points'] + + +.. attribute:: Map.flags + + Filters keys in the map to only those of flag types. Example:: + + map.flags['confirmed'].enable() + del map.flags['attending'] + + +.. attribute:: Map.maps + + Filters keys in the map to only those of map types. Example:: + + map.maps['emails'].registers['home'].set("user@example.com") + del map.maps['spam'] + + +.. attribute:: Map.registers + + Filters keys in the map to only those of register types. Example:: + + map.registers['username'].set_value("riak-user") + del map.registers['access_key'] + +.. attribute:: Map.sets + + Filters keys in the map to only those of set types. Example:: + + map.sets['friends'].add("brett") + del map.sets['favorites'] + +------------------ +Map-only datatypes +------------------ + +Two of the new Data Types may only be embedded in +:py:class:`Map ` objects (in addition to +:py:class:`Map ` itself): + +-------- +Register +-------- + +.. autoclass:: Register + +.. autoattribute:: Register.value +.. automethod:: Register.assign + +---- +Flag +---- + +.. autoclass:: Flag + +.. attribute:: Flag.value + + The current value of the flag. + + :rtype: bool, None + +.. automethod:: Flag.enable +.. automethod:: Flag.disable diff --git a/docsrc/index.rst b/docsrc/index.rst new file mode 100644 index 00000000..612b85b9 --- /dev/null +++ b/docsrc/index.rst @@ -0,0 +1,61 @@ +Riak Python Client +================== + +Tutorial +-------- + +The tutorial documentation has been converted to the `Basho Docs`_ as +the `Taste of Riak: Python`_. The old tutorial_ that used to live here +has been moved to the `Github Wiki`_ and is likely out-of-date. + +.. _`Basho Docs`: http://docs.basho.com/ +.. _`Taste of Riak: Python`: + http://docs.basho.com/riak/latest/dev/taste-of-riak/python/ +.. _tutorial: + https://github.com/basho/riak-python-client/wiki/Tutorial-%28old%29 +.. _`Github Wiki`: https://github.com/basho/riak-python-client/wiki + +Installation +------------ + +#. Ensure Riak installed & running. (``riak ping``) +#. Install the Python client: + + #. If you use Pip_, ``pip install riak``. + #. If you use easy_install_, run ``easy_install riak``. + #. You can download the package off PyPI_, extract it and run + ``python setup.py install``. + +.. _Pip: http://pip.openplans.org/ +.. _easy_install: http://pypi.python.org/pypi/setuptools +.. _PyPI: http://pypi.python.org/pypi/riak/ + +Development +----------- + +All development is done on Github_. Use Issues_ to report +problems or submit contributions. + +.. _Github: https://github.com/basho/riak-python-client/ +.. _Issues: https://github.com/basho/riak-python-client/issues + + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`search` + +Contents +-------- + +.. toctree:: + :maxdepth: 1 + + client + bucket + object + datatypes + query + security + advanced diff --git a/docs/make.bat b/docsrc/make.bat similarity index 100% rename from docs/make.bat rename to docsrc/make.bat diff --git a/docsrc/object.rst b/docsrc/object.rst new file mode 100644 index 00000000..b4da6d66 --- /dev/null +++ b/docsrc/object.rst @@ -0,0 +1,140 @@ +================ +Values & Objects +================ + +.. currentmodule:: riak.riak_object + +Keys in Riak are namespaced into :class:`buckets +`, and their associated values are represented +by :class:`objects `, not to be confused with Python +"objects". A :class:`RiakObject` is a container for the key, the +:ref:`vclock`, the value(s) and any metadata associated with the +value(s). + +Values may also be :class:`datatypes `, but +are not discussed here. + +---------- +RiakObject +---------- + +.. autoclass:: RiakObject + + .. attribute:: key + + The key of this object, a string. If not present, the server + will generate a key the first time this object is stored. + + .. attribute:: bucket + + The :class:`bucket ` to which this + object belongs. + + .. autoattribute:: resolver + .. attribute:: vclock + + The :ref:`vclock` for this object. + + .. autoattribute:: exists + +.. _vclock: + +^^^^^^^^^^^^ +Vector clock +^^^^^^^^^^^^ + +Vector clocks are Riak's means of tracking the relationships between +writes to a key. It is best practice to fetch the latest version of a +key before attempting to modify or overwrite the value; if you do not, +you may create :ref:`siblings` or lose data! The content of a vector +clock is essentially opaque to the user. + +.. autoclass:: VClock + +----------- +Persistence +----------- + +Fetching, storing, and deleting keys are the bread-and-butter of Riak. + +.. automethod:: RiakObject.store +.. automethod:: RiakObject.reload +.. automethod:: RiakObject.delete + +.. _object_accessors: + +------------------ +Value and Metadata +------------------ + +Unless you have enabled :ref:`siblings` via the :attr:`allow_mult +` bucket property, you can +inspect and manipulate the value and metadata of an object directly using these +properties and methods: + +.. autoattribute:: RiakObject.data +.. autoattribute:: RiakObject.encoded_data +.. autoattribute:: RiakObject.content_type +.. autoattribute:: RiakObject.charset +.. autoattribute:: RiakObject.content_encoding +.. autoattribute:: RiakObject.last_modified +.. autoattribute:: RiakObject.etag +.. autoattribute:: RiakObject.usermeta +.. autoattribute:: RiakObject.links +.. autoattribute:: RiakObject.indexes +.. automethod:: RiakObject.add_index +.. automethod:: RiakObject.remove_index +.. automethod:: RiakObject.set_index +.. automethod:: RiakObject.add_link + +.. _siblings: + +-------- +Siblings +-------- + +Because Riak's consistency model is "eventual" (and not linearizable), +there is no way for it to disambiguate writes that happen +concurrently. The :ref:`vclock` helps establish a +"happens after" relationships so that concurrent writes can be +detected, but with the exception of :ref:`datatypes`, Riak has no way +to determine which write has the correct value. + +Instead, when :attr:`allow_mult ` +is ``True``, Riak keeps all writes that appear to be concurrent. Thus, +the contents of a key's value may, in fact, be multiple values, which +are called "siblings". Siblings are modeled in :class:`RiakContent +` objects, which contain all of the same +:ref:`object_accessors` methods and attributes as the parent object. + +.. autoattribute:: RiakObject.siblings + +.. autoclass:: riak.content.RiakContent + +You do not typically have to create :class:`RiakContent +` objects yourself, but they will be created +for you when :meth:`fetching ` objects from Riak. + +.. note:: The :ref:`object_accessors` accessors on :class:`RiakObject` + are actually proxied to the first sibling when the object has only + one. + + +^^^^^^^^^^^^^^^^^^^^^^^ +Conflicts and Resolvers +^^^^^^^^^^^^^^^^^^^^^^^ + +When an object is *not* in conflict, it has only one sibling. When it +is in conflict, you will have to resolve the conflict before it can be +written again. How you choose to resolve the conflict is up to you, +but you can automate the process using a :attr:`resolver +` function. + +.. autofunction:: riak.resolver.default_resolver +.. autofunction:: riak.resolver.last_written_resolver + +If you do not supply a resolver function, or your resolver leaves +multiple siblings present, accessing the :ref:`object_accessors` will +result in a :exc:`ConflictError ` being raised. + +.. autoexception:: riak.ConflictError diff --git a/docsrc/query.rst b/docsrc/query.rst new file mode 100644 index 00000000..e8aaab9d --- /dev/null +++ b/docsrc/query.rst @@ -0,0 +1,438 @@ +============= +Query Methods +============= + +Although most operations you will do involve directly interacting with +known buckets and keys, there are additional ways to get information +out of Riak. + +----------------- +Secondary Indexes +----------------- + +:ref:`Objects ` can be :meth:`tagged +` with :attr:`secondary index +entries `. Those entries can then +be queried over :meth:`the bucket ` +for equality or across ranges.:: + + bucket = client.bucket("index_test") + + # Tag an object with indexes and save + sean = bucket.new("seancribbs") + sean.add_index("fname_bin", "Sean") + sean.add_index("byear_int", 1979) + sean.store() + + # Performs an equality query + seans = bucket.get_index("fname_bin", "Sean") + + # Performs a range query + eighties = bucket.get_index("byear_int", 1980, 1989) + +Secondary indexes are also available via :meth:`MapReduce +`. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Streaming and Paginating Indexes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes the number of results from such a query is too great to +process in one payload, so you can also :meth:`stream the results +`:: + + for keys in bucket.stream_index("bmonth_int", 1): + # keys is a list of matching keys + print(keys) + +Both the regular :meth:`~riak.bucket.RiakBucket.get_index` method and +the :meth:`~riak.bucket.RiakBucket.stream_index` method allow you to +return the index entry along with the matching key as tuples using the +``return_terms`` option:: + + bucket.get_index("byear_int", 1970, 1990, return_terms=True) + # => [(1979, 'seancribbs')] + +You can also limit the number of results using the ``max_results`` +option, which enables pagination:: + + results = bucket.get_index("fname_bin", "S", "T", max_results=20) + +Optionally you can use :meth:`~riak.bucket.RiakBucket.paginate_index` +or :meth:`~riak.bucket.RiakBucket.paginate_stream_index` to create a +generator of paged results:: + + for page in bucket.paginate_stream_index("maestro_bin", "Cribbs"): + for key in page: + do_something(key) + page.close() + +All of these features are implemented using the +:class:`~riak.client.index_page.IndexPage` class, which emulates a +list but also supports streaming and capturing the +:attr:`~riak.client.index_page.IndexPage.continuation`, which is a +sort of pointer to the next page of results:: + + # Detect whether there are more results + if results.has_next_page(): + + # Fetch the next page of results manually + more = bucket.get_index("fname_bin", "S", "T", max_results=20, + continuation=results.continuation) + + # Fetch the next page of results automatically + more = results.next_page() + +.. currentmodule:: riak.client.index_page + +.. autoclass:: IndexPage + + .. autoattribute:: continuation + .. automethod:: has_next_page + .. automethod:: next_page + .. automethod:: __eq__ + .. automethod:: __iter__ + .. automethod:: __getitem__ + +--------- +MapReduce +--------- + +.. currentmodule:: riak.mapreduce + +:class:`RiakMapReduce` allows you to construct query-processing jobs that +are performed mostly in-parallel around the Riak cluster. You can +think of it as a pipeline, where inputs are fed in one end, they pass +through a number of ``map`` and ``reduce`` phases, and then are +returned to the client. + +^^^^^^^^^^^^^^^^^^^^^^ +Constructing the query +^^^^^^^^^^^^^^^^^^^^^^ + +.. autoclass:: RiakMapReduce + +^^^^^^ +Inputs +^^^^^^ + +The first step is to identify the inputs that should be processed. +They can be: + +#. An entire :meth:`bucket ` +#. An entire bucket, with the :meth:`keys filtered by criteria ` +#. A :meth:`list of bucket/key pairs ` or bucket/key/data triples +#. A :meth:`fulltext search query ` +#. A :meth:`secondary-index query ` + +Adding inputs always returns the ``RiakMapReduce`` object so that you +can chain the construction of the query job. + +.. automethod:: RiakMapReduce.add_bucket +.. automethod:: RiakMapReduce.add_key_filters +.. automethod:: RiakMapReduce.add_key_filter + +.. automethod:: RiakMapReduce.add +.. automethod:: RiakMapReduce.add_object +.. automethod:: RiakMapReduce.add_bucket_key_data + +.. automethod:: RiakMapReduce.search +.. automethod:: RiakMapReduce.index + +.. autoclass:: RiakKeyFilter + +^^^^^^ +Phases +^^^^^^ + +The second step is to add processing phases to the query. ``map`` +phases load and process individual keys, returning one or more +results, while ``reduce`` phases operate over collections of results +from previous phases. ``link`` phases are a special type of ``map`` +phase that extract matching :attr:`~riak.riak_object.RiakObject.links` +from the object, usually so they can be used in a subsequent ``map`` +phase. + +Any number of phases can return results directly to the client by +passing ``keep=True``. + +.. automethod:: RiakMapReduce.map +.. automethod:: RiakMapReduce.reduce +.. automethod:: RiakMapReduce.link + +.. autoclass:: RiakMapReducePhase + +.. autoclass:: RiakLinkPhase + +""""""""""""""" +Phase shortcuts +""""""""""""""" + +A number of commonly-used phases are also available as shortcut +methods: + +.. automethod:: RiakMapReduce.map_values +.. automethod:: RiakMapReduce.map_values_json +.. automethod:: RiakMapReduce.reduce_sum +.. automethod:: RiakMapReduce.reduce_min +.. automethod:: RiakMapReduce.reduce_max +.. automethod:: RiakMapReduce.reduce_sort +.. automethod:: RiakMapReduce.reduce_numeric_sort +.. automethod:: RiakMapReduce.reduce_limit +.. automethod:: RiakMapReduce.reduce_slice +.. automethod:: RiakMapReduce.filter_not_found + +^^^^^^^^^ +Execution +^^^^^^^^^ + +Query results can either be executed in one round-trip, or streamed +back to the client. The format of results will depend on the structure +of the ``map`` and ``reduce`` phases the query contains. + +.. automethod:: RiakMapReduce.run +.. automethod:: RiakMapReduce.stream + +^^^^^^^^^^^^^^^^^^^^^ +Shortcut constructors +^^^^^^^^^^^^^^^^^^^^^ + +:class:`~riak.riak_object.RiakObject` contains some shortcut methods +that make it more convenient to begin constructing +:class:`RiakMapReduce` queries. + +.. currentmodule:: riak.riak_object + +.. automethod:: RiakObject.add +.. automethod:: RiakObject.link +.. automethod:: RiakObject.map +.. automethod:: RiakObject.reduce + +.. _yz-label: + +-------------------------- +Riak Search 2.0 (Yokozuna) +-------------------------- + +With Riak 2.0 came the introduction of **Riak Search 2.0**, a.k.a `Yokozuna` +(the top rank in sumo). Riak Search 2.0 is an integration of Solr (for +indexing and querying) and Riak (for storage and distribution). +It allows for distributed, scalable, +fault-tolerant, transparent indexing and querying of Riak values. +After connecting a bucket (or bucket type) to a +`Apache Solr `_ index, you simply write +values (such as JSON, XML, plain text, Data Types, etc.) into Riak as +normal, and then query those indexed values using the Solr API. +Unlike traditional Riak data, however, Solr needs to know the format +of the stored data so it can index it. Solr is a document-based +search engine so it treats each value stored in Riak as a document. + +^^^^^^^^^^^^^^^^^ +Creating a schema +^^^^^^^^^^^^^^^^^ + +The first thing which needs to be done is to define a Solr schema for +your data. Riak Search comes bundled with a default schema named +``_yz_default``. It defaults to many dynamic field types, where the +suffix defines its type. This is an easy path to start development, +but we recommend in production that you define your own schema. + +You can find information about defining your own schema at +`Search Schema +`_, +with a short section dedicated to the `default schema +`_. + +Here is a brief example of creating a custom schema with +:meth:`~riak.client.RiakClient.create_search_schema`:: + + content = """ + + + + + + + + + + + + + _yz_id + + + + """ + schema_name = 'jalapeno' + client.create_search_schema(schema_name, content) + +If you would like to retrieve the current XML Solr schema, +:meth:`~riak.client.RiakClient.get_search_schema` is available:: + + schema = client.get_search_schema('jalapeno') + +^^^^^^^^^^^^ +Solr indexes +^^^^^^^^^^^^ + +Once a schema has been created, then a Solr index must also be created. +This index represents a collection of similar data that you use to perform +queries. When creating an index with +:meth:`~riak.client.RiakClient.create_search_index`, you can optionally +specify a schema. If you do not, the default schema will be used:: + + client.create_search_index('nacho') + +Likewise you can specify a schema, e.g. the index ``"nacho"`` is +associated with the schema ``"jalapeno"``:: + + client.create_search_index('nacho', 'jalapeno') + +Just as easily you can delete an index with +:meth:`~riak.client.RiakClient.delete_search_index`:: + + client.delete_search_index('jalapeno') + +A single index can be retrieved with +:meth:`~riak.client.RiakClient.get_search_index` or all of them +with :meth:`~riak.client.RiakClient.list_search_indexes`:: + + index = client.get_search_index('jalapeno') + name = index['name'] + schema = index['schema'] + indexes = client.list_search_indexes() + first_nval = indexes[0]['n_val'] + +.. note:: Note that index names may only be ASCII values from 32-127 + (spaces, standard punctuation, digits and word characters). + This may change in the future to allow full unicode support. + +More discussion about Riak Search 2.0 Indexes can be found at `Indexes +`_. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Linking a bucket type to an index +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The last step to setting up Riak Search 2.0 is to link a Bucket Type +to a Solr index. This lets Riak know when to index values. This can be +done via the command line:: + + riak-admin bucket-type create spicy '{"props":{"search_index":"jalapeno"}}' + riak-admin bucket-type activate spicy + +Or simply create an empty Bucket Type:: + + riak-admin bucket-type create spicy '{"props":{}}' + riak-admin bucket-type activate spicy + +Then change the bucket properties on the associated bucket or Bucket Type:: + + b = client.bucket('peppers') + b.set_property('search_index', 'jalapeno') + btype = client.bucket_type('spicy') + btype.set_property('search_index', 'jalapeno') + +^^^^^^^^^^^^^^^^^ +Querying an index +^^^^^^^^^^^^^^^^^ + +Once the schema, index and bucket properties have all been properly configured, +adding data is as simple as writing to Riak. Solr is automatically updated. + +To query, on the other hand, is as easy as writing Solr queries. This allows +for the full use of existing Solr tools as well as its rich semantics. + +Here is a brief example of loading and querying data::: + + bucket = self.client.bucket('peppers') + bucket.new("bell", {"name_s": "bell", "scoville_low_i": 0, + "scoville_high_i": 0}).store() + bucket.new("anaheim", {"name_s": "anaheim", "scoville_low_i": 1000, + "scoville_high_i": 2500}).store() + bucket.new("chipotle", {"name_s": "chipotle", "scoville_low_i": 3500, + "scoville_high_i": 10000}).store() + bucket.new("serrano", {"name_s": "serrano", "scoville_low_i": 10000, + "scoville_high_i": 23000}).store() + bucket.new("habanero", {"name_s": "habanero", "scoville_low_i": 100000, + "scoville_high_i": 350000}).store() + results = bucket.search("name_s:/c.*/", index='jalapeno') + # Yields single document 'chipotle' + print(results['docs'][0]['name_s']) + results = bucket.search("scoville_high_i:[20000 TO 500000]") + # Yields two documents + for result in results['docs']: + print(result['name_s']) + results = bucket.search('name_s:*', index='jalapeno', + sort="scoville_low_i desc") + # Yields all documents, sorted in descending order. We take the top one + print("The hottest pepper is {0}".format(results['docs'][0]['name_s'])) + +The results returned by :meth:`~riak.bucket.RiakBucket.search` is a dictionary +with lots of search metadata like the number of results, the maxium +`Lucene Score +`_ +as well as the matching documents. + +When querying on :ref:`datatypes` the datatype is the name of the field +used in Solr since they do not fit into the default schema, e.g.: + +.. code:: + + riak-admin bucket-type create visitors '{"props":{"datatype": "counter}}' + riak-admin bucket-type activate visitors + +.. code:: python + + client.create_search_index('website') + bucket = client.bucket_type('visitors').bucket('hits') + bucket.set_property('search_index', 'website') + + site = bucket.new('bbc.co.uk') + site.increment(80) + site.store() + site = bucket.new('cnn.com') + site.increment(150) + site.store() + site = bucket.new('abc.net.au') + site.increment(24) + site.store() + + results = bucket.search("counter:[10 TO *]", index='website', + sort="counter desc", rows=5) + + # Assume you have a bucket-type named "profiles" that has datatype + # "map". Let's create and search an index containing maps. + client.create_search_index('user-profiles') + bucket = client.bucket_type('profiles').bucket('USA') + bucket.set_property('search_index', 'user-profiles') + + brett = bucket.new() + brett.registers['fname'].assign("Brett") + brett.registers['lname'].assign("Hazen") + brett.sets['emails'].add('spam@basho.com') + brett.counters['visits'].increment() + brett.maps['pages'].counters['homepage'].increment() + brett.update() + + # Note that the field name in the index/schema is the field name in + # the map joined with its type by an underscore. Deeply embedded + # fields are joined with their parent field names by an underscore. + results = bucket.search('lname_register:Hazen AND pages_map_homepage_counter:[1 TO *]', + index='user-profiles') + + +Details on querying Riak Search 2.0 can be found at `Querying +`_. + diff --git a/docsrc/security.rst b/docsrc/security.rst new file mode 100644 index 00000000..b76b82cc --- /dev/null +++ b/docsrc/security.rst @@ -0,0 +1,224 @@ +.. _security-label: + +.. currentmodule:: riak.security + +======== +Security +======== + +Riak 2.0 supports authentication and authorization over encrypted +channels via OpenSSL. This is useful to prevent accidental collisions +between environments (e.g., pointing application software under active +development at the production cluster) and offers protection against +some malicious attacks, although Riak still should not be exposed +directly to any unsecured network. + +Several important caveats when enabling security: + +* There is no support yet for auditing. This is on the roadmap for a future + release. +* Two deprecated features will not work if security is enabled: link + walking and Riak Search 1.0. +* There are restrictions on Erlang modules exposed to MapReduce jobs when + security is enabled. +* Enabling security requires applications be designed to transition + gracefully based on the server response or applications will need to be + halted before security is enabled and brought back online with support + for the new security features. + +-------------------- +Server Configuration +-------------------- + +The server must first be configured to `enable security +`_, +users and `security sources +`_ +must be created, `permissions +`_ +applied and the correct certificates must be installed. An overview +can be found at `Authentication and Authorization +`_. + +-------------------- +Client Configuration +-------------------- + +.. note:: OpenSSL 1.0.1g or later (or patched version built after + 2014-04-01) is required for `pyOpenSSL + `_, which is used + for secure transport in the Riak client. Earlier versions + may not support TLS 1.2, the recommended security protocol. + +On the client, simply create a +:class:`SecurityCreds` object with just a username, +password and CA Certificate file. That would then need to be passed +into the :class:`~riak.client.RiakClient` initializer:: + + creds = SecurityCreds('riakuser', + 'riakpass', + cacert_file='/path/to/ca.crt') + client = RiakClient(credentials=creds) + +The ``credentials`` argument of a :class:`~riak.client.RiakClient` constructor +is a :class:`SecurityCreds` object. If you specify a dictionary +instead, it will be turned into this type:: + + creds = {'username': 'riakuser', + 'password': 'riakpass', + 'cacert_file': '/path/to/ca.crt'} + client = RiakClient(credentials=creds) + +.. note:: A Certifying Authority (CA) Certificate must always be + supplied to :class:`SecurityCreds` by specifying the path to + a CA certificate file via the ``cacert_file`` argument or by + setting the ``cacert`` argument to an `OpenSSL.crypto.X509 + `_ + object. This mitigates MITM (man-in-the-middle) attacks by + ensuring correct certificate validation. + +-------------------- +Authentication Types +-------------------- + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Trust and PAM Authentication +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The most basic authentication would be `Trust-based Authentication +`_ +which is done exclusively on the server side by adding the appropriate +``trust`` security source: + +.. code:: bash + + riak-admin security add-source all 127.0.0.1/32 trust + +`PAM-based Authentication +`_ +is another server-side solution which can be added by a ``pam`` +security source with the name of the service: + +.. code:: bash + + riak-admin security add-source all 127.0.0.1/32 pam service=riak_pam + +Even if you are using Trust authentication or the PAM module doesn't +require a password, you must supply one to the client API. From the +client's perspective, these are equivalent to Password authentication. + +^^^^^^^^^^^^^^^^^^^^^^^ +Password Authentication +^^^^^^^^^^^^^^^^^^^^^^^ + +The next level of security would be simply a username and password for +`Password-based Authentication +`_. +The server needs to first have a user and a ``password`` security source: + +.. code:: bash + + riak-admin security add-user riakuser password=captheorem4life + riak-admin security add-source riakuser 127.0.0.1/32 password + +On the client, simply create a :class:`~SecurityCreds` object or dict +with just a username and password. That would then need to be passed +into the :class:`~riak.client.RiakClient` initializer:: + + creds = {'username': 'riakuser', + 'password': 'riakpass', + 'cacert_file': '/path/to/ca.crt'} + client = RiakClient(credentials=creds) + myBucket = client.bucket('test') + val1 = "#SeanCribbsHoldingThings" + key1 = myBucket.new('hashtag', data=val1) + key1.store() + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Client Certificate Authentication +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you are using the **Protocol Buffers** transport you could also add +a layer of security by using `Certificate-based Authentication +`_. +This time the server requires a ``certificate`` security source:: + + riak-admin security add-source riakuser 127.0.0.1/32 certificate + +When the ``certificate`` source is used, the Riak username must match +the common name, aka ``CN``, that you specified when you generated your +certificate. You can add a ``certificate`` source to any number of clients. + +The :class:`SecurityCreds` must then include the include a client +certificate file and a private key file, too:: + + creds = {'username': 'riakuser', + 'password': 'riakpass', + 'cacert_file': '/path/to/ca.crt', + 'cert_file': '/path/to/client.crt', + 'pkey_file': '/path/to/client.key'} + +.. note:: Username and password are still required for certificate-based + authentication, although the password is ignored. + +Optionally, the certificate or private key may be supplied as a string:: + + with open('/path/to/client.key', 'r') as f: + preloaded_pkey = f.read() + with open('/path/to/client.crt', 'r') as f: + preloaded_cert = f.read() + creds = {'username': 'riakuser', + 'password': 'riakpass', + 'cert': preloaded_cert, + 'pkey': prelocated_pkey} + +------------------ +Additional options +------------------ + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Certificate revocation lists +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Another security option available is a Certificate Revocation List (CRL). +It lists server certificates which, for whatever reason, are no longer +valid. For example, it is discovered that the certificate authority (CA) +had improperly issued a certificate, or if a private-key is thought to +have been compromised. The most common reason for revocation is the user +no longer being in sole possession of the private key (e.g., the token +containing the private key has been lost or stolen):: + + creds = {'username': 'riakuser', + 'password': 'riakpass', + 'cacert_file': '/path/to/ca.crt', + 'crl_file': '/path/to/server.crl'} + +^^^^^^^^^^^^^^ +Cipher options +^^^^^^^^^^^^^^ + +The last interesting setting on :class:`SecurityCreds` is the +``ciphers`` option which is a colon-delimited list of supported +ciphers for encryption:: + + creds = {'username': 'riakuser', + 'password': 'riakpass', + 'ciphers': 'ECDHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA'} + +A more detailed discussion can be found at `Security Ciphers +`_. + +-------------------- +SecurityCreds object +-------------------- + +.. autoclass:: SecurityCreds + + .. autoattribute:: username + .. autoattribute:: password + .. autoattribute:: cacert + .. autoattribute:: crl + .. autoattribute:: cert + .. autoattribute:: pkey + .. autoattribute:: ciphers + .. autoattribute:: ssl_version diff --git a/make.ps1 b/make.ps1 new file mode 100644 index 00000000..6d3c4181 --- /dev/null +++ b/make.ps1 @@ -0,0 +1,20 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$env:RIAK_TEST_HOST = 'riak-test' +$env:RIAK_TEST_PROTOCOL = 'pbc' +$env:RIAK_TEST_PB_PORT = 10017 +$env:RUN_DATATYPES = 1 +$env:RUN_INDEXES = 1 +$env:RUN_POOL = 1 +$env:RUN_YZ = 1 + +flake8 --exclude=riak/pb riak commands.py setup.py version.py +if ($LastExitCode -ne 0) { + throw 'flake8 failed!' +} + +python setup.py test +if ($LastExitCode -ne 0) { + throw 'python tests failed!' +} diff --git a/riak/__init__.py b/riak/__init__.py index ba50fc00..306cf7a0 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -1,58 +1,46 @@ -""" -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---- +""" The Riak API for Python allows you to connect to a Riak instance, create, modify, and delete Riak objects, add and remove links from Riak objects, run Javascript (and Erlang) based Map/Reduce operations, and run Linkwalking operations. - -See the unit_tests.py file for example usage. - -@author Rusty Klophaus (@rklophaus) (rusty@basho.com) -@author Andy Gross (@argv0) (andy@basho.com) -@author Jon Meredith (@jmeredith) (jmeredith@basho.com) -@author Jay Baird (@skatterbean) (jay@mochimedia.com) """ -__all__ = ['RiakBucket', 'RiakNode', 'RiakObject', 'RiakClient', - 'RiakMapReduce', 'RiakKeyFilter', 'RiakLink', 'RiakError', - 'ONE', 'ALL', 'QUORUM', 'key_filter'] +from riak.riak_error import RiakError, ConflictError, ListError +from riak.client import RiakClient +from riak.bucket import RiakBucket, BucketType +from riak.table import Table +from riak.node import RiakNode +from riak.riak_object import RiakObject +from riak.mapreduce import RiakKeyFilter, RiakMapReduce, RiakLink -class RiakError(Exception): - """ - Base class for exceptions generated in the Riak API. - """ - def __init__(self, value): - self.value = value - - def __str__(self): - return repr(self.value) - -from client import RiakClient -from bucket import RiakBucket -from node import RiakNode -from riak_object import RiakObject -from mapreduce import RiakKeyFilter, RiakMapReduce, RiakLink +__all__ = ['RiakBucket', 'Table', 'BucketType', 'RiakNode', + 'RiakObject', 'RiakClient', 'RiakMapReduce', 'RiakKeyFilter', + 'RiakLink', 'RiakError', 'ConflictError', 'ListError', + 'ONE', 'ALL', 'QUORUM', 'key_filter', + 'disable_list_exceptions'] ONE = "one" ALL = "all" QUORUM = "quorum" key_filter = RiakKeyFilter() + +""" +Set to true to allow listing operations +""" +disable_list_exceptions = False diff --git a/riak/benchmark.py b/riak/benchmark.py new file mode 100644 index 00000000..e1f3e55c --- /dev/null +++ b/riak/benchmark.py @@ -0,0 +1,177 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +import gc +import sys +import traceback + +__all__ = ['measure', 'measure_with_rehearsal'] + + +def measure_with_rehearsal(): + """ + Runs a benchmark when used as an iterator, injecting a garbage + collection between iterations. Example:: + + for b in riak.benchmark.measure_with_rehearsal(): + with b.report("pow"): + for _ in range(10000): + math.pow(2,10000) + with b.report("factorial"): + for i in range(100): + math.factorial(i) + """ + return Benchmark(True) + + +def measure(): + """ + Runs a benchmark once when used as a context manager. Example:: + + with riak.benchmark.measure() as b: + with b.report("pow"): + for _ in range(10000): + math.pow(2,10000) + with b.report("factorial"): + for i in range(100): + math.factorial(i) + """ + return Benchmark() + + +class Benchmark(object): + """ + A benchmarking run, which may consist of multiple steps. See + measure_with_rehearsal() and measure() for examples. + """ + def __init__(self, rehearse=False): + """ + Creates a new benchmark reporter. + + :param rehearse: whether to run twice to take counter the effects + of garbage collection + :type rehearse: boolean + """ + self.rehearse = rehearse + if rehearse: + self.count = 2 + else: + self.count = 1 + self._report = None + + def __enter__(self): + if self.rehearse: + raise ValueError("measure_with_rehearsal() cannot be used in with " + "statements, use measure() or the for..in " + "statement") + print_header() + self._report = BenchmarkReport() + self._report.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._report: + return self._report.__exit__(exc_type, exc_val, exc_tb) + else: + print + return True + + def __iter__(self): + return self + + def next(self): + """ + Runs the next iteration of the benchmark. + """ + if self.count == 0: + raise StopIteration + elif self.count > 1: + print_rehearsal_header() + else: + if self.rehearse: + gc.collect() + print("-" * 59) + print() + print_header() + + self.count -= 1 + return self + + def __next__(self): + # Python 3.x Version + return self.next() + + def report(self, name): + """ + Returns a report for the current step of the benchmark. + """ + self._report = None + return BenchmarkReport(name) + + +def print_rehearsal_header(): + """ + Prints the header for the rehearsal phase of a benchmark. + """ + print + print("Rehearsal -------------------------------------------------") + + +def print_report(label, user, system, real): + """ + Prints the report of one step of a benchmark. + """ + print("{:<12s} {:12f} {:12f} ( {:12f} )".format(label, + user, + system, + real)) + + +def print_header(): + """ + Prints the header for the normal phase of a benchmark. + """ + print("{:<12s} {:<12s} {:<12s} ( {:<12s} )" + .format('', 'user', 'system', 'real')) + + +class BenchmarkReport(object): + """ + A labeled step in a benchmark. Acts as a context-manager, printing + its timing results when the context exits. + """ + def __init__(self, name='benchmark'): + self.name = name + self.start = None + + def __enter__(self): + self.start = os.times() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not exc_type: + user1, system1, _, _, real1 = self.start + user2, system2, _, _, real2 = os.times() + print_report(self.name, user2 - user1, system2 - system1, + real2 - real1) + elif exc_type is KeyboardInterrupt: + return False + else: + msg = "EXCEPTION! type: %r val: %r" % (exc_type, exc_val) + print(msg, file=sys.stderr) + traceback.print_tb(exc_tb) + return True if exc_type is None else False diff --git a/riak/benchmarks/multiget.py b/riak/benchmarks/multiget.py new file mode 100644 index 00000000..87a97a6a --- /dev/null +++ b/riak/benchmarks/multiget.py @@ -0,0 +1,65 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import binascii +import os + +import riak.benchmark as benchmark + +from riak import RiakClient +from multiprocessing import cpu_count + +nodes = [ + ('riak-test', 8098, 8087), + # ('riak-test', 10018, 10017), + # ('riak-test', 10028, 10027), + # ('riak-test', 10038, 10037), + # ('riak-test', 10048, 10047), + # ('riak-test', 10058, 10057), +] +client = RiakClient( + nodes=nodes, + protocol='pbc', + multiget_pool_size=128) + +bkeys = [('default', 'multiget', str(key)) for key in range(10000)] + +data = binascii.b2a_hex(os.urandom(1024)) + +print("Benchmarking multiget:") +print(" CPUs: {0}".format(cpu_count())) +print(" Threads: {0}".format(client._multiget_pool._size)) +print(" Keys: {0}".format(len(bkeys))) +print() + +with benchmark.measure() as b: + with b.report('populate'): + for _, bucket, key in bkeys: + client.bucket(bucket).new(key, encoded_data=data, + content_type='text/plain' + ).store() +for b in benchmark.measure_with_rehearsal(): + # client.protocol = 'http' + # with b.report('http seq'): + # for _, bucket, key in bkeys: + # client.bucket(bucket).get(key) + # with b.report('http multi'): + # client.multiget(bkeys) + + client.protocol = 'pbc' + with b.report('pbc seq'): + for _, bucket, key in bkeys: + client.bucket(bucket).get(key) + with b.report('pbc multi'): + client.multiget(bkeys) diff --git a/riak/benchmarks/timeseries.py b/riak/benchmarks/timeseries.py new file mode 100644 index 00000000..5d0f89c3 --- /dev/null +++ b/riak/benchmarks/timeseries.py @@ -0,0 +1,88 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import random +import sys + +import riak.benchmark as benchmark + +from multiprocessing import cpu_count +from riak import RiakClient + +# logger = logging.getLogger() +# logger.level = logging.DEBUG +# logger.addHandler(logging.StreamHandler(sys.stdout)) + +# batch sizes 8, 16, 32, 64, 128, 256 +if len(sys.argv) != 3: + raise AssertionError( + 'first arg is batch size, second arg is true / false' + 'for use_ttb') + +rowcount = 32768 +batchsz = int(sys.argv[1]) +if rowcount % batchsz != 0: + raise AssertionError('rowcount must be divisible by batchsz') +use_ttb = sys.argv[2].lower() == 'true' + +epoch = datetime.datetime.utcfromtimestamp(0) +onesec = datetime.timedelta(0, 1) + +weather = ['typhoon', 'hurricane', 'rain', 'wind', 'snow'] +rows = [] +for i in range(rowcount): + ts = datetime.datetime(2016, 1, 1, 12, 0, 0) + \ + datetime.timedelta(seconds=i) + family_idx = i % batchsz + series_idx = i % batchsz + family = 'hash{:d}'.format(family_idx) + series = 'user{:d}'.format(series_idx) + w = weather[i % len(weather)] + temp = (i % 100) + random.random() + row = [family, series, ts, w, temp] + key = [family, series, ts] + rows.append(row) + +print("Benchmarking timeseries:") +print(" Use TTB: {}".format(use_ttb)) +print("Batch Size: {}".format(batchsz)) +print(" CPUs: {}".format(cpu_count())) +print(" Rows: {}".format(len(rows))) +print() + +tbl = 'GeoCheckin' +h = 'riak-test' +n = [ + {'host': h, 'pb_port': 10017}, + {'host': h, 'pb_port': 10027}, + {'host': h, 'pb_port': 10037}, + {'host': h, 'pb_port': 10047}, + {'host': h, 'pb_port': 10057} +] +client = RiakClient(nodes=n, protocol='pbc', + transport_options={'use_ttb': use_ttb}) +table = client.table(tbl) + +with benchmark.measure() as b: + for i in (1, 2, 3): + with b.report('populate-%d' % i): + for i in range(0, rowcount, batchsz): + x = i + y = i + batchsz + r = rows[x:y] + ts_obj = table.new(r) + result = ts_obj.store() + if result is not True: + raise AssertionError("expected success") diff --git a/riak/bucket.py b/riak/bucket.py index 3b255885..7dde7351 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -1,31 +1,36 @@ -""" -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +# Copyright 2010 Rusty Klophaus +# Copyright 2010 Justin Sheehy +# Copyright 2009 Jay Baird +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from six import string_types, PY2 import mimetypes -from riak.util import deprecateQuorumAccessors, deprecated +from riak.util import lazy_property +from riak.datatypes import TYPES -def deprecateBucketQuorumAccessors(klass): - return deprecateQuorumAccessors(klass, parent='_client') +def bucket_property(name, doc=None): + def _prop_getter(self): + return self.get_property(name) + + def _prop_setter(self, value): + return self.set_property(name, value) + + return property(_prop_getter, _prop_setter, doc=doc) -@deprecateBucketQuorumAccessors class RiakBucket(object): """ The ``RiakBucket`` object allows you to access and change information @@ -33,30 +38,40 @@ class RiakBucket(object): objects within the bucket. """ - SEARCH_PRECOMMIT_HOOK = {"mod": "riak_search_kv_hook", "fun": "precommit"} - - def __init__(self, client, name): + def __init__(self, client, name, bucket_type): """ Returns a new ``RiakBucket`` instance. - :param client: A :class:`RiakClient ` instance + :param client: A :class:`RiakClient ` + instance :type client: :class:`RiakClient ` :param name: The bucket name :type name: string + :param bucket_type: The parent bucket type of this bucket + :type bucket_type: :class:`BucketType` """ - try: - if isinstance(name, basestring): + + if not isinstance(name, string_types): + raise TypeError('Bucket name must be a string') + + if PY2: + try: name = name.encode('ascii') - except UnicodeError: - raise TypeError('Unicode bucket names are not supported.') + except UnicodeError: + raise TypeError('Unicode bucket names are not supported.') + + if not isinstance(bucket_type, BucketType): + raise TypeError('Parent bucket type must be a BucketType instance') self._client = client self.name = name + self.bucket_type = bucket_type self._encoders = {} self._decoders = {} + self._resolver = None def __hash__(self): - return hash((self.name, self._client)) + return hash((self.bucket_type.name, self.name, self._client)) def __eq__(self, other): if isinstance(other, self.__class__): @@ -75,6 +90,8 @@ def get_encoder(self, content_type): Get the encoding function for the provided content type for this bucket. + :param content_type: the requested media type + :type content_type: str :param content_type: Content type requested """ if content_type in self._encoders: @@ -87,9 +104,11 @@ def set_encoder(self, content_type, encoder): Set the encoding function for the provided content type for this bucket. - :param content_type: Content type for encoder - :param encoder: Function to encode with - will be called with - data as single argument. + :param content_type: the requested media type + :type content_type: str + :param encoder: an encoding function, takes a single object + argument and returns a string data as single argument. + :type encoder: function """ self._encoders[content_type] = encoder return self @@ -99,7 +118,9 @@ def get_decoder(self, content_type): Get the decoding function for the provided content type for this bucket. - :param content_type: Content type for decoder + :param content_type: the requested media type + :type content_type: str + :rtype: function """ if content_type in self._decoders: return self._decoders[content_type] @@ -111,33 +132,56 @@ def set_decoder(self, content_type, decoder): Set the decoding function for the provided content type for this bucket. - :param content_type: Content type for decoder - :param decoder: Function to decode with - will be called with - string + :param content_type: the requested media type + :type content_type: str + :param decoder: a decoding function, takes a string and + returns a Python type + :type decoder: function """ self._decoders[content_type] = decoder return self def new(self, key=None, data=None, content_type='application/json', encoded_data=None): - """ - Create a new :class:`RiakObject ` - that will be stored as JSON. A shortcut for manually - instantiating a :class:`RiakObject - `. + """A shortcut for manually instantiating a new + :class:`~riak.riak_object.RiakObject` or a new + :class:`~riak.datatypes.Datatype`, based on the presence and value + of the :attr:`datatype ` bucket property. When + the bucket contains a :class:`~riak.datatypes.Datatype`, all + arguments are ignored except ``key``, otherwise they are used to + initialize the :class:`~riak.riak_object.RiakObject`. :param key: Name of the key. Leaving this to be None (default) will make Riak generate the key on store. - :type key: string - :param data: The data to store. + :type key: str + :param data: The data to store in a + :class:`~riak.riak_object.RiakObject`, see + :attr:`RiakObject.data `. :type data: object - :rtype: :class:`RiakObject ` - """ - try: - if isinstance(data, basestring): - data = data.encode('ascii') - except UnicodeError: - raise TypeError('Unicode data values are not supported.') + :param content_type: The media type of the data stored in the + :class:`~riak.riak_object.RiakObject`, see + :attr:`RiakObject.content_type + `. + :type content_type: str + :param encoded_data: The encoded data to store in a + :class:`~riak.riak_object.RiakObject`, see + :attr:`RiakObject.encoded_data + `. + :type encoded_data: str + :rtype: :class:`~riak.riak_object.RiakObject` or + :class:`~riak.datatypes.Datatype` + + """ + from riak import RiakObject + if self.bucket_type.datatype: + return TYPES[self.bucket_type.datatype](bucket=self, key=key) + + if PY2: + try: + if isinstance(data, string_types): + data = data.encode('ascii') + except UnicodeError: + raise TypeError('Unicode data values are not supported.') obj = RiakObject(self._client, self, key) obj.content_type = content_type @@ -147,30 +191,12 @@ def new(self, key=None, data=None, content_type='application/json', obj.encoded_data = encoded_data return obj - def new_binary(self, key=None, data=None, - content_type='application/octet-stream'): + def get(self, key, r=None, pr=None, timeout=None, include_context=None, + basic_quorum=None, notfound_ok=None, head_only=False): """ - Create a new :class:`RiakObject ` - that will be stored as plain text/binary. A shortcut for - manually instantiating a :class:`RiakObject - `. - - :param key: Name of the key. - :type key: string - :param data: The data to store. - :type data: object - :param content_type: The content type of the object. - :type content_type: string - :rtype: :class:`RiakObject ` - """ - deprecated('RiakBucket.new_binary is deprecated, ' - 'use RiakBucket.new with the encoded_data ' - 'param instead of data') - return self.new(key, encoded_data=data, content_type=content_type) - - def get(self, key, r=None, pr=None): - """ - Retrieve a JSON-encoded object from Riak. + Retrieve a :class:`~riak.riak_object.RiakObject` or + :class:`~riak.datatypes.Datatype`, based on the presence and value + of the :attr:`datatype ` bucket property. :param key: Name of the key. :type key: string @@ -178,127 +204,136 @@ def get(self, key, r=None, pr=None): :type r: integer :param pr: PR-Value of the request (defaults to bucket's PR) :type pr: integer - :rtype: :class:`RiakObject ` - """ - obj = RiakObject(self._client, self, key) - return obj.reload(r=r, pr=pr) + :param timeout: a timeout value in milliseconds + :type timeout: int + :param include_context: if the bucket contains datatypes, include + the opaque context in the result + :type include_context: bool + :param basic_quorum: whether to use the "basic quorum" policy + for not-founds + :type basic_quorum: bool + :param notfound_ok: whether to treat not-found responses as successful + :type notfound_ok: bool + :param head_only: whether to fetch without value, so only metadata + (only available on PB transport) + :type head_only: bool + :rtype: :class:`RiakObject ` or + :class:`~riak.datatypes.Datatype` + + """ + from riak import RiakObject + if self.bucket_type.datatype: + return self._client.fetch_datatype(self, key, r=r, pr=pr, + timeout=timeout, + include_context=include_context, + basic_quorum=basic_quorum, + notfound_ok=notfound_ok) + else: + obj = RiakObject(self._client, self, key) + return obj.reload(r=r, pr=pr, timeout=timeout, + basic_quorum=basic_quorum, + notfound_ok=notfound_ok, + head_only=head_only) - def get_binary(self, key, r=None, pr=None): + def multiget(self, keys, r=None, pr=None, timeout=None, + basic_quorum=None, notfound_ok=None, + head_only=False): """ - Retrieve a binary/string object from Riak. + Retrieves a list of keys belonging to this bucket in parallel. - :param key: Name of the key. - :type key: string - :param r: R-Value of the request (defaults to bucket's R) + :param keys: the keys to fetch + :type keys: list + :param r: R-Value for the requests (defaults to bucket's R) :type r: integer - :param pr: PR-Value of the request (defaults to bucket's PR) + :param pr: PR-Value for the requests (defaults to bucket's PR) :type pr: integer - :rtype: :class:`RiakObject ` - """ - deprecated('RiakBucket.get_binary is deprecated, ' - 'use RiakBucket.get') - return self.get(key, r=r, pr=pr) + :param timeout: a timeout value in milliseconds + :type timeout: int + :param basic_quorum: whether to use the "basic quorum" policy + for not-founds + :type basic_quorum: bool + :param notfound_ok: whether to treat not-found responses as successful + :type notfound_ok: bool + :param head_only: whether to fetch without value, so only metadata + (only available on PB transport) + :type head_only: bool + :rtype: list of :class:`RiakObjects `, + :class:`Datatypes `, or tuples of + bucket_type, bucket, key, and the exception raised on fetch + """ + bkeys = [(self.bucket_type.name, self.name, key) for key in keys] + return self._client.multiget(bkeys, r=r, pr=pr, timeout=timeout, + basic_quorum=basic_quorum, + notfound_ok=notfound_ok, + head_only=head_only) + + def _get_resolver(self): + if callable(self._resolver): + return self._resolver + elif self._resolver is None: + return self._client.resolver + else: + raise TypeError("resolver is not a function") - def _set_n_val(self, nval): - return self.set_property('n_val', nval) + def _set_resolver(self, value): + if value is None or callable(value): + self._resolver = value + else: + raise TypeError("resolver is not a function") - def _get_n_val(self): - return self.get_property('n_val') + resolver = property(_get_resolver, _set_resolver, + doc="""The sibling-resolution function for this + bucket. If the resolver is not set, the + client's resolver will be used.""") - n_val = property(_get_n_val, _set_n_val, doc=""" + n_val = bucket_property('n_val', doc=""" N-value for this bucket, which is the number of replicas that will be written of each object in the bucket. - .. warning:: - - Set this once before you write any data to the bucket, and never - change it again, otherwise unpredictable things could happen. - This should only be used if you know what you are doing. - - :type nval: integer + .. warning:: Set this once before you write any data to the + bucket, and never change it again, otherwise unpredictable + things could happen. This should only be used if you know what + you are doing. """) - def _set_allow_mult(self, bool): - return self.set_property('allow_mult', bool) - - def _get_allow_mult(self): - return self.get_property('allow_mult') - - allow_mult = property(_get_allow_mult, _set_allow_mult, doc=""" + allow_mult = bucket_property('allow_mult', doc=""" If set to True, then writes with conflicting data will be stored - and returned to the client. This situation can be detected by - calling has_siblings() and get_siblings(). + and returned to the client. :type bool: boolean """) - def _set_r(self, val): - return self.set_property('r', val) - - def _get_r(self): - return self.get_property('r') - - r = property(_get_r, _set_r, doc=""" + r = bucket_property('r', doc=""" The default 'read' quorum for this bucket (how many replicas must reply for a successful read). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_pr(self, val): - return self.set_property('pr', val) - - def _get_pr(self): - return self.get_property('pr') - - pr = property(_get_pr, _set_pr, doc=""" + pr = bucket_property('pr', doc=""" The default 'primary read' quorum for this bucket (how many primary replicas are required for a successful read). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_rw(self, val): - return self.set_property('rw', val) - - def _get_rw(self): - return self.get_property('rw') - - rw = property(_get_rw, _set_rw, doc=""" + rw = bucket_property('rw', doc=""" The default 'read' and 'write' quorum for this bucket (equivalent to 'r' and 'w' but for deletes). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_w(self, val): - return self.set_property('w', val) - - def _get_w(self): - return self.get_property('w') - - w = property(_get_w, _set_w, doc=""" + w = bucket_property('w', doc=""" The default 'write' quorum for this bucket (how many replicas must acknowledge receipt of a write). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_dw(self, val): - return self.set_property('dw', val) - - def _get_dw(self): - return self.get_property('dw') - - dw = property(_get_dw, _set_dw, doc=""" + dw = bucket_property('dw', doc=""" The default 'durable write' quorum for this bucket (how many replicas must commit the write). This should be an integer less than the 'n_val' property, or a string of 'one', 'quorum', 'all', or 'default'""") - def _set_pw(self, val): - return self.set_property('pw', val) - - def _get_pw(self): - return self.get_property('pw') - - pw = property(_get_pw, _set_pw, doc=""" + pw = bucket_property('pw', doc=""" The default 'primary write' quorum for this bucket (how many primary replicas are required for a successful write). This should be an integer less than the 'n_val' property, or a string of @@ -323,10 +358,7 @@ def get_property(self, key): :type key: string :rtype: mixed """ - try: - return self.get_properties()[key] - except KeyError: - raise NotImplementedError + return self.get_properties()[key] def set_properties(self, props): """ @@ -348,7 +380,6 @@ def get_properties(self): def clear_properties(self): """ Reset all bucket properties to their defaults. - """ return self._client.clear_bucket_props(self) @@ -356,9 +387,7 @@ def get_keys(self): """ Return all keys within the bucket. - .. warning:: - - At current, this is a very expensive operation. Use with caution. + :rtype: list of keys """ return self._client.get_keys(self) @@ -366,20 +395,31 @@ def stream_keys(self): """ Streams all keys within the bucket through an iterator. - .. warning:: - - At current, this is a very expensive operation. Use with caution. + The caller must close the stream when finished. See + :meth:`RiakClient.stream_keys() + ` for more details. :rtype: iterator """ return self._client.stream_keys(self) def new_from_file(self, key, filename): + """Create a new Riak object in the bucket, using the contents of + the specified file. This is a shortcut for :meth:`new`, where the + ``encoded_data`` and ``content_type`` are set for you. + + .. warning:: This is not supported for buckets that contain + :class:`Datatypes `. + + :param key: the key of the new object + :type key: string + :param filename: the file to read the contents from + :type filename: string + :rtype: :class:`RiakObject ` """ - Create a new Riak object in the bucket, using the content of - the specified file. - """ - binary_data = open(filename, "rb").read() + binary_data = None + with open(filename, 'rb') as f: + binary_data = f.read() mimetype, encoding = mimetypes.guess_type(filename) if encoding: binary_data = bytearray(binary_data, encoding) @@ -387,67 +427,335 @@ def new_from_file(self, key, filename): binary_data = bytearray(binary_data) if not mimetype: mimetype = 'application/octet-stream' - return self.new(key, encoded_data=binary_data, content_type=mimetype) - - def new_binary_from_file(self, key, filename): - deprecated('RiakBucket.new_binary_from_file is deprecated, use ' - 'RiakBucket.new_from_file') - return self.new_from_file(key, filename) + if PY2: + return self.new(key, encoded_data=binary_data, + content_type=mimetype) + else: + return self.new(key, encoded_data=bytes(binary_data), + content_type=mimetype) def search_enabled(self): """ - Returns True if the search precommit hook is enabled for this + Returns True if search indexing is enabled for this bucket. + + .. deprecated:: 2.1.0 (Riak 2.0) + Use :ref:`Riak Search 2.0 ` instead. """ - return self.SEARCH_PRECOMMIT_HOOK in (self.get_property("precommit") or - []) + return self.get_properties().get('search', False) def enable_search(self): """ - Enable search for this bucket by installing the precommit hook to - index objects in it. + Enable search indexing for this bucket. + + .. deprecated:: 2.1.0 (Riak 2.0) + Use :ref:`Riak Search 2.0 ` instead. """ - precommit_hooks = self.get_property("precommit") or [] - if self.SEARCH_PRECOMMIT_HOOK not in precommit_hooks: - self.set_properties({"precommit": - precommit_hooks + - [self.SEARCH_PRECOMMIT_HOOK]}) + if not self.search_enabled(): + self.set_property('search', True) return True def disable_search(self): """ - Disable search for this bucket by removing the precommit hook to - index objects in it. + Disable search indexing for this bucket. + + .. deprecated:: 2.1.0 (Riak 2.0) + Use :ref:`Riak Search 2.0 ` instead. """ - precommit_hooks = self.get_property("precommit") or [] - if self.SEARCH_PRECOMMIT_HOOK in precommit_hooks: - precommit_hooks.remove(self.SEARCH_PRECOMMIT_HOOK) - self.set_properties({"precommit": precommit_hooks}) + if self.search_enabled(): + self.set_property('search', False) return True - def search(self, query, **params): + def search(self, query, index=None, **params): + """ + Queries a search index over objects in this bucket/index. See + :meth:`RiakClient.fulltext_search() + ` for more details. + + :param query: the search query + :type query: string + :param index: the index to search over. Defaults to the bucket's name. + :type index: string or None + :param params: additional query flags + :type params: dict + """ + search_index = index or self.name + return self._client.fulltext_search(search_index, query, **params) + + def get_index(self, index, startkey, endkey=None, return_terms=None, + max_results=None, continuation=None, timeout=None, + term_regex=None): + """ + Queries a secondary index over objects in this bucket, + returning keys or index/key pairs. See + :meth:`RiakClient.get_index() + ` for more details. + """ + return self._client.get_index(self, index, startkey, endkey, + return_terms=return_terms, + max_results=max_results, + continuation=continuation, + timeout=timeout, term_regex=term_regex) + + def paginate_index(self, index, startkey, endkey=None, + return_terms=None, max_results=1000, + continuation=None, timeout=None, term_regex=None): + """ + Paginates through a secondary index over objects in this bucket, + returning keys or index/key pairs. See + :meth:`RiakClient.paginate_index() + ` for more details. + """ + return self._client.paginate_index(self, index, startkey, endkey, + return_terms=return_terms, + max_results=max_results, + continuation=continuation, + timeout=timeout, + term_regex=term_regex) + + def stream_index(self, index, startkey, endkey=None, return_terms=None, + max_results=None, continuation=None, timeout=None, + term_regex=None): + """ + Queries a secondary index over objects in this bucket, + streaming keys or index/key pairs via an iterator. + The caller must close the stream when finished. See + :meth:`RiakClient.stream_index() + ` for more details. + """ + return self._client.stream_index(self, index, startkey, endkey, + return_terms=return_terms, + max_results=max_results, + continuation=continuation, + timeout=timeout, + term_regex=term_regex) + + def paginate_stream_index(self, index, startkey, endkey=None, + return_terms=None, max_results=1000, + continuation=None, timeout=None, + term_regex=None): + """ + Paginates through a secondary index over objects in this bucket, + streaming keys or index/key pairs. The caller must close the stream + when finished. See :meth:`RiakClient.paginate_stream_index() + ` for more details. + """ + return self._client.paginate_stream_index(self, index, startkey, + endkey, + return_terms=return_terms, + max_results=max_results, + continuation=continuation, + timeout=timeout, + term_regex=term_regex) + + def delete(self, key, **kwargs): + """Deletes a key from Riak. Short hand for + ``bucket.new(key).delete()``. See :meth:`RiakClient.delete() + ` for options. + + :param key: The key for the object + :type key: string + :rtype: RiakObject """ - Queries a search index over objects in this bucket/index. + return self.new(key).delete(**kwargs) + + def get_counter(self, key, **kwargs): """ - return self._client.solr.search(self.name, query, **params) + Gets the value of a counter stored in this bucket. See + :meth:`RiakClient.get_counter() + ` for options. + + .. deprecated:: 2.1.0 (Riak 2.0) Riak 1.4-style counters are + deprecated in favor of the :class:`~riak.datatypes.Counter` + datatype. - def get_index(self, index, startkey, endkey=None): + :param key: the key of the counter + :type key: string + :rtype: int """ - Queries a secondary index over objects in this bucket, returning keys. + return self._client.get_counter(self, key, **kwargs) + + def update_counter(self, key, value, **kwargs): """ - return self._client.get_index(self.name, index, startkey, endkey) + Updates the value of a counter stored in this bucket. Positive + values increment the counter, negative values decrement. See + :meth:`RiakClient.update_counter() + ` for options. - def delete(self, key, **kwargs): - """Deletes an object from riak. + .. deprecated:: 2.1.0 (Riak 2.0) Riak 1.4-style counters are + deprecated in favor of the :class:`~riak.datatypes.Counter` + datatype. - Short hand for bucket.new(key).delete() - :param key: The key for the object + :param key: the key of the counter :type key: string - :rtype: RiakObject + :param value: the amount to increment or decrement + :type value: integer """ - return self.new(key).delete(**kwargs) + return self._client.update_counter(self, key, value, **kwargs) + + increment_counter = update_counter + + def get_preflist(self, key): + """ + Retrieve the preflist associated with a given bucket/key + + :param key: Name of the key. + :type key: string + :rtype: list of dict() + """ + return self._client.get_preflist(self, key) def __str__(self): - return ''.format(self.name) + if self.bucket_type.is_default(): + return ''.format(self.name) + else: + return ''.format(self.bucket_type.name, + self.name) + + __repr__ = __str__ + -from riak_object import RiakObject +class BucketType(object): + """ + The ``BucketType`` object allows you to access and change + properties on a Riak bucket type and access buckets within its + namespace. + """ + def __init__(self, client, name): + """ + Returns a new ``BucketType`` instance. + + :param client: A :class:`RiakClient ` + instance + :type client: :class:`RiakClient ` + :param name: The bucket-type's name + :type name: string + """ + self._client = client + self.name = name + + def is_default(self): + """ + Whether this bucket type is the default type, or a user-defined type. + + :rtype: bool + + """ + return self.name == 'default' + + def get_property(self, key): + """ + Retrieve a bucket-type property. + + :param key: The property to retrieve. + :type key: string + :rtype: mixed + """ + return self.get_properties()[key] + + def set_property(self, key, value): + """ + Set a bucket-type property. + + :param key: Property to set. + :type key: string + :param value: Property value. + :type value: mixed + """ + return self.set_properties({key: value}) + + def get_properties(self): + """ + Retrieve a dict of all bucket-type properties. + + :rtype: dict + """ + return self._client.get_bucket_type_props(self) + + def set_properties(self, props): + """ + Set multiple bucket-type properties in one call. + + :param props: A dictionary of properties + :type props: dict + """ + self._client.set_bucket_type_props(self, props) + + def bucket(self, name): + """ + Gets a bucket that belongs to this bucket-type. + + :param name: the bucket name + :type name: str + :rtype: :class:`RiakBucket` + """ + return self._client.bucket(name, self) + + def get_buckets(self, timeout=None): + """ + Get the list of buckets under this bucket-type as + :class:`RiakBucket ` instances. + + .. warning:: Do not use this in production, as it requires + traversing through all keys stored in a cluster. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param timeout: a timeout value in milliseconds + :type timeout: int + :rtype: list of :class:`RiakBucket ` + instances + """ + return self._client.get_buckets(bucket_type=self, timeout=timeout) + + def stream_buckets(self, timeout=None): + """ + Streams the list of buckets under this bucket-type. This is a + generator method that should be iterated over. + + The caller must close the stream when finished. See + :meth:`RiakClient.stream_buckets() + ` for more details. + + .. warning:: Do not use this in production, as it requires + traversing through all keys stored in a cluster. + + :param timeout: a timeout value in milliseconds + :type timeout: int + :rtype: iterator that yields lists of :class:`RiakBucket + ` instances + """ + return self._client.stream_buckets(bucket_type=self, timeout=timeout) + + @lazy_property + def datatype(self): + """ + The assigned datatype for this bucket type, if present. + + :rtype: None or string + """ + if self.is_default(): + return None + else: + return self.get_properties().get('datatype') + + def __str__(self): + return "".format(self.name) + + __repr__ = __str__ + + def __hash__(self): + return hash((self.name, self._client)) + + def __eq__(self, other): + if isinstance(other, self.__class__): + return hash(self) == hash(other) + else: + return False + + def __ne__(self, other): + if isinstance(other, self.__class__): + return hash(self) != hash(other) + else: + return True diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 2dea7a62..7015b48f 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -1,23 +1,16 @@ -""" -Copyright 2011 Basho Technologies, Inc. -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. try: import simplejson as json @@ -25,28 +18,60 @@ import json import random + from weakref import WeakValueDictionary from riak.client.operations import RiakClientOperations from riak.node import RiakNode -from riak.bucket import RiakBucket +from riak.bucket import RiakBucket, BucketType from riak.mapreduce import RiakMapReduceChain -from riak.search import RiakSearch -from riak.transports.http import RiakHttpPool -from riak.transports.pbc import RiakPbcPool -from riak.util import deprecated -from riak.util import deprecateQuorumAccessors -from riak.util import lazy_property +from riak.resolver import default_resolver +from riak.table import Table +from riak.transports.http import HttpPool +from riak.transports.tcp import TcpPool +from riak.security import SecurityCreds +from riak.util import lazy_property, bytes_to_str, str_to_bytes +from six import string_types, PY2 +from riak.client.multi import MultiGetPool, MultiPutPool def default_encoder(obj): """ Default encoder for JSON datatypes, which returns UTF-8 encoded - json instead of the default bloated \uXXXX escaped ASCII strings. + json instead of the default bloated backslash u XXXX escaped ASCII strings. + """ + if isinstance(obj, bytes): + return json.dumps(bytes_to_str(obj), + ensure_ascii=False).encode("utf-8") + else: + return json.dumps(obj, ensure_ascii=False).encode("utf-8") + + +def binary_json_encoder(obj): + """ + Default encoder for JSON datatypes, which returns UTF-8 encoded + json instead of the default bloated backslash u XXXX escaped ASCII strings. + """ + if isinstance(obj, bytes): + return json.dumps(bytes_to_str(obj), + ensure_ascii=False).encode("utf-8") + else: + return json.dumps(obj, ensure_ascii=False).encode("utf-8") + + +def binary_json_decoder(obj): + """ + Default decoder from JSON datatypes. + """ + return json.loads(bytes_to_str(obj)) + + +def binary_encoder_decoder(obj): + """ + Assumes value is already in binary format, so passes unchanged. """ - return json.dumps(obj, ensure_ascii=False).encode("utf-8") + return obj -@deprecateQuorumAccessors class RiakClient(RiakMapReduceChain, RiakClientOperations): """ The ``RiakClient`` object holds information necessary to connect @@ -54,14 +79,17 @@ class RiakClient(RiakMapReduceChain, RiakClientOperations): or by using the methods on related objects. """ - PROTOCOLS = ['http', 'https', 'pbc'] + #: The supported protocols + PROTOCOLS = ['http', 'pbc'] - def __init__(self, protocol='http', transport_options={}, - nodes=None, **unused_args): + def __init__(self, protocol='pbc', transport_options={}, + nodes=None, credentials=None, + multiget_pool_size=None, multiput_pool_size=None, + **kwargs): """ Construct a new ``RiakClient`` object. - :param protocol: the preferred protocol, defaults to 'http' + :param protocol: the preferred protocol, defaults to 'pbc' :type protocol: string :param nodes: a list of node configurations, where each configuration is a dict containing the keys @@ -70,40 +98,55 @@ def __init__(self, protocol='http', transport_options={}, :param transport_options: Optional key-value args to pass to the transport constructor :type transport_options: dict + :param credentials: optional object of security info + :type credentials: :class:`~riak.security.SecurityCreds` or dict + :param multiget_pool_size: the number of threads to use in + :meth:`multiget` operations. Defaults to a factor of the number of + CPUs in the system + :type multiget_pool_size: int + :param multiput_pool_size: the number of threads to use in + :meth:`multiput` operations. Defaults to a factor of the number of + CPUs in the system + :type multiput_pool_size: int """ - unused_args = unused_args.copy() - - if 'port' in unused_args: - deprecated("port option is deprecated, use http_port or pb_port," - " or the nodes option. Your given port of %r will be " - "used as the %s port unless already set" % - (unused_args['port'], protocol)) - unused_args['already_warned_port'] = True - if (protocol in ['http', 'https'] and - 'http_port' not in unused_args): - unused_args['http_port'] = unused_args['port'] - elif protocol == 'pbc' and 'pb_port' not in unused_args: - unused_args['pb_port'] = unused_args['port'] - - if 'transport_class' in unused_args: - deprecated( - "transport_class is deprecated, use the protocol option") + kwargs = kwargs.copy() if nodes is None: - self.nodes = [self._create_node(unused_args), ] + self.nodes = [self._create_node(kwargs), ] else: self.nodes = [self._create_node(n) for n in nodes] - self.protocol = protocol or 'http' - - self._http_pool = RiakHttpPool(self, **transport_options) - self._pb_pool = RiakPbcPool(self, **transport_options) - - self._encoders = {'application/json': default_encoder, - 'text/json': default_encoder} - self._decoders = {'application/json': json.loads, - 'text/json': json.loads} + self._multiget_pool_size = multiget_pool_size + self._multiput_pool_size = multiput_pool_size + self.protocol = protocol or 'pbc' + self._resolver = None + self._credentials = self._create_credentials(credentials) + self._http_pool = HttpPool(self, **transport_options) + self._tcp_pool = TcpPool(self, **transport_options) + self._closed = False + + if PY2: + self._encoders = {'application/json': default_encoder, + 'text/json': default_encoder, + 'text/plain': str} + self._decoders = {'application/json': json.loads, + 'text/json': json.loads, + 'text/plain': str} + else: + self._encoders = {'application/json': binary_json_encoder, + 'text/json': binary_json_encoder, + 'text/plain': str_to_bytes, + 'binary/octet-stream': binary_encoder_decoder} + self._decoders = {'application/json': binary_json_decoder, + 'text/json': binary_json_decoder, + 'text/plain': bytes_to_str, + 'binary/octet-stream': binary_encoder_decoder} self._buckets = WeakValueDictionary() + self._bucket_types = WeakValueDictionary() + self._tables = WeakValueDictionary() + + def __del__(self): + self.close() def _get_protocol(self): return self._protocol @@ -115,40 +158,33 @@ def _set_protocol(self, value): self._protocol = value protocol = property(_get_protocol, _set_protocol, - doc="""Which protocol to prefer, one of PROTOCOLS""") - - def get_transport(self): - """ - Get the transport instance the client is using for it's - connection. DEPRECATED - """ - deprecated("get_transport is deprecated, use client, " + - "bucket, or object methods instead") - return None - - def get_client_id(self): - """ - Get the ``client_id`` for this ``RiakClient`` instance. - DEPRECATED - - :rtype: string - """ - deprecated( - "``get_client_id`` is deprecated, use the ``client_id`` property") - return self.client_id - - def set_client_id(self, client_id): - """ - Set the client_id for this ``RiakClient`` instance. - DEPRECATED + doc=""" + Which protocol to prefer, one of + :attr:`PROTOCOLS + `. Please + note that when one protocol is selected, the + other protocols MAY NOT attempt to connect. + Changing to another protocol will cause a + connection on the next request. + + Some requests are only valid over ``'http'``, + and will always be sent via + those transports, regardless of which protocol + is preferred. + """) + + def _get_resolver(self): + return self._resolver or default_resolver + + def _set_resolver(self, value): + if value is None or callable(value): + self._resolver = value + else: + raise TypeError("resolver is not a function") - :param client_id: The new client_id. - :type client_id: string - """ - deprecated( - "``set_client_id`` is deprecated, use the ``client_id`` property") - self.client_id = client_id - return self + resolver = property(_get_resolver, _set_resolver, + doc=""" The sibling-resolution function for this client. + Defaults to :func:`riak.resolver.default_resolver`.""") def _get_client_id(self): with self._transport() as transport: @@ -157,7 +193,7 @@ def _get_client_id(self): def _set_client_id(self, client_id): for http in self._http_pool: http.client_id = client_id - for pb in self._pb_pool: + for pb in self._tcp_pool: pb.client_id = client_id client_id = property(_get_client_id, _set_client_id, @@ -166,6 +202,10 @@ def _set_client_id(self, client_id): def get_encoder(self, content_type): """ Get the encoding function for the provided content type. + + :param content_type: the requested media type + :type content_type: str + :rtype: function """ return self._encoders.get(content_type) @@ -173,7 +213,10 @@ def set_encoder(self, content_type, encoder): """ Set the encoding function for the provided content type. - :param encoder: + :param content_type: the requested media type + :type content_type: str + :param encoder: an encoding function, takes a single object + argument and returns encoded data :type encoder: function """ self._encoders[content_type] = encoder @@ -181,6 +224,10 @@ def set_encoder(self, content_type, encoder): def get_decoder(self, content_type): """ Get the decoding function for the provided content type. + + :param content_type: the requested media type + :type content_type: str + :rtype: function """ return self._decoders.get(content_type) @@ -188,31 +235,108 @@ def set_decoder(self, content_type, decoder): """ Set the decoding function for the provided content type. - :param decoder: + :param content_type: the requested media type + :type content_type: str + :param decoder: a decoding function, takes encoded data and + returns a Python type :type decoder: function """ self._decoders[content_type] = decoder - def bucket(self, name): + def bucket(self, name, bucket_type='default'): """ Get the bucket by the specified name. Since buckets always exist, - this will always return a :class:`RiakBucket `. + this will always return a + :class:`RiakBucket `. + + If you are using a bucket that is contained in a bucket type, it is + preferable to access it from the bucket type object:: + # Preferred: + client.bucket_type("foo").bucket("bar") + + # Equivalent, but not preferred: + client.bucket("bar", bucket_type="foo") + + :param name: the bucket name + :type name: str + :param bucket_type: the parent bucket-type + :type bucket_type: :class:`BucketType ` + or str :rtype: :class:`RiakBucket ` + + """ + if not isinstance(name, string_types): + raise TypeError('Bucket name must be a string') + + if isinstance(bucket_type, string_types): + bucket_type = self.bucket_type(bucket_type) + elif not isinstance(bucket_type, BucketType): + raise TypeError('bucket_type must be a string ' + 'or riak.bucket.BucketType') + + b = RiakBucket(self, name, bucket_type) + return self._setdefault_handle_none( + self._buckets, (bucket_type, name), b) + + def bucket_type(self, name): + """ + Gets the bucket-type by the specified name. Bucket-types do + not always exist (unlike buckets), but this will always return + a :class:`BucketType ` object. + + :param name: the bucket-type name + :type name: str + :rtype: :class:`BucketType ` + """ + if not isinstance(name, string_types): + raise TypeError('BucketType name must be a string') + + btype = BucketType(self, name) + return self._setdefault_handle_none( + self._bucket_types, name, btype) + + def table(self, name): """ - if name in self._buckets: - return self._buckets[name] + Gets the table by the specified name. Tables do + not always exist (unlike buckets), but this will always return + a :class:`Table ` object. + + :param name: the table name + :type name: str + :rtype: :class:`Table ` + """ + if not isinstance(name, string_types): + raise TypeError('Table name must be a string') + + if name in self._tables: + return self._tables[name] else: - bucket = RiakBucket(self, name) - self._buckets[name] = bucket - return bucket + table = Table(self, name) + self._tables[name] = table + return table - @lazy_property - def solr(self): + def close(self): """ - Returns a RiakSearch object which can access search indexes. + Iterate through all of the connections and close each one. """ - return RiakSearch(self) + if not self._closed: + self._closed = True + self._stop_multi_pools() + if self._http_pool is not None: + self._http_pool.clear() + self._http_pool = None + if self._tcp_pool is not None: + self._tcp_pool.clear() + self._tcp_pool = None + + def _stop_multi_pools(self): + if self._multiget_pool: + self._multiget_pool.stop() + self._multiget_pool = None + if self._multiput_pool: + self._multiput_pool.stop() + self._multiput_pool = None def _create_node(self, n): if isinstance(n, RiakNode): @@ -228,6 +352,20 @@ def _create_node(self, n): raise TypeError("%s is not a valid node configuration" % repr(n)) + def _create_credentials(self, n): + """ + Create security credentials, if necessary. + """ + if not n: + return n + elif isinstance(n, SecurityCreds): + return n + elif isinstance(n, dict): + return SecurityCreds(**n) + else: + raise TypeError("%s is not a valid security configuration" + % repr(n)) + def _choose_node(self, nodes=None): """ Chooses a random node from the list of nodes in the client, @@ -250,6 +388,30 @@ def _error_rate(node): else: return random.choice(good) + def _setdefault_handle_none(self, wvdict, key, value): + # TODO FIXME FUTURE + # This is a workaround for Python issue 19542 + # http://bugs.python.org/issue19542 + rv = wvdict.setdefault(key, value) + if rv is None: + return value + else: + return rv + + @lazy_property + def _multiget_pool(self): + if self._multiget_pool_size: + return MultiGetPool(self._multiget_pool_size) + else: + return None + + @lazy_property + def _multiput_pool(self): + if self._multiput_pool_size: + return MultiPutPool(self._multiput_pool_size) + else: + return None + def __hash__(self): return hash(frozenset([(n.host, n.http_port, n.pb_port) for n in self.nodes])) diff --git a/riak/client/index_page.py b/riak/client/index_page.py new file mode 100644 index 00000000..8e094a66 --- /dev/null +++ b/riak/client/index_page.py @@ -0,0 +1,184 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple, Sequence + + +CONTINUATION = namedtuple('Continuation', ['c']) + + +class IndexPage(Sequence, object): + """ + Encapsulates a single page of results from a secondary index + query, with the ability to iterate over results (if not streamed), + capture the page marker (continuation), and automatically fetch + the next page. + + While users will interact with this object, it will be created + automatically by the client and does not need to be instantiated + elsewhere. + """ + def __init__(self, client, bucket, index, startkey, endkey, return_terms, + max_results, term_regex): + self.client = client + self.bucket = bucket + self.index = index + self.startkey = startkey + self.endkey = endkey + self.return_terms = return_terms + self.max_results = max_results + self.results = None + self.stream = False + self.term_regex = term_regex + + continuation = None + """ + The opaque page marker that is used when fetching the next chunk + of results. The user can simply call :meth:`next_page` to do so, + or pass this to the :meth:`~riak.client.RiakClient.get_index` + method using the ``continuation`` option. + """ + + def __iter__(self): + """ + Emulates the iterator interface. When streaming, this means + delegating to the stream, otherwise iterating over the + existing result set. + """ + if self.results is None: + raise ValueError("No index results to iterate") + + try: + for result in self.results: + if self.stream and isinstance(result, CONTINUATION): + self.continuation = result.c + else: + yield self._inject_term(result) + finally: + if self.stream: + self.results.close() + + def __len__(self): + """ + Returns the length of the captured results. + """ + if self._has_results(): + return len(self.results) + else: + raise ValueError("Streamed index page has no length") + + def __getitem__(self, index): + """ + Fetches an item by index from the captured results. + """ + if self._has_results(): + return self.results[index] + else: + raise ValueError("Streamed index page has no entries") + + def __eq__(self, other): + """ + An IndexPage can pretend to be equal to a list when it has + captured results by simply comparing the internal results to + the passed list. Otherwise the other object needs to be an + equivalent IndexPage. + """ + if isinstance(other, list) and self._has_results(): + return self._inject_term(self.results) == other + elif isinstance(other, IndexPage): + return other.__dict__ == self.__dict__ + else: + return False + + def __ne__(self, other): + """ + Converse of __eq__. + """ + return not self.__eq__(other) + + def has_next_page(self): + """ + Whether there is another page available, i.e. the response + included a continuation. + """ + return self.continuation is not None + + def next_page(self, timeout=None, stream=None): + """ + Fetches the next page using the same parameters as the + original query. + + Note that if streaming was used before, it will be used again + unless overridden. + + :param stream: whether to enable streaming. `True` enables, + `False` disables, `None` uses previous value. + :type stream: boolean + :param timeout: a timeout value in milliseconds, or 'infinity' + :type timeout: int + """ + if not self.continuation: + raise ValueError("Cannot get next index page, no continuation") + + if stream is not None: + self.stream = stream + + args = {'bucket': self.bucket, + 'index': self.index, + 'startkey': self.startkey, + 'endkey': self.endkey, + 'return_terms': self.return_terms, + 'max_results': self.max_results, + 'continuation': self.continuation, + 'timeout': timeout, + 'term_regex': self.term_regex} + + if self.stream: + return self.client.stream_index(**args) + else: + return self.client.get_index(**args) + + def _has_results(self): + """ + When not streaming, have results been assigned? + """ + return not (self.stream or self.results is None) + + def _should_inject_term(self, term): + """ + The index term should be injected when using an equality query + and the return terms option. If the term is already a tuple, + it can be skipped. + """ + return self.return_terms and not self.endkey + + def _inject_term(self, result): + """ + Upgrades a result (streamed or not) to include the index term + when an equality query is used with return_terms. + """ + if self._should_inject_term(result): + if type(result) is list: + return [(self.startkey, r) for r in result] + else: + return (self.startkey, result) + else: + return result + + def __repr__(self): + return "<{!s} {!r}>".format(self.__class__.__name__, self.__dict__) + + def close(self): + if self.stream: + self.results.close() diff --git a/riak/client/multi.py b/riak/client/multi.py new file mode 100644 index 00000000..681d3ec3 --- /dev/null +++ b/riak/client/multi.py @@ -0,0 +1,324 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +from collections import namedtuple +from threading import Thread, Lock, Event +from multiprocessing import cpu_count +from six import PY2 + +from riak.riak_object import RiakObject +from riak.ts_object import TsObject + +if PY2: + from Queue import Queue, Empty +else: + from queue import Queue, Empty + +__all__ = ['multiget', 'multiput', 'MultiGetPool', 'MultiPutPool'] + + +try: + #: The default size of the worker pool, either based on the number + #: of CPUS or defaulting to 6 + POOL_SIZE = cpu_count() +except NotImplementedError: + # Make an educated guess + POOL_SIZE = 6 + +#: A :class:`namedtuple` for tasks that are fed to workers in the +#: multi get pool. +Task = namedtuple('Task', + ['client', 'outq', 'bucket_type', 'bucket', 'key', + 'object', 'options']) + + +#: A :class:`namedtuple` for tasks that are fed to workers in the +#: multi put pool. +PutTask = namedtuple('PutTask', + ['client', 'outq', 'object', 'options']) + + +class MultiPool(object): + """ + Encapsulates a pool of threads. These threads can be used + across many multi requests. + """ + + def __init__(self, size=POOL_SIZE, name='unknown'): + """ + :param size: the desired size of the worker pool + :type size: int + """ + + self._inq = Queue() + self._size = size + self._name = name + self._started = Event() + self._stop = Event() + self._lock = Lock() + self._workers = [] + + def enq(self, task): + """ + Enqueues a fetch task to the pool of workers. This will raise + a RuntimeError if the pool is stopped or in the process of + stopping. + + :param task: the Task object + :type task: Task or PutTask + """ + if not self._stop.is_set(): + self._inq.put(task) + else: + raise RuntimeError("Attempted to enqueue an operation while " + "multi pool was shutdown!") + + def start(self): + """ + Starts the worker threads if they are not already started. + This method is thread-safe and will be called automatically + when executing an operation. + """ + # Check whether we are already started, skip if we are. + if not self._started.is_set(): + # If we are not started, try to capture the lock. + if self._lock.acquire(False): + # If we got the lock, go ahead and start the worker + # threads, set the started flag, and release the lock. + for i in range(self._size): + name = "riak.client.multi-worker-{0}-{1}".format( + self._name, i) + worker = Thread(target=self._worker_method, name=name) + worker.daemon = False + worker.start() + self._workers.append(worker) + self._started.set() + self._lock.release() + else: + # We didn't get the lock, so someone else is already + # starting the worker threads. Wait until they have + # signaled that the threads are started. + self._started.wait() + + def stop(self): + """ + Signals the worker threads to exit and waits on them. + """ + if not self.stopped(): + self._stop.set() + for worker in self._workers: + worker.join() + + def stopped(self): + """ + Detects whether this pool has been stopped. + """ + return self._stop.is_set() + + def __del__(self): + # Ensure that all work in the queue is processed before + # shutting down. + self.stop() + + def _worker_method(self): + raise NotImplementedError + + def _should_quit(self): + """ + Worker threads should exit when the stop flag is set and the + input queue is empty. Once the stop flag is set, new enqueues + are disallowed, meaning that the workers can safely drain the + queue before exiting. + + :rtype: bool + """ + return self.stopped() and self._inq.empty() + + +class MultiGetPool(MultiPool): + def __init__(self, size=POOL_SIZE): + super(MultiGetPool, self).__init__(size=size, name='get') + + def _worker_method(self): + """ + The body of the multi-get worker. Loops until + :meth:`_should_quit` returns ``True``, taking tasks off the + input queue, fetching the object, and putting them on the + output queue. + """ + while not self._should_quit(): + try: + task = self._inq.get(block=True, timeout=0.25) + except TypeError: + if self._should_quit(): + break + else: + raise + except Empty: + continue + + try: + btype = task.client.bucket_type(task.bucket_type) + obj = btype.bucket(task.bucket).get(task.key, **task.options) + task.outq.put(obj) + except KeyboardInterrupt: + raise + except Exception as err: + errdata = (task.bucket_type, task.bucket, task.key, err) + task.outq.put(errdata) + finally: + self._inq.task_done() + + +class MultiPutPool(MultiPool): + def __init__(self, size=POOL_SIZE): + super(MultiPutPool, self).__init__(size=size, name='put') + + def _worker_method(self): + """ + The body of the multi-put worker. Loops until + :meth:`_should_quit` returns ``True``, taking tasks off the + input queue, storing the object, and putting the result on + the output queue. + """ + while not self._should_quit(): + try: + task = self._inq.get(block=True, timeout=0.25) + except TypeError: + if self._should_quit(): + break + else: + raise + except Empty: + continue + + try: + obj = task.object + if isinstance(obj, RiakObject): + rv = task.client.put(obj, **task.options) + elif isinstance(obj, TsObject): + rv = task.client.ts_put(obj, **task.options) + else: + raise ValueError('unknown obj type: %s'.format(type(obj))) + task.outq.put(rv) + except KeyboardInterrupt: + raise + except Exception as err: + errdata = (task.object, err) + task.outq.put(errdata) + finally: + self._inq.task_done() + + +def multiget(client, keys, **options): + """Executes a parallel-fetch across multiple threads. Returns a list + containing :class:`~riak.riak_object.RiakObject` or + :class:`~riak.datatypes.Datatype` instances, or 4-tuples of + bucket-type, bucket, key, and the exception raised. + + If a ``pool`` option is included, the request will use the given worker + pool and not a transient :class:`~riak.client.multi.MultiGetPool`. This + option will be passed by the client if the ``multiget_pool_size`` + option was set on client initialization. + + :param client: the client to use + :type client: :class:`~riak.client.RiakClient` + :param keys: the keys to fetch in parallel + :type keys: list of three-tuples -- bucket_type/bucket/key + :param options: request options to + :meth:`RiakBucket.get ` + :type options: dict + :rtype: list + + """ + transient_pool = False + outq = Queue() + + if 'pool' in options: + pool = options['pool'] + del options['pool'] + else: + pool = MultiGetPool() + transient_pool = True + + try: + pool.start() + for bucket_type, bucket, key in keys: + task = Task(client, outq, bucket_type, bucket, key, None, options) + pool.enq(task) + + results = [] + for _ in range(len(keys)): + if pool.stopped(): + raise RuntimeError( + 'Multi-get operation interrupted by pool ' + 'stopping!') + results.append(outq.get()) + outq.task_done() + finally: + if transient_pool: + pool.stop() + + return results + + +def multiput(client, objs, **options): + """Executes a parallel-store across multiple threads. Returns a list + containing booleans or :class:`~riak.riak_object.RiakObject` + + If a ``pool`` option is included, the request will use the given worker + pool and not a transient :class:`~riak.client.multi.MultiPutPool`. This + option will be passed by the client if the ``multiput_pool_size`` + option was set on client initialization. + + :param client: the client to use + :type client: :class:`RiakClient ` + :param objs: the objects to store in parallel + :type objs: list of `RiakObject ` or + `TsObject ` + :param options: request options to + :meth:`RiakClient.put ` + :type options: dict + :rtype: list + """ + transient_pool = False + outq = Queue() + + if 'pool' in options: + pool = options['pool'] + del options['pool'] + else: + pool = MultiPutPool() + transient_pool = True + + try: + pool.start() + for obj in objs: + task = PutTask(client, outq, obj, options) + pool.enq(task) + + results = [] + for _ in range(len(objs)): + if pool.stopped(): + raise RuntimeError( + 'Multi-put operation interrupted by pool ' + 'stopping!') + results.append(outq.get()) + outq.task_done() + finally: + if transient_pool: + pool.stop() + + return results diff --git a/riak/client/operations.py b/riak/client/operations.py index 516b2d26..0d507f12 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -1,22 +1,27 @@ -""" -Copyright 2012 Basho Technologies, Inc. +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at +import six +import riak.client.multi -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -from transport import RiakClientTransport, retryable, retryableHttpOnly +from riak import ListError +from riak.client.transport import RiakClientTransport, \ + retryable, retryableHttpOnly +from riak.client.index_page import IndexPage +from riak.datatypes import TYPES +from riak.table import Table +from riak.util import bytes_to_str class RiakClientOperations(RiakClientTransport): @@ -24,25 +29,111 @@ class RiakClientOperations(RiakClientTransport): Methods for RiakClient that result in requests sent to the Riak cluster. - Note that all of these methods have an implicit 'transport' + Note that many of these methods have an implicit 'transport' argument that will be prepended automatically as part of the retry logic, and does not need to be supplied by the user. """ @retryable - def get_buckets(self, transport): + def get_buckets(self, transport, bucket_type=None, timeout=None): + """ + get_buckets(bucket_type=None, timeout=None) + + Get the list of buckets as :class:`RiakBucket + ` instances. + + .. warning:: Do not use this in production, as it requires + traversing through all keys stored in a cluster. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param bucket_type: the optional containing bucket type + :type bucket_type: :class:`~riak.bucket.BucketType` + :param timeout: a timeout value in milliseconds + :type timeout: int + :rtype: list of :class:`RiakBucket ` + instances + """ + if not riak.disable_list_exceptions: + raise ListError() + + _validate_timeout(timeout) + + if bucket_type: + bucketfn = self._bucket_type_bucket_builder + else: + bucketfn = self._default_type_bucket_builder + + return [bucketfn(bytes_to_str(name), bucket_type) for name in + transport.get_buckets(bucket_type=bucket_type, + timeout=timeout)] + + def stream_buckets(self, bucket_type=None, timeout=None): """ - Get the list of buckets as RiakBucket instances. - NOTE: Do not use this in production, as it requires traversing through - all keys stored in a cluster. + Streams the list of buckets. This is a generator method that + should be iterated over. + + .. warning:: Do not use this in production, as it requires + traversing through all keys stored in a cluster. + + The caller should explicitly close the returned iterator, + either using :func:`contextlib.closing` or calling ``close()`` + explicitly. Consuming the entire iterator will also close the + stream. If it does not, the associated connection might not be + returned to the pool. Example:: + + from contextlib import closing + + # Using contextlib.closing + with closing(client.stream_buckets()) as buckets: + for bucket_list in buckets: + do_something(bucket_list) + + # Explicit close() + stream = client.stream_buckets() + for bucket_list in stream: + do_something(bucket_list) + stream.close() + + :param bucket_type: the optional containing bucket type + :type bucket_type: :class:`~riak.bucket.BucketType` + :param timeout: a timeout value in milliseconds + :type timeout: int + :rtype: iterator that yields lists of :class:`RiakBucket + ` instances + """ - return [self.bucket(name) for name in transport.get_buckets()] + if not riak.disable_list_exceptions: + raise ListError() + + _validate_timeout(timeout) + + if bucket_type: + bucketfn = self._bucket_type_bucket_builder + else: + bucketfn = self._default_type_bucket_builder + + def make_op(transport): + return transport.stream_buckets( + bucket_type=bucket_type, timeout=timeout) + + for bucket_list in self._stream_with_retry(make_op): + bucket_list = [bucketfn(bytes_to_str(name), bucket_type) + for name in bucket_list] + if len(bucket_list) > 0: + yield bucket_list @retryable def ping(self, transport): """ + ping() + Check if the Riak server for this ``RiakClient`` instance is alive. + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + :rtype: boolean """ return transport.ping() @@ -50,10 +141,19 @@ def ping(self, transport): is_alive = ping @retryable - def get_index(self, transport, bucket, index, startkey, endkey=None): + def get_index(self, transport, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, continuation=None, + timeout=None, term_regex=None): """ + get_index(bucket, index, startkey, endkey=None, return_terms=None,\ + max_results=None, continuation=None, timeout=None,\ + term_regex=None) + Queries a secondary index, returning matching keys. + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + :param bucket: the bucket whose index will be queried :type bucket: RiakBucket :param index: the index to query @@ -62,15 +162,226 @@ def get_index(self, transport, bucket, index, startkey, endkey=None): :type startkey: string, integer :param endkey: the end of the query range (optional if equality) :type endkey: string, integer - :rtype: list + :param return_terms: whether to include the secondary index value + :type return_terms: boolean + :param max_results: the maximum number of results to return (page size) + :type max_results: integer + :param continuation: the opaque continuation returned from a + previous paginated request + :type continuation: string + :param timeout: a timeout value in milliseconds, or 'infinity' + :type timeout: int + :param term_regex: a regular expression used to filter index terms + :type term_regex: string + :rtype: :class:`~riak.client.index_page.IndexPage` + """ + _validate_timeout(timeout, infinity_ok=True) + + page = IndexPage(self, bucket, index, startkey, endkey, + return_terms, max_results, term_regex) + + results, continuation = transport.get_index( + bucket, index, startkey, endkey, return_terms=return_terms, + max_results=max_results, continuation=continuation, + timeout=timeout, term_regex=term_regex) + + page.results = results + page.continuation = continuation + return page + + def paginate_index(self, bucket, index, startkey, endkey=None, + max_results=1000, return_terms=None, + continuation=None, timeout=None, term_regex=None): """ - return transport.get_index(bucket, index, startkey, endkey) + Iterates over a paginated index query. This is equivalent to calling + :meth:`get_index` and then successively calling + :meth:`~riak.client.index_page.IndexPage.next_page` until all + results are exhausted. + + Because limiting the result set is necessary to invoke pagination, + the ``max_results`` option has a default of ``1000``. + + :param bucket: the bucket whose index will be queried + :type bucket: RiakBucket + :param index: the index to query + :type index: string + :param startkey: the sole key to query, or beginning of the query range + :type startkey: string, integer + :param endkey: the end of the query range (optional if equality) + :type endkey: string, integer + :param return_terms: whether to include the secondary index value + :type return_terms: boolean + :param max_results: the maximum number of results to return (page + size), defaults to 1000 + :type max_results: integer + :param continuation: the opaque continuation returned from a + previous paginated request + :type continuation: string + :param timeout: a timeout value in milliseconds, or 'infinity' + :type timeout: int + :param term_regex: a regular expression used to filter index terms + :type term_regex: string + :rtype: generator over instances of + :class:`~riak.client.index_page.IndexPage` + + """ + page = self.get_index(bucket, index, startkey, + endkey=endkey, max_results=max_results, + return_terms=return_terms, + continuation=continuation, + timeout=timeout, term_regex=term_regex) + yield page + while page.has_next_page(): + page = page.next_page() + yield page + + def stream_index(self, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, continuation=None, + timeout=None, term_regex=None): + """ + Queries a secondary index, streaming matching keys through an + iterator. + + The caller should explicitly close the returned iterator, + either using :func:`contextlib.closing` or calling ``close()`` + explicitly. Consuming the entire iterator will also close the + stream. If it does not, the associated connection might not be + returned to the pool. Example:: + + from contextlib import closing + + # Using contextlib.closing + with closing(client.stream_index(mybucket, 'name_bin', + 'Smith')) as index: + for key in index: + do_something(key) + + # Explicit close() + stream = client.stream_index(mybucket, 'name_bin', 'Smith') + for key in stream: + do_something(key) + stream.close() + + :param bucket: the bucket whose index will be queried + :type bucket: RiakBucket + :param index: the index to query + :type index: string + :param startkey: the sole key to query, or beginning of the query range + :type startkey: string, integer + :param endkey: the end of the query range (optional if equality) + :type endkey: string, integer + :param return_terms: whether to include the secondary index value + :type return_terms: boolean + :param max_results: the maximum number of results to return (page size) + :type max_results: integer + :param continuation: the opaque continuation returned from a + previous paginated request + :type continuation: string + :param timeout: a timeout value in milliseconds, or 'infinity' + :type timeout: int + :param term_regex: a regular expression used to filter index terms + :type term_regex: string + :rtype: :class:`~riak.client.index_page.IndexPage` + + """ + # TODO FUTURE: implement "retry on connection closed" + # as in stream_mapred + _validate_timeout(timeout, infinity_ok=True) + + page = IndexPage(self, bucket, index, startkey, endkey, + return_terms, max_results, term_regex) + page.stream = True + resource = self._acquire() + transport = resource.object + page.results = transport.stream_index( + bucket, index, startkey, endkey, return_terms=return_terms, + max_results=max_results, continuation=continuation, + timeout=timeout, term_regex=term_regex) + page.results.attach(resource) + return page + + def paginate_stream_index(self, bucket, index, startkey, endkey=None, + max_results=1000, return_terms=None, + continuation=None, timeout=None, + term_regex=None): + """ + Iterates over a streaming paginated index query. This is equivalent to + calling :meth:`stream_index` and then successively calling + :meth:`~riak.client.index_page.IndexPage.next_page` until all + results are exhausted. + + Because limiting the result set is necessary to invoke + pagination, the ``max_results`` option has a default of ``1000``. + + The caller should explicitly close each yielded page, either using + :func:`contextlib.closing` or calling ``close()`` explicitly. Consuming + the entire page will also close the stream. If it does not, the + associated connection might not be returned to the pool. Example:: + + from contextlib import closing + + # Using contextlib.closing + for page in client.paginate_stream_index(mybucket, 'name_bin', + 'Smith'): + with closing(page): + for key in page: + do_something(key) + + # Explicit close() + for page in client.paginate_stream_index(mybucket, 'name_bin', + 'Smith'): + for key in page: + do_something(key) + page.close() + + :param bucket: the bucket whose index will be queried + :type bucket: RiakBucket + :param index: the index to query + :type index: string + :param startkey: the sole key to query, or beginning of the query range + :type startkey: string, integer + :param endkey: the end of the query range (optional if equality) + :type endkey: string, integer + :param return_terms: whether to include the secondary index value + :type return_terms: boolean + :param max_results: the maximum number of results to return (page + size), defaults to 1000 + :type max_results: integer + :param continuation: the opaque continuation returned from a + previous paginated request + :type continuation: string + :param timeout: a timeout value in milliseconds, or 'infinity' + :type timeout: int + :param term_regex: a regular expression used to filter index terms + :type term_regex: string + :rtype: generator over instances of + :class:`~riak.client.index_page.IndexPage` + + """ + # TODO FUTURE: implement "retry on connection closed" + # as in stream_mapred + page = self.stream_index(bucket, index, startkey, + endkey=endkey, + max_results=max_results, + return_terms=return_terms, + continuation=continuation, + timeout=timeout, + term_regex=term_regex) + yield page + while page.has_next_page(): + page = page.next_page() + yield page @retryable def get_bucket_props(self, transport, bucket): """ + get_bucket_props(bucket) + Fetches bucket properties for the given bucket. + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + :param bucket: the bucket whose properties will be fetched :type bucket: RiakBucket :rtype: dict @@ -80,61 +391,156 @@ def get_bucket_props(self, transport, bucket): @retryable def set_bucket_props(self, transport, bucket, props): """ + set_bucket_props(bucket, props) + Sets bucket properties for the given bucket. + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + :param bucket: the bucket whose properties will be set :type bucket: RiakBucket :param props: the properties to set :type props: dict """ + _validate_bucket_props(props) return transport.set_bucket_props(bucket, props) @retryable def clear_bucket_props(self, transport, bucket): """ + clear_bucket_props(bucket) + Resets bucket properties for the given bucket. + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + :param bucket: the bucket whose properties will be set :type bucket: RiakBucket """ return transport.clear_bucket_props(bucket) @retryable - def get_keys(self, transport, bucket): + def get_bucket_type_props(self, transport, bucket_type): """ + get_bucket_type_props(bucket_type) + + Fetches properties for the given bucket-type. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param bucket_type: the bucket-type whose properties will be fetched + :type bucket_type: BucketType + :rtype: dict + """ + return transport.get_bucket_type_props(bucket_type) + + @retryable + def set_bucket_type_props(self, transport, bucket_type, props): + """ + set_bucket_type_props(bucket_type, props) + + Sets properties for the given bucket-type. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param bucket_type: the bucket-type whose properties will be set + :type bucket_type: BucketType + :param props: the properties to set + :type props: dict + """ + _validate_bucket_props(props) + return transport.set_bucket_type_props(bucket_type, props) + + @retryable + def get_keys(self, transport, bucket, timeout=None): + """ + get_keys(bucket, timeout=None) + Lists all keys in a bucket. - :param bucket: the bucket whose properties will be set + .. warning:: Do not use this in production, as it requires + traversing through all keys stored in a cluster. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param bucket: the bucket whose keys are fetched :type bucket: RiakBucket + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: list """ - return transport.get_keys(bucket) + if not riak.disable_list_exceptions: + raise ListError() + + _validate_timeout(timeout) - def stream_keys(self, bucket): + return transport.get_keys(bucket, timeout=timeout) + + def stream_keys(self, bucket, timeout=None): """ Lists all keys in a bucket via a stream. This is a generator method which should be iterated over. + .. warning:: Do not use this in production, as it requires + traversing through all keys stored in a cluster. + + The caller should explicitly close the returned iterator, + either using :func:`contextlib.closing` or calling ``close()`` + explicitly. Consuming the entire iterator will also close the + stream. If it does not, the associated connection might + not be returned to the pool. Example:: + + from contextlib import closing + + # Using contextlib.closing + with closing(client.stream_keys(mybucket)) as keys: + for key_list in keys: + do_something(key_list) + + # Explicit close() + stream = client.stream_keys(mybucket) + for key_list in stream: + do_something(key_list) + stream.close() :param bucket: the bucket whose properties will be set :type bucket: RiakBucket + :param timeout: a timeout value in milliseconds + :type timeout: int :rtype: iterator """ - with self._transport() as transport: - stream = transport.stream_keys(bucket) - try: - for keylist in stream: - if len(keylist) > 0: - yield keylist - finally: - stream.close() + if not riak.disable_list_exceptions: + raise ListError() + + _validate_timeout(timeout) + + def make_op(transport): + return transport.stream_keys(bucket, timeout=timeout) + + for keylist in self._stream_with_retry(make_op): + if len(keylist) > 0: + if six.PY2: + yield keylist + else: + yield [bytes_to_str(item) for item in keylist] @retryable def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, - if_none_match=None): + if_none_match=None, timeout=None): """ + put(robj, w=None, dw=None, pw=None, return_body=None,\ + if_none_match=None, timeout=None) + Stores an object in the Riak cluster. + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + :param robj: the object to store :type robj: RiakObject :param w: the write quorum @@ -149,63 +555,214 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, :param if_none_match: whether to fail the write if the object exists :type if_none_match: boolean + :param timeout: a timeout value in milliseconds + :type timeout: int """ + _validate_timeout(timeout) return transport.put(robj, w=w, dw=dw, pw=pw, return_body=return_body, - if_none_match=if_none_match) + if_none_match=if_none_match, + timeout=timeout) @retryable - def put_new(self, transport, robj, w=None, dw=None, pw=None, - return_body=None, if_none_match=None): + def ts_describe(self, transport, table): """ - Stores an object in the Riak cluster with a generated key. + ts_describe(table) - :param robj: the object to store - :type robj: RiakObject - :param w: the write quorum - :type w: integer, string, None - :param dw: the durable write quorum - :type dw: integer, string, None - :param pw: the primary write quorum - :type pw: integer, string, None - :param return_body: whether to return the resulting object - after the write - :type return_body: boolean - :param if_none_match: whether to fail the write if the object - exists - :type if_none_match: boolean + Retrieve a time series table description from the Riak cluster. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param table: The timeseries table. + :type table: string or :class:`Table ` + :rtype: :class:`TsObject ` + """ + t = table + if isinstance(t, six.string_types): + t = Table(self, table) + return transport.ts_describe(t) + + @retryable + def ts_get(self, transport, table, key): + """ + ts_get(table, key) + + Retrieve timeseries value by key + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param table: The timeseries table. + :type table: string or :class:`Table ` + :param key: The timeseries value's key. + :type key: list + :rtype: :class:`TsObject ` """ - return transport.put_new(robj, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) + t = table + if isinstance(t, six.string_types): + t = Table(self, table) + return transport.ts_get(t, key) @retryable - def get(self, transport, robj, r=None, pr=None, vtag=None): + def ts_put(self, transport, tsobj): + """ + ts_put(tsobj) + + Stores time series data in the Riak cluster. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param tsobj: the time series object to store + :type tsobj: RiakTsObject + :rtype: boolean + """ + return transport.ts_put(tsobj) + + @retryable + def ts_delete(self, transport, table, key): + """ + ts_delete(table, key) + + Delete timeseries value by key + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param table: The timeseries table. + :type table: string or :class:`Table ` + :param key: The timeseries value's key. + :type key: list or dict + :rtype: boolean + """ + t = table + if isinstance(t, six.string_types): + t = Table(self, table) + return transport.ts_delete(t, key) + + @retryable + def ts_query(self, transport, table, query, interpolations=None): + """ + ts_query(table, query, interpolations=None) + + Queries time series data in the Riak cluster. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param table: The timeseries table. + :type table: string or :class:`Table ` + :param query: The timeseries query. + :type query: string + :rtype: :class:`TsObject ` + """ + t = table + if isinstance(t, six.string_types): + t = Table(self, table) + return transport.ts_query(t, query, interpolations) + + def ts_stream_keys(self, table, timeout=None): + """ + Lists all keys in a time series table via a stream. This is a + generator method which should be iterated over. + + The caller should explicitly close the returned iterator, + either using :func:`contextlib.closing` or calling ``close()`` + explicitly. Consuming the entire iterator will also close the + stream. If it does not, the associated connection might + not be returned to the pool. Example:: + + from contextlib import closing + + # Using contextlib.closing + with closing(client.ts_stream_keys(mytable)) as keys: + for key_list in keys: + do_something(key_list) + + # Explicit close() + stream = client.ts_stream_keys(mytable) + for key_list in stream: + do_something(key_list) + stream.close() + + :param table: the table from which to stream keys + :type table: string or :class:`Table ` + :param timeout: a timeout value in milliseconds + :type timeout: int + :rtype: iterator """ + if not riak.disable_list_exceptions: + raise ListError() + + t = table + if isinstance(t, six.string_types): + t = Table(self, table) + + _validate_timeout(timeout) + + resource = self._acquire() + transport = resource.object + stream = transport.ts_stream_keys(t, timeout) + stream.attach(resource) + try: + for keylist in stream: + if len(keylist) > 0: + yield keylist + finally: + stream.close() + + @retryable + def get(self, transport, robj, r=None, pr=None, timeout=None, + basic_quorum=None, notfound_ok=None, head_only=False): + """ + get(robj, r=None, pr=None, timeout=None) + Fetches the contents of a Riak object. + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + :param robj: the object to fetch :type robj: RiakObject :param r: the read quorum :type r: integer, string, None :param pr: the primary read quorum :type pr: integer, string, None - :param vtag: the specific sibling to fetch - :type vtag: string + :param timeout: a timeout value in milliseconds + :type timeout: int + :param basic_quorum: whether to use the "basic quorum" policy + for not-founds + :type basic_quorum: bool + :param notfound_ok: whether to treat not-found responses as successful + :type notfound_ok: bool + :param head_only: whether to fetch without value, so only metadata + (only available on PB transport) + :type head_only: bool """ - if not isinstance(robj.key, basestring): + _validate_timeout(timeout) + if not isinstance(robj.key, six.string_types): raise TypeError( 'key must be a string, instead got {0}'.format(repr(robj.key))) - return transport.get(robj, r=r, pr=pr, vtag=vtag) + return transport.get(robj, r=r, pr=pr, timeout=timeout, + basic_quorum=basic_quorum, + notfound_ok=notfound_ok, + head_only=head_only) @retryable def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, - pr=None, pw=None): + pr=None, pw=None, timeout=None): """ + delete(robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None,\ + timeout=None) + Deletes an object from Riak. - :param robj: the object to store + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param robj: the object to delete :type robj: RiakObject :param rw: the read/write (delete) quorum :type rw: integer, string, None @@ -219,15 +776,23 @@ def delete(self, transport, robj, rw=None, r=None, w=None, dw=None, :type dw: integer, string, None :param pw: the primary write quorum :type pw: integer, string, None + :param timeout: a timeout value in milliseconds + :type timeout: int """ + _validate_timeout(timeout) return transport.delete(robj, rw=rw, r=r, w=w, dw=dw, pr=pr, - pw=pw) + pw=pw, timeout=timeout) @retryable def mapred(self, transport, inputs, query, timeout): """ + mapred(inputs, query, timeout) + Executes a MapReduce query. + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + :param inputs: the input list/structure :type inputs: list, dict :param query: the list of query phases @@ -236,6 +801,7 @@ def mapred(self, transport, inputs, query, timeout): :type timeout: integer, None :rtype: mixed """ + _validate_timeout(timeout) return transport.mapred(inputs, query, timeout) def stream_mapred(self, inputs, query, timeout): @@ -243,6 +809,25 @@ def stream_mapred(self, inputs, query, timeout): Streams a MapReduce query as (phase, data) pairs. This is a generator method which should be iterated over. + The caller should explicitly close the returned iterator, + either using :func:`contextlib.closing` or calling ``close()`` + explicitly. Consuming the entire iterator will also close the + stream. If it does not, the associated connection might + not be returned to the pool. Example:: + + from contextlib import closing + + # Using contextlib.closing + with closing(mymapred.stream()) as results: + for phase, result in results: + do_something(phase, result) + + # Explicit close() + stream = mymapred.stream() + for phase, result in stream: + do_something(phase, result) + stream.close() + :param inputs: the input list/structure :type inputs: list, dict :param query: the list of query phases @@ -251,33 +836,139 @@ def stream_mapred(self, inputs, query, timeout): :type timeout: integer, None :rtype: iterator """ - with self._transport() as transport: - stream = transport.stream_mapred(inputs, query, timeout) - try: - for phase, data in stream: - yield phase, data - finally: - stream.close() + _validate_timeout(timeout) - @retryableHttpOnly + def make_op(transport): + return transport.stream_mapred(inputs, query, timeout) + + for phase, data in self._stream_with_retry(make_op): + yield phase, data + + @retryable + def create_search_index(self, transport, index, schema=None, n_val=None, + timeout=None): + """ + create_search_index(index, schema=None, n_val=None) + + Create a search index of the given name, and optionally set + a schema. If no schema is set, the default will be used. + + :param index: the name of the index to create + :type index: string + :param schema: the schema that this index will follow + :type schema: string, None + :param n_val: this indexes N value + :type n_val: integer, None + :param timeout: optional timeout (in ms) + :type timeout: integer, None + """ + return transport.create_search_index(index, schema, n_val, timeout) + + @retryable + def get_search_index(self, transport, index): + """ + get_search_index(index) + + Gets a search index of the given name if it exists, which will also + return the schema. Raises a RiakError if no such schema exists. The + returned dict contains keys ``'name'``, ``'schema'`` and + ``'n_val'``. + + :param index: the name of the index to create + :type index: string + :rtype: dict + """ + return transport.get_search_index(index) + + @retryable + def list_search_indexes(self, transport): + """list_search_indexes() + + Gets all search indexes and their schemas. The returned list + contains dicts with keys ``'name'``, ``'schema'`` and ``'n_val'``. + + :return: list of dicts + """ + return transport.list_search_indexes() + + @retryable + def delete_search_index(self, transport, index): + """ + delete_search_index(index) + + Delete the search index that matches the given name. + + :param index: the name of the index to delete + :type index: string + """ + return transport.delete_search_index(index) + + @retryable + def create_search_schema(self, transport, schema, content): + """ + create_search_schema(schema, content) + + Creates a Solr schema of the given name and content. + Content must be valid Solr schema XML. + + :param schema: the name of the schema to create + :type schema: string + :param content: the solr schema xml content + :type content: string + """ + return transport.create_search_schema(schema, content) + + @retryable + def get_search_schema(self, transport, schema): + """ + get_search_schema(schema) + + Gets a search schema of the given name if it exists. + Raises a RiakError if no such schema exists. The schema is + returned as a dict with keys ``'name'`` and ``'content'``. + + :param schema: the name of the schema to get + :type schema: string + + :return: dict + """ + return transport.get_search_schema(schema) + + @retryable def fulltext_search(self, transport, index, query, **params): """ + fulltext_search(index, query, **params) + Performs a full-text search query. + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + :param index: the bucket/index to search over :type index: string :param query: the search query :type query: string :param params: additional query flags :type params: dict + :rtype: dict """ return transport.search(index, query, **params) @retryableHttpOnly def fulltext_add(self, transport, index, docs): """ + fulltext_add(index, docs) + + .. deprecated:: 2.1.0 (Riak 2.0) + Manual index maintenance is not supported for + :ref:`Riak Search 2.0 `. + Adds documents to the full-text index. + .. note:: This request is automatically retried + :attr:`retries` times if it fails due to network error. + Only HTTP will be used for this request. + :param index: the bucket/index in which to index these docs :type index: string :param docs: the list of documents @@ -288,8 +979,18 @@ def fulltext_add(self, transport, index, docs): @retryableHttpOnly def fulltext_delete(self, transport, index, docs=None, queries=None): """ + fulltext_delete(index, docs=None, queries=None) + + .. deprecated:: 2.1.0 (Riak 2.0) + Manual index maintenance is not supported for + :ref:`Riak Search 2.0 `. + Removes documents from the full-text index. + .. note:: This request is automatically retried + :attr:`retries` times if it fails due to network error. + Only HTTP will be used for this request. + :param index: the bucket/index from which to delete :type index: string :param docs: a list of documents (with ids) @@ -298,3 +999,290 @@ def fulltext_delete(self, transport, index, docs=None, queries=None): :type queries: list """ transport.fulltext_delete(index, docs, queries) + + def multiget(self, pairs, **params): + """Fetches many keys in parallel via threads. + + :param pairs: list of bucket_type/bucket/key tuple triples + :type pairs: list + :param params: additional request flags, e.g. r, pr + :type params: dict + :rtype: list of :class:`RiakObjects `, + :class:`Datatypes `, or tuples of + bucket_type, bucket, key, and the exception raised on fetch + """ + if self._multiget_pool: + params['pool'] = self._multiget_pool + return riak.client.multi.multiget(self, pairs, **params) + + def multiput(self, objs, **params): + """ + Stores objects in parallel via threads. + + :param objs: the objects to store + :type objs: list of `RiakObject ` + :param params: additional request flags, e.g. w, dw, pw + :type params: dict + :rtype: list of boolean or + :class:`RiakObjects `, + """ + if self._multiput_pool: + params['pool'] = self._multiput_pool + return riak.client.multi.multiput(self, objs, **params) + + @retryable + def get_counter(self, transport, bucket, key, r=None, pr=None, + basic_quorum=None, notfound_ok=None): + """get_counter(bucket, key, r=None, pr=None, basic_quorum=None,\ + notfound_ok=None) + + Gets the value of a counter. + + .. deprecated:: 2.1.0 (Riak 2.0) Riak 1.4-style counters are + deprecated in favor of the :class:`~riak.datatypes.Counter` + datatype. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param bucket: the bucket of the counter + :type bucket: RiakBucket + :param key: the key of the counter + :type key: string + :param r: the read quorum + :type r: integer, string, None + :param pr: the primary read quorum + :type pr: integer, string, None + :param basic_quorum: whether to use the "basic quorum" policy + for not-founds + :type basic_quorum: bool + :param notfound_ok: whether to treat not-found responses as successful + :type notfound_ok: bool + :rtype: integer + + """ + return transport.get_counter(bucket, key, r=r, pr=pr) + + def update_counter(self, bucket, key, value, w=None, dw=None, pw=None, + returnvalue=False): + """ + update_counter(bucket, key, value, w=None, dw=None, pw=None,\ + returnvalue=False) + + .. deprecated:: 2.1.0 (Riak 2.0) Riak 1.4-style counters are + deprecated in favor of the :class:`~riak.datatypes.Counter` + datatype. + + Updates a counter by the given value. This operation is not + idempotent and so should not be retried automatically. + + :param bucket: the bucket of the counter + :type bucket: RiakBucket + :param key: the key of the counter + :type key: string + :param value: the amount to increment or decrement + :type value: integer + :param w: the write quorum + :type w: integer, string, None + :param dw: the durable write quorum + :type dw: integer, string, None + :param pw: the primary write quorum + :type pw: integer, string, None + :param returnvalue: whether to return the updated value of the counter + :type returnvalue: bool + """ + if not isinstance(value, six.integer_types): + raise TypeError("Counter update amount must be an integer") + if value == 0: + raise ValueError("Cannot increment counter by 0") + + with self._transport() as transport: + return transport.update_counter(bucket, key, value, + w=w, dw=dw, pw=pw, + returnvalue=returnvalue) + + increment_counter = update_counter + + def fetch_datatype(self, bucket, key, r=None, pr=None, + basic_quorum=None, notfound_ok=None, + timeout=None, include_context=None): + """ + Fetches the value of a Riak Datatype. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param bucket: the bucket of the datatype, which must belong to a + :class:`~riak.bucket.BucketType` + :type bucket: :class:`~riak.bucket.RiakBucket` + :param key: the key of the datatype + :type key: string + :param r: the read quorum + :type r: integer, string, None + :param pr: the primary read quorum + :type pr: integer, string, None + :param basic_quorum: whether to use the "basic quorum" policy + for not-founds + :type basic_quorum: bool, None + :param notfound_ok: whether to treat not-found responses as successful + :type notfound_ok: bool, None + :param timeout: a timeout value in milliseconds + :type timeout: int, None + :param include_context: whether to return the opaque context + as well as the value, which is useful for removal operations + on sets and maps + :type include_context: bool, None + :rtype: :class:`~riak.datatypes.Datatype` + """ + dtype, value, context = self._fetch_datatype( + bucket, key, r=r, pr=pr, basic_quorum=basic_quorum, + notfound_ok=notfound_ok, timeout=timeout, + include_context=include_context) + + return TYPES[dtype](bucket=bucket, key=key, value=value, + context=context) + + def update_datatype(self, datatype, w=None, dw=None, pw=None, + return_body=None, timeout=None, + include_context=None): + """ + Sends an update to a Riak Datatype to the server. This operation is not + idempotent and so will not be retried automatically. + + :param datatype: the datatype with pending updates + :type datatype: :class:`~riak.datatypes.Datatype` + :param w: the write quorum + :type w: integer, string, None + :param dw: the durable write quorum + :type dw: integer, string, None + :param pw: the primary write quorum + :type pw: integer, string, None + :param timeout: a timeout value in milliseconds + :type timeout: int + :param include_context: whether to return the opaque context + as well as the value, which is useful for removal operations + on sets and maps + :type include_context: bool + :rtype: tuple of datatype, opaque value and opaque context + + """ + _validate_timeout(timeout) + + with self._transport() as transport: + return transport.update_datatype(datatype, w=w, dw=dw, pw=pw, + return_body=return_body, + timeout=timeout, + include_context=include_context) + + @retryable + def get_preflist(self, transport, bucket, key): + """ + Fetch the preflist for a given bucket and key. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param bucket: the bucket whose index will be queried + :type bucket: RiakBucket + :param key: the key of the preflist + :type key: string + + :return: list of dicts (partition, node, primary) + """ + return transport.get_preflist(bucket, key) + + def _bucket_type_bucket_builder(self, name, bucket_type): + """ + Build a bucket from a bucket type + + :param name: Bucket name + :param bucket_type: A bucket type + :return: A bucket object + """ + return bucket_type.bucket(name) + + def _default_type_bucket_builder(self, name, unused): + """ + Build a bucket for the default bucket type + + :param name: Default bucket name + :param unused: Unused + :return: A bucket object + """ + del unused # Ignored parameters. + return self.bucket(name) + + @retryable + def _fetch_datatype(self, transport, bucket, key, r=None, pr=None, + basic_quorum=None, notfound_ok=None, + timeout=None, include_context=None): + """ + _fetch_datatype(bucket, key, r=None, pr=None, basic_quorum=None, + notfound_ok=None, timeout=None, include_context=None) + + + Fetches the value of a Riak Datatype as raw data. This is used + internally to update already reified Datatype objects. Use the + public version to fetch a reified type. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param bucket: the bucket of the datatype, which must belong to a + :class:`~riak.BucketType` + :type bucket: RiakBucket + :param key: the key of the datatype + :type key: string, None + :param r: the read quorum + :type r: integer, string, None + :param pr: the primary read quorum + :type pr: integer, string, None + :param basic_quorum: whether to use the "basic quorum" policy + for not-founds + :type basic_quorum: bool + :param notfound_ok: whether to treat not-found responses as successful + :type notfound_ok: bool + :param timeout: a timeout value in milliseconds + :type timeout: int + :param include_context: whether to return the opaque context + as well as the value, which is useful for removal operations + on sets and maps + :type include_context: bool + :rtype: tuple of type, value and context + """ + _validate_timeout(timeout) + + return transport.fetch_datatype(bucket, key, r=r, pr=pr, + basic_quorum=basic_quorum, + notfound_ok=notfound_ok, + timeout=timeout, + include_context=include_context) + + +def _validate_bucket_props(props): + if 'hll_precision' in props: + precision = props['hll_precision'] + if precision < 4 or precision > 16: + raise ValueError( + 'hll_precision must be between 4 and 16, inclusive') + + +def _validate_timeout(timeout, infinity_ok=False): + """ + Raises an exception if the given timeout is an invalid value. + """ + if timeout is None: + return + + if timeout == 'infinity': + if infinity_ok: + return + else: + raise ValueError( + 'timeout must be a positive integer ' + '("infinity" is not valid)') + + if isinstance(timeout, six.integer_types) and timeout > 0: + return + + raise ValueError('timeout must be a positive integer') diff --git a/riak/client/transport.py b/riak/client/transport.py index 5c42c55c..ffc705e4 100644 --- a/riak/client/transport.py +++ b/riak/client/transport.py @@ -1,25 +1,43 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from contextlib import contextmanager -from riak.transports.pool import BadResource -from riak.transports.pbc import is_retryable as is_pbc_retryable +from riak.transports.pool import BadResource, ConnectionClosed +from riak.transports.tcp import is_retryable as is_tcp_retryable from riak.transports.http import is_retryable as is_http_retryable -import httplib +from six import PY2 + +import threading + +if PY2: + from httplib import HTTPException +else: + from http.client import HTTPException + +#: The default (global) number of times to retry requests that are +#: retryable. This can be modified locally, per-thread, via the +#: :attr:`RiakClient.retries` property, or using the +#: :attr:`RiakClient.retry_count` method in a ``with`` statement. +DEFAULT_RETRY_COUNT = 3 + + +class _client_locals(threading.local): + """ + A thread-locals object used by the client. + """ + def __init__(self): + self.riak_retries_count = DEFAULT_RETRY_COUNT class RiakClientTransport(object): @@ -27,23 +45,101 @@ class RiakClientTransport(object): Methods for RiakClient related to transport selection and retries. """ - RETRY_COUNT = 3 - # These will be set or redefined by the RiakClient initializer - protocol = 'http' + protocol = 'pbc' _http_pool = None - _pb_pool = None + _tcp_pool = None + _locals = _client_locals() + + def _get_retry_count(self): + return self._locals.riak_retries_count or DEFAULT_RETRY_COUNT + + def _set_retry_count(self, value): + if not isinstance(value, int): + raise TypeError("retries must be an integer") + self._locals.riak_retries_count = value + + __retries_doc = """ + The number of times retryable operations will be attempted + before raising an exception to the caller. Defaults to + ``3``. + + :note: This is a thread-local for safety and + operation-specific modification. To change the + default globally, modify + :data:`riak.client.transport.DEFAULT_RETRY_COUNT`. + """ + + retries = property(_get_retry_count, _set_retry_count, doc=__retries_doc) + + @contextmanager + def retry_count(self, retries): + """ + retry_count(retries) + + Modifies the number of retries for the scope of the ``with`` + statement (in the current thread). + + Example:: + + with client.retry_count(10): + client.ping() + """ + if not isinstance(retries, int): + raise TypeError("retries must be an integer") + + old_retries, self.retries = self.retries, retries + try: + yield + finally: + self.retries = old_retries @contextmanager def _transport(self): """ + _transport() + Yields a single transport to the caller from the default pool, - without retries. + without retries. NB: no need to re-try as this method is only + used by CRDT operations that should never be re-tried. """ pool = self._choose_pool() - with pool.take() as transport: + with pool.transaction() as transport: yield transport + def _acquire(self): + """ + _acquire() + + Acquires a connection from the default pool. + """ + return self._choose_pool().acquire() + + def _stream_with_retry(self, make_op): + first_try = True + while True: + resource = self._acquire() + transport = resource.object + streaming_op = None + try: + streaming_op = make_op(transport) + streaming_op.attach(resource) + for item in streaming_op: + yield item + break + except BadResource as e: + resource.errored = True + # NB: *only* re-try if connection closed happened + # at the start of the streaming op + if first_try and not e.mid_stream: + continue + else: + raise + finally: + first_try = False + if streaming_op: + streaming_op.close() + def _with_retries(self, pool, fn): """ Performs the passed function with retries against the given pool. @@ -58,20 +154,38 @@ def _with_retries(self, pool, fn): def _skip_bad_nodes(transport): return transport._node not in skip_nodes - for retry in range(self.RETRY_COUNT): + retry_count = self.retries - 1 + first_try = True + current_try = 0 + while True: try: - with pool.take(_filter=_skip_bad_nodes) as transport: + with pool.transaction( + _filter=_skip_bad_nodes, + yield_resource=True) as resource: + transport = resource.object try: return fn(transport) - except (IOError, httplib.HTTPException) as e: + except (IOError, HTTPException, ConnectionClosed) as e: + resource.errored = True if _is_retryable(e): transport._node.error_rate.incr(1) skip_nodes.append(transport._node) - raise BadResource(e) + if first_try: + continue + else: + raise BadResource(e) else: - raise e - except BadResource: - continue + raise + except BadResource as e: + if current_try < retry_count: + resource.errored = True + current_try += 1 + continue + else: + # Re-raise the inner exception + raise e.args[0] + finally: + first_try = False def _choose_pool(self, protocol=None): """ @@ -84,12 +198,15 @@ def _choose_pool(self, protocol=None): """ if not protocol: protocol = self.protocol - if protocol in ['http', 'https']: + if protocol == 'http': pool = self._http_pool - elif protocol == 'pbc': - pool = self._pb_pool + elif protocol == 'tcp' or protocol == 'pbc': + pool = self._tcp_pool else: raise ValueError("invalid protocol %s" % protocol) + if pool is None or self._closed: + # NB: GH-500, this can happen if client is closed + raise RuntimeError("Client is closed.") return pool @@ -102,13 +219,14 @@ def _is_retryable(error): :type error: Exception :rtype: boolean """ - return is_pbc_retryable(error) or is_http_retryable(error) + return is_tcp_retryable(error) or is_http_retryable(error) +# http://thecodeship.com/patterns/guide-to-python-function-decorators/ def retryable(fn, protocol=None): """ Wraps a client operation that can be retried according to the set - RETRY_COUNT. Used internally. + :attr:`RiakClient.retries`. Used internally. """ def wrapper(self, *args, **kwargs): pool = self._choose_pool(protocol) @@ -118,6 +236,9 @@ def thunk(transport): return self._with_retries(pool, thunk) + wrapper.__doc__ = fn.__doc__ + wrapper.__repr__ = fn.__repr__ + return wrapper diff --git a/riak/codecs/__init__.py b/riak/codecs/__init__.py new file mode 100644 index 00000000..b824fcc0 --- /dev/null +++ b/riak/codecs/__init__.py @@ -0,0 +1,42 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +import riak.pb.messages + +from riak import RiakError +from riak.codecs.util import parse_pbuf_msg +from riak.util import bytes_to_str + +Msg = collections.namedtuple('Msg', + ['msg_code', 'data', 'resp_code']) + + +class Codec(object): + def parse_msg(self): + raise NotImplementedError('parse_msg not implemented') + + def maybe_incorrect_code(self, resp_code, expect=None): + if expect and resp_code != expect: + raise RiakError("unexpected message code: %d, expected %d" + % (resp_code, expect)) + + def maybe_riak_error(self, msg_code, data=None): + if msg_code == riak.pb.messages.MSG_CODE_ERROR_RESP: + if data is None: + raise RiakError('no error provided!') + else: + err = parse_pbuf_msg(msg_code, data) + raise RiakError(bytes_to_str(err.errmsg)) diff --git a/riak/codecs/http.py b/riak/codecs/http.py new file mode 100644 index 00000000..b981b77a --- /dev/null +++ b/riak/codecs/http.py @@ -0,0 +1,324 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import csv +import six + +from cgi import parse_header +from email import message_from_string +from email.utils import parsedate_tz, mktime_tz +from xml.etree import ElementTree +from riak import RiakError +from riak.content import RiakContent +from riak.riak_object import VClock +from riak.multidict import MultiDict +from riak.transports.http.search import XMLSearchResult +from riak.util import decode_index_value, bytes_to_str + +if six.PY2: + from urllib import unquote_plus +else: + from urllib.parse import unquote_plus + + +# subtract length of "Link: " header string and newline +MAX_LINK_HEADER_SIZE = 8192 - 8 + + +class HttpCodec(object): + """ + Methods for HTTP transport that marshals and unmarshals HTTP + messages. + """ + + def _parse_body(self, robj, response, expected_statuses): + """ + Parse the body of an object response and populate the object. + """ + # If no response given, then return. + if response is None: + return None + + status, headers, data = response + + # Check if the server is down(status==0) + if not status: + m = 'Could not contact Riak Server: http://{0}:{1}!'.format( + self._node.host, self._node.http_port) + raise RiakError(m) + + # Make sure expected code came back + self.check_http_code(status, expected_statuses) + + if 'x-riak-vclock' in headers: + robj.vclock = VClock(headers['x-riak-vclock'], 'base64') + + # If 404(Not Found), then clear the object. + if status == 404: + robj.siblings = [] + return None + # If 201 Created, we need to extract the location and set the + # key on the object. + elif status == 201: + robj.key = headers['location'].strip().split('/')[-1] + # If 300(Siblings), apply the siblings to the object + elif status == 300: + ctype, params = parse_header(headers['content-type']) + if ctype == 'multipart/mixed': + if six.PY3: + data = bytes_to_str(data) + boundary = re.compile('\r?\n--%s(?:--)?\r?\n' % + re.escape(params['boundary'])) + parts = [message_from_string(p) + for p in re.split(boundary, data)[1:-1]] + robj.siblings = [self._parse_sibling(RiakContent(robj), + part.items(), + part.get_payload()) + for part in parts] + + # Invoke sibling-resolution logic + if robj.resolver is not None: + robj.resolver(robj) + + return robj + else: + raise Exception('unexpected sibling response format: {0}'. + format(ctype)) + + robj.siblings = [self._parse_sibling(RiakContent(robj), + headers.items(), + data)] + + return robj + + def _parse_sibling(self, sibling, headers, data): + """ + Parses a single sibling out of a response. + """ + + sibling.exists = True + + # Parse the headers... + for header, value in headers: + header = header.lower() + if header == 'content-type': + sibling.content_type, sibling.charset = \ + self._parse_content_type(value) + elif header == 'etag': + sibling.etag = value + elif header == 'link': + sibling.links = self._parse_links(value) + elif header == 'last-modified': + sibling.last_modified = mktime_tz(parsedate_tz(value)) + elif header.startswith('x-riak-meta-'): + metakey = header.replace('x-riak-meta-', '') + sibling.usermeta[metakey] = value + elif header.startswith('x-riak-index-'): + field = header.replace('x-riak-index-', '') + reader = csv.reader([value], skipinitialspace=True) + for line in reader: + for token in line: + token = decode_index_value(field, token) + sibling.add_index(field, token) + elif header == 'x-riak-deleted': + sibling.exists = False + + sibling.encoded_data = data + + return sibling + + def _to_link_header(self, link): + """ + Convert the link tuple to a link header string. Used internally. + """ + try: + bucket, key, tag = link + except ValueError: + raise RiakError("Invalid link tuple %s" % link) + tag = tag if tag is not None else bucket + url = self.object_path(bucket, key) + header = '<%s>; riaktag="%s"' % (url, tag) + return header + + def _parse_links(self, linkHeaders): + links = [] + oldform = "; ?riaktag=\"([^\"]+)\"" + newform = "; ?riaktag=\"([^\"]+)\"" + for linkHeader in linkHeaders.strip().split(','): + linkHeader = linkHeader.strip() + matches = (re.match(oldform, linkHeader) or + re.match(newform, linkHeader)) + if matches is not None: + link = (unquote_plus(matches.group(2)), + unquote_plus(matches.group(3)), + unquote_plus(matches.group(4))) + links.append(link) + return links + + def _add_links_for_riak_object(self, robject, headers): + links = robject.links + if links: + current_header = '' + for link in links: + header = self._to_link_header(link) + if len(current_header + header) > MAX_LINK_HEADER_SIZE: + headers.add('Link', current_header) + current_header = '' + + if current_header != '': + header = ', ' + header + current_header += header + + headers.add('Link', current_header) + + return headers + + def _build_put_headers(self, robj, if_none_match=False): + """Build the headers for a POST/PUT request.""" + + # Construct the headers... + if robj.charset is not None: + content_type = ('%s; charset="%s"' % + (robj.content_type, robj.charset)) + else: + content_type = robj.content_type + + headers = MultiDict({'Content-Type': content_type, + 'X-Riak-ClientId': self._client_id}) + + # Add the vclock if it exists... + if robj.vclock is not None: + headers['X-Riak-Vclock'] = robj.vclock.encode('base64') + + # Create the header from metadata + self._add_links_for_riak_object(robj, headers) + + for key in robj.usermeta.keys(): + headers['X-Riak-Meta-%s' % key] = robj.usermeta[key] + + for field, value in robj.indexes: + key = 'X-Riak-Index-%s' % field + if key in headers: + headers[key] += ", " + str(value) + else: + headers[key] = str(value) + + if if_none_match: + headers['If-None-Match'] = '*' + + return headers + + def _normalize_json_search_response(self, json): + """ + Normalizes a JSON search response so that PB and HTTP have the + same return value + """ + result = {} + if 'facet_counts' in json: + result['facet_counts'] = json[u'facet_counts'] + if 'grouped' in json: + result['grouped'] = json[u'grouped'] + if 'stats' in json: + result['stats'] = json[u'stats'] + if u'response' in json: + result['num_found'] = json[u'response'][u'numFound'] + result['max_score'] = float(json[u'response'][u'maxScore']) + docs = [] + for doc in json[u'response'][u'docs']: + resdoc = {} + if u'_yz_rk' in doc: + # Is this a Riak 2.0 result? + resdoc = doc + else: + # Riak Search 1.0 Legacy assumptions about format + resdoc[u'id'] = doc[u'id'] + if u'fields' in doc: + for k, v in six.iteritems(doc[u'fields']): + resdoc[k] = v + docs.append(resdoc) + result['docs'] = docs + return result + + def _normalize_xml_search_response(self, xml): + """ + Normalizes an XML search response so that PB and HTTP have the + same return value + """ + target = XMLSearchResult() + parser = ElementTree.XMLParser(target=target) + parser.feed(xml) + return parser.close() + + def _parse_content_type(self, value): + """ + Split the content-type header into two parts: + 1) Actual main/sub encoding type + 2) charset + + :param value: Complete MIME content-type string + """ + content_type, params = parse_header(value) + if 'charset' in params: + charset = params['charset'] + else: + charset = None + return content_type, charset + + def _decode_datatype(self, dtype, value): + if not dtype == 'map': + return value + map = {} + for key in value: + field = self._map_key_to_pair(key) + map[field] = self._decode_datatype(field[1], value[key]) + return map + + def _map_key_to_pair(self, key): + name, _, type = key.rpartition('_') + return (name, type) + + def _map_pair_to_key(self, pair): + return "_".join(pair) + + def _encode_dt_op(self, dtype, op): + if dtype in ('counter', 'register'): + # ('increment', some_int) + # ('assign', some_str) + return dict([op]) + elif dtype == 'flag': + return op + elif dtype == 'set': + set_op = {} + if 'adds' in op: + set_op['add_all'] = op['adds'] + if 'removes' in op: + set_op['remove_all'] = op['removes'] + return set_op + elif dtype == 'hll': + hll_op = {} + if 'adds' in op: + hll_op['add_all'] = op['adds'] + return hll_op + elif dtype == 'map': + map_op = {} + for fop in op: + fopname = fop[0] + fopkey = self._map_pair_to_key(fop[1]) + if fopname in ('add', 'remove'): + map_op.setdefault(fopname, []).append(fopkey) + elif fopname == 'update': + updates = map_op.setdefault(fopname, {}) + updates[fopkey] = self._encode_dt_op(fop[1][1], fop[2]) + return map_op diff --git a/riak/codecs/pbuf.py b/riak/codecs/pbuf.py new file mode 100644 index 00000000..0b4de2a6 --- /dev/null +++ b/riak/codecs/pbuf.py @@ -0,0 +1,1276 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import six + +import riak.pb.messages +import riak.pb.riak_pb2 +import riak.pb.riak_dt_pb2 +import riak.pb.riak_kv_pb2 +import riak.pb.riak_ts_pb2 + +from riak import RiakError +from riak.codecs import Codec, Msg +from riak.codecs.util import parse_pbuf_msg +from riak.content import RiakContent +from riak.pb.riak_ts_pb2 import TsColumnType +from riak.riak_object import VClock +from riak.ts_object import TsColumns +from riak.util import decode_index_value, str_to_bytes, bytes_to_str, \ + unix_time_millis, datetime_from_unix_time_millis +from riak.multidict import MultiDict + + +def _invert(d): + out = {} + for key in d: + value = d[key] + out[value] = key + return out + + +REPL_TO_PY = { + riak.pb.riak_pb2.RpbBucketProps.FALSE: False, + riak.pb.riak_pb2.RpbBucketProps.TRUE: True, + riak.pb.riak_pb2.RpbBucketProps.REALTIME: 'realtime', + riak.pb.riak_pb2.RpbBucketProps.FULLSYNC: 'fullsync' +} + +REPL_TO_PB = _invert(REPL_TO_PY) + +RIAKC_RW_ONE = 4294967294 +RIAKC_RW_QUORUM = 4294967293 +RIAKC_RW_ALL = 4294967292 +RIAKC_RW_DEFAULT = 4294967291 + +QUORUM_TO_PB = {'default': RIAKC_RW_DEFAULT, + 'all': RIAKC_RW_ALL, + 'quorum': RIAKC_RW_QUORUM, + 'one': RIAKC_RW_ONE} + +QUORUM_TO_PY = _invert(QUORUM_TO_PB) + +NORMAL_PROPS = ['n_val', 'allow_mult', 'last_write_wins', 'old_vclock', + 'young_vclock', 'big_vclock', 'small_vclock', 'basic_quorum', + 'notfound_ok', 'search', 'backend', 'search_index', 'datatype', + 'write_once', 'hll_precision'] +COMMIT_HOOK_PROPS = ['precommit', 'postcommit'] +MODFUN_PROPS = ['chash_keyfun', 'linkfun'] +QUORUM_PROPS = ['r', 'pr', 'w', 'pw', 'dw', 'rw'] + +MAP_FIELD_TYPES = { + riak.pb.riak_dt_pb2.MapField.COUNTER: 'counter', + riak.pb.riak_dt_pb2.MapField.SET: 'set', + riak.pb.riak_dt_pb2.MapField.REGISTER: 'register', + riak.pb.riak_dt_pb2.MapField.FLAG: 'flag', + riak.pb.riak_dt_pb2.MapField.MAP: 'map', + 'counter': riak.pb.riak_dt_pb2.MapField.COUNTER, + 'set': riak.pb.riak_dt_pb2.MapField.SET, + 'register': riak.pb.riak_dt_pb2.MapField.REGISTER, + 'flag': riak.pb.riak_dt_pb2.MapField.FLAG, + 'map': riak.pb.riak_dt_pb2.MapField.MAP +} + +DT_FETCH_TYPES = { + riak.pb.riak_dt_pb2.DtFetchResp.COUNTER: 'counter', + riak.pb.riak_dt_pb2.DtFetchResp.SET: 'set', + riak.pb.riak_dt_pb2.DtFetchResp.MAP: 'map', + riak.pb.riak_dt_pb2.DtFetchResp.HLL: 'hll' +} + + +class PbufCodec(Codec): + ''' + Protobuffs Encoding and decoding methods for TcpTransport. + ''' + + def __init__(self, + client_timeouts=False, quorum_controls=False, + tombstone_vclocks=False, bucket_types=False): + if riak.pb is None: + raise NotImplementedError("this codec is not available") + self._client_timeouts = client_timeouts + self._quorum_controls = quorum_controls + self._tombstone_vclocks = tombstone_vclocks + self._bucket_types = bucket_types + + def parse_msg(self, msg_code, data): + return parse_pbuf_msg(msg_code, data) + + def encode_auth(self, username, password): + req = riak.pb.riak_pb2.RpbAuthReq() + req.user = str_to_bytes(username) + req.password = str_to_bytes(password) + mc = riak.pb.messages.MSG_CODE_AUTH_REQ + rc = riak.pb.messages.MSG_CODE_AUTH_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_ping(self): + return Msg(riak.pb.messages.MSG_CODE_PING_REQ, None, + riak.pb.messages.MSG_CODE_PING_RESP) + + def encode_quorum(self, rw): + """ + Converts a symbolic quorum value into its on-the-wire + equivalent. + + :param rw: the quorum + :type rw: string, integer + :rtype: integer + """ + if rw in QUORUM_TO_PB: + return QUORUM_TO_PB[rw] + elif type(rw) is int and rw >= 0: + return rw + else: + return None + + def decode_quorum(self, rw): + """ + Converts a protobuf quorum value to a symbolic value if + necessary. + + :param rw: the quorum + :type rw: int + :rtype int or string + """ + if rw in QUORUM_TO_PY: + return QUORUM_TO_PY[rw] + else: + return rw + + def decode_contents(self, contents, obj): + """ + Decodes the list of siblings from the protobuf representation + into the object. + + :param contents: a list of RpbContent messages + :type contents: list + :param obj: a RiakObject + :type obj: RiakObject + :rtype RiakObject + """ + obj.siblings = [self.decode_content(c, RiakContent(obj)) + for c in contents] + # Invoke sibling-resolution logic + if len(obj.siblings) > 1 and obj.resolver is not None: + obj.resolver(obj) + return obj + + def decode_content(self, rpb_content, sibling): + """ + Decodes a single sibling from the protobuf representation into + a RiakObject. + + :param rpb_content: a single RpbContent message + :type rpb_content: riak.pb.riak_pb2.RpbContent + :param sibling: a RiakContent sibling container + :type sibling: RiakContent + :rtype: RiakContent + """ + + if rpb_content.HasField("deleted") and rpb_content.deleted: + sibling.exists = False + else: + sibling.exists = True + if rpb_content.HasField("content_type"): + sibling.content_type = bytes_to_str(rpb_content.content_type) + if rpb_content.HasField("charset"): + sibling.charset = bytes_to_str(rpb_content.charset) + if rpb_content.HasField("content_encoding"): + sibling.content_encoding = \ + bytes_to_str(rpb_content.content_encoding) + if rpb_content.HasField("vtag"): + sibling.etag = bytes_to_str(rpb_content.vtag) + + sibling.links = [self.decode_link(link) + for link in rpb_content.links] + if rpb_content.HasField("last_mod"): + sibling.last_modified = float(rpb_content.last_mod) + if rpb_content.HasField("last_mod_usecs"): + sibling.last_modified += rpb_content.last_mod_usecs / 1000000.0 + + sibling.usermeta = dict([(bytes_to_str(usermd.key), + bytes_to_str(usermd.value)) + for usermd in rpb_content.usermeta]) + sibling.indexes = set([(bytes_to_str(index.key), + decode_index_value(index.key, index.value)) + for index in rpb_content.indexes]) + sibling.encoded_data = rpb_content.value + + return sibling + + def encode_content(self, robj, rpb_content): + """ + Fills an RpbContent message with the appropriate data and + metadata from a RiakObject. + + :param robj: a RiakObject + :type robj: RiakObject + :param rpb_content: the protobuf message to fill + :type rpb_content: riak.pb.riak_pb2.RpbContent + """ + if robj.content_type: + rpb_content.content_type = str_to_bytes(robj.content_type) + if robj.charset: + rpb_content.charset = str_to_bytes(robj.charset) + if robj.content_encoding: + rpb_content.content_encoding = str_to_bytes(robj.content_encoding) + for uk in robj.usermeta: + pair = rpb_content.usermeta.add() + pair.key = str_to_bytes(uk) + pair.value = str_to_bytes(robj.usermeta[uk]) + for link in robj.links: + pb_link = rpb_content.links.add() + try: + bucket, key, tag = link + except ValueError: + raise RiakError("Invalid link tuple %s" % link) + + pb_link.bucket = str_to_bytes(bucket) + pb_link.key = str_to_bytes(key) + if tag: + pb_link.tag = str_to_bytes(tag) + else: + pb_link.tag = str_to_bytes('') + + for field, value in robj.indexes: + pair = rpb_content.indexes.add() + pair.key = str_to_bytes(field) + pair.value = str_to_bytes(str(value)) + + # Python 2.x data is stored in a string + if six.PY2: + rpb_content.value = str(robj.encoded_data) + else: + rpb_content.value = robj.encoded_data + + def decode_link(self, link): + """ + Decodes an RpbLink message into a tuple + + :param link: an RpbLink message + :type link: riak.pb.riak_pb2.RpbLink + :rtype tuple + """ + + if link.HasField("bucket"): + bucket = bytes_to_str(link.bucket) + else: + bucket = None + if link.HasField("key"): + key = bytes_to_str(link.key) + else: + key = None + if link.HasField("tag"): + tag = bytes_to_str(link.tag) + else: + tag = None + + return (bucket, key, tag) + + def decode_index_value(self, index, value): + """ + Decodes a secondary index value into the correct Python type. + :param index: the name of the index + :type index: str + :param value: the value of the index entry + :type value: str + :rtype str or int + """ + if index.endswith("_int"): + return int(value) + else: + return bytes_to_str(value) + + def encode_bucket_props(self, props, msg): + """ + Encodes a dict of bucket properties into the protobuf message. + + :param props: bucket properties + :type props: dict + :param msg: the protobuf message to fill + :type msg: riak.pb.riak_pb2.RpbSetBucketReq + """ + for prop in NORMAL_PROPS: + if prop in props and props[prop] is not None: + if isinstance(props[prop], six.string_types): + setattr(msg.props, prop, str_to_bytes(props[prop])) + else: + setattr(msg.props, prop, props[prop]) + for prop in COMMIT_HOOK_PROPS: + if prop in props: + setattr(msg.props, 'has_' + prop, True) + self.encode_hooklist(props[prop], getattr(msg.props, prop)) + for prop in MODFUN_PROPS: + if prop in props and props[prop] is not None: + self.encode_modfun(props[prop], getattr(msg.props, prop)) + for prop in QUORUM_PROPS: + if prop in props and props[prop] not in (None, 'default'): + value = self.encode_quorum(props[prop]) + if value is not None: + if isinstance(value, six.string_types): + setattr(msg.props, prop, str_to_bytes(value)) + else: + setattr(msg.props, prop, value) + if 'repl' in props: + msg.props.repl = REPL_TO_PB[props['repl']] + + return msg + + def decode_bucket_props(self, msg): + """ + Decodes the protobuf bucket properties message into a dict. + + :param msg: the protobuf message to decode + :type msg: riak.pb.riak_pb2.RpbBucketProps + :rtype dict + """ + props = {} + for prop in NORMAL_PROPS: + if msg.HasField(prop): + props[prop] = getattr(msg, prop) + if isinstance(props[prop], bytes): + props[prop] = bytes_to_str(props[prop]) + for prop in COMMIT_HOOK_PROPS: + if getattr(msg, 'has_' + prop): + props[prop] = self.decode_hooklist(getattr(msg, prop)) + for prop in MODFUN_PROPS: + if msg.HasField(prop): + props[prop] = self.decode_modfun(getattr(msg, prop)) + for prop in QUORUM_PROPS: + if msg.HasField(prop): + props[prop] = self.decode_quorum(getattr(msg, prop)) + if msg.HasField('repl'): + props['repl'] = REPL_TO_PY[msg.repl] + return props + + def decode_modfun(self, modfun): + """ + Decodes a protobuf modfun pair into a dict with 'mod' and + 'fun' keys. Used in bucket properties. + + :param modfun: the protobuf message to decode + :type modfun: riak.pb.riak_pb2.RpbModFun + :rtype dict + """ + return {'mod': bytes_to_str(modfun.module), + 'fun': bytes_to_str(modfun.function)} + + def encode_modfun(self, props, msg=None): + """ + Encodes a dict with 'mod' and 'fun' keys into a protobuf + modfun pair. Used in bucket properties. + + :param props: the module/function pair + :type props: dict + :param msg: the protobuf message to fill + :type msg: riak.pb.riak_pb2.RpbModFun + :rtype riak.pb.riak_pb2.RpbModFun + """ + if msg is None: + msg = riak.pb.riak_pb2.RpbModFun() + msg.module = str_to_bytes(props['mod']) + msg.function = str_to_bytes(props['fun']) + return msg + + def decode_hooklist(self, hooklist): + """ + Decodes a list of protobuf commit hooks into their python + equivalents. Used in bucket properties. + + :param hooklist: a list of protobuf commit hooks + :type hooklist: list + :rtype list + """ + return [self.decode_hook(hook) for hook in hooklist] + + def encode_hooklist(self, hooklist, msg): + """ + Encodes a list of commit hooks into their protobuf equivalent. + Used in bucket properties. + + :param hooklist: a list of commit hooks + :type hooklist: list + :param msg: a protobuf field that is a list of commit hooks + """ + for hook in hooklist: + pbhook = msg.add() + self.encode_hook(hook, pbhook) + + def decode_hook(self, hook): + """ + Decodes a protobuf commit hook message into a dict. Used in + bucket properties. + + :param hook: the hook to decode + :type hook: riak.pb.riak_pb2.RpbCommitHook + :rtype dict + """ + if hook.HasField('modfun'): + return self.decode_modfun(hook.modfun) + else: + return {'name': bytes_to_str(hook.name)} + + def encode_hook(self, hook, msg): + """ + Encodes a commit hook dict into the protobuf message. Used in + bucket properties. + + :param hook: the hook to encode + :type hook: dict + :param msg: the protobuf message to fill + :type msg: riak.pb.riak_pb2.RpbCommitHook + :rtype riak.pb.riak_pb2.RpbCommitHook + """ + if 'name' in hook: + msg.name = str_to_bytes(hook['name']) + else: + self.encode_modfun(hook, msg.modfun) + return msg + + def encode_index_req(self, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, + continuation=None, timeout=None, term_regex=None, + streaming=False): + """ + Encodes a secondary index request into the protobuf message. + + :param bucket: the bucket whose index to query + :type bucket: string + :param index: the index to query + :type index: string + :param startkey: the value or beginning of the range + :type startkey: integer, string + :param endkey: the end of the range + :type endkey: integer, string + :param return_terms: whether to return the index term with the key + :type return_terms: bool + :param max_results: the maximum number of results to return (page size) + :type max_results: integer + :param continuation: the opaque continuation returned from a + previous paginated request + :type continuation: string + :param timeout: a timeout value in milliseconds, or 'infinity' + :type timeout: int + :param term_regex: a regular expression used to filter index terms + :type term_regex: string + :param streaming: encode as streaming request + :type streaming: bool + :rtype riak.pb.riak_kv_pb2.RpbIndexReq + """ + req = riak.pb.riak_kv_pb2.RpbIndexReq( + bucket=str_to_bytes(bucket.name), + index=str_to_bytes(index)) + self._add_bucket_type(req, bucket.bucket_type) + if endkey is not None: + req.qtype = riak.pb.riak_kv_pb2.RpbIndexReq.range + req.range_min = str_to_bytes(str(startkey)) + req.range_max = str_to_bytes(str(endkey)) + else: + req.qtype = riak.pb.riak_kv_pb2.RpbIndexReq.eq + req.key = str_to_bytes(str(startkey)) + if return_terms is not None: + req.return_terms = return_terms + if max_results: + req.max_results = max_results + if continuation: + req.continuation = str_to_bytes(continuation) + if timeout: + if timeout == 'infinity': + req.timeout = 0 + else: + req.timeout = timeout + if term_regex: + req.term_regex = str_to_bytes(term_regex) + req.stream = streaming + mc = riak.pb.messages.MSG_CODE_INDEX_REQ + rc = riak.pb.messages.MSG_CODE_INDEX_RESP + return Msg(mc, req.SerializeToString(), rc) + + def decode_index_req(self, resp, index, + return_terms=None, max_results=None): + if return_terms and resp.results: + results = [(decode_index_value(index, pair.key), + bytes_to_str(pair.value)) + for pair in resp.results] + else: + results = resp.keys[:] + if six.PY3: + results = [bytes_to_str(key) for key in resp.keys] + + if max_results is not None and resp.HasField('continuation'): + return (results, bytes_to_str(resp.continuation)) + else: + return (results, None) + + def decode_search_index(self, index): + """ + Fills an RpbYokozunaIndex message with the appropriate data. + + :param index: a yz index message + :type index: riak.pb.riak_yokozuna_pb2.RpbYokozunaIndex + :rtype dict + """ + result = {} + result['name'] = bytes_to_str(index.name) + if index.HasField('schema'): + result['schema'] = bytes_to_str(index.schema) + if index.HasField('n_val'): + result['n_val'] = index.n_val + return result + + def _add_bucket_type(self, req, bucket_type): + if bucket_type and not bucket_type.is_default(): + if not self._bucket_types: + raise NotImplementedError( + 'Server does not support bucket-types') + req.type = str_to_bytes(bucket_type.name) + + def encode_search_query(self, req, **kwargs): + if 'rows' in kwargs: + req.rows = kwargs['rows'] + if 'start' in kwargs: + req.start = kwargs['start'] + if 'sort' in kwargs: + req.sort = str_to_bytes(kwargs['sort']) + if 'filter' in kwargs: + req.filter = str_to_bytes(kwargs['filter']) + if 'df' in kwargs: + req.df = str_to_bytes(kwargs['df']) + if 'op' in kwargs: + req.op = str_to_bytes(kwargs['op']) + if 'q.op' in kwargs: + req.op = kwargs['q.op'] + if 'fl' in kwargs: + if isinstance(kwargs['fl'], list): + req.fl.extend([str_to_bytes(fl) for fl in kwargs['fl']]) + else: + req.fl.append(str_to_bytes(kwargs['fl'])) + if 'presort' in kwargs: + req.presort = kwargs['presort'] + + def decode_search_doc(self, doc): + resultdoc = MultiDict() + for pair in doc.fields: + if six.PY2: + ukey = unicode(pair.key, 'utf-8') # noqa + uval = unicode(pair.value, 'utf-8') # noqa + else: + ukey = bytes_to_str(pair.key) + uval = bytes_to_str(pair.value) + resultdoc.add(ukey, uval) + return resultdoc.mixed() + + def decode_dt_fetch(self, resp): + dtype = DT_FETCH_TYPES.get(resp.type) + if dtype is None: + raise ValueError("Unknown datatype on wire: {}".format(resp.type)) + + value = self.decode_dt_value(dtype, resp.value) + + if resp.HasField('context'): + context = resp.context[:] + else: + context = None + + return dtype, value, context + + def decode_dt_value(self, dtype, msg): + if dtype == 'counter': + return msg.counter_value + elif dtype == 'set': + return self.decode_set_value(msg.set_value) + elif dtype == 'hll': + return self.decode_hll_value(msg.hll_value) + elif dtype == 'map': + return self.decode_map_value(msg.map_value) + + def encode_dt_options(self, req, **kwargs): + for q in ['r', 'pr', 'w', 'dw', 'pw']: + if q in kwargs and kwargs[q] is not None: + setattr(req, q, self.encode_quorum(kwargs[q])) + + for o in ['basic_quorum', 'notfound_ok', 'timeout', 'return_body', + 'include_context']: + if o in kwargs and kwargs[o] is not None: + setattr(req, o, kwargs[o]) + + def decode_map_value(self, entries): + out = {} + for entry in entries: + name = bytes_to_str(entry.field.name[:]) + dtype = MAP_FIELD_TYPES[entry.field.type] + if dtype == 'counter': + value = entry.counter_value + elif dtype == 'set': + value = self.decode_set_value(entry.set_value) + elif dtype == 'register': + value = bytes_to_str(entry.register_value[:]) + elif dtype == 'flag': + value = entry.flag_value + elif dtype == 'map': + value = self.decode_map_value(entry.map_value) + else: + raise ValueError( + 'Map may not contain datatype: {}' + .format(dtype)) + out[(name, dtype)] = value + return out + + def decode_set_value(self, set_value): + return [bytes_to_str(string[:]) for string in set_value] + + def decode_hll_value(self, hll_value): + return int(hll_value) + + def encode_dt_op(self, dtype, req, op): + if dtype == 'counter': + req.op.counter_op.increment = op[1] + elif dtype == 'set': + self.encode_set_op(req.op, op) + elif dtype == 'hll': + self.encode_hll_op(req.op, op) + elif dtype == 'map': + self.encode_map_op(req.op.map_op, op) + else: + raise TypeError("Cannot send operation on datatype {!r}". + format(dtype)) + + def encode_set_op(self, msg, op): + if 'adds' in op: + msg.set_op.adds.extend(str_to_bytes(op['adds'])) + if 'removes' in op: + msg.set_op.removes.extend(str_to_bytes(op['removes'])) + + def encode_hll_op(self, msg, op): + if 'adds' in op: + msg.hll_op.adds.extend(str_to_bytes(op['adds'])) + + def encode_map_op(self, msg, ops): + for op in ops: + name, dtype = op[1] + ftype = MAP_FIELD_TYPES[dtype] + if op[0] == 'add': + add = msg.adds.add() + add.name = str_to_bytes(name) + add.type = ftype + elif op[0] == 'remove': + remove = msg.removes.add() + remove.name = str_to_bytes(name) + remove.type = ftype + elif op[0] == 'update': + update = msg.updates.add() + update.field.name = str_to_bytes(name) + update.field.type = ftype + self.encode_map_update(dtype, update, op[2]) + + def encode_map_update(self, dtype, msg, op): + if dtype == 'counter': + # ('increment', some_int) + msg.counter_op.increment = op[1] + elif dtype == 'set': + self.encode_set_op(msg, op) + elif dtype == 'map': + self.encode_map_op(msg.map_op, op) + elif dtype == 'register': + # ('assign', some_str) + msg.register_op = str_to_bytes(op[1]) + elif dtype == 'flag': + if op == 'enable': + msg.flag_op = riak.pb.riak_dt_pb2.MapUpdate.ENABLE + else: + msg.flag_op = riak.pb.riak_dt_pb2.MapUpdate.DISABLE + else: + raise ValueError( + 'Map may not contain datatype: {}' + .format(dtype)) + + def encode_to_ts_cell(self, cell, ts_cell): + if cell is not None: + if isinstance(cell, datetime.datetime): + ts_cell.timestamp_value = unix_time_millis(cell) + elif isinstance(cell, bool): + ts_cell.boolean_value = cell + elif isinstance(cell, six.binary_type): + ts_cell.varchar_value = cell + elif isinstance(cell, six.text_type): + ts_cell.varchar_value = str_to_bytes(cell) + elif isinstance(cell, six.string_types): + ts_cell.varchar_value = str_to_bytes(cell) + elif (isinstance(cell, six.integer_types)): + ts_cell.sint64_value = cell + elif isinstance(cell, float): + ts_cell.double_value = cell + else: + t = type(cell) + raise RiakError("can't serialize type '{}', value '{}'" + .format(t, cell)) + + def encode_timeseries_keyreq(self, table, key, is_delete=False): + key_vals = None + if isinstance(key, list): + key_vals = key + else: + raise ValueError("key must be a list") + + req = riak.pb.riak_ts_pb2.TsGetReq() + mc = riak.pb.messages.MSG_CODE_TS_GET_REQ + rc = riak.pb.messages.MSG_CODE_TS_GET_RESP + if is_delete: + req = riak.pb.riak_ts_pb2.TsDelReq() + mc = riak.pb.messages.MSG_CODE_TS_DEL_REQ + rc = riak.pb.messages.MSG_CODE_TS_DEL_RESP + + req.table = str_to_bytes(table.name) + for cell in key_vals: + ts_cell = req.key.add() + self.encode_to_ts_cell(cell, ts_cell) + return Msg(mc, req.SerializeToString(), rc) + + def encode_timeseries_listkeysreq(self, table, timeout=None): + req = riak.pb.riak_ts_pb2.TsListKeysReq() + req.table = str_to_bytes(table.name) + if self._client_timeouts and timeout: + req.timeout = timeout + mc = riak.pb.messages.MSG_CODE_TS_LIST_KEYS_REQ + rc = riak.pb.messages.MSG_CODE_TS_LIST_KEYS_RESP + return Msg(mc, req.SerializeToString(), rc) + + def validate_timeseries_put_resp(self, resp_code, resp): + if resp is not None: + return True + else: + raise RiakError("missing response object") + + def encode_timeseries_put(self, tsobj): + """ + Fills an TsPutReq message with the appropriate data and + metadata from a TsObject. + + :param tsobj: a TsObject + :type tsobj: TsObject + :param req: the protobuf message to fill + :type req: riak.pb.riak_ts_pb2.TsPutReq + """ + req = riak.pb.riak_ts_pb2.TsPutReq() + req.table = str_to_bytes(tsobj.table.name) + + if tsobj.columns: + raise NotImplementedError("columns are not implemented yet") + + if tsobj.rows and isinstance(tsobj.rows, list): + for row in tsobj.rows: + tsr = req.rows.add() # NB: type TsRow + if not isinstance(row, list): + raise ValueError("TsObject row must be a list of values") + for cell in row: + tsc = tsr.cells.add() # NB: type TsCell + self.encode_to_ts_cell(cell, tsc) + else: + raise RiakError("TsObject requires a list of rows") + + mc = riak.pb.messages.MSG_CODE_TS_PUT_REQ + rc = riak.pb.messages.MSG_CODE_TS_PUT_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_timeseries_query(self, table, query, interpolations=None): + req = riak.pb.riak_ts_pb2.TsQueryReq() + q = query + if '{table}' in q: + q = q.format(table=table.name) + req.query.base = str_to_bytes(q) + mc = riak.pb.messages.MSG_CODE_TS_QUERY_REQ + rc = riak.pb.messages.MSG_CODE_TS_QUERY_RESP + return Msg(mc, req.SerializeToString(), rc) + + def decode_timeseries(self, resp, tsobj, + convert_timestamp=False): + """ + Fills an TsObject with the appropriate data and + metadata from a TsGetResp / TsQueryResp. + + :param resp: the protobuf message from which to process data + :type resp: riak.pb.riak_ts_pb2.TsQueryRsp or + riak.pb.riak_ts_pb2.TsGetResp + :param tsobj: a TsObject + :type tsobj: TsObject + :param convert_timestamp: Convert timestamps to datetime objects + :type tsobj: boolean + """ + if resp.columns is not None: + col_names = [] + col_types = [] + for col in resp.columns: + col_names.append(bytes_to_str(col.name)) + col_type = self.decode_timeseries_col_type(col.type) + col_types.append(col_type) + tsobj.columns = TsColumns(col_names, col_types) + + tsobj.rows = [] + if resp.rows is not None: + for row in resp.rows: + tsobj.rows.append( + self.decode_timeseries_row( + row, resp.columns, convert_timestamp)) + + def decode_timeseries_col_type(self, col_type): + # NB: these match the atom names for column types + if col_type == TsColumnType.Value('VARCHAR'): + return 'varchar' + elif col_type == TsColumnType.Value('SINT64'): + return 'sint64' + elif col_type == TsColumnType.Value('DOUBLE'): + return 'double' + elif col_type == TsColumnType.Value('TIMESTAMP'): + return 'timestamp' + elif col_type == TsColumnType.Value('BOOLEAN'): + return 'boolean' + elif col_type == TsColumnType.Value('BLOB'): + return 'blob' + else: + msg = 'could not decode column type: {}'.format(col_type) + raise RiakError(msg) + + def decode_timeseries_row(self, tsrow, tscols=None, + convert_timestamp=False): + """ + Decodes a TsRow into a list + + :param tsrow: the protobuf TsRow to decode. + :type tsrow: riak.pb.riak_ts_pb2.TsRow + :param tscols: the protobuf TsColumn data to help decode. + :type tscols: list + :rtype list + """ + row = [] + for i, cell in enumerate(tsrow.cells): + col = None + if tscols is not None: + col = tscols[i] + if cell.HasField('varchar_value'): + if col and not (col.type == TsColumnType.Value('VARCHAR') or + col.type == TsColumnType.Value('BLOB')): + raise TypeError('expected VARCHAR or BLOB column') + else: + row.append(cell.varchar_value) + elif cell.HasField('sint64_value'): + if col and col.type != TsColumnType.Value('SINT64'): + raise TypeError('expected SINT64 column') + else: + row.append(cell.sint64_value) + elif cell.HasField('double_value'): + if col and col.type != TsColumnType.Value('DOUBLE'): + raise TypeError('expected DOUBLE column') + else: + row.append(cell.double_value) + elif cell.HasField('timestamp_value'): + if col and col.type != TsColumnType.Value('TIMESTAMP'): + raise TypeError('expected TIMESTAMP column') + else: + dt = cell.timestamp_value + if convert_timestamp: + dt = datetime_from_unix_time_millis( + cell.timestamp_value) + row.append(dt) + elif cell.HasField('boolean_value'): + if col and col.type != TsColumnType.Value('BOOLEAN'): + raise TypeError('expected BOOLEAN column') + else: + row.append(cell.boolean_value) + else: + row.append(None) + return row + + def decode_preflist(self, item): + """ + Decodes a preflist response + + :param preflist: a bucket/key preflist + :type preflist: list of + riak.pb.riak_kv_pb2.RpbBucketKeyPreflistItem + :rtype dict + """ + result = {'partition': item.partition, + 'node': bytes_to_str(item.node), + 'primary': item. primary} + return result + + def encode_get(self, robj, r=None, pr=None, timeout=None, + basic_quorum=None, notfound_ok=None, + head_only=False): + bucket = robj.bucket + req = riak.pb.riak_kv_pb2.RpbGetReq() + if r: + req.r = self.encode_quorum(r) + if self._quorum_controls: + if pr: + req.pr = self.encode_quorum(pr) + if basic_quorum is not None: + req.basic_quorum = basic_quorum + if notfound_ok is not None: + req.notfound_ok = notfound_ok + if self._client_timeouts and timeout: + req.timeout = timeout + if self._tombstone_vclocks: + req.deletedvclock = True + req.bucket = str_to_bytes(bucket.name) + self._add_bucket_type(req, bucket.bucket_type) + req.key = str_to_bytes(robj.key) + req.head = head_only + mc = riak.pb.messages.MSG_CODE_GET_REQ + rc = riak.pb.messages.MSG_CODE_GET_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_put(self, robj, w=None, dw=None, pw=None, + return_body=True, if_none_match=False, + timeout=None): + bucket = robj.bucket + req = riak.pb.riak_kv_pb2.RpbPutReq() + if w: + req.w = self.encode_quorum(w) + if dw: + req.dw = self.encode_quorum(dw) + if self._quorum_controls and pw: + req.pw = self.encode_quorum(pw) + if return_body: + req.return_body = 1 + if if_none_match: + req.if_none_match = 1 + if self._client_timeouts and timeout: + req.timeout = timeout + req.bucket = str_to_bytes(bucket.name) + self._add_bucket_type(req, bucket.bucket_type) + if robj.key: + req.key = str_to_bytes(robj.key) + if robj.vclock: + req.vclock = robj.vclock.encode('binary') + self.encode_content(robj, req.content) + mc = riak.pb.messages.MSG_CODE_PUT_REQ + rc = riak.pb.messages.MSG_CODE_PUT_RESP + return Msg(mc, req.SerializeToString(), rc) + + def decode_get(self, robj, resp): + if resp is not None: + if resp.HasField('vclock'): + robj.vclock = VClock(resp.vclock, 'binary') + # We should do this even if there are no contents, i.e. + # the object is tombstoned + self.decode_contents(resp.content, robj) + else: + # "not found" returns an empty message, + # so let's make sure to clear the siblings + robj.siblings = [] + return robj + + def decode_put(self, robj, resp): + if resp is not None: + if resp.HasField('key'): + robj.key = bytes_to_str(resp.key) + if resp.HasField("vclock"): + robj.vclock = VClock(resp.vclock, 'binary') + if resp.content: + self.decode_contents(resp.content, robj) + elif not robj.key: + raise RiakError("missing response object") + return robj + + def encode_delete(self, robj, rw=None, r=None, + w=None, dw=None, pr=None, pw=None, + timeout=None): + req = riak.pb.riak_kv_pb2.RpbDelReq() + if rw: + req.rw = self.encode_quorum(rw) + if r: + req.r = self.encode_quorum(r) + if w: + req.w = self.encode_quorum(w) + if dw: + req.dw = self.encode_quorum(dw) + + if self._quorum_controls: + if pr: + req.pr = self.encode_quorum(pr) + if pw: + req.pw = self.encode_quorum(pw) + + if self._client_timeouts and timeout: + req.timeout = timeout + + use_vclocks = (self._tombstone_vclocks and + hasattr(robj, 'vclock') and robj.vclock) + if use_vclocks: + req.vclock = robj.vclock.encode('binary') + + bucket = robj.bucket + req.bucket = str_to_bytes(bucket.name) + self._add_bucket_type(req, bucket.bucket_type) + req.key = str_to_bytes(robj.key) + mc = riak.pb.messages.MSG_CODE_DEL_REQ + rc = riak.pb.messages.MSG_CODE_DEL_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_stream_keys(self, bucket, timeout=None): + req = riak.pb.riak_kv_pb2.RpbListKeysReq() + req.bucket = str_to_bytes(bucket.name) + if self._client_timeouts and timeout: + req.timeout = timeout + self._add_bucket_type(req, bucket.bucket_type) + mc = riak.pb.messages.MSG_CODE_LIST_KEYS_REQ + rc = riak.pb.messages.MSG_CODE_LIST_KEYS_RESP + return Msg(mc, req.SerializeToString(), rc) + + def decode_get_keys(self, stream): + keys = [] + for keylist in stream: + for key in keylist: + keys.append(bytes_to_str(key)) + return keys + + def decode_get_server_info(self, resp): + return {'node': bytes_to_str(resp.node), + 'server_version': bytes_to_str(resp.server_version)} + + def encode_get_client_id(self): + mc = riak.pb.messages.MSG_CODE_GET_CLIENT_ID_REQ + rc = riak.pb.messages.MSG_CODE_GET_CLIENT_ID_RESP + return Msg(mc, None, rc) + + def decode_get_client_id(self, resp): + return bytes_to_str(resp.client_id) + + def encode_set_client_id(self, client_id): + req = riak.pb.riak_kv_pb2.RpbSetClientIdReq() + req.client_id = str_to_bytes(client_id) + mc = riak.pb.messages.MSG_CODE_SET_CLIENT_ID_REQ + rc = riak.pb.messages.MSG_CODE_SET_CLIENT_ID_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_get_buckets(self, bucket_type, + timeout=None, streaming=False): + # Bucket streaming landed in the same release as timeouts, so + # we don't need to check the capability. + req = riak.pb.riak_kv_pb2.RpbListBucketsReq() + req.stream = streaming + self._add_bucket_type(req, bucket_type) + if self._client_timeouts and timeout: + req.timeout = timeout + mc = riak.pb.messages.MSG_CODE_LIST_BUCKETS_REQ + rc = riak.pb.messages.MSG_CODE_LIST_BUCKETS_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_get_bucket_props(self, bucket): + req = riak.pb.riak_pb2.RpbGetBucketReq() + req.bucket = str_to_bytes(bucket.name) + self._add_bucket_type(req, bucket.bucket_type) + mc = riak.pb.messages.MSG_CODE_GET_BUCKET_REQ + rc = riak.pb.messages.MSG_CODE_GET_BUCKET_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_set_bucket_props(self, bucket, props): + req = riak.pb.riak_pb2.RpbSetBucketReq() + req.bucket = str_to_bytes(bucket.name) + self._add_bucket_type(req, bucket.bucket_type) + self.encode_bucket_props(props, req) + mc = riak.pb.messages.MSG_CODE_SET_BUCKET_REQ + rc = riak.pb.messages.MSG_CODE_SET_BUCKET_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_clear_bucket_props(self, bucket): + req = riak.pb.riak_pb2.RpbResetBucketReq() + req.bucket = str_to_bytes(bucket.name) + self._add_bucket_type(req, bucket.bucket_type) + mc = riak.pb.messages.MSG_CODE_RESET_BUCKET_REQ + rc = riak.pb.messages.MSG_CODE_RESET_BUCKET_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_get_bucket_type_props(self, bucket_type): + req = riak.pb.riak_pb2.RpbGetBucketTypeReq() + req.type = str_to_bytes(bucket_type.name) + mc = riak.pb.messages.MSG_CODE_GET_BUCKET_TYPE_REQ + rc = riak.pb.messages.MSG_CODE_GET_BUCKET_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_set_bucket_type_props(self, bucket_type, props): + req = riak.pb.riak_pb2.RpbSetBucketTypeReq() + req.type = str_to_bytes(bucket_type.name) + self.encode_bucket_props(props, req) + mc = riak.pb.messages.MSG_CODE_SET_BUCKET_TYPE_REQ + rc = riak.pb.messages.MSG_CODE_SET_BUCKET_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_stream_mapred(self, content): + req = riak.pb.riak_kv_pb2.RpbMapRedReq() + req.request = str_to_bytes(content) + req.content_type = str_to_bytes("application/json") + mc = riak.pb.messages.MSG_CODE_MAP_RED_REQ + rc = riak.pb.messages.MSG_CODE_MAP_RED_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_create_search_index(self, index, schema=None, + n_val=None, timeout=None): + index = str_to_bytes(index) + idx = riak.pb.riak_yokozuna_pb2.RpbYokozunaIndex(name=index) + if schema: + idx.schema = str_to_bytes(schema) + if n_val: + idx.n_val = n_val + req = riak.pb.riak_yokozuna_pb2.RpbYokozunaIndexPutReq(index=idx) + if timeout is not None: + req.timeout = timeout + mc = riak.pb.messages.MSG_CODE_YOKOZUNA_INDEX_PUT_REQ + rc = riak.pb.messages.MSG_CODE_PUT_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_get_search_index(self, index): + req = riak.pb.riak_yokozuna_pb2.RpbYokozunaIndexGetReq( + name=str_to_bytes(index)) + mc = riak.pb.messages.MSG_CODE_YOKOZUNA_INDEX_GET_REQ + rc = riak.pb.messages.MSG_CODE_YOKOZUNA_INDEX_GET_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_list_search_indexes(self): + req = riak.pb.riak_yokozuna_pb2.RpbYokozunaIndexGetReq() + mc = riak.pb.messages.MSG_CODE_YOKOZUNA_INDEX_GET_REQ + rc = riak.pb.messages.MSG_CODE_YOKOZUNA_INDEX_GET_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_delete_search_index(self, index): + req = riak.pb.riak_yokozuna_pb2.RpbYokozunaIndexDeleteReq( + name=str_to_bytes(index)) + mc = riak.pb.messages.MSG_CODE_YOKOZUNA_INDEX_DELETE_REQ + rc = riak.pb.messages.MSG_CODE_DEL_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_create_search_schema(self, schema, content): + scma = riak.pb.riak_yokozuna_pb2.RpbYokozunaSchema( + name=str_to_bytes(schema), + content=str_to_bytes(content)) + req = riak.pb.riak_yokozuna_pb2.RpbYokozunaSchemaPutReq( + schema=scma) + mc = riak.pb.messages.MSG_CODE_YOKOZUNA_SCHEMA_PUT_REQ + rc = riak.pb.messages.MSG_CODE_PUT_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_get_search_schema(self, schema): + req = riak.pb.riak_yokozuna_pb2.RpbYokozunaSchemaGetReq( + name=str_to_bytes(schema)) + mc = riak.pb.messages.MSG_CODE_YOKOZUNA_SCHEMA_GET_REQ + rc = riak.pb.messages.MSG_CODE_YOKOZUNA_SCHEMA_GET_RESP + return Msg(mc, req.SerializeToString(), rc) + + def decode_get_search_schema(self, resp): + result = {} + result['name'] = bytes_to_str(resp.schema.name) + result['content'] = bytes_to_str(resp.schema.content) + return result + + def encode_search(self, index, query, **kwargs): + req = riak.pb.riak_search_pb2.RpbSearchQueryReq( + index=str_to_bytes(index), + q=str_to_bytes(query)) + self.encode_search_query(req, **kwargs) + mc = riak.pb.messages.MSG_CODE_SEARCH_QUERY_REQ + rc = riak.pb.messages.MSG_CODE_SEARCH_QUERY_RESP + return Msg(mc, req.SerializeToString(), rc) + + def decode_search(self, resp): + result = {} + if resp.HasField('max_score'): + result['max_score'] = resp.max_score + if resp.HasField('num_found'): + result['num_found'] = resp.num_found + result['docs'] = [self.decode_search_doc(doc) for doc in resp.docs] + return result + + def encode_get_counter(self, bucket, key, **kwargs): + req = riak.pb.riak_kv_pb2.RpbCounterGetReq() + req.bucket = str_to_bytes(bucket.name) + req.key = str_to_bytes(key) + if kwargs.get('r') is not None: + req.r = self.encode_quorum(kwargs['r']) + if kwargs.get('pr') is not None: + req.pr = self.encode_quorum(kwargs['pr']) + if kwargs.get('basic_quorum') is not None: + req.basic_quorum = kwargs['basic_quorum'] + if kwargs.get('notfound_ok') is not None: + req.notfound_ok = kwargs['notfound_ok'] + mc = riak.pb.messages.MSG_CODE_COUNTER_GET_REQ + rc = riak.pb.messages.MSG_CODE_COUNTER_GET_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_update_counter(self, bucket, key, value, **kwargs): + req = riak.pb.riak_kv_pb2.RpbCounterUpdateReq() + req.bucket = str_to_bytes(bucket.name) + req.key = str_to_bytes(key) + req.amount = value + if kwargs.get('w') is not None: + req.w = self.encode_quorum(kwargs['w']) + if kwargs.get('dw') is not None: + req.dw = self.encode_quorum(kwargs['dw']) + if kwargs.get('pw') is not None: + req.pw = self.encode_quorum(kwargs['pw']) + if kwargs.get('returnvalue') is not None: + req.returnvalue = kwargs['returnvalue'] + mc = riak.pb.messages.MSG_CODE_COUNTER_UPDATE_REQ + rc = riak.pb.messages.MSG_CODE_COUNTER_UPDATE_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_fetch_datatype(self, bucket, key, **kwargs): + req = riak.pb.riak_dt_pb2.DtFetchReq() + req.type = str_to_bytes(bucket.bucket_type.name) + req.bucket = str_to_bytes(bucket.name) + req.key = str_to_bytes(key) + self.encode_dt_options(req, **kwargs) + mc = riak.pb.messages.MSG_CODE_DT_FETCH_REQ + rc = riak.pb.messages.MSG_CODE_DT_FETCH_RESP + return Msg(mc, req.SerializeToString(), rc) + + def encode_update_datatype(self, datatype, **kwargs): + op = datatype.to_op() + type_name = datatype.type_name + if not op: + raise ValueError("No operation to send on datatype {!r}". + format(datatype)) + req = riak.pb.riak_dt_pb2.DtUpdateReq() + req.bucket = str_to_bytes(datatype.bucket.name) + req.type = str_to_bytes(datatype.bucket.bucket_type.name) + if datatype.key: + req.key = str_to_bytes(datatype.key) + if datatype._context: + req.context = datatype._context + self.encode_dt_options(req, **kwargs) + self.encode_dt_op(type_name, req, op) + mc = riak.pb.messages.MSG_CODE_DT_UPDATE_REQ + rc = riak.pb.messages.MSG_CODE_DT_UPDATE_RESP + return Msg(mc, req.SerializeToString(), rc) + + def decode_update_datatype(self, datatype, resp, **kwargs): + type_name = datatype.type_name + if resp.HasField('key'): + datatype.key = resp.key[:] + if resp.HasField('context'): + datatype._context = resp.context[:] + if kwargs.get('return_body'): + datatype._set_value(self.decode_dt_value(type_name, resp)) + + def encode_get_preflist(self, bucket, key): + req = riak.pb.riak_kv_pb2.RpbGetBucketKeyPreflistReq() + req.bucket = str_to_bytes(bucket.name) + req.key = str_to_bytes(key) + req.type = str_to_bytes(bucket.bucket_type.name) + mc = riak.pb.messages.MSG_CODE_GET_BUCKET_KEY_PREFLIST_REQ + rc = riak.pb.messages.MSG_CODE_GET_BUCKET_KEY_PREFLIST_RESP + return Msg(mc, req.SerializeToString(), rc) diff --git a/riak/codecs/ttb.py b/riak/codecs/ttb.py new file mode 100644 index 00000000..70a8b6fc --- /dev/null +++ b/riak/codecs/ttb.py @@ -0,0 +1,228 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import six + +from erlastic import encode, decode +from erlastic.types import Atom + +from riak import RiakError +from riak.codecs import Codec, Msg +from riak.pb.messages import MSG_CODE_TS_TTB_MSG +from riak.ts_object import TsColumns +from riak.util import bytes_to_str, unix_time_millis, \ + datetime_from_unix_time_millis + +udef_a = Atom('undefined') + +rpberrorresp_a = Atom('rpberrorresp') +tsgetreq_a = Atom('tsgetreq') +tsgetresp_a = Atom('tsgetresp') +tsqueryreq_a = Atom('tsqueryreq') +tsqueryresp_a = Atom('tsqueryresp') +tsinterpolation_a = Atom('tsinterpolation') +tsputreq_a = Atom('tsputreq') +tsputresp_a = Atom('tsputresp') +tsdelreq_a = Atom('tsdelreq') +timestamp_a = Atom('timestamp') + + +class TtbCodec(Codec): + ''' + Erlang term-to-binary Encoding and decoding methods for TcpTransport + ''' + + def __init__(self, **unused_args): + super(TtbCodec, self).__init__(**unused_args) + + def parse_msg(self, msg_code, data): + if msg_code != MSG_CODE_TS_TTB_MSG: + raise RiakError("TTB can't parse code: {}".format(msg_code)) + if len(data) > 0: + decoded = decode(data) + self.maybe_err_ttb(decoded) + return decoded + else: + return None + + def maybe_err_ttb(self, err_ttb): + resp_a = err_ttb[0] + if resp_a == rpberrorresp_a: + errmsg = err_ttb[1] + # errcode = err_ttb[2] + raise RiakError(bytes_to_str(errmsg)) + + def encode_to_ts_cell(self, cell): + if cell is None: + return [] + else: + if isinstance(cell, datetime.datetime): + ts = unix_time_millis(cell) + # logging.debug('encoded datetime %s as %s', cell, ts) + return ts + elif isinstance(cell, bool): + return cell + elif isinstance(cell, six.text_type) or \ + isinstance(cell, six.binary_type) or \ + isinstance(cell, six.string_types): + return cell + elif (isinstance(cell, six.integer_types)): + return cell + elif isinstance(cell, float): + return cell + else: + t = type(cell) + raise RiakError("can't serialize type '{}', value '{}'" + .format(t, cell)) + + def encode_timeseries_keyreq(self, table, key, is_delete=False): + key_vals = None + if isinstance(key, list): + key_vals = key + else: + raise ValueError("key must be a list") + + mc = MSG_CODE_TS_TTB_MSG + rc = MSG_CODE_TS_TTB_MSG + req_atom = tsgetreq_a + if is_delete: + req_atom = tsdelreq_a + + # TODO FUTURE add timeout as last param + req = req_atom, table.name, \ + [self.encode_to_ts_cell(k) for k in key_vals], udef_a + return Msg(mc, encode(req), rc) + + def validate_timeseries_put_resp(self, resp_code, resp): + if resp is None and resp_code == MSG_CODE_TS_TTB_MSG: + return True + if resp is not None: + return True + else: + raise RiakError("missing response object") + + def encode_timeseries_put(self, tsobj): + ''' + Returns an Erlang-TTB encoded tuple with the appropriate data and + metadata from a TsObject. + + :param tsobj: a TsObject + :type tsobj: TsObject + :rtype: term-to-binary encoded object + ''' + if tsobj.columns: + raise NotImplementedError('columns are not used') + + if tsobj.rows and isinstance(tsobj.rows, list): + req_rows = [] + for row in tsobj.rows: + req_r = [] + for cell in row: + req_r.append(self.encode_to_ts_cell(cell)) + req_rows.append(tuple(req_r)) + req = tsputreq_a, tsobj.table.name, [], req_rows + mc = MSG_CODE_TS_TTB_MSG + rc = MSG_CODE_TS_TTB_MSG + return Msg(mc, encode(req), rc) + else: + raise RiakError("TsObject requires a list of rows") + + def encode_timeseries_query(self, table, query, interpolations=None): + q = query + if '{table}' in q: + q = q.format(table=table.name) + tsi = tsinterpolation_a, q, [] + req = tsqueryreq_a, tsi, False, udef_a + mc = MSG_CODE_TS_TTB_MSG + rc = MSG_CODE_TS_TTB_MSG + return Msg(mc, encode(req), rc) + + def decode_timeseries(self, resp_ttb, tsobj, + convert_timestamp=False): + """ + Fills an TsObject with the appropriate data and + metadata from a TTB-encoded TsGetResp / TsQueryResp. + + :param resp_ttb: the decoded TTB data + :type resp_ttb: TTB-encoded tsqueryrsp or tsgetresp + :param tsobj: a TsObject + :type tsobj: TsObject + :param convert_timestamp: Convert timestamps to datetime objects + :type tsobj: boolean + """ + if resp_ttb is None: + return tsobj + + self.maybe_err_ttb(resp_ttb) + + # NB: some queries return a BARE 'tsqueryresp' atom + # catch that here: + if resp_ttb == tsqueryresp_a: + return tsobj + + # The response atom is the first element in the response tuple + resp_a = resp_ttb[0] + if resp_a == tsputresp_a: + return + elif resp_a == tsgetresp_a or resp_a == tsqueryresp_a: + resp_data = resp_ttb[1] + if len(resp_data) == 0: + return + elif len(resp_data) == 3: + resp_colnames = resp_data[0] + resp_coltypes = resp_data[1] + tsobj.columns = self.decode_timeseries_cols( + resp_colnames, resp_coltypes) + resp_rows = resp_data[2] + tsobj.rows = [] + for resp_row in resp_rows: + tsobj.rows.append( + self.decode_timeseries_row(resp_row, resp_coltypes, + convert_timestamp)) + else: + raise RiakError( + "Expected 3-tuple in response, got: {}".format(resp_data)) + else: + raise RiakError("Unknown TTB response type: {}".format(resp_a)) + + def decode_timeseries_cols(self, cnames, ctypes): + cnames = [bytes_to_str(cname) for cname in cnames] + ctypes = [str(ctype) for ctype in ctypes] + return TsColumns(cnames, ctypes) + + def decode_timeseries_row(self, tsrow, tsct, convert_timestamp=False): + """ + Decodes a TTB-encoded TsRow into a list + + :param tsrow: the TTB decoded TsRow to decode. + :type tsrow: TTB dncoded row + :param tsct: the TTB decoded column types (atoms). + :type tsct: list + :param convert_timestamp: Convert timestamps to datetime objects + :type tsobj: boolean + :rtype list + """ + row = [] + for i, cell in enumerate(tsrow): + if cell is None: + row.append(None) + elif isinstance(cell, list) and len(cell) == 0: + row.append(None) + else: + if convert_timestamp and tsct[i] == timestamp_a: + row.append(datetime_from_unix_time_millis(cell)) + else: + row.append(cell) + return row diff --git a/riak/codecs/util.py b/riak/codecs/util.py new file mode 100644 index 00000000..1fa492bb --- /dev/null +++ b/riak/codecs/util.py @@ -0,0 +1,24 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import riak.pb.messages + + +def parse_pbuf_msg(msg_code, data): + pbclass = riak.pb.messages.MESSAGE_CLASSES.get(msg_code, None) + if pbclass is None: + return None + pbo = pbclass() + pbo.ParseFromString(data) + return pbo diff --git a/riak/content.py b/riak/content.py new file mode 100644 index 00000000..6eb9e7df --- /dev/null +++ b/riak/content.py @@ -0,0 +1,186 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from riak import RiakError +from six import string_types + + +class RiakContent(object): + """ + The RiakContent holds the metadata and value of a single sibling + within a RiakObject. RiakObjects that have more than one sibling + are considered to be in conflict. + """ + def __init__(self, robject, data=None, encoded_data=None, charset=None, + content_type='application/json', content_encoding=None, + last_modified=None, etag=None, usermeta=None, links=None, + indexes=None, exists=False): + self._robject = robject + self._data = data + self._encoded_data = encoded_data + self.charset = charset + self.content_type = content_type + self.content_encoding = content_encoding + self.last_modified = last_modified + self.etag = etag + self.usermeta = usermeta or {} + self.links = links or [] + self.indexes = indexes or set() + self.exists = exists + + def _get_data(self): + if self._encoded_data is not None and self._data is None: + self._data = self._deserialize(self._encoded_data) + self._encoded_data = None + return self._data + + def _set_data(self, value): + self._encoded_data = None + self._data = value + + data = property(_get_data, _set_data, doc=""" + The data stored in this object, as Python objects. For the raw + data, use the `encoded_data` property. If unset, accessing + this property will result in decoding the `encoded_data` + property into Python values. The decoding is dependent on the + `content_type` property and the bucket's registered decoders. + :type mixed """) + + def _get_encoded_data(self): + if self._data is not None and self._encoded_data is None: + self._encoded_data = self._serialize(self._data) + self._data = None + return self._encoded_data + + def _set_encoded_data(self, value): + self._data = None + self._encoded_data = value + + encoded_data = property(_get_encoded_data, _set_encoded_data, doc=""" + The raw data stored in this object, essentially the encoded + form of the `data` property. If unset, accessing this property + will result in encoding the `data` property into a string. The + encoding is dependent on the `content_type` property and the + bucket's registered encoders. + :type str""") + + def _serialize(self, value): + encoder = self._robject.bucket.get_encoder(self.content_type) + if encoder: + return encoder(value) + elif isinstance(value, string_types): + return value.encode() + else: + raise TypeError('No encoder for non-string data ' + 'with content type "{0}"'. + format(self.content_type)) + + def _deserialize(self, value): + if not value: + return value + decoder = self._robject.bucket.get_decoder(self.content_type) + if decoder: + return decoder(value) + else: + raise TypeError('No decoder for content type "{0}"'. + format(self.content_type)) + + def add_index(self, field, value): + """ + add_index(field, value) + + Tag this object with the specified field/value pair for + indexing. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: :class:`RiakObject ` + """ + if field[-4:] not in ("_bin", "_int"): + raise RiakError("Riak 2i fields must end with either '_bin'" + " or '_int'.") + + self.indexes.add((field, value)) + + return self._robject + + def remove_index(self, field=None, value=None): + """ + remove_index(field=None, value=None) + + Remove the specified field/value pair as an index on this + object. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: :class:`RiakObject ` + """ + if not field and not value: + self.indexes.clear() + elif field and not value: + for index in [x for x in self.indexes if x[0] == field]: + self.indexes.remove(index) + elif field and value: + self.indexes.remove((field, value)) + else: + raise RiakError("Cannot pass value without a field" + " name while removing index") + + return self._robject + + remove_indexes = remove_index + + def set_index(self, field, value): + """ + set_index(field, value) + + Works like :meth:`add_index`, but ensures that there is only + one index on given field. If other found, then removes it + first. + + :param field: The index field. + :type field: string + :param value: The index value. + :type value: string or integer + :rtype: :class:`RiakObject ` + """ + to_rem = set((x for x in self.indexes if x[0] == field)) + self.indexes.difference_update(to_rem) + return self.add_index(field, value) + + def add_link(self, obj, tag=None): + """ + add_link(obj, tag=None) + + Add a link to a RiakObject. + + :param obj: Either a RiakObject or 3 item link tuple consisting + of (bucket, key, tag). + :type obj: mixed + :param tag: Optional link tag. Defaults to bucket name. It is ignored + if ``obj`` is a 3 item link tuple. + :type tag: string + :rtype: :class:`RiakObject ` + """ + if isinstance(obj, tuple): + newlink = obj + else: + newlink = (obj.bucket.name, obj.key, tag) + + self.links.append(newlink) + return self._robject diff --git a/riak/datatypes/__init__.py b/riak/datatypes/__init__.py new file mode 100644 index 00000000..87575a33 --- /dev/null +++ b/riak/datatypes/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .types import TYPES +from .datatype import Datatype +from .counter import Counter +from .flag import Flag +from .register import Register +from .set import Set +from .map import Map +from .errors import ContextRequired +from .hll import Hll + + +__all__ = ['Datatype', 'TYPES', 'ContextRequired', + 'Flag', 'Counter', 'Register', 'Set', 'Map', 'Hll'] diff --git a/riak/datatypes/counter.py b/riak/datatypes/counter.py new file mode 100644 index 00000000..d8c8fd24 --- /dev/null +++ b/riak/datatypes/counter.py @@ -0,0 +1,76 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six + +from riak.datatypes.datatype import Datatype +from riak.datatypes import TYPES + + +class Counter(Datatype): + """ + A convergent datatype that represents a counter which can be + incremented or decremented. This type can stand on its own or be + embedded within a :class:`~riak.datatypes.Map`. + """ + + type_name = 'counter' + _type_error_msg = "Counters can only be integers" + + def _post_init(self): + self._increment = 0 + + def _default_value(self): + return 0 + + @Datatype.modified.getter + def modified(self): + """ + Whether this counter has staged increments. + """ + return self._increment is not 0 + + def to_op(self): + """ + Extracts the mutation operation from the counter + :rtype: int, None + """ + if not self._increment == 0: + return ('increment', self._increment) + + def increment(self, amount=1): + """ + Increments the counter by one or the given amount. + + :param amount: the amount to increment the counter + :type amount: int + """ + self._raise_if_badtype(amount) + self._increment += amount + + def decrement(self, amount=1): + """ + Decrements the counter by one or the given amount. + + :param amount: the amount to decrement the counter + :type amount: int + """ + self._raise_if_badtype(amount) + self._increment -= amount + + def _check_type(self, new_value): + return isinstance(new_value, six.integer_types) + + +TYPES['counter'] = Counter diff --git a/riak/datatypes/datatype.py b/riak/datatypes/datatype.py new file mode 100644 index 00000000..2303dba7 --- /dev/null +++ b/riak/datatypes/datatype.py @@ -0,0 +1,229 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .errors import ContextRequired +from . import TYPES + + +class Datatype(object): + """ + Base class for all convergent datatype wrappers. You will not use + this class directly, but it does define some methods are common to + all datatype wrappers. + """ + + #: The string "name" of this datatype. Each datatype should set this. + type_name = None + + #: The message included in the exception raised when the value is of + #: incorrect type. See also :meth:`_check_type`. + _type_error_msg = "Invalid value type" + + def __init__(self, bucket=None, key=None, value=None, context=None): + self.bucket = bucket + self.key = key + self._context = context + if value is not None: + self._set_value(value) + else: + self._set_value(self._default_value()) + self._post_init() + + # Properties + + @property + def value(self): + """ + The pure, immutable value of this datatype, as a Python value, + which is unique for each datatype. + + **NB**: Do not use this property to mutate data, as it will not + have any effect. Use the methods of the individual type to affect + changes. This value is guaranteed to be independent of any internal + data representation. + """ + return self._value + + @property + def context(self): + """ + The opaque context for this type, if it was previously fetched. + + :rtype: str + """ + if self._context: + return self._context[:] + + @property + def modified(self): + """ + Whether this datatype has staged local modifications. + + :rtype: bool + """ + raise NotImplementedError + + # Lifecycle methods + + def reload(self, **params): + """ + Reloads the datatype from Riak. + + .. warning: This clears any local modifications you might have + made. + + :param r: the read quorum + :type r: integer, string, None + :param pr: the primary read quorum + :type pr: integer, string, None + :param basic_quorum: whether to use the "basic quorum" policy + for not-founds + :type basic_quorum: bool + :param notfound_ok: whether to treat not-found responses as successful + :type notfound_ok: bool + :param timeout: a timeout value in milliseconds + :type timeout: int + :param include_context: whether to return the opaque context + as well as the value, which is useful for removal operations + on sets and maps + :type include_context: bool + :rtype: :class:`Datatype` + """ + if not self.bucket: + raise ValueError('bucket property not assigned') + + if not self.key: + raise ValueError('key property not assigned') + + dtype, value, context = self.bucket._client._fetch_datatype( + self.bucket, self.key, **params) + + if not dtype == self.type_name: + raise TypeError("Expected datatype {} but " + "got datatype {}".format(self.__class__, + TYPES[dtype])) + + self.clear() + self._context = context + self._set_value(value) + return self + + def delete(self, **params): + """ + Deletes the datatype from Riak. See :meth:`RiakClient.delete() + ` for options. + """ + self.clear() + self._context = None + self._set_value(self._default_value()) + self.bucket._client.delete(self, **params) + return self + + def update(self, **params): + """ + Sends locally staged mutations to Riak. + + :param w: W-value, wait for this many partitions to respond + before returning to client. + :type w: integer + :param dw: DW-value, wait for this many partitions to + confirm the write before returning to client. + :type dw: integer + :param pw: PW-value, require this many primary partitions to + be available before performing the put + :type pw: integer + :param return_body: if the newly stored object should be + retrieved, defaults to True + :type return_body: bool + :param include_context: whether to return the new opaque + context when `return_body` is `True` + :type include_context: bool + :param timeout: a timeout value in milliseconds + :type timeout: int + :rtype: a subclass of :class:`~riak.datatypes.Datatype` + """ + if not self.modified: + raise ValueError("No operation to perform") + + params.setdefault('return_body', True) + self.bucket._client.update_datatype(self, **params) + self.clear() + + return self + + store = update + + def clear(self): + """ + Removes all locally staged mutations. + """ + self._post_init() + + def to_op(self): + """ + Extracts the mutation operation from this datatype, if any. + Each type must implement this method, returning the + appropriate operation, or `None` if there is no queued + mutation. + """ + raise NotImplementedError + + # Private stuff + + def _check_type(self, new_value): + """ + Checks that initial values of the type are appropriate. Each + type must implement this method. + + :rtype: bool + """ + raise NotImplementedError + + def _coerce_value(self, new_value): + """ + Coerces the input value into the internal representation for + the type. Datatypes may override this method. + """ + return new_value + + def _raise_if_badtype(self, new_value): + if not self._check_type(new_value): + raise TypeError(self._type_error_msg) + + def __str__(self): + return str(self.value) + + def _set_value(self, value): + self._raise_if_badtype(value) + self._value = self._coerce_value(value) + + def _default_value(self): + """ + Returns what the initial value of an empty datatype should be. + """ + raise NotImplementedError + + def _post_init(self): + """ + Called at the end of :meth:`__init__` so that subclasses can tweak + their own setup without overriding the constructor. + """ + pass + + def _require_context(self): + """ + Raises an exception if the context is not present + """ + if not self._context: + raise ContextRequired() diff --git a/riak/datatypes/errors.py b/riak/datatypes/errors.py new file mode 100644 index 00000000..16be5589 --- /dev/null +++ b/riak/datatypes/errors.py @@ -0,0 +1,30 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from riak import RiakError + + +class ContextRequired(RiakError): + """ + This exception is raised when removals of map fields and set + entries are attempted and the datatype hasn't been initialized + with a context. + """ + + _default_message = ("A context is required for remove operations, " + "fetch the datatype first") + + def __init__(self, message=None): + super(ContextRequired, self).__init__(message or + self._default_message) diff --git a/riak/datatypes/flag.py b/riak/datatypes/flag.py new file mode 100644 index 00000000..bfb869a2 --- /dev/null +++ b/riak/datatypes/flag.py @@ -0,0 +1,67 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from riak.datatypes.datatype import Datatype +from riak.datatypes import TYPES + + +class Flag(Datatype): + """ + A convergent datatype that represents a boolean value that can be + enabled or disabled, and may only be embedded in :class:`Map` + instances. + """ + + type_name = 'flag' + _type_error_msg = "Flags can only be booleans" + + def _post_init(self): + self._op = None + + def _default_value(self): + return False + + @Datatype.modified.getter + def modified(self): + """ + Whether this flag has staged toggles. + """ + return self._op is not None + + def enable(self): + """ + Turns the flag on, effectively setting its value to ``True``. + """ + self._op = 'enable' + + def disable(self): + """ + Turns the flag off, effectively setting its value to ``False``. + """ + self._require_context() + self._op = 'disable' + + def to_op(self): + """ + Extracts the mutation operation from the flag. + + :rtype: bool, None + """ + return self._op + + def _check_type(self, new_value): + return isinstance(new_value, bool) + + +TYPES['flag'] = Flag diff --git a/riak/datatypes/hll.py b/riak/datatypes/hll.py new file mode 100644 index 00000000..1d962731 --- /dev/null +++ b/riak/datatypes/hll.py @@ -0,0 +1,81 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six + +from .datatype import Datatype +from riak.datatypes import TYPES + +__all__ = ['Hll'] + + +class Hll(Datatype): + """A convergent datatype representing a HyperLogLog set. + Currently strings are the only supported value type. + Example:: + + myhll.add('barista') + myhll.add('roaster') + myhll.add('brewer') + """ + + type_name = 'hll' + _type_error_msg = 'Hlls can only be integers' + + def _post_init(self): + self._adds = set() + + def _default_value(self): + return 0 + + @Datatype.modified.getter + def modified(self): + """ + Whether this HyperLogLog has staged adds. + """ + return len(self._adds) > 0 + + def to_op(self): + """ + Extracts the modification operation from the Hll. + + :rtype: dict, None + """ + if not self._adds: + return None + changes = {} + if self._adds: + changes['adds'] = list(self._adds) + return changes + + def add(self, element): + """ + Adds an element to the HyperLogLog. Datatype cardinality will + be updated when the object is saved. + + :param element: the element to add + :type element: str + """ + if not isinstance(element, six.string_types): + raise TypeError("Hll elements can only be strings") + self._adds.add(element) + + def _coerce_value(self, new_value): + return int(new_value) + + def _check_type(self, new_value): + return isinstance(new_value, six.integer_types) + + +TYPES['hll'] = Hll diff --git a/riak/datatypes/map.py b/riak/datatypes/map.py new file mode 100644 index 00000000..b5b790bf --- /dev/null +++ b/riak/datatypes/map.py @@ -0,0 +1,302 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import Mapping +from riak.util import lazy_property +from .datatype import Datatype +from riak.datatypes import TYPES + + +class TypedMapView(Mapping): + """ + Implements a sort of view over a :class:`Map`, filtered by the embedded + datatype. + """ + + def __init__(self, parent, datatype): + self.map = parent + self.datatype = datatype + + # Mapping API + def __getitem__(self, key): + """ + Fetches an item from the parent :class:`Map` scoped by this view's + datatype. + + :param key: the key of the item + :type key: str + :rtype: :class:`~riak.datatypes.Datatype` + """ + return self.map[(key, self.datatype)] + + def __iter__(self): + """ + Iterates over all keys in the :class:`Map` scoped by this view's + datatype. + """ + for key in self.map.value: + name, datatype = key + if datatype == self.datatype: + yield name + + def __len__(self): + """ + Returns the number of keys in this map scoped by this view's datatype. + """ + return len(iter(self)) + + def __contains__(self, key): + """ + Determines whether the given key with this view's datatype is in the + parent :class:`Map`. + """ + return (key, self.datatype) in self.map + + # From the MutableMapping API + def __delitem__(self, key): + """ + Removes the key with this view's datatype from the parent :class:`Map`. + """ + del self.map[(key, self.datatype)] + + +class Map(Mapping, Datatype): + """A convergent datatype that acts as a key-value datastructure. Keys + are pairs of ``(name, datatype)`` where ``name`` is a string and + ``datatype`` is the datatype name. Values are other convergent + datatypes, represented by any concrete type in this module. + + You cannot set values in the map directly (it does not implement + ``__setitem__``), but you may add new empty values or access + non-existing values directly via bracket syntax. If a key is not in the + original value of the map when accessed, fetching the key will cause + its associated value to be created.:: + + map[('name', 'register')] + + Keys and their associated values may be deleted from the map as + you would in a dict:: + + del map[('emails', 'set')] + + Convenience accessors exist that partition the map's keys by + datatype and implement the :class:`~collections.Mapping` + behavior as well as supporting deletion:: + + map.sets['emails'] + map.registers['name'] + del map.counters['likes'] + """ + + type_name = 'map' + _type_error_msg = "Map must be a dict with (name, type) keys" + + def _default_value(self): + return dict() + + def _post_init(self): + self._removes = set() + self._updates = {} + + @lazy_property + def counters(self): + """ + Filters keys in the map to only those of counter types. Example:: + + map.counters['views'].increment() + del map.counters['points'] + """ + return TypedMapView(self, 'counter') + + @lazy_property + def flags(self): + """ + Filters keys in the map to only those of flag types. Example:: + + map.flags['confirmed'].enable() + del map.flags['attending'] + """ + return TypedMapView(self, 'flag') + + @lazy_property + def maps(self): + """ + Filters keys in the map to only those of map types. Example:: + + map.maps['emails'].registers['home'].set("user@example.com") + del map.maps['spam'] + """ + return TypedMapView(self, 'map') + + @lazy_property + def registers(self): + """ + Filters keys in the map to only those of register types. Example:: + + map.registers['username'].set_value("riak-user") + del map.registers['access_key'] + """ + return TypedMapView(self, 'register') + + @lazy_property + def sets(self): + """ + Filters keys in the map to only those of set types. Example:: + + map.sets['friends'].add("brett") + del map.sets['favorites'] + """ + return TypedMapView(self, 'set') + + def __contains__(self, key): + """ + A map contains a key if that key exists in the original value + or has been added or mutated. + + :rtype: bool + """ + self._check_key(key) + return (key in self._value) or (key in self._updates) + + # collections.Mapping API + def __getitem__(self, key): + """ + Fetches a convergent datatype at the given key. + + .. note: If the key is not in the map, a new empty datatype + will be inserted at that key and returned. If the key was + previously deleted, that mutation will be discarded. + + :param key: the key of the value to fetch + :type key: tuple + :rtype: :class:`Datatype` matching the datatype in the key + """ + self._check_key(key) + if key in self._value: + return self._value[key] + else: + # If the key does not exist, we assume they are wanting to + # create a new one with that name/type. + if key not in self._updates: + self._updates[key] = TYPES[key[1]](context=self.context) + return self._updates[key] + + def __iter__(self): + """ + Iterates over the *immutable* original value of the map. + """ + return iter(self.value) + + def __len__(self): + """ + Returns the size of the original value of the map. + """ + return len(self._value) + + def __delitem__(self, key): + """ + Deletes a key from the map. If you have previously mutated the + datatype associated with this key, those mutations will be + discarded. + + .. note: You may delete keys that are not entries in the map. + If the Riak server does not find the entry in the set, an + error may be returned to the client. For safety, always + submit removal operations with a context. + + :param key: the key to remove + :type key: tuple + """ + # NB: deleting a key only marks it deleted, and you can delete + # things that don't appear in the value! + self._check_key(key) + self._require_context() + self._removes.add(key) + + def _check_key(self, key): + """ + Ensures well-formedness of a key. + """ + if not len(key) == 2: + raise TypeError('invalid key: %r' % key) + elif key[1] not in TYPES: + raise TypeError('invalid datatype: %s' % key[1]) + + # Datatype API + @Datatype.value.getter + def value(self): + """ + Returns a copy of the original map's value. Nested values are + pure Python values as returned by :attr:`Datatype.value` from + the nested types. + + :rtype: dict + """ + pvalue = {} + for key in self._value: + pvalue[key] = self._value[key].value + return pvalue + + @Datatype.modified.getter + def modified(self): + """ + Whether the map has staged local modifications. + """ + if self._removes: + return True + for v in self._value: + if self._value[v].modified: + return True + for v in self._updates: + if self._updates[v].modified: + return True + return False + + def to_op(self): + """ + Extracts the modification operation(s) from the map. + + :rtype: list, None + """ + removes = [('remove', r) for r in self._removes] + value_updates = list(self._extract_updates(self._value)) + new_updates = list(self._extract_updates(self._updates)) + all_updates = removes + value_updates + new_updates + if all_updates: + return all_updates + else: + return None + + def _check_type(self, value): + for key in value: + try: + self._check_key(key) + except: + return False + return True + + def _coerce_value(self, new_value): + cvalue = {} + for key in new_value: + cvalue[key] = TYPES[key[1]](value=new_value[key], + context=self._context) + return cvalue + + def _extract_updates(self, d): + for key in d: + if d[key].modified: + yield ('update', key, d[key].to_op()) + + +TYPES['map'] = Map diff --git a/riak/datatypes/register.py b/riak/datatypes/register.py new file mode 100644 index 00000000..247a2a52 --- /dev/null +++ b/riak/datatypes/register.py @@ -0,0 +1,79 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import Sized +from riak.datatypes.datatype import Datatype +from six import string_types +from riak.datatypes import TYPES + + +class Register(Sized, Datatype): + """ + A convergent datatype that represents an opaque string that is set + with last-write-wins semantics, and may only be embedded in + :class:`~riak.datatypes.Map` instances. + """ + + type_name = 'register' + _type_error_msg = "Registers can only be strings" + + def _post_init(self): + self._new_value = None + + def _default_value(self): + return "" + + @Datatype.value.getter + def value(self): + """ + Returns a copy of the original value of the register. + + :rtype: str + """ + return self._value[:] + + @Datatype.modified.getter + def modified(self): + """ + Whether this register has staged assignment. + """ + return self._new_value is not None + + def to_op(self): + """ + Extracts the mutation operation from the register. + + :rtype: str, None + """ + if self._new_value is not None: + return ('assign', self._new_value) + + def assign(self, new_value): + """ + Assigns a new value to the register. + + :param new_value: the new value for the register + :type new_value: str + """ + self._raise_if_badtype(new_value) + self._new_value = new_value + + def __len__(self): + return len(self.value) + + def _check_type(self, new_value): + return isinstance(new_value, string_types) + + +TYPES['register'] = Register diff --git a/riak/datatypes/set.py b/riak/datatypes/set.py new file mode 100644 index 00000000..19829cf3 --- /dev/null +++ b/riak/datatypes/set.py @@ -0,0 +1,132 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +from .datatype import Datatype +from six import string_types +from riak.datatypes import TYPES + +__all__ = ['Set'] + + +class Set(collections.Set, Datatype): + """A convergent datatype representing a Set with observed-remove + semantics. Currently strings are the only supported value type. + Example:: + + myset.add('barista') + myset.add('roaster') + myset.add('brewer') + + Likewise they can simply be removed:: + + myset.discard('barista') + + This datatype also implements the `Set ABC + `_, meaning it + supports ``len()``, ``in``, and iteration. + + """ + + type_name = 'set' + _type_error_msg = "Sets can only be iterables of strings" + + def _post_init(self): + self._adds = set() + self._removes = set() + + def _default_value(self): + return frozenset() + + @Datatype.modified.getter + def modified(self): + """ + Whether this set has staged adds or removes. + """ + return len(self._removes | self._adds) > 0 + + def to_op(self): + """ + Extracts the modification operation from the set. + + :rtype: dict, None + """ + if not self._adds and not self._removes: + return None + changes = {} + if self._adds: + changes['adds'] = list(self._adds) + if self._removes: + changes['removes'] = list(self._removes) + return changes + + # collections.Set API, operates only on the immutable version + def __contains__(self, element): + return element in self.value + + def __iter__(self): + return iter(self.value) + + def __len__(self): + return len(self.value) + + # Sort of like collections.MutableSet API, without the additional + # methods. + def add(self, element): + """ + Adds an element to the set. + + .. note: You may add elements that already exist in the set. + This may be used as an "assertion" that the element is a + member. + + :param element: the element to add + :type element: str + """ + _check_element(element) + self._adds.add(element) + + def discard(self, element): + """ + Removes an element from the set. + + .. note: You may remove elements from the set that are not + present, but a context from the server is required. + + :param element: the element to remove + :type element: str + """ + _check_element(element) + self._require_context() + self._removes.add(element) + + def _coerce_value(self, new_value): + return frozenset(new_value) + + def _check_type(self, new_value): + if not isinstance(new_value, collections.Iterable): + return False + for element in new_value: + if not isinstance(element, string_types): + return False + return True + + +def _check_element(element): + if not isinstance(element, string_types): + raise TypeError("Set elements can only be strings") + + +TYPES['set'] = Set diff --git a/riak/datatypes/types.py b/riak/datatypes/types.py new file mode 100644 index 00000000..b9761294 --- /dev/null +++ b/riak/datatypes/types.py @@ -0,0 +1,18 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#: A dict from :attr:`type names ` to the +#: class that implements them. This is used inside :class:`Map` to +#: initialize new values. +TYPES = {} diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 5792aab7..1b604663 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -1,26 +1,29 @@ -""" -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - +# Copyright 2010 Rusty Klophaus +# Copyright 2010 Justin Sheehy +# Copyright 2009 Jay Baird +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function from collections import Iterable, namedtuple -from riak import RiakError +from six import string_types, PY2 + +import riak + +#: Links are just bucket/key/tag tuples, this class provides a +#: backwards-compatible format: ``RiakLink(bucket, key, tag)`` RiakLink = namedtuple("RiakLink", ("bucket", "key", "tag")) @@ -34,8 +37,9 @@ class RiakMapReduce(object): def __init__(self, client): """ Construct a Map/Reduce object. - :param client: A RiakClient object. - :type client: RiakClient + + :param client: the client that will perform the query + :type client: :class:`~riak.client.RiakClient` """ self._client = client self._phases = [] @@ -43,7 +47,7 @@ def __init__(self, client): self._key_filters = [] self._input_mode = None - def add(self, arg1, arg2=None, arg3=None): + def add(self, arg1, arg2=None, arg3=None, bucket_type=None): """ Add inputs to a map/reduce operation. This method takes three different forms, depending on the provided inputs. You can @@ -57,15 +61,18 @@ def add(self, arg1, arg2=None, arg3=None): :type arg2: string, list, None :param arg3: key data for this input (must be convertible to JSON) :type arg3: string, list, dict, None - :rtype: RiakMapReduce + :param bucket_type: Optional name of a bucket type + :type bucket_type: string, None + :rtype: :class:`RiakMapReduce` """ + from riak.riak_object import RiakObject if (arg2 is None) and (arg3 is None): if isinstance(arg1, RiakObject): return self.add_object(arg1) else: - return self.add_bucket(arg1) + return self.add_bucket(arg1, bucket_type) else: - return self.add_bucket_key_data(arg1, arg2, arg3) + return self.add_bucket_key_data(arg1, arg2, arg3, bucket_type) def add_object(self, obj): """ @@ -73,11 +80,11 @@ def add_object(self, obj): :param obj: the object to add :type obj: RiakObject - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ return self.add_bucket_key_data(obj._bucket._name, obj._key, None) - def add_bucket_key_data(self, bucket, key, data): + def add_bucket_key_data(self, bucket, key, data, bucket_type=None): """ Adds a bucket/key/keydata triple to the inputs. @@ -87,7 +94,9 @@ def add_bucket_key_data(self, bucket, key, data): :type key: string :param data: the key-specific data :type data: string, list, dict, None - :rtype: RiakMapReduce + :param bucket_type: Optional name of a bucket type + :type bucket_type: string, None + :rtype: :class:`RiakMapReduce` """ if self._input_mode == 'bucket': raise ValueError('Already added a bucket, can\'t add an object.') @@ -95,23 +104,43 @@ def add_bucket_key_data(self, bucket, key, data): raise ValueError('Already added a query, can\'t add an object.') else: if isinstance(key, Iterable) and \ - not isinstance(key, basestring): - for k in key: - self._inputs.append([bucket, k, data]) + not isinstance(key, string_types): + if bucket_type is not None: + for k in key: + self._inputs.append([bucket, k, data, bucket_type]) + else: + for k in key: + self._inputs.append([bucket, k, data]) else: - self._inputs.append([bucket, key, data]) + if bucket_type is not None: + self._inputs.append([bucket, key, data, bucket_type]) + else: + self._inputs.append([bucket, key, data]) return self - def add_bucket(self, bucket): + def add_bucket(self, bucket, bucket_type=None): """ Adds all keys in a bucket to the inputs. :param bucket: the bucket :type bucket: string - :rtype: RiakMapReduce + :param bucket_type: Optional name of a bucket type + :type bucket_type: string, None + :rtype: :class:`RiakMapReduce` """ + if not riak.disable_list_exceptions: + raise riak.ListError() self._input_mode = 'bucket' - self._inputs = bucket + if isinstance(bucket, riak.RiakBucket): + if bucket.bucket_type.is_default(): + self._inputs = {'bucket': bucket.name} + else: + self._inputs = {'bucket': [bucket.bucket_type.name, + bucket.name]} + elif bucket_type is not None and bucket_type != "default": + self._inputs = {'bucket': [bucket_type, bucket]} + else: + self._inputs = {'bucket': bucket} return self def add_key_filters(self, key_filters): @@ -120,7 +149,7 @@ def add_key_filters(self, key_filters): :param key_filters: a list of filters :type key_filters: list - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if self._input_mode == 'query': raise ValueError('Key filters are not supported in a query.') @@ -134,7 +163,7 @@ def add_key_filter(self, *args): :param args: a filter :type args: list - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if self._input_mode == 'query': raise ValueError('Key filters are not supported in a query.') @@ -142,24 +171,24 @@ def add_key_filter(self, *args): self._key_filters.append(args) return self - def search(self, bucket, query): + def search(self, index, query): """ Begin a map/reduce operation using a Search. This command will return an error unless executed against a Riak Search cluster. - :param bucket: The bucket over which to perform the search - :type bucket: string + :param index: The Solr index used in the search + :type index: string :param query: The search query :type query: string - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ self._input_mode = 'query' - self._inputs = {'module': 'riak_search', - 'function': 'mapred_search', - 'arg': [bucket, query]} + self._inputs = {'bucket': index, + 'index': index, + 'query': query} return self - def index(self, bucket, index, startkey, endkey=None): + def index(self, bucket, index, startkey, endkey=None, bucket_type=None): """ Begin a map/reduce operation using a Secondary Index query. @@ -173,6 +202,9 @@ def index(self, bucket, index, startkey, endkey=None): :type startkey: string, integer :param endkey: The end key of index range (if doing a range query) :type endkey: string, integer, None + :param bucket_type: Optional name of a bucket type + :type bucket_type: string, None + :rtype: :class:`RiakMapReduce` """ self._input_mode = 'query' @@ -185,6 +217,8 @@ def index(self, bucket, index, startkey, endkey=None): 'index': index, 'start': startkey, 'end': endkey} + if bucket_type is not None: + self._inputs['bucket'] = [bucket_type, bucket] return self def link(self, bucket='_', tag='_', keep=False): @@ -192,7 +226,7 @@ def link(self, bucket='_', tag='_', keep=False): Add a link phase to the map/reduce operation. :param bucket: Bucket name (default '_', which means all - buckets) + buckets) :type bucket: string :param tag: Tag (default '_', which means any tag) :type tag: string @@ -200,7 +234,7 @@ def link(self, bucket='_', tag='_', keep=False): the map/reduce. (default False, unless this is the last step in the phase) :type keep: boolean - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ self._phases.append(RiakLinkPhase(bucket, tag, keep)) return self @@ -217,7 +251,7 @@ def map(self, function, options=None): :param options: phase options, containing 'language', 'keep' flag, and/or 'arg'. :type options: dict - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if options is None: options = dict() @@ -245,7 +279,7 @@ def reduce(self, function, options=None): :type function: string, list :param options: phase options, containing 'language', 'keep' flag, and/or 'arg'. - :rtype: RiakMapReduce + :rtype: :class:`RiakMapReduce` """ if options is None: options = dict() @@ -265,8 +299,8 @@ def reduce(self, function, options=None): def run(self, timeout=None): """ Run the map/reduce operation synchronously. Returns a list of - results, or a list of links if the last phase is a - link phase. + results, or a list of links if the last phase is a link phase. + Shortcut for :meth:`riak.client.RiakClient.mapred`. :param timeout: Timeout in milliseconds :type timeout: integer, None @@ -276,19 +310,20 @@ def run(self, timeout=None): try: result = self._client.mapred(self._inputs, query, timeout) - except RiakError as e: + except riak.RiakError as e: if 'worker_startup_failed' in e.value: for phase in self._phases: if phase._language == 'erlang': if type(phase._function) is str: - raise RiakError('May have tried erlang strfun ' - 'when not allowed\n' - 'original error: ' + e.value) + raise riak.RiakError( + 'May have tried erlang strfun ' + 'when not allowed\n' + 'original error: ' + e.value) raise e # If the last phase is NOT a link phase, then return the result. - if not (link_results_flag - or isinstance(self._phases[-1], RiakLinkPhase)): + if not (link_results_flag or + isinstance(self._phases[-1], RiakLinkPhase)): return result # If there are no results, then return an empty list. @@ -309,11 +344,12 @@ def run(self, timeout=None): def stream(self, timeout=None): """ - Streams the MapReduce query (returns an iterator). + Streams the MapReduce query (returns an iterator). Shortcut + for :meth:`riak.client.RiakClient.stream_mapred`. :param timeout: Timeout in milliseconds :type timeout: integer - :rtype: iterator + :rtype: iterator that yields (phase_num, data) tuples """ query, lrf = self._normalize_query() return self._client.stream_mapred(self._inputs, query, timeout) @@ -340,16 +376,14 @@ def _normalize_query(self): keep_flag = True query.append(phase.to_array()) - if (len(self._key_filters) > 0): - bucket_name = None - if (type(self._inputs) == str): - bucket_name = self._inputs - elif (type(self._inputs) == RiakBucket): - bucket_name = self._inputs.name - - if (bucket_name is not None): - self._inputs = {'bucket': bucket_name, - 'key_filters': self._key_filters} + # Is this a bucket-only input? If so, we need to switch + # to be a string or array, i.e., no keyword (to keep Riak happy) + # Also add the key filter, if necessary + if isinstance(self._inputs, dict) and len(self._inputs) == 1: + if len(self._key_filters) > 0: + self._inputs['key_filters'] = self._key_filters + else: + self._inputs = self._inputs['bucket'] return query, link_results_flag @@ -521,7 +555,7 @@ def __init__(self, type, function, language, keep, arg): :type arg: string, dict, list """ try: - if isinstance(function, basestring): + if isinstance(function, string_types) and PY2: function = function.encode('ascii') except UnicodeError: raise TypeError('Unicode encoded functions are not supported.') @@ -547,7 +581,7 @@ def to_array(self): if isinstance(self._function, list): stepdef['bucket'] = self._function[0] stepdef['key'] = self._function[1] - elif isinstance(self._function, str): + elif isinstance(self._function, string_types): if ("{" in self._function): stepdef['source'] = self._function else: @@ -557,7 +591,8 @@ def to_array(self): stepdef['module'] = self._function[0] stepdef['function'] = self._function[1] - elif (self._language == 'erlang' and isinstance(self._function, str)): + elif (self._language == 'erlang' and + isinstance(self._function, string_types)): stepdef['source'] = self._function return {self._type: stepdef} @@ -569,13 +604,14 @@ class RiakLinkPhase(object): map/reduce operation. Normally you won't need to use this object directly, but instead - call ``link`` on RiakMapReduce objects to add instances to the - query. + call :meth:`RiakMapReduce.link` on RiakMapReduce objects to add + instances to the query. """ def __init__(self, bucket, tag, keep): """ Construct a RiakLinkPhase object. + :param bucket: - The bucket name :type bucket: string :param tag: The tag @@ -599,7 +635,25 @@ def to_array(self): class RiakKeyFilter(object): + """ + A helper class for building up lists of key filters. Unknown + methods are treated as filters to be added; ``&`` and ``|`` create + conjunctions and disjunctions, respectively. ``+`` concatenates filters. + + Example:: + + f1 = RiakKeyFilter().starts_with('2005') + f2 = RiakKeyFilter().ends_with('-01') + f3 = f1 & f2 + print(f3) + # => [['and', [['starts_with', '2005']], [['ends_with', '-01']]]] + """ + def __init__(self, *args): + """ + :param args: a list of arguments to be treated as a filter. + :type args: list + """ if args: self._filters = [list(args)] else: @@ -646,15 +700,24 @@ class RiakMapReduceChain(object): Mixin to add chaining from the client object directly into a MapReduce operation. """ - def add(self, *args): + def add(self, arg1, arg2=None, arg3=None, bucket_type=None): """ Start assembling a Map/Reduce operation. A shortcut for :func:`RiakMapReduce.add`. + :param arg1: the object or bucket to add + :type arg1: RiakObject, string + :param arg2: a key or list of keys to add (if a bucket is + given in arg1) + :type arg2: string, list, None + :param arg3: key data for this input (must be convertible to JSON) + :type arg3: string, list, dict, None + :param bucket_type: Optional name of a bucket type + :type bucket_type: string, None :rtype: :class:`RiakMapReduce` """ mr = RiakMapReduce(self) - return mr.add(*args) + return mr.add(arg1, arg2, arg3, bucket_type) def search(self, *args): """ @@ -665,18 +728,31 @@ def search(self, *args): :rtype: :class:`RiakMapReduce` """ + mr = RiakMapReduce(self) return mr.search(*args) - def index(self, *args): + def index(self, bucket, index, startkey, endkey=None, bucket_type=None): """ Start assembling a Map/Reduce operation based on secondary index query results. + :param bucket: The bucket over which to perform the query + :type bucket: string + :param index: The index to use for query + :type index: string + :param startkey: The start key of index range, or the + value which all entries must equal + :type startkey: string, integer + :param endkey: The end key of index range (if doing a range query) + :type endkey: string, integer, None + :param bucket_type: Optional name of a bucket type + :type bucket_type: string, None :rtype: :class:`RiakMapReduce` """ + mr = RiakMapReduce(self) - return mr.index(*args) + return mr.index(bucket, index, startkey, endkey, bucket_type) def link(self, *args): """ @@ -707,6 +783,3 @@ def reduce(self, *args): """ mr = RiakMapReduce(self) return mr.reduce(*args) - -from riak.riak_object import RiakObject -from riak.bucket import RiakBucket diff --git a/riak/multidict.py b/riak/multidict.py index 14761dde..0df28f80 100644 --- a/riak/multidict.py +++ b/riak/multidict.py @@ -1,10 +1,23 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # (c) 2005 Ian Bicking and contributors; written for Paste # (http://pythonpaste.org) Licensed under the MIT license: # http://www.opensource.org/licenses/mit-license.php -from UserDict import DictMixin -class MultiDict(DictMixin): +class MultiDict(dict): """ An ordered dictionary that can have multiple values for each key. @@ -20,13 +33,13 @@ def __init__(self, *args, **kw): if hasattr(args[0], 'iteritems'): items = list(args[0].iteritems()) elif hasattr(args[0], 'items'): - items = args[0].items() + items = list(args[0].items()) else: items = list(args[0]) self._items = items else: self._items = [] - self._items.extend(kw.iteritems()) + self._items.extend(list(kw.items())) def __getitem__(self, key): for k, v in self._items: @@ -174,9 +187,9 @@ def __repr__(self): def __len__(self): return len(self._items) - ## - ## All the iteration: - ## + # + # All the iteration: + # def keys(self): return [k for k, v in self._items] diff --git a/riak/node.py b/riak/node.py index 08268fd9..eecffe69 100644 --- a/riak/node.py +++ b/riak/node.py @@ -1,24 +1,21 @@ -""" -Copyright 2012 Basho Technologies, Inc. +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" import math import time + from threading import RLock -from riak.util import deprecated class Decaying(object): @@ -89,10 +86,6 @@ def __init__(self, host='127.0.0.1', http_port=8098, pb_port=8087, :param pb_port: the Protcol Buffers port of the node :type pb_port: integer """ - - if 'port' in unused_args and not 'already_warned_port' in unused_args: - deprecated("port option is deprecated, use http_port or pb_port") - self.host = host self.http_port = http_port self.pb_port = pb_port diff --git a/riak/pb/__init__.py b/riak/pb/__init__.py new file mode 100644 index 00000000..9b867bc5 --- /dev/null +++ b/riak/pb/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/riak/pb/messages.py b/riak/pb/messages.py new file mode 100644 index 00000000..b8f1e91e --- /dev/null +++ b/riak/pb/messages.py @@ -0,0 +1,187 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a generated file. DO NOT EDIT. + +""" +Constants and mappings between Riak protocol codes and messages. +""" + +import riak.pb.riak_dt_pb2 +import riak.pb.riak_kv_pb2 +import riak.pb.riak_pb2 +import riak.pb.riak_search_pb2 +import riak.pb.riak_ts_pb2 +import riak.pb.riak_yokozuna_pb2 + +# Protocol codes +MSG_CODE_ERROR_RESP = 0 +MSG_CODE_PING_REQ = 1 +MSG_CODE_PING_RESP = 2 +MSG_CODE_GET_CLIENT_ID_REQ = 3 +MSG_CODE_GET_CLIENT_ID_RESP = 4 +MSG_CODE_SET_CLIENT_ID_REQ = 5 +MSG_CODE_SET_CLIENT_ID_RESP = 6 +MSG_CODE_GET_SERVER_INFO_REQ = 7 +MSG_CODE_GET_SERVER_INFO_RESP = 8 +MSG_CODE_GET_REQ = 9 +MSG_CODE_GET_RESP = 10 +MSG_CODE_PUT_REQ = 11 +MSG_CODE_PUT_RESP = 12 +MSG_CODE_DEL_REQ = 13 +MSG_CODE_DEL_RESP = 14 +MSG_CODE_LIST_BUCKETS_REQ = 15 +MSG_CODE_LIST_BUCKETS_RESP = 16 +MSG_CODE_LIST_KEYS_REQ = 17 +MSG_CODE_LIST_KEYS_RESP = 18 +MSG_CODE_GET_BUCKET_REQ = 19 +MSG_CODE_GET_BUCKET_RESP = 20 +MSG_CODE_SET_BUCKET_REQ = 21 +MSG_CODE_SET_BUCKET_RESP = 22 +MSG_CODE_MAP_RED_REQ = 23 +MSG_CODE_MAP_RED_RESP = 24 +MSG_CODE_INDEX_REQ = 25 +MSG_CODE_INDEX_RESP = 26 +MSG_CODE_SEARCH_QUERY_REQ = 27 +MSG_CODE_SEARCH_QUERY_RESP = 28 +MSG_CODE_RESET_BUCKET_REQ = 29 +MSG_CODE_RESET_BUCKET_RESP = 30 +MSG_CODE_GET_BUCKET_TYPE_REQ = 31 +MSG_CODE_SET_BUCKET_TYPE_REQ = 32 +MSG_CODE_GET_BUCKET_KEY_PREFLIST_REQ = 33 +MSG_CODE_GET_BUCKET_KEY_PREFLIST_RESP = 34 +MSG_CODE_CS_BUCKET_REQ = 40 +MSG_CODE_CS_BUCKET_RESP = 41 +MSG_CODE_INDEX_BODY_RESP = 42 +MSG_CODE_COUNTER_UPDATE_REQ = 50 +MSG_CODE_COUNTER_UPDATE_RESP = 51 +MSG_CODE_COUNTER_GET_REQ = 52 +MSG_CODE_COUNTER_GET_RESP = 53 +MSG_CODE_YOKOZUNA_INDEX_GET_REQ = 54 +MSG_CODE_YOKOZUNA_INDEX_GET_RESP = 55 +MSG_CODE_YOKOZUNA_INDEX_PUT_REQ = 56 +MSG_CODE_YOKOZUNA_INDEX_DELETE_REQ = 57 +MSG_CODE_YOKOZUNA_SCHEMA_GET_REQ = 58 +MSG_CODE_YOKOZUNA_SCHEMA_GET_RESP = 59 +MSG_CODE_YOKOZUNA_SCHEMA_PUT_REQ = 60 +MSG_CODE_COVERAGE_REQ = 70 +MSG_CODE_COVERAGE_RESP = 71 +MSG_CODE_DT_FETCH_REQ = 80 +MSG_CODE_DT_FETCH_RESP = 81 +MSG_CODE_DT_UPDATE_REQ = 82 +MSG_CODE_DT_UPDATE_RESP = 83 +MSG_CODE_TS_QUERY_REQ = 90 +MSG_CODE_TS_QUERY_RESP = 91 +MSG_CODE_TS_PUT_REQ = 92 +MSG_CODE_TS_PUT_RESP = 93 +MSG_CODE_TS_DEL_REQ = 94 +MSG_CODE_TS_DEL_RESP = 95 +MSG_CODE_TS_GET_REQ = 96 +MSG_CODE_TS_GET_RESP = 97 +MSG_CODE_TS_LIST_KEYS_REQ = 98 +MSG_CODE_TS_LIST_KEYS_RESP = 99 +MSG_CODE_TS_COVERAGE_REQ = 100 +MSG_CODE_TS_COVERAGE_RESP = 101 +MSG_CODE_TS_COVERAGE_ENTRY = 102 +MSG_CODE_TS_RANGE = 103 +MSG_CODE_TS_TTB_MSG = 104 +MSG_CODE_AUTH_REQ = 253 +MSG_CODE_AUTH_RESP = 254 +MSG_CODE_START_TLS = 255 + +# Mapping from code to protobuf class +MESSAGE_CLASSES = { + MSG_CODE_ERROR_RESP: riak.pb.riak_pb2.RpbErrorResp, + MSG_CODE_PING_REQ: None, + MSG_CODE_PING_RESP: None, + MSG_CODE_GET_CLIENT_ID_REQ: None, + MSG_CODE_GET_CLIENT_ID_RESP: riak.pb.riak_kv_pb2.RpbGetClientIdResp, + MSG_CODE_SET_CLIENT_ID_REQ: riak.pb.riak_kv_pb2.RpbSetClientIdReq, + MSG_CODE_SET_CLIENT_ID_RESP: None, + MSG_CODE_GET_SERVER_INFO_REQ: None, + MSG_CODE_GET_SERVER_INFO_RESP: riak.pb.riak_pb2.RpbGetServerInfoResp, + MSG_CODE_GET_REQ: riak.pb.riak_kv_pb2.RpbGetReq, + MSG_CODE_GET_RESP: riak.pb.riak_kv_pb2.RpbGetResp, + MSG_CODE_PUT_REQ: riak.pb.riak_kv_pb2.RpbPutReq, + MSG_CODE_PUT_RESP: riak.pb.riak_kv_pb2.RpbPutResp, + MSG_CODE_DEL_REQ: riak.pb.riak_kv_pb2.RpbDelReq, + MSG_CODE_DEL_RESP: None, + MSG_CODE_LIST_BUCKETS_REQ: riak.pb.riak_kv_pb2.RpbListBucketsReq, + MSG_CODE_LIST_BUCKETS_RESP: riak.pb.riak_kv_pb2.RpbListBucketsResp, + MSG_CODE_LIST_KEYS_REQ: riak.pb.riak_kv_pb2.RpbListKeysReq, + MSG_CODE_LIST_KEYS_RESP: riak.pb.riak_kv_pb2.RpbListKeysResp, + MSG_CODE_GET_BUCKET_REQ: riak.pb.riak_pb2.RpbGetBucketReq, + MSG_CODE_GET_BUCKET_RESP: riak.pb.riak_pb2.RpbGetBucketResp, + MSG_CODE_SET_BUCKET_REQ: riak.pb.riak_pb2.RpbSetBucketReq, + MSG_CODE_SET_BUCKET_RESP: None, + MSG_CODE_MAP_RED_REQ: riak.pb.riak_kv_pb2.RpbMapRedReq, + MSG_CODE_MAP_RED_RESP: riak.pb.riak_kv_pb2.RpbMapRedResp, + MSG_CODE_INDEX_REQ: riak.pb.riak_kv_pb2.RpbIndexReq, + MSG_CODE_INDEX_RESP: riak.pb.riak_kv_pb2.RpbIndexResp, + MSG_CODE_SEARCH_QUERY_REQ: riak.pb.riak_search_pb2.RpbSearchQueryReq, + MSG_CODE_SEARCH_QUERY_RESP: riak.pb.riak_search_pb2.RpbSearchQueryResp, + MSG_CODE_RESET_BUCKET_REQ: riak.pb.riak_pb2.RpbResetBucketReq, + MSG_CODE_RESET_BUCKET_RESP: None, + MSG_CODE_GET_BUCKET_TYPE_REQ: riak.pb.riak_pb2.RpbGetBucketTypeReq, + MSG_CODE_SET_BUCKET_TYPE_REQ: riak.pb.riak_pb2.RpbSetBucketTypeReq, + MSG_CODE_GET_BUCKET_KEY_PREFLIST_REQ: + riak.pb.riak_kv_pb2.RpbGetBucketKeyPreflistReq, + MSG_CODE_GET_BUCKET_KEY_PREFLIST_RESP: + riak.pb.riak_kv_pb2.RpbGetBucketKeyPreflistResp, + MSG_CODE_CS_BUCKET_REQ: riak.pb.riak_kv_pb2.RpbCSBucketReq, + MSG_CODE_CS_BUCKET_RESP: riak.pb.riak_kv_pb2.RpbCSBucketResp, + MSG_CODE_INDEX_BODY_RESP: riak.pb.riak_kv_pb2.RpbIndexBodyResp, + MSG_CODE_COUNTER_UPDATE_REQ: riak.pb.riak_kv_pb2.RpbCounterUpdateReq, + MSG_CODE_COUNTER_UPDATE_RESP: riak.pb.riak_kv_pb2.RpbCounterUpdateResp, + MSG_CODE_COUNTER_GET_REQ: riak.pb.riak_kv_pb2.RpbCounterGetReq, + MSG_CODE_COUNTER_GET_RESP: riak.pb.riak_kv_pb2.RpbCounterGetResp, + MSG_CODE_YOKOZUNA_INDEX_GET_REQ: + riak.pb.riak_yokozuna_pb2.RpbYokozunaIndexGetReq, + MSG_CODE_YOKOZUNA_INDEX_GET_RESP: + riak.pb.riak_yokozuna_pb2.RpbYokozunaIndexGetResp, + MSG_CODE_YOKOZUNA_INDEX_PUT_REQ: + riak.pb.riak_yokozuna_pb2.RpbYokozunaIndexPutReq, + MSG_CODE_YOKOZUNA_INDEX_DELETE_REQ: + riak.pb.riak_yokozuna_pb2.RpbYokozunaIndexDeleteReq, + MSG_CODE_YOKOZUNA_SCHEMA_GET_REQ: + riak.pb.riak_yokozuna_pb2.RpbYokozunaSchemaGetReq, + MSG_CODE_YOKOZUNA_SCHEMA_GET_RESP: + riak.pb.riak_yokozuna_pb2.RpbYokozunaSchemaGetResp, + MSG_CODE_YOKOZUNA_SCHEMA_PUT_REQ: + riak.pb.riak_yokozuna_pb2.RpbYokozunaSchemaPutReq, + MSG_CODE_COVERAGE_REQ: riak.pb.riak_kv_pb2.RpbCoverageReq, + MSG_CODE_COVERAGE_RESP: riak.pb.riak_kv_pb2.RpbCoverageResp, + MSG_CODE_DT_FETCH_REQ: riak.pb.riak_dt_pb2.DtFetchReq, + MSG_CODE_DT_FETCH_RESP: riak.pb.riak_dt_pb2.DtFetchResp, + MSG_CODE_DT_UPDATE_REQ: riak.pb.riak_dt_pb2.DtUpdateReq, + MSG_CODE_DT_UPDATE_RESP: riak.pb.riak_dt_pb2.DtUpdateResp, + MSG_CODE_TS_QUERY_REQ: riak.pb.riak_ts_pb2.TsQueryReq, + MSG_CODE_TS_QUERY_RESP: riak.pb.riak_ts_pb2.TsQueryResp, + MSG_CODE_TS_PUT_REQ: riak.pb.riak_ts_pb2.TsPutReq, + MSG_CODE_TS_PUT_RESP: riak.pb.riak_ts_pb2.TsPutResp, + MSG_CODE_TS_DEL_REQ: riak.pb.riak_ts_pb2.TsDelReq, + MSG_CODE_TS_DEL_RESP: riak.pb.riak_ts_pb2.TsDelResp, + MSG_CODE_TS_GET_REQ: riak.pb.riak_ts_pb2.TsGetReq, + MSG_CODE_TS_GET_RESP: riak.pb.riak_ts_pb2.TsGetResp, + MSG_CODE_TS_LIST_KEYS_REQ: riak.pb.riak_ts_pb2.TsListKeysReq, + MSG_CODE_TS_LIST_KEYS_RESP: riak.pb.riak_ts_pb2.TsListKeysResp, + MSG_CODE_TS_COVERAGE_REQ: riak.pb.riak_ts_pb2.TsCoverageReq, + MSG_CODE_TS_COVERAGE_RESP: riak.pb.riak_ts_pb2.TsCoverageResp, + MSG_CODE_TS_COVERAGE_ENTRY: riak.pb.riak_ts_pb2.TsCoverageEntry, + MSG_CODE_TS_RANGE: riak.pb.riak_ts_pb2.TsRange, + MSG_CODE_TS_TTB_MSG: None, + MSG_CODE_AUTH_REQ: riak.pb.riak_pb2.RpbAuthReq, + MSG_CODE_AUTH_RESP: None, + MSG_CODE_START_TLS: None +} diff --git a/riak/pb/riak_dt_pb2.py b/riak/pb/riak_dt_pb2.py new file mode 100644 index 00000000..ba9a590d --- /dev/null +++ b/riak/pb/riak_dt_pb2.py @@ -0,0 +1,999 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from six import * +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: riak_dt.proto + +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='riak_dt.proto', + package='', + serialized_pb='\n\rriak_dt.proto\"\x85\x01\n\x08MapField\x12\x0c\n\x04name\x18\x01 \x02(\x0c\x12$\n\x04type\x18\x02 \x02(\x0e\x32\x16.MapField.MapFieldType\"E\n\x0cMapFieldType\x12\x0b\n\x07\x43OUNTER\x10\x01\x12\x07\n\x03SET\x10\x02\x12\x0c\n\x08REGISTER\x10\x03\x12\x08\n\x04\x46LAG\x10\x04\x12\x07\n\x03MAP\x10\x05\"\x98\x01\n\x08MapEntry\x12\x18\n\x05\x66ield\x18\x01 \x02(\x0b\x32\t.MapField\x12\x15\n\rcounter_value\x18\x02 \x01(\x12\x12\x11\n\tset_value\x18\x03 \x03(\x0c\x12\x16\n\x0eregister_value\x18\x04 \x01(\x0c\x12\x12\n\nflag_value\x18\x05 \x01(\x08\x12\x1c\n\tmap_value\x18\x06 \x03(\x0b\x32\t.MapEntry\"\xcf\x01\n\nDtFetchReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\x0c\n\x04type\x18\x03 \x02(\x0c\x12\t\n\x01r\x18\x04 \x01(\r\x12\n\n\x02pr\x18\x05 \x01(\r\x12\x14\n\x0c\x62\x61sic_quorum\x18\x06 \x01(\x08\x12\x13\n\x0bnotfound_ok\x18\x07 \x01(\x08\x12\x0f\n\x07timeout\x18\x08 \x01(\r\x12\x15\n\rsloppy_quorum\x18\t \x01(\x08\x12\r\n\x05n_val\x18\n \x01(\r\x12\x1d\n\x0finclude_context\x18\x0b \x01(\x08:\x04true\"x\n\x07\x44tValue\x12\x15\n\rcounter_value\x18\x01 \x01(\x12\x12\x11\n\tset_value\x18\x02 \x03(\x0c\x12\x1c\n\tmap_value\x18\x03 \x03(\x0b\x32\t.MapEntry\x12\x11\n\thll_value\x18\x04 \x01(\x04\x12\x12\n\ngset_value\x18\x05 \x03(\x0c\"\x9a\x01\n\x0b\x44tFetchResp\x12\x0f\n\x07\x63ontext\x18\x01 \x01(\x0c\x12#\n\x04type\x18\x02 \x02(\x0e\x32\x15.DtFetchResp.DataType\x12\x17\n\x05value\x18\x03 \x01(\x0b\x32\x08.DtValue\"<\n\x08\x44\x61taType\x12\x0b\n\x07\x43OUNTER\x10\x01\x12\x07\n\x03SET\x10\x02\x12\x07\n\x03MAP\x10\x03\x12\x07\n\x03HLL\x10\x04\x12\x08\n\x04GSET\x10\x05\"\x1e\n\tCounterOp\x12\x11\n\tincrement\x18\x01 \x01(\x12\"&\n\x05SetOp\x12\x0c\n\x04\x61\x64\x64s\x18\x01 \x03(\x0c\x12\x0f\n\x07removes\x18\x02 \x03(\x0c\"\x16\n\x06GSetOp\x12\x0c\n\x04\x61\x64\x64s\x18\x01 \x03(\x0c\"\x15\n\x05HllOp\x12\x0c\n\x04\x61\x64\x64s\x18\x01 \x03(\x0c\"\xd1\x01\n\tMapUpdate\x12\x18\n\x05\x66ield\x18\x01 \x02(\x0b\x32\t.MapField\x12\x1e\n\ncounter_op\x18\x02 \x01(\x0b\x32\n.CounterOp\x12\x16\n\x06set_op\x18\x03 \x01(\x0b\x32\x06.SetOp\x12\x13\n\x0bregister_op\x18\x04 \x01(\x0c\x12\"\n\x07\x66lag_op\x18\x05 \x01(\x0e\x32\x11.MapUpdate.FlagOp\x12\x16\n\x06map_op\x18\x06 \x01(\x0b\x32\x06.MapOp\"!\n\x06\x46lagOp\x12\n\n\x06\x45NABLE\x10\x01\x12\x0b\n\x07\x44ISABLE\x10\x02\"@\n\x05MapOp\x12\x1a\n\x07removes\x18\x01 \x03(\x0b\x32\t.MapField\x12\x1b\n\x07updates\x18\x02 \x03(\x0b\x32\n.MapUpdate\"\x88\x01\n\x04\x44tOp\x12\x1e\n\ncounter_op\x18\x01 \x01(\x0b\x32\n.CounterOp\x12\x16\n\x06set_op\x18\x02 \x01(\x0b\x32\x06.SetOp\x12\x16\n\x06map_op\x18\x03 \x01(\x0b\x32\x06.MapOp\x12\x16\n\x06hll_op\x18\x04 \x01(\x0b\x32\x06.HllOp\x12\x18\n\x07gset_op\x18\x05 \x01(\x0b\x32\x07.GSetOp\"\xf1\x01\n\x0b\x44tUpdateReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0c\n\x04type\x18\x03 \x02(\x0c\x12\x0f\n\x07\x63ontext\x18\x04 \x01(\x0c\x12\x11\n\x02op\x18\x05 \x02(\x0b\x32\x05.DtOp\x12\t\n\x01w\x18\x06 \x01(\r\x12\n\n\x02\x64w\x18\x07 \x01(\r\x12\n\n\x02pw\x18\x08 \x01(\r\x12\x1a\n\x0breturn_body\x18\t \x01(\x08:\x05\x66\x61lse\x12\x0f\n\x07timeout\x18\n \x01(\r\x12\x15\n\rsloppy_quorum\x18\x0b \x01(\x08\x12\r\n\x05n_val\x18\x0c \x01(\r\x12\x1d\n\x0finclude_context\x18\r \x01(\x08:\x04true\"\x9b\x01\n\x0c\x44tUpdateResp\x12\x0b\n\x03key\x18\x01 \x01(\x0c\x12\x0f\n\x07\x63ontext\x18\x02 \x01(\x0c\x12\x15\n\rcounter_value\x18\x03 \x01(\x12\x12\x11\n\tset_value\x18\x04 \x03(\x0c\x12\x1c\n\tmap_value\x18\x05 \x03(\x0b\x32\t.MapEntry\x12\x11\n\thll_value\x18\x06 \x01(\x04\x12\x12\n\ngset_value\x18\x07 \x03(\x0c\x42#\n\x17\x63om.basho.riak.protobufB\x08RiakDtPB') + + + +_MAPFIELD_MAPFIELDTYPE = _descriptor.EnumDescriptor( + name='MapFieldType', + full_name='MapField.MapFieldType', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='COUNTER', index=0, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='SET', index=1, number=2, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='REGISTER', index=2, number=3, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FLAG', index=3, number=4, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MAP', index=4, number=5, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=82, + serialized_end=151, +) + +_DTFETCHRESP_DATATYPE = _descriptor.EnumDescriptor( + name='DataType', + full_name='DtFetchResp.DataType', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='COUNTER', index=0, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='SET', index=1, number=2, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MAP', index=2, number=3, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='HLL', index=3, number=4, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='GSET', index=4, number=5, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=735, + serialized_end=795, +) + +_MAPUPDATE_FLAGOP = _descriptor.EnumDescriptor( + name='FlagOp', + full_name='MapUpdate.FlagOp', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='ENABLE', index=0, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DISABLE', index=1, number=2, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=1093, + serialized_end=1126, +) + + +_MAPFIELD = _descriptor.Descriptor( + name='MapField', + full_name='MapField', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='MapField.name', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='MapField.type', index=1, + number=2, type=14, cpp_type=8, label=2, + has_default_value=False, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _MAPFIELD_MAPFIELDTYPE, + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=18, + serialized_end=151, +) + + +_MAPENTRY = _descriptor.Descriptor( + name='MapEntry', + full_name='MapEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='field', full_name='MapEntry.field', index=0, + number=1, type=11, cpp_type=10, label=2, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='counter_value', full_name='MapEntry.counter_value', index=1, + number=2, type=18, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='set_value', full_name='MapEntry.set_value', index=2, + number=3, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='register_value', full_name='MapEntry.register_value', index=3, + number=4, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='flag_value', full_name='MapEntry.flag_value', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='map_value', full_name='MapEntry.map_value', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=154, + serialized_end=306, +) + + +_DTFETCHREQ = _descriptor.Descriptor( + name='DtFetchReq', + full_name='DtFetchReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='DtFetchReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='DtFetchReq.key', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='DtFetchReq.type', index=2, + number=3, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='r', full_name='DtFetchReq.r', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pr', full_name='DtFetchReq.pr', index=4, + number=5, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='basic_quorum', full_name='DtFetchReq.basic_quorum', index=5, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='notfound_ok', full_name='DtFetchReq.notfound_ok', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timeout', full_name='DtFetchReq.timeout', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='sloppy_quorum', full_name='DtFetchReq.sloppy_quorum', index=8, + number=9, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='n_val', full_name='DtFetchReq.n_val', index=9, + number=10, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='include_context', full_name='DtFetchReq.include_context', index=10, + number=11, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=True, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=309, + serialized_end=516, +) + + +_DTVALUE = _descriptor.Descriptor( + name='DtValue', + full_name='DtValue', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='counter_value', full_name='DtValue.counter_value', index=0, + number=1, type=18, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='set_value', full_name='DtValue.set_value', index=1, + number=2, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='map_value', full_name='DtValue.map_value', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='hll_value', full_name='DtValue.hll_value', index=3, + number=4, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='gset_value', full_name='DtValue.gset_value', index=4, + number=5, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=518, + serialized_end=638, +) + + +_DTFETCHRESP = _descriptor.Descriptor( + name='DtFetchResp', + full_name='DtFetchResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='context', full_name='DtFetchResp.context', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='DtFetchResp.type', index=1, + number=2, type=14, cpp_type=8, label=2, + has_default_value=False, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='DtFetchResp.value', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _DTFETCHRESP_DATATYPE, + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=641, + serialized_end=795, +) + + +_COUNTEROP = _descriptor.Descriptor( + name='CounterOp', + full_name='CounterOp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='increment', full_name='CounterOp.increment', index=0, + number=1, type=18, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=797, + serialized_end=827, +) + + +_SETOP = _descriptor.Descriptor( + name='SetOp', + full_name='SetOp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='adds', full_name='SetOp.adds', index=0, + number=1, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='removes', full_name='SetOp.removes', index=1, + number=2, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=829, + serialized_end=867, +) + + +_GSETOP = _descriptor.Descriptor( + name='GSetOp', + full_name='GSetOp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='adds', full_name='GSetOp.adds', index=0, + number=1, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=869, + serialized_end=891, +) + + +_HLLOP = _descriptor.Descriptor( + name='HllOp', + full_name='HllOp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='adds', full_name='HllOp.adds', index=0, + number=1, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=893, + serialized_end=914, +) + + +_MAPUPDATE = _descriptor.Descriptor( + name='MapUpdate', + full_name='MapUpdate', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='field', full_name='MapUpdate.field', index=0, + number=1, type=11, cpp_type=10, label=2, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='counter_op', full_name='MapUpdate.counter_op', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='set_op', full_name='MapUpdate.set_op', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='register_op', full_name='MapUpdate.register_op', index=3, + number=4, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='flag_op', full_name='MapUpdate.flag_op', index=4, + number=5, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='map_op', full_name='MapUpdate.map_op', index=5, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _MAPUPDATE_FLAGOP, + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=917, + serialized_end=1126, +) + + +_MAPOP = _descriptor.Descriptor( + name='MapOp', + full_name='MapOp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='removes', full_name='MapOp.removes', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='updates', full_name='MapOp.updates', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1128, + serialized_end=1192, +) + + +_DTOP = _descriptor.Descriptor( + name='DtOp', + full_name='DtOp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='counter_op', full_name='DtOp.counter_op', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='set_op', full_name='DtOp.set_op', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='map_op', full_name='DtOp.map_op', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='hll_op', full_name='DtOp.hll_op', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='gset_op', full_name='DtOp.gset_op', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1195, + serialized_end=1331, +) + + +_DTUPDATEREQ = _descriptor.Descriptor( + name='DtUpdateReq', + full_name='DtUpdateReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='DtUpdateReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='DtUpdateReq.key', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='DtUpdateReq.type', index=2, + number=3, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='context', full_name='DtUpdateReq.context', index=3, + number=4, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='op', full_name='DtUpdateReq.op', index=4, + number=5, type=11, cpp_type=10, label=2, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='w', full_name='DtUpdateReq.w', index=5, + number=6, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='dw', full_name='DtUpdateReq.dw', index=6, + number=7, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pw', full_name='DtUpdateReq.pw', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='return_body', full_name='DtUpdateReq.return_body', index=8, + number=9, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timeout', full_name='DtUpdateReq.timeout', index=9, + number=10, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='sloppy_quorum', full_name='DtUpdateReq.sloppy_quorum', index=10, + number=11, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='n_val', full_name='DtUpdateReq.n_val', index=11, + number=12, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='include_context', full_name='DtUpdateReq.include_context', index=12, + number=13, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=True, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1334, + serialized_end=1575, +) + + +_DTUPDATERESP = _descriptor.Descriptor( + name='DtUpdateResp', + full_name='DtUpdateResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='DtUpdateResp.key', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='context', full_name='DtUpdateResp.context', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='counter_value', full_name='DtUpdateResp.counter_value', index=2, + number=3, type=18, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='set_value', full_name='DtUpdateResp.set_value', index=3, + number=4, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='map_value', full_name='DtUpdateResp.map_value', index=4, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='hll_value', full_name='DtUpdateResp.hll_value', index=5, + number=6, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='gset_value', full_name='DtUpdateResp.gset_value', index=6, + number=7, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1578, + serialized_end=1733, +) + +_MAPFIELD.fields_by_name['type'].enum_type = _MAPFIELD_MAPFIELDTYPE +_MAPFIELD_MAPFIELDTYPE.containing_type = _MAPFIELD; +_MAPENTRY.fields_by_name['field'].message_type = _MAPFIELD +_MAPENTRY.fields_by_name['map_value'].message_type = _MAPENTRY +_DTVALUE.fields_by_name['map_value'].message_type = _MAPENTRY +_DTFETCHRESP.fields_by_name['type'].enum_type = _DTFETCHRESP_DATATYPE +_DTFETCHRESP.fields_by_name['value'].message_type = _DTVALUE +_DTFETCHRESP_DATATYPE.containing_type = _DTFETCHRESP; +_MAPUPDATE.fields_by_name['field'].message_type = _MAPFIELD +_MAPUPDATE.fields_by_name['counter_op'].message_type = _COUNTEROP +_MAPUPDATE.fields_by_name['set_op'].message_type = _SETOP +_MAPUPDATE.fields_by_name['flag_op'].enum_type = _MAPUPDATE_FLAGOP +_MAPUPDATE.fields_by_name['map_op'].message_type = _MAPOP +_MAPUPDATE_FLAGOP.containing_type = _MAPUPDATE; +_MAPOP.fields_by_name['removes'].message_type = _MAPFIELD +_MAPOP.fields_by_name['updates'].message_type = _MAPUPDATE +_DTOP.fields_by_name['counter_op'].message_type = _COUNTEROP +_DTOP.fields_by_name['set_op'].message_type = _SETOP +_DTOP.fields_by_name['map_op'].message_type = _MAPOP +_DTOP.fields_by_name['hll_op'].message_type = _HLLOP +_DTOP.fields_by_name['gset_op'].message_type = _GSETOP +_DTUPDATEREQ.fields_by_name['op'].message_type = _DTOP +_DTUPDATERESP.fields_by_name['map_value'].message_type = _MAPENTRY +DESCRIPTOR.message_types_by_name['MapField'] = _MAPFIELD +DESCRIPTOR.message_types_by_name['MapEntry'] = _MAPENTRY +DESCRIPTOR.message_types_by_name['DtFetchReq'] = _DTFETCHREQ +DESCRIPTOR.message_types_by_name['DtValue'] = _DTVALUE +DESCRIPTOR.message_types_by_name['DtFetchResp'] = _DTFETCHRESP +DESCRIPTOR.message_types_by_name['CounterOp'] = _COUNTEROP +DESCRIPTOR.message_types_by_name['SetOp'] = _SETOP +DESCRIPTOR.message_types_by_name['GSetOp'] = _GSETOP +DESCRIPTOR.message_types_by_name['HllOp'] = _HLLOP +DESCRIPTOR.message_types_by_name['MapUpdate'] = _MAPUPDATE +DESCRIPTOR.message_types_by_name['MapOp'] = _MAPOP +DESCRIPTOR.message_types_by_name['DtOp'] = _DTOP +DESCRIPTOR.message_types_by_name['DtUpdateReq'] = _DTUPDATEREQ +DESCRIPTOR.message_types_by_name['DtUpdateResp'] = _DTUPDATERESP + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class MapField(_message.Message): + DESCRIPTOR = _MAPFIELD + + # @@protoc_insertion_point(class_scope:MapField) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class MapEntry(_message.Message): + DESCRIPTOR = _MAPENTRY + + # @@protoc_insertion_point(class_scope:MapEntry) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class DtFetchReq(_message.Message): + DESCRIPTOR = _DTFETCHREQ + + # @@protoc_insertion_point(class_scope:DtFetchReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class DtValue(_message.Message): + DESCRIPTOR = _DTVALUE + + # @@protoc_insertion_point(class_scope:DtValue) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class DtFetchResp(_message.Message): + DESCRIPTOR = _DTFETCHRESP + + # @@protoc_insertion_point(class_scope:DtFetchResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class CounterOp(_message.Message): + DESCRIPTOR = _COUNTEROP + + # @@protoc_insertion_point(class_scope:CounterOp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class SetOp(_message.Message): + DESCRIPTOR = _SETOP + + # @@protoc_insertion_point(class_scope:SetOp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class GSetOp(_message.Message): + DESCRIPTOR = _GSETOP + + # @@protoc_insertion_point(class_scope:GSetOp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class HllOp(_message.Message): + DESCRIPTOR = _HLLOP + + # @@protoc_insertion_point(class_scope:HllOp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class MapUpdate(_message.Message): + DESCRIPTOR = _MAPUPDATE + + # @@protoc_insertion_point(class_scope:MapUpdate) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class MapOp(_message.Message): + DESCRIPTOR = _MAPOP + + # @@protoc_insertion_point(class_scope:MapOp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class DtOp(_message.Message): + DESCRIPTOR = _DTOP + + # @@protoc_insertion_point(class_scope:DtOp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class DtUpdateReq(_message.Message): + DESCRIPTOR = _DTUPDATEREQ + + # @@protoc_insertion_point(class_scope:DtUpdateReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class DtUpdateResp(_message.Message): + DESCRIPTOR = _DTUPDATERESP + + # @@protoc_insertion_point(class_scope:DtUpdateResp) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), '\n\027com.basho.riak.protobufB\010RiakDtPB') +# @@protoc_insertion_point(module_scope) diff --git a/riak/pb/riak_kv_pb2.py b/riak/pb/riak_kv_pb2.py new file mode 100644 index 00000000..f1832df6 --- /dev/null +++ b/riak/pb/riak_kv_pb2.py @@ -0,0 +1,1987 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from six import * +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: riak_kv.proto + +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + +import riak.pb.riak_pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='riak_kv.proto', + package='', + serialized_pb='\n\rriak_kv.proto\x1a\nriak.proto\"\'\n\x12RpbGetClientIdResp\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"&\n\x11RpbSetClientIdReq\x12\x11\n\tclient_id\x18\x01 \x02(\x0c\"\xe9\x01\n\tRpbGetReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\t\n\x01r\x18\x03 \x01(\r\x12\n\n\x02pr\x18\x04 \x01(\r\x12\x14\n\x0c\x62\x61sic_quorum\x18\x05 \x01(\x08\x12\x13\n\x0bnotfound_ok\x18\x06 \x01(\x08\x12\x13\n\x0bif_modified\x18\x07 \x01(\x0c\x12\x0c\n\x04head\x18\x08 \x01(\x08\x12\x15\n\rdeletedvclock\x18\t \x01(\x08\x12\x0f\n\x07timeout\x18\n \x01(\r\x12\x15\n\rsloppy_quorum\x18\x0b \x01(\x08\x12\r\n\x05n_val\x18\x0c \x01(\r\x12\x0c\n\x04type\x18\r \x01(\x0c\"M\n\nRpbGetResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\x12\x11\n\tunchanged\x18\x03 \x01(\x08\"\xa6\x02\n\tRpbPutReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0e\n\x06vclock\x18\x03 \x01(\x0c\x12\x1c\n\x07\x63ontent\x18\x04 \x02(\x0b\x32\x0b.RpbContent\x12\t\n\x01w\x18\x05 \x01(\r\x12\n\n\x02\x64w\x18\x06 \x01(\r\x12\x13\n\x0breturn_body\x18\x07 \x01(\x08\x12\n\n\x02pw\x18\x08 \x01(\r\x12\x17\n\x0fif_not_modified\x18\t \x01(\x08\x12\x15\n\rif_none_match\x18\n \x01(\x08\x12\x13\n\x0breturn_head\x18\x0b \x01(\x08\x12\x0f\n\x07timeout\x18\x0c \x01(\r\x12\x0c\n\x04\x61sis\x18\r \x01(\x08\x12\x15\n\rsloppy_quorum\x18\x0e \x01(\x08\x12\r\n\x05n_val\x18\x0f \x01(\r\x12\x0c\n\x04type\x18\x10 \x01(\x0c\"G\n\nRpbPutResp\x12\x1c\n\x07\x63ontent\x18\x01 \x03(\x0b\x32\x0b.RpbContent\x12\x0e\n\x06vclock\x18\x02 \x01(\x0c\x12\x0b\n\x03key\x18\x03 \x01(\x0c\"\xc3\x01\n\tRpbDelReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\n\n\x02rw\x18\x03 \x01(\r\x12\x0e\n\x06vclock\x18\x04 \x01(\x0c\x12\t\n\x01r\x18\x05 \x01(\r\x12\t\n\x01w\x18\x06 \x01(\r\x12\n\n\x02pr\x18\x07 \x01(\r\x12\n\n\x02pw\x18\x08 \x01(\r\x12\n\n\x02\x64w\x18\t \x01(\r\x12\x0f\n\x07timeout\x18\n \x01(\r\x12\x15\n\rsloppy_quorum\x18\x0b \x01(\x08\x12\r\n\x05n_val\x18\x0c \x01(\r\x12\x0c\n\x04type\x18\r \x01(\x0c\"B\n\x11RpbListBucketsReq\x12\x0f\n\x07timeout\x18\x01 \x01(\r\x12\x0e\n\x06stream\x18\x02 \x01(\x08\x12\x0c\n\x04type\x18\x03 \x01(\x0c\"3\n\x12RpbListBucketsResp\x12\x0f\n\x07\x62uckets\x18\x01 \x03(\x0c\x12\x0c\n\x04\x64one\x18\x02 \x01(\x08\"?\n\x0eRpbListKeysReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0f\n\x07timeout\x18\x02 \x01(\r\x12\x0c\n\x04type\x18\x03 \x01(\x0c\"-\n\x0fRpbListKeysResp\x12\x0c\n\x04keys\x18\x01 \x03(\x0c\x12\x0c\n\x04\x64one\x18\x02 \x01(\x08\"5\n\x0cRpbMapRedReq\x12\x0f\n\x07request\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x02(\x0c\">\n\rRpbMapRedResp\x12\r\n\x05phase\x18\x01 \x01(\r\x12\x10\n\x08response\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\"\xf9\x02\n\x0bRpbIndexReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\r\n\x05index\x18\x02 \x02(\x0c\x12*\n\x05qtype\x18\x03 \x02(\x0e\x32\x1b.RpbIndexReq.IndexQueryType\x12\x0b\n\x03key\x18\x04 \x01(\x0c\x12\x11\n\trange_min\x18\x05 \x01(\x0c\x12\x11\n\trange_max\x18\x06 \x01(\x0c\x12\x14\n\x0creturn_terms\x18\x07 \x01(\x08\x12\x0e\n\x06stream\x18\x08 \x01(\x08\x12\x13\n\x0bmax_results\x18\t \x01(\r\x12\x14\n\x0c\x63ontinuation\x18\n \x01(\x0c\x12\x0f\n\x07timeout\x18\x0b \x01(\r\x12\x0c\n\x04type\x18\x0c \x01(\x0c\x12\x12\n\nterm_regex\x18\r \x01(\x0c\x12\x17\n\x0fpagination_sort\x18\x0e \x01(\x08\x12\x15\n\rcover_context\x18\x0f \x01(\x0c\x12\x13\n\x0breturn_body\x18\x10 \x01(\x08\"#\n\x0eIndexQueryType\x12\x06\n\x02\x65q\x10\x00\x12\t\n\x05range\x10\x01\"[\n\x0cRpbIndexResp\x12\x0c\n\x04keys\x18\x01 \x03(\x0c\x12\x19\n\x07results\x18\x02 \x03(\x0b\x32\x08.RpbPair\x12\x14\n\x0c\x63ontinuation\x18\x03 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x04 \x01(\x08\"X\n\x10RpbIndexBodyResp\x12 \n\x07objects\x18\x01 \x03(\x0b\x32\x0f.RpbIndexObject\x12\x14\n\x0c\x63ontinuation\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\"\xd8\x01\n\x0eRpbCSBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x11\n\tstart_key\x18\x02 \x02(\x0c\x12\x0f\n\x07\x65nd_key\x18\x03 \x01(\x0c\x12\x18\n\nstart_incl\x18\x04 \x01(\x08:\x04true\x12\x17\n\x08\x65nd_incl\x18\x05 \x01(\x08:\x05\x66\x61lse\x12\x14\n\x0c\x63ontinuation\x18\x06 \x01(\x0c\x12\x13\n\x0bmax_results\x18\x07 \x01(\r\x12\x0f\n\x07timeout\x18\x08 \x01(\r\x12\x0c\n\x04type\x18\t \x01(\x0c\x12\x15\n\rcover_context\x18\n \x01(\x0c\"W\n\x0fRpbCSBucketResp\x12 \n\x07objects\x18\x01 \x03(\x0b\x32\x0f.RpbIndexObject\x12\x14\n\x0c\x63ontinuation\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\":\n\x0eRpbIndexObject\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\x1b\n\x06object\x18\x02 \x02(\x0b\x32\x0b.RpbGetResp\"\xf5\x01\n\nRpbContent\x12\r\n\x05value\x18\x01 \x02(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x02 \x01(\x0c\x12\x0f\n\x07\x63harset\x18\x03 \x01(\x0c\x12\x18\n\x10\x63ontent_encoding\x18\x04 \x01(\x0c\x12\x0c\n\x04vtag\x18\x05 \x01(\x0c\x12\x17\n\x05links\x18\x06 \x03(\x0b\x32\x08.RpbLink\x12\x10\n\x08last_mod\x18\x07 \x01(\r\x12\x16\n\x0elast_mod_usecs\x18\x08 \x01(\r\x12\x1a\n\x08usermeta\x18\t \x03(\x0b\x32\x08.RpbPair\x12\x19\n\x07indexes\x18\n \x03(\x0b\x32\x08.RpbPair\x12\x0f\n\x07\x64\x65leted\x18\x0b \x01(\x08\"3\n\x07RpbLink\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\x0c\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0b\n\x03tag\x18\x03 \x01(\x0c\"z\n\x13RpbCounterUpdateReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\x0e\n\x06\x61mount\x18\x03 \x02(\x12\x12\t\n\x01w\x18\x04 \x01(\r\x12\n\n\x02\x64w\x18\x05 \x01(\r\x12\n\n\x02pw\x18\x06 \x01(\r\x12\x13\n\x0breturnvalue\x18\x07 \x01(\x08\"%\n\x14RpbCounterUpdateResp\x12\r\n\x05value\x18\x01 \x01(\x12\"q\n\x10RpbCounterGetReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\t\n\x01r\x18\x03 \x01(\r\x12\n\n\x02pr\x18\x04 \x01(\r\x12\x14\n\x0c\x62\x61sic_quorum\x18\x05 \x01(\x08\x12\x13\n\x0bnotfound_ok\x18\x06 \x01(\x08\"\"\n\x11RpbCounterGetResp\x12\r\n\x05value\x18\x01 \x01(\x12\"G\n\x1aRpbGetBucketKeyPreflistReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0b\n\x03key\x18\x02 \x02(\x0c\x12\x0c\n\x04type\x18\x03 \x01(\x0c\"J\n\x1bRpbGetBucketKeyPreflistResp\x12+\n\x08preflist\x18\x01 \x03(\x0b\x32\x19.RpbBucketKeyPreflistItem\"L\n\x18RpbBucketKeyPreflistItem\x12\x11\n\tpartition\x18\x01 \x02(\x03\x12\x0c\n\x04node\x18\x02 \x02(\x0c\x12\x0f\n\x07primary\x18\x03 \x02(\x08\"x\n\x0eRpbCoverageReq\x12\x0c\n\x04type\x18\x01 \x01(\x0c\x12\x0e\n\x06\x62ucket\x18\x02 \x02(\x0c\x12\x16\n\x0emin_partitions\x18\x03 \x01(\r\x12\x15\n\rreplace_cover\x18\x04 \x01(\x0c\x12\x19\n\x11unavailable_cover\x18\x05 \x03(\x0c\"5\n\x0fRpbCoverageResp\x12\"\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x11.RpbCoverageEntry\"Z\n\x10RpbCoverageEntry\x12\n\n\x02ip\x18\x01 \x02(\x0c\x12\x0c\n\x04port\x18\x02 \x02(\r\x12\x15\n\rkeyspace_desc\x18\x03 \x01(\x0c\x12\x15\n\rcover_context\x18\x04 \x02(\x0c\x42#\n\x17\x63om.basho.riak.protobufB\x08RiakKvPB') + + + +_RPBINDEXREQ_INDEXQUERYTYPE = _descriptor.EnumDescriptor( + name='IndexQueryType', + full_name='RpbIndexReq.IndexQueryType', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='eq', index=0, number=0, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='range', index=1, number=1, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=1688, + serialized_end=1723, +) + + +_RPBGETCLIENTIDRESP = _descriptor.Descriptor( + name='RpbGetClientIdResp', + full_name='RpbGetClientIdResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='client_id', full_name='RpbGetClientIdResp.client_id', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=29, + serialized_end=68, +) + + +_RPBSETCLIENTIDREQ = _descriptor.Descriptor( + name='RpbSetClientIdReq', + full_name='RpbSetClientIdReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='client_id', full_name='RpbSetClientIdReq.client_id', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=70, + serialized_end=108, +) + + +_RPBGETREQ = _descriptor.Descriptor( + name='RpbGetReq', + full_name='RpbGetReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbGetReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='RpbGetReq.key', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='r', full_name='RpbGetReq.r', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pr', full_name='RpbGetReq.pr', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='basic_quorum', full_name='RpbGetReq.basic_quorum', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='notfound_ok', full_name='RpbGetReq.notfound_ok', index=5, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='if_modified', full_name='RpbGetReq.if_modified', index=6, + number=7, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='head', full_name='RpbGetReq.head', index=7, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='deletedvclock', full_name='RpbGetReq.deletedvclock', index=8, + number=9, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timeout', full_name='RpbGetReq.timeout', index=9, + number=10, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='sloppy_quorum', full_name='RpbGetReq.sloppy_quorum', index=10, + number=11, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='n_val', full_name='RpbGetReq.n_val', index=11, + number=12, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='RpbGetReq.type', index=12, + number=13, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=111, + serialized_end=344, +) + + +_RPBGETRESP = _descriptor.Descriptor( + name='RpbGetResp', + full_name='RpbGetResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='content', full_name='RpbGetResp.content', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='vclock', full_name='RpbGetResp.vclock', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='unchanged', full_name='RpbGetResp.unchanged', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=346, + serialized_end=423, +) + + +_RPBPUTREQ = _descriptor.Descriptor( + name='RpbPutReq', + full_name='RpbPutReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbPutReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='RpbPutReq.key', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='vclock', full_name='RpbPutReq.vclock', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='content', full_name='RpbPutReq.content', index=3, + number=4, type=11, cpp_type=10, label=2, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='w', full_name='RpbPutReq.w', index=4, + number=5, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='dw', full_name='RpbPutReq.dw', index=5, + number=6, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='return_body', full_name='RpbPutReq.return_body', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pw', full_name='RpbPutReq.pw', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='if_not_modified', full_name='RpbPutReq.if_not_modified', index=8, + number=9, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='if_none_match', full_name='RpbPutReq.if_none_match', index=9, + number=10, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='return_head', full_name='RpbPutReq.return_head', index=10, + number=11, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timeout', full_name='RpbPutReq.timeout', index=11, + number=12, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='asis', full_name='RpbPutReq.asis', index=12, + number=13, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='sloppy_quorum', full_name='RpbPutReq.sloppy_quorum', index=13, + number=14, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='n_val', full_name='RpbPutReq.n_val', index=14, + number=15, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='RpbPutReq.type', index=15, + number=16, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=426, + serialized_end=720, +) + + +_RPBPUTRESP = _descriptor.Descriptor( + name='RpbPutResp', + full_name='RpbPutResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='content', full_name='RpbPutResp.content', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='vclock', full_name='RpbPutResp.vclock', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='RpbPutResp.key', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=722, + serialized_end=793, +) + + +_RPBDELREQ = _descriptor.Descriptor( + name='RpbDelReq', + full_name='RpbDelReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbDelReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='RpbDelReq.key', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='rw', full_name='RpbDelReq.rw', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='vclock', full_name='RpbDelReq.vclock', index=3, + number=4, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='r', full_name='RpbDelReq.r', index=4, + number=5, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='w', full_name='RpbDelReq.w', index=5, + number=6, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pr', full_name='RpbDelReq.pr', index=6, + number=7, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pw', full_name='RpbDelReq.pw', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='dw', full_name='RpbDelReq.dw', index=8, + number=9, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timeout', full_name='RpbDelReq.timeout', index=9, + number=10, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='sloppy_quorum', full_name='RpbDelReq.sloppy_quorum', index=10, + number=11, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='n_val', full_name='RpbDelReq.n_val', index=11, + number=12, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='RpbDelReq.type', index=12, + number=13, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=796, + serialized_end=991, +) + + +_RPBLISTBUCKETSREQ = _descriptor.Descriptor( + name='RpbListBucketsReq', + full_name='RpbListBucketsReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='timeout', full_name='RpbListBucketsReq.timeout', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='stream', full_name='RpbListBucketsReq.stream', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='RpbListBucketsReq.type', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=993, + serialized_end=1059, +) + + +_RPBLISTBUCKETSRESP = _descriptor.Descriptor( + name='RpbListBucketsResp', + full_name='RpbListBucketsResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='buckets', full_name='RpbListBucketsResp.buckets', index=0, + number=1, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='done', full_name='RpbListBucketsResp.done', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1061, + serialized_end=1112, +) + + +_RPBLISTKEYSREQ = _descriptor.Descriptor( + name='RpbListKeysReq', + full_name='RpbListKeysReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbListKeysReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timeout', full_name='RpbListKeysReq.timeout', index=1, + number=2, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='RpbListKeysReq.type', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1114, + serialized_end=1177, +) + + +_RPBLISTKEYSRESP = _descriptor.Descriptor( + name='RpbListKeysResp', + full_name='RpbListKeysResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='keys', full_name='RpbListKeysResp.keys', index=0, + number=1, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='done', full_name='RpbListKeysResp.done', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1179, + serialized_end=1224, +) + + +_RPBMAPREDREQ = _descriptor.Descriptor( + name='RpbMapRedReq', + full_name='RpbMapRedReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='request', full_name='RpbMapRedReq.request', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='content_type', full_name='RpbMapRedReq.content_type', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1226, + serialized_end=1279, +) + + +_RPBMAPREDRESP = _descriptor.Descriptor( + name='RpbMapRedResp', + full_name='RpbMapRedResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='phase', full_name='RpbMapRedResp.phase', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='response', full_name='RpbMapRedResp.response', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='done', full_name='RpbMapRedResp.done', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1281, + serialized_end=1343, +) + + +_RPBINDEXREQ = _descriptor.Descriptor( + name='RpbIndexReq', + full_name='RpbIndexReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbIndexReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='index', full_name='RpbIndexReq.index', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='qtype', full_name='RpbIndexReq.qtype', index=2, + number=3, type=14, cpp_type=8, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='RpbIndexReq.key', index=3, + number=4, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='range_min', full_name='RpbIndexReq.range_min', index=4, + number=5, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='range_max', full_name='RpbIndexReq.range_max', index=5, + number=6, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='return_terms', full_name='RpbIndexReq.return_terms', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='stream', full_name='RpbIndexReq.stream', index=7, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='max_results', full_name='RpbIndexReq.max_results', index=8, + number=9, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='continuation', full_name='RpbIndexReq.continuation', index=9, + number=10, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timeout', full_name='RpbIndexReq.timeout', index=10, + number=11, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='RpbIndexReq.type', index=11, + number=12, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='term_regex', full_name='RpbIndexReq.term_regex', index=12, + number=13, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pagination_sort', full_name='RpbIndexReq.pagination_sort', index=13, + number=14, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='cover_context', full_name='RpbIndexReq.cover_context', index=14, + number=15, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='return_body', full_name='RpbIndexReq.return_body', index=15, + number=16, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _RPBINDEXREQ_INDEXQUERYTYPE, + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1346, + serialized_end=1723, +) + + +_RPBINDEXRESP = _descriptor.Descriptor( + name='RpbIndexResp', + full_name='RpbIndexResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='keys', full_name='RpbIndexResp.keys', index=0, + number=1, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='results', full_name='RpbIndexResp.results', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='continuation', full_name='RpbIndexResp.continuation', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='done', full_name='RpbIndexResp.done', index=3, + number=4, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1725, + serialized_end=1816, +) + + +_RPBINDEXBODYRESP = _descriptor.Descriptor( + name='RpbIndexBodyResp', + full_name='RpbIndexBodyResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='objects', full_name='RpbIndexBodyResp.objects', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='continuation', full_name='RpbIndexBodyResp.continuation', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='done', full_name='RpbIndexBodyResp.done', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1818, + serialized_end=1906, +) + + +_RPBCSBUCKETREQ = _descriptor.Descriptor( + name='RpbCSBucketReq', + full_name='RpbCSBucketReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbCSBucketReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='start_key', full_name='RpbCSBucketReq.start_key', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='end_key', full_name='RpbCSBucketReq.end_key', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='start_incl', full_name='RpbCSBucketReq.start_incl', index=3, + number=4, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=True, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='end_incl', full_name='RpbCSBucketReq.end_incl', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='continuation', full_name='RpbCSBucketReq.continuation', index=5, + number=6, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='max_results', full_name='RpbCSBucketReq.max_results', index=6, + number=7, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timeout', full_name='RpbCSBucketReq.timeout', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='RpbCSBucketReq.type', index=8, + number=9, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='cover_context', full_name='RpbCSBucketReq.cover_context', index=9, + number=10, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1909, + serialized_end=2125, +) + + +_RPBCSBUCKETRESP = _descriptor.Descriptor( + name='RpbCSBucketResp', + full_name='RpbCSBucketResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='objects', full_name='RpbCSBucketResp.objects', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='continuation', full_name='RpbCSBucketResp.continuation', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='done', full_name='RpbCSBucketResp.done', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=2127, + serialized_end=2214, +) + + +_RPBINDEXOBJECT = _descriptor.Descriptor( + name='RpbIndexObject', + full_name='RpbIndexObject', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='RpbIndexObject.key', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='object', full_name='RpbIndexObject.object', index=1, + number=2, type=11, cpp_type=10, label=2, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=2216, + serialized_end=2274, +) + + +_RPBCONTENT = _descriptor.Descriptor( + name='RpbContent', + full_name='RpbContent', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='value', full_name='RpbContent.value', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='content_type', full_name='RpbContent.content_type', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='charset', full_name='RpbContent.charset', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='content_encoding', full_name='RpbContent.content_encoding', index=3, + number=4, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='vtag', full_name='RpbContent.vtag', index=4, + number=5, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='links', full_name='RpbContent.links', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='last_mod', full_name='RpbContent.last_mod', index=6, + number=7, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='last_mod_usecs', full_name='RpbContent.last_mod_usecs', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='usermeta', full_name='RpbContent.usermeta', index=8, + number=9, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='indexes', full_name='RpbContent.indexes', index=9, + number=10, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='deleted', full_name='RpbContent.deleted', index=10, + number=11, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=2277, + serialized_end=2522, +) + + +_RPBLINK = _descriptor.Descriptor( + name='RpbLink', + full_name='RpbLink', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbLink.bucket', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='RpbLink.key', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='tag', full_name='RpbLink.tag', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=2524, + serialized_end=2575, +) + + +_RPBCOUNTERUPDATEREQ = _descriptor.Descriptor( + name='RpbCounterUpdateReq', + full_name='RpbCounterUpdateReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbCounterUpdateReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='RpbCounterUpdateReq.key', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='amount', full_name='RpbCounterUpdateReq.amount', index=2, + number=3, type=18, cpp_type=2, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='w', full_name='RpbCounterUpdateReq.w', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='dw', full_name='RpbCounterUpdateReq.dw', index=4, + number=5, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pw', full_name='RpbCounterUpdateReq.pw', index=5, + number=6, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='returnvalue', full_name='RpbCounterUpdateReq.returnvalue', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=2577, + serialized_end=2699, +) + + +_RPBCOUNTERUPDATERESP = _descriptor.Descriptor( + name='RpbCounterUpdateResp', + full_name='RpbCounterUpdateResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='value', full_name='RpbCounterUpdateResp.value', index=0, + number=1, type=18, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=2701, + serialized_end=2738, +) + + +_RPBCOUNTERGETREQ = _descriptor.Descriptor( + name='RpbCounterGetReq', + full_name='RpbCounterGetReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbCounterGetReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='RpbCounterGetReq.key', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='r', full_name='RpbCounterGetReq.r', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pr', full_name='RpbCounterGetReq.pr', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='basic_quorum', full_name='RpbCounterGetReq.basic_quorum', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='notfound_ok', full_name='RpbCounterGetReq.notfound_ok', index=5, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=2740, + serialized_end=2853, +) + + +_RPBCOUNTERGETRESP = _descriptor.Descriptor( + name='RpbCounterGetResp', + full_name='RpbCounterGetResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='value', full_name='RpbCounterGetResp.value', index=0, + number=1, type=18, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=2855, + serialized_end=2889, +) + + +_RPBGETBUCKETKEYPREFLISTREQ = _descriptor.Descriptor( + name='RpbGetBucketKeyPreflistReq', + full_name='RpbGetBucketKeyPreflistReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbGetBucketKeyPreflistReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='RpbGetBucketKeyPreflistReq.key', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='RpbGetBucketKeyPreflistReq.type', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=2891, + serialized_end=2962, +) + + +_RPBGETBUCKETKEYPREFLISTRESP = _descriptor.Descriptor( + name='RpbGetBucketKeyPreflistResp', + full_name='RpbGetBucketKeyPreflistResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='preflist', full_name='RpbGetBucketKeyPreflistResp.preflist', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=2964, + serialized_end=3038, +) + + +_RPBBUCKETKEYPREFLISTITEM = _descriptor.Descriptor( + name='RpbBucketKeyPreflistItem', + full_name='RpbBucketKeyPreflistItem', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='partition', full_name='RpbBucketKeyPreflistItem.partition', index=0, + number=1, type=3, cpp_type=2, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='node', full_name='RpbBucketKeyPreflistItem.node', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='primary', full_name='RpbBucketKeyPreflistItem.primary', index=2, + number=3, type=8, cpp_type=7, label=2, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=3040, + serialized_end=3116, +) + + +_RPBCOVERAGEREQ = _descriptor.Descriptor( + name='RpbCoverageReq', + full_name='RpbCoverageReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='type', full_name='RpbCoverageReq.type', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbCoverageReq.bucket', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='min_partitions', full_name='RpbCoverageReq.min_partitions', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='replace_cover', full_name='RpbCoverageReq.replace_cover', index=3, + number=4, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='unavailable_cover', full_name='RpbCoverageReq.unavailable_cover', index=4, + number=5, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=3118, + serialized_end=3238, +) + + +_RPBCOVERAGERESP = _descriptor.Descriptor( + name='RpbCoverageResp', + full_name='RpbCoverageResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='entries', full_name='RpbCoverageResp.entries', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=3240, + serialized_end=3293, +) + + +_RPBCOVERAGEENTRY = _descriptor.Descriptor( + name='RpbCoverageEntry', + full_name='RpbCoverageEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='ip', full_name='RpbCoverageEntry.ip', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='port', full_name='RpbCoverageEntry.port', index=1, + number=2, type=13, cpp_type=3, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='keyspace_desc', full_name='RpbCoverageEntry.keyspace_desc', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='cover_context', full_name='RpbCoverageEntry.cover_context', index=3, + number=4, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=3295, + serialized_end=3385, +) + +_RPBGETRESP.fields_by_name['content'].message_type = _RPBCONTENT +_RPBPUTREQ.fields_by_name['content'].message_type = _RPBCONTENT +_RPBPUTRESP.fields_by_name['content'].message_type = _RPBCONTENT +_RPBINDEXREQ.fields_by_name['qtype'].enum_type = _RPBINDEXREQ_INDEXQUERYTYPE +_RPBINDEXREQ_INDEXQUERYTYPE.containing_type = _RPBINDEXREQ; +_RPBINDEXRESP.fields_by_name['results'].message_type = riak.pb.riak_pb2._RPBPAIR +_RPBINDEXBODYRESP.fields_by_name['objects'].message_type = _RPBINDEXOBJECT +_RPBCSBUCKETRESP.fields_by_name['objects'].message_type = _RPBINDEXOBJECT +_RPBINDEXOBJECT.fields_by_name['object'].message_type = _RPBGETRESP +_RPBCONTENT.fields_by_name['links'].message_type = _RPBLINK +_RPBCONTENT.fields_by_name['usermeta'].message_type = riak.pb.riak_pb2._RPBPAIR +_RPBCONTENT.fields_by_name['indexes'].message_type = riak.pb.riak_pb2._RPBPAIR +_RPBGETBUCKETKEYPREFLISTRESP.fields_by_name['preflist'].message_type = _RPBBUCKETKEYPREFLISTITEM +_RPBCOVERAGERESP.fields_by_name['entries'].message_type = _RPBCOVERAGEENTRY +DESCRIPTOR.message_types_by_name['RpbGetClientIdResp'] = _RPBGETCLIENTIDRESP +DESCRIPTOR.message_types_by_name['RpbSetClientIdReq'] = _RPBSETCLIENTIDREQ +DESCRIPTOR.message_types_by_name['RpbGetReq'] = _RPBGETREQ +DESCRIPTOR.message_types_by_name['RpbGetResp'] = _RPBGETRESP +DESCRIPTOR.message_types_by_name['RpbPutReq'] = _RPBPUTREQ +DESCRIPTOR.message_types_by_name['RpbPutResp'] = _RPBPUTRESP +DESCRIPTOR.message_types_by_name['RpbDelReq'] = _RPBDELREQ +DESCRIPTOR.message_types_by_name['RpbListBucketsReq'] = _RPBLISTBUCKETSREQ +DESCRIPTOR.message_types_by_name['RpbListBucketsResp'] = _RPBLISTBUCKETSRESP +DESCRIPTOR.message_types_by_name['RpbListKeysReq'] = _RPBLISTKEYSREQ +DESCRIPTOR.message_types_by_name['RpbListKeysResp'] = _RPBLISTKEYSRESP +DESCRIPTOR.message_types_by_name['RpbMapRedReq'] = _RPBMAPREDREQ +DESCRIPTOR.message_types_by_name['RpbMapRedResp'] = _RPBMAPREDRESP +DESCRIPTOR.message_types_by_name['RpbIndexReq'] = _RPBINDEXREQ +DESCRIPTOR.message_types_by_name['RpbIndexResp'] = _RPBINDEXRESP +DESCRIPTOR.message_types_by_name['RpbIndexBodyResp'] = _RPBINDEXBODYRESP +DESCRIPTOR.message_types_by_name['RpbCSBucketReq'] = _RPBCSBUCKETREQ +DESCRIPTOR.message_types_by_name['RpbCSBucketResp'] = _RPBCSBUCKETRESP +DESCRIPTOR.message_types_by_name['RpbIndexObject'] = _RPBINDEXOBJECT +DESCRIPTOR.message_types_by_name['RpbContent'] = _RPBCONTENT +DESCRIPTOR.message_types_by_name['RpbLink'] = _RPBLINK +DESCRIPTOR.message_types_by_name['RpbCounterUpdateReq'] = _RPBCOUNTERUPDATEREQ +DESCRIPTOR.message_types_by_name['RpbCounterUpdateResp'] = _RPBCOUNTERUPDATERESP +DESCRIPTOR.message_types_by_name['RpbCounterGetReq'] = _RPBCOUNTERGETREQ +DESCRIPTOR.message_types_by_name['RpbCounterGetResp'] = _RPBCOUNTERGETRESP +DESCRIPTOR.message_types_by_name['RpbGetBucketKeyPreflistReq'] = _RPBGETBUCKETKEYPREFLISTREQ +DESCRIPTOR.message_types_by_name['RpbGetBucketKeyPreflistResp'] = _RPBGETBUCKETKEYPREFLISTRESP +DESCRIPTOR.message_types_by_name['RpbBucketKeyPreflistItem'] = _RPBBUCKETKEYPREFLISTITEM +DESCRIPTOR.message_types_by_name['RpbCoverageReq'] = _RPBCOVERAGEREQ +DESCRIPTOR.message_types_by_name['RpbCoverageResp'] = _RPBCOVERAGERESP +DESCRIPTOR.message_types_by_name['RpbCoverageEntry'] = _RPBCOVERAGEENTRY + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbGetClientIdResp(_message.Message): + DESCRIPTOR = _RPBGETCLIENTIDRESP + + # @@protoc_insertion_point(class_scope:RpbGetClientIdResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbSetClientIdReq(_message.Message): + DESCRIPTOR = _RPBSETCLIENTIDREQ + + # @@protoc_insertion_point(class_scope:RpbSetClientIdReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbGetReq(_message.Message): + DESCRIPTOR = _RPBGETREQ + + # @@protoc_insertion_point(class_scope:RpbGetReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbGetResp(_message.Message): + DESCRIPTOR = _RPBGETRESP + + # @@protoc_insertion_point(class_scope:RpbGetResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbPutReq(_message.Message): + DESCRIPTOR = _RPBPUTREQ + + # @@protoc_insertion_point(class_scope:RpbPutReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbPutResp(_message.Message): + DESCRIPTOR = _RPBPUTRESP + + # @@protoc_insertion_point(class_scope:RpbPutResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbDelReq(_message.Message): + DESCRIPTOR = _RPBDELREQ + + # @@protoc_insertion_point(class_scope:RpbDelReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbListBucketsReq(_message.Message): + DESCRIPTOR = _RPBLISTBUCKETSREQ + + # @@protoc_insertion_point(class_scope:RpbListBucketsReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbListBucketsResp(_message.Message): + DESCRIPTOR = _RPBLISTBUCKETSRESP + + # @@protoc_insertion_point(class_scope:RpbListBucketsResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbListKeysReq(_message.Message): + DESCRIPTOR = _RPBLISTKEYSREQ + + # @@protoc_insertion_point(class_scope:RpbListKeysReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbListKeysResp(_message.Message): + DESCRIPTOR = _RPBLISTKEYSRESP + + # @@protoc_insertion_point(class_scope:RpbListKeysResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbMapRedReq(_message.Message): + DESCRIPTOR = _RPBMAPREDREQ + + # @@protoc_insertion_point(class_scope:RpbMapRedReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbMapRedResp(_message.Message): + DESCRIPTOR = _RPBMAPREDRESP + + # @@protoc_insertion_point(class_scope:RpbMapRedResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbIndexReq(_message.Message): + DESCRIPTOR = _RPBINDEXREQ + + # @@protoc_insertion_point(class_scope:RpbIndexReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbIndexResp(_message.Message): + DESCRIPTOR = _RPBINDEXRESP + + # @@protoc_insertion_point(class_scope:RpbIndexResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbIndexBodyResp(_message.Message): + DESCRIPTOR = _RPBINDEXBODYRESP + + # @@protoc_insertion_point(class_scope:RpbIndexBodyResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbCSBucketReq(_message.Message): + DESCRIPTOR = _RPBCSBUCKETREQ + + # @@protoc_insertion_point(class_scope:RpbCSBucketReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbCSBucketResp(_message.Message): + DESCRIPTOR = _RPBCSBUCKETRESP + + # @@protoc_insertion_point(class_scope:RpbCSBucketResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbIndexObject(_message.Message): + DESCRIPTOR = _RPBINDEXOBJECT + + # @@protoc_insertion_point(class_scope:RpbIndexObject) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbContent(_message.Message): + DESCRIPTOR = _RPBCONTENT + + # @@protoc_insertion_point(class_scope:RpbContent) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbLink(_message.Message): + DESCRIPTOR = _RPBLINK + + # @@protoc_insertion_point(class_scope:RpbLink) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbCounterUpdateReq(_message.Message): + DESCRIPTOR = _RPBCOUNTERUPDATEREQ + + # @@protoc_insertion_point(class_scope:RpbCounterUpdateReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbCounterUpdateResp(_message.Message): + DESCRIPTOR = _RPBCOUNTERUPDATERESP + + # @@protoc_insertion_point(class_scope:RpbCounterUpdateResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbCounterGetReq(_message.Message): + DESCRIPTOR = _RPBCOUNTERGETREQ + + # @@protoc_insertion_point(class_scope:RpbCounterGetReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbCounterGetResp(_message.Message): + DESCRIPTOR = _RPBCOUNTERGETRESP + + # @@protoc_insertion_point(class_scope:RpbCounterGetResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbGetBucketKeyPreflistReq(_message.Message): + DESCRIPTOR = _RPBGETBUCKETKEYPREFLISTREQ + + # @@protoc_insertion_point(class_scope:RpbGetBucketKeyPreflistReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbGetBucketKeyPreflistResp(_message.Message): + DESCRIPTOR = _RPBGETBUCKETKEYPREFLISTRESP + + # @@protoc_insertion_point(class_scope:RpbGetBucketKeyPreflistResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbBucketKeyPreflistItem(_message.Message): + DESCRIPTOR = _RPBBUCKETKEYPREFLISTITEM + + # @@protoc_insertion_point(class_scope:RpbBucketKeyPreflistItem) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbCoverageReq(_message.Message): + DESCRIPTOR = _RPBCOVERAGEREQ + + # @@protoc_insertion_point(class_scope:RpbCoverageReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbCoverageResp(_message.Message): + DESCRIPTOR = _RPBCOVERAGERESP + + # @@protoc_insertion_point(class_scope:RpbCoverageResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbCoverageEntry(_message.Message): + DESCRIPTOR = _RPBCOVERAGEENTRY + + # @@protoc_insertion_point(class_scope:RpbCoverageEntry) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), '\n\027com.basho.riak.protobufB\010RiakKvPB') +# @@protoc_insertion_point(module_scope) diff --git a/riak/pb/riak_pb2.py b/riak/pb/riak_pb2.py new file mode 100644 index 00000000..72dba122 --- /dev/null +++ b/riak/pb/riak_pb2.py @@ -0,0 +1,807 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from six import * +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: riak.proto + +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='riak.proto', + package='', + serialized_pb='\n\nriak.proto\"/\n\x0cRpbErrorResp\x12\x0e\n\x06\x65rrmsg\x18\x01 \x02(\x0c\x12\x0f\n\x07\x65rrcode\x18\x02 \x02(\r\"<\n\x14RpbGetServerInfoResp\x12\x0c\n\x04node\x18\x01 \x01(\x0c\x12\x16\n\x0eserver_version\x18\x02 \x01(\x0c\"%\n\x07RpbPair\x12\x0b\n\x03key\x18\x01 \x02(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c\"/\n\x0fRpbGetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0c\n\x04type\x18\x02 \x01(\x0c\"2\n\x10RpbGetBucketResp\x12\x1e\n\x05props\x18\x01 \x02(\x0b\x32\x0f.RpbBucketProps\"O\n\x0fRpbSetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x1e\n\x05props\x18\x02 \x02(\x0b\x32\x0f.RpbBucketProps\x12\x0c\n\x04type\x18\x03 \x01(\x0c\"1\n\x11RpbResetBucketReq\x12\x0e\n\x06\x62ucket\x18\x01 \x02(\x0c\x12\x0c\n\x04type\x18\x02 \x01(\x0c\"#\n\x13RpbGetBucketTypeReq\x12\x0c\n\x04type\x18\x01 \x02(\x0c\"C\n\x13RpbSetBucketTypeReq\x12\x0c\n\x04type\x18\x01 \x02(\x0c\x12\x1e\n\x05props\x18\x02 \x02(\x0b\x32\x0f.RpbBucketProps\"-\n\tRpbModFun\x12\x0e\n\x06module\x18\x01 \x02(\x0c\x12\x10\n\x08\x66unction\x18\x02 \x02(\x0c\"9\n\rRpbCommitHook\x12\x1a\n\x06modfun\x18\x01 \x01(\x0b\x32\n.RpbModFun\x12\x0c\n\x04name\x18\x02 \x01(\x0c\"\xc7\x05\n\x0eRpbBucketProps\x12\r\n\x05n_val\x18\x01 \x01(\r\x12\x12\n\nallow_mult\x18\x02 \x01(\x08\x12\x17\n\x0flast_write_wins\x18\x03 \x01(\x08\x12!\n\tprecommit\x18\x04 \x03(\x0b\x32\x0e.RpbCommitHook\x12\x1c\n\rhas_precommit\x18\x05 \x01(\x08:\x05\x66\x61lse\x12\"\n\npostcommit\x18\x06 \x03(\x0b\x32\x0e.RpbCommitHook\x12\x1d\n\x0ehas_postcommit\x18\x07 \x01(\x08:\x05\x66\x61lse\x12 \n\x0c\x63hash_keyfun\x18\x08 \x01(\x0b\x32\n.RpbModFun\x12\x1b\n\x07linkfun\x18\t \x01(\x0b\x32\n.RpbModFun\x12\x12\n\nold_vclock\x18\n \x01(\r\x12\x14\n\x0cyoung_vclock\x18\x0b \x01(\r\x12\x12\n\nbig_vclock\x18\x0c \x01(\r\x12\x14\n\x0csmall_vclock\x18\r \x01(\r\x12\n\n\x02pr\x18\x0e \x01(\r\x12\t\n\x01r\x18\x0f \x01(\r\x12\t\n\x01w\x18\x10 \x01(\r\x12\n\n\x02pw\x18\x11 \x01(\r\x12\n\n\x02\x64w\x18\x12 \x01(\r\x12\n\n\x02rw\x18\x13 \x01(\r\x12\x14\n\x0c\x62\x61sic_quorum\x18\x14 \x01(\x08\x12\x13\n\x0bnotfound_ok\x18\x15 \x01(\x08\x12\x0f\n\x07\x62\x61\x63kend\x18\x16 \x01(\x0c\x12\x0e\n\x06search\x18\x17 \x01(\x08\x12)\n\x04repl\x18\x18 \x01(\x0e\x32\x1b.RpbBucketProps.RpbReplMode\x12\x14\n\x0csearch_index\x18\x19 \x01(\x0c\x12\x10\n\x08\x64\x61tatype\x18\x1a \x01(\x0c\x12\x12\n\nconsistent\x18\x1b \x01(\x08\x12\x12\n\nwrite_once\x18\x1c \x01(\x08\x12\x15\n\rhll_precision\x18\x1d \x01(\r\">\n\x0bRpbReplMode\x12\t\n\x05\x46\x41LSE\x10\x00\x12\x0c\n\x08REALTIME\x10\x01\x12\x0c\n\x08\x46ULLSYNC\x10\x02\x12\x08\n\x04TRUE\x10\x03\",\n\nRpbAuthReq\x12\x0c\n\x04user\x18\x01 \x02(\x0c\x12\x10\n\x08password\x18\x02 \x02(\x0c\x42!\n\x17\x63om.basho.riak.protobufB\x06RiakPB') + + + +_RPBBUCKETPROPS_RPBREPLMODE = _descriptor.EnumDescriptor( + name='RpbReplMode', + full_name='RpbBucketProps.RpbReplMode', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='FALSE', index=0, number=0, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='REALTIME', index=1, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FULLSYNC', index=2, number=2, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='TRUE', index=3, number=3, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=1259, + serialized_end=1321, +) + + +_RPBERRORRESP = _descriptor.Descriptor( + name='RpbErrorResp', + full_name='RpbErrorResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='errmsg', full_name='RpbErrorResp.errmsg', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='errcode', full_name='RpbErrorResp.errcode', index=1, + number=2, type=13, cpp_type=3, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=14, + serialized_end=61, +) + + +_RPBGETSERVERINFORESP = _descriptor.Descriptor( + name='RpbGetServerInfoResp', + full_name='RpbGetServerInfoResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='node', full_name='RpbGetServerInfoResp.node', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='server_version', full_name='RpbGetServerInfoResp.server_version', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=63, + serialized_end=123, +) + + +_RPBPAIR = _descriptor.Descriptor( + name='RpbPair', + full_name='RpbPair', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='RpbPair.key', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='RpbPair.value', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=125, + serialized_end=162, +) + + +_RPBGETBUCKETREQ = _descriptor.Descriptor( + name='RpbGetBucketReq', + full_name='RpbGetBucketReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbGetBucketReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='RpbGetBucketReq.type', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=164, + serialized_end=211, +) + + +_RPBGETBUCKETRESP = _descriptor.Descriptor( + name='RpbGetBucketResp', + full_name='RpbGetBucketResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='props', full_name='RpbGetBucketResp.props', index=0, + number=1, type=11, cpp_type=10, label=2, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=213, + serialized_end=263, +) + + +_RPBSETBUCKETREQ = _descriptor.Descriptor( + name='RpbSetBucketReq', + full_name='RpbSetBucketReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbSetBucketReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='props', full_name='RpbSetBucketReq.props', index=1, + number=2, type=11, cpp_type=10, label=2, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='RpbSetBucketReq.type', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=265, + serialized_end=344, +) + + +_RPBRESETBUCKETREQ = _descriptor.Descriptor( + name='RpbResetBucketReq', + full_name='RpbResetBucketReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='bucket', full_name='RpbResetBucketReq.bucket', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='RpbResetBucketReq.type', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=346, + serialized_end=395, +) + + +_RPBGETBUCKETTYPEREQ = _descriptor.Descriptor( + name='RpbGetBucketTypeReq', + full_name='RpbGetBucketTypeReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='type', full_name='RpbGetBucketTypeReq.type', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=397, + serialized_end=432, +) + + +_RPBSETBUCKETTYPEREQ = _descriptor.Descriptor( + name='RpbSetBucketTypeReq', + full_name='RpbSetBucketTypeReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='type', full_name='RpbSetBucketTypeReq.type', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='props', full_name='RpbSetBucketTypeReq.props', index=1, + number=2, type=11, cpp_type=10, label=2, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=434, + serialized_end=501, +) + + +_RPBMODFUN = _descriptor.Descriptor( + name='RpbModFun', + full_name='RpbModFun', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='module', full_name='RpbModFun.module', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='function', full_name='RpbModFun.function', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=503, + serialized_end=548, +) + + +_RPBCOMMITHOOK = _descriptor.Descriptor( + name='RpbCommitHook', + full_name='RpbCommitHook', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='modfun', full_name='RpbCommitHook.modfun', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='name', full_name='RpbCommitHook.name', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=550, + serialized_end=607, +) + + +_RPBBUCKETPROPS = _descriptor.Descriptor( + name='RpbBucketProps', + full_name='RpbBucketProps', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='n_val', full_name='RpbBucketProps.n_val', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='allow_mult', full_name='RpbBucketProps.allow_mult', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='last_write_wins', full_name='RpbBucketProps.last_write_wins', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='precommit', full_name='RpbBucketProps.precommit', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='has_precommit', full_name='RpbBucketProps.has_precommit', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='postcommit', full_name='RpbBucketProps.postcommit', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='has_postcommit', full_name='RpbBucketProps.has_postcommit', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='chash_keyfun', full_name='RpbBucketProps.chash_keyfun', index=7, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='linkfun', full_name='RpbBucketProps.linkfun', index=8, + number=9, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='old_vclock', full_name='RpbBucketProps.old_vclock', index=9, + number=10, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='young_vclock', full_name='RpbBucketProps.young_vclock', index=10, + number=11, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='big_vclock', full_name='RpbBucketProps.big_vclock', index=11, + number=12, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='small_vclock', full_name='RpbBucketProps.small_vclock', index=12, + number=13, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pr', full_name='RpbBucketProps.pr', index=13, + number=14, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='r', full_name='RpbBucketProps.r', index=14, + number=15, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='w', full_name='RpbBucketProps.w', index=15, + number=16, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pw', full_name='RpbBucketProps.pw', index=16, + number=17, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='dw', full_name='RpbBucketProps.dw', index=17, + number=18, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='rw', full_name='RpbBucketProps.rw', index=18, + number=19, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='basic_quorum', full_name='RpbBucketProps.basic_quorum', index=19, + number=20, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='notfound_ok', full_name='RpbBucketProps.notfound_ok', index=20, + number=21, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='backend', full_name='RpbBucketProps.backend', index=21, + number=22, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='search', full_name='RpbBucketProps.search', index=22, + number=23, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='repl', full_name='RpbBucketProps.repl', index=23, + number=24, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='search_index', full_name='RpbBucketProps.search_index', index=24, + number=25, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='datatype', full_name='RpbBucketProps.datatype', index=25, + number=26, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='consistent', full_name='RpbBucketProps.consistent', index=26, + number=27, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='write_once', full_name='RpbBucketProps.write_once', index=27, + number=28, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='hll_precision', full_name='RpbBucketProps.hll_precision', index=28, + number=29, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _RPBBUCKETPROPS_RPBREPLMODE, + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=610, + serialized_end=1321, +) + + +_RPBAUTHREQ = _descriptor.Descriptor( + name='RpbAuthReq', + full_name='RpbAuthReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='user', full_name='RpbAuthReq.user', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='password', full_name='RpbAuthReq.password', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1323, + serialized_end=1367, +) + +_RPBGETBUCKETRESP.fields_by_name['props'].message_type = _RPBBUCKETPROPS +_RPBSETBUCKETREQ.fields_by_name['props'].message_type = _RPBBUCKETPROPS +_RPBSETBUCKETTYPEREQ.fields_by_name['props'].message_type = _RPBBUCKETPROPS +_RPBCOMMITHOOK.fields_by_name['modfun'].message_type = _RPBMODFUN +_RPBBUCKETPROPS.fields_by_name['precommit'].message_type = _RPBCOMMITHOOK +_RPBBUCKETPROPS.fields_by_name['postcommit'].message_type = _RPBCOMMITHOOK +_RPBBUCKETPROPS.fields_by_name['chash_keyfun'].message_type = _RPBMODFUN +_RPBBUCKETPROPS.fields_by_name['linkfun'].message_type = _RPBMODFUN +_RPBBUCKETPROPS.fields_by_name['repl'].enum_type = _RPBBUCKETPROPS_RPBREPLMODE +_RPBBUCKETPROPS_RPBREPLMODE.containing_type = _RPBBUCKETPROPS; +DESCRIPTOR.message_types_by_name['RpbErrorResp'] = _RPBERRORRESP +DESCRIPTOR.message_types_by_name['RpbGetServerInfoResp'] = _RPBGETSERVERINFORESP +DESCRIPTOR.message_types_by_name['RpbPair'] = _RPBPAIR +DESCRIPTOR.message_types_by_name['RpbGetBucketReq'] = _RPBGETBUCKETREQ +DESCRIPTOR.message_types_by_name['RpbGetBucketResp'] = _RPBGETBUCKETRESP +DESCRIPTOR.message_types_by_name['RpbSetBucketReq'] = _RPBSETBUCKETREQ +DESCRIPTOR.message_types_by_name['RpbResetBucketReq'] = _RPBRESETBUCKETREQ +DESCRIPTOR.message_types_by_name['RpbGetBucketTypeReq'] = _RPBGETBUCKETTYPEREQ +DESCRIPTOR.message_types_by_name['RpbSetBucketTypeReq'] = _RPBSETBUCKETTYPEREQ +DESCRIPTOR.message_types_by_name['RpbModFun'] = _RPBMODFUN +DESCRIPTOR.message_types_by_name['RpbCommitHook'] = _RPBCOMMITHOOK +DESCRIPTOR.message_types_by_name['RpbBucketProps'] = _RPBBUCKETPROPS +DESCRIPTOR.message_types_by_name['RpbAuthReq'] = _RPBAUTHREQ + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbErrorResp(_message.Message): + DESCRIPTOR = _RPBERRORRESP + + # @@protoc_insertion_point(class_scope:RpbErrorResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbGetServerInfoResp(_message.Message): + DESCRIPTOR = _RPBGETSERVERINFORESP + + # @@protoc_insertion_point(class_scope:RpbGetServerInfoResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbPair(_message.Message): + DESCRIPTOR = _RPBPAIR + + # @@protoc_insertion_point(class_scope:RpbPair) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbGetBucketReq(_message.Message): + DESCRIPTOR = _RPBGETBUCKETREQ + + # @@protoc_insertion_point(class_scope:RpbGetBucketReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbGetBucketResp(_message.Message): + DESCRIPTOR = _RPBGETBUCKETRESP + + # @@protoc_insertion_point(class_scope:RpbGetBucketResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbSetBucketReq(_message.Message): + DESCRIPTOR = _RPBSETBUCKETREQ + + # @@protoc_insertion_point(class_scope:RpbSetBucketReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbResetBucketReq(_message.Message): + DESCRIPTOR = _RPBRESETBUCKETREQ + + # @@protoc_insertion_point(class_scope:RpbResetBucketReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbGetBucketTypeReq(_message.Message): + DESCRIPTOR = _RPBGETBUCKETTYPEREQ + + # @@protoc_insertion_point(class_scope:RpbGetBucketTypeReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbSetBucketTypeReq(_message.Message): + DESCRIPTOR = _RPBSETBUCKETTYPEREQ + + # @@protoc_insertion_point(class_scope:RpbSetBucketTypeReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbModFun(_message.Message): + DESCRIPTOR = _RPBMODFUN + + # @@protoc_insertion_point(class_scope:RpbModFun) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbCommitHook(_message.Message): + DESCRIPTOR = _RPBCOMMITHOOK + + # @@protoc_insertion_point(class_scope:RpbCommitHook) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbBucketProps(_message.Message): + DESCRIPTOR = _RPBBUCKETPROPS + + # @@protoc_insertion_point(class_scope:RpbBucketProps) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbAuthReq(_message.Message): + DESCRIPTOR = _RPBAUTHREQ + + # @@protoc_insertion_point(class_scope:RpbAuthReq) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), '\n\027com.basho.riak.protobufB\006RiakPB') +# @@protoc_insertion_point(module_scope) diff --git a/riak/pb/riak_search_pb2.py b/riak/pb/riak_search_pb2.py new file mode 100644 index 00000000..b20adbfc --- /dev/null +++ b/riak/pb/riak_search_pb2.py @@ -0,0 +1,224 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from six import * +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: riak_search.proto + +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + +import riak.pb.riak_pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='riak_search.proto', + package='', + serialized_pb='\n\x11riak_search.proto\x1a\nriak.proto\"(\n\x0cRpbSearchDoc\x12\x18\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x08.RpbPair\"\x9d\x01\n\x11RpbSearchQueryReq\x12\t\n\x01q\x18\x01 \x02(\x0c\x12\r\n\x05index\x18\x02 \x02(\x0c\x12\x0c\n\x04rows\x18\x03 \x01(\r\x12\r\n\x05start\x18\x04 \x01(\r\x12\x0c\n\x04sort\x18\x05 \x01(\x0c\x12\x0e\n\x06\x66ilter\x18\x06 \x01(\x0c\x12\n\n\x02\x64\x66\x18\x07 \x01(\x0c\x12\n\n\x02op\x18\x08 \x01(\x0c\x12\n\n\x02\x66l\x18\t \x03(\x0c\x12\x0f\n\x07presort\x18\n \x01(\x0c\"W\n\x12RpbSearchQueryResp\x12\x1b\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\r.RpbSearchDoc\x12\x11\n\tmax_score\x18\x02 \x01(\x02\x12\x11\n\tnum_found\x18\x03 \x01(\rB\'\n\x17\x63om.basho.riak.protobufB\x0cRiakSearchPB') + + + + +_RPBSEARCHDOC = _descriptor.Descriptor( + name='RpbSearchDoc', + full_name='RpbSearchDoc', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='fields', full_name='RpbSearchDoc.fields', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=33, + serialized_end=73, +) + + +_RPBSEARCHQUERYREQ = _descriptor.Descriptor( + name='RpbSearchQueryReq', + full_name='RpbSearchQueryReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='q', full_name='RpbSearchQueryReq.q', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='index', full_name='RpbSearchQueryReq.index', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='rows', full_name='RpbSearchQueryReq.rows', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='start', full_name='RpbSearchQueryReq.start', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='sort', full_name='RpbSearchQueryReq.sort', index=4, + number=5, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='filter', full_name='RpbSearchQueryReq.filter', index=5, + number=6, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='df', full_name='RpbSearchQueryReq.df', index=6, + number=7, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='op', full_name='RpbSearchQueryReq.op', index=7, + number=8, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='fl', full_name='RpbSearchQueryReq.fl', index=8, + number=9, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='presort', full_name='RpbSearchQueryReq.presort', index=9, + number=10, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=76, + serialized_end=233, +) + + +_RPBSEARCHQUERYRESP = _descriptor.Descriptor( + name='RpbSearchQueryResp', + full_name='RpbSearchQueryResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='docs', full_name='RpbSearchQueryResp.docs', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='max_score', full_name='RpbSearchQueryResp.max_score', index=1, + number=2, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='num_found', full_name='RpbSearchQueryResp.num_found', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=235, + serialized_end=322, +) + +_RPBSEARCHDOC.fields_by_name['fields'].message_type = riak.pb.riak_pb2._RPBPAIR +_RPBSEARCHQUERYRESP.fields_by_name['docs'].message_type = _RPBSEARCHDOC +DESCRIPTOR.message_types_by_name['RpbSearchDoc'] = _RPBSEARCHDOC +DESCRIPTOR.message_types_by_name['RpbSearchQueryReq'] = _RPBSEARCHQUERYREQ +DESCRIPTOR.message_types_by_name['RpbSearchQueryResp'] = _RPBSEARCHQUERYRESP + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbSearchDoc(_message.Message): + DESCRIPTOR = _RPBSEARCHDOC + + # @@protoc_insertion_point(class_scope:RpbSearchDoc) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbSearchQueryReq(_message.Message): + DESCRIPTOR = _RPBSEARCHQUERYREQ + + # @@protoc_insertion_point(class_scope:RpbSearchQueryReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbSearchQueryResp(_message.Message): + DESCRIPTOR = _RPBSEARCHQUERYRESP + + # @@protoc_insertion_point(class_scope:RpbSearchQueryResp) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), '\n\027com.basho.riak.protobufB\014RiakSearchPB') +# @@protoc_insertion_point(module_scope) diff --git a/riak/pb/riak_ts_pb2.py b/riak/pb/riak_ts_pb2.py new file mode 100644 index 00000000..5033db67 --- /dev/null +++ b/riak/pb/riak_ts_pb2.py @@ -0,0 +1,934 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from six import * +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: riak_ts.proto + +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + +import riak.pb.riak_pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='riak_ts.proto', + package='', + serialized_pb='\n\rriak_ts.proto\x1a\nriak.proto\"[\n\nTsQueryReq\x12\x1f\n\x05query\x18\x01 \x01(\x0b\x32\x10.TsInterpolation\x12\x15\n\x06stream\x18\x02 \x01(\x08:\x05\x66\x61lse\x12\x15\n\rcover_context\x18\x03 \x01(\x0c\"^\n\x0bTsQueryResp\x12%\n\x07\x63olumns\x18\x01 \x03(\x0b\x32\x14.TsColumnDescription\x12\x14\n\x04rows\x18\x02 \x03(\x0b\x32\x06.TsRow\x12\x12\n\x04\x64one\x18\x03 \x01(\x08:\x04true\"@\n\x08TsGetReq\x12\r\n\x05table\x18\x01 \x02(\x0c\x12\x14\n\x03key\x18\x02 \x03(\x0b\x32\x07.TsCell\x12\x0f\n\x07timeout\x18\x03 \x01(\r\"H\n\tTsGetResp\x12%\n\x07\x63olumns\x18\x01 \x03(\x0b\x32\x14.TsColumnDescription\x12\x14\n\x04rows\x18\x02 \x03(\x0b\x32\x06.TsRow\"V\n\x08TsPutReq\x12\r\n\x05table\x18\x01 \x02(\x0c\x12%\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x14.TsColumnDescription\x12\x14\n\x04rows\x18\x03 \x03(\x0b\x32\x06.TsRow\"\x0b\n\tTsPutResp\"P\n\x08TsDelReq\x12\r\n\x05table\x18\x01 \x02(\x0c\x12\x14\n\x03key\x18\x02 \x03(\x0b\x32\x07.TsCell\x12\x0e\n\x06vclock\x18\x03 \x01(\x0c\x12\x0f\n\x07timeout\x18\x04 \x01(\r\"\x0b\n\tTsDelResp\"A\n\x0fTsInterpolation\x12\x0c\n\x04\x62\x61se\x18\x01 \x02(\x0c\x12 \n\x0einterpolations\x18\x02 \x03(\x0b\x32\x08.RpbPair\"@\n\x13TsColumnDescription\x12\x0c\n\x04name\x18\x01 \x02(\x0c\x12\x1b\n\x04type\x18\x02 \x02(\x0e\x32\r.TsColumnType\"\x1f\n\x05TsRow\x12\x16\n\x05\x63\x65lls\x18\x01 \x03(\x0b\x32\x07.TsCell\"{\n\x06TsCell\x12\x15\n\rvarchar_value\x18\x01 \x01(\x0c\x12\x14\n\x0csint64_value\x18\x02 \x01(\x12\x12\x17\n\x0ftimestamp_value\x18\x03 \x01(\x12\x12\x15\n\rboolean_value\x18\x04 \x01(\x08\x12\x14\n\x0c\x64ouble_value\x18\x05 \x01(\x01\"/\n\rTsListKeysReq\x12\r\n\x05table\x18\x01 \x02(\x0c\x12\x0f\n\x07timeout\x18\x02 \x01(\r\"4\n\x0eTsListKeysResp\x12\x14\n\x04keys\x18\x01 \x03(\x0b\x32\x06.TsRow\x12\x0c\n\x04\x64one\x18\x02 \x01(\x08\"q\n\rTsCoverageReq\x12\x1f\n\x05query\x18\x01 \x01(\x0b\x32\x10.TsInterpolation\x12\r\n\x05table\x18\x02 \x02(\x0c\x12\x15\n\rreplace_cover\x18\x03 \x01(\x0c\x12\x19\n\x11unavailable_cover\x18\x04 \x03(\x0c\"3\n\x0eTsCoverageResp\x12!\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x10.TsCoverageEntry\"[\n\x0fTsCoverageEntry\x12\n\n\x02ip\x18\x01 \x02(\x0c\x12\x0c\n\x04port\x18\x02 \x02(\r\x12\x15\n\rcover_context\x18\x03 \x02(\x0c\x12\x17\n\x05range\x18\x04 \x01(\x0b\x32\x08.TsRange\"\x93\x01\n\x07TsRange\x12\x12\n\nfield_name\x18\x01 \x02(\x0c\x12\x13\n\x0blower_bound\x18\x02 \x02(\x12\x12\x1d\n\x15lower_bound_inclusive\x18\x03 \x02(\x08\x12\x13\n\x0bupper_bound\x18\x04 \x02(\x12\x12\x1d\n\x15upper_bound_inclusive\x18\x05 \x02(\x08\x12\x0c\n\x04\x64\x65sc\x18\x06 \x02(\x0c*Y\n\x0cTsColumnType\x12\x0b\n\x07VARCHAR\x10\x00\x12\n\n\x06SINT64\x10\x01\x12\n\n\x06\x44OUBLE\x10\x02\x12\r\n\tTIMESTAMP\x10\x03\x12\x0b\n\x07\x42OOLEAN\x10\x04\x12\x08\n\x04\x42LOB\x10\x05\x42#\n\x17\x63om.basho.riak.protobufB\x08RiakTsPB') + +_TSCOLUMNTYPE = _descriptor.EnumDescriptor( + name='TsColumnType', + full_name='TsColumnType', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='VARCHAR', index=0, number=0, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='SINT64', index=1, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DOUBLE', index=2, number=2, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='TIMESTAMP', index=3, number=3, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='BOOLEAN', index=4, number=4, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='BLOB', index=5, number=5, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=1359, + serialized_end=1448, +) + +TsColumnType = enum_type_wrapper.EnumTypeWrapper(_TSCOLUMNTYPE) +VARCHAR = 0 +SINT64 = 1 +DOUBLE = 2 +TIMESTAMP = 3 +BOOLEAN = 4 +BLOB = 5 + + + +_TSQUERYREQ = _descriptor.Descriptor( + name='TsQueryReq', + full_name='TsQueryReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='query', full_name='TsQueryReq.query', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='stream', full_name='TsQueryReq.stream', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='cover_context', full_name='TsQueryReq.cover_context', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=29, + serialized_end=120, +) + + +_TSQUERYRESP = _descriptor.Descriptor( + name='TsQueryResp', + full_name='TsQueryResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='columns', full_name='TsQueryResp.columns', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='rows', full_name='TsQueryResp.rows', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='done', full_name='TsQueryResp.done', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=True, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=122, + serialized_end=216, +) + + +_TSGETREQ = _descriptor.Descriptor( + name='TsGetReq', + full_name='TsGetReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='table', full_name='TsGetReq.table', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='TsGetReq.key', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timeout', full_name='TsGetReq.timeout', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=218, + serialized_end=282, +) + + +_TSGETRESP = _descriptor.Descriptor( + name='TsGetResp', + full_name='TsGetResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='columns', full_name='TsGetResp.columns', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='rows', full_name='TsGetResp.rows', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=284, + serialized_end=356, +) + + +_TSPUTREQ = _descriptor.Descriptor( + name='TsPutReq', + full_name='TsPutReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='table', full_name='TsPutReq.table', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='columns', full_name='TsPutReq.columns', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='rows', full_name='TsPutReq.rows', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=358, + serialized_end=444, +) + + +_TSPUTRESP = _descriptor.Descriptor( + name='TsPutResp', + full_name='TsPutResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=446, + serialized_end=457, +) + + +_TSDELREQ = _descriptor.Descriptor( + name='TsDelReq', + full_name='TsDelReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='table', full_name='TsDelReq.table', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='key', full_name='TsDelReq.key', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='vclock', full_name='TsDelReq.vclock', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timeout', full_name='TsDelReq.timeout', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=459, + serialized_end=539, +) + + +_TSDELRESP = _descriptor.Descriptor( + name='TsDelResp', + full_name='TsDelResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=541, + serialized_end=552, +) + + +_TSINTERPOLATION = _descriptor.Descriptor( + name='TsInterpolation', + full_name='TsInterpolation', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='base', full_name='TsInterpolation.base', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='interpolations', full_name='TsInterpolation.interpolations', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=554, + serialized_end=619, +) + + +_TSCOLUMNDESCRIPTION = _descriptor.Descriptor( + name='TsColumnDescription', + full_name='TsColumnDescription', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='TsColumnDescription.name', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='type', full_name='TsColumnDescription.type', index=1, + number=2, type=14, cpp_type=8, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=621, + serialized_end=685, +) + + +_TSROW = _descriptor.Descriptor( + name='TsRow', + full_name='TsRow', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='cells', full_name='TsRow.cells', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=687, + serialized_end=718, +) + + +_TSCELL = _descriptor.Descriptor( + name='TsCell', + full_name='TsCell', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='varchar_value', full_name='TsCell.varchar_value', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='sint64_value', full_name='TsCell.sint64_value', index=1, + number=2, type=18, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timestamp_value', full_name='TsCell.timestamp_value', index=2, + number=3, type=18, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='boolean_value', full_name='TsCell.boolean_value', index=3, + number=4, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='double_value', full_name='TsCell.double_value', index=4, + number=5, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=720, + serialized_end=843, +) + + +_TSLISTKEYSREQ = _descriptor.Descriptor( + name='TsListKeysReq', + full_name='TsListKeysReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='table', full_name='TsListKeysReq.table', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timeout', full_name='TsListKeysReq.timeout', index=1, + number=2, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=845, + serialized_end=892, +) + + +_TSLISTKEYSRESP = _descriptor.Descriptor( + name='TsListKeysResp', + full_name='TsListKeysResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='keys', full_name='TsListKeysResp.keys', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='done', full_name='TsListKeysResp.done', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=894, + serialized_end=946, +) + + +_TSCOVERAGEREQ = _descriptor.Descriptor( + name='TsCoverageReq', + full_name='TsCoverageReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='query', full_name='TsCoverageReq.query', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='table', full_name='TsCoverageReq.table', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='replace_cover', full_name='TsCoverageReq.replace_cover', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='unavailable_cover', full_name='TsCoverageReq.unavailable_cover', index=3, + number=4, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=948, + serialized_end=1061, +) + + +_TSCOVERAGERESP = _descriptor.Descriptor( + name='TsCoverageResp', + full_name='TsCoverageResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='entries', full_name='TsCoverageResp.entries', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1063, + serialized_end=1114, +) + + +_TSCOVERAGEENTRY = _descriptor.Descriptor( + name='TsCoverageEntry', + full_name='TsCoverageEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='ip', full_name='TsCoverageEntry.ip', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='port', full_name='TsCoverageEntry.port', index=1, + number=2, type=13, cpp_type=3, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='cover_context', full_name='TsCoverageEntry.cover_context', index=2, + number=3, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='range', full_name='TsCoverageEntry.range', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1116, + serialized_end=1207, +) + + +_TSRANGE = _descriptor.Descriptor( + name='TsRange', + full_name='TsRange', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='field_name', full_name='TsRange.field_name', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='lower_bound', full_name='TsRange.lower_bound', index=1, + number=2, type=18, cpp_type=2, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='lower_bound_inclusive', full_name='TsRange.lower_bound_inclusive', index=2, + number=3, type=8, cpp_type=7, label=2, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='upper_bound', full_name='TsRange.upper_bound', index=3, + number=4, type=18, cpp_type=2, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='upper_bound_inclusive', full_name='TsRange.upper_bound_inclusive', index=4, + number=5, type=8, cpp_type=7, label=2, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='desc', full_name='TsRange.desc', index=5, + number=6, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=1210, + serialized_end=1357, +) + +_TSQUERYREQ.fields_by_name['query'].message_type = _TSINTERPOLATION +_TSQUERYRESP.fields_by_name['columns'].message_type = _TSCOLUMNDESCRIPTION +_TSQUERYRESP.fields_by_name['rows'].message_type = _TSROW +_TSGETREQ.fields_by_name['key'].message_type = _TSCELL +_TSGETRESP.fields_by_name['columns'].message_type = _TSCOLUMNDESCRIPTION +_TSGETRESP.fields_by_name['rows'].message_type = _TSROW +_TSPUTREQ.fields_by_name['columns'].message_type = _TSCOLUMNDESCRIPTION +_TSPUTREQ.fields_by_name['rows'].message_type = _TSROW +_TSDELREQ.fields_by_name['key'].message_type = _TSCELL +_TSINTERPOLATION.fields_by_name['interpolations'].message_type = riak.pb.riak_pb2._RPBPAIR +_TSCOLUMNDESCRIPTION.fields_by_name['type'].enum_type = _TSCOLUMNTYPE +_TSROW.fields_by_name['cells'].message_type = _TSCELL +_TSLISTKEYSRESP.fields_by_name['keys'].message_type = _TSROW +_TSCOVERAGEREQ.fields_by_name['query'].message_type = _TSINTERPOLATION +_TSCOVERAGERESP.fields_by_name['entries'].message_type = _TSCOVERAGEENTRY +_TSCOVERAGEENTRY.fields_by_name['range'].message_type = _TSRANGE +DESCRIPTOR.message_types_by_name['TsQueryReq'] = _TSQUERYREQ +DESCRIPTOR.message_types_by_name['TsQueryResp'] = _TSQUERYRESP +DESCRIPTOR.message_types_by_name['TsGetReq'] = _TSGETREQ +DESCRIPTOR.message_types_by_name['TsGetResp'] = _TSGETRESP +DESCRIPTOR.message_types_by_name['TsPutReq'] = _TSPUTREQ +DESCRIPTOR.message_types_by_name['TsPutResp'] = _TSPUTRESP +DESCRIPTOR.message_types_by_name['TsDelReq'] = _TSDELREQ +DESCRIPTOR.message_types_by_name['TsDelResp'] = _TSDELRESP +DESCRIPTOR.message_types_by_name['TsInterpolation'] = _TSINTERPOLATION +DESCRIPTOR.message_types_by_name['TsColumnDescription'] = _TSCOLUMNDESCRIPTION +DESCRIPTOR.message_types_by_name['TsRow'] = _TSROW +DESCRIPTOR.message_types_by_name['TsCell'] = _TSCELL +DESCRIPTOR.message_types_by_name['TsListKeysReq'] = _TSLISTKEYSREQ +DESCRIPTOR.message_types_by_name['TsListKeysResp'] = _TSLISTKEYSRESP +DESCRIPTOR.message_types_by_name['TsCoverageReq'] = _TSCOVERAGEREQ +DESCRIPTOR.message_types_by_name['TsCoverageResp'] = _TSCOVERAGERESP +DESCRIPTOR.message_types_by_name['TsCoverageEntry'] = _TSCOVERAGEENTRY +DESCRIPTOR.message_types_by_name['TsRange'] = _TSRANGE + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsQueryReq(_message.Message): + DESCRIPTOR = _TSQUERYREQ + + # @@protoc_insertion_point(class_scope:TsQueryReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsQueryResp(_message.Message): + DESCRIPTOR = _TSQUERYRESP + + # @@protoc_insertion_point(class_scope:TsQueryResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsGetReq(_message.Message): + DESCRIPTOR = _TSGETREQ + + # @@protoc_insertion_point(class_scope:TsGetReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsGetResp(_message.Message): + DESCRIPTOR = _TSGETRESP + + # @@protoc_insertion_point(class_scope:TsGetResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsPutReq(_message.Message): + DESCRIPTOR = _TSPUTREQ + + # @@protoc_insertion_point(class_scope:TsPutReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsPutResp(_message.Message): + DESCRIPTOR = _TSPUTRESP + + # @@protoc_insertion_point(class_scope:TsPutResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsDelReq(_message.Message): + DESCRIPTOR = _TSDELREQ + + # @@protoc_insertion_point(class_scope:TsDelReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsDelResp(_message.Message): + DESCRIPTOR = _TSDELRESP + + # @@protoc_insertion_point(class_scope:TsDelResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsInterpolation(_message.Message): + DESCRIPTOR = _TSINTERPOLATION + + # @@protoc_insertion_point(class_scope:TsInterpolation) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsColumnDescription(_message.Message): + DESCRIPTOR = _TSCOLUMNDESCRIPTION + + # @@protoc_insertion_point(class_scope:TsColumnDescription) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsRow(_message.Message): + DESCRIPTOR = _TSROW + + # @@protoc_insertion_point(class_scope:TsRow) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsCell(_message.Message): + DESCRIPTOR = _TSCELL + + # @@protoc_insertion_point(class_scope:TsCell) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsListKeysReq(_message.Message): + DESCRIPTOR = _TSLISTKEYSREQ + + # @@protoc_insertion_point(class_scope:TsListKeysReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsListKeysResp(_message.Message): + DESCRIPTOR = _TSLISTKEYSRESP + + # @@protoc_insertion_point(class_scope:TsListKeysResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsCoverageReq(_message.Message): + DESCRIPTOR = _TSCOVERAGEREQ + + # @@protoc_insertion_point(class_scope:TsCoverageReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsCoverageResp(_message.Message): + DESCRIPTOR = _TSCOVERAGERESP + + # @@protoc_insertion_point(class_scope:TsCoverageResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsCoverageEntry(_message.Message): + DESCRIPTOR = _TSCOVERAGEENTRY + + # @@protoc_insertion_point(class_scope:TsCoverageEntry) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class TsRange(_message.Message): + DESCRIPTOR = _TSRANGE + + # @@protoc_insertion_point(class_scope:TsRange) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), '\n\027com.basho.riak.protobufB\010RiakTsPB') +# @@protoc_insertion_point(module_scope) diff --git a/riak/pb/riak_yokozuna_pb2.py b/riak/pb/riak_yokozuna_pb2.py new file mode 100644 index 00000000..6cc20395 --- /dev/null +++ b/riak/pb/riak_yokozuna_pb2.py @@ -0,0 +1,386 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from six import * +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: riak_yokozuna.proto + +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='riak_yokozuna.proto', + package='', + serialized_pb='\n\x13riak_yokozuna.proto\"?\n\x10RpbYokozunaIndex\x12\x0c\n\x04name\x18\x01 \x02(\x0c\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x12\r\n\x05n_val\x18\x03 \x01(\r\"&\n\x16RpbYokozunaIndexGetReq\x12\x0c\n\x04name\x18\x01 \x01(\x0c\";\n\x17RpbYokozunaIndexGetResp\x12 \n\x05index\x18\x01 \x03(\x0b\x32\x11.RpbYokozunaIndex\"K\n\x16RpbYokozunaIndexPutReq\x12 \n\x05index\x18\x01 \x02(\x0b\x32\x11.RpbYokozunaIndex\x12\x0f\n\x07timeout\x18\x02 \x01(\r\")\n\x19RpbYokozunaIndexDeleteReq\x12\x0c\n\x04name\x18\x01 \x02(\x0c\"2\n\x11RpbYokozunaSchema\x12\x0c\n\x04name\x18\x01 \x02(\x0c\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\"=\n\x17RpbYokozunaSchemaPutReq\x12\"\n\x06schema\x18\x01 \x02(\x0b\x32\x12.RpbYokozunaSchema\"\'\n\x17RpbYokozunaSchemaGetReq\x12\x0c\n\x04name\x18\x01 \x02(\x0c\">\n\x18RpbYokozunaSchemaGetResp\x12\"\n\x06schema\x18\x01 \x02(\x0b\x32\x12.RpbYokozunaSchemaB)\n\x17\x63om.basho.riak.protobufB\x0eRiakYokozunaPB') + + + + +_RPBYOKOZUNAINDEX = _descriptor.Descriptor( + name='RpbYokozunaIndex', + full_name='RpbYokozunaIndex', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='RpbYokozunaIndex.name', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='schema', full_name='RpbYokozunaIndex.schema', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='n_val', full_name='RpbYokozunaIndex.n_val', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=23, + serialized_end=86, +) + + +_RPBYOKOZUNAINDEXGETREQ = _descriptor.Descriptor( + name='RpbYokozunaIndexGetReq', + full_name='RpbYokozunaIndexGetReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='RpbYokozunaIndexGetReq.name', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=88, + serialized_end=126, +) + + +_RPBYOKOZUNAINDEXGETRESP = _descriptor.Descriptor( + name='RpbYokozunaIndexGetResp', + full_name='RpbYokozunaIndexGetResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='index', full_name='RpbYokozunaIndexGetResp.index', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=128, + serialized_end=187, +) + + +_RPBYOKOZUNAINDEXPUTREQ = _descriptor.Descriptor( + name='RpbYokozunaIndexPutReq', + full_name='RpbYokozunaIndexPutReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='index', full_name='RpbYokozunaIndexPutReq.index', index=0, + number=1, type=11, cpp_type=10, label=2, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='timeout', full_name='RpbYokozunaIndexPutReq.timeout', index=1, + number=2, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=189, + serialized_end=264, +) + + +_RPBYOKOZUNAINDEXDELETEREQ = _descriptor.Descriptor( + name='RpbYokozunaIndexDeleteReq', + full_name='RpbYokozunaIndexDeleteReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='RpbYokozunaIndexDeleteReq.name', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=266, + serialized_end=307, +) + + +_RPBYOKOZUNASCHEMA = _descriptor.Descriptor( + name='RpbYokozunaSchema', + full_name='RpbYokozunaSchema', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='RpbYokozunaSchema.name', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='content', full_name='RpbYokozunaSchema.content', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=309, + serialized_end=359, +) + + +_RPBYOKOZUNASCHEMAPUTREQ = _descriptor.Descriptor( + name='RpbYokozunaSchemaPutReq', + full_name='RpbYokozunaSchemaPutReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='schema', full_name='RpbYokozunaSchemaPutReq.schema', index=0, + number=1, type=11, cpp_type=10, label=2, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=361, + serialized_end=422, +) + + +_RPBYOKOZUNASCHEMAGETREQ = _descriptor.Descriptor( + name='RpbYokozunaSchemaGetReq', + full_name='RpbYokozunaSchemaGetReq', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='RpbYokozunaSchemaGetReq.name', index=0, + number=1, type=12, cpp_type=9, label=2, + has_default_value=False, default_value="", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=424, + serialized_end=463, +) + + +_RPBYOKOZUNASCHEMAGETRESP = _descriptor.Descriptor( + name='RpbYokozunaSchemaGetResp', + full_name='RpbYokozunaSchemaGetResp', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='schema', full_name='RpbYokozunaSchemaGetResp.schema', index=0, + number=1, type=11, cpp_type=10, label=2, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=465, + serialized_end=527, +) + +_RPBYOKOZUNAINDEXGETRESP.fields_by_name['index'].message_type = _RPBYOKOZUNAINDEX +_RPBYOKOZUNAINDEXPUTREQ.fields_by_name['index'].message_type = _RPBYOKOZUNAINDEX +_RPBYOKOZUNASCHEMAPUTREQ.fields_by_name['schema'].message_type = _RPBYOKOZUNASCHEMA +_RPBYOKOZUNASCHEMAGETRESP.fields_by_name['schema'].message_type = _RPBYOKOZUNASCHEMA +DESCRIPTOR.message_types_by_name['RpbYokozunaIndex'] = _RPBYOKOZUNAINDEX +DESCRIPTOR.message_types_by_name['RpbYokozunaIndexGetReq'] = _RPBYOKOZUNAINDEXGETREQ +DESCRIPTOR.message_types_by_name['RpbYokozunaIndexGetResp'] = _RPBYOKOZUNAINDEXGETRESP +DESCRIPTOR.message_types_by_name['RpbYokozunaIndexPutReq'] = _RPBYOKOZUNAINDEXPUTREQ +DESCRIPTOR.message_types_by_name['RpbYokozunaIndexDeleteReq'] = _RPBYOKOZUNAINDEXDELETEREQ +DESCRIPTOR.message_types_by_name['RpbYokozunaSchema'] = _RPBYOKOZUNASCHEMA +DESCRIPTOR.message_types_by_name['RpbYokozunaSchemaPutReq'] = _RPBYOKOZUNASCHEMAPUTREQ +DESCRIPTOR.message_types_by_name['RpbYokozunaSchemaGetReq'] = _RPBYOKOZUNASCHEMAGETREQ +DESCRIPTOR.message_types_by_name['RpbYokozunaSchemaGetResp'] = _RPBYOKOZUNASCHEMAGETRESP + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbYokozunaIndex(_message.Message): + DESCRIPTOR = _RPBYOKOZUNAINDEX + + # @@protoc_insertion_point(class_scope:RpbYokozunaIndex) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbYokozunaIndexGetReq(_message.Message): + DESCRIPTOR = _RPBYOKOZUNAINDEXGETREQ + + # @@protoc_insertion_point(class_scope:RpbYokozunaIndexGetReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbYokozunaIndexGetResp(_message.Message): + DESCRIPTOR = _RPBYOKOZUNAINDEXGETRESP + + # @@protoc_insertion_point(class_scope:RpbYokozunaIndexGetResp) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbYokozunaIndexPutReq(_message.Message): + DESCRIPTOR = _RPBYOKOZUNAINDEXPUTREQ + + # @@protoc_insertion_point(class_scope:RpbYokozunaIndexPutReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbYokozunaIndexDeleteReq(_message.Message): + DESCRIPTOR = _RPBYOKOZUNAINDEXDELETEREQ + + # @@protoc_insertion_point(class_scope:RpbYokozunaIndexDeleteReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbYokozunaSchema(_message.Message): + DESCRIPTOR = _RPBYOKOZUNASCHEMA + + # @@protoc_insertion_point(class_scope:RpbYokozunaSchema) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbYokozunaSchemaPutReq(_message.Message): + DESCRIPTOR = _RPBYOKOZUNASCHEMAPUTREQ + + # @@protoc_insertion_point(class_scope:RpbYokozunaSchemaPutReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbYokozunaSchemaGetReq(_message.Message): + DESCRIPTOR = _RPBYOKOZUNASCHEMAGETREQ + + # @@protoc_insertion_point(class_scope:RpbYokozunaSchemaGetReq) + +@add_metaclass(_reflection.GeneratedProtocolMessageType) +class RpbYokozunaSchemaGetResp(_message.Message): + DESCRIPTOR = _RPBYOKOZUNASCHEMAGETRESP + + # @@protoc_insertion_point(class_scope:RpbYokozunaSchemaGetResp) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), '\n\027com.basho.riak.protobufB\016RiakYokozunaPB') +# @@protoc_insertion_point(module_scope) diff --git a/riak/resolver.py b/riak/resolver.py new file mode 100644 index 00000000..6e245ff3 --- /dev/null +++ b/riak/resolver.py @@ -0,0 +1,40 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def default_resolver(riak_object): + """ + The default conflict-resolution function, which does nothing. To + implement a resolver, define a function that sets the + :attr:`siblings ` property + on the passed :class:`RiakObject ` + instance to a list containing a single :class:`RiakContent + ` object. + + :param riak_object: an object-in-conflict that will be resolved + :type riak_object: :class:`RiakObject ` + """ + pass + + +def last_written_resolver(riak_object): + """ + A conflict-resolution function that resolves by selecting the most + recently-modified sibling by timestamp. + + :param riak_object: an object-in-conflict that will be resolved + :type riak_object: :class:`RiakObject ` + """ + riak_object.siblings = [max(riak_object.siblings, + key=lambda x: x.last_modified), ] diff --git a/riak/riak_error.py b/riak/riak_error.py new file mode 100644 index 00000000..4fe0ce05 --- /dev/null +++ b/riak/riak_error.py @@ -0,0 +1,49 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class RiakError(Exception): + """ + Base class for exceptions generated in the Riak API. + """ + def __init__(self, *args, **kwargs): + super(RiakError, self).__init__(*args, **kwargs) + if len(args) > 0: + self.value = args[0] + else: + self.value = 'unknown' + + def __str__(self): + return repr(self.value) + + +class ConflictError(RiakError): + """ + Raised when an operation is attempted on a + :class:`~riak.riak_object.RiakObject` that has more than one + sibling. + """ + def __init__(self, message='Object in conflict'): + super(ConflictError, self).__init__(message) + + +class ListError(RiakError): + """ + Raised when a list operation is attempted and + riak.disable_list_exceptions is false. + """ + def __init__(self, message='Bucket and key list operations ' + 'are expensive and should not be ' + 'used in production.'): + super(ListError, self).__init__(message) diff --git a/riak/riak_object.py b/riak/riak_object.py index dc057c36..ab9650ca 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -1,24 +1,102 @@ -""" -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from riak import ConflictError +from riak.content import RiakContent +import base64 +from six import string_types, PY2 +from riak.mapreduce import RiakMapReduce + + +def content_property(name, doc=None): + """ + Delegates a property to the first sibling in a RiakObject, raising + an error when the object is in conflict. + """ + def _setter(self, value): + if len(self.siblings) == 0: + # In this case, assume that what the user wants is to + # create a new sibling inside an empty object. + self.siblings = [RiakContent(self)] + if len(self.siblings) != 1: + raise ConflictError() + setattr(self.siblings[0], name, value) + + def _getter(self): + if len(self.siblings) == 0: + return + if len(self.siblings) != 1: + raise ConflictError() + return getattr(self.siblings[0], name) + + return property(_getter, _setter, doc=doc) + + +def content_method(name): + """ + Delegates a method to the first sibling in a RiakObject, raising + an error when the object is in conflict. + """ + def _delegate(self, *args, **kwargs): + if len(self.siblings) != 1: + raise ConflictError() + return getattr(self.siblings[0], name).__call__(*args, **kwargs) + + _delegate.__doc__ = getattr(RiakContent, name).__doc__ + + return _delegate -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +class VClock(object): + """ + A representation of a vector clock received from Riak. + """ -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" -from riak import RiakError -from riak.util import deprecated + if PY2: + _decoders = { + 'base64': base64.b64decode, + 'binary': str + } + + _encoders = { + 'base64': base64.b64encode, + 'binary': str + } + else: + _decoders = { + 'base64': base64.b64decode, + 'binary': bytes + } + + _encoders = { + 'base64': base64.b64encode, + 'binary': bytes + } + + def __init__(self, value, encoding): + self._vclock = self._decoders[encoding].__call__(value) + + def encode(self, encoding): + if encoding in self._encoders: + return self._encoders[encoding].__call__(self._vclock) + else: + raise ValueError('{} is not a valid vector clock encoding'. + format(encoding)) + + def __repr__(self): + return '<{} {}>'.format(self.__class__.__name__, + self.encode('base64')) class RiakObject(object): @@ -38,30 +116,26 @@ def __init__(self, client, bucket, key=None): is generated by the server when :func:`store` is called. :type key: string """ - try: - if isinstance(key, basestring): - key = key.encode('ascii') - except UnicodeError: - raise TypeError('Unicode keys are not supported.') + if PY2: + try: + if isinstance(key, string_types): + key = key.encode('ascii') + except UnicodeError: + raise TypeError('Unicode keys are not supported.') if key is not None and len(key) == 0: raise ValueError('Key name must either be "None"' ' or a non-empty string.') + self._resolver = None self.client = client self.bucket = bucket self.key = key - self._data = None - self._encoded_data = None self.vclock = None - self.charset = None - self.content_type = 'application/json' - self.content_encoding = None - self.usermeta = {} - self.indexes = set() - self.links = [] - self.siblings = [] - self.exists = False + self.siblings = [RiakContent(self)] + + #: The list of sibling values contained in this object + siblings = [] def __hash__(self): return hash((self.key, self.bucket, self.vclock)) @@ -78,153 +152,100 @@ def __ne__(self, other): else: return True - def _get_data(self): - if self._encoded_data is not None and self._data is None: - self._data = self._deserialize(self._encoded_data) - self._encoded_data = None - return self._data - - def _set_data(self, value): - self._encoded_data = None - self._data = value - - data = property(_get_data, _set_data, doc=""" + data = content_property('data', doc=""" The data stored in this object, as Python objects. For the raw data, use the `encoded_data` property. If unset, accessing this property will result in decoding the `encoded_data` property into Python values. The decoding is dependent on the `content_type` property and the bucket's registered decoders. - :type mixed """) - - def get_encoded_data(self): - deprecated("`get_encoded_data` is deprecated, use the `encoded_data`" - " property") - return self.encoded_data + """) - def set_encoded_data(self, value): - deprecated("`set_encoded_data` is deprecated, use the `encoded_data`" - " property") - self.encoded_data = value - - def _get_encoded_data(self): - if self._data is not None and self._encoded_data is None: - self._encoded_data = self._serialize(self._data) - self._data = None - return self._encoded_data - - def _set_encoded_data(self, value): - self._data = None - self._encoded_data = value - - encoded_data = property(_get_encoded_data, _set_encoded_data, doc=""" + encoded_data = content_property('encoded_data', doc=""" The raw data stored in this object, essentially the encoded form of the `data` property. If unset, accessing this property will result in encoding the `data` property into a string. The encoding is dependent on the `content_type` property and the bucket's registered encoders. - :type basestring""") - - def _serialize(self, value): - encoder = self.bucket.get_encoder(self.content_type) - if encoder: - return encoder(value) - elif isinstance(value, basestring): - return value.encode() - else: - raise TypeError('No encoder for non-string data ' - 'with content type "{0}"'. - format(self.content_type)) - - def _deserialize(self, value): - decoder = self.bucket.get_decoder(self.content_type) - if decoder: - return decoder(value) - else: - raise TypeError('No decoder for content type "{0}"'. - format(self.content_type)) + """) - def add_index(self, field, value): - """ - Tag this object with the specified field/value pair for - indexing. - - :param field: The index field. - :type field: string - :param value: The index value. - :type value: string or integer - :rtype: RiakObject - """ - if field[-4:] not in ("_bin", "_int"): - raise RiakError("Riak 2i fields must end with either '_bin'" - " or '_int'.") + charset = content_property('charset', doc=""" + The character set of the encoded data as a string + """) - self.indexes.add((field, value)) + content_type = content_property('content_type', doc=""" + The MIME media type of the encoded data as a string + """) - return self + content_encoding = content_property('content_encoding', doc=""" + The encoding (compression) of the encoded data. Valid values + are identity, deflate, gzip + """) - def remove_index(self, field=None, value=None): - """ - Remove the specified field/value pair as an index on this - object. + last_modified = content_property('last_modified', """ + The UNIX timestamp of the modification time of this value. + """) - :param field: The index field. - :type field: string - :param value: The index value. - :type value: string or integer - :rtype: RiakObject - """ - if not field and not value: - self.indexes.clear() - elif field and not value: - for index in [x for x in self.indexes if x[0] == field]: - self.indexes.remove(index) - elif field and value: - self.indexes.remove((field, value)) - else: - raise RiakError("Cannot pass value without a field" - " name while removing index") + etag = content_property('etag', """ + A unique entity-tag for the value. + """) - return self + usermeta = content_property('usermeta', doc=""" + Arbitrary user-defined metadata dict, mapping strings to strings. + """) - def set_index(self, field, value): - """ - Works like add_index, but ensures that there is only one index on given field. - If other found, then removes it first. + links = content_property('links', doc=""" + A set of bucket/key/tag 3-tuples representing links to other + keys. + """) - :param field: The index field. - :type field: string - :param value: The index value. - :type value: string or integer - :rtype: RiakObject - """ - to_rem = set((x for x in self.indexes if x[0] == field)) - self.indexes.difference_update(to_rem) - return self.add_index(field, value) + indexes = content_property('indexes', doc=""" + The set of secondary index entries, consisting of + index-name/value tuples + """) + add_index = content_method('add_index') + remove_index = content_method('remove_index') remove_indexes = remove_index + set_index = content_method('set_index') + add_link = content_method('add_link') - def add_link(self, obj, tag=None): - """ - Add a link to a RiakObject. - - :param obj: Either a RiakObject or 3 item link tuple consisting - of (bucket, key, tag). - :type obj: mixed - :param tag: Optional link tag. Defaults to bucket name. It is ignored - if ``obj`` is a 3 item link tuple. - :type tag: string - :rtype: RiakObject - """ - if isinstance(obj, tuple): - newlink = obj + def _exists(self): + if len(self.siblings) == 0: + return False + elif len(self.siblings) > 1: + # Even if all of the siblings are tombstones, the object + # essentially exists. + return True else: - newlink = (obj.bucket.name, obj.key, tag) + return self.siblings[0].exists + + exists = property(_exists, None, doc=""" + Whether the object exists. This is only ``False`` when there + are no siblings (the object was not found), or the solitary + sibling is a tombstone. + """) + + def _get_resolver(self): + if callable(self._resolver): + return self._resolver + elif self._resolver is None: + return self.bucket.resolver + else: + raise TypeError("resolver is not a function") - self.links.append(newlink) - return self + def _set_resolver(self, value): + if value is None or callable(value): + self._resolver = value + else: + raise TypeError("resolver is not a function") + + resolver = property(_get_resolver, _set_resolver, + doc="""The sibling-resolution function for this + object. If the resolver is not set, the + bucket's resolver will be used.""") def store(self, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): + if_none_match=False, timeout=None): """ Store the object in Riak. When this operation completes, the object could contain new metadata and possibly new data if Riak @@ -247,55 +268,59 @@ def store(self, w=None, dw=None, pw=None, return_body=True, :param if_none_match: Should the object be stored only if there is no key previously defined :type if_none_match: bool - :rtype: RiakObject """ - if (self.siblings and not self._data - and not self._encoded_data and not self.vclock): - raise RiakError("Attempting to store an invalid object," - "store one of the siblings instead") - - if self.key is None: - result = self.client.put_new( - self, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) - self._populate(result) - else: - result = self.client.put(self, w=w, dw=dw, pw=pw, - return_body=return_body, - if_none_match=if_none_match) - if result is not None and result != ('', []): - self._populate(result) + :param timeout: a timeout value in milliseconds + :type timeout: int + :rtype: :class:`RiakObject` """ + if len(self.siblings) != 1: + raise ConflictError("Attempting to store an invalid object, " + "resolve the siblings first") + + self.client.put(self, w=w, dw=dw, pw=pw, + return_body=return_body, + if_none_match=if_none_match, + timeout=timeout) return self - def reload(self, r=None, pr=None, vtag=None): + def reload(self, r=None, pr=None, timeout=None, basic_quorum=None, + notfound_ok=None, head_only=False): """ Reload the object from Riak. When this operation completes, the object could contain new metadata and a new value, if the object was updated in Riak since it was last retrieved. + .. note:: Even if the key is not found in Riak, this will + return a :class:`RiakObject`. Check the :attr:`exists` + property to see if the key was found. + :param r: R-Value, wait for this many partitions to respond before returning to client. :type r: integer - :rtype: RiakObject + :param pr: PR-value, require this many primary partitions to + be available before performing the read that + precedes the put + :type pr: integer + :param timeout: a timeout value in milliseconds + :type timeout: int + :param basic_quorum: whether to use the "basic quorum" policy + for not-founds + :type basic_quorum: bool + :param notfound_ok: whether to treat not-found responses as successful + :type notfound_ok: bool + :param head_only: whether to fetch without value, so only metadata + (only available on PB transport) + :type head_only: bool + :rtype: :class:`RiakObject` """ - result = self.client.get(self, r=r, pr=pr, vtag=vtag) - if result and result != ('', []): - self._populate(result) - else: - self.clear() - + self.client.get(self, r=r, pr=pr, timeout=timeout, head_only=head_only) return self - def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + def delete(self, r=None, w=None, dw=None, pr=None, pw=None, + timeout=None): """ Delete this object from Riak. - :param rw: RW-value. Wait until this many partitions have - deleted the object before responding. (deprecated in Riak - 1.0+, use R/W/DW) - :type rw: integer :param r: R-value, wait for this many partitions to read object before performing the put :type r: integer @@ -312,10 +337,13 @@ def delete(self, rw=None, r=None, w=None, dw=None, pr=None, pw=None): :param pw: PW-value, require this many primary partitions to be available before performing the put :type pw: integer - :rtype: RiakObject + :param timeout: a timeout value in milliseconds + :type timeout: int + :rtype: :class:`RiakObject` """ - self.client.delete(self, rw=rw, r=r, w=w, dw=dw, pr=pr, pw=pw) + self.client.delete(self, r=r, w=w, dw=dw, pr=pr, pw=pw, + timeout=timeout) self.clear() return self @@ -325,72 +353,35 @@ def clear(self): :rtype: RiakObject """ - self.headers = [] - self.links = [] - self.data = None - self.exists = False self.siblings = [] return self - def _populate(self, result): - """ - Populate the object based on the return from get. - - If None returned, then object is not found - If a tuple of vclock, contents then one or more - whole revisions of the key were found - If a list of vtags is returned there are multiple - sibling that need to be retrieved with get. - """ - if result is None or result is self: - return self - elif type(result) is RiakObject: - self.clear() - self.__dict__ = result.__dict__.copy() - else: - raise RiakError("do not know how to handle type %s" % type(result)) - - def get_sibling(self, i, r=None, pr=None): - """ - Retrieve a sibling by sibling number. - - :param i: Sibling number. - :type i: integer - :param r: R-Value. Wait until this many partitions - have responded before returning to client. - :type r: integer - :rtype: RiakObject. - """ - if isinstance(self.siblings[i], RiakObject): - return self.siblings[i] - else: - # Run the request... - vtag = self.siblings[i] - obj = RiakObject(self.client, self.bucket, self.key) - obj.reload(r=r, pr=pr, vtag=vtag) - - # And make sure it knows who its siblings are - self.siblings[i] = obj - obj.siblings = self.siblings - return obj - - def add(self, *args): + def add(self, arg1, arg2=None, arg3=None, bucket_type=None): """ Start assembling a Map/Reduce operation. - A shortcut for :func:`RiakMapReduce.add`. - - :rtype: RiakMapReduce + A shortcut for :meth:`~riak.mapreduce.RiakMapReduce.add`. + + :param arg1: the object or bucket to add + :type arg1: RiakObject, string + :param arg2: a key or list of keys to add (if a bucket is + given in arg1) + :type arg2: string, list, None + :param arg3: key data for this input (must be convertible to JSON) + :type arg3: string, list, dict, None + :param bucket_type: Optional name of a bucket type + :type bucket_type: string, None + :rtype: :class:`~riak.mapreduce.RiakMapReduce` """ mr = RiakMapReduce(self.client) - mr.add(self.bucket.name, self.key) - return mr.add(*args) + mr.add(self.bucket.name, self.key, bucket_type=bucket_type) + return mr.add(arg1, arg2, arg3, bucket_type) def link(self, *args): """ Start assembling a Map/Reduce operation. - A shortcut for :func:`RiakMapReduce.link`. + A shortcut for :meth:`~riak.mapreduce.RiakMapReduce.link`. - :rtype: RiakMapReduce + :rtype: :class:`~riak.mapreduce.RiakMapReduce` """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) @@ -399,9 +390,9 @@ def link(self, *args): def map(self, *args): """ Start assembling a Map/Reduce operation. - A shortcut for :func:`RiakMapReduce.map`. + A shortcut for :meth:`~riak.mapreduce.RiakMapReduce.map`. - :rtype: RiakMapReduce + :rtype: :class:`~riak.mapreduce.RiakMapReduce` """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) @@ -410,12 +401,10 @@ def map(self, *args): def reduce(self, *args): """ Start assembling a Map/Reduce operation. - A shortcut for :func:`RiakMapReduce.reduce`. + A shortcut for :meth:`~riak.mapreduce.RiakMapReduce.reduce`. - :rtype: RiakMapReduce + :rtype: :class:`~riak.mapreduce.RiakMapReduce` """ mr = RiakMapReduce(self.client) mr.add(self.bucket.name, self.key) return mr.reduce(*args) - -from riak.mapreduce import RiakMapReduce diff --git a/riak/search.py b/riak/search.py deleted file mode 100644 index 0ada3d1b..00000000 --- a/riak/search.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Copyright 2010 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - - -class RiakSearch(object): - """ - A wrapper around Riak Search-related client operations. See - :func:`RiakClient.solr`. - """ - - def __init__(self, client, **unused_args): - self._client = client - - def add(self, index, *docs): - """ - Adds documents to a fulltext index. Shortcut and backwards - compatibility for :func:`RiakClientOperations.fulltext_add`. - """ - self._client.fulltext_add(index, docs=docs) - - index = add - - def delete(self, index, docs=None, queries=None): - """ - Removes documents from a fulltext index. Shortcut and backwards - compatibility for :func:`RiakClientOperations.fulltext_delete`. - """ - self._client.fulltext_delete(index, docs=docs, queries=queries) - - remove = delete - - def search(self, index, query, **params): - """ - Searches a fulltext index. Shortcut and backwards - compatibility for :func:`RiakClientOperations.fulltext_search`. - """ - return self._client.fulltext_search(index, query, **params) - - select = search diff --git a/riak/security.py b/riak/security.py new file mode 100644 index 00000000..d048f008 --- /dev/null +++ b/riak/security.py @@ -0,0 +1,291 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ssl +import warnings +from riak import RiakError +from riak.util import str_to_long + +if hasattr(ssl, 'SSLContext'): + # For Python >= 2.7.9 and Python 3.x + USE_STDLIB_SSL = True +else: + # For Python 2.6 and <= 2.7.8 + USE_STDLIB_SSL = False + +if not USE_STDLIB_SSL: + import OpenSSL.SSL + from OpenSSL import crypto + +OPENSSL_VERSION_101G = 268439679 +if hasattr(ssl, 'OPENSSL_VERSION_NUMBER'): + # For Python 2.7 and Python 3.x + sslver = ssl.OPENSSL_VERSION_NUMBER + # Be sure to use at least OpenSSL 1.0.1g + tls_12 = hasattr(ssl, 'PROTOCOL_TLSv1_2') + if sslver < OPENSSL_VERSION_101G or not tls_12: + verstring = ssl.OPENSSL_VERSION + msg = "{0} (>= 1.0.1g required), TLS 1.2 support: {1}" \ + .format(verstring, tls_12) + warnings.warn(msg, UserWarning) + if hasattr(ssl, 'PROTOCOL_TLSv1_2'): + DEFAULT_TLS_VERSION = ssl.PROTOCOL_TLSv1_2 + elif hasattr(ssl, 'PROTOCOL_TLSv1_1'): + DEFAULT_TLS_VERSION = ssl.PROTOCOL_TLSv1_1 + elif hasattr(ssl, 'PROTOCOL_TLSv1'): + DEFAULT_TLS_VERSION = ssl.PROTOCOL_TLSv1 + else: + DEFAULT_TLS_VERSION = ssl.PROTOCOL_SSLv23 + +else: + # For Python 2.6 + sslver = OpenSSL.SSL.OPENSSL_VERSION_NUMBER + # Be sure to use at least OpenSSL 1.0.1g + tls_12 = hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD') + if (sslver < OPENSSL_VERSION_101G) or tls_12: + verstring = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) + msg = "{0} (>= 1.0.1g required), TLS 1.2 support: {1}" \ + .format(verstring, tls_12) + warnings.warn(msg, UserWarning) + if hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): + DEFAULT_TLS_VERSION = OpenSSL.SSL.TLSv1_2_METHOD + elif hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'): + DEFAULT_TLS_VERSION = OpenSSL.SSL.TLSv1_1_METHOD + elif hasattr(OpenSSL.SSL, 'TLSv1_METHOD'): + DEFAULT_TLS_VERSION = OpenSSL.SSL.TLSv1_METHOD + else: + DEFAULT_TLS_VERSION = OpenSSL.SSL.SSLv23_METHOD + + +class SecurityError(RiakError): + """ + Raised when there is an issue establishing security. + """ + def __init__(self, message="Security error"): + super(SecurityError, self).__init__(message) + + +class SecurityCreds: + def __init__(self, + username=None, + password=None, + pkey_file=None, + pkey=None, + cert_file=None, + cert=None, + cacert_file=None, + cacert=None, + crl_file=None, + crl=None, + ciphers=None, + ssl_version=DEFAULT_TLS_VERSION): + """ + Container class for security-related settings + + :param username: Riak Security username + :type username: str + :param password: Riak Security password + :type password: str + :param pkey_file: Full path to security key file + :type pkey_file: str + :param key: Loaded security key file + :type key: :class:`OpenSSL.crypto.PKey` + :param cert_file: Full path to certificate file + :type cert_file: str + :param cert: Loaded client certificate + :type cert: :class:`OpenSSL.crypto.X509` + :param cacert_file: Full path to CA certificate file + :type cacert_file: str + :param cacert: Loaded CA certificate + :type cacert: :class:`OpenSSL.crypto.X509` + :param crl_file: Full path to revoked certificates file + :type crl_file: str + :param crl: Loaded revoked certificates list + :type crl: :class:`OpenSSL.crypto.CRL` + :param ciphers: List of supported SSL ciphers + :type ciphers: str + :param ssl_version: OpenSSL security version + :type ssl_version: int + """ + self._username = username + self._password = password + self._pkey_file = pkey_file + self._pkey = pkey + self._cert_file = cert_file + self._cert = cert + self._cacert_file = cacert_file + self._cacert = cacert + self._crl_file = crl_file + self._crl = crl + self._ciphers = ciphers + self._ssl_version = ssl_version + + @property + def username(self): + """ + Riak Username + + :rtype: str + """ + return self._username + + @property + def password(self): + """ + Riak Password + + :rtype: str + """ + return self._password + + @property + def pkey_file(self): + """ + Client Private Key file + + :rtype: str + """ + return self._pkey_file + + @property + def cert_file(self): + """ + Client Certificate file + + :rtype: str + """ + return self._cert_file + + @property + def cacert_file(self): + """ + Certifying Authority (CA) Certificate file + + :rtype: str + """ + return self._cacert_file + + @property + def crl_file(self): + """ + Certificate Revocation List file + + :rtype: str + """ + return self._crl_file + + @property + def ciphers(self): + """ + Colon-delimited list of supported ciphers + + :rtype: str + """ + return self._ciphers + + @property + def ssl_version(self): + """SSL/TLS Protocol to use + + :rtype: an int constant from OpenSSL, like + :data:`OpenSSL.SSL.TLSv1_2_METHOD` + """ + return self._ssl_version + + if not USE_STDLIB_SSL: + @property + def pkey(self): + """ + Client Private key + + :rtype: :class:`OpenSSL.crypto.PKey` + """ + return self._cached_cert('_pkey', crypto.load_privatekey) + + @property + def cert(self): + """ + Client Certificate + + :rtype: :class:`OpenSSL.crypto.X509` + """ + return self._cached_cert('_cert', crypto.load_certificate) + + @property + def cacert(self): + """ + Certifying Authority (CA) Certificate + + :rtype: :class:`OpenSSL.crypto.X509` + """ + return self._cached_cert('_cacert', crypto.load_certificate) + + @property + def crl(self): + """ + Certificate Revocation List + + :rtype: :class:`OpenSSL.crypto.CRL` + """ + return self._cached_cert('_crl', crypto.load_crl) + + def _cached_cert(self, key, loader): + # If the key is associated with a file, + # then lazily load and cache it + key_file = getattr(self, key + "_file") + if (getattr(self, key) is None) and (key_file is not None): + cert_list = [] + # The _file may be a list of files + if not isinstance(key_file, list): + key_file = [key_file] + for filename in key_file: + with open(filename, 'rb') as f: + cert_list.append(loader(OpenSSL.SSL.FILETYPE_PEM, + f.read())) + # If it is not a list, just store the first element + if len(cert_list) == 1: + cert_list = cert_list[0] + setattr(self, key, cert_list) + return getattr(self, key) + + def _has_credential(self, key): + """ + ``True`` if a credential or filename value has been supplied for + the given property. + + :param key: which configuration property to check for + :type key: str + :rtype: bool + """ + internal_key = "_" + key + return (getattr(self, internal_key) is not None) or \ + (getattr(self, internal_key + "_file") is not None) + + def _check_revoked_cert(self, ssl_socket): + """ + Checks whether the server certificate on the passed socket has been + revoked by checking the CRL. + + :param ssl_socket: the SSL/TLS socket + :rtype: bool + :raises SecurityError: when the certificate has been revoked + """ + if not self._has_credential('crl'): + return True + + servcert = ssl_socket.get_peer_certificate() + servserial = servcert.get_serial_number() + for rev in self.crl.get_revoked(): + if servserial == str_to_long(rev.get_serial(), 16): + raise SecurityError("Server certificate has been revoked") diff --git a/riak/table.py b/riak/table.py new file mode 100644 index 00000000..d4006503 --- /dev/null +++ b/riak/table.py @@ -0,0 +1,110 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from six import string_types, PY2 + + +class Table(object): + """ + The ``Table`` object allows you to access properties on a Riak + timeseries table and query timeseries data. + """ + def __init__(self, client, name): + """ + Returns a new ``Table`` instance. + + :param client: A :class:`RiakClient ` + instance + :type client: :class:`RiakClient ` + :param name: The table's name + :type name: string + """ + if not isinstance(name, string_types): + raise TypeError('Table name must be a string') + + if PY2: + try: + name = name.encode('ascii') + except UnicodeError: + raise TypeError('Unicode table names are not supported.') + + self._client = client + self.name = name + + def __str__(self): + return self.name + + def __repr__(self): + return self.name + + def new(self, rows, columns=None): + """ + A shortcut for manually instantiating a new + :class:`~riak.ts_object.TsObject` + + :param rows: An list of lists with timeseries data + :type rows: list + :param columns: An list of Column names and types. Optional. + :type columns: list + :rtype: :class:`~riak.ts_object.TsObject` + """ + from riak.ts_object import TsObject + + return TsObject(self._client, self, rows, columns) + + def describe(self): + """ + Retrieves a timeseries table's description. + + :rtype: :class:`TsObject ` + """ + return self._client.ts_describe(self) + + def get(self, key): + """ + Gets a value from a timeseries table. + + :param key: The timeseries value's key. + :type key: list + :rtype: :class:`TsObject ` + """ + return self._client.ts_get(self, key) + + def delete(self, key): + """ + Deletes a value from a timeseries table. + + :param key: The timeseries value's key. + :type key: list or dict + :rtype: boolean + """ + return self._client.ts_delete(self, key) + + def query(self, query, interpolations=None): + """ + Queries a timeseries table. + + :param query: The timeseries query. + :type query: string + :rtype: :class:`TsObject ` + """ + return self._client.ts_query(self, query, interpolations) + + def stream_keys(self, timeout=None): + """ + Streams keys from a timeseries table. + + :rtype: list + """ + return self._client.ts_stream_keys(self, timeout) diff --git a/riak/test_server.py b/riak/test_server.py index 73df069d..99d68742 100644 --- a/riak/test_server.py +++ b/riak/test_server.py @@ -1,3 +1,18 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function import os.path import threading import string @@ -6,8 +21,10 @@ import shutil import socket import time +import stat from subprocess import Popen, PIPE from riak.util import deep_merge +from six import string_types try: bytes @@ -28,14 +45,14 @@ def __repr__(self): def __eq__(self, other): return self.str == other - def __cmp__(self, other): - return cmp(self.str, other) + def __lt__(self, other): + return self.str < other def erlang_config(hash, depth=1): def printable(item): k, v = item - if isinstance(v, str): + if isinstance(v, string_types): p = '"%s"' % v elif isinstance(v, dict): p = erlang_config(v, depth + 1) @@ -125,6 +142,7 @@ def __init__(self, tmp_dir="/tmp/riak/test_server", def prepare(self): if not self._prepared: + self.touch_ssl_distribution_args() self.create_temp_directories() self._riak_script = os.path.join(self._temp_bin, "riak") self.write_riak_script() @@ -190,7 +208,7 @@ def wait_for_startup(self): try: socket.create_connection((self._http_ip(), self._http_port()), 1.0) - except socket.error, (value, message): + except IOError: pass else: listening = True @@ -199,7 +217,7 @@ def wait_for_erlang_prompt(self): prompted = False buffer = "" while not prompted: - line = self._server.stdout.read(1) + line = self._server.stdout.readline() if len(line) > 0: buffer += line if re.search(r"\(%s\)\d+>" % self.vm_args["-name"], buffer): @@ -231,7 +249,9 @@ def write_riak_script(self): temp_bin_file.write(line) - os.fchmod(temp_bin_file.fileno(), 0755) + os.fchmod(temp_bin_file.fileno(), + stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) def write_vm_args(self): with open(self._vm_args_path(), 'wb') as vm_args: @@ -243,6 +263,15 @@ def write_app_config(self): app_config.write(erlang_config(self.app_config)) app_config.write(".") + def touch_ssl_distribution_args(self): + # To make sure that the ssl_distribution.args file is present, + # the control script in the source node has to have been run at + # least once. Running the `chkconfig` command is innocuous + # enough to accomplish this without other side-effects. + script = os.path.join(self.bin_dir, "riak") + Popen([script, "chkconfig"], + stdin=PIPE, stdout=PIPE, stderr=PIPE).communicate() + def _kv_backend(self): return self.app_config["riak_kv"]["storage_backend"] diff --git a/riak/tests/__init__.py b/riak/tests/__init__.py index e69de29b..be547be6 100644 --- a/riak/tests/__init__.py +++ b/riak/tests/__init__.py @@ -0,0 +1,121 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import socket +import sys + +from riak.test_server import TestServer +from riak.security import SecurityCreds + +USE_TEST_SERVER = int(os.environ.get('USE_TEST_SERVER', '0')) +if USE_TEST_SERVER: + HTTP_PORT = 9000 + PB_PORT = 9002 + test_server = TestServer() + test_server.cleanup() + test_server.prepare() + test_server.start() + +try: + __import__('riak.pb') + HAVE_PROTO = True +except ImportError: + HAVE_PROTO = False + + +def hostname_resolves(hostname): + try: + socket.gethostbyname(hostname) + return 1 + except socket.error: + return 0 + + +distutils_debug = os.environ.get('DISTUTILS_DEBUG', '0') +if distutils_debug == '1': + logger = logging.getLogger() + logger.level = logging.DEBUG + logger.addHandler(logging.StreamHandler(sys.stdout)) + +HOST = os.environ.get('RIAK_TEST_HOST', '127.0.0.1') + +PROTOCOL = os.environ.get('RIAK_TEST_PROTOCOL', 'pbc') + +PB_HOST = os.environ.get('RIAK_TEST_PB_HOST', HOST) +PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) + +HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) +HTTP_PORT = int(os.environ.get('RIAK_TEST_HTTP_PORT', '8098')) + +# these ports are used to simulate errors, there shouldn't +# be anything listening on either port. +DUMMY_HTTP_PORT = int(os.environ.get('DUMMY_HTTP_PORT', '1023')) +DUMMY_PB_PORT = int(os.environ.get('DUMMY_PB_PORT', '1022')) + +RUN_BTYPES = int(os.environ.get('RUN_BTYPES', '0')) +RUN_DATATYPES = int(os.environ.get('RUN_DATATYPES', '0')) +RUN_CLIENT = int(os.environ.get('RUN_CLIENT', '0')) +RUN_INDEXES = int(os.environ.get('RUN_INDEXES', '0')) +RUN_KV = int(os.environ.get('RUN_KV', '0')) +RUN_MAPREDUCE = int(os.environ.get('RUN_MAPREDUCE', '0')) +RUN_POOL = int(os.environ.get('RUN_POOL', '0')) +RUN_RESOLVE = int(os.environ.get('RUN_RESOLVE', '0')) +RUN_SEARCH = int(os.environ.get('RUN_SEARCH', '0')) +RUN_TIMESERIES = int(os.environ.get('RUN_TIMESERIES', '0')) +RUN_YZ = int(os.environ.get('RUN_YZ', '0')) + +if PROTOCOL != 'pbc': + RUN_TIMESERIES = 0 + +RUN_SECURITY = int(os.environ.get('RUN_SECURITY', '0')) +if RUN_SECURITY: + h = 'riak-test' + if hostname_resolves(h): + HOST = PB_HOST = HTTP_HOST = h + else: + raise AssertionError( + 'RUN_SECURITY requires that the host name' + + ' "riak-test" resolves to the IP address of a Riak node' + + ' with security enabled.') + +SECURITY_USER = os.environ.get('RIAK_TEST_SECURITY_USER', 'riakpass') +SECURITY_PASSWD = os.environ.get('RIAK_TEST_SECURITY_PASSWD', 'Test1234') + +SECURITY_CACERT = os.environ.get('RIAK_TEST_SECURITY_CACERT', + 'tools/test-ca/certs/cacert.pem') +SECURITY_REVOKED = os.environ.get('RIAK_TEST_SECURITY_REVOKED', + 'tools/test-ca/crl/crl.pem') +SECURITY_BAD_CERT = os.environ.get('RIAK_TEST_SECURITY_BAD_CERT', + 'tools/test-ca/certs/badcert.pem') +# Certificate-based Authentication only supported by PBC +SECURITY_KEY = os.environ.get( + 'RIAK_TEST_SECURITY_KEY', + 'tools/test-ca/private/riakuser-client-cert-key.pem') +SECURITY_CERT = os.environ.get('RIAK_TEST_SECURITY_CERT', + 'tools/test-ca/certs/riakuser-client-cert.pem') +SECURITY_CERT_USER = os.environ.get('RIAK_TEST_SECURITY_CERT_USER', + 'riakuser') + +SECURITY_CIPHERS = 'DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:' + \ + 'DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:AES128-SHA256:' + \ + 'AES128-SHA:AES256-SHA256:AES256-SHA:RC4-SHA' + +SECURITY_CREDS = None +if RUN_SECURITY: + SECURITY_CREDS = SecurityCreds(username=SECURITY_USER, + password=SECURITY_PASSWD, + cacert_file=SECURITY_CACERT, + ciphers=SECURITY_CIPHERS) diff --git a/riak/tests/base.py b/riak/tests/base.py new file mode 100644 index 00000000..9aaf4e69 --- /dev/null +++ b/riak/tests/base.py @@ -0,0 +1,81 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +import logging +import random +import riak + +from riak.client import RiakClient +from riak.tests import HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS + + +class IntegrationTestBase(object): + host = None + pb_port = None + http_port = None + credentials = None + + @staticmethod + def randint(): + return random.randint(1, 999999) + + @staticmethod + def randname(length=12): + out = '' + for i in range(length): + out += chr(random.randint(ord('a'), ord('z'))) + return out + + @classmethod + def create_client(cls, host=None, http_port=None, pb_port=None, + protocol=None, credentials=None, **kwargs): + host = host or HOST + http_port = http_port or HTTP_PORT + pb_port = pb_port or PB_PORT + + if protocol is None: + if hasattr(cls, 'protocol') and (cls.protocol is not None): + protocol = cls.protocol + else: + protocol = PROTOCOL + + cls.protocol = protocol + + credentials = credentials or SECURITY_CREDS + + if hasattr(cls, 'client_options'): + kwargs.update(cls.client_options) + + logger = logging.getLogger() + logger.debug("RiakClient(protocol='%s', host='%s', pb_port='%d', " + "http_port='%d', credentials='%s', kwargs='%s')", + protocol, host, pb_port, http_port, credentials, kwargs) + + return RiakClient(protocol=protocol, + host=host, + http_port=http_port, + credentials=credentials, + pb_port=pb_port, + **kwargs) + + def setUp(self): + riak.disable_list_exceptions = True + self.bucket_name = self.randname() + self.key_name = self.randname() + self.client = self.create_client() + + def tearDown(self): + riak.disable_list_exceptions = False + self.client.close() diff --git a/riak/tests/comparison.py b/riak/tests/comparison.py new file mode 100644 index 00000000..aa1d21cc --- /dev/null +++ b/riak/tests/comparison.py @@ -0,0 +1,134 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +from six import PY2, PY3 +import collections +import warnings + + +class Comparison(object): + ''' + Provide a cross-version object comparison operator + since its name changed between Python 2.x and Python 3.x + ''' + + if PY3: + # Stolen from Python 2.7.8's unittest + _Mismatch = collections.namedtuple('Mismatch', 'actual expected value') + + def _count_diff_all_purpose(self, actual, expected): + ''' + Returns list of (cnt_act, cnt_exp, elem) + triples where the counts differ + ''' + # elements need not be hashable + s, t = list(actual), list(expected) + m, n = len(s), len(t) + NULL = object() + result = [] + for i, elem in enumerate(s): + if elem is NULL: + continue + cnt_s = cnt_t = 0 + for j in range(i, m): + if s[j] == elem: + cnt_s += 1 + s[j] = NULL + for j, other_elem in enumerate(t): + if other_elem == elem: + cnt_t += 1 + t[j] = NULL + if cnt_s != cnt_t: + diff = self._Mismatch(cnt_s, cnt_t, elem) + result.append(diff) + + for i, elem in enumerate(t): + if elem is NULL: + continue + cnt_t = 0 + for j in range(i, n): + if t[j] == elem: + cnt_t += 1 + t[j] = NULL + diff = self._Mismatch(0, cnt_t, elem) + result.append(diff) + return result + + def _count_diff_hashable(self, actual, expected): + ''' + Returns list of (cnt_act, cnt_exp, elem) triples + where the counts differ + ''' + # elements must be hashable + s, t = self._ordered_count(actual), self._ordered_count(expected) + result = [] + for elem, cnt_s in s.items(): + cnt_t = t.get(elem, 0) + if cnt_s != cnt_t: + diff = self._Mismatch(cnt_s, cnt_t, elem) + result.append(diff) + for elem, cnt_t in t.items(): + if elem not in s: + diff = self._Mismatch(0, cnt_t, elem) + result.append(diff) + return result + + def _ordered_count(self, iterable): + 'Return dict of element counts, in the order they were first seen' + c = collections.OrderedDict() + for elem in iterable: + c[elem] = c.get(elem, 0) + 1 + return c + + def assertItemsEqual(self, expected_seq, actual_seq, msg=None): + """An unordered sequence specific comparison. It asserts that + actual_seq and expected_seq have the same element counts. + Equivalent to:: + + self.assertEqual(Counter(iter(actual_seq)), + Counter(iter(expected_seq))) + + Asserts that each element has the same count in both sequences. + Example: + - [0, 1, 1] and [1, 0, 1] compare equal. + - [0, 0, 1] and [0, 1] compare unequal. + """ + first_seq, second_seq = list(expected_seq), list(actual_seq) + with warnings.catch_warnings(): + try: + first = collections.Counter(first_seq) + second = collections.Counter(second_seq) + except TypeError: + # Handle case with unhashable elements + differences = self._count_diff_all_purpose(first_seq, + second_seq) + else: + if first == second: + return + differences = self._count_diff_hashable(first_seq, + second_seq) + + if differences: + standardMsg = 'Element counts were not equal:\n' + lines = ['First has %d, Second has %d: %r' % + diff for diff in differences] + diffMsg = '\n'.join(lines) + standardMsg = self._truncateMessage(standardMsg, diffMsg) + + def assert_raises_regex(self, exception, regexp): + if PY2: + return self.assertRaisesRegexp(exception, regexp) + else: + return self.assertRaisesRegex(exception, regexp) diff --git a/riak/tests/pool-grinder.py b/riak/tests/pool-grinder.py index a7d73c83..19cb71d7 100755 --- a/riak/tests/pool-grinder.py +++ b/riak/tests/pool-grinder.py @@ -1,12 +1,30 @@ #!/usr/bin/env python - -from Queue import Queue -from threading import Thread #, currentThread +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +from six import PY2 +from threading import Thread import sys -sys.path.append("../transports/") -from pool import Pool #, BadResource +from pool import Pool from random import SystemRandom from time import sleep +if PY2: + from Queue import Queue +else: + from queue import Queue +sys.path.append("../transports/") class SimplePool(Pool): @@ -37,12 +55,12 @@ def test(): def _run(): psleep = rand.uniform(0.05, 0.1) - with pool.take() as a: + with pool.transaction() as a: started.put(1) started.join() a.append(rand.uniform(0, 1)) if psleep > 1: - print psleep + print(psleep) sleep(psleep) for i in range(n): @@ -61,34 +79,35 @@ def _run(): thr.join() if set(pool.elements) != set(touched): - print set(pool.elements) - set(touched) + print(set(pool.elements) - set(touched)) return False else: return True + ret = True count = 0 while ret: ret = test() count += 1 - print count + print(count) # INSTRUMENTED FUNCTION # def __claim_elements(self): -# #print 'waiting for self lock' +# #print('waiting for self lock') # with self.lock: # if self.__all_claimed(): # and self.unlocked: -# #print 'waiting on releaser lock' +# #print('waiting on releaser lock') # with self.releaser: -# print 'waiting for release' -# print 'targets', self.targets -# print 'tomb', self.targets[0].tomb -# print 'claimed', self.targets[0].claimed -# print self.releaser -# print self.lock -# print self.unlocked +# print('waiting for release'') +# print('targets', self.targets) +# print('tomb', self.targets[0].tomb) +# print('claimed', self.targets[0].claimed) +# print(self.releaser) +# print(self.lock) +# print(self.unlocked) # self.releaser.wait(1) # for element in self.targets: # if element.tomb: diff --git a/riak/tests/suite.py b/riak/tests/suite.py index 97f3532c..782be4a0 100644 --- a/riak/tests/suite.py +++ b/riak/tests/suite.py @@ -1,10 +1,19 @@ -import os.path -import platform +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -if platform.python_version() < '2.7': - unittest = __import__('unittest2') -else: - import unittest +import os.path +import unittest def additional_tests(): diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index d303eb4b..01f02aee 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -1,31 +1,40 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # -*- coding: utf-8 -*- -import os -import platform -if platform.python_version() < '2.7': - unittest = __import__('unittest2') -else: - import unittest +import unittest from riak import RiakError - -SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '0')) +from riak.tests import RUN_INDEXES +from riak.tests.base import IntegrationTestBase -class TwoITests(object): +class TwoITests(IntegrationTestBase, unittest.TestCase): def is_2i_supported(self): # Immediate test to see if 2i is even supported w/ the backend try: - self.client.index('foo', 'bar_bin', 'baz').run() + self.client.get_index('foo', 'bar_bin', 'baz') return True except Exception as e: if "indexes_not_supported" in str(e): return False return True # it failed, but is supported! - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_secondary_index_store(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") # Create a new object with indexes... bucket = self.client.bucket(self.bucket_name) @@ -103,10 +112,10 @@ def test_secondary_index_store(self): # Clean up... bucket.get('mykey1').delete() - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_set_indexes(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) foo = bucket.new('foo', 1) @@ -121,10 +130,10 @@ def test_set_indexes(self): self.assertEqual(1, len(result)) self.assertEqual('foo', str(result[0])) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_remove_indexes(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) bar = bucket.new('bar', 1).add_index('bar_int', 1)\ @@ -181,76 +190,49 @@ def test_remove_indexes(self): self.assertEqual(1, len([x for x in bar.indexes if x[0] == 'baz_bin'])) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_secondary_index_query(self): if not self.is_2i_supported(): - return True - - bucket = self.client.bucket(self.bucket_name) + raise unittest.SkipTest("2I not supported") - bucket.\ - new('mykey1', 'data1').\ - add_index('field1_bin', 'val1').\ - add_index('field2_int', 1001).\ - store() - bucket.\ - new('mykey2', 'data1').\ - add_index('field1_bin', 'val2').\ - add_index('field2_int', 1002).\ - store() - bucket.\ - new('mykey3', 'data1').\ - add_index('field1_bin', 'val3').\ - add_index('field2_int', 1003).\ - store() - bucket.\ - new('mykey4', 'data1').\ - add_index('field1_bin', 'val4').\ - add_index('field2_int', 1004).\ - store() + bucket, o1, o2, o3, o4 = self._create_index_objects() # Test an equality query... results = bucket.get_index('field1_bin', 'val2') - self.assertEquals(1, len(results)) - self.assertEquals('mykey2', str(results[0])) + self.assertEqual(1, len(results)) + self.assertEqual(o2.key, str(results[0])) # Test a range query... results = bucket.get_index('field1_bin', 'val2', 'val4') vals = set([str(key) for key in results]) - self.assertEquals(3, len(results)) - self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) + self.assertEqual(3, len(results)) + self.assertEqual(set([o2.key, o3.key, o4.key]), vals) # Test an equality query... results = bucket.get_index('field2_int', 1002) - self.assertEquals(1, len(results)) - self.assertEquals('mykey2', str(results[0])) + self.assertEqual(1, len(results)) + self.assertEqual(o2.key, str(results[0])) # Test a range query... results = bucket.get_index('field2_int', 1002, 1004) vals = set([str(key) for key in results]) - self.assertEquals(3, len(results)) - self.assertEquals(set(['mykey2', 'mykey3', 'mykey4']), vals) + self.assertEqual(3, len(results)) + self.assertEqual(set([o2.key, o3.key, o4.key]), vals) - # Clean up... - bucket.get('mykey1').delete() - bucket.get('mykey2').delete() - bucket.get('mykey3').delete() - bucket.get('mykey4').delete() - - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_secondary_index_invalid_name(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) with self.assertRaises(RiakError): bucket.new('k', 'a').add_index('field1', 'value1') - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_set_index(self): if not self.is_2i_supported(): - return True + raise unittest.SkipTest("2I not supported") bucket = self.client.bucket(self.bucket_name) obj = bucket.new('bar', 1) @@ -264,3 +246,275 @@ def test_set_index(self): self.assertEqual(set((('bar_int', 3), ('bar2_int', 1))), obj.indexes) obj.set_index('bar2_int', 10) self.assertEqual(set((('bar_int', 3), ('bar2_int', 10))), obj.indexes) + + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') + def test_stream_index(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + keys = [] + for entries in bucket.stream_index('field1_bin', 'val1', 'val3'): + keys.extend(entries) + + self.assertEqual(sorted([o1.key, o2.key, o3.key]), sorted(keys)) + + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') + def test_index_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # Test synchronous index query + pairs = bucket.get_index('field1_bin', 'val2', 'val4', + return_terms=True) + + self.assertEqual([('val2', o2.key), + ('val3', o3.key), + ('val4', o4.key)], sorted(pairs)) + + # Test streaming index query + spairs = [] + for chunk in bucket.stream_index('field2_int', 1002, 1004, + return_terms=True): + spairs.extend(chunk) + + self.assertEqual([(1002, o2.key), (1003, o3.key), (1004, o4.key)], + sorted(spairs)) + + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') + def test_index_pagination(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + results = bucket.get_index('field1_bin', 'val0', 'val5', + max_results=2) + # Number of results =< page size + self.assertLessEqual(2, len(results)) + # Results are in-order + self.assertEqual([o1.key, o2.key], results) + + # Continuation/next page present when page size smaller than + # total results size + self.assertIsNotNone(results.continuation) + self.assertTrue(results.has_next_page()) + + # Retrieving next page gets more results + page2 = results.next_page() + self.assertLessEqual(2, len(page2)) + self.assertEqual([o3.key, o4.key], page2) + + # iterate over the entire query + presults = [] + pagecount = 0 + for page in bucket.paginate_index('field1_bin', 'val0', 'val5', + max_results=2): + pagecount += 1 + presults.extend(page.results) + + self.assertEqual(3, pagecount) + self.assertEqual([o1.key, o2.key, o3.key, o4.key], presults) + + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') + def test_index_pagination_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # ========= Above steps work for return-terms ========== + results = bucket.get_index('field1_bin', 'val0', 'val5', + max_results=2, return_terms=True) + # Number of results =< page size + self.assertLessEqual(2, len(results)) + # Results are in-order + self.assertEqual([('val1', o1.key), ('val2', o2.key)], results) + + # Continuation/next page present when page size smaller than + # total results size + self.assertIsNotNone(results.continuation) + self.assertTrue(results.has_next_page()) + + # Retrieving next page gets more results + page2 = results.next_page() + self.assertLessEqual(2, len(results)) + self.assertEqual([('val3', o3.key), ('val4', o4.key)], page2) + + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') + def test_index_pagination_stream(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # ========= Above steps work for streaming ========== + stream = bucket.stream_index('field1_bin', 'val0', 'val5', + max_results=2) + results = [] + for result in stream: + results.extend(result) + + # Number of results =< page size + self.assertLessEqual(2, len(results)) + # Results are in-order + self.assertEqual([o1.key, o2.key], results) + + # Continuation/next page present when page size smaller than + # total results size + self.assertIsNotNone(stream.continuation) + self.assertTrue(stream.has_next_page()) + + # Retrieving next page gets more results + results = [] + for result in stream.next_page(): + results.extend(result) + self.assertLessEqual(2, len(results)) + self.assertEqual([o3.key, o4.key], results) + + # iterate over the entire query, streaming each page + presults = [] + pagecount = 0 + for page in bucket.paginate_stream_index('field1_bin', 'val0', 'val5', + max_results=2): + pagecount += 1 + for result in page: + presults.extend(result) + + self.assertEqual(3, pagecount) + self.assertEqual([o1.key, o2.key, o3.key, o4.key], presults) + + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') + def test_index_pagination_stream_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # ========= Above steps work for streaming with return-terms ========== + + stream = bucket.stream_index('field1_bin', 'val0', 'val5', + max_results=2, return_terms=True) + results = [] + for result in stream: + results.extend(result) + + # Number of results =< page size + self.assertLessEqual(2, len(results)) + # Results are in-order + self.assertEqual([('val1', o1.key), ('val2', o2.key)], results) + + # Continuation/next page present when page size smaller than + # total results size + self.assertIsNotNone(stream.continuation) + self.assertTrue(stream.has_next_page()) + + # Retrieving next page gets more results + results = [] + for result in stream.next_page(): + results.extend(result) + self.assertLessEqual(2, len(results)) + self.assertEqual([('val3', o3.key), ('val4', o4.key)], results) + + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') + def test_index_eq_query_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + results = bucket.get_index('field2_int', 1001, return_terms=True) + self.assertEqual([(1001, o1.key)], results) + + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') + def test_index_eq_query_stream_return_terms(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + results = [] + for item in bucket.stream_index('field2_int', 1001, return_terms=True): + results.extend(item) + + self.assertEqual([(1001, o1.key)], results) + + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') + def test_index_timeout(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + # Disable timeouts since they are too racy + # with self.assertRaises(RiakError): + # bucket.get_index('field1_bin', 'val1', timeout=1) + # + # with self.assertRaises(RiakError): + # for i in bucket.stream_index('field1_bin', 'val1', timeout=1): + # pass + + # This should not raise + self.assertEqual([o1.key], bucket.get_index('field1_bin', 'val1', + timeout='infinity')) + + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') + def test_index_regex(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects() + + results = [] + for item in bucket.stream_index('field1_bin', 'val0', + 'val5', term_regex='.*l2', + return_terms=True): + results.extend(item) + + self.assertEqual([('val2', o2.key)], results) + + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') + def test_index_falsey_endkey_gh378(self): + if not self.is_2i_supported(): + raise unittest.SkipTest("2I is not supported") + + bucket, o1, o2, o3, o4 = self._create_index_objects(int_sign=-1) + + results = [] + for item in bucket.stream_index('field2_int', -10000, 0): + results.extend(item) + + self.assertEqual(set([o4.key, o3.key, o2.key, o1.key]), + set(results)) + + def _create_index_objects(self, int_sign=1): + """ + Creates a number of index objects to be used in 2i test + """ + bucket = self.client.bucket(self.bucket_name) + + o1 = bucket.\ + new(self.randname(), 'data1').\ + add_index('field1_bin', 'val1').\ + add_index('field2_int', int_sign*1001).\ + store() + o2 = bucket.\ + new(self.randname(), 'data1').\ + add_index('field1_bin', 'val2').\ + add_index('field2_int', int_sign*1002).\ + store() + o3 = bucket.\ + new(self.randname(), 'data1').\ + add_index('field1_bin', 'val3').\ + add_index('field2_int', int_sign*1003).\ + store() + o4 = bucket.\ + new(self.randname(), 'data1').\ + add_index('field1_bin', 'val4').\ + add_index('field2_int', int_sign*1004).\ + store() + + return bucket, o1, o2, o3, o4 diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py deleted file mode 100644 index 1894cba9..00000000 --- a/riak/tests/test_all.py +++ /dev/null @@ -1,269 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import with_statement - -import os -import random -import platform - -if platform.python_version() < '2.7': - unittest = __import__('unittest2') -else: - import unittest - -from riak.client import RiakClient -from riak.mapreduce import RiakKeyFilter -from riak import key_filter - -from riak.test_server import TestServer - -from riak.tests.test_search import SearchTests, \ - EnableSearchTests, SolrSearchTests -from riak.tests.test_mapreduce import MapReduceAliasTests, \ - ErlangMapReduceTests, JSMapReduceTests, LinkTests, MapReduceStreamTests -from riak.tests.test_kv import BasicKVTests, KVFileTests, \ - HTTPBucketPropsTest, PbcBucketPropsTest -from riak.tests.test_2i import TwoITests - -try: - __import__('riak_pb') - HAVE_PROTO = True -except ImportError: - HAVE_PROTO = False - -HOST = os.environ.get('RIAK_TEST_HOST', '127.0.0.1') - -PB_HOST = os.environ.get('RIAK_TEST_PB_HOST', HOST) -PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) - -HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) -HTTP_PORT = int(os.environ.get('RIAK_TEST_HTTP_PORT', '8098')) - -USE_TEST_SERVER = int(os.environ.get('USE_TEST_SERVER', '0')) - -if USE_TEST_SERVER: - HTTP_PORT = 9000 - PB_PORT = 9002 - test_server = TestServer() - test_server.cleanup() - test_server.prepare() - test_server.start() - -testrun_search_bucket = None -testrun_props_bucket = None -testrun_sibs_bucket = None - - -def setUpModule(): - global testrun_search_bucket, testrun_props_bucket, \ - testrun_sibs_bucket - - c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT) - - testrun_props_bucket = 'propsbucket' - testrun_sibs_bucket = 'sibsbucket' - c.bucket(testrun_sibs_bucket).allow_mult = True - - if not int(os.environ.get('SKIP_SEARCH', '0')): - testrun_search_bucket = 'searchbucket' - b = c.bucket(testrun_search_bucket) - b.enable_search() - - -def tearDownModule(): - c = RiakClient(protocol='http', host=HTTP_HOST, http_port=HTTP_PORT) - if not int(os.environ.get('SKIP_SEARCH', '0')): - b = c.bucket(testrun_search_bucket) - b.clear_properties() - b = c.bucket(testrun_sibs_bucket) - b.clear_properties() - b = c.bucket(testrun_props_bucket) - b.clear_properties() - - -class BaseTestCase(object): - - host = None - pb_port = None - http_port = None - - @staticmethod - def randint(): - return random.randint(1, 999999) - - @staticmethod - def randname(length=12): - out = '' - for i in range(length): - out += chr(random.randint(ord('a'), ord('z'))) - return out - - def create_client(self, host=None, http_port=None, pb_port=None, - protocol=None, **client_args): - host = host or self.host or HOST - http_port = http_port or self.http_port or HTTP_PORT - pb_port = pb_port or self.pb_port or PB_PORT - protocol = protocol or self.protocol - return RiakClient(protocol=protocol, - host=host, - http_port=http_port, - pb_port=pb_port, **client_args) - - def setUp(self): - self.bucket_name = self.randname() - self.key_name = self.randname() - self.search_bucket = testrun_search_bucket - self.sibs_bucket = testrun_sibs_bucket - self.props_bucket = testrun_props_bucket - - self.client = self.create_client() - - -class RiakPbcTransportTestCase(BasicKVTests, - KVFileTests, - PbcBucketPropsTest, - TwoITests, - LinkTests, - ErlangMapReduceTests, - JSMapReduceTests, - MapReduceAliasTests, - MapReduceStreamTests, - SearchTests, - BaseTestCase, - unittest.TestCase): - - def setUp(self): - if not HAVE_PROTO: - self.skipTest('protobuf is unavailable') - self.host = PB_HOST - self.pb_port = PB_PORT - self.protocol = 'pbc' - self.http_client = self.create_client(HTTP_HOST, - http_port=HTTP_PORT) - super(RiakPbcTransportTestCase, self).setUp() - - def test_uses_client_id_if_given(self): - zero_client_id = "\0\0\0\0" - c = self.create_client(client_id=zero_client_id) - self.assertEqual(zero_client_id, c.client_id) - - def test_bucket_search_enabled(self): - with self.assertRaises(NotImplementedError): - bucket = self.client.bucket(self.bucket_name) - bucket.search_enabled() - - def test_enable_search_commit_hook(self): - with self.assertRaises(NotImplementedError): - bucket = self.client.bucket(self.bucket_name) - bucket.enable_search() - - -class RiakHttpTransportTestCase(BasicKVTests, - KVFileTests, - HTTPBucketPropsTest, - TwoITests, - LinkTests, - ErlangMapReduceTests, - JSMapReduceTests, - MapReduceAliasTests, - MapReduceStreamTests, - EnableSearchTests, - SolrSearchTests, - SearchTests, - BaseTestCase, - unittest.TestCase): - - def setUp(self): - self.host = HTTP_HOST - self.http_port = HTTP_PORT - self.protocol = 'http' - super(RiakHttpTransportTestCase, self).setUp() - - def test_no_returnbody(self): - bucket = self.client.bucket(self.bucket_name) - o = bucket.new(self.key_name, "bar").store(return_body=False) - self.assertEqual(o.vclock, None) - - def test_too_many_link_headers_shouldnt_break_http(self): - bucket = self.client.bucket(self.bucket_name) - o = bucket.new("lots_of_links", "My god, it's full of links!") - for i in range(0, 400): - link = ("other", "key%d" % i, "next") - o.add_link(link) - - o.store() - stored_object = bucket.get("lots_of_links") - self.assertEqual(len(stored_object.links), 400) - - def test_clear_bucket_properties(self): - bucket = self.client.bucket(self.props_bucket) - bucket.allow_mult = True - self.assertTrue(bucket.allow_mult) - bucket.n_val = 1 - self.assertEqual(bucket.n_val, 1) - # Test setting clearing properties... - - self.assertTrue(bucket.clear_properties()) - self.assertFalse(bucket.allow_mult) - self.assertEqual(bucket.n_val, 3) - - -class FilterTests(unittest.TestCase): - def test_simple(self): - f1 = RiakKeyFilter("tokenize", "-", 1) - self.assertEqual(f1._filters, [["tokenize", "-", 1]]) - - def test_add(self): - f1 = RiakKeyFilter("tokenize", "-", 1) - f2 = RiakKeyFilter("eq", "2005") - f3 = f1 + f2 - self.assertEqual(list(f3), [["tokenize", "-", 1], ["eq", "2005"]]) - - def test_and(self): - f1 = RiakKeyFilter("starts_with", "2005-") - f2 = RiakKeyFilter("ends_with", "-01") - f3 = f1 & f2 - self.assertEqual(list(f3), - [["and", - [["starts_with", "2005-"]], - [["ends_with", "-01"]]]]) - - def test_multi_and(self): - f1 = RiakKeyFilter("starts_with", "2005-") - f2 = RiakKeyFilter("ends_with", "-01") - f3 = RiakKeyFilter("matches", "-11-") - f4 = f1 & f2 & f3 - self.assertEqual(list(f4), [["and", - [["starts_with", "2005-"]], - [["ends_with", "-01"]], - [["matches", "-11-"]], - ]]) - - def test_or(self): - f1 = RiakKeyFilter("starts_with", "2005-") - f2 = RiakKeyFilter("ends_with", "-01") - f3 = f1 | f2 - self.assertEqual(list(f3), [["or", [["starts_with", "2005-"]], - [["ends_with", "-01"]]]]) - - def test_multi_or(self): - f1 = RiakKeyFilter("starts_with", "2005-") - f2 = RiakKeyFilter("ends_with", "-01") - f3 = RiakKeyFilter("matches", "-11-") - f4 = f1 | f2 | f3 - self.assertEqual(list(f4), [["or", - [["starts_with", "2005-"]], - [["ends_with", "-01"]], - [["matches", "-11-"]], - ]]) - - def test_chaining(self): - f1 = key_filter.tokenize("-", 1).eq("2005") - f2 = key_filter.tokenize("-", 2).eq("05") - f3 = f1 & f2 - self.assertEqual(list(f3), [["and", - [["tokenize", "-", 1], ["eq", "2005"]], - [["tokenize", "-", 2], ["eq", "05"]] - ]]) - -if __name__ == '__main__': - unittest.main() diff --git a/riak/tests/test_btypes.py b/riak/tests/test_btypes.py new file mode 100644 index 00000000..67cd6568 --- /dev/null +++ b/riak/tests/test_btypes.py @@ -0,0 +1,192 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from riak import RiakError, RiakObject +from riak.bucket import RiakBucket, BucketType +from riak.tests import RUN_BTYPES +from riak.tests.base import IntegrationTestBase +from riak.tests.comparison import Comparison + + +@unittest.skipUnless(RUN_BTYPES, "RUN_BTYPES is 0") +class BucketTypeTests(IntegrationTestBase, unittest.TestCase, Comparison): + def test_btype_init(self): + btype = self.client.bucket_type('foo') + self.assertIsInstance(btype, BucketType) + self.assertEqual('foo', btype.name) + self.assertIs(btype, self.client.bucket_type('foo')) + + def test_btype_get_bucket(self): + btype = self.client.bucket_type('foo') + bucket = btype.bucket(self.bucket_name) + self.assertIsInstance(bucket, RiakBucket) + self.assertIs(btype, bucket.bucket_type) + self.assertIs(bucket, + self.client.bucket_type('foo').bucket(self.bucket_name)) + self.assertIsNot(bucket, self.client.bucket(self.bucket_name)) + + def test_btype_default(self): + defbtype = self.client.bucket_type('default') + othertype = self.client.bucket_type('foo') + self.assertTrue(defbtype.is_default()) + self.assertFalse(othertype.is_default()) + + def test_btype_repr(self): + defbtype = self.client.bucket_type("default") + othertype = self.client.bucket_type("foo") + self.assertEqual("", str(defbtype)) + self.assertEqual("", str(othertype)) + self.assertEqual("", repr(defbtype)) + self.assertEqual("", repr(othertype)) + + def test_btype_get_props(self): + defbtype = self.client.bucket_type("default") + btype = self.client.bucket_type('no_siblings') + with self.assertRaises(ValueError): + defbtype.get_properties() + + props = btype.get_properties() + self.assertIsInstance(props, dict) + self.assertIn('n_val', props) + self.assertEqual(3, props['n_val']) + + def test_btype_set_props(self): + defbtype = self.client.bucket_type("default") + btype = self.client.bucket_type('no_siblings') + with self.assertRaises(ValueError): + defbtype.set_properties({'allow_mult': True}) + + oldprops = btype.get_properties() + try: + btype.set_properties({'allow_mult': True}) + newprops = btype.get_properties() + self.assertIsInstance(newprops, dict) + self.assertIn('allow_mult', newprops) + self.assertTrue(newprops['allow_mult']) + if 'claimant' in oldprops: # HTTP hack + del oldprops['claimant'] + finally: + btype.set_properties(oldprops) + + def test_btype_set_props_immutable(self): + btype = self.client.bucket_type("maps") + with self.assertRaises(RiakError): + btype.set_property('datatype', 'counter') + + def test_btype_list_buckets(self): + btype = self.client.bucket_type('no_siblings') + bucket = btype.bucket(self.bucket_name) + obj = bucket.new(self.key_name) + obj.data = [1, 2, 3] + obj.store() + + self.assertIn(bucket, btype.get_buckets()) + buckets = [] + for nested_buckets in btype.stream_buckets(): + buckets.extend(nested_buckets) + + self.assertIn(bucket, buckets) + + def test_btype_list_keys(self): + btype = self.client.bucket_type('no_siblings') + bucket = btype.bucket(self.bucket_name) + + obj = bucket.new(self.key_name) + obj.data = [1, 2, 3] + obj.store() + + self.assertIn(self.key_name, bucket.get_keys()) + keys = [] + for keylist in bucket.stream_keys(): + keys.extend(keylist) + + self.assertIn(self.key_name, keys) + + def test_default_btype_list_buckets(self): + default_btype = self.client.bucket_type("default") + bucket = default_btype.bucket(self.bucket_name) + obj = bucket.new(self.key_name) + obj.data = [1, 2, 3] + obj.store() + + self.assertIn(bucket, default_btype.get_buckets()) + buckets = [] + for nested_buckets in default_btype.stream_buckets(): + buckets.extend(nested_buckets) + + self.assertIn(bucket, buckets) + + self.assertItemsEqual(buckets, self.client.get_buckets()) + + def test_default_btype_list_keys(self): + btype = self.client.bucket_type("default") + bucket = btype.bucket(self.bucket_name) + + obj = bucket.new(self.key_name) + obj.data = [1, 2, 3] + obj.store() + + self.assertIn(self.key_name, bucket.get_keys()) + keys = [] + for keylist in bucket.stream_keys(): + keys.extend(keylist) + + self.assertIn(self.key_name, keys) + + oldapikeys = self.client.get_keys(self.client.bucket(self.bucket_name)) + self.assertItemsEqual(keys, oldapikeys) + + def test_multiget_bucket_types(self): + btype = self.client.bucket_type('no_siblings') + bucket = btype.bucket(self.bucket_name) + + for i in range(100): + obj = bucket.new(self.key_name + str(i)) + obj.data = {'id': i} + obj.store() + + mget = bucket.multiget([self.key_name + str(i) for i in range(100)]) + for mobj in mget: + self.assertIsInstance(mobj, RiakObject) + self.assertEqual(bucket, mobj.bucket) + self.assertEqual(btype, mobj.bucket.bucket_type) + + def test_write_once_bucket_type(self): + bt = 'write_once' + skey = 'write_once-init' + btype = self.client.bucket_type(bt) + bucket = btype.bucket(bt) + try: + sobj = bucket.get(skey) + except RiakError as e: + raise unittest.SkipTest(e) + if not sobj.exists: + for i in range(100): + o = bucket.new(self.key_name + str(i)) + o.data = {'id': i} + o.store() + o = bucket.new(skey, data={'id': skey}) + o.store() + + mget = bucket.multiget([self.key_name + str(i) for i in range(100)]) + for mobj in mget: + self.assertIsInstance(mobj, RiakObject) + self.assertEqual(bucket, mobj.bucket) + self.assertEqual(btype, mobj.bucket.bucket_type) + + props = btype.get_properties() + self.assertIn('write_once', props) + self.assertEqual(True, props['write_once']) diff --git a/riak/tests/test_client.py b/riak/tests/test_client.py new file mode 100644 index 00000000..001520d2 --- /dev/null +++ b/riak/tests/test_client.py @@ -0,0 +1,342 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from six import PY2 +from threading import Thread +from riak.riak_object import RiakObject +from riak.transports.tcp import TcpTransport +from riak.tests import DUMMY_HTTP_PORT, DUMMY_PB_PORT, \ + RUN_POOL, RUN_CLIENT +from riak.tests.base import IntegrationTestBase + +if PY2: + from Queue import Queue +else: + from queue import Queue + + +@unittest.skipUnless(RUN_CLIENT, 'RUN_CLIENT is 0') +class ClientTests(IntegrationTestBase, unittest.TestCase): + def test_can_set_tcp_keepalive(self): + if self.protocol == 'pbc': + topts = {'socket_keepalive': True} + c = self.create_client(transport_options=topts) + for i, r in enumerate(c._tcp_pool.resources): + self.assertIsInstance(r, TcpTransport) + self.assertTrue(r._socket_keepalive) + c.close() + else: + pass + + def test_uses_client_id_if_given(self): + if self.protocol == 'pbc': + zero_client_id = "\0\0\0\0" + c = self.create_client(client_id=zero_client_id) + self.assertEqual(zero_client_id, c.client_id) + c.close() + else: + pass + + def test_request_retries(self): + # We guess at some ports that will be unused by Riak or + # anything else. + client = self.create_client(http_port=DUMMY_HTTP_PORT, + pb_port=DUMMY_PB_PORT) + + # If retries are exhausted, the final result should also be an + # error. + self.assertRaises(IOError, client.ping) + client.close() + + def test_request_retries_configurable(self): + # We guess at some ports that will be unused by Riak or + # anything else. + client = self.create_client(http_port=DUMMY_HTTP_PORT, + pb_port=DUMMY_PB_PORT) + + # Change the retry count + client.retries = 10 + self.assertEqual(10, client.retries) + + # The retry count should be a thread local + retries = Queue() + + def _target(): + retries.put(client.retries) + retries.join() + + th = Thread(target=_target) + th.start() + self.assertEqual(3, retries.get(block=True)) + retries.task_done() + th.join() + + # Modify the retries in a with statement + with client.retry_count(5): + self.assertEqual(5, client.retries) + self.assertRaises(IOError, client.ping) + client.close() + + def test_timeout_validation(self): + bucket = self.client.bucket(self.bucket_name) + key = self.key_name + obj = bucket.new(key) + for bad in [0, -1, False, "foo"]: + with self.assertRaises(ValueError): + self.client.get_buckets(timeout=bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_buckets(timeout=bad): + pass + + with self.assertRaises(ValueError): + self.client.get_keys(bucket, timeout=bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_keys(bucket, timeout=bad): + pass + + with self.assertRaises(ValueError): + self.client.put(obj, timeout=bad) + + with self.assertRaises(ValueError): + self.client.get(obj, timeout=bad) + + with self.assertRaises(ValueError): + self.client.delete(obj, timeout=bad) + + with self.assertRaises(ValueError): + self.client.mapred([], [], bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_mapred([], [], bad): + pass + + with self.assertRaises(ValueError): + self.client.get_index(bucket, 'field1_bin', 'val1', 'val4', + timeout=bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_index(bucket, 'field1_bin', 'val1', + 'val4', timeout=bad): + pass + + def test_close_stops_operation_requests(self): + c = self.create_client() + c.ping() + c.close() + self.assertRaises(RuntimeError, c.ping) + + def test_multiget_bucket(self): + """ + Multiget operations can be invoked on buckets. + """ + keys = [self.key_name, self.randname(), self.randname()] + for key in keys: + if PY2: + self.client.bucket(self.bucket_name)\ + .new(key, encoded_data=key, content_type="text/plain")\ + .store() + else: + self.client.bucket(self.bucket_name)\ + .new(key, data=key, + content_type="text/plain").store() + results = self.client.bucket(self.bucket_name).multiget(keys) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertTrue(obj.exists) + if PY2: + self.assertEqual(obj.key, obj.encoded_data) + else: + self.assertEqual(obj.key, obj.data) + + def test_multiget_errors(self): + """ + Unrecoverable errors are captured along with the bucket/key + and not propagated. + """ + keys = [self.key_name, self.randname(), self.randname()] + client = self.create_client(http_port=DUMMY_HTTP_PORT, + pb_port=DUMMY_PB_PORT) + results = client.bucket(self.bucket_name).multiget(keys) + for failure in results: + self.assertIsInstance(failure, tuple) + self.assertEqual(failure[0], 'default') + self.assertEqual(failure[1], self.bucket_name) + self.assertIn(failure[2], keys) + if PY2: + self.assertIsInstance(failure[3], StandardError) # noqa + else: + self.assertIsInstance(failure[3], Exception) + client.close() + + def test_multiput_errors(self): + """ + Unrecoverable errors are captured along with the bucket/key + and not propagated. + """ + client = self.create_client(http_port=DUMMY_HTTP_PORT, + pb_port=DUMMY_PB_PORT) + bucket = client.bucket(self.bucket_name) + k1 = self.randname() + k2 = self.randname() + o1 = RiakObject(client, bucket, k1) + o2 = RiakObject(client, bucket, k2) + + if PY2: + o1.encoded_data = k1 + o2.encoded_data = k2 + else: + o1.data = k1 + o2.data = k2 + + objs = [o1, o2] + for robj in objs: + robj.content_type = 'text/plain' + + results = client.multiput(objs, return_body=True) + for failure in results: + self.assertIsInstance(failure, tuple) + self.assertIsInstance(failure[0], RiakObject) + if PY2: + self.assertIsInstance(failure[1], StandardError) # noqa + else: + self.assertIsInstance(failure[1], Exception) + client.close() + + def test_multiget_notfounds(self): + """ + Not founds work in multiget just the same as get. + """ + keys = [("default", self.bucket_name, self.key_name), + ("default", self.bucket_name, self.randname())] + results = self.client.multiget(keys) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertFalse(obj.exists) + + def test_multiget_pool_size(self): + """ + The pool size for multigets can be configured at client initiation + time. Multiget still works as expected. + """ + client = self.create_client(multiget_pool_size=2) + self.assertEqual(2, client._multiget_pool._size) + + keys = [self.key_name, self.randname(), self.randname()] + for key in keys: + if PY2: + client.bucket(self.bucket_name)\ + .new(key, encoded_data=key, content_type="text/plain")\ + .store() + else: + client.bucket(self.bucket_name)\ + .new(key, data=key, content_type="text/plain")\ + .store() + + results = client.bucket(self.bucket_name).multiget(keys) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertTrue(obj.exists) + if PY2: + self.assertEqual(obj.key, obj.encoded_data) + else: + self.assertEqual(obj.key, obj.data) + client.close() + + def test_multiput_pool_size(self): + """ + The pool size for multiputs can be configured at client initiation + time. Multiput still works as expected. + """ + client = self.create_client(multiput_pool_size=2) + self.assertEqual(2, client._multiput_pool._size) + + bucket = client.bucket(self.bucket_name) + k1 = self.randname() + k2 = self.randname() + o1 = RiakObject(client, bucket, k1) + o2 = RiakObject(client, bucket, k2) + + if PY2: + o1.encoded_data = k1 + o2.encoded_data = k2 + else: + o1.data = k1 + o2.data = k2 + + objs = [o1, o2] + for robj in objs: + robj.content_type = 'text/plain' + + results = client.multiput(objs, return_body=True) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertTrue(obj.exists) + self.assertEqual(obj.content_type, 'text/plain') + if PY2: + self.assertEqual(obj.key, obj.encoded_data) + else: + self.assertEqual(obj.key, obj.data) + client.close() + + def test_multiput_pool_options(self): + sz = 4 + client = self.create_client(multiput_pool_size=sz) + self.assertEqual(sz, client._multiput_pool._size) + + bucket = client.bucket(self.bucket_name) + k1 = self.randname() + k2 = self.randname() + o1 = RiakObject(client, bucket, k1) + o2 = RiakObject(client, bucket, k2) + + if PY2: + o1.encoded_data = k1 + o2.encoded_data = k2 + else: + o1.data = k1 + o2.data = k2 + + objs = [o1, o2] + for robj in objs: + robj.content_type = 'text/plain' + + results = client.multiput(objs, return_body=False) + for obj in results: + if client.protocol == 'pbc': + self.assertIsInstance(obj, RiakObject) + self.assertFalse(obj.exists) + self.assertEqual(obj.content_type, 'text/plain') + else: + self.assertIsNone(obj) + client.close() + + @unittest.skipUnless(RUN_POOL, 'RUN_POOL is 0') + def test_pool_close(self): + """ + Iterate over the connection pool and close all connections. + """ + # Do something to add to the connection pool + self.test_multiget_bucket() + if self.client.protocol == 'pbc': + self.assertGreater(len(self.client._tcp_pool.resources), 1) + else: + self.assertGreater(len(self.client._http_pool.resources), 1) + # Now close them all up + self.client.close() + self.assertEqual(len(self.client._http_pool.resources), 0) + self.assertEqual(len(self.client._tcp_pool.resources), 0) diff --git a/riak/tests/test_comparison.py b/riak/tests/test_comparison.py index 297ca5fa..8aac4ef8 100644 --- a/riak/tests/test_comparison.py +++ b/riak/tests/test_comparison.py @@ -1,32 +1,89 @@ -import platform - -if platform.python_version() < '2.7': - unittest = __import__('unittest2') -else: - import unittest +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +import unittest from riak.riak_object import RiakObject -from riak.bucket import RiakBucket -from riak.tests.test_all import BaseTestCase +from riak.bucket import RiakBucket, BucketType +from riak.tests.base import IntegrationTestBase + + +class BucketTypeRichComparisonTest(unittest.TestCase): + def test_btype_eq(self): + a = BucketType('client', 'a') + b = BucketType('client', 'a') + c = BucketType(None, 'a') + d = BucketType(None, 'a') + self.assertEqual(a, b) + self.assertEqual(c, d) + + def test_btype_nq(self): + a = BucketType('client', 'a') + b = BucketType('client', 'b') + c = BucketType(None, 'a') + d = BucketType(None, 'a') + self.assertNotEqual(a, b, "matched with different name, same client") + self.assertNotEqual(a, c, "matched with different client, same name") + self.assertNotEqual(b, d, "matched with nothing in common") + + def test_btype_hash(self): + a = BucketType('client', 'a') + b = BucketType('client', 'a') + c = BucketType('client', 'c') + d = BucketType('client2', 'a') + self.assertEqual(hash(a), hash(b), + 'same bucket type has different hashes') + self.assertNotEqual(hash(a), hash(c), + 'different bucket has same hash') + self.assertNotEqual(hash(a), hash(d), + 'same bucket type, different client has same hash') class RiakBucketRichComparisonTest(unittest.TestCase): def test_bucket_eq(self): - a = RiakBucket('client', 'a') - b = RiakBucket('client', 'a') + default_bt = BucketType(None, "default") + foo_bt = BucketType(None, "foo") + a = RiakBucket('client', 'a', default_bt) + b = RiakBucket('client', 'a', default_bt) + c = RiakBucket('client', 'a', foo_bt) + d = RiakBucket('client', 'a', foo_bt) self.assertEqual(a, b) + self.assertEqual(c, d) def test_bucket_nq(self): - a = RiakBucket('client', 'a') - b = RiakBucket('client', 'b') + default_bt = BucketType(None, "default") + foo_bt = BucketType(None, "foo") + a = RiakBucket('client', 'a', default_bt) + b = RiakBucket('client', 'b', default_bt) + c = RiakBucket('client', 'a', foo_bt) self.assertNotEqual(a, b, 'matched with a different bucket') + self.assertNotEqual(a, c, 'matched with a different bucket type') def test_bucket_hash(self): - a = RiakBucket('client', 'a') - b = RiakBucket('client', 'a') - c = RiakBucket('client', 'c') - self.assertEqual(hash(a), hash(b), 'same bucket has different hashes') - self.assertNotEqual(hash(a), hash(c), 'different bucket has same hash') + default_bt = BucketType(None, "default") + foo_bt = BucketType(None, "foo") + a = RiakBucket('client', 'a', default_bt) + b = RiakBucket('client', 'a', default_bt) + c = RiakBucket('client', 'c', default_bt) + d = RiakBucket('client', 'a', foo_bt) + self.assertEqual(hash(a), hash(b), + 'same bucket has different hashes') + self.assertNotEqual(hash(a), hash(c), + 'different bucket has same hash') + self.assertNotEqual(hash(a), hash(d), + 'same bucket, different bucket type has same hash') class RiakObjectComparisonTest(unittest.TestCase): @@ -34,6 +91,12 @@ def test_object_eq(self): a = RiakObject(None, 'bucket', 'key') b = RiakObject(None, 'bucket', 'key') self.assertEqual(a, b) + default_bt = BucketType(None, "default") + bucket_a = RiakBucket('client', 'a', default_bt) + bucket_b = RiakBucket('client', 'a', default_bt) + c = RiakObject(None, bucket_a, 'key') + d = RiakObject(None, bucket_b, 'key') + self.assertEqual(c, d) def test_object_nq(self): a = RiakObject(None, 'bucket', 'key') @@ -41,6 +104,13 @@ def test_object_nq(self): c = RiakObject(None, 'not bucket', 'key') self.assertNotEqual(a, b, 'matched with different keys') self.assertNotEqual(a, c, 'matched with different buckets') + default_bt = BucketType(None, "default") + foo_bt = BucketType(None, "foo") + bucket_a = RiakBucket('client', 'a', default_bt) + bucket_b = RiakBucket('client', 'a', foo_bt) + c = RiakObject(None, bucket_a, 'key') + d = RiakObject(None, bucket_b, 'key') + self.assertNotEqual(c, d) def test_object_hash(self): a = RiakObject(None, 'bucket', 'key') @@ -49,6 +119,23 @@ def test_object_hash(self): self.assertEqual(hash(a), hash(b), 'same object has different hashes') self.assertNotEqual(hash(a), hash(c), 'different object has same hash') + default_bt = BucketType(None, "default") + foo_bt = BucketType(None, "foo") + bucket_a = RiakBucket('client', 'a', default_bt) + bucket_b = RiakBucket('client', 'a', foo_bt) + d = RiakObject(None, bucket_a, 'key') + e = RiakObject(None, bucket_a, 'key') + f = RiakObject(None, bucket_b, 'key') + g = RiakObject(None, bucket_b, 'not key') + self.assertEqual(hash(d), hash(e), + 'same object, same bucket_type has different hashes') + self.assertNotEqual(hash(e), hash(f), + 'same object, different bucket type has the ' + 'same hash') + self.assertNotEqual(hash(d), hash(g), + 'different object, different bucket ' + 'type has same hash') + def test_object_valid_key(self): a = RiakObject(None, 'bucket', 'key') self.assertIsInstance(a, RiakObject, 'valid key name is rejected') @@ -59,12 +146,14 @@ def test_object_valid_key(self): self.assertIsNone(b, 'empty object key not allowed') -class RiakClientComparisonTest(unittest.TestCase, BaseTestCase): +class RiakClientComparisonTest(IntegrationTestBase, unittest.TestCase): def test_client_eq(self): self.protocol = 'http' a = self.create_client(host='host1', http_port=11) b = self.create_client(host='host1', http_port=11) self.assertEqual(a, b) + a.close() + b.close() def test_client_nq(self): self.protocol = 'http' @@ -73,6 +162,9 @@ def test_client_nq(self): c = self.create_client(host='host1', http_port=12) self.assertNotEqual(a, b, 'matched with different hosts') self.assertNotEqual(a, c, 'matched with different ports') + a.close() + b.close() + c.close() def test_client_hash(self): self.protocol = 'http' @@ -81,6 +173,10 @@ def test_client_hash(self): c = self.create_client(host='host2', http_port=11) self.assertEqual(hash(a), hash(b), 'same object has different hashes') self.assertNotEqual(hash(a), hash(c), 'different object has same hash') + a.close() + b.close() + c.close() + if __name__ == '__main__': unittest.main() diff --git a/riak/tests/test_datatypes.py b/riak/tests/test_datatypes.py new file mode 100644 index 00000000..17aa4bf2 --- /dev/null +++ b/riak/tests/test_datatypes.py @@ -0,0 +1,549 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +import unittest +import riak.datatypes as datatypes + +from riak import RiakError, RiakBucket, BucketType, RiakObject +from riak.tests import RUN_DATATYPES +from riak.tests.base import IntegrationTestBase +from riak.tests.comparison import Comparison + + +class DatatypeUnitTestBase(object): + dtype = None + bucket = RiakBucket(None, 'test', BucketType(None, 'datatypes')) + + def op(self, dtype): + raise NotImplementedError + + def check_op_output(self, op): + raise NotImplementedError + + def test_new_type_is_clean(self): + newtype = self.dtype(self.bucket, 'key') + self.assertIsNone(newtype.to_op()) + + def test_modified_type_has_op(self): + newtype = self.dtype(self.bucket, 'key') + self.op(newtype) + self.assertIsNotNone(newtype.to_op()) + + def test_protected_attrs_not_settable(self): + newtype = self.dtype(self.bucket, 'key') + for i in ('value', 'context'): + with self.assertRaises(AttributeError): + setattr(newtype, i, 'foo') + + def test_modified_type_has_unmodified_value(self): + newtype = self.dtype(self.bucket, 'key') + oldvalue = newtype.value + self.op(newtype) + self.assertEqual(oldvalue, newtype.value) + + def test_op_output(self): + newtype = self.dtype(self.bucket, 'key') + self.op(newtype) + op = newtype.to_op() + self.check_op_output(op) + + +class FlagUnitTests(DatatypeUnitTestBase, unittest.TestCase): + dtype = datatypes.Flag + + def op(self, dtype): + dtype.enable() + + def check_op_output(self, op): + self.assertEqual('enable', op) + + def test_disables_require_context(self): + dtype = self.dtype(self.bucket, 'key') + with self.assertRaises(datatypes.ContextRequired): + dtype.disable() + + dtype._context = 'blah' + dtype.disable() + self.assertTrue(dtype.modified) + + +class RegisterUnitTests(DatatypeUnitTestBase, unittest.TestCase): + dtype = datatypes.Register + + def op(self, dtype): + dtype.assign('foobarbaz') + + def check_op_output(self, op): + self.assertEqual(('assign', 'foobarbaz'), op) + + +class CounterUnitTests(DatatypeUnitTestBase, unittest.TestCase): + dtype = datatypes.Counter + + def op(self, dtype): + dtype.increment(5) + + def check_op_output(self, op): + self.assertEqual(('increment', 5), op) + + +class SetUnitTests(DatatypeUnitTestBase, unittest.TestCase, Comparison): + dtype = datatypes.Set + + def op(self, dtype): + dtype._context = "foo" + dtype.add('foo') + dtype.discard('foo') + dtype.add('bar') + + def check_op_output(self, op): + self.assertIn('adds', op) + self.assertItemsEqual(op['adds'], ['bar', 'foo']) + self.assertIn('removes', op) + self.assertIn('foo', op['removes']) + + def test_removes_require_context(self): + dtype = self.dtype(self.bucket, 'key') + with self.assertRaises(datatypes.ContextRequired): + dtype.discard('foo') + dtype._context = 'blah' + dtype.discard('foo') + self.assertTrue(dtype.modified) + + +class HllUnitTests(DatatypeUnitTestBase, unittest.TestCase, Comparison): + dtype = datatypes.Hll + + def op(self, dtype): + dtype._context = 'hll_context' + dtype.add('foo') + dtype.add('bar') + + def check_op_output(self, op): + self.assertIn('adds', op) + self.assertItemsEqual(op['adds'], ['bar', 'foo']) + + +class MapUnitTests(DatatypeUnitTestBase, unittest.TestCase): + dtype = datatypes.Map + + def op(self, dtype): + dtype.counters['a'].increment(2) + dtype.registers['b'].assign('testing') + dtype.flags['c'].enable() + dtype.maps['d'][('e', 'set')].add('deep value') + dtype.maps['f'].counters['g'] + dtype.maps['h'].maps['i'].flags['j'] + + def check_op_output(self, op): + self.assertIn(('update', ('a', 'counter'), ('increment', 2)), op) + self.assertIn(('update', ('b', 'register'), ('assign', 'testing')), op) + self.assertIn(('update', ('c', 'flag'), 'enable'), op) + self.assertIn(('update', ('d', 'map'), [('update', ('e', 'set'), + {'adds': ['deep value']})]), + op) + self.assertNotIn(('update', ('f', 'map'), None), op) + self.assertNotIn(('update', ('h', 'map'), [('update', ('i', 'map'), + None)]), op) + + def test_removes_require_context(self): + dtype = self.dtype(self.bucket, 'key') + with self.assertRaises(datatypes.ContextRequired): + del dtype.sets['foo'] + + with self.assertRaises(datatypes.ContextRequired): + dtype.sets['bar'].discard('xyz') + + with self.assertRaises(datatypes.ContextRequired): + del dtype.maps['baz'].registers['quux'] + + dtype._context = 'blah' + del dtype.sets['foo'] + self.assertTrue(dtype.modified) + + +@unittest.skipUnless(RUN_DATATYPES, 'RUN_DATATYPES is 0') +class HllDatatypeIntegrationTests(IntegrationTestBase, + unittest.TestCase): + @classmethod + def setUpClass(cls): + super(HllDatatypeIntegrationTests, cls).setUpClass() + client = cls.create_client() + try: + btype = client.bucket_type('hlls') + btype.get_properties() + except RiakError as e: + raise unittest.SkipTest(e) + finally: + client.close() + + def test_fetch_bucket_type_props(self): + btype = self.client.bucket_type('hlls') + props = btype.get_properties() + self.assertEqual(14, props['hll_precision']) + + def test_set_same_hll_precision(self): + btype = self.client.bucket_type('hlls') + btype.set_property('hll_precision', 14) + props = btype.get_properties() + self.assertEqual(14, props['hll_precision']) + + def test_set_larger_hll_precision(self): + btype = self.client.bucket_type('hlls') + with self.assertRaises(RiakError): + btype.set_property('hll_precision', 15) + + def test_set_invalid_hll_precision(self): + btype = self.client.bucket_type('hlls') + with self.assertRaises(ValueError): + btype.set_property('hll_precision', 3) + with self.assertRaises(ValueError): + btype.set_property('hll_precision', 17) + with self.assertRaises(ValueError): + btype.set_property('hll_precision', 0) + + def test_dt_hll(self): + btype = self.client.bucket_type('hlls') + props = btype.get_properties() + self.assertEqual(14, props['hll_precision']) + bucket = btype.bucket(self.bucket_name) + myhll = datatypes.Hll(bucket, self.key_name) + myhll.add('user1') + myhll.add('user2') + myhll.add('foo') + myhll.add('bar') + myhll.add('baz') + myhll.add('user1') + self.assertEqual(5, len(myhll._adds)) + + myhll.store() + self.assertEqual(5, myhll.value) + + otherhll = bucket.get(self.key_name) + self.assertEqual(5, otherhll.value) + + +@unittest.skipUnless(RUN_DATATYPES, 'RUN_DATATYPES is 0') +class DatatypeIntegrationTests(IntegrationTestBase, + unittest.TestCase, + Comparison): + def test_dt_counter(self): + btype = self.client.bucket_type('counters') + bucket = btype.bucket(self.bucket_name) + mycount = datatypes.Counter(bucket, self.key_name) + mycount.increment(5) + mycount.store() + + othercount = bucket.get(self.key_name) + self.assertEqual(5, othercount.value) + + othercount.decrement(3) + othercount.store(return_body=True) + + mycount.reload() + self.assertEqual(2, mycount.value) + + def test_dt_set(self): + btype = self.client.bucket_type('sets') + bucket = btype.bucket(self.bucket_name) + myset = datatypes.Set(bucket, self.key_name) + myset.add('Sean') + myset.add('Brett') + myset.store() + + otherset = bucket.get(self.key_name) + + self.assertIn('Sean', otherset) + self.assertIn('Brett', otherset) + + otherset.add('Russell') + otherset.discard('Sean') + otherset.store(return_body=True) + + myset.reload() + self.assertIn('Russell', myset) + self.assertIn('Brett', myset) + self.assertNotIn('Sean', myset) + + def test_dt_map(self): + btype = self.client.bucket_type('maps') + bucket = btype.bucket(self.bucket_name) + mymap = datatypes.Map(bucket, self.key_name) + + mymap.counters['a'].increment(2) + mymap.registers['b'].assign('testing') + mymap.flags['c'].enable() + mymap.maps['d'][('e', 'set')].add('deep value') + mymap.store() + + othermap = bucket.get(self.key_name) + + self.assertIn('a', othermap.counters) + self.assertIn('b', othermap.registers) + self.assertIn('c', othermap.flags) + self.assertIn('d', othermap.maps) + + self.assertEqual(2, othermap.counters['a'].value) + self.assertEqual('testing', othermap.registers['b'].value) + self.assertTrue(othermap.flags['c'].value) + self.assertEqual({('e', 'set'): frozenset(['deep value'])}, + othermap.maps['d'].value) + self.assertEqual(frozenset([]), othermap.sets['f'].value) + + othermap.sets['f'].add('thing1') + othermap.sets['f'].add('thing2') + del othermap.counters['a'] + othermap.store(return_body=True) + + mymap.reload() + self.assertNotIn('a', mymap.counters) + self.assertIn('f', mymap.sets) + self.assertItemsEqual(['thing1', 'thing2'], mymap.sets['f'].value) + + def test_dt_set_remove_without_context(self): + btype = self.client.bucket_type('sets') + bucket = btype.bucket(self.bucket_name) + set = datatypes.Set(bucket, self.key_name) + + set.add("X") + set.add("Y") + set.add("Z") + with self.assertRaises(datatypes.ContextRequired): + set.discard("Y") + + def test_dt_set_remove_fetching_context(self): + btype = self.client.bucket_type('sets') + bucket = btype.bucket(self.bucket_name) + set = datatypes.Set(bucket, self.key_name) + + set.add('X') + set.add('Y') + set.store() + + set.reload() + set.discard('bogus') + set.store() + + set2 = bucket.get(self.key_name) + self.assertItemsEqual(['X', 'Y'], set2.value) + + def test_dt_set_add_twice(self): + btype = self.client.bucket_type('sets') + bucket = btype.bucket(self.bucket_name) + set = datatypes.Set(bucket, self.key_name) + + set.add('X') + set.add('Y') + set.store() + + set.reload() + set.add('X') + set.store() + + set2 = bucket.get(self.key_name) + self.assertItemsEqual(['X', 'Y'], set2.value) + + def test_dt_set_add_wins_in_same_op(self): + btype = self.client.bucket_type('sets') + bucket = btype.bucket(self.bucket_name) + set = datatypes.Set(bucket, self.key_name) + + set.add('X') + set.add('Y') + set.store() + + set.reload() + set.add('X') + set.discard('X') + set.store() + + set2 = bucket.get(self.key_name) + self.assertItemsEqual(['X', 'Y'], set2.value) + + def test_dt_set_add_wins_in_same_op_reversed(self): + btype = self.client.bucket_type('sets') + bucket = btype.bucket(self.bucket_name) + set = datatypes.Set(bucket, self.key_name) + + set.add('X') + set.add('Y') + set.store() + + set.reload() + set.discard('X') + set.add('X') + set.store() + + set2 = bucket.get(self.key_name) + self.assertItemsEqual(['X', 'Y'], set2.value) + + def test_dt_set_remove_old_context(self): + btype = self.client.bucket_type('sets') + bucket = btype.bucket(self.bucket_name) + set = datatypes.Set(bucket, self.key_name) + + set.add('X') + set.add('Y') + set.store() + + set.reload() + + set_parallel = datatypes.Set(bucket, self.key_name) + set_parallel.add('Z') + set_parallel.store() + + set.discard('Z') + set.store() + + set2 = bucket.get(self.key_name) + self.assertItemsEqual(['X', 'Y', 'Z'], set2.value) + + def test_dt_set_remove_updated_context(self): + btype = self.client.bucket_type('sets') + bucket = btype.bucket(self.bucket_name) + set = datatypes.Set(bucket, self.key_name) + + set.add('X') + set.add('Y') + set.store() + + set_parallel = datatypes.Set(bucket, self.key_name) + set_parallel.add('Z') + set_parallel.store() + + set.reload() + set.discard('Z') + set.store() + + set2 = bucket.get(self.key_name) + self.assertItemsEqual(['X', 'Y'], set2.value) + + def test_dt_map_remove_set_update_same_op(self): + btype = self.client.bucket_type('maps') + bucket = btype.bucket(self.bucket_name) + map = datatypes.Map(bucket, self.key_name) + + map.sets['set'].add("X") + map.sets['set'].add("Y") + map.store() + + map.reload() + del map.sets['set'] + map.sets['set'].add("Z") + map.store() + + map2 = bucket.get(self.key_name) + self.assertItemsEqual(["Z"], map2.sets['set']) + + def test_dt_map_remove_counter_increment_same_op(self): + btype = self.client.bucket_type('maps') + bucket = btype.bucket(self.bucket_name) + map = datatypes.Map(bucket, self.key_name) + + map.counters['counter'].increment(5) + map.store() + + map.reload() + self.assertEqual(5, map.counters['counter'].value) + map.counters['counter'].increment(2) + del map.counters['counter'] + map.store() + + map2 = bucket.get(self.key_name) + self.assertEqual(2, map2.counters['counter'].value) + + def test_dt_map_remove_map_update_same_op(self): + btype = self.client.bucket_type('maps') + bucket = btype.bucket(self.bucket_name) + map = datatypes.Map(bucket, self.key_name) + + map.maps['map'].sets['set'].add("X") + map.maps['map'].sets['set'].add("Y") + map.store() + + map.reload() + del map.maps['map'] + map.maps['map'].sets['set'].add("Z") + map.store() + + map2 = bucket.get(self.key_name) + self.assertItemsEqual(["Z"], map2.maps['map'].sets['set']) + + def test_dt_set_return_body_true_default(self): + btype = self.client.bucket_type('sets') + bucket = btype.bucket(self.bucket_name) + myset = bucket.new(self.key_name) + myset.add('X') + myset.store(return_body=False) + with self.assertRaises(datatypes.ContextRequired): + myset.discard('X') + + myset.add('Y') + myset.store() + self.assertItemsEqual(myset.value, ['X', 'Y']) + + myset.discard('X') + myset.store() + self.assertItemsEqual(myset.value, ['Y']) + + def test_dt_map_return_body_true_default(self): + btype = self.client.bucket_type('maps') + bucket = btype.bucket(self.bucket_name) + mymap = bucket.new(self.key_name) + mymap.sets['a'].add('X') + mymap.store(return_body=False) + with self.assertRaises(datatypes.ContextRequired): + mymap.sets['a'].discard('X') + with self.assertRaises(datatypes.ContextRequired): + del mymap.sets['a'] + + mymap.sets['a'].add('Y') + mymap.store() + self.assertItemsEqual(mymap.sets['a'].value, ['X', 'Y']) + + mymap.sets['a'].discard('X') + mymap.store() + self.assertItemsEqual(mymap.sets['a'].value, ['Y']) + + del mymap.sets['a'] + mymap.store() + + self.assertEqual(mymap.value, {}) + + def test_delete_datatype(self): + ctype = self.client.bucket_type('counters') + cbucket = ctype.bucket(self.bucket_name) + counter = cbucket.new(self.key_name) + counter.increment(5) + counter.store() + + stype = self.client.bucket_type('sets') + sbucket = stype.bucket(self.bucket_name) + set_ = sbucket.new(self.key_name) + set_.add("Brett") + set_.store() + + mtype = self.client.bucket_type('maps') + mbucket = mtype.bucket(self.bucket_name) + map_ = mbucket.new(self.key_name) + map_.sets['people'].add('Sean') + map_.store() + + for t in [counter, set_, map_]: + t.delete() + obj = RiakObject(self.client, t.bucket, t.key) + self.client.get(obj) + self.assertFalse(obj.exists, + "{0} exists after deletion".format(t.type_name)) diff --git a/riak/tests/test_datetime.py b/riak/tests/test_datetime.py new file mode 100644 index 00000000..f3367179 --- /dev/null +++ b/riak/tests/test_datetime.py @@ -0,0 +1,44 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +import datetime +import unittest + +from riak.util import epoch, epoch_tz, \ + unix_time_millis + +# NB: without tzinfo, this is UTC +ts0 = datetime.datetime(2015, 1, 1, 12, 1, 2, 987000) +ts0_ts = 1420113662987 +ts0_ts_pst = 1420142462987 + + +class DatetimeUnitTests(unittest.TestCase): + def test_get_unix_time_without_tzinfo(self): + self.assertIsNone(epoch.tzinfo) + self.assertIsNotNone(epoch_tz.tzinfo) + self.assertIsNone(ts0.tzinfo) + utm = unix_time_millis(ts0) + self.assertEqual(utm, ts0_ts) + + def test_get_unix_time_with_tzinfo(self): + try: + import pytz + tz = pytz.timezone('America/Los_Angeles') + ts0_pst = tz.localize(ts0) + utm = unix_time_millis(ts0_pst) + self.assertEqual(utm, ts0_ts_pst) + except ImportError: + pass diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py index 0ab7fde4..bf0c0c7b 100644 --- a/riak/tests/test_feature_detection.py +++ b/riak/tests/test_feature_detection.py @@ -1,27 +1,19 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -import platform - -if platform.python_version() < '2.7': - unittest = __import__('unittest2') -else: - import unittest +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +import unittest from riak.transports.feature_detect import FeatureDetection @@ -42,9 +34,8 @@ class FeatureDetectionTest(unittest.TestCase): def test_implements_server_version(self): t = IncompleteTransport() - def get_server_version(): + with self.assertRaises(NotImplementedError): t.server_version - self.assertRaises(NotImplementedError, get_server_version) def test_pre_10(self): t = DummyTransport("0.14.2") @@ -55,6 +46,15 @@ def test_pre_10(self): self.assertFalse(t.quorum_controls()) self.assertFalse(t.tombstone_vclocks()) self.assertFalse(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) + self.assertFalse(t.stream_indexes()) + self.assertFalse(t.index_term_regex()) + self.assertFalse(t.bucket_types()) + self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) def test_10(self): t = DummyTransport("1.0.3") @@ -65,6 +65,15 @@ def test_10(self): self.assertTrue(t.quorum_controls()) self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) + self.assertFalse(t.stream_indexes()) + self.assertFalse(t.index_term_regex()) + self.assertFalse(t.bucket_types()) + self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) def test_11(self): t = DummyTransport("1.1.4") @@ -75,6 +84,15 @@ def test_11(self): self.assertTrue(t.quorum_controls()) self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) + self.assertFalse(t.stream_indexes()) + self.assertFalse(t.index_term_regex()) + self.assertFalse(t.bucket_types()) + self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) def test_12(self): t = DummyTransport("1.2.0") @@ -85,6 +103,15 @@ def test_12(self): self.assertTrue(t.quorum_controls()) self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) + self.assertFalse(t.stream_indexes()) + self.assertFalse(t.index_term_regex()) + self.assertFalse(t.bucket_types()) + self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) def test_12_loose(self): t = DummyTransport("1.2.1p3") @@ -95,6 +122,91 @@ def test_12_loose(self): self.assertTrue(t.quorum_controls()) self.assertTrue(t.tombstone_vclocks()) self.assertTrue(t.pb_head()) + self.assertFalse(t.pb_clear_bucket_props()) + self.assertFalse(t.pb_all_bucket_props()) + self.assertFalse(t.counters()) + self.assertFalse(t.stream_indexes()) + self.assertFalse(t.index_term_regex()) + self.assertFalse(t.bucket_types()) + self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) + + def test_14(self): + t = DummyTransport("1.4.0rc1") + self.assertTrue(t.phaseless_mapred()) + self.assertTrue(t.pb_indexes()) + self.assertTrue(t.pb_search()) + self.assertTrue(t.pb_conditionals()) + self.assertTrue(t.quorum_controls()) + self.assertTrue(t.tombstone_vclocks()) + self.assertTrue(t.pb_head()) + self.assertTrue(t.pb_clear_bucket_props()) + self.assertTrue(t.pb_all_bucket_props()) + self.assertTrue(t.counters()) + self.assertTrue(t.stream_indexes()) + self.assertFalse(t.index_term_regex()) + self.assertFalse(t.bucket_types()) + self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) + + def test_144(self): + t = DummyTransport("1.4.6") + self.assertTrue(t.phaseless_mapred()) + self.assertTrue(t.pb_indexes()) + self.assertTrue(t.pb_search()) + self.assertTrue(t.pb_conditionals()) + self.assertTrue(t.quorum_controls()) + self.assertTrue(t.tombstone_vclocks()) + self.assertTrue(t.pb_head()) + self.assertTrue(t.pb_clear_bucket_props()) + self.assertTrue(t.pb_all_bucket_props()) + self.assertTrue(t.counters()) + self.assertTrue(t.stream_indexes()) + self.assertTrue(t.index_term_regex()) + self.assertFalse(t.bucket_types()) + self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) + + def test_20(self): + t = DummyTransport("2.0.1") + self.assertTrue(t.phaseless_mapred()) + self.assertTrue(t.pb_indexes()) + self.assertTrue(t.pb_search()) + self.assertTrue(t.pb_conditionals()) + self.assertTrue(t.quorum_controls()) + self.assertTrue(t.tombstone_vclocks()) + self.assertTrue(t.pb_head()) + self.assertTrue(t.pb_clear_bucket_props()) + self.assertTrue(t.pb_all_bucket_props()) + self.assertTrue(t.counters()) + self.assertTrue(t.stream_indexes()) + self.assertTrue(t.index_term_regex()) + self.assertTrue(t.bucket_types()) + self.assertTrue(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) + + def test_21(self): + t = DummyTransport("2.1.0") + self.assertTrue(t.phaseless_mapred()) + self.assertTrue(t.pb_indexes()) + self.assertTrue(t.pb_search()) + self.assertTrue(t.pb_conditionals()) + self.assertTrue(t.quorum_controls()) + self.assertTrue(t.tombstone_vclocks()) + self.assertTrue(t.pb_head()) + self.assertTrue(t.pb_clear_bucket_props()) + self.assertTrue(t.pb_all_bucket_props()) + self.assertTrue(t.counters()) + self.assertTrue(t.stream_indexes()) + self.assertTrue(t.index_term_regex()) + self.assertTrue(t.bucket_types()) + self.assertTrue(t.datatypes()) + self.assertTrue(t.preflists()) + self.assertTrue(t.write_once()) if __name__ == '__main__': diff --git a/riak/tests/test_filters.py b/riak/tests/test_filters.py new file mode 100644 index 00000000..f4a77db0 --- /dev/null +++ b/riak/tests/test_filters.py @@ -0,0 +1,78 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +import unittest + +from riak.mapreduce import RiakKeyFilter +from riak import key_filter + + +class FilterTests(unittest.TestCase): + def test_simple(self): + f1 = RiakKeyFilter("tokenize", "-", 1) + self.assertEqual(f1._filters, [["tokenize", "-", 1]]) + + def test_add(self): + f1 = RiakKeyFilter("tokenize", "-", 1) + f2 = RiakKeyFilter("eq", "2005") + f3 = f1 + f2 + self.assertEqual(list(f3), [["tokenize", "-", 1], ["eq", "2005"]]) + + def test_and(self): + f1 = RiakKeyFilter("starts_with", "2005-") + f2 = RiakKeyFilter("ends_with", "-01") + f3 = f1 & f2 + self.assertEqual(list(f3), + [["and", + [["starts_with", "2005-"]], + [["ends_with", "-01"]]]]) + + def test_multi_and(self): + f1 = RiakKeyFilter("starts_with", "2005-") + f2 = RiakKeyFilter("ends_with", "-01") + f3 = RiakKeyFilter("matches", "-11-") + f4 = f1 & f2 & f3 + self.assertEqual(list(f4), [["and", + [["starts_with", "2005-"]], + [["ends_with", "-01"]], + [["matches", "-11-"]], + ]]) + + def test_or(self): + f1 = RiakKeyFilter("starts_with", "2005-") + f2 = RiakKeyFilter("ends_with", "-01") + f3 = f1 | f2 + self.assertEqual(list(f3), [["or", [["starts_with", "2005-"]], + [["ends_with", "-01"]]]]) + + def test_multi_or(self): + f1 = RiakKeyFilter("starts_with", "2005-") + f2 = RiakKeyFilter("ends_with", "-01") + f3 = RiakKeyFilter("matches", "-11-") + f4 = f1 | f2 | f3 + self.assertEqual(list(f4), [["or", + [["starts_with", "2005-"]], + [["ends_with", "-01"]], + [["matches", "-11-"]], + ]]) + + def test_chaining(self): + f1 = key_filter.tokenize("-", 1).eq("2005") + f2 = key_filter.tokenize("-", 2).eq("05") + f3 = f1 & f2 + self.assertEqual(list(f3), [["and", + [["tokenize", "-", 1], ["eq", "2005"]], + [["tokenize", "-", 2], ["eq", "05"]] + ]]) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 6fb644c1..63206c95 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -1,16 +1,69 @@ # -*- coding: utf-8 -*- -import os -import cPickle +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import copy +import os +import sys +import unittest + +from six import string_types, PY2, PY3 +from time import sleep +from riak import ConflictError, RiakError, ListError +from riak import RiakClient, RiakBucket, BucketType +from riak.resolver import default_resolver, last_written_resolver +from riak.tests import RUN_KV, RUN_RESOLVE, PROTOCOL +from riak.tests.base import IntegrationTestBase +from riak.tests.comparison import Comparison try: import simplejson as json except ImportError: import json +if PY2: + import cPickle + test_pickle_dumps = cPickle.dumps + test_pickle_loads = cPickle.loads +else: + import pickle + test_pickle_dumps = pickle.dumps + test_pickle_loads = pickle.loads + + +testrun_sibs_bucket = 'sibsbucket' +testrun_props_bucket = 'propsbucket' + + +def setUpModule(): + if not RUN_KV: + return + c = IntegrationTestBase.create_client() + c.bucket(testrun_sibs_bucket).allow_mult = True + c.close() -class NotJsonSerializable(object): +def tearDownModule(): + if not RUN_KV: + return + c = IntegrationTestBase.create_client() + c.bucket(testrun_sibs_bucket).clear_properties() + c.bucket(testrun_props_bucket).clear_properties() + c.close() + + +class NotJsonSerializable(object): def __init__(self, *args, **kwargs): self.args = list(args) self.kwargs = kwargs @@ -27,13 +80,73 @@ def __eq__(self, other): value2_args = copy.copy(other.args) value1_args.sort() value2_args.sort() - for i in xrange(len(value1_args)): + for i in range(len(value1_args)): if value1_args[i] != value2_args[i]: return False return True -class BasicKVTests(object): +class KVUnitTests(unittest.TestCase): + def test_list_keys_exception(self): + c = RiakClient() + bt = BucketType(c, 'test') + b = RiakBucket(c, 'test', bt) + with self.assertRaises(ListError): + b.get_keys() + + def test_stream_buckets_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + bs = [] + for bl in c.stream_buckets(): + bs.extend(bl) + + def test_stream_keys_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + ks = [] + for kl in c.stream_keys('test'): + ks.extend(kl) + + def test_ts_stream_keys_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + ks = [] + for kl in c.ts_stream_keys('test'): + ks.extend(kl) + + +@unittest.skipUnless(RUN_KV, 'RUN_KV is 0') +class BasicKVTests(IntegrationTestBase, unittest.TestCase, Comparison): + def test_no_returnbody(self): + bucket = self.client.bucket(self.bucket_name) + o = bucket.new(self.key_name, "bar").store(return_body=False) + self.assertEqual(o.vclock, None) + + @unittest.skipUnless(PROTOCOL == 'pbc', 'Only available on pbc') + def test_get_no_returnbody(self): + bucket = self.client.bucket(self.bucket_name) + o = bucket.new(self.key_name, "Ain't no body") + o.store() + + stored_object = bucket.get(self.key_name, head_only=True) + self.assertFalse(stored_object.data) + + list_of_objects = bucket.multiget([self.key_name], head_only=True) + for stored_object in list_of_objects: + self.assertFalse(stored_object.data) + + def test_many_link_headers_should_work_fine(self): + bucket = self.client.bucket(self.bucket_name) + o = bucket.new("lots_of_links", "My god, it's full of links!") + for i in range(0, 300): + link = ("other", "key%d" % i, "next") + o.add_link(link) + + o.store() + stored_object = bucket.get("lots_of_links") + self.assertEqual(len(stored_object.links), 300) + def test_is_alive(self): self.assertTrue(self.client.is_alive()) @@ -50,18 +163,34 @@ def test_store_and_get(self): # unicode objects are fine, as long as they don't # contain any non-ASCII chars - self.client.bucket(unicode(self.bucket_name)) - self.assertRaises(TypeError, self.client.bucket, u'búcket') - self.assertRaises(TypeError, self.client.bucket, 'búcket') + if PY2: + self.client.bucket(unicode(self.bucket_name)) # noqa + else: + self.client.bucket(self.bucket_name) + if PY2: + self.assertRaises(TypeError, self.client.bucket, u'búcket') + self.assertRaises(TypeError, self.client.bucket, 'búcket') + else: + self.client.bucket(u'búcket') + self.client.bucket('búcket') bucket.get(u'foo') - self.assertRaises(TypeError, bucket.get, u'føø') - self.assertRaises(TypeError, bucket.get, 'føø') - - self.assertRaises(TypeError, bucket.new, u'foo', 'éå') - self.assertRaises(TypeError, bucket.new, u'foo', 'éå') - self.assertRaises(TypeError, bucket.new, 'foo', u'éå') - self.assertRaises(TypeError, bucket.new, 'foo', u'éå') + if PY2: + self.assertRaises(TypeError, bucket.get, u'føø') + self.assertRaises(TypeError, bucket.get, 'føø') + + self.assertRaises(TypeError, bucket.new, u'foo', 'éå') + self.assertRaises(TypeError, bucket.new, u'foo', 'éå') + self.assertRaises(TypeError, bucket.new, 'foo', u'éå') + self.assertRaises(TypeError, bucket.new, 'foo', u'éå') + else: + bucket.get(u'føø') + bucket.get('føø') + + bucket.new(u'foo', 'éå') + bucket.new(u'foo', 'éå') + bucket.new('foo', u'éå') + bucket.new('foo', u'éå') obj2 = bucket.new('baz', rand, 'application/json') obj2.charset = 'UTF-8' @@ -69,20 +198,73 @@ def test_store_and_get(self): obj2 = bucket.get('baz') self.assertEqual(obj2.data, rand) + def test_store_obj_with_unicode(self): + bucket = self.client.bucket(self.bucket_name) + data = {u'føø': u'éå'} + obj = bucket.new('foo', data) + obj.store() + obj = bucket.get('foo') + self.assertEqual(obj.data, data) + + def test_store_unicode_string(self): + bucket = self.client.bucket(self.bucket_name) + data = u"some unicode data: \u00c6" + obj = bucket.new(self.key_name, encoded_data=data.encode('utf-8'), + content_type='text/plain') + obj.charset = 'utf-8' + obj.store() + obj2 = bucket.get(self.key_name) + self.assertEqual(data, obj2.encoded_data.decode('utf-8')) + + def test_string_bucket_name(self): + # Things that are not strings cannot be bucket names + for bad in (12345, True, None, {}, []): + with self.assert_raises_regex(TypeError, 'must be a string'): + self.client.bucket(bad) + + with self.assert_raises_regex(TypeError, 'must be a string'): + RiakBucket(self.client, bad, None) + + # Unicode bucket names are not supported in Python 2.x, + # if they can't be encoded to ASCII. This should be changed in a + # future release. + if PY2: + with self.assert_raises_regex(TypeError, + 'Unicode bucket names ' + 'are not supported'): + self.client.bucket(u'føø') + else: + self.client.bucket(u'føø') + + # This is fine, since it's already ASCII + self.client.bucket('ASCII') + def test_generate_key(self): # Ensure that Riak generates a random key when # the key passed to bucket.new() is None. - bucket = self.client.bucket('random_key_bucket') - existing_keys = bucket.get_keys() + bucket = self.client.bucket(self.bucket_name) o = bucket.new(None, data={}) self.assertIsNone(o.key) o.store() self.assertIsNotNone(o.key) self.assertNotIn('/', o.key) - self.assertNotIn(o.key, existing_keys) - self.assertEqual(len(bucket.get_keys()), len(existing_keys) + 1) + existing_keys = bucket.get_keys() + self.assertEqual(len(existing_keys), 1) + + def maybe_store_keys(self): + skey = 'rkb-init' + bucket = self.client.bucket('random_key_bucket') + sobj = bucket.get(skey) + if sobj.exists: + return + for key in range(1, 1000): + o = bucket.new(None, data={}) + o.store() + o = bucket.new(skey, data={}) + o.store() def test_stream_keys(self): + self.maybe_store_keys() bucket = self.client.bucket('random_key_bucket') regular_keys = bucket.get_keys() self.assertNotEqual(len(regular_keys), 0) @@ -90,11 +272,23 @@ def test_stream_keys(self): for keylist in bucket.stream_keys(): self.assertNotEqual([], keylist) for key in keylist: - self.assertIsInstance(key, basestring) + self.assertIsInstance(key, string_types) streamed_keys += keylist self.assertEqual(sorted(regular_keys), sorted(streamed_keys)) + def test_stream_keys_timeout(self): + self.maybe_store_keys() + bucket = self.client.bucket('random_key_bucket') + streamed_keys = [] + with self.assertRaises(RiakError): + for keylist in self.client.stream_keys(bucket, timeout=1): + self.assertNotEqual([], keylist) + for key in keylist: + self.assertIsInstance(key, string_types) + streamed_keys += keylist + def test_stream_keys_abort(self): + self.maybe_store_keys() bucket = self.client.bucket('random_key_bucket') regular_keys = bucket.get_keys() self.assertNotEqual(len(regular_keys), 0) @@ -106,6 +300,7 @@ def test_stream_keys_abort(self): # If the stream was closed correctly, this will not error robj = bucket.get(regular_keys[0]) + self.assertEqual(len(robj.siblings), 1) self.assertEqual(True, robj.exists) def test_bad_key(self): @@ -124,6 +319,10 @@ def test_binary_store_and_get(self): bucket = self.client.bucket(self.bucket_name) # Store as binary, retrieve as binary, then compare... rand = str(self.randint()) + if PY2: + rand = bytes(rand) + else: + rand = bytes(rand, 'utf-8') obj = bucket.new(self.key_name, encoded_data=rand, content_type='text/plain') obj.store() @@ -136,23 +335,28 @@ def test_binary_store_and_get(self): obj = bucket.new(key2, data) obj.store() obj = bucket.get(key2) - self.assertEqual(data, json.loads(obj.encoded_data)) + self.assertEqual(data, json.loads(obj.encoded_data.decode())) def test_blank_binary_204(self): bucket = self.client.bucket(self.bucket_name) # this should *not* raise an error - obj = bucket.new('foo2', encoded_data='', content_type='text/plain') + empty = "" + if PY2: + empty = bytes(empty) + else: + empty = bytes(empty, 'utf-8') + obj = bucket.new('foo2', encoded_data=empty, content_type='text/plain') obj.store() obj = bucket.get('foo2') self.assertTrue(obj.exists) - self.assertEqual(obj.encoded_data, '') + self.assertEqual(obj.encoded_data, empty) def test_custom_bucket_encoder_decoder(self): bucket = self.client.bucket(self.bucket_name) # Teach the bucket how to pickle - bucket.set_encoder('application/x-pickle', cPickle.dumps) - bucket.set_decoder('application/x-pickle', cPickle.loads) + bucket.set_encoder('application/x-pickle', test_pickle_dumps) + bucket.set_decoder('application/x-pickle', test_pickle_loads) data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} obj = bucket.new(self.key_name, data, 'application/x-pickle') obj.store() @@ -162,8 +366,8 @@ def test_custom_bucket_encoder_decoder(self): def test_custom_client_encoder_decoder(self): bucket = self.client.bucket(self.bucket_name) # Teach the client how to pickle - self.client.set_encoder('application/x-pickle', cPickle.dumps) - self.client.set_decoder('application/x-pickle', cPickle.loads) + self.client.set_encoder('application/x-pickle', test_pickle_dumps) + self.client.set_decoder('application/x-pickle', test_pickle_loads) data = {'array': [1, 2, 3], 'badforjson': NotJsonSerializable(1, 3)} obj = bucket.new(self.key_name, data, 'application/x-pickle') obj.store() @@ -171,9 +375,12 @@ def test_custom_client_encoder_decoder(self): self.assertEqual(data, obj2.data) def test_unknown_content_type_encoder_decoder(self): - # Teach the bucket how to pickle + # Bypass the content_type encoders bucket = self.client.bucket(self.bucket_name) data = "some funny data" + if PY3: + # Python 3.x needs to store binaries + data = data.encode() obj = bucket.new(self.key_name, encoded_data=data, content_type='application/x-frobnicator') @@ -181,11 +388,20 @@ def test_unknown_content_type_encoder_decoder(self): obj2 = bucket.get(self.key_name) self.assertEqual(data, obj2.encoded_data) + def test_text_plain_encoder_decoder(self): + bucket = self.client.bucket(self.bucket_name) + data = "some funny data" + obj = bucket.new(self.key_name, data, content_type='text/plain') + obj.store() + obj2 = bucket.get(self.key_name) + self.assertEqual(data, obj2.data) + def test_missing_object(self): bucket = self.client.bucket(self.bucket_name) obj = bucket.get(self.key_name) self.assertFalse(obj.exists) - self.assertEqual(obj.data, None) + # Object with no siblings should not raise the ConflictError + self.assertIsNone(obj.data) def test_delete(self): bucket = self.client.bucket(self.bucket_name) @@ -210,22 +426,28 @@ def test_bucket_delete(self): self.assertFalse(obj.exists) def test_set_bucket_properties(self): - bucket = self.client.bucket(self.props_bucket) + bucket = self.client.bucket(testrun_props_bucket) # Test setting allow mult... bucket.allow_mult = True # Test setting nval... bucket.n_val = 1 - bucket2 = self.create_client().bucket(self.props_bucket) + c2 = self.create_client() + bucket2 = c2.bucket(testrun_props_bucket) self.assertTrue(bucket2.allow_mult) self.assertEqual(bucket2.n_val, 1) # Test setting multiple properties... bucket.set_properties({"allow_mult": False, "n_val": 2}) - bucket3 = self.create_client().bucket(self.props_bucket) + c3 = self.create_client() + bucket3 = c3.bucket(testrun_props_bucket) self.assertFalse(bucket3.allow_mult) self.assertEqual(bucket3.n_val, 2) + # clean up! + c2.close() + c3.close() + def test_if_none_match(self): bucket = self.client.bucket(self.bucket_name) obj = bucket.get(self.key_name) @@ -243,54 +465,130 @@ def test_if_none_match(self): def test_siblings(self): # Set up the bucket, clear any existing object... - bucket = self.client.bucket(self.sibs_bucket) + bucket = self.client.bucket(testrun_sibs_bucket) obj = bucket.get(self.key_name) bucket.allow_mult = True # Even if it previously existed, let's store a base resolved version # from which we can diverge by sending a stale vclock. - obj.encoded_data = 'start' - obj.content_type = 'application/octet-stream' + obj.data = 'start' + obj.content_type = 'text/plain' obj.store() - # Store the same object five times... - # First run through should overwrite the datum 'start' above - other_client = self.create_client() - other_bucket = other_client.bucket(self.sibs_bucket) + vals = set(self.generate_siblings(obj, count=5)) - vals = set() - for i in range(5): - while True: - randval = self.randint() - if str(randval) not in vals: - break + # Make sure the object has five siblings... + obj = bucket.get(self.key_name) + self.assertEqual(len(obj.siblings), 5) - other_obj = other_bucket.new(key=self.key_name, - encoded_data=str(randval), - content_type='text/plain') - other_obj.vclock = obj.vclock - other_obj.store() - vals.add(str(randval)) + # When the object is in conflict, using the shortcut methods + # should raise the ConflictError + with self.assertRaises(ConflictError): + obj.data + + # Get each of the values - make sure they match what was + # assigned + vals2 = set([sibling.data for sibling in obj.siblings]) + self.assertEqual(vals, vals2) + + # Resolve the conflict, and then do a get... + resolved_sibling = obj.siblings[3] + obj.siblings = [resolved_sibling] + self.assertEqual(len(obj.siblings), 1) + obj.store() + + self.assertEqual(len(obj.siblings), 1) + self.assertEqual(obj.data, resolved_sibling.data) + + @unittest.skipUnless(RUN_RESOLVE, "RUN_RESOLVE is 0") + def test_resolution(self): + bucket = self.client.bucket(testrun_sibs_bucket) + obj = bucket.get(self.key_name) + bucket.allow_mult = True + + # Even if it previously existed, let's store a base resolved version + # from which we can diverge by sending a stale vclock. + obj.data = 'start' + obj.content_type = 'text/plain' + obj.store() + + vals = self.generate_siblings(obj, count=5, delay=1.01) - # Make sure the object has itself plus four siblings... + # Make sure the object has five siblings when using the + # default resolver obj = bucket.get(self.key_name) obj.reload() - self.assertTrue(bool(obj.siblings)) self.assertEqual(len(obj.siblings), 5) - # Get each of the values - make sure they match what was assigned - vals2 = set() - for i in xrange(len(obj.siblings)): - vals2.add(obj.get_sibling(i).encoded_data) - self.assertEqual(vals, vals2) + # Setting the resolver on the client object to use the + # "last-write-wins" behavior + self.client.resolver = last_written_resolver + obj.reload() + self.assertEqual(obj.resolver, last_written_resolver) + self.assertEqual(1, len(obj.siblings)) + self.assertEqual(obj.data, vals[-1]) - # Resolve the conflict, and then do a get... - obj3 = obj.get_sibling(3) - obj3.store() + # Set the resolver on the bucket to the default resolver, + # overriding the resolver on the client + bucket.resolver = default_resolver + obj.reload() + self.assertEqual(obj.resolver, default_resolver) + self.assertEqual(len(obj.siblings), 5) + + # Define our own custom resolver on the object that returns + # the maximum value, overriding the bucket and client resolvers + def max_value_resolver(obj): + obj.siblings = [max(obj.siblings, key=lambda s: s.data), ] + obj.resolver = max_value_resolver obj.reload() - self.assertEqual(len(obj.siblings), 0) - self.assertEqual(obj.encoded_data, obj3.encoded_data) + self.assertEqual(obj.resolver, max_value_resolver) + self.assertEqual(obj.data, max(vals)) + + # Setting the resolver to None on all levels reverts to the + # default resolver. + obj.resolver = None + self.assertEqual(obj.resolver, default_resolver) # set by bucket + bucket.resolver = None + self.assertEqual(obj.resolver, last_written_resolver) # set by client + self.client.resolver = None + self.assertEqual(obj.resolver, default_resolver) # reset + self.assertEqual(bucket.resolver, default_resolver) # reset + self.assertEqual(self.client.resolver, default_resolver) # reset + + @unittest.skipUnless(RUN_RESOLVE, "RUN_RESOLVE is 0") + def test_resolution_default(self): + # If no resolver is setup, be sure to resolve to default_resolver + bucket = self.client.bucket(testrun_sibs_bucket) + self.assertEqual(self.client.resolver, default_resolver) + self.assertEqual(bucket.resolver, default_resolver) + + def test_tombstone_siblings(self): + # Set up the bucket, clear any existing object... + bucket = self.client.bucket(testrun_sibs_bucket) + obj = bucket.get(self.key_name) + bucket.allow_mult = True + + obj.data = 'start' + obj.content_type = 'text/plain' + obj.store(return_body=True) + + obj.delete() + + vals = set(self.generate_siblings(obj, count=4)) + + obj = bucket.get(self.key_name) + + # TODO this used to be 5, only + siblen = len(obj.siblings) + self.assertTrue(siblen == 4 or siblen == 5) + + non_tombstones = 0 + for sib in obj.siblings: + if sib.exists: + non_tombstones += 1 + self.assertTrue(not sib.exists or sib.data in vals) + self.assertEqual(non_tombstones, 4) def test_store_of_missing_object(self): bucket = self.client.bucket(self.bucket_name) @@ -307,11 +605,17 @@ def test_store_of_missing_object(self): # for binary objects o = bucket.get(self.randname()) self.assertEqual(o.exists, False) - o.encoded_data = "1234567890" + if PY2: + o.encoded_data = "1234567890" + else: + o.encoded_data = "1234567890".encode() o.content_type = 'application/octet-stream' o = o.store() - self.assertEqual(o.encoded_data, "1234567890") + if PY2: + self.assertEqual(o.encoded_data, "1234567890") + else: + self.assertEqual(o.encoded_data, "1234567890".encode()) self.assertEqual(o.content_type, "application/octet-stream") o.delete() @@ -330,10 +634,82 @@ def test_list_buckets(self): buckets = self.client.get_buckets() self.assertTrue(self.bucket_name in [x.name for x in buckets]) + def test_stream_buckets(self): + bucket = self.client.bucket(self.bucket_name) + bucket.new(self.key_name, data={"foo": "one", + "bar": "baz"}).store() + buckets = [] + for bucket_list in self.client.stream_buckets(): + buckets.extend(bucket_list) + + self.assertTrue(self.bucket_name in [x.name for x in buckets]) + + def test_stream_buckets_abort(self): + bucket = self.client.bucket(self.bucket_name) + bucket.new(self.key_name, data={"foo": "one", + "bar": "baz"}).store() + try: + for bucket_list in self.client.stream_buckets(): + raise RuntimeError("abort") + except RuntimeError: + pass -class HTTPBucketPropsTest(object): + robj = bucket.get(self.key_name) + self.assertTrue(robj.exists) + self.assertEqual(len(robj.siblings), 1) + + def test_get_params(self): + bucket = self.client.bucket(self.bucket_name) + bucket.new(self.key_name, data={"foo": "one", + "bar": "baz"}).store() + + bucket.get(self.key_name, basic_quorum=False) + bucket.get(self.key_name, basic_quorum=True) + bucket.get(self.key_name, notfound_ok=True) + bucket.get(self.key_name, notfound_ok=False) + + missing = bucket.get('missing-key', notfound_ok=True, + basic_quorum=True) + self.assertFalse(missing.exists) + + def test_preflist(self): + nodes = ['riak@127.0.0.1', 'dev1@127.0.0.1'] + bucket = self.client.bucket(self.bucket_name) + bucket.new(self.key_name, data={"foo": "one", + "bar": "baz"}).store() + try: + preflist = bucket.get_preflist(self.key_name) + preflist2 = self.client.get_preflist(bucket, self.key_name) + for pref in (preflist, preflist2): + self.assertEqual(len(pref), 3) + self.assertIn(pref[0]['node'], nodes) + [self.assertTrue(node['primary']) for node in pref] + except NotImplementedError as e: + raise unittest.SkipTest(e) + + def generate_siblings(self, original, count=5, delay=None): + vals = [] + for _ in range(count): + while True: + randval = str(self.randint()) + if randval not in vals: + break + + other_obj = original.bucket.new(key=original.key, + data=randval, + content_type='text/plain') + other_obj.vclock = original.vclock + other_obj.store() + vals.append(randval) + if delay: + sleep(delay) + return vals + + +@unittest.skipUnless(RUN_KV, 'RUN_KV is 0') +class BucketPropsTest(IntegrationTestBase, unittest.TestCase): def test_rw_settings(self): - bucket = self.client.bucket(self.props_bucket) + bucket = self.client.bucket(testrun_props_bucket) self.assertEqual(bucket.r, "quorum") self.assertEqual(bucket.w, "quorum") self.assertEqual(bucket.dw, "quorum") @@ -358,7 +734,7 @@ def test_rw_settings(self): bucket.clear_properties() def test_primary_quora(self): - bucket = self.client.bucket(self.props_bucket) + bucket = self.client.bucket(testrun_props_bucket) self.assertEqual(bucket.pr, 0) self.assertEqual(bucket.pw, 0) @@ -371,57 +747,36 @@ def test_primary_quora(self): bucket.set_properties({'pr': 0, 'pw': 0}) bucket.clear_properties() + def test_clear_bucket_properties(self): + bucket = self.client.bucket(testrun_props_bucket) + bucket.allow_mult = True + self.assertTrue(bucket.allow_mult) + bucket.n_val = 1 + self.assertEqual(bucket.n_val, 1) + # Test setting clearing properties... -class PbcBucketPropsTest(object): - def test_rw_settings(self): - bucket = self.client.bucket(self.props_bucket) - with self.assertRaises(NotImplementedError): - bucket.r - with self.assertRaises(NotImplementedError): - bucket.w - with self.assertRaises(NotImplementedError): - bucket.dw - with self.assertRaises(NotImplementedError): - bucket.rw - - with self.assertRaises(NotImplementedError): - bucket.r = 2 - with self.assertRaises(NotImplementedError): - bucket.w = 2 - with self.assertRaises(NotImplementedError): - bucket.dw = 2 - with self.assertRaises(NotImplementedError): - bucket.rw = 2 - with self.assertRaises(NotImplementedError): - bucket.clear_properties() - - def test_primary_quora(self): - bucket = self.client.bucket(self.props_bucket) - with self.assertRaises(NotImplementedError): - bucket.pr - with self.assertRaises(NotImplementedError): - bucket.pw - - with self.assertRaises(NotImplementedError): - bucket.pr = 2 - with self.assertRaises(NotImplementedError): - bucket.pw = 2 + self.assertTrue(bucket.clear_properties()) + self.assertFalse(bucket.allow_mult) + self.assertEqual(bucket.n_val, 3) -class KVFileTests(object): +@unittest.skipUnless(RUN_KV, 'RUN_KV is 0') +class KVFileTests(IntegrationTestBase, unittest.TestCase): def test_store_binary_object_from_file(self): bucket = self.client.bucket(self.bucket_name) - filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') - obj = bucket.new_from_file(self.key_name, filepath) + obj = bucket.new_from_file(self.key_name, __file__) obj.store() obj = bucket.get(self.key_name) self.assertNotEqual(obj.encoded_data, None) - self.assertEqual(obj.content_type, "text/x-python") + is_win32 = sys.platform == 'win32' + self.assertTrue(obj.content_type == 'text/x-python' or + (is_win32 and obj.content_type == 'text/plain') or + obj.content_type == 'application/x-python-code') def test_store_binary_object_from_file_should_use_default_mimetype(self): bucket = self.client.bucket(self.bucket_name) filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), - os.pardir, os.pardir, 'THANKS') + os.pardir, os.pardir, 'README.md') obj = bucket.new_from_file(self.key_name, filepath) obj.store() obj = bucket.get(self.key_name) @@ -430,7 +785,38 @@ def test_store_binary_object_from_file_should_use_default_mimetype(self): def test_store_binary_object_from_file_should_fail_if_file_not_found(self): bucket = self.client.bucket(self.bucket_name) with self.assertRaises(IOError): - bucket.new_from_file('not_found_from_file', 'FILE_NOT_FOUND') - obj = bucket.get('not_found_from_file') - self.assertEqual(obj.encoded_data, None) + bucket.new_from_file(self.key_name, 'FILE_NOT_FOUND') + obj = bucket.get(self.key_name) + # self.assertEqual(obj.encoded_data, None) self.assertFalse(obj.exists) + + +@unittest.skipUnless(RUN_KV, 'RUN_KV is 0') +class CounterTests(IntegrationTestBase, unittest.TestCase): + def test_counter_requires_allow_mult(self): + bucket = self.client.bucket(self.bucket_name) + if bucket.allow_mult: + bucket.allow_mult = False + self.assertFalse(bucket.allow_mult) + + with self.assertRaises(Exception): + bucket.update_counter(self.key_name, 10) + + def test_counter_ops(self): + bucket = self.client.bucket(testrun_sibs_bucket) + self.assertTrue(bucket.allow_mult) + + # Non-existent counter has no value + self.assertEqual(None, bucket.get_counter(self.key_name)) + + # Update the counter + bucket.update_counter(self.key_name, 10) + self.assertEqual(10, bucket.get_counter(self.key_name)) + + # Update with returning the value + self.assertEqual(15, bucket.update_counter(self.key_name, 5, + returnvalue=True)) + + # Now try decrementing + self.assertEqual(10, bucket.update_counter(self.key_name, -5, + returnvalue=True)) diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 8f2ad92a..bfdfc7dd 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -1,20 +1,71 @@ # -*- coding: utf-8 -*- - +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest + +from six import PY2 from riak.mapreduce import RiakMapReduce -from riak import key_filter, RiakError +from riak import key_filter, RiakClient, RiakError, ListError +from riak.tests import RUN_MAPREDUCE, RUN_SECURITY, RUN_YZ +from riak.tests.base import IntegrationTestBase +from riak.tests.test_yokozuna import wait_for_yz_index +from riak.tests.yz_setup import yzSetUp, yzTearDown + + +testrun_yz_mr = {'btype': 'mr', + 'bucket': 'mrbucket', + 'index': 'mrbucket'} + + +def setUpModule(): + yzSetUp(testrun_yz_mr) + + +def tearDownModule(): + yzTearDown(testrun_yz_mr) -class LinkTests(object): +class MapReduceUnitTests(unittest.TestCase): + def test_mapred_bucket_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + c.add('bucket') + + +@unittest.skipUnless(RUN_MAPREDUCE, 'RUN_MAPREDUCE is 0') +class LinkTests(IntegrationTestBase, unittest.TestCase): def test_store_and_get_links(self): # Create the object... bucket = self.client.bucket(self.bucket_name) - bucket.new(key="test_store_and_get_links", encoded_data='2', - content_type='application/octet-stream') \ - .add_link(bucket.new("foo1")) \ - .add_link(bucket.new("foo2"), "tag") \ - .add_link(bucket.new("foo3"), "tag2!@#%^&*)") \ - .store() - obj = bucket.get("test_store_and_get_links") + if PY2: + bucket.new(key=self.key_name, encoded_data='2', + content_type='application/octet-stream') \ + .add_link(bucket.new("foo1")) \ + .add_link(bucket.new("foo2"), "tag") \ + .add_link(bucket.new("foo3"), "tag2!@#%^&*)") \ + .store() + else: + bucket.new(key=self.key_name, data='2', + content_type='application/octet-stream') \ + .add_link(bucket.new("foo1")) \ + .add_link(bucket.new("foo2"), "tag") \ + .add_link(bucket.new("foo3"), "tag2!@#%^&*)") \ + .store() + obj = bucket.get(self.key_name) links = obj.links self.assertEqual(len(links), 3) for bucket, key, tag in links: @@ -44,6 +95,9 @@ def test_set_links(self): self.assertEqual(links[2][1], "foo2") self.assertEqual(links[2][2], "tag2") + # "Link walking is deprecated in Riak 2.0 and is not compatible + # with security." + @unittest.skipIf(RUN_SECURITY, 'RUN_SECURITY is set') def test_link_walking(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -59,7 +113,8 @@ def test_link_walking(self): self.assertEqual(len(results), 1) -class ErlangMapReduceTests(object): +@unittest.skipUnless(RUN_MAPREDUCE, 'RUN_MAPREDUCE is 0') +class ErlangMapReduceTests(IntegrationTestBase, unittest.TestCase): def test_erlang_map_reduce(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -76,6 +131,23 @@ def test_erlang_map_reduce(self): .run() self.assertEqual(len(result), 2) + def test_erlang_map_reduce_bucket_type(self): + # Create the object... + btype = self.client.bucket_type('no_siblings') + bucket = btype.bucket(self.bucket_name) + bucket.new("foo", 2).store() + bucket.new("bar", 2).store() + bucket.new("baz", 4).store() + # Run the map... + result = self.client \ + .add(self.bucket_name, "foo", bucket_type='no_siblings') \ + .add(self.bucket_name, "bar", bucket_type='no_siblings') \ + .add(self.bucket_name, "baz", bucket_type='no_siblings') \ + .map(["riak_kv_mapreduce", "map_object_value"]) \ + .reduce(["riak_kv_mapreduce", "reduce_set_union"]) \ + .run() + self.assertEqual(len(result), 2) + def test_erlang_source_map_reduce(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -83,6 +155,7 @@ def test_erlang_source_map_reduce(self): bucket.new("bar", 3).store() bucket.new("baz", 4).store() strfun_allowed = True + result = [] # Run the map... try: result = self.client \ @@ -93,11 +166,41 @@ def test_erlang_source_map_reduce(self): Value = riak_object:get_value(Object), [Value] end.""", {'language': 'erlang'}).run() + except RiakError as e: + if e.value.startswith('May have tried'): + strfun_allowed = False + else: + print("test_erlang_source_map_reduce {}".format(e.value)) + if strfun_allowed: + self.assertIn('2', result) + self.assertIn('3', result) + self.assertIn('4', result) + + def test_erlang_source_map_reduce_bucket_type(self): + # Create the object... + btype = self.client.bucket_type('no_siblings') + bucket = btype.bucket(self.bucket_name) + bucket.new("foo", 2).store() + bucket.new("bar", 3).store() + bucket.new("baz", 4).store() + strfun_allowed = True + # Run the map... + try: + result = self.client \ + .add(self.bucket_name, "foo", bucket_type='no_siblings') \ + .add(self.bucket_name, "bar", bucket_type='no_siblings') \ + .add(self.bucket_name, "baz", bucket_type='no_siblings') \ + .map("""fun(Object, _KD, _A) -> + Value = riak_object:get_value(Object), + [Value] + end.""", {'language': 'erlang'}).run() except RiakError as e: if e.value.startswith('May have tried'): strfun_allowed = False if strfun_allowed: - self.assertEqual(result, ['2', '3', '4']) + self.assertIn('2', result) + self.assertIn('3', result) + self.assertIn('4', result) def test_client_exceptional_paths(self): bucket = self.client.bucket(self.bucket_name) @@ -105,23 +208,25 @@ def test_client_exceptional_paths(self): bucket.new("bar", 2).store() bucket.new("baz", 4).store() - #adding a b-key pair to a bucket input + # adding a b-key pair to a bucket input with self.assertRaises(ValueError): mr = self.client.add(self.bucket_name) mr.add(self.bucket_name, 'bar') - #adding a b-key pair to a query input + # adding a b-key pair to a query input with self.assertRaises(ValueError): mr = self.client.search(self.bucket_name, 'fleh') mr.add(self.bucket_name, 'bar') - #adding a key filter to a query input + # adding a key filter to a query input with self.assertRaises(ValueError): mr = self.client.search(self.bucket_name, 'fleh') mr.add_key_filter("tokenize", "-", 1) -class JSMapReduceTests(object): +@unittest.skipUnless(RUN_MAPREDUCE, 'RUN_MAPREDUCE is 0') +class JSMapReduceTests(IntegrationTestBase, unittest.TestCase): + def test_javascript_source_map(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -135,20 +240,29 @@ def test_javascript_source_map(self): # test ASCII-encodable unicode is accepted mr.map(u"function (v) { return [JSON.parse(v.values[0].data)]; }") - # test non-ASCII-encodable unicode is rejected - self.assertRaises(TypeError, mr.map, - u""" - function (v) { - /* æ */ - return [JSON.parse(v.values[0].data)]; - }""") - - # test non-ASCII-encodable string is rejected - self.assertRaises(TypeError, mr.map, - """function (v) { - /* æ */ - return [JSON.parse(v.values[0].data)]; - }""") + # test non-ASCII-encodable unicode is rejected in Python 2.x + if PY2: + self.assertRaises(TypeError, mr.map, + u""" + function (v) { + /* æ */ + return [JSON.parse(v.values[0].data)]; + }""") + else: + mr = self.client.add(self.bucket_name, "foo") + result = mr.map("""function (v) { + /* æ */ + return [JSON.parse(v.values[0].data)]; + }""").run() + self.assertEqual(result, [2]) + + # test non-ASCII-encodable string is rejected in Python 2.x + if PY2: + self.assertRaises(TypeError, mr.map, + """function (v) { + /* æ */ + return [JSON.parse(v.values[0].data)]; + }""") def test_javascript_named_map(self): # Create the object... @@ -161,6 +275,18 @@ def test_javascript_named_map(self): .run() self.assertEqual(result, [2]) + def test_javascript_named_map_bucket_type(self): + # Create the object... + btype = self.client.bucket_type('no_siblings') + bucket = btype.bucket(self.bucket_name) + bucket.new("foo", 2).store() + # Run the map... + result = self.client \ + .add(self.bucket_name, "foo", bucket_type='no_siblings') \ + .map("Riak.mapValuesJson") \ + .run() + self.assertEqual(result, [2]) + def test_javascript_source_map_reduce(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -177,6 +303,23 @@ def test_javascript_source_map_reduce(self): .run() self.assertEqual(result, [3]) + def test_javascript_source_map_reduce_bucket_type(self): + # Create the object... + btype = self.client.bucket_type('no_siblings') + bucket = btype.bucket(self.bucket_name) + bucket.new("foo", 2).store() + bucket.new("bar", 3).store() + bucket.new("baz", 4).store() + # Run the map... + result = self.client \ + .add(self.bucket_name, "foo", bucket_type='no_siblings') \ + .add(self.bucket_name, "bar", bucket_type='no_siblings') \ + .add(self.bucket_name, "baz", bucket_type='no_siblings') \ + .map("function (v) { return [1]; }") \ + .reduce("Riak.reduceSum") \ + .run() + self.assertEqual(result, [3]) + def test_javascript_named_map_reduce(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -193,6 +336,23 @@ def test_javascript_named_map_reduce(self): .run() self.assertEqual(result, [9]) + def test_javascript_named_map_reduce_bucket_type(self): + # Create the object... + btype = self.client.bucket_type('no_siblings') + bucket = btype.bucket(self.bucket_name) + bucket.new("foo", 2).store() + bucket.new("bar", 3).store() + bucket.new("baz", 4).store() + # Run the map... + result = self.client \ + .add(self.bucket_name, "foo", bucket_type='no_siblings') \ + .add(self.bucket_name, "bar", bucket_type='no_siblings') \ + .add(self.bucket_name, "baz", bucket_type='no_siblings') \ + .map("Riak.mapValuesJson") \ + .reduce("Riak.reduceSum") \ + .run() + self.assertEqual(result, [9]) + def test_javascript_bucket_map_reduce(self): # Create the object... bucket = self.client.bucket("bucket_%s" % self.randint()) @@ -207,6 +367,21 @@ def test_javascript_bucket_map_reduce(self): .run() self.assertEqual(result, [9]) + def test_javascript_bucket_map_reduceP_bucket_type(self): + # Create the object... + btype = self.client.bucket_type('no_siblings') + bucket = btype.bucket("bucket_%s" % self.randint()) + bucket.new("foo", 2).store() + bucket.new("bar", 3).store() + bucket.new("baz", 4).store() + # Run the map... + result = self.client \ + .add(bucket.name, bucket_type='no_siblings') \ + .map("Riak.mapValuesJson") \ + .reduce("Riak.reduceSum") \ + .run() + self.assertEqual(result, [9]) + def test_javascript_arg_map_reduce(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -223,6 +398,23 @@ def test_javascript_arg_map_reduce(self): .run() self.assertEqual(result, [10]) + def test_javascript_arg_map_reduce_bucket_type(self): + # Create the object... + btype = self.client.bucket_type('no_siblings') + bucket = btype.bucket(self.bucket_name) + bucket.new("foo", 2).store() + # Run the map... + result = self.client \ + .add(self.bucket_name, "foo", 5, bucket_type='no_siblings') \ + .add(self.bucket_name, "foo", 10, bucket_type='no_siblings') \ + .add(self.bucket_name, "foo", 15, bucket_type='no_siblings') \ + .add(self.bucket_name, "foo", -15, bucket_type='no_siblings') \ + .add(self.bucket_name, "foo", -5, bucket_type='no_siblings') \ + .map("function(v, arg) { return [arg]; }") \ + .reduce("Riak.reduceSum") \ + .run() + self.assertEqual(result, [10]) + def test_key_filters(self): bucket = self.client.bucket("kftest") bucket.new("basho-20101215", 1).store() @@ -238,6 +430,22 @@ def test_key_filters(self): self.assertEqual(result, ["yahoo-20090613"]) + def test_key_filters_bucket_type(self): + btype = self.client.bucket_type('no_siblings') + bucket = btype.bucket("kftest") + bucket.new("basho-20101215", 1).store() + bucket.new("google-20110103", 2).store() + bucket.new("yahoo-20090613", 3).store() + + result = self.client \ + .add("kftest", bucket_type='no_siblings') \ + .add_key_filters([["tokenize", "-", 2]]) \ + .add_key_filter("ends_with", "0613") \ + .map("function (v, keydata) { return [v.key]; }") \ + .run() + + self.assertEqual(result, ["yahoo-20090613"]) + def test_key_filters_f_chain(self): bucket = self.client.bucket("kftest") bucket.new("basho-20101215", 1).store() @@ -334,17 +542,67 @@ def test_mr_list_add_mix(self): u'"fooval2"', u'"fooval3"']) - -class MapReduceAliasTests(object): + @unittest.skipUnless(RUN_YZ, 'RUN_YZ is 0') + def test_mr_search(self): + """ + Try a successful map/reduce from search results. + """ + btype = self.client.bucket_type(testrun_yz_mr['btype']) + bucket = btype.bucket(testrun_yz_mr['bucket']) + bucket.new("Pebbles", {"name_s": "Fruity Pebbles", + "maker_s": "Post", + "sugar_i": 9, + "calories_i": 110, + "fruit_b": True}).store() + bucket.new("Loops", {"name_s": "Froot Loops", + "maker_s": "Kellogg's", + "sugar_i": 12, + "calories_i": 110, + "fruit_b": True}).store() + bucket.new("Charms", {"name_s": "Lucky Charms", + "maker_s": "General Mills", + "sugar_i": 10, + "calories_i": 110, + "fruit_b": False}).store() + bucket.new("Count", {"name_s": "Count Chocula", + "maker_s": "General Mills", + "sugar_i": 9, + "calories_i": 100, + "fruit_b": False}).store() + bucket.new("Crunch", {"name_s": "Cap'n Crunch", + "maker_s": "Quaker Oats", + "sugar_i": 12, + "calories_i": 110, + "fruit_b": False}).store() + # Wait for Solr to catch up + wait_for_yz_index(bucket, "Crunch") + mr = RiakMapReduce(self.client).search(testrun_yz_mr['bucket'], + 'fruit_b:false') + mr.map("""function(v) { + var solr_doc = JSON.parse(v.values[0].data); + return [solr_doc["calories_i"]]; }""") + result = mr.reduce('function(values, arg) ' + + '{ return [values.sort()[0]]; }').run() + self.assertEqual(result, [100]) + + +@unittest.skipUnless(RUN_MAPREDUCE, 'RUN_MAPREDUCE is 0') +class MapReduceAliasTests(IntegrationTestBase, unittest.TestCase): """This tests the map reduce aliases""" def test_map_values(self): # Add a value to the bucket bucket = self.client.bucket(self.bucket_name) - bucket.new('one', encoded_data='value_1', - content_type='text/plain').store() - bucket.new('two', encoded_data='value_2', - content_type='text/plain').store() + if PY2: + bucket.new('one', encoded_data='value_1', + content_type='text/plain').store() + bucket.new('two', encoded_data='value_2', + content_type='text/plain').store() + else: + bucket.new('one', data='value_1', + content_type='text/plain').store() + bucket.new('two', data='value_2', + content_type='text/plain').store() # Create a map reduce object and use one and two as inputs mr = self.client.add(self.bucket_name, 'one')\ @@ -524,7 +782,8 @@ def test_filter_not_found(self): self.assertEqual(sorted(result), [1, 2]) -class MapReduceStreamTests(object): +@unittest.skipUnless(RUN_MAPREDUCE, 'RUN_MAPREDUCE is 0') +class MapReduceStreamTests(IntegrationTestBase, unittest.TestCase): def test_stream_results(self): bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() @@ -555,4 +814,7 @@ def test_stream_cleanoperationsup(self): # This should not raise an exception obj = bucket.get('one') - self.assertEqual(1, obj.data) + if PY2: + self.assertEqual('1', obj.encoded_data) + else: + self.assertEqual(b'1', obj.encoded_data) diff --git a/riak/tests/test_misc.py b/riak/tests/test_misc.py new file mode 100644 index 00000000..3660720e --- /dev/null +++ b/riak/tests/test_misc.py @@ -0,0 +1,42 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + + +class MiscTests(unittest.TestCase): + def test_timeout_validation(self): + from riak.client.operations import _validate_timeout + # valid cases + try: + _validate_timeout(None) + _validate_timeout(None, infinity_ok=True) + _validate_timeout('infinity', infinity_ok=True) + _validate_timeout(1234) + _validate_timeout(1234567898765432123456789) + except ValueError: + self.fail('_validate_timeout() unexpectedly raised ValueError') + # invalid cases + with self.assertRaises(ValueError): + _validate_timeout('infinity') + with self.assertRaises(ValueError): + _validate_timeout('infinity-foo') + with self.assertRaises(ValueError): + _validate_timeout('foobarbaz') + with self.assertRaises(ValueError): + _validate_timeout('1234') + with self.assertRaises(ValueError): + _validate_timeout(0) + with self.assertRaises(ValueError): + _validate_timeout(12.34) diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index 61793a31..346b2645 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -1,33 +1,34 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -import platform -from Queue import Queue +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +import unittest + +from six import PY2 from threading import Thread, currentThread -from riak.transports.pool import Pool, BadResource from random import SystemRandom from time import sleep -if platform.python_version() < '2.7': - unittest = __import__('unittest2') +from riak import RiakError +from riak.tests import RUN_POOL +from riak.tests.comparison import Comparison +from riak.transports.pool import Pool, BadResource + +if PY2: + from Queue import Queue else: - import unittest -import os + from queue import Queue class SimplePool(Pool): @@ -48,15 +49,30 @@ def create_resource(self): return [] -@unittest.skipIf(os.environ.get('SKIP_POOL'), - 'Skipping connection pool tests') -class PoolTest(unittest.TestCase): +@unittest.skipUnless(RUN_POOL, 'RUN_POOL is 0') +class PoolTest(unittest.TestCase, Comparison): + + def test_can_raise_bad_resource(self): + ex_msg = 'exception-message!' + with self.assertRaises(BadResource) as cm: + raise BadResource(ex_msg) + ex = cm.exception + self.assertEqual(ex.args[0], ex_msg) + + def test_bad_resource_inner_exception(self): + ex_msg = 'exception-message!' + ex = RiakError(ex_msg) + with self.assertRaises(BadResource) as cm: + raise BadResource(ex) + br_ex = cm.exception + self.assertEqual(br_ex.args[0], ex) + def test_yields_new_object_when_empty(self): """ The pool should create new resources as needed. """ pool = SimplePool() - with pool.take() as element: + with pool.transaction() as element: self.assertEqual([1], element) def test_yields_same_object_in_serial_access(self): @@ -66,15 +82,15 @@ def test_yields_same_object_in_serial_access(self): """ pool = SimplePool() - with pool.take() as element: + with pool.transaction() as element: self.assertEqual([1], element) element.append(2) - with pool.take() as element2: - self.assertEqual(1, len(pool.elements)) + with pool.transaction() as element2: + self.assertEqual(1, len(pool.resources)) self.assertEqual([1, 2], element2) - self.assertEqual(1, len(pool.elements)) + self.assertEqual(1, len(pool.resources)) def test_reentrance(self): """ @@ -82,11 +98,11 @@ def test_reentrance(self): while one is already claimed in the same code path. """ pool = SimplePool() - with pool.take() as first: + with pool.transaction() as first: self.assertEqual([1], first) - with pool.take() as second: + with pool.transaction() as second: self.assertEqual([2], second) - with pool.take() as third: + with pool.transaction() as third: self.assertEqual([3], third) def test_unlocks_when_exception_raised(self): @@ -96,12 +112,12 @@ def test_unlocks_when_exception_raised(self): """ pool = SimplePool() try: - with pool.take(): - with pool.take(): + with pool.transaction(): + with pool.transaction(): raise RuntimeError except: - self.assertEqual(2, len(pool.elements)) - for e in pool.elements: + self.assertEqual(2, len(pool.resources)) + for e in pool.resources: self.assertFalse(e.claimed) def test_removes_bad_resource(self): @@ -110,18 +126,18 @@ def test_removes_bad_resource(self): user code throwing a BadResource exception. """ pool = SimplePool() - with pool.take() as element: - self.assertEqual([1], element) - element.append(2) + with pool.transaction() as resource: + self.assertEqual([1], resource) + resource.append(2) try: - with pool.take(): + with pool.transaction(): raise BadResource except BadResource: - self.assertEqual(0, len(pool.elements)) - with pool.take() as goodie: + self.assertEqual(0, len(pool.resources)) + with pool.transaction() as goodie: self.assertEqual([2], goodie) - def test_filter_skips_unmatching_elements(self): + def test_filter_skips_unmatching_resources(self): """ The _filter parameter should cause the pool to yield the first unclaimed resource that passes the filter. @@ -130,11 +146,11 @@ def filtereven(numlist): return numlist[0] % 2 == 0 pool = SimplePool() - with pool.take(): - with pool.take(): + with pool.transaction(): + with pool.transaction(): pass - with pool.take(_filter=filtereven) as f: + with pool.transaction(_filter=filtereven) as f: self.assertEqual([2], f) def test_requires_filter_to_be_callable(self): @@ -146,7 +162,7 @@ def test_requires_filter_to_be_callable(self): pool = SimplePool() with self.assertRaises(TypeError): - with pool.take(_filter=badfilter): + with pool.transaction(_filter=badfilter): pass def test_yields_default_when_empty(self): @@ -155,9 +171,31 @@ def test_yields_default_when_empty(self): resources are free. """ pool = SimplePool() - with pool.take(default='default') as x: + with pool.transaction(default='default') as x: self.assertEqual('default', x) + def test_manual_release(self): + """ + The pool should allow resources to be acquired and released + manually, without giving them out twice. + """ + pool = SimplePool() + a = pool.acquire() + self.assertEqual([1], a.object) + with pool.transaction() as b: + self.assertEqual([2], b) + with pool.transaction() as c: + self.assertEqual([2], c) + pool.release(a) + with pool.transaction() as d: + self.assertEqual([1], d) + e = pool.acquire() + with pool.transaction() as f: + self.assertEqual([2], f) + e.release() + with pool.transaction() as g: + self.assertEqual([1], g) + def test_thread_safety(self): """ The pool should allocate n objects for n concurrent operations. @@ -169,7 +207,7 @@ def test_thread_safety(self): threads = [] def _run(): - with pool.take() as resource: + with pool.transaction() as resource: readyq.put(1) resource.append(currentThread()) finishq.get(True) @@ -190,11 +228,11 @@ def _run(): for thr in threads: thr.join() - self.assertEqual(n, len(pool.elements)) - for element in pool.elements: - self.assertFalse(element.claimed) - self.assertEqual(1, len(element.object)) - self.assertIn(element.object[0], threads) + self.assertEqual(n, len(pool.resources)) + for resource in pool.resources: + self.assertFalse(resource.claimed) + self.assertEqual(1, len(resource.object)) + self.assertIn(resource.object[0], threads) def test_iteration(self): """ @@ -213,7 +251,7 @@ def test_iteration(self): def _run(): psleep = rand.uniform(0.05, 0.1) - with pool.take() as a: + with pool.transaction() as a: started.put(1) started.join() a.append(rand.uniform(0, 1)) @@ -228,13 +266,13 @@ def _run(): started.get() started.task_done() - for element in pool: - touched.append(element) + for resource in pool: + touched.append(resource) for thr in threads: thr.join() - self.assertItemsEqual(pool.elements, touched) + self.assertItemsEqual(pool.resources, touched) def test_clear(self): """ @@ -250,7 +288,7 @@ def test_clear(self): pool = SimplePool() def worker_run(): - with pool.take(): + with pool.transaction(): startq.put(1) startq.join() sleep(rand.uniform(0, 0.5)) @@ -287,11 +325,11 @@ def pusher_run(): t.join() # Make sure that the pool resources are gone - self.assertEqual(0, len(pool.elements)) + self.assertEqual(0, len(pool.resources)) def test_stress(self): """ - Runs a large number of threads doing operations with elements + Runs a large number of threads doing operations with resources checked out, ensuring properties of the pool. """ rand = SystemRandom() @@ -303,7 +341,7 @@ def test_stress(self): def _run(): for i in range(rounds): - with pool.take() as a: + with pool.transaction() as a: self.assertEqual([], a) a.append(currentThread()) self.assertEqual([currentThread()], a) @@ -325,5 +363,6 @@ def _run(): for th in threads: th.join() + if __name__ == '__main__': unittest.main() diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index 25da929d..efc5aa65 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -1,134 +1,169 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # -*- coding: utf-8 -*- -import os -import platform -if platform.python_version() < '2.7': - unittest = __import__('unittest2') -else: - import unittest +from __future__ import print_function + +import unittest + +from riak.tests import RUN_SEARCH, RUN_YZ +from riak.tests.base import IntegrationTestBase + +testrun_search_bucket = 'searchbucket' + -SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '0')) +def setUpModule(): + if RUN_SEARCH and not RUN_YZ: + c = IntegrationTestBase.create_client() + b = c.bucket(testrun_search_bucket) + b.enable_search() + c.close() -class EnableSearchTests(object): - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') +def tearDownModule(): + if RUN_SEARCH and not RUN_YZ: + c = IntegrationTestBase.create_client() + b = c.bucket(testrun_search_bucket) + b.clear_properties() + c.close() + + +@unittest.skipUnless(RUN_SEARCH, 'RUN_SEARCH is 0') +class EnableSearchTests(IntegrationTestBase, unittest.TestCase): def test_bucket_search_enabled(self): bucket = self.client.bucket(self.bucket_name) self.assertFalse(bucket.search_enabled()) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_enable_search_commit_hook(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.clear_properties() - self.assertFalse(self.create_client(). - bucket(self.search_bucket). - search_enabled()) + + c = self.create_client() + self.assertFalse(c.bucket(testrun_search_bucket).search_enabled()) + c.close() + bucket.enable_search() - self.assertTrue(self.create_client(). - bucket(self.search_bucket). - search_enabled()) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + c = self.create_client() + self.assertTrue(c.bucket(testrun_search_bucket).search_enabled()) + c.close() + def test_disable_search_commit_hook(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.clear_properties() bucket.enable_search() - self.assertTrue(self.create_client().bucket(self.search_bucket) - .search_enabled()) + + c = self.create_client() + self.assertTrue(c.bucket(testrun_search_bucket).search_enabled()) + c.close() + bucket.disable_search() - self.assertFalse(self.create_client().bucket(self.search_bucket) - .search_enabled()) + + c = self.create_client() + self.assertFalse(c.bucket(testrun_search_bucket).search_enabled()) + c.close() + bucket.enable_search() -class SolrSearchTests(object): - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') +@unittest.skipUnless(RUN_SEARCH, 'RUN_SEARCH is 0') +class SolrSearchTests(IntegrationTestBase, unittest.TestCase): def test_add_document_to_index(self): - self.client.solr.add(self.search_bucket, - {"id": "doc", "username": "tony"}) - results = self.client.solr.search(self.search_bucket, - "username:tony") - self.assertEquals("tony", results['docs'][0]['username']) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') - def test_add_multiple_documents_to_iindex(self): - self.client.solr.add(self.search_bucket, - {"id": "dizzy", "username": "dizzy"}, - {"id": "russell", "username": "russell"}) - results = self.client.solr\ - .search(self.search_bucket, "username:russell OR username:dizzy") - self.assertEquals(2, len(results['docs'])) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + self.client.fulltext_add(testrun_search_bucket, + [{"id": "doc", "username": "tony"}]) + results = self.client.fulltext_search(testrun_search_bucket, + "username:tony") + self.assertEqual("tony", results['docs'][0]['username']) + + def test_add_multiple_documents_to_index(self): + self.client.fulltext_add( + testrun_search_bucket, + [{"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}]) + results = self.client.fulltext_search( + testrun_search_bucket, "username:russell OR username:dizzy") + self.assertEqual(2, len(results['docs'])) + def test_delete_documents_from_search_by_id(self): - self.client.solr.add(self.search_bucket, - {"id": "dizzy", "username": "dizzy"}, - {"id": "russell", "username": "russell"}) - self.client.solr.delete(self.search_bucket, docs=["dizzy"]) - results = self.client.solr\ - .search(self.search_bucket, "username:russell OR username:dizzy") - self.assertEquals(1, len(results['docs'])) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + self.client.fulltext_add( + testrun_search_bucket, + [{"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}]) + self.client.fulltext_delete(testrun_search_bucket, docs=["dizzy"]) + results = self.client.fulltext_search( + testrun_search_bucket, "username:russell OR username:dizzy") + self.assertEqual(1, len(results['docs'])) + def test_delete_documents_from_search_by_query(self): - self.client.solr.add(self.search_bucket, - {"id": "dizzy", "username": "dizzy"}, - {"id": "russell", "username": "russell"}) - self.client.solr\ - .delete(self.search_bucket, - queries=["username:dizzy", "username:russell"]) - results = self.client.solr\ - .search(self.search_bucket, "username:russell OR username:dizzy") - self.assertEquals(0, len(results['docs'])) - - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + self.client.fulltext_add( + testrun_search_bucket, + [{"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}]) + self.client.fulltext_delete( + testrun_search_bucket, + queries=["username:dizzy", "username:russell"]) + results = self.client.fulltext_search( + testrun_search_bucket, "username:russell OR username:dizzy") + self.assertEqual(0, len(results['docs'])) + def test_delete_documents_from_search_by_query_and_id(self): - self.client.solr.add(self.search_bucket, - {"id": "dizzy", "username": "dizzy"}, - {"id": "russell", "username": "russell"}) - self.client.solr.delete(self.search_bucket, - docs=["dizzy"], - queries=["username:russell"]) - results = self.client.solr\ - .search(self.search_bucket, - "username:russell OR username:dizzy") - self.assertEquals(0, len(results['docs'])) - - -class SearchTests(object): - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + self.client.fulltext_add( + testrun_search_bucket, + [{"id": "dizzy", "username": "dizzy"}, + {"id": "russell", "username": "russell"}]) + self.client.fulltext_delete( + testrun_search_bucket, + docs=["dizzy"], + queries=["username:russell"]) + results = self.client.fulltext_search( + testrun_search_bucket, + "username:russell OR username:dizzy") + self.assertEqual(0, len(results['docs'])) + + +@unittest.skipUnless(RUN_SEARCH, 'RUN_SEARCH is 0') +class SearchTests(IntegrationTestBase, unittest.TestCase): def test_solr_search_from_bucket(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage") - self.assertEquals(1, len(results['docs'])) + self.assertEqual(1, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params_from_bucket(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage", wt="xml") - self.assertEquals(1, len(results['docs'])) + self.assertEqual(1, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr.search(self.search_bucket, - "username:roidrage", wt="xml") - self.assertEquals(1, len(results['docs'])) + results = self.client.fulltext_search( + testrun_search_bucket, + "username:roidrage", wt="xml") + self.assertEqual(1, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() - results = self.client.solr.search(self.search_bucket, - "username:roidrage") - self.assertEquals(1, len(results["docs"])) + results = self.client.fulltext_search(testrun_search_bucket, + "username:roidrage") + self.assertEqual(1, len(results["docs"])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_search_integration(self): # Create some objects to search across... - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("one", {"foo": "one", "bar": "red"}).store() bucket.new("two", {"foo": "two", "bar": "green"}).store() bucket.new("three", {"foo": "three", "bar": "blue"}).store() @@ -136,17 +171,17 @@ def test_search_integration(self): bucket.new("five", {"foo": "five", "bar": "yellow"}).store() # Run some operations... - results = self.client.solr.search(self.search_bucket, - "foo:one OR foo:two") + results = self.client.fulltext_search(testrun_search_bucket, + "foo:one OR foo:two") if (len(results) == 0): - print "\n\nNot running test \"testSearchIntegration()\".\n" - print """Please ensure that you have installed the Riak + print("\n\nNot running test \"testSearchIntegration()\".\n") + print("""Please ensure that you have installed the Riak Search hook on bucket \"searchbucket\" by running - \"bin/search-cmd install searchbucket\".\n\n""" + \"bin/search-cmd install searchbucket\".\n\n""") return self.assertEqual(len(results['docs']), 2) query = "(foo:one OR foo:two OR foo:three OR foo:four) AND\ (NOT bar:green)" - results = self.client.solr.search(self.search_bucket, query) + results = self.client.fulltext_search(testrun_search_bucket, query) self.assertEqual(len(results['docs']), 3) diff --git a/riak/tests/test_security.py b/riak/tests/test_security.py new file mode 100644 index 00000000..d9e1ee10 --- /dev/null +++ b/riak/tests/test_security.py @@ -0,0 +1,168 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +import sys +import unittest + +from riak.tests import RUN_SECURITY, SECURITY_USER, SECURITY_PASSWD, \ + SECURITY_CACERT, SECURITY_KEY, SECURITY_CERT, SECURITY_REVOKED, \ + SECURITY_CERT_USER, SECURITY_BAD_CERT, SECURITY_CIPHERS +from riak.security import SecurityCreds +from riak.tests.base import IntegrationTestBase + + +class SecurityTests(IntegrationTestBase, unittest.TestCase): + @unittest.skipIf(RUN_SECURITY, 'RUN_SECURITY is 1') + def test_security_disabled(self): + """ + Test valid security settings without security enabled + """ + topts = {'timeout': 1} + # NB: can't use SECURITY_CREDS here since they won't be set + # if RUN_SECURITY is UN-set + creds = SecurityCreds(username='foo', password='bar') + client = self.create_client(credentials=creds, + transport_options=topts) + myBucket = client.bucket('test') + val1 = "foobar" + key1 = myBucket.new('x', data=val1) + with self.assertRaises(Exception): + key1.store() + client.close() + + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') + def test_security_basic_connection(self): + myBucket = self.client.bucket('test') + val1 = "foobar" + key1 = myBucket.new('x', data=val1) + key1.store() + myBucket.get('x') + + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') + def test_security_bad_user(self): + creds = SecurityCreds(username='foo', + password=SECURITY_PASSWD, + cacert_file=SECURITY_CACERT, + ciphers=SECURITY_CIPHERS) + client = self.create_client(credentials=creds) + with self.assertRaises(Exception): + client.get_buckets() + client.close() + + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') + def test_security_bad_password(self): + creds = SecurityCreds(username=SECURITY_USER, + password='foo', + cacert_file=SECURITY_CACERT, + ciphers=SECURITY_CIPHERS) + client = self.create_client(credentials=creds) + with self.assertRaises(Exception): + client.get_buckets() + client.close() + + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') + def test_security_invalid_cert(self): + creds = SecurityCreds(username=SECURITY_USER, + password=SECURITY_PASSWD, + cacert_file='/tmp/foo', + ciphers=SECURITY_CIPHERS) + client = self.create_client(credentials=creds) + with self.assertRaises(Exception): + client.get_buckets() + client.close() + + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') + def test_security_password_without_cacert(self): + creds = SecurityCreds(username=SECURITY_USER, + password=SECURITY_PASSWD, + ciphers=SECURITY_CIPHERS) + client = self.create_client(credentials=creds) + with self.assertRaises(Exception): + myBucket = client.bucket('test') + val1 = "foobar" + key1 = myBucket.new('x', data=val1) + key1.store() + client.close() + + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') + def test_security_cert_authentication(self): + creds = SecurityCreds(username=SECURITY_CERT_USER, + ciphers=SECURITY_CIPHERS, + cert_file=SECURITY_CERT, + pkey_file=SECURITY_KEY, + cacert_file=SECURITY_CACERT) + client = self.create_client(credentials=creds) + myBucket = client.bucket('test') + val1 = "foobar2" + key1 = myBucket.new('x', data=val1) + # Certificate Authentication is currently only supported + # by Protocol Buffers + if self.protocol == 'pbc': + key1.store() + myBucket.get('x') + else: + with self.assertRaises(Exception): + key1.store() + myBucket.get('x') + client.close() + + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') + def test_security_revoked_cert(self): + creds = SecurityCreds(username=SECURITY_USER, + password=SECURITY_PASSWD, + ciphers=SECURITY_CIPHERS, + cacert_file=SECURITY_CACERT, + crl_file=SECURITY_REVOKED) + # Currently Python >= 2.7.9 and Python 3.x native CRL doesn't seem to + # work as advertised + if sys.version_info >= (2, 7, 9): + return + client = self.create_client(credentials=creds) + with self.assertRaises(Exception): + client.get_buckets() + client.close() + + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') + def test_security_bad_ca_cert(self): + creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, + ciphers=SECURITY_CIPHERS, + cacert_file=SECURITY_BAD_CERT) + client = self.create_client(credentials=creds) + with self.assertRaises(Exception): + client.get_buckets() + client.close() + + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') + def test_security_ciphers(self): + creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, + ciphers=SECURITY_CIPHERS, + cacert_file=SECURITY_CACERT) + client = self.create_client(credentials=creds) + myBucket = client.bucket('test') + val1 = "foobar" + key1 = myBucket.new('x', data=val1) + key1.store() + myBucket.get('x') + client.close() + + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') + def test_security_bad_ciphers(self): + creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, + cacert_file=SECURITY_CACERT, + ciphers='ECDHE-RSA-AES256-GCM-SHA384') + client = self.create_client(credentials=creds) + with self.assertRaises(Exception): + client.get_buckets() + client.close() diff --git a/riak/tests/test_server_test.py b/riak/tests/test_server_test.py index 98826d2c..2b5cfc48 100644 --- a/riak/tests/test_server_test.py +++ b/riak/tests/test_server_test.py @@ -1,7 +1,24 @@ -from riak.test_server import TestServer +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys import unittest +from riak.test_server import TestServer + +@unittest.skipIf(sys.platform == 'win32', 'Windows is not supported') class TestServerTestCase(unittest.TestCase): def setUp(self): self.test_server = TestServer() @@ -10,44 +27,44 @@ def tearDown(self): pass def test_options_defaults(self): - self.assertEquals( + self.assertEqual( self.test_server.app_config["riak_core"]["handoff_port"], 9001) - self.assertEquals( + self.assertEqual( self.test_server.app_config["riak_kv"]["pb_ip"], "127.0.0.1") def test_merge_riak_core_options(self): self.test_server = TestServer(riak_core={"handoff_port": 10000}) - self.assertEquals( + self.assertEqual( self.test_server.app_config["riak_core"]["handoff_port"], 10000) def test_merge_riak_search_options(self): self.test_server = TestServer( riak_search={"search_backend": "riak_search_backend"}) - self.assertEquals( + self.assertEqual( self.test_server.app_config["riak_search"]["search_backend"], "riak_search_backend") def test_merge_riak_kv_options(self): self.test_server = TestServer(riak_kv={"pb_ip": "192.168.2.1"}) - self.assertEquals(self.test_server.app_config["riak_kv"]["pb_ip"], - "192.168.2.1") + self.assertEqual(self.test_server.app_config["riak_kv"]["pb_ip"], + "192.168.2.1") def test_merge_vmargs(self): self.test_server = TestServer(vm_args={"-P": 65000}) - self.assertEquals(self.test_server.vm_args["-P"], 65000) + self.assertEqual(self.test_server.vm_args["-P"], 65000) def test_set_ring_state_dir(self): - self.assertEquals( + self.assertEqual( self.test_server.app_config["riak_core"]["ring_state_dir"], "/tmp/riak/test_server/data/ring") def test_set_default_tmp_dir(self): - self.assertEquals(self.test_server.temp_dir, "/tmp/riak/test_server") + self.assertEqual(self.test_server.temp_dir, "/tmp/riak/test_server") def test_set_non_default_tmp_dir(self): tmp_dir = '/not/the/default/dir' server = TestServer(tmp_dir=tmp_dir) - self.assertEquals(server.temp_dir, tmp_dir) + self.assertEqual(server.temp_dir, tmp_dir) def suite(): diff --git a/riak/tests/test_timeseries_pbuf.py b/riak/tests/test_timeseries_pbuf.py new file mode 100644 index 00000000..8cffa1c7 --- /dev/null +++ b/riak/tests/test_timeseries_pbuf.py @@ -0,0 +1,511 @@ +# -*- coding: utf-8 -*- +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import six +import unittest + +import riak.pb.riak_ts_pb2 +from riak.pb.riak_ts_pb2 import TsColumnType + +from riak import RiakError +from riak.codecs.pbuf import PbufCodec +from riak.table import Table +from riak.tests import RUN_TIMESERIES +from riak.tests.base import IntegrationTestBase +from riak.ts_object import TsObject +from riak.util import str_to_bytes, bytes_to_str, \ + unix_time_millis, datetime_from_unix_time_millis, \ + is_timeseries_supported + +table_name = 'GeoCheckin' + +bd0 = '时间序列' +bd1 = 'временные ряды' + +blob0 = b'\x00\x01\x02\x03\x04\x05\x06\x07' + +fiveMins = datetime.timedelta(0, 300) +# NB: last arg is microseconds, 987ms expressed +ts0 = datetime.datetime(2015, 1, 1, 12, 0, 0, 987000) +ex0ms = 1420113600987 + +ts1 = ts0 + fiveMins +ex1ms = 1420113900987 + + +@unittest.skipUnless(is_timeseries_supported(), + 'Timeseries not supported by this Python version') +class TimeseriesPbufUnitTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ts0ms = unix_time_millis(ts0) + if cls.ts0ms != ex0ms: + raise AssertionError( + 'expected {:d} to equal {:d}'.format(cls.ts0ms, ex0ms)) + + cls.ts1ms = unix_time_millis(ts1) + if cls.ts1ms != ex1ms: + raise AssertionError( + 'expected {:d} to equal {:d}'.format(cls.ts1ms, ex1ms)) + + cls.rows = [ + [bd0, 0, 1.2, ts0, True, None], + [bd1, 3, 4.5, ts1, False, blob0] + ] + cls.test_key = ['hash1', 'user2', ts0] + cls.table = Table(None, table_name) + + def validate_keyreq(self, req): + self.assertEqual(self.table.name, bytes_to_str(req.table)) + self.assertEqual(len(self.test_key), len(req.key)) + self.assertEqual('hash1', bytes_to_str(req.key[0].varchar_value)) + self.assertEqual('user2', bytes_to_str(req.key[1].varchar_value)) + self.assertEqual(self.ts0ms, req.key[2].timestamp_value) + + def test_encode_decode_timestamp(self): + ts0ms = unix_time_millis(ts0) + self.assertEqual(ts0ms, ex0ms) + ts0_d = datetime_from_unix_time_millis(ts0ms) + self.assertEqual(ts0, ts0_d) + + def test_encode_data_for_get(self): + c = PbufCodec() + msg = c.encode_timeseries_keyreq( + self.table, self.test_key, is_delete=False) + req = riak.pb.riak_ts_pb2.TsGetReq() + req.ParseFromString(msg.data) + self.validate_keyreq(req) + + def test_encode_data_for_delete(self): + c = PbufCodec() + msg = c.encode_timeseries_keyreq( + self.table, self.test_key, is_delete=True) + req = riak.pb.riak_ts_pb2.TsDelReq() + req.ParseFromString(msg.data) + self.validate_keyreq(req) + + def test_encode_data_for_put(self): + c = PbufCodec() + tsobj = TsObject(None, self.table, self.rows, None) + msg = c.encode_timeseries_put(tsobj) + req = riak.pb.riak_ts_pb2.TsPutReq() + req.ParseFromString(msg.data) + + # NB: expected, actual + self.assertEqual(self.table.name, bytes_to_str(req.table)) + self.assertEqual(len(self.rows), len(req.rows)) + + r0 = req.rows[0] + self.assertEqual(bytes_to_str(r0.cells[0].varchar_value), + self.rows[0][0]) + self.assertEqual(r0.cells[1].sint64_value, self.rows[0][1]) + self.assertEqual(r0.cells[2].double_value, self.rows[0][2]) + self.assertEqual(r0.cells[3].timestamp_value, self.ts0ms) + self.assertEqual(r0.cells[4].boolean_value, self.rows[0][4]) + self.assertFalse(r0.cells[5].HasField('varchar_value')) + + r1 = req.rows[1] + self.assertEqual(bytes_to_str(r1.cells[0].varchar_value), + self.rows[1][0]) + self.assertEqual(r1.cells[1].sint64_value, self.rows[1][1]) + self.assertEqual(r1.cells[2].double_value, self.rows[1][2]) + self.assertEqual(r1.cells[3].timestamp_value, self.ts1ms) + self.assertEqual(r1.cells[4].boolean_value, self.rows[1][4]) + self.assertEqual(r1.cells[5].varchar_value, self.rows[1][5]) + + def test_encode_data_for_listkeys(self): + c = PbufCodec(client_timeouts=True) + msg = c.encode_timeseries_listkeysreq(self.table, 1234) + req = riak.pb.riak_ts_pb2.TsListKeysReq() + req.ParseFromString(msg.data) + self.assertEqual(self.table.name, bytes_to_str(req.table)) + self.assertEqual(1234, req.timeout) + + def test_decode_data_from_query(self): + tqr = riak.pb.riak_ts_pb2.TsQueryResp() + + c0 = tqr.columns.add() + c0.name = str_to_bytes('col_varchar') + c0.type = TsColumnType.Value('VARCHAR') + c1 = tqr.columns.add() + c1.name = str_to_bytes('col_integer') + c1.type = TsColumnType.Value('SINT64') + c2 = tqr.columns.add() + c2.name = str_to_bytes('col_double') + c2.type = TsColumnType.Value('DOUBLE') + c3 = tqr.columns.add() + c3.name = str_to_bytes('col_timestamp') + c3.type = TsColumnType.Value('TIMESTAMP') + c4 = tqr.columns.add() + c4.name = str_to_bytes('col_boolean') + c4.type = TsColumnType.Value('BOOLEAN') + c5 = tqr.columns.add() + c5.name = str_to_bytes('col_blob') + c5.type = TsColumnType.Value('BLOB') + + r0 = tqr.rows.add() + r0c0 = r0.cells.add() + r0c0.varchar_value = str_to_bytes(self.rows[0][0]) + r0c1 = r0.cells.add() + r0c1.sint64_value = self.rows[0][1] + r0c2 = r0.cells.add() + r0c2.double_value = self.rows[0][2] + r0c3 = r0.cells.add() + r0c3.timestamp_value = self.ts0ms + r0c4 = r0.cells.add() + r0c4.boolean_value = self.rows[0][4] + r0.cells.add() + + r1 = tqr.rows.add() + r1c0 = r1.cells.add() + r1c0.varchar_value = str_to_bytes(self.rows[1][0]) + r1c1 = r1.cells.add() + r1c1.sint64_value = self.rows[1][1] + r1c2 = r1.cells.add() + r1c2.double_value = self.rows[1][2] + r1c3 = r1.cells.add() + r1c3.timestamp_value = self.ts1ms + r1c4 = r1.cells.add() + r1c4.boolean_value = self.rows[1][4] + r1c5 = r1.cells.add() + r1c5.varchar_value = self.rows[1][5] + + tsobj = TsObject(None, self.table) + c = PbufCodec() + c.decode_timeseries(tqr, tsobj, True) + + self.assertEqual(len(tsobj.rows), len(self.rows)) + self.assertEqual(len(tsobj.columns.names), len(tqr.columns)) + self.assertEqual(len(tsobj.columns.types), len(tqr.columns)) + + cn, ct = tsobj.columns + self.assertEqual(cn[0], 'col_varchar') + self.assertEqual(ct[0], 'varchar') + self.assertEqual(cn[1], 'col_integer') + self.assertEqual(ct[1], 'sint64') + self.assertEqual(cn[2], 'col_double') + self.assertEqual(ct[2], 'double') + self.assertEqual(cn[3], 'col_timestamp') + self.assertEqual(ct[3], 'timestamp') + self.assertEqual(cn[4], 'col_boolean') + self.assertEqual(ct[4], 'boolean') + self.assertEqual(cn[5], 'col_blob') + self.assertEqual(ct[5], 'blob') + + r0 = tsobj.rows[0] + self.assertEqual(bytes_to_str(r0[0]), self.rows[0][0]) + self.assertEqual(r0[1], self.rows[0][1]) + self.assertEqual(r0[2], self.rows[0][2]) + self.assertEqual(r0[3], ts0) + self.assertEqual(r0[4], self.rows[0][4]) + self.assertEqual(r0[5], self.rows[0][5]) + + r1 = tsobj.rows[1] + self.assertEqual(bytes_to_str(r1[0]), self.rows[1][0]) + self.assertEqual(r1[1], self.rows[1][1]) + self.assertEqual(r1[2], self.rows[1][2]) + self.assertEqual(r1[3], ts1) + self.assertEqual(r1[4], self.rows[1][4]) + self.assertEqual(r1[5], self.rows[1][5]) + + +@unittest.skipUnless(is_timeseries_supported() and RUN_TIMESERIES, + 'Timeseries not supported by this Python version' + ' or RUN_TIMESERIES is 0') +class TimeseriesPbufTests(IntegrationTestBase, unittest.TestCase): + client_options = {'transport_options': + {'use_ttb': False, 'ts_convert_timestamp': True}} + + @classmethod + def setUpClass(cls): + super(TimeseriesPbufTests, cls).setUpClass() + cls.now = datetime.datetime.utcfromtimestamp(144379690.987000) + fiveMinsAgo = cls.now - fiveMins + tenMinsAgo = fiveMinsAgo - fiveMins + fifteenMinsAgo = tenMinsAgo - fiveMins + twentyMinsAgo = fifteenMinsAgo - fiveMins + twentyFiveMinsAgo = twentyMinsAgo - fiveMins + + client = cls.create_client() + table = client.table(table_name) + rows = [ + ['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3], + ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], + ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], + ['hash1', 'user2', fiveMinsAgo, 'wind', None], + ['hash1', 'user2', cls.now, 'snow', 20.1] + ] + try: + ts_obj = table.new(rows) + result = ts_obj.store() + except (RiakError, NotImplementedError) as e: + raise unittest.SkipTest(e) + finally: + client.close() + if result is not True: + raise AssertionError("expected success") + + cls.nowMsec = unix_time_millis(cls.now) + cls.fiveMinsAgo = fiveMinsAgo + cls.twentyMinsAgo = twentyMinsAgo + cls.twentyFiveMinsAgo = twentyFiveMinsAgo + cls.tenMinsAgoMsec = unix_time_millis(tenMinsAgo) + cls.twentyMinsAgoMsec = unix_time_millis(twentyMinsAgo) + cls.numCols = len(rows[0]) + cls.rows = rows + encoded_rows = [ + [str_to_bytes('hash1'), str_to_bytes('user2'), + twentyFiveMinsAgo, str_to_bytes('typhoon'), 90.3], + [str_to_bytes('hash1'), str_to_bytes('user2'), + twentyMinsAgo, str_to_bytes('hurricane'), 82.3], + [str_to_bytes('hash1'), str_to_bytes('user2'), + fifteenMinsAgo, str_to_bytes('rain'), 79.0], + [str_to_bytes('hash1'), str_to_bytes('user2'), + fiveMinsAgo, str_to_bytes('wind'), None], + [str_to_bytes('hash1'), str_to_bytes('user2'), + cls.now, str_to_bytes('snow'), 20.1] + ] + cls.encoded_rows = encoded_rows + + def validate_len(self, ts_obj, elen): + if isinstance(elen, tuple): + self.assertIn(len(ts_obj.columns.names), elen) + self.assertIn(len(ts_obj.columns.types), elen) + self.assertIn(len(ts_obj.rows), elen) + else: + self.assertEqual(len(ts_obj.columns.names), elen) + self.assertEqual(len(ts_obj.columns.types), elen) + self.assertEqual(len(ts_obj.rows), elen) + + def validate_data(self, ts_obj): + if ts_obj.columns is not None: + self.assertEqual(len(ts_obj.columns.names), self.numCols) + self.assertEqual(len(ts_obj.columns.types), self.numCols) + self.assertEqual(len(ts_obj.rows), 1) + row = ts_obj.rows[0] + self.assertEqual(bytes_to_str(row[0]), 'hash1') + self.assertEqual(bytes_to_str(row[1]), 'user2') + self.assertEqual(row[2], self.fiveMinsAgo) + self.assertEqual(row[2].microsecond, 987000) + self.assertEqual(bytes_to_str(row[3]), 'wind') + self.assertIsNone(row[4]) + + def test_insert_data_via_sql(self): + query = """ + INSERT INTO GeoCheckin_Wide + (geohash, user, time, weather, temperature, uv_index, observed) + VALUES + ('hash3', 'user3', 1460203200000, 'tornado', 43.5, 128, True); + """ + ts_obj = self.client.ts_query('GeoCheckin_Wide', query) + self.assertIsNotNone(ts_obj) + self.validate_len(ts_obj, 0) + + def test_query_that_creates_table_using_interpolation(self): + table = self.randname() + query = """CREATE TABLE test-{table} ( + geohash varchar not null, + user varchar not null, + time timestamp not null, + weather varchar not null, + temperature double, + PRIMARY KEY((geohash, user, quantum(time, 15, m)), + geohash, user, time)) + """ + ts_obj = self.client.ts_query(table, query) + self.assertIsNotNone(ts_obj) + self.validate_len(ts_obj, 0) + + def test_query_that_returns_table_description(self): + fmt = 'DESCRIBE {table}' + query = fmt.format(table=table_name) + ts_obj = self.client.ts_query(table_name, query) + self.assertIsNotNone(ts_obj) + self.validate_len(ts_obj, (5, 7, 8)) + + def test_query_that_returns_table_description_using_interpolation(self): + query = 'Describe {table}' + ts_obj = self.client.ts_query(table_name, query) + self.assertIsNotNone(ts_obj) + self.validate_len(ts_obj, (5, 7, 8)) + + def test_query_description_via_table(self): + query = 'describe {table}' + table = Table(self.client, table_name) + ts_obj = table.query(query) + self.assertIsNotNone(ts_obj) + self.validate_len(ts_obj, (5, 7, 8)) + + def test_get_description(self): + ts_obj = self.client.ts_describe(table_name) + self.assertIsNotNone(ts_obj) + self.validate_len(ts_obj, (5, 7, 8)) + + def test_get_description_via_table(self): + table = Table(self.client, table_name) + ts_obj = table.describe() + self.assertIsNotNone(ts_obj) + self.validate_len(ts_obj, (5, 7, 8)) + + def test_query_that_returns_no_data(self): + fmt = """ + select * from {table} where + time > 0 and time < 10 and + geohash = 'hash1' and + user = 'user1' + """ + query = fmt.format(table=table_name) + ts_obj = self.client.ts_query(table_name, query) + self.validate_len(ts_obj, 0) + + def test_query_that_returns_no_data_using_interpolation(self): + query = """ + select * from {table} where + time > 0 and time < 10 and + geohash = 'hash1' and + user = 'user1' + """ + ts_obj = self.client.ts_query(table_name, query) + self.validate_len(ts_obj, 0) + + def test_query_that_matches_some_data(self): + fmt = """ + select * from {table} where + time > {t1} and time < {t2} and + geohash = 'hash1' and + user = 'user2' + """ + query = fmt.format( + table=table_name, + t1=self.tenMinsAgoMsec, + t2=self.nowMsec) + ts_obj = self.client.ts_query(table_name, query) + self.validate_data(ts_obj) + + def test_query_that_matches_some_data_using_interpolation(self): + fmt = """ + select * from {{table}} where + time > {t1} and time < {t2} and + geohash = 'hash1' and + user = 'user2' + """ + query = fmt.format( + t1=self.tenMinsAgoMsec, + t2=self.nowMsec) + ts_obj = self.client.ts_query(table_name, query) + self.validate_data(ts_obj) + + def test_query_that_matches_more_data(self): + fmt = """ + select * from {table} where + time >= {t1} and time <= {t2} and + geohash = 'hash1' and + user = 'user2' + """ + query = fmt.format( + table=table_name, + t1=self.twentyMinsAgoMsec, + t2=self.nowMsec) + ts_obj = self.client.ts_query(table_name, query) + j = 0 + for i, want in enumerate(self.encoded_rows): + if want[2] == self.twentyFiveMinsAgo: + continue + got = ts_obj.rows[j] + j += 1 + self.assertListEqual(got, want) + + def test_get_with_invalid_key(self): + key = ['hash1', 'user2'] + with self.assertRaises(RiakError): + self.client.ts_get(table_name, key) + + def test_get_single_value(self): + key = ['hash1', 'user2', self.fiveMinsAgo] + ts_obj = self.client.ts_get(table_name, key) + self.assertIsNotNone(ts_obj) + self.validate_data(ts_obj) + + def test_get_single_value_via_table(self): + key = ['hash1', 'user2', self.fiveMinsAgo] + table = Table(self.client, table_name) + ts_obj = table.get(key) + self.assertIsNotNone(ts_obj) + self.validate_data(ts_obj) + + def test_stream_keys(self): + table = Table(self.client, table_name) + streamed_keys = [] + for keylist in table.stream_keys(): + self.validate_keylist(streamed_keys, keylist) + self.assertGreater(len(streamed_keys), 0) + + def test_stream_keys_from_string_table(self): + streamed_keys = [] + for keylist in self.client.ts_stream_keys(table_name): + self.validate_keylist(streamed_keys, keylist) + self.assertGreater(len(streamed_keys), 0) + + def validate_keylist(self, streamed_keys, keylist): + self.assertNotEqual([], keylist) + streamed_keys += keylist + for key in keylist: + self.assertIsInstance(key, list) + self.assertEqual(len(key), 3) + self.assertEqual(bytes_to_str(key[0]), 'hash1') + self.assertEqual(bytes_to_str(key[1]), 'user2') + self.assertIsInstance(key[2], datetime.datetime) + + def test_delete_single_value(self): + key = ['hash1', 'user2', self.twentyFiveMinsAgo] + rslt = self.client.ts_delete(table_name, key) + self.assertTrue(rslt) + ts_obj = self.client.ts_get(table_name, key) + self.assertIsNotNone(ts_obj) + self.assertEqual(len(ts_obj.rows), 0) + self.assertEqual(len(ts_obj.columns.names), 0) + self.assertEqual(len(ts_obj.columns.types), 0) + + def test_create_error_via_put(self): + table = Table(self.client, table_name) + ts_obj = table.new([]) + with self.assertRaises(RiakError): + ts_obj.store() + + def test_store_and_fetch_gh_483(self): + now = datetime.datetime(2015, 1, 1, 12, 0, 0) + table = self.client.table(table_name) + rows = [ + ['hash1', 'user2', now, 'frazzle', 12.3] + ] + + ts_obj = table.new(rows) + result = ts_obj.store() + self.assertTrue(result) + + k = ['hash1', 'user2', now] + ts_obj = self.client.ts_get(table_name, k) + self.assertIsNotNone(ts_obj) + ts_cols = ts_obj.columns + self.assertEqual(len(ts_cols.names), 5) + self.assertEqual(len(ts_cols.types), 5) + self.assertEqual(len(ts_obj.rows), 1) + + row = ts_obj.rows[0] + self.assertEqual(len(row), 5) + exp = [six.b('hash1'), six.b('user2'), now, + six.b('frazzle'), 12.3] + self.assertEqual(row, exp) diff --git a/riak/tests/test_timeseries_ttb.py b/riak/tests/test_timeseries_ttb.py new file mode 100644 index 00000000..d2434799 --- /dev/null +++ b/riak/tests/test_timeseries_ttb.py @@ -0,0 +1,308 @@ +# -*- coding: utf-8 -*- +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import six +import unittest + +from erlastic import decode, encode +from erlastic.types import Atom + +from riak import RiakError +from riak.table import Table +from riak.tests import RUN_TIMESERIES +from riak.ts_object import TsObject +from riak.codecs.ttb import TtbCodec +from riak.util import str_to_bytes, bytes_to_str, \ + unix_time_millis, is_timeseries_supported +from riak.tests.base import IntegrationTestBase + +rpberrorresp_a = Atom('rpberrorresp') +tsgetreq_a = Atom('tsgetreq') +tsgetresp_a = Atom('tsgetresp') +tsputreq_a = Atom('tsputreq') + +udef_a = Atom('undefined') +varchar_a = Atom('varchar') +sint64_a = Atom('sint64') +double_a = Atom('double') +timestamp_a = Atom('timestamp') +boolean_a = Atom('boolean') + +table_name = 'GeoCheckin' + +str0 = 'ascii-0' +str1 = 'ascii-1' + +bd0 = six.u('时间序列') +bd1 = six.u('временные ряды') + +blob0 = b'\x00\x01\x02\x03\x04\x05\x06\x07' + +fiveMins = datetime.timedelta(0, 300) +ts0 = datetime.datetime(2015, 1, 1, 12, 1, 2, 987000) +ts1 = ts0 + fiveMins + + +@unittest.skipUnless(is_timeseries_supported(), + 'Timeseries not supported by this Python version') +class TimeseriesTtbUnitTests(unittest.TestCase): + def setUp(self): + self.table = Table(None, table_name) + + def test_encode_data_for_get(self): + keylist = [ + str_to_bytes('hash1'), str_to_bytes('user2'), unix_time_millis(ts0) + ] + req = tsgetreq_a, str_to_bytes(table_name), keylist, udef_a + req_test = encode(req) + + test_key = ['hash1', 'user2', ts0] + c = TtbCodec() + msg = c.encode_timeseries_keyreq(self.table, test_key) + self.assertEqual(req_test, msg.data) + + # {tsgetresp, + # { + # [<<"geohash">>, <<"user">>, <<"time">>, + # <<"weather">>, <<"temperature">>, <<"blob">>], + # [varchar, varchar, timestamp, varchar, double, blob], + # [(<<"hash1">>, <<"user2">>, 144378190987, <<"typhoon">>, + # 90.3, <<0,1,2,3,4,5,6,7>>)] + # } + # } + def test_decode_data_from_get(self): + colnames = ["varchar", "sint64", "double", "timestamp", + "boolean", "varchar", "varchar", "blob"] + coltypes = [varchar_a, sint64_a, double_a, timestamp_a, + boolean_a, varchar_a, varchar_a] + r0 = (bd0, 0, 1.2, unix_time_millis(ts0), True, + [], str1, None, None) + r1 = (bd1, 3, 4.5, unix_time_millis(ts1), False, + [], str1, None, blob0) + rows = [r0, r1] + # { tsgetresp, { [colnames], [coltypes], [rows] } } + data_t = colnames, coltypes, rows + rsp_data = tsgetresp_a, data_t + rsp_ttb = encode(rsp_data) + + tsobj = TsObject(None, self.table) + c = TtbCodec() + c.decode_timeseries(decode(rsp_ttb), tsobj) + + for i in range(0, 1): + dr = rows[i] + r = tsobj.rows[i] # encoded + self.assertEqual(r[0], dr[0].encode('utf-8')) + self.assertEqual(r[1], dr[1]) + self.assertEqual(r[2], dr[2]) + # NB *not* decoding timestamps + # dt = datetime_from_unix_time_millis(dr[3]) + self.assertEqual(r[3], dr[3]) + if i == 0: + self.assertEqual(r[4], True) + else: + self.assertEqual(r[4], False) + self.assertEqual(r[5], None) + self.assertEqual(r[6], dr[6].encode('ascii')) + self.assertEqual(r[7], None) + self.assertEqual(r[8], dr[8]) + + def test_encode_data_for_put(self): + r0 = (bd0, 0, 1.2, unix_time_millis(ts0), True, []) + r1 = (bd1, 3, 4.5, unix_time_millis(ts1), False, []) + rows = [r0, r1] + req = tsputreq_a, str_to_bytes(table_name), [], rows + req_test = encode(req) + + rows_to_encode = [ + [bd0, 0, 1.2, ts0, True, None], + [bd1, 3, 4.5, ts1, False, None] + ] + + tsobj = TsObject(None, self.table, rows_to_encode, None) + c = TtbCodec() + msg = c.encode_timeseries_put(tsobj) + self.assertEqual(req_test, msg.data) + + +@unittest.skipUnless(is_timeseries_supported() and RUN_TIMESERIES, + 'Timeseries not supported by this Python version' + ' or RUN_TIMESERIES is 0') +class TimeseriesTtbTests(IntegrationTestBase, unittest.TestCase): + client_options = {'transport_options': + {'use_ttb': True, 'ts_convert_timestamp': True}} + + @classmethod + def setUpClass(cls): + super(TimeseriesTtbTests, cls).setUpClass() + client = cls.create_client() + skey = 'test-key' + btype = client.bucket_type(table_name) + bucket = btype.bucket(table_name) + try: + bucket.get(skey) + except (RiakError, NotImplementedError) as e: + raise unittest.SkipTest(e) + finally: + client.close() + + def validate_len(self, ts_obj, elen): + if isinstance(elen, tuple): + self.assertIn(len(ts_obj.columns.names), elen) + self.assertIn(len(ts_obj.columns.types), elen) + self.assertIn(len(ts_obj.rows), elen) + else: + self.assertEqual(len(ts_obj.columns.names), elen) + self.assertEqual(len(ts_obj.columns.types), elen) + self.assertEqual(len(ts_obj.rows), elen) + + def test_insert_data_via_sql(self): + query = """ + INSERT INTO GeoCheckin_Wide + (geohash, user, time, weather, temperature, uv_index, observed) + VALUES + ('hash3', 'user3', 1460203200000, 'tornado', 43.5, 128, True); + """ + ts_obj = self.client.ts_query('GeoCheckin_Wide', query) + self.assertIsNotNone(ts_obj) + self.validate_len(ts_obj, 0) + + def test_query_that_creates_table_using_interpolation(self): + table = self.randname() + query = """CREATE TABLE test-{table} ( + geohash varchar not null, + user varchar not null, + time timestamp not null, + weather varchar not null, + temperature double, + PRIMARY KEY((geohash, user, quantum(time, 15, m)), + geohash, user, time)) + """ + ts_obj = self.client.ts_query(table, query) + self.assertIsNotNone(ts_obj) + self.assertFalse(hasattr(ts_obj, 'ts_cols')) + self.assertIsNone(ts_obj.rows) + + def test_query_that_returns_table_description(self): + fmt = 'DESCRIBE {table}' + query = fmt.format(table=table_name) + ts_obj = self.client.ts_query(table_name, query) + self.assertIsNotNone(ts_obj) + self.validate_len(ts_obj, (5, 7, 8)) + + def test_store_and_fetch_gh_483(self): + now = datetime.datetime(2015, 1, 1, 12, 0, 0) + table = self.client.table(table_name) + rows = [ + ['hash1', 'user2', now, 'frazzle', 12.3] + ] + + ts_obj = table.new(rows) + result = ts_obj.store() + self.assertTrue(result) + + k = ['hash1', 'user2', now] + ts_obj = self.client.ts_get(table_name, k) + self.assertIsNotNone(ts_obj) + ts_cols = ts_obj.columns + self.assertEqual(len(ts_cols.names), 5) + self.assertEqual(len(ts_cols.types), 5) + self.assertEqual(len(ts_obj.rows), 1) + + row = ts_obj.rows[0] + self.assertEqual(len(row), 5) + exp = [six.b('hash1'), six.b('user2'), now, + six.b('frazzle'), 12.3] + self.assertEqual(row, exp) + + def test_store_and_fetch_and_query(self): + now = datetime.datetime.utcfromtimestamp(144379690.987000) + fiveMinsAgo = now - fiveMins + tenMinsAgo = fiveMinsAgo - fiveMins + fifteenMinsAgo = tenMinsAgo - fiveMins + twentyMinsAgo = fifteenMinsAgo - fiveMins + twentyFiveMinsAgo = twentyMinsAgo - fiveMins + + table = self.client.table(table_name) + rows = [ + ['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3], + ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], + ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], + ['hash1', 'user2', fiveMinsAgo, 'wind', None], + ['hash1', 'user2', now, 'snow', 20.1] + ] + # NB: response data is binary + exp_rows = [ + [six.b('hash1'), six.b('user2'), twentyFiveMinsAgo, + six.b('typhoon'), 90.3], + [six.b('hash1'), six.b('user2'), twentyMinsAgo, + six.b('hurricane'), 82.3], + [six.b('hash1'), six.b('user2'), fifteenMinsAgo, + six.b('rain'), 79.0], + [six.b('hash1'), six.b('user2'), fiveMinsAgo, + six.b('wind'), None], + [six.b('hash1'), six.b('user2'), now, + six.b('snow'), 20.1] + ] + ts_obj = table.new(rows) + result = ts_obj.store() + self.assertTrue(result) + + for i, r in enumerate(rows): + k = r[0:3] + ts_obj = self.client.ts_get(table_name, k) + self.assertIsNotNone(ts_obj) + ts_cols = ts_obj.columns + self.assertEqual(len(ts_cols.names), 5) + self.assertEqual(len(ts_cols.types), 5) + self.assertEqual(len(ts_obj.rows), 1) + row = ts_obj.rows[0] + exp = exp_rows[i] + self.assertEqual(len(row), 5) + self.assertEqual(row, exp) + + fmt = """ + select * from {table} where + time > {t1} and time < {t2} and + geohash = 'hash1' and + user = 'user2' + """ + query = fmt.format( + table=table_name, + t1=unix_time_millis(tenMinsAgo), + t2=unix_time_millis(now)) + ts_obj = self.client.ts_query(table_name, query) + if ts_obj.columns is not None: + self.assertEqual(len(ts_obj.columns.names), 5) + self.assertEqual(len(ts_obj.columns.types), 5) + self.assertEqual(len(ts_obj.rows), 1) + row = ts_obj.rows[0] + self.assertEqual(bytes_to_str(row[0]), 'hash1') + self.assertEqual(bytes_to_str(row[1]), 'user2') + self.assertEqual(row[2], fiveMinsAgo) + self.assertEqual(row[2].microsecond, 987000) + self.assertEqual(bytes_to_str(row[3]), 'wind') + self.assertIsNone(row[4]) + + def test_create_error_via_put(self): + table = Table(self.client, table_name) + ts_obj = table.new([]) + with self.assertRaises(RiakError) as cm: + ts_obj.store() + logging.debug( + "[test_timeseries_ttb] saw exception: {}" + .format(cm.exception)) diff --git a/riak/tests/test_util.py b/riak/tests/test_util.py new file mode 100644 index 00000000..766c82fa --- /dev/null +++ b/riak/tests/test_util.py @@ -0,0 +1,94 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import unittest + +from riak.util import is_timeseries_supported, \ + datetime_from_unix_time_millis, \ + unix_time_millis + + +class UtilUnitTests(unittest.TestCase): + # NB: + # 144379690 secs, 987 msecs past epoch + # 144379690987 total msecs past epoch + def test_conv_ms_timestamp_to_datetime_and_back(self): + if is_timeseries_supported(): + # this is what would be stored in Riak TS + v = 144379690987 + dt = datetime_from_unix_time_millis(v) + + # This is how Python represents the above + utp = 144379690.987000 + dtp = datetime.datetime.utcfromtimestamp(utp) + self.assertEqual(dt, dtp) + + utm = unix_time_millis(dt) + self.assertEqual(v, utm) + else: + pass + + def test_conv_datetime_to_unix_millis(self): + # This is the "native" Python unix timestamp including + # microseconds, as float. timedelta "total_seconds()" + # returns a value like this + if is_timeseries_supported(): + v = 144379690.987000 + d = datetime.datetime.utcfromtimestamp(v) + utm = unix_time_millis(d) + self.assertEqual(utm, 144379690987) + else: + pass + + def test_unix_millis_validation(self): + v = 144379690.987 + with self.assertRaises(ValueError): + datetime_from_unix_time_millis(v) + + def test_unix_millis_small_value(self): + if is_timeseries_supported(): + # this is what would be stored in Riak TS + v = 1001 + dt = datetime_from_unix_time_millis(v) + + # This is how Python represents the above + utp = 1.001 + dtp = datetime.datetime.utcfromtimestamp(utp) + self.assertEqual(dt, dtp) + + utm = unix_time_millis(dt) + self.assertEqual(v, utm) + else: + pass + + def test_is_timeseries_supported(self): + v = (2, 7, 10) + self.assertEqual(True, is_timeseries_supported(v)) + v = (2, 7, 11) + self.assertEqual(True, is_timeseries_supported(v)) + v = (2, 7, 12) + self.assertEqual(True, is_timeseries_supported(v)) + v = (3, 3, 6) + self.assertEqual(False, is_timeseries_supported(v)) + v = (3, 4, 3) + self.assertEqual(False, is_timeseries_supported(v)) + v = (3, 4, 4) + self.assertEqual(True, is_timeseries_supported(v)) + v = (3, 4, 5) + self.assertEqual(True, is_timeseries_supported(v)) + v = (3, 5, 0) + self.assertEqual(False, is_timeseries_supported(v)) + v = (3, 5, 1) + self.assertEqual(True, is_timeseries_supported(v)) diff --git a/riak/tests/test_yokozuna.py b/riak/tests/test_yokozuna.py new file mode 100644 index 00000000..a1823774 --- /dev/null +++ b/riak/tests/test_yokozuna.py @@ -0,0 +1,220 @@ +# -*- coding: utf-8 -*- +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from riak.tests import RUN_YZ +from riak.tests.base import IntegrationTestBase +from riak.tests.comparison import Comparison +from riak.tests.yz_setup import yzSetUp, yzTearDown + + +def wait_for_yz_index(bucket, key, index=None): + """ + Wait until Solr index has been updated and a value returns from a query. + + :param bucket: Bucket to which indexed value is written + :type bucket: RiakBucket + :param key: Key to which value was written + :type key: str + """ + while len(bucket.search('_yz_rk:' + key, index=index)['docs']) == 0: + pass + + +# YZ index on bucket of the same name +testrun_yz = {'btype': None, 'bucket': 'yzbucket', 'index': 'yzbucket'} +# YZ index on bucket of a different name +testrun_yz_index = {'btype': None, + 'bucket': 'yzindexbucket', + 'index': 'yzindex'} + + +def setUpModule(): + yzSetUp(testrun_yz, testrun_yz_index) + + +def tearDownModule(): + yzTearDown(testrun_yz, testrun_yz_index) + + +@unittest.skipUnless(RUN_YZ, 'RUN_YZ is 0') +class YZSearchTests(IntegrationTestBase, unittest.TestCase, Comparison): + def test_yz_search_from_bucket(self): + bucket = self.client.bucket(testrun_yz['bucket']) + bucket.new("user", {"user_s": "Z"}).store() + wait_for_yz_index(bucket, "user") + results = bucket.search("user_s:Z") + self.assertEqual(1, len(results['docs'])) + # TODO: check that docs return useful info + result = results['docs'][0] + self.assertIn('_yz_rk', result) + self.assertEqual(u'user', result['_yz_rk']) + self.assertIn('_yz_rb', result) + self.assertEqual(testrun_yz['bucket'], result['_yz_rb']) + self.assertIn('score', result) + self.assertIn('user_s', result) + self.assertEqual(u'Z', result['user_s']) + + def test_yz_search_index_using_bucket(self): + bucket = self.client.bucket(testrun_yz_index['bucket']) + bucket.new("feliz", + {"name_s": "Felix", "species_s": "Felis catus"}).store() + wait_for_yz_index(bucket, "feliz", index=testrun_yz_index['index']) + results = bucket.search('name_s:Felix', + index=testrun_yz_index['index']) + self.assertEqual(1, len(results['docs'])) + + def test_yz_search_index_using_wrong_bucket(self): + bucket = self.client.bucket(testrun_yz_index['bucket']) + bucket.new("feliz", + {"name_s": "Felix", "species_s": "Felis catus"}).store() + wait_for_yz_index(bucket, "feliz", index=testrun_yz_index['index']) + with self.assertRaises(Exception): + bucket.search('name_s:Felix') + + def test_yz_get_search_index(self): + index = self.client.get_search_index(testrun_yz['bucket']) + self.assertEqual(testrun_yz['bucket'], index['name']) + self.assertEqual('_yz_default', index['schema']) + self.assertEqual(3, index['n_val']) + with self.assertRaises(Exception): + self.client.get_search_index('NOT' + testrun_yz['bucket']) + + def test_yz_delete_search_index(self): + # expected to fail, since there's an attached bucket + with self.assertRaises(Exception): + self.client.delete_search_index(testrun_yz['bucket']) + # detatch bucket from index then delete + b = self.client.bucket(testrun_yz['bucket']) + b.set_property('search_index', '_dont_index_') + self.assertTrue(self.client.delete_search_index(testrun_yz['bucket'])) + # create it again + self.client.create_search_index(testrun_yz['bucket'], '_yz_default', 3) + b = self.client.bucket(testrun_yz['bucket']) + b.set_property('search_index', testrun_yz['bucket']) + # Wait for index to apply + indexes = [] + while testrun_yz['bucket'] not in indexes: + indexes = [i['name'] for i in self.client.list_search_indexes()] + + def test_yz_list_search_indexes(self): + indexes = self.client.list_search_indexes() + self.assertIn(testrun_yz['bucket'], [item['name'] for item in indexes]) + self.assertLessEqual(1, len(indexes)) + + def test_yz_create_schema(self): + content = """ + + + + + + + + + + + + + _yz_id + + + + """ + schema_name = self.randname() + self.assertTrue(self.client.create_search_schema(schema_name, content)) + schema = self.client.get_search_schema(schema_name) + self.assertEqual(schema_name, schema['name']) + self.assertEqual(content, schema['content']) + + def test_yz_create_bad_schema(self): + bad_content = """ + `. + """ + def _server_version(self): """ Gets the server version from the server. To be implemented by the individual transport class. - :rtype string + + :rtype: string """ raise NotImplementedError def phaseless_mapred(self): """ Whether MapReduce requests can be submitted without phases. - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.1] @@ -48,14 +61,23 @@ def pb_indexes(self): Whether secondary index queries are supported over Protocol Buffers - :rtype bool + :rtype: bool """ return self.server_version >= versions[1.2] + def pb_search_admin(self): + """ + Whether search administration is supported over Protocol Buffers + + :rtype: bool + """ + return self.server_version >= versions[2.0] + def pb_search(self): """ Whether search queries are supported over Protocol Buffers - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1.2] @@ -63,7 +85,8 @@ def pb_conditionals(self): """ Whether conditional fetch/store semantics are supported over Protocol Buffers - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1] @@ -71,14 +94,16 @@ def quorum_controls(self): """ Whether additional quorums and FSM controls are available, e.g. primary quorums, basic_quorum, notfound_ok - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1] def tombstone_vclocks(self): """ Whether 'not found' responses might include vclocks - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1] @@ -86,10 +111,101 @@ def pb_head(self): """ Whether partial-fetches (vclock and metadata only) are supported over Protocol Buffers - :rtype bool + + :rtype: bool """ return self.server_version >= versions[1] + def pb_clear_bucket_props(self): + """ + Whether bucket properties can be cleared over Protocol + Buffers. + + :rtype: bool + """ + return self.server_version >= versions[1.4] + + def pb_all_bucket_props(self): + """ + Whether all normal bucket properties are supported over + Protocol Buffers. + + :rtype: bool + """ + return self.server_version >= versions[1.4] + + def counters(self): + """ + Whether CRDT counters are supported. + + :rtype: bool + """ + return self.server_version >= versions[1.4] + + def bucket_stream(self): + """ + Whether streaming bucket lists are supported. + + :rtype: bool + """ + return self.server_version >= versions[1.4] + + def client_timeouts(self): + """ + Whether client-supplied timeouts are supported. + + :rtype: bool + """ + return self.server_version >= versions[1.4] + + def stream_indexes(self): + """ + Whether secondary indexes support streaming responses. + + :rtype: bool + """ + return self.server_version >= versions[1.4] + + def index_term_regex(self): + """ + Whether secondary indexes supports a regexp term filter. + + :rtype: bool + """ + return self.server_version >= versions[1.44] + + def bucket_types(self): + """ + Whether bucket-types are supported. + + :rtype: bool + """ + return self.server_version >= versions[2.0] + + def datatypes(self): + """ + Whether datatypes are supported. + + :rtype: bool + """ + return self.server_version >= versions[2.0] + + def preflists(self): + """ + Whether bucket/key preflists are supported. + + :rtype: bool + """ + return self.server_version >= versions[2.1] + + def write_once(self): + """ + Whether write-once operations are supported. + + :rtype: bool + """ + return self.server_version >= versions[2.1] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index 50ed56bb..68797b7b 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -1,57 +1,182 @@ -""" -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -import httplib +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import socket +import select + +from six import PY2 +from riak.security import SecurityError, USE_STDLIB_SSL from riak.transports.pool import Pool -from riak.transports.http.transport import RiakHttpTransport +from riak.transports.http.transport import HttpTransport + +if USE_STDLIB_SSL: + import ssl + from riak.transports.security import configure_ssl_context +else: + import OpenSSL.SSL + from riak.transports.security import RiakWrappedSocket,\ + configure_pyopenssl_context + +if PY2: + from httplib import HTTPConnection, \ + NotConnected, \ + IncompleteRead, \ + ImproperConnectionState, \ + BadStatusLine, \ + HTTPSConnection +else: + from http.client import HTTPConnection, \ + HTTPSConnection, \ + NotConnected, \ + IncompleteRead, \ + ImproperConnectionState, \ + BadStatusLine + + +class NoNagleHTTPConnection(HTTPConnection): + """ + Setup a connection class which does not use Nagle - deal with + latency on PUT requests lower than MTU + """ + def connect(self): + """ + Set TCP_NODELAY on socket + """ + HTTPConnection.connect(self) + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + +# Inspired by +# http://code.activestate.com/recipes/577548-https-httplib-client-connection-with-certificate-v/ +class RiakHTTPSConnection(HTTPSConnection): + def __init__(self, + host, + port, + credentials, + pkey_file=None, + cert_file=None, + timeout=None): + """ + Class to make a HTTPS connection, + with support for full client-based SSL Authentication + :param host: Riak host name + :type host: str + :param port: Riak host port number + :type port: int + :param credentials: Security Credential settings + :type credentials: SecurityCreds + :param pkey_file: PEM formatted file that contains your private key + :type pkey_file: str + :param cert_file: PEM formatted certificate chain file + :type cert_file: str + :param timeout: Number of seconds before timing out + :type timeout: int + """ + if PY2: + # NB: it appears that pkey_file / cert_file are never set + # in riak/transports/http/connection.py#_connect() method + pkf = pkey_file + if pkf is None and credentials is not None: + pkf = credentials._pkey_file -class RiakHttpPool(Pool): + cf = cert_file + if cf is None and credentials is not None: + cf = credentials._cert_file + + HTTPSConnection.__init__(self, + host, + port, + key_file=pkf, + cert_file=cf) + else: + super(RiakHTTPSConnection, self). \ + __init__(host=host, + port=port, + key_file=credentials._pkey_file, + cert_file=credentials._cert_file) + self.pkey_file = pkey_file + self.cert_file = cert_file + self.credentials = credentials + self.timeout = timeout + + def connect(self): + """ + Connect to a host on a given (SSL) port using PyOpenSSL. + """ + sock = socket.create_connection((self.host, self.port), self.timeout) + if not USE_STDLIB_SSL: + ssl_ctx = configure_pyopenssl_context(self.credentials) + + # attempt to upgrade the socket to TLS + cxn = OpenSSL.SSL.Connection(ssl_ctx, sock) + cxn.set_connect_state() + while True: + try: + cxn.do_handshake() + except OpenSSL.SSL.WantReadError: + select.select([sock], [], []) + continue + except OpenSSL.SSL.Error as e: + raise SecurityError('bad handshake - ' + str(e)) + break + + self.sock = RiakWrappedSocket(cxn, sock) + self.credentials._check_revoked_cert(self.sock) + else: + ssl_ctx = configure_ssl_context(self.credentials) + if self.timeout is not None: + sock.settimeout(self.timeout) + self.sock = ssl.SSLSocket(sock=sock, + keyfile=self.credentials.pkey_file, + certfile=self.credentials.cert_file, + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=self.credentials.cacert_file, + ciphers=self.credentials.ciphers, + server_hostname=self.host) + self.sock.context = ssl_ctx + + +class HttpPool(Pool): """ A pool of HTTP(S) transport connections. """ def __init__(self, client, **options): self.client = client self.options = options - if client.protocol == 'https': - self.connection_class = httplib.HTTPSConnection - else: - self.connection_class = httplib.HTTPConnection - super(RiakHttpPool, self).__init__() + self.connection_class = NoNagleHTTPConnection + if self.client._credentials: + self.connection_class = RiakHTTPSConnection + + super(HttpPool, self).__init__() def create_resource(self): node = self.client._choose_node() - return RiakHttpTransport(node=node, - client=self.client, - connection_class=self.connection_class, - **self.options) + return HttpTransport(node=node, + client=self.client, + connection_class=self.connection_class, + **self.options) def destroy_resource(self, transport): transport.close() CONN_CLOSED_ERRORS = ( - httplib.NotConnected, - httplib.IncompleteRead, - httplib.ImproperConnectionState, - httplib.BadStatusLine + NotConnected, + IncompleteRead, + ImproperConnectionState, + BadStatusLine ) diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py index 3262b8d3..d1c16281 100644 --- a/riak/transports/http/connection.py +++ b/riak/transports/http/connection.py @@ -1,43 +1,54 @@ -""" -Copyright 2012 Basho Technologies, Inc. +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at +import base64 -http://www.apache.org/licenses/LICENSE-2.0 +from six import PY2 +from riak.util import str_to_bytes -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +if PY2: + from httplib import NotConnected, HTTPConnection +else: + from http.client import NotConnected, HTTPConnection -import httplib - -class RiakHttpConnection(object): +class HttpConnection(object): """ - Connection and low-level request methods for RiakHttpTransport. + Connection and low-level request methods for HttpTransport. """ def _request(self, method, uri, headers={}, body='', stream=False): """ - Given a Method, URL, Headers, and Body, perform and HTTP request, - and return a 2-tuple containing a dictionary of response headers - and the response body. + Given a Method, URL, Headers, and Body, perform and HTTP + request, and return a 3-tuple containing the response status, + response headers (as httplib.HTTPMessage), and response body. """ response = None + headers.setdefault('Accept', + 'multipart/mixed, application/json, */*;q=0.5') + + if self._client._credentials: + self._security_auth_headers(self._client._credentials.username, + self._client._credentials.password, + headers) + try: self._connection.request(method, uri, body, headers) - response = self._connection.getresponse() - - response_headers = {'http_code': response.status} - for (key, value) in response.getheaders(): - response_headers[key.lower()] = value + try: + response = self._connection.getresponse(buffering=True) + except TypeError: + response = self._connection.getresponse() if stream: # The caller is responsible for fully reading the @@ -49,11 +60,30 @@ def _request(self, method, uri, headers={}, body='', stream=False): if response and not stream: response.close() - return response_headers, response_body + return response.status, response.msg, response_body def _connect(self): - self._connection = self._connection_class(self._node.host, - self._node.http_port) + """ + Use the appropriate connection class; optionally with security. + """ + timeout = None + if self._options is not None and 'timeout' in self._options: + timeout = self._options['timeout'] + + if self._client._credentials: + self._connection = self._connection_class( + host=self._node.host, + port=self._node.http_port, + credentials=self._client._credentials, + timeout=timeout) + else: + self._connection = self._connection_class( + host=self._node.host, + port=self._node.http_port, + timeout=timeout) + # Forces the population of stats and resources before any + # other requests are made. + self.server_version def close(self): """ @@ -61,9 +91,25 @@ def close(self): """ try: self._connection.close() - except httplib.NotConnected: + except NotConnected: pass - # These are set by the RiakHttpTransport initializer - _connection_class = httplib.HTTPConnection + # These are set by the HttpTransport initializer + _connection_class = HTTPConnection _node = None + + def _security_auth_headers(self, username, password, headers): + """ + Add in the requisite HTTP Authentication Headers + + :param username: Riak Security Username + :type str + :param password: Riak Security Password + :type str + :param headers: Dictionary of headers + :type dict + """ + userColonPassword = username + ":" + password + b64UserColonPassword = base64. \ + b64encode(str_to_bytes(userColonPassword)).decode("ascii") + headers['Authorization'] = 'Basic %s' % b64UserColonPassword diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index d46fb414..2b53ae7c 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -1,30 +1,32 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import re -from urllib import quote_plus, urlencode + +from six import PY2 from riak import RiakError -from riak.util import lazy_property +from riak.util import lazy_property, bytes_to_str + +if PY2: + from urllib import quote_plus, urlencode +else: + from urllib.parse import quote_plus, urlencode -class RiakHttpResources(object): +class HttpResources(object): """ - Methods for RiakHttpTransport related to URL generation, i.e. + Methods for HttpTransport related to URL generation, i.e. creating the proper paths. """ @@ -37,80 +39,181 @@ def stats_path(self): def mapred_path(self, **options): return mkpath(self.riak_kv_wm_mapred, **options) - def bucket_list_path(self, **options): - query = options.copy() - query.update(buckets=True) - if self.riak_kv_wm_buckets: - return mkpath(self.riak_kv_wm_buckets, **query) + def bucket_list_path(self, bucket_type=None, **options): + query = {'buckets': True} + query.update(options) + if self.riak_kv_wm_bucket_type and bucket_type: + return mkpath("/types", quote_plus(bucket_type), + "buckets", **query) + elif self.riak_kv_wm_buckets: + return mkpath("/buckets", **query) else: return mkpath(self.riak_kv_wm_raw, **query) - def bucket_properties_path(self, bucket, **options): - if self.riak_kv_wm_buckets: - return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), + def bucket_properties_path(self, bucket, bucket_type=None, **options): + if self.riak_kv_wm_bucket_type and bucket_type: + return mkpath("/types", quote_plus(bucket_type), "buckets", + quote_plus(bucket), "props", **options) + elif self.riak_kv_wm_buckets: + return mkpath("/buckets", quote_plus(bucket), "props", **options) else: query = options.copy() - query.update(props=True, keys=True) + query.update(props=True, keys=False) return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), **query) - def key_list_path(self, bucket, **options): + def bucket_type_properties_path(self, bucket_type, **options): + return mkpath("/types", quote_plus(bucket_type), "props", + **options) + + def key_list_path(self, bucket, bucket_type=None, **options): query = {'keys': True, 'props': False} query.update(options) + if self.riak_kv_wm_bucket_type and bucket_type: + return mkpath("/types", quote_plus(bucket_type), "buckets", + quote_plus(bucket), "keys", **query) if self.riak_kv_wm_buckets: - return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), "keys", + return mkpath("/buckets", quote_plus(bucket), "keys", **query) else: return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), **query) - def object_path(self, bucket, key=None, **options): + def object_path(self, bucket, key=None, bucket_type=None, **options): if key: key = quote_plus(key) - if self.riak_kv_wm_buckets: - return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), "keys", + if self.riak_kv_wm_bucket_type and bucket_type: + return mkpath("/types", quote_plus(bucket_type), "buckets", + quote_plus(bucket), "keys", key, **options) + elif self.riak_kv_wm_buckets: + return mkpath("/buckets", quote_plus(bucket), "keys", key, **options) else: return mkpath(self.riak_kv_wm_raw, quote_plus(bucket), key, **options) - # TODO: link_walk_path is undefined here because there is no path - # to it in the client without using MapReduce. - - def index_path(self, bucket, index, start, finish=None, **options): + def index_path(self, bucket, index, start, finish=None, bucket_type=None, + **options): if not self.riak_kv_wm_buckets: raise RiakError("Indexes are unsupported by this Riak node") - if finish: + if finish is not None: finish = quote_plus(str(finish)) - return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), - "index", quote_plus(index), quote_plus(str(start)), - finish, **options) + if self.riak_kv_wm_bucket_type and bucket_type: + return mkpath("/types", quote_plus(bucket_type), + "buckets", quote_plus(bucket), + "index", quote_plus(index), quote_plus(str(start)), + finish, **options) + else: + return mkpath("/buckets", quote_plus(bucket), + "index", quote_plus(index), quote_plus(str(start)), + finish, **options) + + def search_index_path(self, index=None, **options): + """ + Builds a Yokozuna search index URL. + + :param index: optional name of a yz index + :type index: string + :param options: optional list of additional arguments + :type index: dict + :rtype URL string + """ + if not self.yz_wm_index: + raise RiakError("Yokozuna search is unsupported by this Riak node") + if index: + quote_plus(index) + return mkpath(self.yz_wm_index, "index", index, **options) + + def search_schema_path(self, index, **options): + """ + Builds a Yokozuna search Solr schema URL. + + :param index: a name of a yz solr schema + :type index: string + :param options: optional list of additional arguments + :type index: dict + :rtype URL string + """ + if not self.yz_wm_schema: + raise RiakError("Yokozuna search is unsupported by this Riak node") + return mkpath(self.yz_wm_schema, "schema", quote_plus(index), + **options) def solr_select_path(self, index, query, **options): - if not self.riak_solr_searcher_wm: - raise RiakError("Riak Search is unsupported by this Riak node") - qs = {'q': query, 'wt': 'json'} + if not self.riak_solr_searcher_wm and not self.yz_wm_search: + raise RiakError("Search is unsupported by this Riak node") + qs = {'q': query, 'wt': 'json', 'fl': '*,score'} qs.update(options) if index: index = quote_plus(index) - return mkpath(self.riak_solr_searcher_wm, index, "select", **qs) + return mkpath("/solr", index, "select", **qs) def solr_update_path(self, index): if not self.riak_solr_searcher_wm: - raise RiakError("Riak Search is unsupported by this Riak node") + raise RiakError("Riak Search 1 is unsupported by this Riak node") if index: index = quote_plus(index) return mkpath(self.riak_solr_indexer_wm, index, "update") - def luwak_path(self, key=None): - if not self.luwak_wm_file: - raise RiakError("Luwak is unsupported by this Riak node") + def counters_path(self, bucket, key, **options): + if not self.riak_kv_wm_counter: + raise RiakError("Counters are unsupported by this Riak node") + + return mkpath(self.riak_kv_wm_buckets, quote_plus(bucket), "counters", + quote_plus(key), **options) + + def datatypes_path(self, bucket_type, bucket, key=None, **options): + if not self.bucket_types(): + raise RiakError("Datatypes are unsupported by this Riak node") if key: key = quote_plus(key) - return mkpath(self.luwak_wm_file, key) + + return mkpath("/types", quote_plus(bucket_type), "buckets", + quote_plus(bucket), "datatypes", key, **options) + + def preflist_path(self, bucket, key, bucket_type=None, **options): + """ + Generate the URL for bucket/key preflist information + + :param bucket: Name of a Riak bucket + :type bucket: string + :param key: Name of a Key + :type key: string + :param bucket_type: Optional Riak Bucket Type + :type bucket_type: None or string + :rtype URL string + """ + if not self.riak_kv_wm_preflist: + raise RiakError("Preflists are unsupported by this Riak node") + if self.riak_kv_wm_bucket_type and bucket_type: + return mkpath("/types", quote_plus(bucket_type), + "buckets", quote_plus(bucket), + "keys", quote_plus(key), + "preflist", **options) + else: + return mkpath("/buckets", quote_plus(bucket), + "keys", quote_plus(key), + "preflist", **options) + + # Feature detection overrides + def bucket_types(self): + return self.riak_kv_wm_bucket_type is not None + + def index_term_regex(self): + if self.riak_kv_wm_bucket_type is not None: + return True + else: + return super(HttpResources, self).index_term_regex() + + # Resource root paths + @lazy_property + def riak_kv_wm_bucket_type(self): + if 'riak_kv_wm_bucket_type' in self.resources: + return "/types" @lazy_property def riak_kv_wm_buckets(self): - return self.resources.get('riak_kv_wm_index') + if 'riak_kv_wm_buckets' in self.resources: + return "/buckets" @lazy_property def riak_kv_wm_raw(self): @@ -141,8 +244,28 @@ def riak_solr_indexer_wm(self): return self.resources.get('riak_solr_indexer_wm') @lazy_property - def luwak_wm_file(self): - return self.resources.get('luwak_wm_file') + def riak_kv_wm_counter(self): + return self.resources.get('riak_kv_wm_counter') + + @lazy_property + def riak_kv_wm_preflist(self): + return self.resources.get('riak_kv_wm_preflist') + + @lazy_property + def yz_wm_search(self): + return self.resources.get('yz_wm_search') + + @lazy_property + def yz_wm_extract(self): + return self.resources.get('yz_wm_extract') + + @lazy_property + def yz_wm_schema(self): + return self.resources.get('yz_wm_schema') + + @lazy_property + def yz_wm_index(self): + return self.resources.get('yz_wm_index') @lazy_property def resources(self): @@ -155,7 +278,7 @@ def mkpath(*segments, **query): and a dict. """ # Remove empty segments (e.g. no key specified) - segments = [s for s in segments if s is not None] + segments = [bytes_to_str(s) for s in segments if s is not None] # Join the segments into a path pathstring = '/'.join(segments) # Remove extra slashes @@ -167,7 +290,7 @@ def mkpath(*segments, **query): if query[key] in [False, True]: _query[key] = str(query[key]).lower() elif query[key] is not None: - if isinstance(query[key], unicode): + if PY2 and isinstance(query[key], unicode): # noqa _query[key] = query[key].encode('utf-8') else: _query[key] = query[key] diff --git a/riak/transports/http/search.py b/riak/transports/http/search.py index 4e6c69e6..d43688f6 100644 --- a/riak/transports/http/search.py +++ b/riak/transports/http/search.py @@ -1,3 +1,18 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + class XMLSearchResult(object): # Match tags that are document fields fieldtags = ['str', 'int', 'date'] diff --git a/riak/transports/http/stream.py b/riak/transports/http/stream.py index 2f1620f9..590565f2 100644 --- a/riak/transports/http/stream.py +++ b/riak/transports/http/stream.py @@ -1,29 +1,29 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import json -import string import re + from cgi import parse_header from email import message_from_string +from riak.util import decode_index_value +from riak.client.index_page import CONTINUATION +from riak import RiakError +from six import PY2 -class RiakHttpStream(object): +class HttpStream(object): """ Base class for HTTP streaming iterators. """ @@ -34,48 +34,81 @@ def __init__(self, response): self.response = response self.buffer = '' self.response_done = False + self.resource = None def __iter__(self): return self def _read(self): chunk = self.response.read(self.BLOCK_SIZE) - if chunk == '': - self.response_done = True - self.buffer += chunk + if PY2: + if chunk == '': + self.response_done = True + self.buffer += chunk + else: + if chunk == b'': + self.response_done = True + self.buffer += chunk.decode('utf-8') + + def __next__(self): + raise NotImplementedError def next(self): raise NotImplementedError + def attach(self, resource): + self.resource = resource + def close(self): - pass + self.resource.release() -class RiakHttpKeyStream(RiakHttpStream): - """ - Streaming iterator for list-keys over HTTP - """ +class HttpJsonStream(HttpStream): + _json_field = None def next(self): + # Python 2.x Version while '}' not in self.buffer and not self.response_done: self._read() if '}' in self.buffer: - idx = string.index(self.buffer, '}') + 1 + idx = self.buffer.index('}') + 1 chunk = self.buffer[:idx] self.buffer = self.buffer[idx:] - keys = json.loads(chunk)[u'keys'] - return keys + jsdict = json.loads(chunk) + if 'error' in jsdict: + self.close() + raise RiakError(jsdict['error']) + field = jsdict[self._json_field] + return field else: raise StopIteration + def __next__(self): + # Python 3.x Version + return self.next() + -class RiakHttpMultipartStream(RiakHttpStream): +class HttpKeyStream(HttpJsonStream): + """ + Streaming iterator for list-keys over HTTP + """ + _json_field = u'keys' + + +class HttpBucketStream(HttpJsonStream): + """ + Streaming iterator for list-buckets over HTTP + """ + _json_field = u'buckets' + + +class HttpMultipartStream(HttpStream): """ Streaming iterator for multipart messages over HTTP """ def __init__(self, response): - super(RiakHttpMultipartStream, self).__init__(response) + super(HttpMultipartStream, self).__init__(response) ctypehdr = response.getheader('content-type') _, params = parse_header(ctypehdr) self.boundary_re = re.compile('\r?\n--%s(?:--)?\r?\n' % @@ -99,6 +132,10 @@ def next(self): else: raise StopIteration + def __next__(self): + # Python 3.x Version + return self.next() + def try_match(self): self.next_boundary = self.boundary_re.search(self.buffer) return self.next_boundary @@ -114,12 +151,48 @@ def read_until_boundary(self): self._read() -class RiakHttpMapReduceStream(RiakHttpMultipartStream): +class HttpMapReduceStream(HttpMultipartStream): """ Streaming iterator for MapReduce over HTTP """ def next(self): - message = super(RiakHttpMapReduceStream, self).next() + message = super(HttpMapReduceStream, self).next() payload = json.loads(message.get_payload()) return payload['phase'], payload['data'] + + def __next__(self): + # Python 3.x Version + return self.next() + + +class HttpIndexStream(HttpMultipartStream): + """ + Streaming iterator for secondary indexes over HTTP + """ + + def __init__(self, response, index, return_terms): + super(HttpIndexStream, self).__init__(response) + self.index = index + self.return_terms = return_terms + + def next(self): + message = super(HttpIndexStream, self).next() + payload = json.loads(message.get_payload()) + if u'error' in payload: + raise RiakError(payload[u'error']) + elif u'keys' in payload: + return payload[u'keys'] + elif u'results' in payload: + structs = payload[u'results'] + # Format is {"results":[{"2ikey":"primarykey"}, ...]} + return [self._decode_pair(list(d.items())[0]) for d in structs] + elif u'continuation' in payload: + return CONTINUATION(payload[u'continuation']) + + def __next__(self): + # Python 3.x Version + return self.next() + + def _decode_pair(self, pair): + return (decode_index_value(self.index, pair[0]), pair[1]) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index ee147f3c..7cba2681 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -1,71 +1,66 @@ -""" -Copyright 2012 Basho Technologies, Inc. -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. try: import simplejson as json except ImportError: import json -import urllib -import re -import csv -import httplib -from email.message import Message -from riak.transports.transport import RiakTransport -from riak.transports.http.resources import RiakHttpResources -from riak.transports.http.connection import RiakHttpConnection -from riak.transports.http.search import XMLSearchResult -from riak.transports.http.stream import ( - RiakHttpKeyStream, - RiakHttpMapReduceStream) -from riak import RiakError -from riak.multidict import MultiDict -from xml.etree import ElementTree +from six import PY2 from xml.dom.minidom import Document +from riak import RiakError +from riak.codecs.http import HttpCodec +from riak.transports.transport import Transport +from riak.transports.http.resources import HttpResources +from riak.transports.http.connection import HttpConnection +from riak.transports.http.stream import ( + HttpKeyStream, + HttpMapReduceStream, + HttpBucketStream, + HttpIndexStream) +from riak.security import SecurityError +from riak.util import decode_index_value, bytes_to_str, str_to_long -# subtract length of "Link: " header string and newline -MAX_LINK_HEADER_SIZE = 8192 - 8 +if PY2: + from httplib import HTTPConnection +else: + from http.client import HTTPConnection -class RiakHttpTransport(RiakHttpConnection, RiakHttpResources, RiakTransport): +class HttpTransport(Transport, + HttpConnection, HttpResources, HttpCodec): """ - The RiakHttpTransport object holds information necessary to + The HttpTransport object holds information necessary to connect to Riak via HTTP. """ def __init__(self, node=None, client=None, - connection_class=httplib.HTTPConnection, + connection_class=HTTPConnection, client_id=None, - **unused_options): + **options): """ Construct a new HTTP connection to Riak. """ - super(RiakHttpTransport, self).__init__() + super(HttpTransport, self).__init__() self._client = client self._node = node self._connection_class = connection_class self._client_id = client_id + self._options = options if not self._client_id: self._client_id = self.make_random_client_id() self._connect() @@ -74,17 +69,17 @@ def ping(self): """ Check server is alive over HTTP """ - response = self._request('GET', self.ping_path()) - return(response is not None) and (response[1] == 'OK') + status, _, body = self._request('GET', self.ping_path()) + return(status is not None) and (bytes_to_str(body) == 'OK') def stats(self): """ Gets performance statistics and server information """ - response = self._request('GET', self.stats_path(), - {'Accept': 'application/json'}) - if response[0]['http_code'] is 200: - return json.loads(response[1]) + status, _, body = self._request('GET', self.stats_path(), + {'Accept': 'application/json'}) + if status == 200: + return json.loads(bytes_to_str(body)) else: return None @@ -92,7 +87,11 @@ def stats(self): def _server_version(self): stats = self.stats() if stats is not None: - return stats['riak_kv_version'] + s = stats['riak_kv_version'] + if s.startswith('riak_ts-'): + return stats['riak_pb_version'] + else: + return s # If stats is disabled, we can't assume the Riak version # is >= 1.1. However, we can assume the new URL scheme is # at least version 1.0 @@ -106,189 +105,241 @@ def get_resources(self): Gets a JSON mapping of server-side resource names to paths :rtype dict """ - response = self._request('GET', '/', {'Accept': 'application/json'}) - if response[0]['http_code'] is 200: - return json.loads(response[1]) + status, _, body = self._request('GET', '/', + {'Accept': 'application/json'}) + if status == 200: + tmp, resources = json.loads(bytes_to_str(body)), {} + for k in tmp: + # The keys and values returned by json.loads() are unicode, + # which will cause problems when passed into httplib later + # (expecting bytes both in Python 2.x and 3.x). + # We just encode the resource paths into bytes, with an + # encoding consistent with what the resources module expects. + resources[k] = tmp[k].encode('utf-8') + return resources else: return {} - def get(self, robj, r=None, pr=None, vtag=None): + def get(self, robj, r=None, pr=None, timeout=None, basic_quorum=None, + notfound_ok=None, head_only=False): """ Get a bucket/key from the server """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'r': r, 'pr': pr, 'vtag': vtag} - url = self.object_path(robj.bucket.name, robj.key, **params) + params = {'r': r, 'pr': pr, 'timeout': timeout, + 'basic_quorum': basic_quorum, + 'notfound_ok': notfound_ok} + + bucket_type = self._get_bucket_type(robj.bucket.bucket_type) + + url = self.object_path(robj.bucket.name, robj.key, + bucket_type=bucket_type, **params) response = self._request('GET', url) - return self.parse_body(robj, response, [200, 300, 404]) + return self._parse_body(robj, response, [200, 300, 404]) def put(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): + if_none_match=False, timeout=None): """ - Serialize put request and deserialize response + Puts a (possibly new) object. """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} - url = self.object_path(robj.bucket.name, robj.key, **params) - headers = self._build_put_headers(robj) - - # TODO: use a more general 'prevent_stale_writes' semantics, - # which is a superset of the if_none_match semantics. - if if_none_match: - headers["If-None-Match"] = "*" - content = robj.encoded_data - return self.do_put(url, headers, content, robj, return_body) - - def do_put(self, url, headers, content, robj, return_body=False): - if robj.key is None: - response = self._request('POST', url, headers, content) + params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw, + 'timeout': timeout} + + bucket_type = self._get_bucket_type(robj.bucket.bucket_type) + + url = self.object_path(robj.bucket.name, robj.key, + bucket_type=bucket_type, + **params) + headers = self._build_put_headers(robj, if_none_match=if_none_match) + if PY2: + content = bytearray(robj.encoded_data) else: - response = self._request('PUT', url, headers, content) + content = robj.encoded_data - if return_body: - return self.parse_body(robj, response, [200, 201, 204, 300]) + if robj.key is None: + expect = [201] + method = 'POST' else: - self.check_http_code(response, [204]) - return None + expect = [204] + method = 'PUT' - def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """Put a new object into the Riak store, returning its (new) key.""" - # We could detect quorum_controls here but HTTP ignores - # unknown flags/params. - params = {'returnbody': return_body, 'w': w, 'dw': dw, 'pw': pw} - url = self.object_path(robj.bucket.name, **params) - headers = self._build_put_headers(robj) - # TODO: use a more general 'prevent_stale_writes' semantics, - # which is a superset of the if_none_match semantics. - if if_none_match: - headers["If-None-Match"] = "*" - content = robj.encoded_data - response = self._request('POST', url, headers, content) - location = response[0]['location'] - idx = location.rindex('/') - robj.key = location[(idx + 1):] + response = self._request(method, url, headers, content) if return_body: - return self.parse_body(robj, response, [201]) + return self._parse_body(robj, response, [200, 201, 204, 300]) else: - self.check_http_code(response, [201]) + self.check_http_code(response[0], expect) return None - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None, + timeout=None): """ Delete an object. """ # We could detect quorum_controls here but HTTP ignores # unknown flags/params. - params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw} + params = {'rw': rw, 'r': r, 'w': w, 'dw': dw, 'pr': pr, 'pw': pw, + 'timeout': timeout} headers = {} - url = self.object_path(robj.bucket.name, robj.key, **params) - if self.tombstone_vclocks() and robj.vclock is not None: - headers['X-Riak-Vclock'] = robj.vclock + + bucket_type = self._get_bucket_type(robj.bucket.bucket_type) + + url = self.object_path(robj.bucket.name, robj.key, + bucket_type=bucket_type, **params) + use_vclocks = (self.tombstone_vclocks() and hasattr(robj, 'vclock') and + robj.vclock is not None) + if use_vclocks: + headers['X-Riak-Vclock'] = robj.vclock.encode('base64') response = self._request('DELETE', url, headers) - self.check_http_code(response, [204, 404]) + self.check_http_code(response[0], [204, 404]) return self - def get_keys(self, bucket): + def get_keys(self, bucket, timeout=None): """ Fetch a list of keys for the bucket """ - url = self.key_list_path(bucket.name) - response = self._request('GET', url) + bucket_type = self._get_bucket_type(bucket.bucket_type) + url = self.key_list_path(bucket.name, bucket_type=bucket_type, + timeout=timeout) + status, _, body = self._request('GET', url) - headers, encoded_props = response[0:2] - if headers['http_code'] == 200: - props = json.loads(encoded_props) + if status == 200: + props = json.loads(bytes_to_str(body)) return props['keys'] else: - raise Exception('Error listing keys.') + raise RiakError('Error listing keys.') - def stream_keys(self, bucket): - url = self.key_list_path(bucket.name, keys='stream') - headers, response = self._request('GET', url, stream=True) + def stream_keys(self, bucket, timeout=None): + bucket_type = self._get_bucket_type(bucket.bucket_type) + url = self.key_list_path(bucket.name, bucket_type=bucket_type, + keys='stream', timeout=timeout) + status, headers, response = self._request('GET', url, stream=True) - if headers['http_code'] == 200: - return RiakHttpKeyStream(response) + if status == 200: + return HttpKeyStream(response) else: - raise Exception('Error listing keys.') + raise RiakError('Error listing keys.') - def get_buckets(self): + def get_buckets(self, bucket_type=None, timeout=None): """ Fetch a list of all buckets """ - url = self.bucket_list_path() - response = self._request('GET', url) + bucket_type = self._get_bucket_type(bucket_type) + url = self.bucket_list_path(bucket_type=bucket_type, + timeout=timeout) + status, headers, body = self._request('GET', url) - headers, encoded_props = response[0:2] - if headers['http_code'] == 200: - props = json.loads(encoded_props) + if status == 200: + props = json.loads(bytes_to_str(body)) return props['buckets'] else: - raise Exception('Error getting buckets.') + raise RiakError('Error getting buckets.') + + def stream_buckets(self, bucket_type=None, timeout=None): + """ + Stream list of buckets through an iterator + """ + if not self.bucket_stream(): + raise NotImplementedError('Streaming list-buckets is not ' + "supported on %s" % + self.server_version.vstring) + bucket_type = self._get_bucket_type(bucket_type) + url = self.bucket_list_path(bucket_type=bucket_type, + buckets="stream", timeout=timeout) + status, headers, response = self._request('GET', url, stream=True) + + if status == 200: + return HttpBucketStream(response) + else: + raise RiakError('Error listing buckets.') def get_bucket_props(self, bucket): """ Get properties for a bucket """ - # Run the request... - url = self.bucket_properties_path(bucket.name) - response = self._request('GET', url) + bucket_type = self._get_bucket_type(bucket.bucket_type) + url = self.bucket_properties_path(bucket.name, + bucket_type=bucket_type) + status, headers, body = self._request('GET', url) - headers = response[0] - encoded_props = response[1] - if headers['http_code'] == 200: - props = json.loads(encoded_props) + if status == 200: + props = json.loads(bytes_to_str(body)) return props['props'] else: - raise Exception('Error getting bucket properties.') + raise RiakError('Error getting bucket properties.') def set_bucket_props(self, bucket, props): """ Set the properties on the bucket object given """ - url = self.bucket_properties_path(bucket.name) + bucket_type = self._get_bucket_type(bucket.bucket_type) + url = self.bucket_properties_path(bucket.name, + bucket_type=bucket_type) headers = {'Content-Type': 'application/json'} content = json.dumps({'props': props}) # Run the request... - response = self._request('PUT', url, headers, content) - - # Handle the response... - if response is None: - raise Exception('Error setting bucket properties.') + status, _, body = self._request('PUT', url, headers, content) - # Check the response value... - status = response[0]['http_code'] - if status != 204: - raise Exception('Error setting bucket properties.') + if status == 401: + raise SecurityError('Not authorized to set bucket properties.') + elif status != 204: + raise RiakError('Error setting bucket properties.') return True def clear_bucket_props(self, bucket): """ reset the properties on the bucket object given """ + bucket_type = self._get_bucket_type(bucket.bucket_type) + url = self.bucket_properties_path(bucket.name, + bucket_type=bucket_type) url = self.bucket_properties_path(bucket.name) headers = {'Content-Type': 'application/json'} # Run the request... - response = self._request('DELETE', url, headers, None) - - # Handle the response... - if response is None: - raise Exception('Error clearing bucket properties.') + status, _, _ = self._request('DELETE', url, headers, None) - # Check the response value... - status = response[0]['http_code'] if status == 204: return True elif status == 405: return False else: - raise Exception('Error %s clearing bucket properties.' + raise RiakError('Error %s clearing bucket properties.' % status) + def get_bucket_type_props(self, bucket_type): + """ + Get properties for a bucket-type + """ + self._check_bucket_types(bucket_type) + url = self.bucket_type_properties_path(bucket_type.name) + status, headers, body = self._request('GET', url) + + if status == 200: + props = json.loads(bytes_to_str(body)) + return props['props'] + else: + raise RiakError('Error getting bucket-type properties.') + + def set_bucket_type_props(self, bucket_type, props): + """ + Set the properties on the bucket-type + """ + self._check_bucket_types(bucket_type) + url = self.bucket_type_properties_path(bucket_type.name) + headers = {'Content-Type': 'application/json'} + content = json.dumps({'props': props}) + + # Run the request... + status, _, _ = self._request('PUT', url, headers, content) + + if status != 204: + raise RiakError('Error setting bucket-type properties.') + return True + def mapred(self, inputs, query, timeout=None): """ Run a MapReduce query. @@ -299,16 +350,15 @@ def mapred(self, inputs, query, timeout=None): # Do the request... url = self.mapred_path() headers = {'Content-Type': 'application/json'} - response = self._request('POST', url, headers, content) + status, headers, body = self._request('POST', url, headers, content) # Make sure the expected status code came back... - status = response[0]['http_code'] if status != 200: raise RiakError( 'Error running MapReduce operation. Headers: %s Body: %s' % - (repr(response[0]), repr(response[1]))) + (repr(headers), repr(body))) - result = json.loads(response[1]) + result = json.loads(bytes_to_str(body)) return result def stream_mapred(self, inputs, query, timeout=None): @@ -316,26 +366,238 @@ def stream_mapred(self, inputs, query, timeout=None): url = self.mapred_path(chunked=True) reqheaders = {'Content-Type': 'application/json'} - headers, response = self._request('POST', url, reqheaders, - content, stream=True) + status, headers, response = self._request('POST', url, reqheaders, + content, stream=True) - if headers['http_code'] is 200: - return RiakHttpMapReduceStream(response) + if status == 200: + return HttpMapReduceStream(response) else: - raise Exception( + raise RiakError( 'Error running MapReduce operation. Headers: %s Body: %s' % (repr(headers), repr(response.read()))) - def get_index(self, bucket, index, startkey, endkey=None): + def get_index(self, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, continuation=None, + timeout=None, term_regex=None): """ Performs a secondary index query. """ - url = self.index_path(bucket, index, startkey, endkey) - response = self._request('GET', url) - headers, data = response - self.check_http_code(response, [200]) - json_data = json.loads(data) - return json_data[u'keys'][:] + if term_regex and not self.index_term_regex(): + raise NotImplementedError("Secondary index term_regex is not " + "supported on %s" % + self.server_version.vstring) + + if timeout == 'infinity': + timeout = 0 + + params = {'return_terms': return_terms, 'max_results': max_results, + 'continuation': continuation, 'timeout': timeout, + 'term_regex': term_regex} + bucket_type = self._get_bucket_type(bucket.bucket_type) + url = self.index_path(bucket.name, index, startkey, endkey, + bucket_type=bucket_type, **params) + status, headers, body = self._request('GET', url) + self.check_http_code(status, [200]) + json_data = json.loads(bytes_to_str(body)) + if return_terms and u'results' in json_data: + results = [] + for result in json_data[u'results'][:]: + term, key = list(result.items())[0] + results.append((decode_index_value(index, term), key),) + else: + results = json_data[u'keys'][:] + + if max_results and u'continuation' in json_data: + return (results, json_data[u'continuation']) + else: + return (results, None) + + def stream_index(self, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, continuation=None, + timeout=None, term_regex=None): + """ + Streams a secondary index query. + """ + if not self.stream_indexes(): + raise NotImplementedError("Secondary index streaming is not " + "supported on %s" % + self.server_version.vstring) + + if term_regex and not self.index_term_regex(): + raise NotImplementedError("Secondary index term_regex is not " + "supported on %s" % + self.server_version.vstring) + + if timeout == 'infinity': + timeout = 0 + + params = {'return_terms': return_terms, 'stream': True, + 'max_results': max_results, 'continuation': continuation, + 'timeout': timeout, 'term_regex': term_regex} + bucket_type = self._get_bucket_type(bucket.bucket_type) + url = self.index_path(bucket.name, index, startkey, endkey, + bucket_type=bucket_type, **params) + status, headers, response = self._request('GET', url, stream=True) + + if status == 200: + return HttpIndexStream(response, index, return_terms) + else: + raise RiakError('Error streaming secondary index.') + + def create_search_index(self, index, schema=None, n_val=None, + timeout=None): + """ + Create a Solr search index for Yokozuna. + + :param index: a name of a yz index + :type index: string + :param schema: XML of Solr schema + :type schema: string + :param n_val: N value of the write + :type n_val: int + :param timeout: optional timeout (in ms) + :type timeout: integer, None + + :rtype boolean + """ + if not self.yz_wm_index: + raise NotImplementedError("Search 2.0 administration is not " + "supported for this version") + + url = self.search_index_path(index) + headers = {'Content-Type': 'application/json'} + content_dict = dict() + if schema: + content_dict['schema'] = schema + if n_val: + content_dict['n_val'] = n_val + if timeout: + content_dict['timeout'] = timeout + content = json.dumps(content_dict) + + # Run the request... + status, _, _ = self._request('PUT', url, headers, content) + + if status != 204: + raise RiakError('Error setting Search 2.0 index.') + return True + + def get_search_index(self, index): + """ + Fetch the specified Solr search index for Yokozuna. + + :param index: a name of a yz index + :type index: string + + :rtype string + """ + if not self.yz_wm_index: + raise NotImplementedError("Search 2.0 administration is not " + "supported for this version") + + url = self.search_index_path(index) + + # Run the request... + status, headers, body = self._request('GET', url) + + if status == 200: + return json.loads(bytes_to_str(body)) + else: + raise RiakError('Error getting Search 2.0 index.') + + def list_search_indexes(self): + """ + Return a list of Solr search indexes from Yokozuna. + + :rtype list of dicts + """ + if not self.yz_wm_index: + raise NotImplementedError("Search 2.0 administration is not " + "supported for this version") + + url = self.search_index_path() + + # Run the request... + status, headers, body = self._request('GET', url) + + if status == 200: + json_data = json.loads(bytes_to_str(body)) + # Return a list of dictionaries + return json_data + else: + raise RiakError('Error getting Search 2.0 index.') + + def delete_search_index(self, index): + """ + Fetch the specified Solr search index for Yokozuna. + + :param index: a name of a yz index + :type index: string + + :rtype boolean + """ + if not self.yz_wm_index: + raise NotImplementedError("Search 2.0 administration is not " + "supported for this version") + + url = self.search_index_path(index) + + # Run the request... + status, _, _ = self._request('DELETE', url) + + if status != 204: + raise RiakError('Error setting Search 2.0 index.') + return True + + def create_search_schema(self, schema, content): + """ + Create a new Solr schema for Yokozuna. + + :param schema: name of Solr schema + :type schema: string + :param content: actual defintion of schema (XML) + :type content: string + + :rtype boolean + """ + if not self.yz_wm_schema: + raise NotImplementedError("Search 2.0 administration is not " + "supported for this version") + + url = self.search_schema_path(schema) + headers = {'Content-Type': 'application/xml'} + + # Run the request... + status, header, body = self._request('PUT', url, headers, content) + + if status != 204: + raise RiakError('Error creating Search 2.0 schema.') + return True + + def get_search_schema(self, schema): + """ + Fetch a Solr schema from Yokozuna. + + :param schema: name of Solr schema + :type schema: string + + :rtype dict + """ + if not self.yz_wm_schema: + raise NotImplementedError("Search 2.0 administration is not " + "supported for this version") + url = self.search_schema_path(schema) + + # Run the request... + status, _, body = self._request('GET', url) + + if status == 200: + result = {} + result['name'] = schema + result['content'] = bytes_to_str(body) + return result + else: + raise RiakError('Error getting Search 2.0 schema.') def search(self, index, query, **params): """ @@ -351,11 +613,10 @@ def search(self, index, query, **params): options.update(params) url = self.solr_select_path(index, query, **options) - response = self._request('GET', url) - headers, data = response - self.check_http_code(response, [200]) + status, headers, data = self._request('GET', url) + self.check_http_code(status, [200]) if 'json' in headers['content-type']: - results = json.loads(data) + results = json.loads(bytes_to_str(data)) return self._normalize_json_search_response(results) elif 'xml' in headers['content-type']: return self._normalize_xml_search_response(data) @@ -409,235 +670,147 @@ def fulltext_delete(self, index, docs=None, queries=None): {'Content-Type': 'text/xml'}, xml.toxml().encode('utf-8')) - def check_http_code(self, response, expected_statuses): - status = response[0]['http_code'] - if not status in expected_statuses: - raise Exception('Expected status %s, received %s : %s' % - (expected_statuses, status, response[1])) + def get_counter(self, bucket, key, **options): + if not bucket.bucket_type.is_default(): + raise NotImplementedError("Counters are not " + "supported with bucket-types, " + "use datatypes instead.") - def parse_body(self, robj, response, expected_statuses): - """ - Parse the body of an object response and populate the object. - """ - # If no response given, then return. - if response is None: + if not self.counters(): + raise NotImplementedError("Counters are not " + "supported on %s" % + self.server_version.vstring) + + url = self.counters_path(bucket.name, key, **options) + status, headers, body = self._request('GET', url) + + self.check_http_code(status, [200, 404]) + if status == 200: + return str_to_long(body.strip()) + elif status == 404: return None - # Make sure expected code came back - self.check_http_code(response, expected_statuses) + def update_counter(self, bucket, key, amount, **options): + if not bucket.bucket_type.is_default(): + raise NotImplementedError("Counters are not " + "supported with bucket-types, " + "use datatypes instead.") + + if not self.counters(): + raise NotImplementedError("Counters are not " + "supported on %s" % + self.server_version.vstring) + + return_value = 'returnvalue' in options and options['returnvalue'] + headers = {'Content-Type': 'text/plain'} + url = self.counters_path(bucket.name, key, **options) + status, headers, body = self._request('POST', url, headers, + str(amount)) + if return_value and status == 200: + return str_to_long(body.strip()) + elif status == 204: + return True + else: + self.check_http_code(status, [200, 204]) + + def fetch_datatype(self, bucket, key, **options): + if not self.datatypes(): + raise NotImplementedError("Datatypes are not supported.") + if bucket.bucket_type.is_default(): + raise NotImplementedError( + 'Datatypes cannot be used in the default bucket-type.') + + url = self.datatypes_path(bucket.bucket_type.name, bucket.name, key, + **options) + status, headers, body = self._request('GET', url) + + self.check_http_code(status, [200, 404]) + response = json.loads(bytes_to_str(body)) + dtype = response['type'] + if status == 404: + return (dtype, None, None) + else: + return (dtype, self._decode_datatype(dtype, response['value']), + response.get('context')) + + def update_datatype(self, datatype, **options): + if not self.datatypes(): + raise NotImplementedError('Datatypes are not supported.') + if datatype.bucket.bucket_type.is_default(): + raise NotImplementedError( + 'Datatypes cannot be used in the default bucket-type.') + + op = datatype.to_op() + context = datatype.context + type_name = datatype.type_name + if not op: + raise ValueError("No operation to send on datatype {!r}". + format(datatype)) + + if type_name not in ('counter', 'set', 'hll', 'map'): + raise TypeError("Cannot send operation on datatype {!r}". + format(type_name)) + + if 'return_body' in options: + options['returnbody'] = options['return_body'] + + url = self.datatypes_path(datatype.bucket.bucket_type.name, + datatype.bucket.name, + datatype.key, **options) + headers = {'Content-Type': 'application/json'} + opdict = self._encode_dt_op(type_name, op) + if context: + opdict['context'] = context + payload = json.dumps(opdict) - # Update the object... - headers = response[0] - data = response[1] - status = headers['http_code'] + status, headers, body = self._request('POST', url, headers, payload) - # Check if the server is down(status==0) - if not status: - ### we need the host/port that was used. - m = 'Could not contact Riak Server: http://$HOST:$PORT !' - raise RiakError(m) + self.check_http_code(status, [200, 201, 204]) - # If 404(Not Found), then clear the object. - if status == 404: - return None + if status == 201: + datatype.key = headers['location'].strip().split('/')[-1] - # If 300(Siblings), then return the list of siblings - elif status == 300: - # Parse and get rid of 'Siblings:' string in element 0 - siblings = data.strip().split('\n') - siblings.pop(0) - robj.siblings = siblings - robj.exists = True - robj.vclock = headers['x-riak-vclock'] - return robj - - #no sibs - robj.siblings = [] - - # Parse the headers... - links = [] - for header, value in headers.iteritems(): - if header == 'content-type': - robj.content_type, robj.charset = \ - self._parse_content_type(value) - elif header == 'content-encoding': - robj.content_encoding = value - elif header == 'etag': - robj.etag = value - elif header == 'link': - self._parse_links(links, headers['link']) - elif header == 'last-modified': - robj.last_modified = value - elif header.startswith('x-riak-meta-'): - metakey = header.replace('x-riak-meta-', '') - robj.usermeta[metakey] = value - elif header.startswith('x-riak-index-'): - field = header.replace('x-riak-index-', '') - reader = csv.reader([value], skipinitialspace=True) - for line in reader: - for token in line: - if field.endswith("_int"): - token = int(token) - robj.add_index(field, token) - elif header == 'x-riak-vclock': - robj.vclock = value - elif header == 'x-riak-deleted': - robj.deleted = True - if links: - robj.links = links - - robj.encoded_data = data - - robj.exists = True - return robj - - def to_link_header(self, link): - """ - Convert the link tuple to a link header string. Used internally. - """ - try: - bucket, key, tag = link - except ValueError: - raise RiakError("Invalid link tuple %s" % link) - tag = tag if tag is not None else bucket - url = self.object_path(bucket, key) - header = '<%s>; riaktag="%s"' % (url, tag) - return header - - def _parse_links(self, links, linkHeaders): - oldform = "; ?riaktag=\"([^\"]+)\"" - newform = "; ?riaktag=\"([^\"]+)\"" - for linkHeader in linkHeaders.strip().split(','): - linkHeader = linkHeader.strip() - matches = (re.match(oldform, linkHeader) or - re.match(newform, linkHeader)) - if matches is not None: - link = (urllib.unquote_plus(matches.group(2)), - urllib.unquote_plus(matches.group(3)), - urllib.unquote_plus(matches.group(4))) - links.append(link) - return links - - def _add_links_for_riak_object(self, robject, headers): - links = robject.links - if links: - current_header = '' - for link in links: - header = self.to_link_header(link) - if len(current_header + header) > MAX_LINK_HEADER_SIZE: - headers.add('Link', current_header) - current_header = '' - - if current_header != '': - header = ', ' + header - current_header += header - - headers.add('Link', current_header) - - return headers - - # Utility functions used by Riak library. - - def _build_put_headers(self, robj): - """Build the headers for a POST/PUT request.""" - - # Construct the headers... - if robj.charset is not None: - content_type = ('%s; charset="%s"' % - (robj.content_type, robj.charset)) - else: - content_type = robj.content_type - headers = MultiDict({'Accept': 'text/plain, */*; q=0.5', - 'Content-Type': content_type, - 'X-Riak-ClientId': self._client_id}) - # Add the vclock if it exists... - if robj.vclock is not None: - headers['X-Riak-Vclock'] = robj.vclock - - # Create the header from metadata - self._add_links_for_riak_object(robj, headers) - - for key, value in robj.usermeta.iteritems(): - headers['X-Riak-Meta-%s' % key] = value - - for field, value in robj.indexes: - key = 'X-Riak-Index-%s' % field - if key in headers: - headers[key] += ", " + str(value) - else: - headers[key] = str(value) - - return headers - - def _normalize_json_search_response(self, json): - """ - Normalizes a JSON search response so that PB and HTTP have the - same return value - """ - result = {} - if u'response' in json: - result['num_found'] = json[u'response'][u'numFound'] - result['max_score'] = float(json[u'response'][u'maxScore']) - docs = [] - for doc in json[u'response'][u'docs']: - resdoc = {u'id': doc[u'id']} - if u'fields' in doc: - for k, v in doc[u'fields'].iteritems(): - resdoc[k] = v - docs.append(resdoc) - result['docs'] = docs - return result + if status != 204: + response = json.loads(bytes_to_str(body)) + datatype._context = response.get('context') + datatype._set_value(self._decode_datatype(type_name, + response['value'])) - def _normalize_xml_search_response(self, xml): - """ - Normalizes an XML search response so that PB and HTTP have the - same return value - """ - target = XMLSearchResult() - parser = ElementTree.XMLParser(target=target) - parser.feed(xml) - return parser.close() + return True - def _parse_content_type(self, value): + def get_preflist(self, bucket, key): """ - Split the content-type header into two parts: - 1) Actual main/sub encoding type - 2) charset + Get the preflist for a bucket/key - :param value: Complete MIME content-type string + :param bucket: Riak Bucket + :type bucket: :class:`~riak.bucket.RiakBucket` + :param key: Riak Key + :type key: string + :rtype: list of dicts """ - message = Message() - message.set_type(value) - - content_type = message.get_content_type() - charset = message.get_content_charset(None) + if not self.preflists(): + raise NotImplementedError("fetching preflists is not supported.") + bucket_type = self._get_bucket_type(bucket.bucket_type) + url = self.preflist_path(bucket.name, key, bucket_type=bucket_type) + status, headers, body = self._request('GET', url) - return content_type, charset + if status == 200: + preflist = json.loads(bytes_to_str(body)) + return preflist['preflist'] + else: + raise RiakError('Error getting bucket/key preflist.') - @classmethod - def build_headers(cls, headers): - return ['%s: %s' % (header, value) - for header, value in headers.iteritems()] + def check_http_code(self, status, expected_statuses): + if status not in expected_statuses: + raise RiakError('Expected status %s, received %s' % + (expected_statuses, status)) - @classmethod - def parse_http_headers(cls, headers): - """ - Parse an HTTP Header string into an associative array of - response headers. - """ - retVal = {} - fields = headers.split("\n") - for field in fields: - matches = re.match("([^:]+):(.+)", field) - if matches is None: - continue - key = matches.group(1).lower() - value = matches.group(2).strip() - if key in retVal.keys(): - if isinstance(retVal[key], list): - retVal[key].append(value) - else: - retVal[key] = [retVal[key]].append(value) - else: - retVal[key] = value - return retVal + def _get_bucket_type(self, bucket_type): + if bucket_type is None: + return None + if bucket_type.is_default(): + return None + elif not self.bucket_types(): + raise NotImplementedError('Server does not support bucket-types') + else: + return bucket_type.name diff --git a/riak/transports/pbc/__init__.py b/riak/transports/pbc/__init__.py deleted file mode 100644 index fc8914b6..00000000 --- a/riak/transports/pbc/__init__.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Copyright 2012 Basho Technologies, Inc. -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -import errno -import socket -from riak.transports.pool import Pool -from riak.transports.pbc.transport import RiakPbcTransport - - -class RiakPbcPool(Pool): - """ - A resource pool of PBC transports. - """ - def __init__(self, client, **options): - super(RiakPbcPool, self).__init__() - self._client = client - self._options = options - - def create_resource(self): - node = self._client._choose_node() - return RiakPbcTransport(node=node, - client=self._client, - **self._options) - - def destroy_resource(self, pbc): - pbc.close() - -# These are a specific set of socket errors -# that could be raised on send/recv that indicate -# that the socket is closed or reset, and is not -# usable. On seeing any of these errors, the socket -# should be closed, and the connection re-established. -CONN_CLOSED_ERRORS = ( - errno.EHOSTUNREACH, - errno.ECONNRESET, - errno.ECONNREFUSED, - errno.ECONNABORTED, - errno.ETIMEDOUT, - errno.EBADF, - errno.EPIPE -) - - -def is_retryable(err): - """ - Determines if the given exception is something that is - network/socket-related and should thus cause the PBC connection to - close and the operation retried on another node. - - :rtype: boolean - """ - if isinstance(err, socket.error): - code = err.args[0] - return code in CONN_CLOSED_ERRORS - else: - return False diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py deleted file mode 100644 index d41d8ed6..00000000 --- a/riak/transports/pbc/codec.py +++ /dev/null @@ -1,154 +0,0 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" -import riak_pb -from riak.riak_object import RiakObject - -RIAKC_RW_ONE = 4294967294 -RIAKC_RW_QUORUM = 4294967293 -RIAKC_RW_ALL = 4294967292 -RIAKC_RW_DEFAULT = 4294967291 - - -class RiakPbcCodec(object): - """ - Protobuffs Encoding and decoding methods for RiakPbcTransport. - """ - - rw_names = { - 'default': RIAKC_RW_DEFAULT, - 'all': RIAKC_RW_ALL, - 'quorum': RIAKC_RW_QUORUM, - 'one': RIAKC_RW_ONE - } - - def __init__(self, **unused_args): - if riak_pb is None: - raise NotImplementedError("this transport is not available") - super(RiakPbcCodec, self).__init__(**unused_args) - - def translate_rw_val(self, rw): - """ - Converts a symbolic quorum value into its on-the-wire - equivalent. - - :param rw: the quorum - :type rw: string, integer - :rtype: integer - """ - val = self.rw_names.get(rw) - if val is None: - return rw - elif type(rw) is int and rw >= 0: - return val - else: - return None - - def decode_content(self, rpb_content, robj): - """ - Decodes a single sibling from the protobuf representation into - a RiakObject. - - :rtype: (RiakObject) - """ - - if rpb_content.HasField("deleted"): - robj.deleted = True - if rpb_content.HasField("content_type"): - robj.content_type = rpb_content.content_type - if rpb_content.HasField("charset"): - robj.charset = rpb_content.charset - if rpb_content.HasField("content_encoding"): - robj.content_encoding = rpb_content.content_encoding - if rpb_content.HasField("vtag"): - robj.vtag = rpb_content.vtag - links = [] - for link in rpb_content.links: - if link.HasField("bucket"): - bucket = link.bucket - else: - bucket = None - if link.HasField("key"): - key = link.key - else: - key = None - if link.HasField("tag"): - tag = link.tag - else: - tag = None - links.append((bucket, key, tag)) - if links: - robj.links = links - if rpb_content.HasField("last_mod"): - robj.last_mod = rpb_content.last_mod - if rpb_content.HasField("last_mod_usecs"): - robj.last_mod_usecs = rpb_content.last_mod_usecs - usermeta = {} - for usermd in rpb_content.usermeta: - usermeta[usermd.key] = usermd.value - if len(usermeta) > 0: - robj.usermeta = usermeta - indexes = set() - for index in rpb_content.indexes: - if index.key.endswith("_int"): - indexes.add((index.key, int(index.value))) - else: - indexes.add((index.key, index.value)) - - if len(indexes) > 0: - robj.indexes = indexes - - robj.encoded_data = rpb_content.value - robj.exists = True - - return robj - - def encode_content(self, robj, rpb_content): - """ - Fills an RpbContent message with the appropriate data and - metadata from a RiakObject. - """ - if robj.content_type: - rpb_content.content_type = robj.content_type - if robj.charset: - rpb_content.charset = robj.charset - if robj.content_encoding: - rpb_content.content_encoding = robj.content_encoding - for uk in robj.usermeta: - pair = rpb_content.usermeta.add() - pair.key = uk - pair.value = robj.usermeta[uk] - for link in robj.links: - pb_link = rpb_content.links.add() - try: - bucket, key, tag = link - except ValueError: - raise RiakError("Invalid link tuple %s" % link) - - pb_link.bucket = bucket - pb_link.key = key - if tag: - pb_link.tag = tag - else: - pb_link.tag = '' - - for field, value in robj.indexes: - pair = rpb_content.indexes.add() - pair.key = field - pair.value = str(value) - - rpb_content.value = str(robj.encoded_data) diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py deleted file mode 100644 index 86e409b4..00000000 --- a/riak/transports/pbc/connection.py +++ /dev/null @@ -1,111 +0,0 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -import socket -import struct -from riak import RiakError -from messages import ( - MESSAGE_CLASSES, - MSG_CODE_ERROR_RESP -) - - -class RiakPbcConnection(object): - """ - Connection-related methods for RiakPbcTransport. - """ - - def _encode_msg(self, msg_code, msg=None): - if msg is None: - return struct.pack("!iB", 1, msg_code) - msgstr = msg.SerializeToString() - slen = len(msgstr) - hdr = struct.pack("!iB", 1 + slen, msg_code) - return hdr + msgstr - - def _request(self, msg_code, msg=None, expect=None): - self._send_msg(msg_code, msg) - return self._recv_msg(expect) - - def _send_msg(self, msg_code, msg): - self._socket.send(self._encode_msg(msg_code, msg)) - - def _recv_msg(self, expect=None): - self._recv_pkt() - msg_code, = struct.unpack("B", self._inbuf[:1]) - - if msg_code is MSG_CODE_ERROR_RESP: - err = self._parse_msg(msg_code, self._inbuf[1:]) - raise RiakError(err.errmsg) - elif msg_code in MESSAGE_CLASSES: - msg = self._parse_msg(msg_code, self._inbuf[1:]) - else: - raise Exception("unknown msg code %s" % msg_code) - - if expect and msg_code != expect: - raise RiakError("unexpected protocol buffer message code: %d, %r" - % (msg_code, msg)) - return msg_code, msg - - def _recv_pkt(self): - nmsglen = self._socket.recv(4) - if len(nmsglen) != 4: - raise RiakError( - "Socket returned short packet length %d - expected 4" - % len(nmsglen)) - msglen, = struct.unpack('!i', nmsglen) - self._inbuf_len = msglen - self._inbuf = '' - while len(self._inbuf) < msglen: - want_len = min(8192, msglen - len(self._inbuf)) - recv_buf = self._socket.recv(want_len) - if not recv_buf: - break - self._inbuf += recv_buf - if len(self._inbuf) != self._inbuf_len: - raise RiakError("Socket returned short packet %d - expected %d" - % (len(self._inbuf), self._inbuf_len)) - - def _connect(self): - if self._timeout: - self._socket = socket.create_connection(self._address, self._timeout) - else: - self._socket = socket.create_connection(self._address) - - def close(self): - """ - Closes the underlying socket of the PB connection. - """ - self._socket.shutdown(socket.SHUT_RDWR) - - def _parse_msg(self, code, packet): - try: - pbclass = MESSAGE_CLASSES[code] - except KeyError: - pbclass = None - - if pbclass is None: - return None - - pbo = pbclass() - pbo.ParseFromString(packet) - return pbo - - # These are set in the RiakPbcTransport initializer - _address = None - _timeout = None diff --git a/riak/transports/pbc/messages.py b/riak/transports/pbc/messages.py deleted file mode 100644 index fb9dba5c..00000000 --- a/riak/transports/pbc/messages.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -import riak_pb - - -# Protocol codes -MSG_CODE_ERROR_RESP = 0 -MSG_CODE_PING_REQ = 1 -MSG_CODE_PING_RESP = 2 -MSG_CODE_GET_CLIENT_ID_REQ = 3 -MSG_CODE_GET_CLIENT_ID_RESP = 4 -MSG_CODE_SET_CLIENT_ID_REQ = 5 -MSG_CODE_SET_CLIENT_ID_RESP = 6 -MSG_CODE_GET_SERVER_INFO_REQ = 7 -MSG_CODE_GET_SERVER_INFO_RESP = 8 -MSG_CODE_GET_REQ = 9 -MSG_CODE_GET_RESP = 10 -MSG_CODE_PUT_REQ = 11 -MSG_CODE_PUT_RESP = 12 -MSG_CODE_DEL_REQ = 13 -MSG_CODE_DEL_RESP = 14 -MSG_CODE_LIST_BUCKETS_REQ = 15 -MSG_CODE_LIST_BUCKETS_RESP = 16 -MSG_CODE_LIST_KEYS_REQ = 17 -MSG_CODE_LIST_KEYS_RESP = 18 -MSG_CODE_GET_BUCKET_REQ = 19 -MSG_CODE_GET_BUCKET_RESP = 20 -MSG_CODE_SET_BUCKET_REQ = 21 -MSG_CODE_SET_BUCKET_RESP = 22 -MSG_CODE_MAPRED_REQ = 23 -MSG_CODE_MAPRED_RESP = 24 -MSG_CODE_INDEX_REQ = 25 -MSG_CODE_INDEX_RESP = 26 -MSG_CODE_SEARCH_QUERY_REQ = 27 -MSG_CODE_SEARCH_QUERY_RESP = 28 - -# These responses don't include messages -EMPTY_RESPONSES = [ - MSG_CODE_PING_RESP, - MSG_CODE_SET_CLIENT_ID_RESP, - MSG_CODE_DEL_RESP, - MSG_CODE_SET_BUCKET_RESP -] - -# Mapping from code to protobuf class -MESSAGE_CLASSES = { - MSG_CODE_ERROR_RESP: riak_pb.RpbErrorResp, - MSG_CODE_PING_REQ: None, - MSG_CODE_PING_RESP: None, - MSG_CODE_GET_CLIENT_ID_REQ: None, - MSG_CODE_GET_CLIENT_ID_RESP: riak_pb.RpbGetClientIdResp, - MSG_CODE_SET_CLIENT_ID_REQ: riak_pb.RpbSetClientIdReq, - MSG_CODE_SET_CLIENT_ID_RESP: None, - MSG_CODE_GET_SERVER_INFO_REQ: None, - MSG_CODE_GET_SERVER_INFO_RESP: riak_pb.RpbGetServerInfoResp, - MSG_CODE_GET_REQ: riak_pb.RpbGetReq, - MSG_CODE_GET_RESP: riak_pb.RpbGetResp, - MSG_CODE_PUT_REQ: riak_pb.RpbPutReq, - MSG_CODE_PUT_RESP: riak_pb.RpbPutResp, - MSG_CODE_DEL_REQ: riak_pb.RpbDelReq, - MSG_CODE_DEL_RESP: None, - MSG_CODE_LIST_BUCKETS_REQ: None, - MSG_CODE_LIST_BUCKETS_RESP: riak_pb.RpbListBucketsResp, - MSG_CODE_LIST_KEYS_REQ: riak_pb.RpbListKeysReq, - MSG_CODE_LIST_KEYS_RESP: riak_pb.RpbListKeysResp, - MSG_CODE_GET_BUCKET_REQ: riak_pb.RpbGetBucketReq, - MSG_CODE_GET_BUCKET_RESP: riak_pb.RpbGetBucketResp, - MSG_CODE_SET_BUCKET_REQ: riak_pb.RpbSetBucketReq, - MSG_CODE_SET_BUCKET_RESP: None, - MSG_CODE_MAPRED_REQ: riak_pb.RpbMapRedReq, - MSG_CODE_MAPRED_RESP: riak_pb.RpbMapRedResp, - MSG_CODE_INDEX_REQ: riak_pb.RpbIndexReq, - MSG_CODE_INDEX_RESP: riak_pb.RpbIndexResp, - MSG_CODE_SEARCH_QUERY_REQ: riak_pb.RpbSearchQueryReq, - MSG_CODE_SEARCH_QUERY_RESP: riak_pb.RpbSearchQueryResp -} diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py deleted file mode 100644 index 02c69918..00000000 --- a/riak/transports/pbc/stream.py +++ /dev/null @@ -1,98 +0,0 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - - -import json -from riak.transports.pbc.messages import ( - MSG_CODE_LIST_KEYS_RESP, - MSG_CODE_MAPRED_RESP -) - - -class RiakPbcStream(object): - """ - Used internally by RiakPbcTransport to implement streaming - operations. Implements the iterator interface. - """ - - _expect = None - - def __init__(self, transport): - self.finished = False - self.transport = transport - - def __iter__(self): - return self - - def next(self): - if self.finished: - raise StopIteration - - msg_code, resp = self.transport._recv_msg(self._expect) - if(self._is_done(resp)): - self.finished = True - - return resp - - def _is_done(self, response): - # This could break if new messages don't name the field the - # same thing. - return response.done - - def close(self): - # We have to drain the socket to make sure that we don't get - # weird responses when some other request comes after a - # failed/prematurely-terminated one. - try: - while self.next(): - pass - except StopIteration: - pass - - -class RiakPbcKeyStream(RiakPbcStream): - """ - Used internally by RiakPbcTransport to implement key-list streams. - """ - - _expect = MSG_CODE_LIST_KEYS_RESP - - def next(self): - response = super(RiakPbcKeyStream, self).next() - - if response.done and len(response.keys) is 0: - raise StopIteration - - return response.keys - - -class RiakPbcMapredStream(RiakPbcStream): - """ - Used internally by RiakPbcTransport to implement MapReduce - streams. - """ - - _expect = MSG_CODE_MAPRED_RESP - - def next(self): - response = super(RiakPbcMapredStream, self).next() - - if response.done and not response.HasField('response'): - raise StopIteration - - return response.phase, json.loads(response.response) diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py deleted file mode 100644 index 542e97fb..00000000 --- a/riak/transports/pbc/transport.py +++ /dev/null @@ -1,424 +0,0 @@ -""" -Copyright 2012 Basho Technologies, Inc. -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -import riak_pb -from riak import RiakError -from riak.transports.transport import RiakTransport -from connection import RiakPbcConnection -from stream import RiakPbcKeyStream, RiakPbcMapredStream -from codec import RiakPbcCodec -from riak.riak_object import RiakObject - -from messages import ( - MSG_CODE_PING_REQ, - MSG_CODE_PING_RESP, - MSG_CODE_GET_CLIENT_ID_REQ, - MSG_CODE_GET_CLIENT_ID_RESP, - MSG_CODE_SET_CLIENT_ID_REQ, - MSG_CODE_SET_CLIENT_ID_RESP, - MSG_CODE_GET_SERVER_INFO_REQ, - MSG_CODE_GET_SERVER_INFO_RESP, - MSG_CODE_GET_REQ, - MSG_CODE_GET_RESP, - MSG_CODE_PUT_REQ, - MSG_CODE_PUT_RESP, - MSG_CODE_DEL_REQ, - MSG_CODE_DEL_RESP, - MSG_CODE_LIST_BUCKETS_REQ, - MSG_CODE_LIST_BUCKETS_RESP, - MSG_CODE_LIST_KEYS_REQ, - MSG_CODE_GET_BUCKET_REQ, - MSG_CODE_GET_BUCKET_RESP, - MSG_CODE_SET_BUCKET_REQ, - MSG_CODE_SET_BUCKET_RESP, - MSG_CODE_MAPRED_REQ, - MSG_CODE_INDEX_REQ, - MSG_CODE_INDEX_RESP, - MSG_CODE_SEARCH_QUERY_REQ, - MSG_CODE_SEARCH_QUERY_RESP -) - - -class RiakPbcTransport(RiakTransport, RiakPbcConnection, RiakPbcCodec): - """ - The RiakPbcTransport object holds a connection to the protocol - buffers interface on the riak server. - """ - - def __init__(self, node=None, client=None, timeout=None, *unused_options): - """ - Construct a new RiakPbcTransport object. - """ - super(RiakPbcTransport, self).__init__() - - self._client = client - self._node = node - self._address = (node.host, node.pb_port) - self._timeout = timeout - self._connect() - - # FeatureDetection API - def _server_version(self): - return self.get_server_info()['server_version'] - - def ping(self): - """ - Ping the remote server - """ - - msg_code, msg = self._request(MSG_CODE_PING_REQ) - if msg_code == MSG_CODE_PING_RESP: - return True - else: - return False - - def get_server_info(self): - """ - Get information about the server - """ - msg_code, resp = self._request(MSG_CODE_GET_SERVER_INFO_REQ, - expect=MSG_CODE_GET_SERVER_INFO_RESP) - return {'node': resp.node, 'server_version': resp.server_version} - - def _get_client_id(self): - msg_code, resp = self._request(MSG_CODE_GET_CLIENT_ID_REQ, - expect=MSG_CODE_GET_CLIENT_ID_RESP) - return resp.client_id - - def _set_client_id(self, client_id): - req = riak_pb.RpbSetClientIdReq() - req.client_id = client_id - - msg_code, resp = self._request(MSG_CODE_SET_CLIENT_ID_REQ, req, - MSG_CODE_SET_CLIENT_ID_RESP) - - self._client_id = client_id - - client_id = property(_get_client_id, _set_client_id, - doc="""the client ID for this connection""") - - def _decoded_contents(self, resp, old_obj): - contents = [] - for c in resp.content: - new_obj = RiakObject(old_obj.client, old_obj.bucket, old_obj.key) - new_obj.vclock = resp.vclock - contents.append(self.decode_content(c, new_obj)) - if contents: - ret = contents[0] - if len(contents) > 1: - ret.siblings = contents[:] - return ret - else: - old_obj.exists = False - return old_obj - - def get(self, robj, r=None, pr=None, vtag=None): - """ - Serialize get request and deserialize response - """ - if vtag is not None: - raise RiakError("PB transport does not support vtags") - - bucket = robj.bucket - - req = riak_pb.RpbGetReq() - if r: - req.r = self.translate_rw_val(r) - if self.quorum_controls() and pr: - req.pr = self.translate_rw_val(pr) - - if self.tombstone_vclocks(): - req.deletedvclock = 1 - - req.bucket = bucket.name - req.key = robj.key - - msg_code, resp = self._request(MSG_CODE_GET_REQ, req) - if msg_code == MSG_CODE_GET_RESP: - return self._decoded_contents(resp, robj) - else: - return None - - def put(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """ - Serialize get request and deserialize response - """ - bucket = robj.bucket - - req = riak_pb.RpbPutReq() - if w: - req.w = self.translate_rw_val(w) - if dw: - req.dw = self.translate_rw_val(dw) - if self.quorum_controls() and pw: - req.pw = self.translate_rw_val(pw) - - if return_body: - req.return_body = 1 - if if_none_match: - req.if_none_match = 1 - - req.bucket = bucket.name - req.key = robj.key - if robj.vclock: - req.vclock = robj.vclock - - self.encode_content(robj, req.content) - - msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, - MSG_CODE_PUT_RESP) - if resp is not None: - return self._decoded_contents(resp, robj) - - def put_new(self, robj, w=None, dw=None, pw=None, return_body=True, - if_none_match=False): - """Put a new object into the Riak store, returning its (new) key. - - If return_meta is False, then the vlock and metadata return values - will be None. - - @return robj - """ - # Note that this won't work on 0.14 nodes. - bucket = robj.bucket - - req = riak_pb.RpbPutReq() - if w: - req.w = self.translate_rw_val(w) - if dw: - req.dw = self.translate_rw_val(dw) - if self.quorum_controls() and pw: - req.pw = self.translate_rw_val(pw) - - if return_body: - req.return_body = 1 - if if_none_match: - req.if_none_match = 1 - - req.bucket = bucket.name - - self.encode_content(robj, req.content) - - msg_code, resp = self._request(MSG_CODE_PUT_REQ, req, - MSG_CODE_PUT_RESP) - if not resp: - raise RiakError("missing response object") - if len(resp.content) != 1: - raise RiakError("siblings were returned from object creation") - - robj.key = resp.key - robj.vclock = resp.vclock - content = self.decode_content(resp.content[0], robj) - return content - - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None): - """ - Serialize get request and deserialize response - """ - bucket = robj.bucket - - req = riak_pb.RpbDelReq() - if rw: - req.rw = self.translate_rw_val(rw) - if r: - req.r = self.translate_rw_val(r) - if w: - req.w = self.translate_rw_val(w) - if dw: - req.dw = self.translate_rw_val(dw) - - if self.quorum_controls(): - if pr: - req.pr = self.translate_rw_val(pr) - if pw: - req.pw = self.translate_rw_val(pw) - - if self.tombstone_vclocks() and robj.vclock: - req.vclock = robj.vclock - - req.bucket = bucket.name - req.key = robj.key - - msg_code, resp = self._request(MSG_CODE_DEL_REQ, req, - MSG_CODE_DEL_RESP) - return self - - def get_keys(self, bucket): - """ - Lists all keys within a bucket. - """ - keys = [] - for keylist in self.stream_keys(bucket): - for key in keylist: - keys.append(key) - - return keys - - def stream_keys(self, bucket): - """ - Streams keys from a bucket, returning an iterator that yields - lists of keys. - """ - req = riak_pb.RpbListKeysReq() - req.bucket = bucket.name - - self._send_msg(MSG_CODE_LIST_KEYS_REQ, req) - - return RiakPbcKeyStream(self) - - def get_buckets(self): - """ - Serialize bucket listing request and deserialize response - """ - msg_code, resp = self._request(MSG_CODE_LIST_BUCKETS_REQ, - expect=MSG_CODE_LIST_BUCKETS_RESP) - return resp.buckets - - def get_bucket_props(self, bucket): - """ - Serialize bucket property request and deserialize response - """ - req = riak_pb.RpbGetBucketReq() - req.bucket = bucket.name - - msg_code, resp = self._request(MSG_CODE_GET_BUCKET_REQ, req, - MSG_CODE_GET_BUCKET_RESP) - props = {} - if resp.props.HasField('n_val'): - props['n_val'] = resp.props.n_val - if resp.props.HasField('allow_mult'): - props['allow_mult'] = resp.props.allow_mult - - return props - - def set_bucket_props(self, bucket, props): - """ - Serialize set bucket property request and deserialize response - """ - req = riak_pb.RpbSetBucketReq() - req.bucket = bucket.name - for key in props: - if key not in ['n_val', 'allow_mult']: - raise NotImplementedError - - if 'n_val' in props: - req.props.n_val = props['n_val'] - if 'allow_mult' in props: - req.props.allow_mult = props['allow_mult'] - - msg_code, resp = self._request(MSG_CODE_SET_BUCKET_REQ, req, - MSG_CODE_SET_BUCKET_RESP) - return self - - def mapred(self, inputs, query, timeout=None): - # dictionary of phase results - each content should be an encoded array - # which is appended to the result for that phase. - result = {} - for phase, content in self.stream_mapred(inputs, query, timeout): - if phase in result: - result[phase] += content - else: - result[phase] = content - - # If a single result - return the same as the HTTP interface does - # otherwise return all the phase information - if not len(result): - return None - elif len(result) == 1: - return result[max(result.keys())] - else: - return result - - def stream_mapred(self, inputs, query, timeout=None): - # Construct the job, optionally set the timeout... - content = self._construct_mapred_json(inputs, query, timeout) - - req = riak_pb.RpbMapRedReq() - req.request = content - req.content_type = "application/json" - - self._send_msg(MSG_CODE_MAPRED_REQ, req) - - return RiakPbcMapredStream(self) - - def get_index(self, bucket, index, startkey, endkey=None): - if not self.pb_indexes(): - return self._get_index_mapred_emu(bucket, index, startkey, endkey) - - req = riak_pb.RpbIndexReq(bucket=bucket, index=index) - if endkey: - req.qtype = riak_pb.RpbIndexReq.range - req.range_min = str(startkey) - req.range_max = str(endkey) - else: - req.qtype = riak_pb.RpbIndexReq.eq - req.key = str(startkey) - - msg_code, resp = self._request(MSG_CODE_INDEX_REQ, req, - MSG_CODE_INDEX_RESP) - return resp.keys - - def search(self, index, query, **params): - if not self.pb_search(): - return self._search_mapred_emu(index, query) - - req = riak_pb.RpbSearchQueryReq(index=index, q=query) - if 'rows' in params: - req.rows = params['rows'] - if 'start' in params: - req.start = params['start'] - if 'sort' in params: - req.sort = params['sort'] - if 'filter' in params: - req.filter = params['filter'] - if 'df' in params: - req.df = params['df'] - if 'op' in params: - req.op = params['op'] - if 'q.op' in params: - req.op = params['q.op'] - if 'fl' in params: - if isinstance(params['fl'], list): - req.fl.extend(params['fl']) - else: - req.fl.append(params['fl']) - if 'presort' in params: - req.presort = params['presort'] - - msg_code, resp = self._request(MSG_CODE_SEARCH_QUERY_REQ, req, - MSG_CODE_SEARCH_QUERY_RESP) - - result = {} - if resp.HasField('max_score'): - result['max_score'] = resp.max_score - if resp.HasField('num_found'): - result['num_found'] = resp.num_found - docs = [] - for doc in resp.docs: - resultdoc = {} - for pair in doc.fields: - ukey = unicode(pair.key, 'utf-8') - uval = unicode(pair.value, 'utf-8') - resultdoc[ukey] = uval - docs.append(resultdoc) - result['docs'] = docs - return result diff --git a/riak/transports/pool.py b/riak/transports/pool.py index 13d5cdf7..38a87b43 100644 --- a/riak/transports/pool.py +++ b/riak/transports/pool.py @@ -1,52 +1,87 @@ -""" -Copyright 2012 Basho Technologies, Inc. +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at +import threading - http://www.apache.org/licenses/LICENSE-2.0 +from contextlib import contextmanager -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" -from contextlib import contextmanager -import threading +class BadResource(Exception): + """ + Users of a :class:`Pool` should raise this error when the pool + resource currently in-use is bad and should be removed from the + pool. + + :param mid_stream: did this exception happen mid-streaming op? + :type mid_stream: boolean + """ + def __init__(self, ex, mid_stream=False): + super(BadResource, self).__init__(ex) + self.mid_stream = mid_stream -# This file is a rough port of the Innertube Ruby library -class BadResource(StandardError): +class ConnectionClosed(BadResource): """ - Users of a Pool should raise this error when the pool element - currently in-use is bad and should be removed from the pool. + Users of a :class:`Pool` should raise this error when the pool + resource currently in-use has been closed and should be removed + from the pool. + + :param mid_stream: did this exception happen mid-streaming op? + :type mid_stream: boolean """ - pass + def __init__(self, ex, mid_stream=False): + super(ConnectionClosed, self).__init__(ex, mid_stream) -class Element(object): +class Resource(object): """ - A member of the Pool, a container for the actual resource being - pooled and a marker for whether the resource is currently claimed. + A member of the :class:`Pool`, a container for the actual resource + being pooled and a marker for whether the resource is currently + claimed. """ - def __init__(self, obj): + def __init__(self, obj, pool): """ - Creates a new Element, wrapping the passed object as the + Creates a new Resource, wrapping the passed object as the pooled resource. + :param obj: the resource to wrap :type obj: object """ """The wrapped pool resource.""" self.object = obj + """Whether the resource is currently in use.""" self.claimed = False + """The pool that this resource belongs to.""" + self.pool = pool + + """True if this Resource errored.""" + self.errored = False + + def release(self): + """ + Releases this resource back to the pool it came from. + """ + if self.errored: + self.pool.delete_resource(self) + else: + self.pool.release(self) + class Pool(object): """ @@ -55,13 +90,14 @@ class Pool(object): the create_resource and destroy_resource functions that are responsible for creating and cleaning up the resources in the pool, respectively. Claiming a resource of the pool for a block of - code is done using a with statement on the take method. The take - method also allows filtering of the pool and supplying a default - value to be used as the resource if no elements are free. + code is done using a with statement on the transaction method. The + transaction method also allows filtering of the pool and supplying + a default value to be used as the resource if no resources are + free. - Example: + Example:: - from riak.Pool import Pool, BadResource + from riak.transports.pool import Pool class ListPool(Pool): def create_resource(self): return [] @@ -71,34 +107,36 @@ def destroy_resource(self): pass pool = ListPool() - with pool.take() as resource: + with pool.transaction() as resource: resource.append(1) - with pool.take() as resource2: - print repr(resource2) # should be [1] + with pool.transaction() as resource2: + print(repr(resource2)) # should be [1] """ def __init__(self): """ Creates a new Pool. This should be called manually if you - override the __init__ method in a subclass. + override the :meth:`__init__` method in a subclass. """ self.lock = threading.RLock() self.releaser = threading.Condition(self.lock) - self.elements = list() + self.resources = list() - @contextmanager - def take(self, _filter=None, default=None): + def acquire(self, _filter=None, default=None): """ - Claims a resource from the pool for use in a thread-safe, - reentrant manner (as part of a with statement). Resources are + acquire(_filter=None, default=None) + + Claims a resource from the pool for manual use. Resources are created as needed when all members of the pool are claimed or - the pool is empty. + the pool is empty. Most of the time you will want to use + :meth:`transaction`. :param _filter: a filter that can be used to select a member of the pool :type _filter: callable :param default: a value that will be used instead of calling - create_resource if a new resource needs to be created + :meth:`create_resource` if a new resource needs to be created + :rtype: Resource """ if not _filter: def _filter(obj): @@ -106,56 +144,95 @@ def _filter(obj): elif not callable(_filter): raise TypeError("_filter is not a callable") - element = None + resource = None with self.lock: - for e in self.elements: + for e in self.resources: if not e.claimed and _filter(e.object): - element = e + resource = e break - if element is None: + if resource is None: if default is not None: - element = Element(default) + resource = Resource(default, self) else: - element = Element(self.create_resource()) - self.elements.append(element) - element.claimed = True + resource = Resource(self.create_resource(), self) + self.resources.append(resource) + resource.claimed = True + return resource + + def release(self, resource): + """release(resource) + + Returns a resource to the pool. Most of the time you will want + to use :meth:`transaction`, but if you use :meth:`acquire`, + you must release the acquired resource back to the pool when + finished. Failure to do so could result in deadlock. + + :param resource: Resource + """ + with self.releaser: + resource.claimed = False + self.releaser.notify_all() + + @contextmanager + def transaction(self, _filter=None, default=None, yield_resource=False): + """ + transaction(_filter=None, default=None) + + Claims a resource from the pool for use in a thread-safe, + reentrant manner (as part of a with statement). Resources are + created as needed when all members of the pool are claimed or + the pool is empty. + + :param _filter: a filter that can be used to select a member + of the pool + :type _filter: callable + :param default: a value that will be used instead of calling + :meth:`create_resource` if a new resource needs to be created + :param yield_resource: set to True to yield the Resource object + itself + :type yield_resource: boolean + """ + resource = self.acquire(_filter=_filter, default=default) try: - yield element.object + if yield_resource: + yield resource + else: + yield resource.object + if resource.errored: + self.delete_resource(resource) except BadResource: - self.delete_element(element) + self.delete_resource(resource) raise finally: - with self.releaser: - element.claimed = False - self.releaser.notify_all() + self.release(resource) - def delete_element(self, element): + def delete_resource(self, resource): """ - Deletes the element from the pool and destroys the associated + Deletes the resource from the pool and destroys the associated resource. Not usually needed by users of the pool, but called internally when BadResource is raised. - :param element: the element to remove - :type element: Element + :param resource: the resource to remove + :type resource: Resource """ with self.lock: - self.elements.remove(element) - self.destroy_resource(element.object) - del element + self.resources.remove(resource) + self.destroy_resource(resource.object) + del resource def __iter__(self): """ - Iterator callback to iterate over the elements of the pool. + Iterator callback to iterate over the resources of the pool. """ return PoolIterator(self) def clear(self): """ - Removes all resources from the pool, calling delete_element + Removes all resources from the pool, calling :meth:`delete_resource` with each one so that the resources are cleaned up. """ - for element in self: - self.delete_element(element) + for resource in self: + self.delete_resource(resource) def create_resource(self): """ @@ -192,7 +269,7 @@ class PoolIterator(object): def __init__(self, pool): with pool.lock: - self.targets = pool.elements[:] + self.targets = pool.resources[:] self.unlocked = [] self.lock = pool.lock self.releaser = pool.releaser @@ -201,25 +278,30 @@ def __iter__(self): return self def next(self): + # Python 2.x version if len(self.targets) == 0: raise StopIteration if len(self.unlocked) == 0: - self.__claim_elements() + self.__claim_resources() return self.unlocked.pop(0) - def __claim_elements(self): + def __next__(self): + # Python 3.x version + return self.next() + + def __claim_resources(self): with self.lock: with self.releaser: if self.__all_claimed(): self.releaser.wait() - for element in self.targets: - if not element.claimed: - self.targets.remove(element) - self.unlocked.append(element) - element.claimed = True + for resource in self.targets: + if not resource.claimed: + self.targets.remove(resource) + self.unlocked.append(resource) + resource.claimed = True def __all_claimed(self): - for element in self.targets: - if not element.claimed: + for resource in self.targets: + if not resource.claimed: return False return True diff --git a/riak/transports/security.py b/riak/transports/security.py new file mode 100644 index 00000000..01cf6315 --- /dev/null +++ b/riak/transports/security.py @@ -0,0 +1,337 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import socket +from riak.security import SecurityError, USE_STDLIB_SSL +if USE_STDLIB_SSL: + import ssl +else: + import OpenSSL.SSL + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO + + +def verify_cb(conn, cert, errnum, depth, ok): + """ + The default OpenSSL certificate verification callback. + """ + if not ok: + raise SecurityError("Could not verify CA certificate {0}" + .format(cert.get_subject())) + return ok + + +if USE_STDLIB_SSL: + def configure_ssl_context(credentials): + """ + Set various options on the SSL context for Python >= 2.7.9 and 3.x. + + N.B. versions earlier than 3.4 may not support all security + measures, e.g., hostname check. + + :param credentials: Riak Security Credentials + :type credentials: :class:`~riak.security.SecurityCreds` + :rtype :class:`~ssl.SSLContext` + """ + + ssl_ctx = ssl.SSLContext(credentials.ssl_version) + ssl_ctx.verify_mode = ssl.CERT_REQUIRED + if hasattr(ssl_ctx, 'check_hostname'): + ssl_ctx.check_hostname = True + if credentials.cacert_file is None: + raise SecurityError("cacert_file is required in SecurityCreds") + if credentials.ciphers is not None: + ssl_ctx.set_ciphers(credentials.ciphers) + + ssl_ctx.load_verify_locations(credentials.cacert_file) + if credentials.ciphers is not None: + ssl_ctx.set_ciphers(credentials.ciphers) + + pkeyfile = credentials.pkey_file + certfile = credentials.cert_file + if pkeyfile and not certfile: + raise SecurityError("cert_file must be specified with pkey_file") + if certfile and not pkeyfile: + pkeyfile = certfile + if certfile: + ssl_ctx.load_cert_chain(certfile, pkeyfile) + # TODO https://bugs.python.org/issue8813 + if credentials.crl_file is not None: + ssl_ctx.load_verify_locations(credentials.crl_file) + ssl_ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF + + # SSLv2 considered harmful. + ssl_ctx.options |= ssl.OP_NO_SSLv2 + + # SSLv3 has problematic security and is only required for really old + # clients such as IE6 on Windows XP + ssl_ctx.options |= ssl.OP_NO_SSLv3 + + # disable compression to prevent CRIME attacks (OpenSSL 1.0+) + ssl_ctx.options |= ssl.OP_NO_COMPRESSION + + return ssl_ctx + +else: + def configure_pyopenssl_context(credentials): + """ + Set various options on the SSL context for Python <= 2.7.8. + + :param credentials: Riak Security Credentials + :type credentials: :class:`~riak.security.SecurityCreds` + :rtype ssl_ctx: :class:`~OpenSSL.SSL.Context` + """ + + ssl_ctx = OpenSSL.SSL.Context(credentials.ssl_version) + if credentials._has_credential('pkey'): + ssl_ctx.use_privatekey(credentials.pkey) + if credentials._has_credential('cert'): + ssl_ctx.use_certificate(credentials.cert) + if credentials._has_credential('cacert'): + store = ssl_ctx.get_cert_store() + cacerts = credentials.cacert + if not isinstance(cacerts, list): + cacerts = [cacerts] + for cacert in cacerts: + store.add_cert(cacert) + else: + raise SecurityError("cacert_file is required in SecurityCreds") + ciphers = credentials.ciphers + if ciphers is not None: + ssl_ctx.set_cipher_list(ciphers) + # Demand a certificate + ssl_ctx.set_verify(OpenSSL.SSL.VERIFY_PEER | + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, + verify_cb) + return ssl_ctx + + # Inspired by + # https://github.com/shazow/urllib3/blob/master/urllib3/contrib/pyopenssl.py + class RiakWrappedSocket(socket.socket): + def __init__(self, connection, socket): + """ + API-compatibility wrapper for Python OpenSSL's Connection-class. + + :param connection: OpenSSL connection + :type connection: OpenSSL.SSL.Connection + :param socket: Underlying already connected socket + :type socket: socket + """ + self.connection = connection + self.socket = socket + + def fileno(self): + return self.socket.fileno() + + def makefile(self, mode, bufsize=-1): + return fileobject(self.connection, mode, bufsize) + + def settimeout(self, timeout): + return self.socket.settimeout(timeout) + + def sendall(self, data): + # SSL seems to need bytes, so force the data to byte encoding + return self.connection.sendall(bytes(data)) + + def close(self): + try: + return self.connection.shutdown() + except OpenSSL.SSL.Error as err: + if err.args == ([],): + return False + else: + raise err + + # Blatantly Stolen from + # https://github.com/shazow/urllib3/blob/master/urllib3/contrib/pyopenssl.py + # which is basically a port of the `socket._fileobject` class + class fileobject(socket._fileobject): + """ + Extension of the socket module's fileobject to use PyOpenSSL. + """ + + def read(self, size=-1): + # Use max, disallow tiny reads in a loop as they are very + # inefficient. We never leave read() with any leftover data from + # a new recv() call in our internal buffer. + rbufsize = max(self._rbufsize, self.default_bufsize) + # Our use of StringIO rather than lists of string objects returned + # by recv() minimizes memory usage and fragmentation that occurs + # when rbufsize is large compared to the typical return value of + # recv(). + buf = self._rbuf + buf.seek(0, 2) # seek end + if size < 0: + # Read until EOF + self._rbuf = StringIO() # reset _rbuf. we consume it via buf. + while True: + try: + data = self._sock.recv(rbufsize) + except OpenSSL.SSL.WantReadError: + continue + if not data: + break + buf.write(data) + return buf.getvalue() + else: + # Read until size bytes or EOF seen, whichever comes first + buf_len = buf.tell() + if buf_len >= size: + # Already have size bytes in our buffer? Extract and + # return. + buf.seek(0) + rv = buf.read(size) + self._rbuf = StringIO() + self._rbuf.write(buf.read()) + return rv + + self._rbuf = StringIO() # reset _rbuf. we consume it via buf. + while True: + left = size - buf_len + # recv() will malloc the amount of memory given as its + # parameter even though it often returns much less data + # than that. The returned data string is short lived + # as we copy it into a StringIO and free it. This avoids + # fragmentation issues on many platforms. + try: + data = self._sock.recv(left) + except OpenSSL.SSL.WantReadError: + continue + if not data: + break + n = len(data) + if n == size and not buf_len: + # Shortcut. Avoid buffer data copies when: + # - We have no data in our buffer. + # AND + # - Our call to recv returned exactly the + # number of bytes we were asked to read. + return data + if n == left: + buf.write(data) + # del data # explicit free + break + assert n <= left, "recv(%d) returned %d bytes" % (left, n) + buf.write(data) + buf_len += n + # del data # explicit free + # assert buf_len == buf.tell() + # Moved del outside of loop to keep pyflakes happy + if data: + del data + return buf.getvalue() + + def readline(self, size=-1): + data = None + buf = self._rbuf + buf.seek(0, 2) # seek end + if buf.tell() > 0: + # check if we already have it in our buffer + buf.seek(0) + bline = buf.readline(size) + if bline.endswith('\n') or len(bline) == size: + self._rbuf = StringIO() + self._rbuf.write(buf.read()) + return bline + del bline + if size < 0: + # Read until \n or EOF, whichever comes first + if self._rbufsize <= 1: + # Speed up unbuffered case + buf.seek(0) + buffers = [buf.read()] + # reset _rbuf. we consume it via buf. + self._rbuf = StringIO() + data = None + recv = self._sock.recv + while True: + try: + while data != "\n": + data = recv(1) + if not data: + break + buffers.append(data) + except OpenSSL.SSL.WantReadError: + continue + break + return "".join(buffers) + + buf.seek(0, 2) # seek end + self._rbuf = StringIO() # reset _rbuf. we consume it via buf. + while True: + try: + data = self._sock.recv(self._rbufsize) + except OpenSSL.SSL.WantReadError: + continue + if not data: + break + nl = data.find('\n') + if nl >= 0: + nl += 1 + buf.write(data[:nl]) + self._rbuf.write(data[nl:]) + # del data + break + buf.write(data) + # Moved del outside of loop to keep pyflakes happy + if data: + del data + return buf.getvalue() + else: + # Read until size bytes or \n or EOF seen, whichever comes 1st + buf.seek(0, 2) # seek end + buf_len = buf.tell() + if buf_len >= size: + buf.seek(0) + rv = buf.read(size) + self._rbuf = StringIO() + self._rbuf.write(buf.read()) + return rv + self._rbuf = StringIO() # reset _rbuf. we consume it via buf. + while True: + try: + data = self._sock.recv(self._rbufsize) + except OpenSSL.SSL.WantReadError: + continue + if not data: + break + left = size - buf_len + # did we just receive a newline? + nl = data.find('\n', 0, left) + if nl >= 0: + nl += 1 + # save the excess data to _rbuf + self._rbuf.write(data[nl:]) + if buf_len: + buf.write(data[:nl]) + break + else: + # Shortcut. Avoid data copy through buf when + # returning a substring of our first recv(). + return data[:nl] + n = len(data) + if n == size and not buf_len: + # Shortcut. Avoid data copy through buf when + # returning exactly all of our first recv(). + return data + if n >= left: + buf.write(data[:left]) + self._rbuf.write(data[left:]) + break + buf.write(data) + buf_len += n + # assert buf_len == buf.tell() + return buf.getvalue() diff --git a/riak/transports/tcp/__init__.py b/riak/transports/tcp/__init__.py new file mode 100644 index 00000000..d58add2e --- /dev/null +++ b/riak/transports/tcp/__init__.py @@ -0,0 +1,75 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import errno +import socket + +from riak.transports.pool import Pool, ConnectionClosed +from riak.transports.tcp.transport import TcpTransport + + +class TcpPool(Pool): + """ + A resource pool of TCP transports. + """ + def __init__(self, client, **options): + super(TcpPool, self).__init__() + self._client = client + self._options = options + + def create_resource(self): + node = self._client._choose_node() + return TcpTransport(node=node, + client=self._client, + **self._options) + + def destroy_resource(self, tcp): + tcp.close() + + +# These are a specific set of socket errors +# that could be raised on send/recv that indicate +# that the socket is closed or reset, and is not +# usable. On seeing any of these errors, the socket +# should be closed, and the connection re-established. +CONN_CLOSED_ERRORS = ( + errno.EHOSTUNREACH, + errno.ECONNRESET, + errno.ECONNREFUSED, + errno.ECONNABORTED, + errno.ETIMEDOUT, + errno.EBADF, + errno.EPIPE +) + + +def is_retryable(err): + """ + Determines if the given exception is something that is + network/socket-related and should thus cause the TCP connection to + close and the operation retried on another node. + + :rtype: boolean + """ + if isinstance(err, ConnectionClosed): + # NB: only retryable if we're not mid-streaming + if err.mid_stream: + return False + else: + return True + elif isinstance(err, socket.error): + code = err.args[0] + return code in CONN_CLOSED_ERRORS + else: + return False diff --git a/riak/transports/tcp/connection.py b/riak/transports/tcp/connection.py new file mode 100644 index 00000000..13c02cf4 --- /dev/null +++ b/riak/transports/tcp/connection.py @@ -0,0 +1,283 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import errno +import logging +import socket +import struct +import six +import riak.pb.riak_pb2 +import riak.pb.messages + +from riak import RiakError +from riak.codecs.pbuf import PbufCodec +from riak.security import SecurityError, USE_STDLIB_SSL +from riak.transports.pool import BadResource, ConnectionClosed + +if USE_STDLIB_SSL: + import ssl + from riak.transports.security import configure_ssl_context +else: + from OpenSSL.SSL import Connection + from riak.transports.security import configure_pyopenssl_context + + +class TcpConnection(object): + def __init__(self): + self.bytes_required = False + + """ + Connection-related methods for TcpTransport. + """ + def _encode_msg(self, msg_code, data=None): + if data is None: + return struct.pack("!iB", 1, msg_code) + hdr = struct.pack("!iB", 1 + len(data), msg_code) + return hdr + data + + def _send_recv(self, msg_code, data=None): + self._send_msg(msg_code, data) + return self._recv_msg() + + def _non_connect_send_recv(self, msg_code, data=None): + """ + Similar to self._send_recv, but doesn't try to initiate a connection, + thus preventing an infinite loop. + """ + self._non_connect_send_msg(msg_code, data) + return self._recv_msg() + + def _non_connect_send_recv_msg(self, msg): + self._non_connect_send_msg(msg.msg_code, msg.data) + return self._recv_msg() + + def _non_connect_send_msg(self, msg_code, data): + """ + Similar to self._send, but doesn't try to initiate a connection, + thus preventing an infinite loop. + """ + try: + self._socket.sendall(self._encode_msg(msg_code, data)) + except (IOError, socket.error) as e: + if e.errno == errno.EPIPE: + raise ConnectionClosed(e) + else: + raise + + def _send_msg(self, msg_code, data): + self._connect() + self._non_connect_send_msg(msg_code, data) + + def _init_security(self): + """ + Initialize a secure connection to the server. + """ + if not self._starttls(): + raise SecurityError("Could not start TLS connection") + # _ssh_handshake() will throw an exception upon failure + self._ssl_handshake() + if not self._auth(): + raise SecurityError("Could not authorize connection") + + def _starttls(self): + """ + Exchange a STARTTLS message with Riak to initiate secure communications + return True is Riak responds with a STARTTLS response, False otherwise + """ + resp_code, _ = self._non_connect_send_recv( + riak.pb.messages.MSG_CODE_START_TLS) + if resp_code == riak.pb.messages.MSG_CODE_START_TLS: + return True + else: + return False + + def _auth(self): + """ + Perform an authorization request against Riak + returns True upon success, False otherwise + Note: Riak will sleep for a short period of time upon a failed + auth request/response to prevent denial of service attacks + """ + codec = PbufCodec() + username = self._client._credentials.username + password = self._client._credentials.password + if not password: + password = '' + msg = codec.encode_auth(username, password) + resp_code, _ = self._non_connect_send_recv_msg(msg) + if resp_code == riak.pb.messages.MSG_CODE_AUTH_RESP: + return True + else: + return False + + if not USE_STDLIB_SSL: + def _ssl_handshake(self): + """ + Perform an SSL handshake w/ the server. + Precondition: a successful STARTTLS exchange has + taken place with Riak + returns True upon success, otherwise an exception is raised + """ + if self._client._credentials: + try: + ssl_ctx = configure_pyopenssl_context(self. + _client._credentials) + # attempt to upgrade the socket to SSL + ssl_socket = Connection(ssl_ctx, self._socket) + ssl_socket.set_connect_state() + ssl_socket.do_handshake() + # ssl handshake successful + self._socket = ssl_socket + + self._client._credentials._check_revoked_cert(ssl_socket) + return True + except Exception as e: + # fail if *any* exceptions are thrown during SSL handshake + raise SecurityError(e) + else: + def _ssl_handshake(self): + """ + Perform an SSL handshake w/ the server. + Precondition: a successful STARTTLS exchange has + taken place with Riak + returns True upon success, otherwise an exception is raised + """ + credentials = self._client._credentials + if credentials: + try: + ssl_ctx = configure_ssl_context(credentials) + host = self._address[0] + ssl_socket = ssl.SSLSocket(sock=self._socket, + keyfile=credentials.pkey_file, + certfile=credentials.cert_file, + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=credentials. + cacert_file, + ciphers=credentials.ciphers, + server_hostname=host) + ssl_socket.context = ssl_ctx + # ssl handshake successful + ssl_socket.do_handshake() + self._socket = ssl_socket + return True + except ssl.SSLError as e: + raise SecurityError(e) + except Exception as e: + # fail if *any* exceptions are thrown during SSL handshake + raise SecurityError(e) + + def _recv_msg(self, mid_stream=False): + """ + :param mid_stream: are we receiving in a streaming operation? + :type mid_stream: boolean + """ + try: + msgbuf = self._recv_pkt() + except BadResource as e: + e.mid_stream = mid_stream + raise + except socket.timeout as e: + # A timeout can leave the socket in an inconsistent state because + # it might still receive the data later and mix up with a + # subsequent request. + # https://github.com/basho/riak-python-client/issues/425 + raise BadResource(e, mid_stream) + mv = memoryview(msgbuf) + mcb = mv[0:1] + if self.bytes_required: + mcb = mcb.tobytes() + try: + msg_code, = struct.unpack("B", mcb) + except struct.error: + # NB: Python 2.7.3 requires this + # http://bugs.python.org/issue10212 + msg_code, = struct.unpack("B", mv[0:1].tobytes()) + self.bytes_required = True + data = mv[1:].tobytes() + return (msg_code, data) + + def _recv_pkt(self): + # TODO FUTURE re-use buffer + msglen_buf = self._recv(4) + # NB: msg length is an unsigned int + if self.bytes_required: + msglen_buf = bytes(msglen_buf) + try: + msglen, = struct.unpack('!I', msglen_buf) + except struct.error: + # NB: Python 2.7.3 requires this + # http://bugs.python.org/issue10212 + msglen, = struct.unpack('!I', bytes(msglen_buf)) + self.bytes_required = True + return self._recv(msglen) + + def _recv(self, msglen): + # TODO FUTURE re-use buffer + # http://stackoverflow.com/a/15964489 + msgbuf = bytearray(msglen) + view = memoryview(msgbuf) + nread = 0 + toread = msglen + while toread: + nbytes = self._socket.recv_into(view, toread) + # https://docs.python.org/2/howto/sockets.html#using-a-socket + # https://github.com/basho/riak-python-client/issues/399 + if nbytes == 0: + msg = 'socket recv returned zero bytes unexpectedly, ' \ + 'expected {}'.format(toread) + ex = RiakError(msg) + raise ConnectionClosed(ex) + view = view[nbytes:] # slicing views is cheap + toread -= nbytes + nread += nbytes + if nread != msglen: + raise RiakError("Socket returned short packet %d - expected %d" + % (nread, msglen)) + return msgbuf + + def _connect(self): + if not self._socket: + if self._timeout: + self._socket = socket.create_connection(self._address, + self._timeout) + else: + self._socket = socket.create_connection(self._address) + if self._socket_tcp_options: + ka_opts = self._socket_tcp_options + for k, v in six.iteritems(ka_opts): + self._socket.setsockopt(socket.SOL_TCP, k, v) + if self._socket_keepalive: + self._socket.setsockopt( + socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + if self._client._credentials: + self._init_security() + + def close(self): + """ + Closes the underlying socket of the PB connection. + """ + if self._socket: + if USE_STDLIB_SSL: + # NB: Python 2.7.8 and earlier does not have a compatible + # shutdown() method due to the SSL lib + try: + self._socket.shutdown(socket.SHUT_RDWR) + except EnvironmentError: + # NB: sometimes these exceptions are raised if the initial + # connection didn't succeed correctly, or if shutdown() is + # called after the connection dies + logging.debug('Exception occurred while shutting ' + 'down socket.', exc_info=True) + self._socket.close() + del self._socket diff --git a/riak/transports/tcp/stream.py b/riak/transports/tcp/stream.py new file mode 100644 index 00000000..95436825 --- /dev/null +++ b/riak/transports/tcp/stream.py @@ -0,0 +1,214 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import riak.pb.messages + +from riak.util import decode_index_value, bytes_to_str +from riak.client.index_page import CONTINUATION +from riak.codecs.ttb import TtbCodec +from six import PY2 + + +class PbufStream(object): + """ + Used internally by TcpTransport to implement streaming + operations. Implements the iterator interface. + """ + + _expect = None + + def __init__(self, transport, codec): + self.finished = False + self.transport = transport + self.codec = codec + self.resource = None + self._mid_stream = False + + def __iter__(self): + return self + + def next(self): + if self.finished: + raise StopIteration + + try: + resp_code, data = self.transport._recv_msg( + mid_stream=self._mid_stream) + self.codec.maybe_riak_error(resp_code, data) + expect = self._expect + self.codec.maybe_incorrect_code(resp_code, expect) + resp = self.codec.parse_msg(expect, data) + except: + self.finished = True + raise + finally: + self._mid_stream = True + + if self._is_done(resp): + self.finished = True + + return resp + + def __next__(self): + # Python 3.x Version + return self.next() + + def _is_done(self, response): + # This could break if new messages don't name the field the + # same thing. + return response.done + + def attach(self, resource): + self.resource = resource + + def close(self): + # We have to drain the socket to make sure that we don't get + # weird responses when some other request comes after a + # failed/prematurely-terminated one. + try: + while self.next(): + pass + except StopIteration: + pass + self.resource.release() + + +class PbufKeyStream(PbufStream): + """ + Used internally by TcpTransport to implement key-list streams. + """ + + _expect = riak.pb.messages.MSG_CODE_LIST_KEYS_RESP + + def next(self): + response = super(PbufKeyStream, self).next() + + if response.done and len(response.keys) is 0: + raise StopIteration + + return response.keys + + def __next__(self): + # Python 3.x Version + return self.next() + + +class PbufMapredStream(PbufStream): + """ + Used internally by TcpTransport to implement MapReduce + streams. + """ + + _expect = riak.pb.messages.MSG_CODE_MAP_RED_RESP + + def next(self): + response = super(PbufMapredStream, self).next() + + if response.done and not response.HasField('response'): + raise StopIteration + + return response.phase, json.loads(bytes_to_str(response.response)) + + def __next__(self): + # Python 3.x Version + return self.next() + + +class PbufBucketStream(PbufStream): + """ + Used internally by TcpTransport to implement key-list streams. + """ + + _expect = riak.pb.messages.MSG_CODE_LIST_BUCKETS_RESP + + def next(self): + response = super(PbufBucketStream, self).next() + + if response.done and len(response.buckets) is 0: + raise StopIteration + + return response.buckets + + def __next__(self): + # Python 3.x Version + return self.next() + + +class PbufIndexStream(PbufStream): + """ + Used internally by TcpTransport to implement Secondary Index + streams. + """ + + _expect = riak.pb.messages.MSG_CODE_INDEX_RESP + + def __init__(self, transport, codec, index, return_terms=False): + super(PbufIndexStream, self).__init__(transport, codec) + self.index = index + self.return_terms = return_terms + + def next(self): + response = super(PbufIndexStream, self).next() + + if response.done and not (response.keys or + response.results or + response.continuation): + raise StopIteration + + if self.return_terms and response.results: + return [(decode_index_value(self.index, r.key), + bytes_to_str(r.value)) + for r in response.results] + elif response.keys: + if PY2: + return response.keys[:] + else: + return [bytes_to_str(key) for key in response.keys] + elif response.continuation: + return CONTINUATION(bytes_to_str(response.continuation)) + + def __next__(self): + # Python 3.x Version + return self.next() + + +class PbufTsKeyStream(PbufStream, TtbCodec): + """ + Used internally by TcpTransport to implement TS key-list streams. + """ + + _expect = riak.pb.messages.MSG_CODE_TS_LIST_KEYS_RESP + + def __init__(self, transport, codec, convert_timestamp=False): + super(PbufTsKeyStream, self).__init__(transport, codec) + self._convert_timestamp = convert_timestamp + + def next(self): + response = super(PbufTsKeyStream, self).next() + + if response.done and len(response.keys) is 0: + raise StopIteration + + keys = [] + for tsrow in response.keys: + keys.append(self.codec.decode_timeseries_row(tsrow, + convert_timestamp=self._convert_timestamp)) + + return keys + + def __next__(self): + # Python 3.x Version + return self.next() diff --git a/riak/transports/tcp/transport.py b/riak/transports/tcp/transport.py new file mode 100644 index 00000000..5d3a1599 --- /dev/null +++ b/riak/transports/tcp/transport.py @@ -0,0 +1,574 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six + +import riak.pb.messages + +from riak import RiakError +from riak.codecs import Codec, Msg +from riak.codecs.pbuf import PbufCodec +from riak.codecs.ttb import TtbCodec +from riak.pb.messages import MSG_CODE_TS_TTB_MSG +from riak.transports.pool import BadResource +from riak.transports.transport import Transport +from riak.ts_object import TsObject + +from riak.transports.tcp.connection import TcpConnection +from riak.transports.tcp.stream import (PbufKeyStream, + PbufMapredStream, + PbufBucketStream, + PbufIndexStream, + PbufTsKeyStream) + + +class TcpTransport(Transport, TcpConnection): + """ + The TcpTransport object holds a connection to the TCP + socket on the Riak server. + """ + def __init__(self, + node=None, + client=None, + timeout=None, + **kwargs): + super(TcpTransport, self).__init__() + + self._client = client + self._node = node + self._address = (node.host, node.pb_port) + self._timeout = timeout + self._socket = None + self._pbuf_c = None + self._ttb_c = None + self._socket_tcp_options = \ + kwargs.get('socket_tcp_options', {}) + self._socket_keepalive = \ + kwargs.get('socket_keepalive', False) + self._ts_convert_timestamp = \ + kwargs.get('ts_convert_timestamp', False) + self._use_ttb = \ + kwargs.get('use_ttb', True) + + def _get_pbuf_codec(self): + if not self._pbuf_c: + self._pbuf_c = PbufCodec( + self.client_timeouts(), self.quorum_controls(), + self.tombstone_vclocks(), self.bucket_types()) + return self._pbuf_c + + def _get_ttb_codec(self): + if self._use_ttb: + if not self._ttb_c: + self._ttb_c = TtbCodec() + codec = self._ttb_c + else: + codec = self._get_pbuf_codec() + return codec + + def _get_codec(self, msg_code): + if msg_code == MSG_CODE_TS_TTB_MSG: + codec = self._get_ttb_codec() + elif msg_code == riak.pb.messages.MSG_CODE_TS_GET_REQ: + codec = self._get_ttb_codec() + elif msg_code == riak.pb.messages.MSG_CODE_TS_PUT_REQ: + codec = self._get_ttb_codec() + elif msg_code == riak.pb.messages.MSG_CODE_TS_QUERY_REQ: + codec = self._get_ttb_codec() + else: + codec = self._get_pbuf_codec() + return codec + + # FeatureDetection API + def _server_version(self): + server_info = self.get_server_info() + ver = server_info['server_version'] + (maj, min, patch) = [int(v) for v in ver.split('.')] + if maj == 0: + import datetime + now = datetime.datetime.now() + if now.year == 2016: + # GH-471 As of 20160509 Riak TS OSS 1.3.0 returns '0.8.0' as + # the version string. + return '2.1.1' + return ver + + def ping(self): + """ + Ping the remote server + """ + msg_code = riak.pb.messages.MSG_CODE_PING_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_ping() + resp_code, _ = self._request(msg, codec) + if resp_code == riak.pb.messages.MSG_CODE_PING_RESP: + return True + else: + return False + + def get_server_info(self): + """ + Get information about the server + """ + # NB: can't do it this way due to recursion + # codec = self._get_codec(ttb_supported=False) + codec = PbufCodec() + msg = Msg(riak.pb.messages.MSG_CODE_GET_SERVER_INFO_REQ, None, + riak.pb.messages.MSG_CODE_GET_SERVER_INFO_RESP) + resp_code, resp = self._request(msg, codec) + return codec.decode_get_server_info(resp) + + def _get_client_id(self): + msg_code = riak.pb.messages.MSG_CODE_GET_CLIENT_ID_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_get_client_id() + resp_code, resp = self._request(msg, codec) + return codec.decode_get_client_id(resp) + + def _set_client_id(self, client_id): + msg_code = riak.pb.messages.MSG_CODE_SET_CLIENT_ID_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_set_client_id(client_id) + resp_code, resp = self._request(msg, codec) + self._client_id = client_id + + client_id = property(_get_client_id, _set_client_id, + doc="""the client ID for this connection""") + + def get(self, robj, r=None, pr=None, timeout=None, basic_quorum=None, + notfound_ok=None, head_only=False): + """ + Serialize get request and deserialize response + """ + msg_code = riak.pb.messages.MSG_CODE_GET_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_get(robj, r, pr, + timeout, basic_quorum, + notfound_ok, head_only) + resp_code, resp = self._request(msg, codec) + return codec.decode_get(robj, resp) + + def put(self, robj, w=None, dw=None, pw=None, return_body=True, + if_none_match=False, timeout=None): + msg_code = riak.pb.messages.MSG_CODE_PUT_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_put(robj, w, dw, pw, return_body, + if_none_match, timeout) + resp_code, resp = self._request(msg, codec) + return codec.decode_put(robj, resp) + + def ts_describe(self, table): + query = 'DESCRIBE {table}'.format(table=table.name) + return self.ts_query(table, query) + + def ts_get(self, table, key): + msg_code = MSG_CODE_TS_TTB_MSG + codec = self._get_codec(msg_code) + msg = codec.encode_timeseries_keyreq(table, key) + resp_code, resp = self._request(msg, codec) + tsobj = TsObject(self._client, table) + codec.decode_timeseries(resp, tsobj, + self._ts_convert_timestamp) + return tsobj + + def ts_put(self, tsobj): + msg_code = MSG_CODE_TS_TTB_MSG + codec = self._get_codec(msg_code) + msg = codec.encode_timeseries_put(tsobj) + resp_code, resp = self._request(msg, codec) + return codec.validate_timeseries_put_resp(resp_code, resp) + + def ts_delete(self, table, key): + msg_code = riak.pb.messages.MSG_CODE_TS_DEL_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_timeseries_keyreq(table, key, is_delete=True) + resp_code, resp = self._request(msg, codec) + if resp is not None: + return True + else: + raise RiakError("missing response object") + + def ts_query(self, table, query, interpolations=None): + msg_code = riak.pb.messages.MSG_CODE_TS_QUERY_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_timeseries_query(table, query, interpolations) + resp_code, resp = self._request(msg, codec) + tsobj = TsObject(self._client, table) + codec.decode_timeseries(resp, tsobj, + self._ts_convert_timestamp) + return tsobj + + def ts_stream_keys(self, table, timeout=None): + """ + Streams keys from a timeseries table, returning an iterator that + yields lists of keys. + """ + msg_code = riak.pb.messages.MSG_CODE_TS_LIST_KEYS_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_timeseries_listkeysreq(table, timeout) + self._send_msg(msg.msg_code, msg.data) + return PbufTsKeyStream(self, codec, self._ts_convert_timestamp) + + def delete(self, robj, rw=None, r=None, w=None, dw=None, + pr=None, pw=None, timeout=None): + msg_code = riak.pb.messages.MSG_CODE_DEL_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_delete(robj, rw, r, w, dw, pr, pw, timeout) + resp_code, resp = self._request(msg, codec) + return self + + def get_keys(self, bucket, timeout=None): + """ + Lists all keys within a bucket. + """ + msg_code = riak.pb.messages.MSG_CODE_LIST_KEYS_REQ + codec = self._get_codec(msg_code) + stream = self.stream_keys(bucket, timeout=timeout) + return codec.decode_get_keys(stream) + + def stream_keys(self, bucket, timeout=None): + """ + Streams keys from a bucket, returning an iterator that yields + lists of keys. + """ + msg_code = riak.pb.messages.MSG_CODE_LIST_KEYS_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_stream_keys(bucket, timeout) + self._send_msg(msg.msg_code, msg.data) + return PbufKeyStream(self, codec) + + def get_buckets(self, bucket_type=None, timeout=None): + """ + Serialize bucket listing request and deserialize response + """ + msg_code = riak.pb.messages.MSG_CODE_LIST_BUCKETS_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_get_buckets(bucket_type, + timeout, streaming=False) + resp_code, resp = self._request(msg, codec) + return resp.buckets + + def stream_buckets(self, bucket_type=None, timeout=None): + """ + Stream list of buckets through an iterator + """ + if not self.bucket_stream(): + raise NotImplementedError('Streaming list-buckets is not ' + 'supported') + msg_code = riak.pb.messages.MSG_CODE_LIST_BUCKETS_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_get_buckets(bucket_type, + timeout, streaming=True) + self._send_msg(msg.msg_code, msg.data) + return PbufBucketStream(self, codec) + + def get_bucket_props(self, bucket): + """ + Serialize bucket property request and deserialize response + """ + msg_code = riak.pb.messages.MSG_CODE_GET_BUCKET_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_get_bucket_props(bucket) + resp_code, resp = self._request(msg, codec) + return codec.decode_bucket_props(resp.props) + + def set_bucket_props(self, bucket, props): + """ + Serialize set bucket property request and deserialize response + """ + if not self.pb_all_bucket_props(): + for key in props: + if key not in ('n_val', 'allow_mult'): + raise NotImplementedError('Server only supports n_val and ' + 'allow_mult properties over PBC') + msg_code = riak.pb.messages.MSG_CODE_SET_BUCKET_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_set_bucket_props(bucket, props) + resp_code, resp = self._request(msg, codec) + return True + + def clear_bucket_props(self, bucket): + """ + Clear bucket properties, resetting them to their defaults + """ + if not self.pb_clear_bucket_props(): + return False + msg_code = riak.pb.messages.MSG_CODE_RESET_BUCKET_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_clear_bucket_props(bucket) + self._request(msg, codec) + return True + + def get_bucket_type_props(self, bucket_type): + """ + Fetch bucket-type properties + """ + self._check_bucket_types(bucket_type) + msg_code = riak.pb.messages.MSG_CODE_GET_BUCKET_TYPE_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_get_bucket_type_props(bucket_type) + resp_code, resp = self._request(msg, codec) + return codec.decode_bucket_props(resp.props) + + def set_bucket_type_props(self, bucket_type, props): + """ + Set bucket-type properties + """ + self._check_bucket_types(bucket_type) + msg_code = riak.pb.messages.MSG_CODE_SET_BUCKET_TYPE_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_set_bucket_type_props(bucket_type, props) + resp_code, resp = self._request(msg, codec) + return True + + def mapred(self, inputs, query, timeout=None): + # dictionary of phase results - each content should be an encoded array + # which is appended to the result for that phase. + result = {} + for phase, content in self.stream_mapred(inputs, query, timeout): + if phase in result: + result[phase] += content + else: + result[phase] = content + # If a single result - return the same as the HTTP interface does + # otherwise return all the phase information + if not len(result): + return None + elif len(result) == 1: + return result[max(result.keys())] + else: + return result + + def stream_mapred(self, inputs, query, timeout=None): + # Construct the job, optionally set the timeout... + msg_code = riak.pb.messages.MSG_CODE_MAP_RED_REQ + codec = self._get_codec(msg_code) + content = self._construct_mapred_json(inputs, query, timeout) + msg = codec.encode_stream_mapred(content) + self._send_msg(msg.msg_code, msg.data) + return PbufMapredStream(self, codec) + + def get_index(self, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, continuation=None, + timeout=None, term_regex=None): + # TODO FUTURE NUKE THIS MAPRED + if not self.pb_indexes(): + return self._get_index_mapred_emu(bucket, index, startkey, endkey) + + if term_regex and not self.index_term_regex(): + raise NotImplementedError("Secondary index term_regex is not " + "supported") + + msg_code = riak.pb.messages.MSG_CODE_INDEX_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_index_req(bucket, index, startkey, endkey, + return_terms, max_results, + continuation, timeout, + term_regex, streaming=False) + resp_code, resp = self._request(msg, codec) + return codec.decode_index_req(resp, index, + return_terms, max_results) + + def stream_index(self, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, continuation=None, + timeout=None, term_regex=None): + if not self.stream_indexes(): + raise NotImplementedError("Secondary index streaming is not " + "supported") + if term_regex and not self.index_term_regex(): + raise NotImplementedError("Secondary index term_regex is not " + "supported") + msg_code = riak.pb.messages.MSG_CODE_INDEX_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_index_req(bucket, index, startkey, endkey, + return_terms, max_results, + continuation, timeout, + term_regex, streaming=True) + self._send_msg(msg.msg_code, msg.data) + return PbufIndexStream(self, codec, index, return_terms) + + def create_search_index(self, index, schema=None, n_val=None, + timeout=None): + if not self.pb_search_admin(): + raise NotImplementedError("Search 2.0 administration is not " + "supported for this version") + msg_code = riak.pb.messages.MSG_CODE_YOKOZUNA_INDEX_PUT_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_create_search_index(index, schema, n_val, timeout) + self._request(msg, codec) + return True + + def get_search_index(self, index): + if not self.pb_search_admin(): + raise NotImplementedError("Search 2.0 administration is not " + "supported for this version") + msg_code = riak.pb.messages.MSG_CODE_YOKOZUNA_INDEX_GET_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_get_search_index(index) + resp_code, resp = self._request(msg, codec) + if len(resp.index) > 0: + return codec.decode_search_index(resp.index[0]) + else: + raise RiakError('notfound') + + def list_search_indexes(self): + if not self.pb_search_admin(): + raise NotImplementedError("Search 2.0 administration is not " + "supported for this version") + msg_code = riak.pb.messages.MSG_CODE_YOKOZUNA_INDEX_GET_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_list_search_indexes() + resp_code, resp = self._request(msg, codec) + return [codec.decode_search_index(index) for index in resp.index] + + def delete_search_index(self, index): + if not self.pb_search_admin(): + raise NotImplementedError("Search 2.0 administration is not " + "supported for this version") + msg_code = riak.pb.messages.MSG_CODE_YOKOZUNA_INDEX_DELETE_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_delete_search_index(index) + self._request(msg, codec) + return True + + def create_search_schema(self, schema, content): + if not self.pb_search_admin(): + raise NotImplementedError("Search 2.0 administration is not " + "supported for this version") + msg_code = riak.pb.messages.MSG_CODE_YOKOZUNA_SCHEMA_PUT_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_create_search_schema(schema, content) + self._request(msg, codec) + return True + + def get_search_schema(self, schema): + if not self.pb_search_admin(): + raise NotImplementedError("Search 2.0 administration is not " + "supported for this version") + msg_code = riak.pb.messages.MSG_CODE_YOKOZUNA_SCHEMA_GET_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_get_search_schema(schema) + resp_code, resp = self._request(msg, codec) + return codec.decode_get_search_schema(resp) + + def search(self, index, query, **kwargs): + # TODO FUTURE NUKE THIS MAPRED + if not self.pb_search(): + return self._search_mapred_emu(index, query) + if six.PY2 and isinstance(query, unicode): # noqa + query = query.encode('utf8') + msg_code = riak.pb.messages.MSG_CODE_SEARCH_QUERY_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_search(index, query, **kwargs) + resp_code, resp = self._request(msg, codec) + return codec.decode_search(resp) + + def get_counter(self, bucket, key, **kwargs): + if not bucket.bucket_type.is_default(): + raise NotImplementedError("Counters are not " + "supported with bucket-types, " + "use datatypes instead.") + if not self.counters(): + raise NotImplementedError("Counters are not supported") + msg_code = riak.pb.messages.MSG_CODE_COUNTER_GET_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_get_counter(bucket, key, **kwargs) + resp_code, resp = self._request(msg, codec) + if resp.HasField('value'): + return resp.value + else: + return None + + def update_counter(self, bucket, key, value, **kwargs): + if not bucket.bucket_type.is_default(): + raise NotImplementedError("Counters are not " + "supported with bucket-types, " + "use datatypes instead.") + if not self.counters(): + raise NotImplementedError("Counters are not supported") + msg_code = riak.pb.messages.MSG_CODE_COUNTER_UPDATE_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_update_counter(bucket, key, value, **kwargs) + resp_code, resp = self._request(msg, codec) + if resp.HasField('value'): + return resp.value + else: + return True + + def fetch_datatype(self, bucket, key, **kwargs): + if bucket.bucket_type.is_default(): + raise NotImplementedError("Datatypes cannot be used in the default" + " bucket-type.") + if not self.datatypes(): + raise NotImplementedError("Datatypes are not supported.") + msg_code = riak.pb.messages.MSG_CODE_DT_FETCH_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_fetch_datatype(bucket, key, **kwargs) + resp_code, resp = self._request(msg, codec) + return codec.decode_dt_fetch(resp) + + def update_datatype(self, datatype, **kwargs): + if datatype.bucket.bucket_type.is_default(): + raise NotImplementedError("Datatypes cannot be used in the default" + " bucket-type.") + if not self.datatypes(): + raise NotImplementedError("Datatypes are not supported.") + msg_code = riak.pb.messages.MSG_CODE_DT_UPDATE_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_update_datatype(datatype, **kwargs) + resp_code, resp = self._request(msg, codec) + codec.decode_update_datatype(datatype, resp, **kwargs) + return True + + def get_preflist(self, bucket, key): + """ + Get the preflist for a bucket/key + + :param bucket: Riak Bucket + :type bucket: :class:`~riak.bucket.RiakBucket` + :param key: Riak Key + :type key: string + :rtype: list of dicts + """ + if not self.preflists(): + raise NotImplementedError("fetching preflists is not supported.") + msg_code = riak.pb.messages.MSG_CODE_GET_BUCKET_KEY_PREFLIST_REQ + codec = self._get_codec(msg_code) + msg = codec.encode_get_preflist(bucket, key) + resp_code, resp = self._request(msg, codec) + return [codec.decode_preflist(item) for item in resp.preflist] + + def _request(self, msg, codec=None): + if isinstance(msg, Msg): + msg_code = msg.msg_code + data = msg.data + expect = msg.resp_code + else: + raise ValueError('expected a Msg argument') + + if not isinstance(codec, Codec): + raise ValueError('expected a Codec argument') + + resp_code, data = self._send_recv(msg_code, data) + # NB: decodes errors with msg code 0 + codec.maybe_riak_error(resp_code, data) + codec.maybe_incorrect_code(resp_code, expect) + if resp_code == MSG_CODE_TS_TTB_MSG or \ + resp_code in riak.pb.messages.MESSAGE_CLASSES: + msg = codec.parse_msg(resp_code, data) + else: + # NB: raise a BadResource to ensure this connection is + # closed and not re-used + raise BadResource('unknown msg code {}'.format(resp_code)) + return resp_code, msg diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 53615853..258d24e8 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -1,34 +1,32 @@ -""" -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import base64 import random import threading -import platform import os import json -from feature_detect import FeatureDetection +import platform +from six import PY2 +from riak.transports.feature_detect import FeatureDetection -class RiakTransport(FeatureDetection): + +class Transport(FeatureDetection): """ - Class to encapsulate transport details + Class to encapsulate transport details and methods. All protocol + transports are subclasses of this class. """ def _get_client_id(self): @@ -45,8 +43,13 @@ def make_random_client_id(self): """ Returns a random client identifier """ - return ('py_%s' % - base64.b64encode(str(random.randint(1, 0x40000000)))) + if PY2: + return ('py_%s' % + base64.b64encode(str(random.randint(1, 0x40000000)))) + else: + return ('py_%s' % + base64.b64encode(bytes(str(random.randint(1, 0x40000000)), + 'ascii'))) @classmethod def make_fixed_client_id(self): @@ -61,79 +64,115 @@ def make_fixed_client_id(self): def ping(self): """ Ping the remote server - @return boolean """ raise NotImplementedError - def get(self, robj, r=None, vtag=None): + def get(self, robj, r=None, pr=None, timeout=None, basic_quorum=None, + notfound_ok=None, head_only=False): + """ + Fetches an object. + """ + raise NotImplementedError + + def put(self, robj, w=None, dw=None, pw=None, return_body=None, + if_none_match=None, timeout=None): + """ + Stores an object. + """ + raise NotImplementedError + + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, + pw=None, timeout=None): """ - Serialize get request and deserialize response - @return (vclock=None, [(metadata, value)]=None) + Deletes an object. """ raise NotImplementedError - def put(self, robj, w=None, dw=None, return_body=True): + def ts_describe(self, table): """ - Serialize put request and deserialize response - if 'content' - is true, retrieve the updated metadata/content - @return (vclock=None, [(metadata, value)]=None) + Retrieves a timeseries table description. """ raise NotImplementedError - def put_new(self, robj, w=None, dw=None, return_meta=True): - """Put a new object into the Riak store, returning its (new) key. + def ts_get(self, table, key): + """ + Retrieves a timeseries object. + """ + raise NotImplementedError - If return_meta is False, then the vlock and metadata return values - will be None. + def ts_put(self, tsobj): + """ + Stores a timeseries object. + """ + raise NotImplementedError - @return (key, vclock, metadata) + def ts_delete(self, table, key): + """ + Deletes a timeseries object. """ raise NotImplementedError - def delete(self, robj, rw=None): + def ts_query(self, table, query, interpolations=None): """ - Serialize delete request and deserialize response - @return true + Query timeseries data. """ raise NotImplementedError - def get_buckets(self): + def ts_stream_keys(self, table, timeout=None): """ - Serialize get buckets request and deserialize response - @return dict() + Streams the list of keys for the table through an iterator. + """ + raise NotImplementedError + + def get_buckets(self, bucket_type=None, timeout=None): + """ + Gets the list of buckets as strings. + """ + raise NotImplementedError + + def stream_buckets(self, bucket_type=None, timeout=None): + """ + Streams the list of buckets through an iterator """ raise NotImplementedError def get_bucket_props(self, bucket): """ - Serialize get bucket property request and deserialize response - @return dict() + Fetches properties for the given bucket. """ raise NotImplementedError def set_bucket_props(self, bucket, props): """ - Serialize set bucket property request and deserialize response - bucket = bucket object - props = dictionary of properties - @return boolean + Sets properties on the given bucket. + """ + raise NotImplementedError + + def get_bucket_type_props(self, bucket_type): + """ + Fetches properties for the given bucket-type. + """ + raise NotImplementedError + + def set_bucket_type_props(self, bucket_type, props): + """ + Sets properties on the given bucket-type. """ raise NotImplementedError def clear_bucket_props(self, bucket): """ Reset bucket properties to their defaults - bucket = bucket object """ raise NotImplementedError - def get_keys(self, bucket): + def get_keys(self, bucket, timeout=None): """ Lists all keys within the given bucket. """ raise NotImplementedError - def stream_keys(self, bucket): + def stream_keys(self, bucket, timeout=None): """ Streams the list of keys for the bucket through an iterator. """ @@ -165,18 +204,65 @@ def get_client_id(self): """ raise NotImplementedError + def create_search_index(self, index, schema=None, n_val=None, + timeout=None): + """ + Creates a yokozuna search index. + """ + raise NotImplementedError + + def get_search_index(self, index): + """ + Returns a yokozuna search index or None. + """ + raise NotImplementedError + + def list_search_indexes(self): + """ + Lists all yokozuna search indexes. + """ + raise NotImplementedError + + def delete_search_index(self, index): + """ + Deletes a yokozuna search index. + """ + raise NotImplementedError + + def create_search_schema(self, schema, content): + """ + Creates a yokozuna search schema. + """ + raise NotImplementedError + + def get_search_schema(self, schema): + """ + Returns a yokozuna search schema. + """ + raise NotImplementedError + def search(self, index, query, **params): """ Performs a search query. """ raise NotImplementedError - def get_index(self, bucket, index, startkey, endkey=None): + def get_index(self, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, continuation=None, + timeout=None, term_regex=None): """ Performs a secondary index query. """ raise NotImplementedError + def stream_index(self, bucket, index, startkey, endkey=None, + return_terms=None, max_results=None, continuation=None, + timeout=None): + """ + Streams a secondary index query. + """ + raise NotImplementedError + def fulltext_add(self, index, *docs): """ Adds documents to the full-text index. @@ -189,6 +275,41 @@ def fulltext_delete(self, index, docs=None, queries=None): """ raise NotImplementedError + def get_counter(self, bucket, key, r=None, pr=None, basic_quorum=None, + notfound_ok=None): + """ + Gets the value of a counter. + """ + raise NotImplementedError + + def update_counter(self, bucket, key, value, w=None, dw=None, pw=None, + returnvalue=False): + """ + Updates a counter by the given value. + """ + raise NotImplementedError + + def fetch_datatype(self, bucket, key, r=None, pr=None, basic_quorum=None, + notfound_ok=None, timeout=None, include_context=None): + """ + Fetches a Riak Datatype. + """ + raise NotImplementedError + + def update_datatype(self, datatype, w=None, dw=None, pw=None, + return_body=None, timeout=None, include_context=None): + """ + Updates a Riak Datatype by sending local operations to the server. + """ + raise NotImplementedError + + def get_preflist(self, bucket, key): + """ + Fetches the preflist for a bucket/key. + """ + raise NotImplementedError + + # TODO FUTURE NUKE THIS MAPRED def _search_mapred_emu(self, index, query): """ Emulates a search request via MapReduce. Used in the case @@ -214,6 +335,7 @@ def _search_mapred_emu(self, index, query): result['docs'].append({u'id': key}) return result + # TODO FUTURE NUKE THIS MAPRED def _get_index_mapred_emu(self, bucket, index, startkey, endkey=None): """ Emulates a secondary index request via MapReduce. Used in the @@ -250,3 +372,9 @@ def _construct_mapred_json(self, inputs, query, timeout=None): content = json.dumps(job) return content + + def _check_bucket_types(self, bucket_type): + if not self.bucket_types(): + raise NotImplementedError('Server does not support bucket-types') + if bucket_type.is_default(): + raise ValueError('Cannot manipulate the default bucket-type') diff --git a/riak/ts_object.py b/riak/ts_object.py new file mode 100644 index 00000000..2c7fddf5 --- /dev/null +++ b/riak/ts_object.py @@ -0,0 +1,65 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +from riak import RiakError +from riak.table import Table + +TsColumns = collections.namedtuple('TsColumns', ['names', 'types']) + + +class TsObject(object): + """ + The TsObject holds information about Timeseries data, plus the data + itself. + """ + def __init__(self, client, table, rows=None, columns=None): + """ + Construct a new TsObject. + + :param client: A RiakClient object. + :type client: :class:`RiakClient ` + :param table: The table for the timeseries data as a Table object. + :type table: :class:`Table` + :param rows: An list of lists with timeseries data + :type rows: list + :param columns: A TsColumns tuple. Optional + :type columns: :class:`TsColumns` + """ + + if not isinstance(table, Table): + raise ValueError('table must be an instance of Table.') + + self.client = client + self.table = table + + if rows is not None and not isinstance(rows, list): + raise RiakError("TsObject rows parameter must be a list.") + else: + self.rows = rows + + if columns is not None and \ + not isinstance(columns, TsColumns): + raise RiakError( + "TsObject columns parameter must be a TsColumns instance") + else: + self.columns = columns + + def store(self): + """ + Store the timeseries data in Riak. + :rtype: boolean + """ + return self.client.ts_put(self) diff --git a/riak/tz.py b/riak/tz.py new file mode 100644 index 00000000..fc44e32d --- /dev/null +++ b/riak/tz.py @@ -0,0 +1,33 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import tzinfo, timedelta + +ZERO = timedelta(0) + + +class UTC(tzinfo): + """UTC""" + + def utcoffset(self, dt): + return ZERO + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return ZERO + + +utc = UTC() diff --git a/riak/util.py b/riak/util.py index 448ee8d7..9101275b 100644 --- a/riak/util.py +++ b/riak/util.py @@ -1,23 +1,58 @@ -""" -Copyright 2010 Basho Technologies, Inc. +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import datetime +import sys +import warnings -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at +from collections import Mapping +from six import string_types, PY2 - http://www.apache.org/licenses/LICENSE-2.0 +epoch = datetime.datetime.utcfromtimestamp(0) +try: + import pytz + epoch_tz = pytz.utc.localize(epoch) +except ImportError: + from riak.tz import utc + epoch_tz = datetime.datetime.fromtimestamp(0, tz=utc) -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" -import warnings -from collections import Mapping +def unix_time_millis(dt): + if dt.tzinfo: + td = dt - epoch_tz + else: + td = dt - epoch + tdms = ((td.days * 24 * 3600) + td.seconds) * 1000 + ms = td.microseconds // 1000 + return tdms + ms + + +def datetime_from_unix_time_millis(ut): + if isinstance(ut, float): + raise ValueError('unix timestamp must not be a float, ' + 'it must be total milliseconds since ' + 'epoch as an integer') + utms = ut / 1000.0 + return datetime.datetime.utcfromtimestamp(utms) + + +def is_timeseries_supported(v=None): + if v is None: + v = sys.version_info + return v < (3,) or (v[:3] >= (3, 4, 4) and v[:3] != (3, 5, 0)) def quacks_like_dict(object): @@ -32,7 +67,7 @@ def deep_merge(a, b): >>> a = {'a': 1, 'b': {1: 1, 2: 2}, 'd': 6} >>> b = {'c': 3, 'b': {2: 7}, 'd': {'z': [1, 2, 3]}} - >>> c = merge(a, b) + >>> c = deep_merge(a, b) >>> from pprint import pprint; pprint(c) {'a': 1, 'b': {1: 1, 2: 7}, 'c': 3, 'd': {'z': [1, 2, 3]}} """ @@ -46,8 +81,8 @@ def deep_merge(a, b): if key not in current_dst: current_dst[key] = current_src[key] else: - if (quacks_like_dict(current_src[key]) - and quacks_like_dict(current_dst[key])): + if (quacks_like_dict(current_src[key]) and + quacks_like_dict(current_dst[key])): stack.append((current_dst[key], current_src[key])) else: current_dst[key] = current_src[key] @@ -55,64 +90,18 @@ def deep_merge(a, b): def deprecated(message, stacklevel=3): - warnings.warn(message, UserWarning, stacklevel=stacklevel) - -QUORUMS = ['r', 'pr', 'w', 'dw', 'pw', 'rw'] -QDEPMESSAGE = """ -Quorum accessors on type %s are deprecated. Use request-specific -parameters or bucket properties instead. -""" - - -def deprecateQuorumAccessors(klass, parent=None): """ - Adds deprecation warnings for the quorum get_* and set_* - accessors, informing the user to switch to the appropriate bucket - properties or requests parameters. + Prints a deprecation warning to the console. """ - for q in QUORUMS: - __deprecateQuorumAccessor(klass, parent, q) - return klass - - -def __deprecateQuorumAccessor(klass, parent, quorum): - propname = "_%s" % quorum - getter_name = "get_%s" % quorum - setter_name = "set_%s" % quorum - if not parent: - def direct_getter(self, val=None): - deprecated(QDEPMESSAGE % klass.__name__) - if val: - return val - return getattr(self, propname, "default") - - getter = direct_getter - else: - def parent_getter(self, val=None): - deprecated(QDEPMESSAGE % klass.__name__) - if val: - return val - parentInstance = getattr(self, parent) - return getattr(self, propname, - getattr(parentInstance, propname, "default")) - - getter = parent_getter - - def setter(self, value): - deprecated(QDEPMESSAGE % klass.__name__) - setattr(self, propname, value) - return self - - setattr(klass, getter_name, getter) - setattr(klass, setter_name, setter) + warnings.warn(message, UserWarning, stacklevel=stacklevel) class lazy_property(object): ''' - meant to be used for lazy evaluation of an object attribute. - property should represent non-mutable data, as it replaces itself. + A method decorator meant to be used for lazy evaluation and + memoization of an object attribute. The property should represent + immutable data, as it replaces itself on first access. ''' - def __init__(self, fget): self.fget = fget self.func_name = fget.__name__ @@ -123,3 +112,39 @@ def __get__(self, obj, cls): value = self.fget(obj) setattr(obj, self.func_name, value) return value + + +def decode_index_value(index, value): + if "_int" in bytes_to_str(index): + return str_to_long(value) + elif PY2: + return str(value) + else: + return bytes_to_str(value) + + +def bytes_to_str(value, encoding='utf-8'): + if isinstance(value, string_types) or value is None: + return value + elif isinstance(value, list): + return [bytes_to_str(elem) for elem in value] + else: + return value.decode(encoding) + + +def str_to_bytes(value, encoding='utf-8'): + if PY2 or value is None: + return value + elif isinstance(value, list): + return [str_to_bytes(elem) for elem in value] + else: + return value.encode(encoding) + + +def str_to_long(value, base=10): + if value is None: + return None + elif PY2: + return long(value, base) # noqa + else: + return int(value, base) diff --git a/riak_pb b/riak_pb new file mode 160000 index 00000000..cb15cc47 --- /dev/null +++ b/riak_pb @@ -0,0 +1 @@ +Subproject commit cb15cc4770f3748289ba56245d62b1c0d07c33f7 diff --git a/setup.py b/setup.py index 06393638..37eb8da0 100755 --- a/setup.py +++ b/setup.py @@ -1,32 +1,46 @@ #!/usr/bin/env python -import glob -import os -import subprocess -import platform + +import codecs +import sys + from setuptools import setup, find_packages +from version import get_version +from commands import setup_timeseries, build_messages + +install_requires = ['six >= 1.8.0', 'basho_erlastic >= 2.1.1'] +requires = ['six(>=1.8.0)', 'basho_erlastic(>= 2.1.1)'] + +if sys.version_info[:3] <= (2, 7, 9): + install_requires.append("pyOpenSSL >= 0.14") + requires.append("pyOpenSSL(>=0.14)") + +if sys.version_info[:3] <= (3, 0, 0): + install_requires.append('protobuf >=2.4.1, <2.7.0') + requires.append('protobuf(>=2.4.1, <2.7.0)') +else: + install_requires.append('python3_protobuf >=2.4.1, <2.6.0') + requires.append('python3_protobuf(>=2.4.1, <2.6.0)') -def make_docs(): - if not os.path.exists('docs'): - os.mkdir('docs') - subprocess.call(['pydoc', '-w', 'riak']) - for name in glob.glob('*.html'): - os.rename(name, 'docs/%s' % name) +with codecs.open('README.md', 'r', 'utf-8') as f: + readme_md = f.read() -install_requires = ["riak_pb >=1.2.0, < 1.3.0"] -requires = ["riak_pb(>=1.2.0,<1.3.0)"] -tests_require = [] -if platform.python_version() < '2.7': - tests_require.append("unittest2") +try: + import pypandoc + long_description = pypandoc.convert('README.md', 'rst') + with codecs.open('README.rst', 'w', 'utf-8') as f: + f.write(long_description) +except(IOError, ImportError): + long_description = readme_md setup( name='riak', - version='1.5.1', - packages = find_packages(), - requires = requires, - install_requires = install_requires, - tests_require = tests_require, - package_data = {'riak' : ['erl_src/*']}, + version=get_version(), + packages=find_packages(), + requires=requires, + install_requires=install_requires, + package_data={'riak': ['erl_src/*']}, description='Python client for Riak', + long_description=long_description, zip_safe=True, options={'easy_install': {'allow_hosts': 'pypi.python.org'}}, include_package_data=True, @@ -36,8 +50,16 @@ def make_docs(): author_email='clients@basho.com', test_suite='riak.tests.suite', url='https://github.com/basho/riak-python-client', - classifiers = ['License :: OSI Approved :: Apache Software License', - 'Intended Audience :: Developers', - 'Operating System :: OS Independent', - 'Topic :: Database'] + cmdclass={ + 'build_messages': build_messages, + 'setup_timeseries': setup_timeseries + }, + classifiers=['License :: OSI Approved :: Apache Software License', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Topic :: Database'] ) diff --git a/tools b/tools new file mode 160000 index 00000000..1f54803c --- /dev/null +++ b/tools @@ -0,0 +1 @@ +Subproject commit 1f54803ca7912a41a0ec47c0028c259b97475e1f diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..f411b799 --- /dev/null +++ b/tox.ini @@ -0,0 +1,14 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. + +[tox] +envlist = py2, py3 + +[testenv] +install_command = pip install --upgrade {packages} +commands = {envpython} setup.py test +deps = + pip + pytz +passenv = RUN_* SKIP_* RIAK_* diff --git a/version.py b/version.py new file mode 100644 index 00000000..ca6a019c --- /dev/null +++ b/version.py @@ -0,0 +1,103 @@ +# Copyright 2010-present Basho Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Gets the current version number. +If in a git repository, it is the current git tag. +Otherwise it is the one contained in the PKG-INFO file. + +To use this script, simply import it in your setup.py file +and use the results of get_version() as your package version:: + + from version import * + + setup( + version=get_version() + ) +""" + +from __future__ import print_function +from os.path import dirname, isdir, join +import re +from subprocess import CalledProcessError, Popen, PIPE + +try: + from subprocess import check_output +except ImportError: + def check_output(*popenargs, **kwargs): + """Run command with arguments and return its output as a byte string. + + If the exit code was non-zero it raises a CalledProcessError. The + CalledProcessError object will have the return code in the returncode + attribute and output in the output attribute. + + The arguments are the same as for the Popen constructor. Example: + + >>> check_output(["ls", "-l", "/dev/null"]) + 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' + + The stdout argument is not allowed as it is used internally. + To capture standard error in the result, use stderr=STDOUT. + + >>> import sys + >>> check_output(["/bin/sh", "-c", + ... "ls -l non_existent_file ; exit 0"], + ... stderr=sys.stdout) + 'ls: non_existent_file: No such file or directory\n' + """ + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be ' + 'overridden.') + process = Popen(stdout=PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise CalledProcessError(retcode, cmd) + return output + +version_re = re.compile('^Version: (.+)$', re.M) + +__all__ = ['get_version'] + + +def get_version(): + d = dirname(__file__) + + if isdir(join(d, '.git')): + # Get the version using "git describe". + cmd = 'git describe --tags --match [0-9]*'.split() + try: + version = check_output(cmd).decode().strip() + except CalledProcessError: + print('Unable to get version number from git tags') + exit(1) + + # PEP 386 compatibility + if '-' in version: + version = '.post'.join(version.split('-')[:2]) + + else: + # Extract the version from the PKG-INFO file. + import codecs + with codecs.open(join(d, 'PKG-INFO'), 'r', 'utf-8') as f: + version = version_re.search(f.read()).group(1) + + return version + + +if __name__ == '__main__': + print(get_version())