From 68fb9a52cbb824955d90d19f267b136261ae54ea Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 11 Apr 2017 11:57:42 +0100 Subject: [PATCH 001/185] Catch error if throw_on_exists flag is False for document create --- src/cloudant/database.py | 11 ++++++++--- tests/unit/database_tests.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/cloudant/database.py b/src/cloudant/database.py index bae7e5c1..28f02c61 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -157,10 +157,15 @@ def create_document(self, data, throw_on_exists=False): doc = DesignDocument(self, docid) else: doc = Document(self, docid) - if throw_on_exists and doc.exists(): - raise CloudantDatabaseException(409, docid) doc.update(data) - doc.create() + try: + doc.create() + except HTTPError as error: + if error.response.status_code == 409: + if throw_on_exists: + raise CloudantDatabaseException(409, docid) + else: + raise super(CouchDatabase, self).__setitem__(doc['_id'], doc) return doc diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index f9abac99..3a5459b0 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -267,6 +267,17 @@ def test_create_document_with_id(self): 'Document with id julia06 already exists.' ) + def test_create_document_that_already_exists(self): + """ + Test creating a document that already exists + """ + data = {'_id': 'julia'} + doc = self.db.create_document(data) + self.assertEqual(self.db['julia'], doc) + self.assertTrue(doc['_rev'].startswith('1-')) + # attempt to recreate document + self.db.create_document(data, throw_on_exists=False) + def test_create_document_without_id(self): """ Test creating a document without supplying a document id From 2722aa67b0f6b6b0ce33a7ed703040b51e0a29c5 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 18 Apr 2017 12:54:20 +0100 Subject: [PATCH 002/185] Fix pylint failures --- src/cloudant/_common_util.py | 9 ++++----- src/cloudant/client.py | 4 ++-- src/cloudant/database.py | 8 ++++---- src/cloudant/result.py | 4 ++-- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 6acab04a..5a08ee74 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -201,9 +201,8 @@ def _py_to_couch_translate(key, val): return {key: val} elif val is None: return {key: None} - else: - arg_converter = TYPE_CONVERTERS.get(type(val)) - return {key: arg_converter(val)} + arg_converter = TYPE_CONVERTERS.get(type(val)) + return {key: arg_converter(val)} except Exception as ex: raise CloudantArgumentError(136, key, ex) @@ -299,7 +298,7 @@ def __init__(self, username, password, server_url, **kwargs): self._server_url = server_url self._timeout = kwargs.get('timeout', None) - def request(self, method, url, **kwargs): + def request(self, method, url, **kwargs): # pylint: disable=W0221 """ Overrides ``requests.Session.request`` to perform a POST to the _session endpoint to renew Session cookie authentication settings and @@ -338,7 +337,7 @@ def __init__(self, username, password, server_url, **kwargs): self._server_url = server_url self._timeout = kwargs.get('timeout', None) - def request(self, method, url, **kwargs): + def request(self, method, url, **kwargs): # pylint: disable=W0221 """ Overrides ``requests.Session.request`` to set the timeout. """ diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 1cbd428d..e70e9af3 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -371,8 +371,8 @@ def get(self, key, default=None, remote=False): if db.exists(): super(CouchDB, self).__setitem__(key, db) return db - else: - return default + + return default def __setitem__(self, key, value, remote=False): """ diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 28f02c61..90d3030d 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -331,8 +331,8 @@ def get_view_result(self, ddoc_id, view_name, raw_result=False, **kwargs): return view(**kwargs) elif kwargs: return Result(view, **kwargs) - else: - return view.result + + return view.result def create(self, throw_on_exists=False): """ @@ -1237,8 +1237,8 @@ def get_query_result(self, selector, fields=None, raw_result=False, return query(**kwargs) if kwargs: return QueryResult(query, **kwargs) - else: - return query.result + + return query.result def get_search_result(self, ddoc_id, index_name, **query_params): """ diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 58aa743f..8cac3ae4 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -201,7 +201,7 @@ def __getitem__(self, arg): data = None if isinstance(arg, int): data = self._handle_result_by_index(arg) - elif isinstance(arg, STRTYPE) or isinstance(arg, list): + elif isinstance(arg, (STRTYPE, list)): data = self._handle_result_by_key(arg) elif isinstance(arg, ResultByKey): data = self._handle_result_by_key(arg()) @@ -345,7 +345,7 @@ def __iter__(self): ) result = self._parse_data(response) skip += int(self._page_size) - if len(result) > 0: + if result: for row in result: yield row del result From 38037b22675f548ea760c9e81eedf67f2b9e0cf6 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 11 Apr 2017 15:18:27 +0100 Subject: [PATCH 003/185] Fix _all_docs call where keys is an empty list --- src/cloudant/_common_util.py | 4 ++-- tests/unit/database_tests.py | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 5a08ee74..98c5da71 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -237,11 +237,11 @@ def get_docs(r_session, url, encoder=None, headers=None, **params): """ keys_list = params.pop('keys', None) keys = None - if keys_list: + if keys_list is not None: keys = json.dumps({'keys': keys_list}, cls=encoder) f_params = python_to_couch(params) resp = None - if keys: + if keys is not None: # If we're using POST we are sending JSON so add the header if headers is None: headers = {} diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index 3a5459b0..c9267976 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -432,6 +432,15 @@ def test_all_docs_post(self): keys_returned = [row['key'] for row in rows] self.assertTrue(all(x in keys_returned for x in keys_list)) + def test_all_docs_post_empty_key_list(self): + """ + Test the all_docs POST request functionality using empty keys param + """ + self.populate_db_with_documents() + # Request all_docs using an empty key list + rows = self.db.all_docs(keys=[]).get('rows') + self.assertEqual(len(rows), 0) + def test_all_docs_post_multiple_params(self): """ Test the all_docs POST request functionality using keys and other params From c784cd38fb1830879882c2d941f7b906efad20e8 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 19 Apr 2017 10:00:46 +0100 Subject: [PATCH 004/185] Update CHANGES.rst --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 305ad920..cf1226ec 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -3,6 +3,8 @@ - [FIXED] Fixed ``TypeError`` when setting revision limits on Python>=3.6. - [FIXED] Fixed the ``exists()`` double check on ``client.py`` and ``database.py``. - [FIXED] Fixed Cloudant exception code 409 with 412 when creating a database that already exists. +- [FIXED] Catch error if ``throw_on_exists`` flag is ``False`` for document create. +- [FIXED] Fixed /_all_docs call where ``keys`` is an empty list. 2.4.0 (2017-02-14) ================== From 1b1ff9e187ab4b94798e48bd17057c6c179a5d8f Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 5 Jun 2017 13:39:35 +0100 Subject: [PATCH 005/185] Fix pylint import error in adapters.py --- src/cloudant/adapters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloudant/adapters.py b/src/cloudant/adapters.py index 7428bbe0..c9072515 100644 --- a/src/cloudant/adapters.py +++ b/src/cloudant/adapters.py @@ -18,7 +18,7 @@ """ from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util import Retry +from urllib3.util import Retry class Replay429Adapter(HTTPAdapter): """ From a988a28a7715949edd1bd0f50a4f8b7e3e41f96c Mon Sep 17 00:00:00 2001 From: Matthias A Lee Date: Tue, 6 Jun 2017 09:48:26 -0400 Subject: [PATCH 006/185] non-UTF8 chars (like \u2013) in your map/reduce will make the ddoc fetch puke (#299) * if for some reason you have non-UTF8 chars in your ddocs (like: \u2013) _Code.__new__ will puke. to fix this I just added encoding to UTF8 before creating the UTF8 string to be returned * fixed for non-string --- CHANGES.rst | 1 + src/cloudant/_common_util.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index cf1226ec..a834f025 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,6 @@ 2.5.0 (Unreleased) ================== +- [FIXED] Fixed crash caused by non-UTF8 chars in ddocs - [FIXED] Fixed ``TypeError`` when setting revision limits on Python>=3.6. - [FIXED] Fixed the ``exists()`` double check on ``client.py`` and ``database.py``. - [FIXED] Fixed Cloudant exception code 409 with 412 when creating a database that already exists. diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 98c5da71..0e75e766 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -283,6 +283,8 @@ class _Code(str): codifying map and reduce Javascript content. """ def __new__(cls, code): + if isinstance(code, unicode): + return str.__new__(cls, code.encode('utf8')) return str.__new__(cls, code) class InfiniteSession(Session): From ce75776008b93ba340fd79ec54ee224ef16fac59 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 13 Jun 2017 17:56:39 +0100 Subject: [PATCH 007/185] UTF-8 encode ddoc unicode strings in Py2 --- src/cloudant/_common_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 0e75e766..6dcf331d 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -283,7 +283,7 @@ class _Code(str): codifying map and reduce Javascript content. """ def __new__(cls, code): - if isinstance(code, unicode): + if type(code).__name__ == 'unicode': return str.__new__(cls, code.encode('utf8')) return str.__new__(cls, code) From e4741c114e56741846943d3b2891f97c018b7f66 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 13 Jun 2017 22:15:38 +0100 Subject: [PATCH 008/185] Import urllib3.util.Retry from requests.packages --- src/cloudant/adapters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cloudant/adapters.py b/src/cloudant/adapters.py index c9072515..b5bd6964 100644 --- a/src/cloudant/adapters.py +++ b/src/cloudant/adapters.py @@ -18,7 +18,7 @@ """ from requests.adapters import HTTPAdapter -from urllib3.util import Retry +from requests.packages import urllib3 class Replay429Adapter(HTTPAdapter): """ @@ -33,7 +33,7 @@ class Replay429Adapter(HTTPAdapter): :param float initialBackoff: time in seconds for the first backoff. """ def __init__(self, retries=3, initialBackoff=0.25): - super(Replay429Adapter, self).__init__(max_retries=Retry( + super(Replay429Adapter, self).__init__(max_retries=urllib3.util.Retry( # Configure the number of retries for status codes total=retries, # No retries for connect|read errors From c1fa2df172dccf686438235a750cedacbeed710b Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 28 Jun 2017 16:44:44 +0100 Subject: [PATCH 009/185] Add all docs iterator test --- src/cloudant/_2to3.py | 3 +++ tests/unit/database_tests.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/cloudant/_2to3.py b/src/cloudant/_2to3.py index 595c0fc3..2e52af9b 100644 --- a/src/cloudant/_2to3.py +++ b/src/cloudant/_2to3.py @@ -32,6 +32,9 @@ # pylint: disable=undefined-variable LONGTYPE = long if PY2 else int +# pylint: disable=undefined-variable +UNICHR = unichr if PY2 else chr + if PY2: # pylint: disable=wrong-import-position,no-name-in-module,import-error,unused-import from urllib import quote as url_quote, quote_plus as url_quote_plus diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index c9267976..cf6e74d5 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -29,6 +29,7 @@ import os import uuid +from cloudant._2to3 import UNICHR from cloudant.result import Result, QueryResult from cloudant.error import CloudantArgumentError, CloudantDatabaseException from cloudant.document import Document @@ -605,6 +606,39 @@ def test_document_iteration_over_fetch_limit(self): self.assertEqual(doc['name'], 'julia') self.assertEqual(doc['age'], int(id[len(id) - 3: len(id)])) + def test_document_iteration_completeness(self): + """ + Test __iter__ works as expected, fetching all documents from the + database. + """ + for _ in self.db: + self.fail('There should be no documents in the database yet!!') + + # sample code point ranges + include_ranges = [ + (0x0023, 0x0026), + (0x00A1, 0x00AC), + (0x0370, 0x0377), + (0x037A, 0x037E), + (0x0384, 0x038A), + (0x16A0, 0x16F0), + (0x2C60, 0x2C7F) + ] + + all_docs = [{'_id': UNICHR(i) + UNICHR(j)} for a, b in include_ranges + for i in range(a, b) + for j in range(a, b)] + batch_size = 500 + for i in range(0, len(all_docs), batch_size): + self.db.bulk_docs(all_docs[i:i+batch_size]) + + doc_count = 0 + for i, doc in enumerate(self.db): + doc_count += 1 + self.assertEqual(doc['_id'], all_docs[i]['_id']) + + self.assertEqual(doc_count, len(all_docs)) + def test_document_iteration_returns_valid_documents(self): """ This test will check that the __iter__ method returns documents that are From 106c19055add4a1012f1f15fa391d5c3a0289acd Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 28 Jun 2017 16:46:13 +0100 Subject: [PATCH 010/185] Use unicode Null U+0000 in startkey for all docs iterator --- src/cloudant/database.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 90d3030d..0beab0f9 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -630,18 +630,20 @@ def __iter__(self, remote=True): if not remote: super(CouchDatabase, self).__iter__() else: - next_startkey = '0' + # Use unicode Null U+0000 as the initial lower bound to ensure any + # document id could exist in the results set. + next_startkey = u'\u0000' while next_startkey is not None: docs = self.all_docs( - limit=self._fetch_limit + 1, # Get one extra doc - # to use as - # next_startkey + limit=self._fetch_limit, include_docs=True, startkey=next_startkey ).get('rows', []) - if len(docs) > self._fetch_limit: - next_startkey = docs.pop()['id'] + if len(docs) >= self._fetch_limit: + # Ensure the next document batch contains ids that sort + # strictly higher than the previous document id fetched. + next_startkey = docs[-1]['id'] + u'\u0000' else: # This is the last batch of docs, so we set # ourselves up to break out of the while loop From abd620ad8d1b3088bf08eda15e5506c9cbf6211c Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 29 Jun 2017 21:07:42 +0100 Subject: [PATCH 011/185] Update CHANGES.rst --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index a834f025..a25e9ffb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,7 @@ - [FIXED] Fixed Cloudant exception code 409 with 412 when creating a database that already exists. - [FIXED] Catch error if ``throw_on_exists`` flag is ``False`` for document create. - [FIXED] Fixed /_all_docs call where ``keys`` is an empty list. +- [FIXED] Issue where docs with IDs that sorted lower than 0 were not returned when iterating through _all_docs. 2.4.0 (2017-02-14) ================== From 4d537a3ff43c92a73bad1d46cdc8f3d5f08a16dd Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 6 Jul 2017 14:31:53 +0100 Subject: [PATCH 012/185] Prepare python-cloudant==2.5.0 release --- CHANGES.rst | 6 +++--- docs/conf.py | 4 ++-- setup.py | 2 +- src/cloudant/__init__.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index a25e9ffb..761892c8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,10 +1,10 @@ -2.5.0 (Unreleased) +2.5.0 (2017-07-06) ================== -- [FIXED] Fixed crash caused by non-UTF8 chars in ddocs +- [FIXED] Fixed crash caused by non-UTF8 chars in design documents. - [FIXED] Fixed ``TypeError`` when setting revision limits on Python>=3.6. - [FIXED] Fixed the ``exists()`` double check on ``client.py`` and ``database.py``. - [FIXED] Fixed Cloudant exception code 409 with 412 when creating a database that already exists. -- [FIXED] Catch error if ``throw_on_exists`` flag is ``False`` for document create. +- [FIXED] Catch error if ``throw_on_exists`` flag is ``False`` for creating a document. - [FIXED] Fixed /_all_docs call where ``keys`` is an empty list. - [FIXED] Issue where docs with IDs that sorted lower than 0 were not returned when iterating through _all_docs. diff --git a/docs/conf.py b/docs/conf.py index 6099c07d..8db31f57 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.5.0.dev' +version = '2.5.0' # The full version, including alpha/beta/rc tags. -release = '2.5.0.dev' +release = '2.5.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index beeb376f..f29f338a 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ 'include_package_data': True, 'install_requires': requirements, 'name': 'cloudant', - 'version': '2.5.0.dev', + 'version': '2.5.0', 'author': 'IBM', 'author_email': 'alfinkel@us.ibm.com', 'url': 'https://github.com/cloudant/python-cloudant', diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 9400ab2a..6ddd584f 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.5.0.dev' +__version__ = '2.5.0' # pylint: disable=wrong-import-position import contextlib From 4a11c70586cbe0c872b6acb5605c886af13fdba6 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 6 Jul 2017 15:58:25 +0100 Subject: [PATCH 013/185] Start next development phase at 2.6.0.dev --- CHANGES.rst | 3 +++ docs/conf.py | 4 ++-- setup.py | 2 +- src/cloudant/__init__.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 761892c8..e4cdb221 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,6 @@ +2.6.0 (Unreleased) +================== + 2.5.0 (2017-07-06) ================== - [FIXED] Fixed crash caused by non-UTF8 chars in design documents. diff --git a/docs/conf.py b/docs/conf.py index 8db31f57..b7536ab1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.5.0' +version = '2.6.0.dev' # The full version, including alpha/beta/rc tags. -release = '2.5.0' +release = '2.6.0.dev' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index f29f338a..8d4a6712 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ 'include_package_data': True, 'install_requires': requirements, 'name': 'cloudant', - 'version': '2.5.0', + 'version': '2.6.0.dev', 'author': 'IBM', 'author_email': 'alfinkel@us.ibm.com', 'url': 'https://github.com/cloudant/python-cloudant', diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 6ddd584f..cdf02898 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.5.0' +__version__ = '2.6.0.dev' # pylint: disable=wrong-import-position import contextlib From 27f731da6def4a0d9dcd6e3087a893716c586091 Mon Sep 17 00:00:00 2001 From: cclauss Date: Sat, 8 Jul 2017 13:26:51 +0200 Subject: [PATCH 014/185] Make examples compatible with Python 3 https://docs.python.org/3/whatsnew/3.0.html --- docs/getting_started.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 815d0452..93e3c1c8 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -56,8 +56,8 @@ Connecting with a client # Perform client tasks... session = client.session() - print 'Username: {0}'.format(session['userCtx']['name']) - print 'Databases: {0}'.format(client.all_dbs()) + print('Username: {0}'.format(session['userCtx']['name'])) + print('Databases: {0}'.format(client.all_dbs())) # Disconnect from the server client.disconnect() @@ -176,7 +176,7 @@ Creating a database # You can check that the database exists if my_database.exists(): - print 'SUCCESS!!' + print('SUCCESS!!') Opening a database ^^^^^^^^^^^^^^^^^^ @@ -228,7 +228,7 @@ Creating a document # Check that the document exists in the database if my_document.exists(): - print 'SUCCESS!!' + print('SUCCESS!!') Retrieving a document ^^^^^^^^^^^^^^^^^^^^^ @@ -244,7 +244,7 @@ classes are sub-classes of ``dict``, this is accomplished through standard my_document = my_database['julia30'] # Display the document - print my_document + print(my_document) Retrieve all documents ^^^^^^^^^^^^^^^^^^^^^^ @@ -256,7 +256,7 @@ to retrieve all documents in a database. # Get all of the documents from my_database for document in my_database: - print document + print(document) Update a document ^^^^^^^^^^^^^^^^^ @@ -333,7 +333,7 @@ object already exists. # Iterate over the result collection for result in result_collection: - print result + print(result) **************** Context managers @@ -367,13 +367,13 @@ and ``couchdb_admin_party`` context helpers. # Perform client tasks... session = client.session() - print 'Username: {0}'.format(session['userCtx']['name']) - print 'Databases: {0}'.format(client.all_dbs()) + print('Username: {0}'.format(session['userCtx']['name'])) + print('Databases: {0}'.format(client.all_dbs())) # Create a database my_database = client.create_database('my_database') if my_database.exists(): - print 'SUCCESS!!' + print('SUCCESS!!') # You can open an existing database del my_database @@ -387,12 +387,12 @@ and ``couchdb_admin_party`` context helpers. doc['pets'] = ['cat', 'dog', 'frog'] # Display a Document - print my_database['julia30'] + print(my_database['julia30']) # Delete the database client.delete_database('my_database') - print 'Databases: {0}'.format(client.all_dbs()) + print('Databases: {0}'.format(client.all_dbs())) **************** Endpoint access @@ -422,4 +422,4 @@ Cloudant/CouchDB server. This example assumes that either a ``Cloudant`` or a response = client.r_session.get(end_point, params=params) # Display the response content - print response.json() + print(response.json()) From 59f86ad4f7504c0a3b9077aa323c7a329dee55a1 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 10 Jul 2017 11:29:11 +0100 Subject: [PATCH 015/185] Update CONTRIBUTING.rst with DCO instructions --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- CONTRIBUTING.rst | 29 ++++++++++++----------------- DCO1.1.txt | 25 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 18 deletions(-) create mode 100644 DCO1.1.txt diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6a0b96ea..184340f6 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,6 @@ Thanks for your hard work, please ensure all items are complete before opening. -- [ ] You have signed the CLA as per the instructions in [CONTRIBUTING.rst](https://github.com/cloudant/python-cloudant/blob/master/CONTRIBUTING.rst#contributor-license-agreement) +- [ ] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](https://github.com/cloudant/python-cloudant/blob/master/DCO1.1.txt) - [ ] You have added tests for any code changes - [ ] You have updated the [CHANGES.rst](https://github.com/cloudant/python-cloudant/blob/master/CHANGES.rst) - [ ] You have completed the PR template below: diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 7c0106b5..c7520ee7 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -3,26 +3,21 @@ Developing this library Python-Cloudant Client Library is written in Python. -============================= -Contributor License Agreement -============================= +=============================== +Developer Certificate of Origin +=============================== -In order for us to accept pull-requests, the contributor must first complete -a Contributor License Agreement (CLA). This clarifies the intellectual -property license granted with any contribution. It is for your protection as a -Contributor as well as the protection of IBM and its customers; it does not -change your rights to use your own Contributions for any other purpose. +In order for us to accept pull-requests, the contributor must sign-off a +[Developer Certificate of Origin (DCO)](DCO1.1.txt). This clarifies the +intellectual property license granted with any contribution. It is for your +protection as a Contributor as well as the protection of IBM and its customers; +it does not change your rights to use your own Contributions for any other +purpose. -This is a quick process: one option is signing using Preview on a Mac, -then sending a copy to us via email. +Please read the agreement and acknowledge it by ticking the appropriate box in +the PR text, for example: -You can download the CLAs here: - -- `Individual `_ -- `Corporate `_ - -If you are an IBMer, please contact us directly as the contribution process is -slightly different. +- [x] Tick to sign-off your agreement to the Developer Certificate of Origin (DCO) 1.1 ====================== Development Quickstart diff --git a/DCO1.1.txt b/DCO1.1.txt new file mode 100644 index 00000000..f440e6fa --- /dev/null +++ b/DCO1.1.txt @@ -0,0 +1,25 @@ +Developer's Certificate of Origin 1.1 + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + + (b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + + (c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + + (d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. From 9a07ee4ce7b0c4f9badf1af4dc3a65b11073aba1 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Fri, 4 Aug 2017 11:41:53 +0100 Subject: [PATCH 016/185] Fix client construction in cloudant_bluemix context manager Change `cloudant_user` and `auth_token` to positional arguments. --- CHANGES.rst | 1 + src/cloudant/__init__.py | 4 ++-- tests/unit/client_tests.py | 28 ++++++++++++++++++++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index e4cdb221..6dfe4455 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,6 @@ 2.6.0 (Unreleased) ================== +- [FIXED] Fixed client construction in ``cloudant_bluemix`` context manager. 2.5.0 (2017-07-06) ================== diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index cdf02898..ade849a9 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -122,8 +122,8 @@ def cloudant_bluemix(vcap_services, instance_name=None, **kwargs): """ service = CloudFoundryService(vcap_services, instance_name) cloudant_session = Cloudant( - username=service.username, - password=service.password, + service.username, + service.password, url=service.url, **kwargs ) diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 797c90b9..137c26fb 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -30,7 +30,7 @@ from requests import ConnectTimeout -from cloudant import cloudant, couchdb, couchdb_admin_party +from cloudant import cloudant, cloudant_bluemix, couchdb, couchdb_admin_party from cloudant.client import Cloudant, CouchDB from cloudant.error import CloudantArgumentError, CloudantClientException from cloudant.feed import Feed, InfiniteFeed @@ -502,7 +502,31 @@ def test_cloudant_context_helper(self): self.assertIsInstance(c.r_session, requests.Session) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) - + + def test_cloudant_bluemix_context_helper(self): + """ + Test that the cloudant_bluemix context helper works as expected. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [{ + 'credentials': { + 'username': self.user, + 'password': self.pwd, + 'host': '{0}.cloudant.com'.format(self.account), + 'port': 443, + 'url': self.url + }, + 'name': instance_name, + }]} + + try: + with cloudant_bluemix(vcap_services, instance_name=instance_name) as c: + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + self.assertEquals(c.session()['userCtx']['name'], self.user) + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + def test_constructor_with_account(self): """ Test instantiating a client object using an account name From 9b5e532accb674925d5465e45f294b6c5a656b27 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Mon, 7 Aug 2017 09:47:04 -0400 Subject: [PATCH 017/185] Fixed validation for feed options to accept zero as a valid value --- CHANGES.rst | 1 + src/cloudant/feed.py | 2 +- tests/unit/changes_tests.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6dfe4455..f3c211c8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,7 @@ 2.6.0 (Unreleased) ================== - [FIXED] Fixed client construction in ``cloudant_bluemix`` context manager. +- [FIXED] Fixed validation for feed options to accept zero as a valid value. 2.5.0 (2017-07-06) ================== diff --git a/src/cloudant/feed.py b/src/cloudant/feed.py index f4797a38..44fa3525 100644 --- a/src/cloudant/feed.py +++ b/src/cloudant/feed.py @@ -116,7 +116,7 @@ def _validate(self, key, val, arg_types): if (not isinstance(val, arg_types[key]) or (isinstance(val, bool) and int in arg_types[key])): raise CloudantArgumentError(117, key, arg_types[key]) - if isinstance(val, int) and val <= 0 and not isinstance(val, bool): + if isinstance(val, int) and val < 0 and not isinstance(val, bool): raise CloudantArgumentError(118, key, val) if key == 'feed': valid_vals = ('continuous', 'normal', 'longpoll') diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index be78c520..8558c156 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -353,6 +353,20 @@ def test_get_feed_using_since_now(self): expected = set(['julia003', 'julia004', 'julia005']) self.assertSetEqual(set([x['id'] for x in changes]), expected) + def test_get_feed_using_since_zero(self): + """ + Test getting content back for a feed using since set to zero + """ + self.populate_db_with_documents(3) + feed = Feed(self.db, since=0) + changes = list() + for change in feed: + self.assertSetEqual(set(change.keys()), {'seq', 'changes', 'id'}) + changes.append(change) + expected = set(['julia{0:03d}'.format(i) for i in range(3)]) + self.assertSetEqual(set([x['id'] for x in changes]), expected) + self.assertTrue(str(feed.last_seq).startswith('3')) + def test_get_feed_using_timeout(self): """ Test getting content back for a feed using timeout From 17d8c6306a3855da3518023472b5ced36705b8de Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 8 Aug 2017 11:13:06 +0100 Subject: [PATCH 018/185] Implement Cloudant.bluemix client class method Added `Cloudant.bluemix()` client class method for instantiating a Cloudant Bluemix client. --- CHANGES.rst | 1 + src/cloudant/client.py | 31 ++++++++++- tests/unit/client_tests.py | 108 +++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6dfe4455..afbb6520 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,6 @@ 2.6.0 (Unreleased) ================== +- [NEW] Added ``Cloudant.bluemix()`` class method to the Cloudant client allowing service credentials to be passed using the CloudFoundry VCAP_SERVICES environment variable. - [FIXED] Fixed client construction in ``cloudant_bluemix`` context manager. 2.5.0 (2017-07-06) diff --git a/src/cloudant/client.py b/src/cloudant/client.py index e70e9af3..3c7da688 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -31,7 +31,8 @@ USER_AGENT, append_response_error_content, InfiniteSession, - ClientSession) + ClientSession, + CloudFoundryService) class CouchDB(dict): @@ -764,3 +765,31 @@ def _write_cors_configuration(self, config): resp.raise_for_status() return resp.json() + + @classmethod + def bluemix(cls, vcap_services, instance_name=None, **kwargs): + """ + Create a Cloudant session using a VCAP_SERVICES environment variable. + + :param vcap_services: VCAP_SERVICES environment variable + :type vcap_services: dict or str + :param str instance_name: Optional Bluemix instance name. Only required + if multiple Cloudant instances are available. + + Example usage: + + .. code-block:: python + + import os + from cloudant.client import Cloudant + + client = Cloudant.bluemix(os.getenv('VCAP_SERVICES'), + 'Cloudant NoSQL DB') + + print client.all_dbs() + """ + service = CloudFoundryService(vcap_services, instance_name) + return Cloudant(service.username, + service.password, + url=service.url, + **kwargs) diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 137c26fb..b2dbcbf0 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -539,6 +539,114 @@ def test_constructor_with_account(self): 'https://{0}.cloudant.com'.format(self.account) ) + def test_bluemix_constructor(self): + """ + Test instantiating a client object using a VCAP_SERVICES environment + variable. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [{ + 'credentials': { + 'username': self.user, + 'password': self.pwd, + 'host': '{0}.cloudant.com'.format(self.account), + 'port': 443, + 'url': self.url + }, + 'name': instance_name + }]} + + # create Cloudant Bluemix client + c = Cloudant.bluemix(vcap_services) + + try: + c.connect() + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + self.assertEquals(c.session()['userCtx']['name'], self.user) + + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + + finally: + c.disconnect() + + def test_bluemix_constructor_specify_instance_name(self): + """ + Test instantiating a client object using a VCAP_SERVICES environment + variable and specifying which instance name to use. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [{ + 'credentials': { + 'username': self.user, + 'password': self.pwd, + 'host': '{0}.cloudant.com'.format(self.account), + 'port': 443, + 'url': self.url + }, + 'name': instance_name + }]} + + # create Cloudant Bluemix client + c = Cloudant.bluemix(vcap_services, instance_name=instance_name) + + try: + c.connect() + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + self.assertEquals(c.session()['userCtx']['name'], self.user) + + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + + finally: + c.disconnect() + + def test_bluemix_constructor_with_multiple_services(self): + """ + Test instantiating a client object using a VCAP_SERVICES environment + variable that contains multiple services. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [ + { + 'credentials': { + 'username': self.user, + 'password': self.pwd, + 'host': '{0}.cloudant.com'.format(self.account), + 'port': 443, + 'url': self.url + }, + 'name': instance_name + }, + { + 'credentials': { + 'username': 'foo', + 'password': 'bar', + 'host': 'baz.com', + 'port': 1234, + 'url': 'https://foo:bar@baz.com:1234' + }, + 'name': 'Cloudant NoSQL DB-yu' + } + ]} + + # create Cloudant Bluemix client + c = Cloudant.bluemix(vcap_services, instance_name=instance_name) + + try: + c.connect() + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + self.assertEquals(c.session()['userCtx']['name'], self.user) + + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + + finally: + c.disconnect() + def test_connect_headers(self): """ Test that the appropriate request headers are set From 4418a0d16510569a4eef638114d8f7baef398134 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 10 Aug 2017 11:23:14 +0100 Subject: [PATCH 019/185] Prepare for 2.6.0 release --- CHANGES.rst | 2 +- docs/conf.py | 4 ++-- setup.py | 2 +- src/cloudant/__init__.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 632bead9..83d8e7e0 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -2.6.0 (Unreleased) +2.6.0 (2017-08-10) ================== - [NEW] Added ``Cloudant.bluemix()`` class method to the Cloudant client allowing service credentials to be passed using the CloudFoundry VCAP_SERVICES environment variable. - [FIXED] Fixed client construction in ``cloudant_bluemix`` context manager. diff --git a/docs/conf.py b/docs/conf.py index b7536ab1..b03f4ddc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.6.0.dev' +version = '2.6.0' # The full version, including alpha/beta/rc tags. -release = '2.6.0.dev' +release = '2.6.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 8d4a6712..b9fd26a5 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ 'include_package_data': True, 'install_requires': requirements, 'name': 'cloudant', - 'version': '2.6.0.dev', + 'version': '2.6.0', 'author': 'IBM', 'author_email': 'alfinkel@us.ibm.com', 'url': 'https://github.com/cloudant/python-cloudant', diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index ade849a9..1bd702af 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.6.0.dev' +__version__ = '2.6.0' # pylint: disable=wrong-import-position import contextlib From d007eb8e01ed3ab46d8188b647266e652d32ec14 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 10 Aug 2017 14:12:15 +0100 Subject: [PATCH 020/185] Start next development phase at 2.6.1.dev --- CHANGES.rst | 3 +++ docs/conf.py | 4 ++-- setup.py | 2 +- src/cloudant/__init__.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 83d8e7e0..edcc9c76 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,6 @@ +Unreleased +========== + 2.6.0 (2017-08-10) ================== - [NEW] Added ``Cloudant.bluemix()`` class method to the Cloudant client allowing service credentials to be passed using the CloudFoundry VCAP_SERVICES environment variable. diff --git a/docs/conf.py b/docs/conf.py index b03f4ddc..d993fc17 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.6.0' +version = '2.6.1.dev' # The full version, including alpha/beta/rc tags. -release = '2.6.0' +release = '2.6.1.dev' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index b9fd26a5..d381c2a5 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ 'include_package_data': True, 'install_requires': requirements, 'name': 'cloudant', - 'version': '2.6.0', + 'version': '2.6.1.dev', 'author': 'IBM', 'author_email': 'alfinkel@us.ibm.com', 'url': 'https://github.com/cloudant/python-cloudant', diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 1bd702af..8cb9f75f 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.6.0' +__version__ = '2.6.1.dev' # pylint: disable=wrong-import-position import contextlib From ce75c57e2da8c5a00ca445b74c56e9752e38a683 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 14 Aug 2017 11:43:57 +0100 Subject: [PATCH 021/185] Use `'/'.join()` to concatenate URL parts --- CHANGES.rst | 1 + src/cloudant/client.py | 44 ++++++++++++++---------------------- src/cloudant/database.py | 31 ++++++++++++------------- src/cloudant/document.py | 15 ++++++------ src/cloudant/index.py | 5 ++-- src/cloudant/query.py | 3 +-- src/cloudant/view.py | 5 ++-- tests/unit/database_tests.py | 10 ++++---- tests/unit/document_tests.py | 21 ++++++----------- tests/unit/index_tests.py | 3 +-- tests/unit/query_tests.py | 3 +-- tests/unit/view_tests.py | 3 +-- 12 files changed, 58 insertions(+), 86 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index edcc9c76..70c1f562 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,6 @@ Unreleased ========== +- [IMPROVED] Updated ``posixpath.join`` references to use ``'/'.join`` when concatenating URL parts. 2.6.0 (2017-08-10) ================== diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 3c7da688..7c69c461 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -18,7 +18,6 @@ """ import base64 import json -import posixpath from ._2to3 import bytes_, unicode_ from .database import CloudantDatabase, CouchDatabase @@ -138,7 +137,7 @@ def session(self): """ if self.admin_party: return None - sess_url = posixpath.join(self.server_url, '_session') + sess_url = '/'.join((self.server_url, '_session')) resp = self.r_session.get(sess_url) resp.raise_for_status() sess_data = resp.json() @@ -164,7 +163,7 @@ def session_login(self, user, passwd): """ if self.admin_party: return - sess_url = posixpath.join(self.server_url, '_session') + sess_url = '/'.join((self.server_url, '_session')) resp = self.r_session.post( sess_url, data={ @@ -182,7 +181,7 @@ def session_logout(self): """ if self.admin_party: return - sess_url = posixpath.join(self.server_url, '_session') + sess_url = '/'.join((self.server_url, '_session')) resp = self.r_session.delete(sess_url) resp.raise_for_status() @@ -207,7 +206,7 @@ def all_dbs(self): :returns: List of database names for the client """ - url = posixpath.join(self.server_url, '_all_dbs') + url = '/'.join((self.server_url, '_all_dbs')) resp = self.r_session.get(url) resp.raise_for_status() return resp.json() @@ -560,9 +559,7 @@ def _usage_endpoint(self, endpoint, year=None, month=None): try: if int(year) > 0 and int(month) in range(1, 13): resp = self.r_session.get( - posixpath.join( - endpoint, str(int(year)), str(int(month))) - ) + '/'.join((endpoint, str(int(year)), str(int(month))))) else: err = True except (ValueError, TypeError): @@ -587,7 +584,7 @@ def bill(self, year=None, month=None): :returns: Billing data in JSON format """ - endpoint = posixpath.join(self.server_url, '_api', 'v2', 'bill') + endpoint = '/'.join((self.server_url, '_api', 'v2', 'bill')) return self._usage_endpoint(endpoint, year, month) def volume_usage(self, year=None, month=None): @@ -604,9 +601,8 @@ def volume_usage(self, year=None, month=None): :returns: Volume usage data in JSON format """ - endpoint = posixpath.join( - self.server_url, '_api', 'v2', 'usage', 'data_volume' - ) + endpoint = '/'.join(( + self.server_url, '_api', 'v2', 'usage', 'data_volume')) return self._usage_endpoint(endpoint, year, month) def requests_usage(self, year=None, month=None): @@ -623,9 +619,8 @@ def requests_usage(self, year=None, month=None): :returns: Requests usage data in JSON format """ - endpoint = posixpath.join( - self.server_url, '_api', 'v2', 'usage', 'requests' - ) + endpoint = '/'.join(( + self.server_url, '_api', 'v2', 'usage', 'requests')) return self._usage_endpoint(endpoint, year, month) def shared_databases(self): @@ -635,9 +630,8 @@ def shared_databases(self): :returns: List of database names """ - endpoint = posixpath.join( - self.server_url, '_api', 'v2', 'user', 'shared_databases' - ) + endpoint = '/'.join(( + self.server_url, '_api', 'v2', 'user', 'shared_databases')) resp = self.r_session.get(endpoint) resp.raise_for_status() data = resp.json() @@ -649,9 +643,7 @@ def generate_api_key(self): :returns: API key/pass pair in JSON format """ - endpoint = posixpath.join( - self.server_url, '_api', 'v2', 'api_keys' - ) + endpoint = '/'.join((self.server_url, '_api', 'v2', 'api_keys')) resp = self.r_session.post(endpoint) resp.raise_for_status() return resp.json() @@ -662,9 +654,8 @@ def cors_configuration(self): :returns: CORS data in JSON format """ - endpoint = posixpath.join( - self.server_url, '_api', 'v2', 'user', 'config', 'cors' - ) + endpoint = '/'.join(( + self.server_url, '_api', 'v2', 'user', 'config', 'cors')) resp = self.r_session.get(endpoint) resp.raise_for_status() @@ -754,9 +745,8 @@ def _write_cors_configuration(self, config): :returns: CORS configuration update status in JSON format """ - endpoint = posixpath.join( - self.server_url, '_api', 'v2', 'user', 'config', 'cors' - ) + endpoint = '/'.join(( + self.server_url, '_api', 'v2', 'user', 'config', 'cors')) resp = self.r_session.put( endpoint, data=json.dumps(config, cls=self.encoder), diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 0beab0f9..47c2a1fd 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -17,7 +17,6 @@ """ import json import contextlib -import posixpath from requests.exceptions import HTTPError @@ -84,10 +83,8 @@ def database_url(self): :returns: Database URL """ - return posixpath.join( - self._database_host, - url_quote_plus(self.database_name) - ) + return '/'.join(( + self._database_host, url_quote_plus(self.database_name))) @property def creds(self): @@ -189,7 +186,7 @@ def design_documents(self): :returns: All design documents found in this database in JSON format """ - url = posixpath.join(self.database_url, '_all_docs') + url = '/'.join((self.database_url, '_all_docs')) query = "startkey=\"_design\"&endkey=\"_design0\"&include_docs=true" resp = self.r_session.get(url, params=query) resp.raise_for_status() @@ -203,7 +200,7 @@ def list_design_documents(self): :returns: List of names for all design documents in this database """ - url = posixpath.join(self.database_url, '_all_docs') + url = '/'.join((self.database_url, '_all_docs')) query = "startkey=\"_design\"&endkey=\"_design0\"" resp = self.r_session.get(url, params=query) resp.raise_for_status() @@ -675,7 +672,7 @@ def bulk_docs(self, docs): :returns: Bulk document creation/update status in JSON format """ - url = posixpath.join(self.database_url, '_bulk_docs') + url = '/'.join((self.database_url, '_bulk_docs')) data = {'docs': docs} headers = {'Content-Type': 'application/json'} resp = self.r_session.post( @@ -698,7 +695,7 @@ def missing_revisions(self, doc_id, *revisions): :returns: List of missing document revision values """ - url = posixpath.join(self.database_url, '_missing_revs') + url = '/'.join((self.database_url, '_missing_revs')) data = {doc_id: list(revisions)} resp = self.r_session.post( @@ -727,7 +724,7 @@ def revisions_diff(self, doc_id, *revisions): :returns: The revision differences in JSON format """ - url = posixpath.join(self.database_url, '_revs_diff') + url = '/'.join((self.database_url, '_revs_diff')) data = {doc_id: list(revisions)} resp = self.r_session.post( @@ -746,7 +743,7 @@ def get_revision_limit(self): :returns: Revision limit value for the current remote database """ - url = posixpath.join(self.database_url, '_revs_limit') + url = '/'.join((self.database_url, '_revs_limit')) resp = self.r_session.get(url) resp.raise_for_status() @@ -767,7 +764,7 @@ def set_revision_limit(self, limit): :returns: Revision limit set operation status in JSON format """ - url = posixpath.join(self.database_url, '_revs_limit') + url = '/'.join((self.database_url, '_revs_limit')) resp = self.r_session.put(url, data=json.dumps(limit, cls=self.client.encoder)) resp.raise_for_status() @@ -781,7 +778,7 @@ def view_cleanup(self): :returns: View cleanup status in JSON format """ - url = posixpath.join(self.database_url, '_view_cleanup') + url = '/'.join((self.database_url, '_view_cleanup')) resp = self.r_session.post( url, headers={'Content-Type': 'application/json'} @@ -958,8 +955,8 @@ def security_url(self): :returns: Security document URL """ - parts = ['_api', 'v2', 'db', self.database_name, '_security'] - url = posixpath.join(self._database_host, *parts) + url = '/'.join((self._database_host, '_api', 'v2', 'db', + self.database_name, '_security')) return url def share_database(self, username, roles=None): @@ -1040,7 +1037,7 @@ def shards(self): :returns: Shard information retrieval status in JSON format """ - url = posixpath.join(self.database_url, '_shards') + url = '/'.join((self.database_url, '_shards')) resp = self.r_session.get(url) resp.raise_for_status() @@ -1059,7 +1056,7 @@ def get_query_indexes(self, raw_result=False): :returns: The query indexes in the database """ - url = posixpath.join(self.database_url, '_index') + url = '/'.join((self.database_url, '_index')) resp = self.r_session.get(url) resp.raise_for_status() diff --git a/src/cloudant/document.py b/src/cloudant/document.py index 500acc18..8b93c92f 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -16,7 +16,6 @@ API module/class for interacting with a document in a database. """ import json -import posixpath import requests from requests.exceptions import HTTPError @@ -87,19 +86,19 @@ def document_url(self): # handle design document url if self._document_id.startswith('_design/'): - return posixpath.join( + return '/'.join(( self._database_host, url_quote_plus(self._database_name), '_design', url_quote(self._document_id[8:], safe='') - ) + )) # handle document url - return posixpath.join( + return '/'.join(( self._database_host, url_quote_plus(self._database_name), url_quote(self._document_id, safe='') - ) + )) def exists(self): """ @@ -389,7 +388,7 @@ def get_attachment( """ # need latest rev self.fetch() - attachment_url = posixpath.join(self.document_url, attachment) + attachment_url = '/'.join((self.document_url, attachment)) if headers is None: headers = {'If-Match': self['_rev']} else: @@ -432,7 +431,7 @@ def delete_attachment(self, attachment, headers=None): """ # need latest rev self.fetch() - attachment_url = posixpath.join(self.document_url, attachment) + attachment_url = '/'.join((self.document_url, attachment)) if headers is None: headers = {'If-Match': self['_rev']} else: @@ -473,7 +472,7 @@ def put_attachment(self, attachment, content_type, data, headers=None): """ # need latest rev self.fetch() - attachment_url = posixpath.join(self.document_url, attachment) + attachment_url = '/'.join((self.document_url, attachment)) if headers is None: headers = { 'If-Match': self['_rev'], diff --git a/src/cloudant/index.py b/src/cloudant/index.py index e44fd258..5f34b76b 100644 --- a/src/cloudant/index.py +++ b/src/cloudant/index.py @@ -16,7 +16,6 @@ API module for managing/viewing query indexes. """ -import posixpath import json from ._2to3 import STRTYPE, iteritems_ @@ -60,7 +59,7 @@ def index_url(self): :returns: Index URL """ - return posixpath.join(self._database.database_url, '_index') + return '/'.join((self._database.database_url, '_index')) @property def design_document_id(self): @@ -166,7 +165,7 @@ def delete(self): ddoc_id = self._ddoc_id if ddoc_id.startswith('_design/'): ddoc_id = ddoc_id[8:] - url = posixpath.join(self.index_url, ddoc_id, self._type, self._name) + url = '/'.join((self.index_url, ddoc_id, self._type, self._name)) resp = self._r_session.delete(url) resp.raise_for_status() return diff --git a/src/cloudant/query.py b/src/cloudant/query.py index a72ec6a7..b5bd2b58 100644 --- a/src/cloudant/query.py +++ b/src/cloudant/query.py @@ -16,7 +16,6 @@ API module for composing and executing Cloudant queries. """ -import posixpath import json import contextlib @@ -105,7 +104,7 @@ def url(self): :returns: Query URL """ - return posixpath.join(self._database.database_url, '_find') + return '/'.join((self._database.database_url, '_find')) def __call__(self, **kwargs): """ diff --git a/src/cloudant/view.py b/src/cloudant/view.py index 37449e8d..685600e5 100644 --- a/src/cloudant/view.py +++ b/src/cloudant/view.py @@ -16,7 +16,6 @@ API module for interacting with a view in a design document. """ import contextlib -import posixpath from ._2to3 import STRTYPE from ._common_util import codify, get_docs @@ -168,11 +167,11 @@ def url(self): :returns: View URL """ - return posixpath.join( + return '/'.join(( self.design_doc.document_url, '_view', self.view_name - ) + )) def __call__(self, **kwargs): """ diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index cf6e74d5..1ea6bec6 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -25,7 +25,6 @@ import unittest import mock import requests -import posixpath import os import uuid @@ -149,8 +148,8 @@ def test_retrieve_db_url(self): """ self.assertEqual( self.db.database_url, - posixpath.join(self.client.server_url, self.test_dbname) - ) + '/'.join((self.client.server_url, self.test_dbname)) + ) def test_retrieve_creds(self): """ @@ -233,8 +232,7 @@ def test_retrieve_db_metadata(self): same. Therefore comparing keys is a valid test of this functionality. """ resp = self.db.r_session.get( - posixpath.join(self.client.server_url, self.test_dbname) - ) + '/'.join((self.client.server_url, self.test_dbname))) expected = resp.json() actual = self.db.metadata() self.assertListEqual(list(actual.keys()), list(expected.keys())) @@ -656,7 +654,7 @@ def test_document_iteration_returns_valid_documents(self): # A valid document must have a document_url self.assertEqual( doc.document_url, - posixpath.join(self.db.database_url, doc['_id']) + '/'.join((self.db.database_url, doc['_id'])) ) if isinstance(doc, DesignDocument): self.assertEqual(doc['_id'], '_design/ddoc001') diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 2d76389f..93e3659d 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -24,7 +24,6 @@ import unittest import mock -import posixpath import json import requests import os @@ -115,7 +114,7 @@ def test_document_url(self): """ doc = Document(self.db, 'julia006') self.assertEqual( - doc.document_url, posixpath.join(self.db.database_url, 'julia006') + doc.document_url, '/'.join((self.db.database_url, 'julia006')) ) def test_document_url_encodes_correctly(self): @@ -124,10 +123,8 @@ def test_document_url_encodes_correctly(self): """ doc = Document(self.db, 'http://example.com') self.assertEqual( - doc.document_url, posixpath.join( - self.db.database_url, - 'http%3A%2F%2Fexample.com' - ) + doc.document_url, + '/'.join((self.db.database_url, 'http%3A%2F%2Fexample.com')) ) def test_design_document_url(self): @@ -137,10 +134,8 @@ def test_design_document_url(self): """ doc = Document(self.db, '_design/ddoc001') self.assertEqual( - doc.document_url, posixpath.join( - self.db.database_url, - '_design/ddoc001' - ) + doc.document_url, + '/'.join((self.db.database_url, '_design/ddoc001')) ) def test_design_document_url_encodes_correctly(self): @@ -149,10 +144,8 @@ def test_design_document_url_encodes_correctly(self): """ doc = Document(self.db, '_design/http://example.com') self.assertEqual( - doc.document_url, posixpath.join( - self.db.database_url, - '_design/http%3A%2F%2Fexample.com' - ) + doc.document_url, + '/'.join((self.db.database_url, '_design/http%3A%2F%2Fexample.com')) ) def test_constructor_without_docid(self): diff --git a/tests/unit/index_tests.py b/tests/unit/index_tests.py index 398e3080..28696286 100644 --- a/tests/unit/index_tests.py +++ b/tests/unit/index_tests.py @@ -25,7 +25,6 @@ import unittest import mock import os -import posixpath import requests from cloudant.index import Index, TextIndex, SpecialIndex @@ -122,7 +121,7 @@ def test_retrieve_index_url(self): index = Index(self.db) self.assertEqual( index.index_url, - posixpath.join(self.db.database_url, '_index') + '/'.join((self.db.database_url, '_index')) ) def test_index_to_dictionary(self): diff --git a/tests/unit/query_tests.py b/tests/unit/query_tests.py index 214fecc0..72fd35f1 100644 --- a/tests/unit/query_tests.py +++ b/tests/unit/query_tests.py @@ -22,7 +22,6 @@ import unittest import os -import posixpath from cloudant.query import Query from cloudant.result import QueryResult @@ -78,7 +77,7 @@ def test_retrieve_query_url(self): query = Query(self.db) self.assertEqual( query.url, - posixpath.join(self.db.database_url, '_find') + '/'.join((self.db.database_url, '_find')) ) def test_callable_with_invalid_argument(self): diff --git a/tests/unit/view_tests.py b/tests/unit/view_tests.py index 39d7efb4..35bac6e3 100644 --- a/tests/unit/view_tests.py +++ b/tests/unit/view_tests.py @@ -24,7 +24,6 @@ import unittest import mock -import posixpath import requests import os @@ -182,7 +181,7 @@ def test_retrieve_view_url(self): view = View(ddoc, 'view001') self.assertEqual( view.url, - posixpath.join(ddoc.document_url, '_view/view001') + '/'.join((ddoc.document_url, '_view/view001')) ) def test_get_view_callable_raw_json(self): From 95d67d9868ce442beb42b502a421bdb25deee57c Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 14 Aug 2017 10:57:49 +0100 Subject: [PATCH 022/185] Update Document CM usage in docs/getting_started.rst --- docs/getting_started.rst | 47 ++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 93e3c1c8..17fe3e60 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -342,27 +342,27 @@ Context managers Now that we've gone through the basics, let's take a look at how to simplify the process of connection, database acquisition, and document management through the use of Python *with* blocks and this library's context managers. -Handling your business using *with* blocks saves you from having to connect and + +Handling your business using *with* blocks saves you from having to connect and disconnect your client as well as saves you from having to perform a lot of fetch and save operations as the context managers handle these operations for -you. This example uses the ``cloudant`` context helper to illustrate the +you. + +This example uses the ``cloudant`` context helper to illustrate the process but identical functionality exists for CouchDB through the ``couchdb`` and ``couchdb_admin_party`` context helpers. .. code-block:: python - # cloudant context helper from cloudant import cloudant - # couchdb context helper + # ...or use CouchDB variant # from cloudant import couchdb - from cloudant.document import Document - # Perform a connect upon entry and a disconnect upon exit of the block with cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME) as client: - # CouchDB variant + # ...or use CouchDB variant # with couchdb(USERNAME, PASSWORD, url=COUCHDB_URL) as client: # Perform client tasks... @@ -378,9 +378,22 @@ and ``couchdb_admin_party`` context helpers. # You can open an existing database del my_database my_database = client['my_database'] - + +The following example uses the ``Document`` context manager. Here we make +multiple updates to a single document. Note that we don't save to the server +after each update. We only save once to the server upon exiting the ``Document`` +context manager. + + .. code-block:: python + + from cloudant import cloudant + from cloudant.document import Document + + with cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME) as client: + + my_database = client.create_database('my_database') + # Performs a fetch upon entry and a save upon exit of this block - # Use this context manager to create or update a Document with Document(my_database, 'julia30') as doc: doc['name'] = 'Julia' doc['age'] = 30 @@ -394,6 +407,22 @@ and ``couchdb_admin_party`` context helpers. print('Databases: {0}'.format(client.all_dbs())) +Always use the ``_deleted`` document property to delete a document from within +a ``Document`` context manager. For example: + + .. code-block:: python + + with Document(my_database, 'julia30') as doc: + doc['_deleted'] = True + +*You can also delete non underscore prefixed document keys to reduce the size of the request.* + +.. warning:: Don't use the ``doc.delete()`` method inside your ``Document`` + context manager. This method immediately deletes the document on + the server and clears the local document dictionary. A new, empty + document is still saved to the server upon exiting the context + manager. + **************** Endpoint access **************** From 3242823f197c23c0d838b00ee46c67e2face8e8d Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 14 Aug 2017 12:02:25 +0100 Subject: [PATCH 023/185] Add ``Result.all()`` convenience method --- CHANGES.rst | 1 + src/cloudant/result.py | 12 ++++++++++++ tests/unit/result_tests.py | 10 ++++++++++ 3 files changed, 23 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 70c1f562..cfae6908 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,6 @@ Unreleased ========== +- [NEW] Added ``Result.all()`` convenience method. - [IMPROVED] Updated ``posixpath.join`` references to use ``'/'.join`` when concatenating URL parts. 2.6.0 (2017-08-10) diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 8cac3ae4..f4be5f1f 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -359,6 +359,18 @@ def _parse_data(self, data): """ return data.get('rows', []) + def all(self): + """ + Retrieve all results. + + Specifying a ``limit`` parameter in the ``Result`` constructor will + limit the number of documents returned. Be aware that the ``page_size`` + parameter is not honoured. + + :return: results data as list in JSON format. + """ + return self[:] + class QueryResult(Result): """ Provides a index key accessible, sliceable and iterable interface to query diff --git a/tests/unit/result_tests.py b/tests/unit/result_tests.py index f33ce0e4..a7ede996 100644 --- a/tests/unit/result_tests.py +++ b/tests/unit/result_tests.py @@ -248,6 +248,16 @@ def test_get_item_slice_no_start_no_stop(self): {'key': 'julia002', 'id': 'julia002', 'value': 1}] self.assertEqual(result[:], expected) + def test_get_all_items(self): + """ + Test that all results can be retrieved. + """ + result = Result(self.view001, limit=3) + expected = [{'key': 'julia000', 'id': 'julia000', 'value': 1}, + {'key': 'julia001', 'id': 'julia001', 'value': 1}, + {'key': 'julia002', 'id': 'julia002', 'value': 1}] + self.assertEqual(result.all(), expected) + def test_get_item_invalid_index_slice(self): """ Test that when invalid start and stop values are provided in a slice From 090afb735223c0fed4b29c6428385b2de35cb034 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 26 Jun 2017 17:48:39 +0100 Subject: [PATCH 024/185] Add IAM authentication support --- src/cloudant/_2to3.py | 2 + src/cloudant/__init__.py | 15 +++ src/cloudant/_common_util.py | 190 +++++++++++++++++++++++++++++------ src/cloudant/client.py | 42 ++++++-- 4 files changed, 205 insertions(+), 44 deletions(-) diff --git a/src/cloudant/_2to3.py b/src/cloudant/_2to3.py index 2e52af9b..26f995ac 100644 --- a/src/cloudant/_2to3.py +++ b/src/cloudant/_2to3.py @@ -39,6 +39,7 @@ # pylint: disable=wrong-import-position,no-name-in-module,import-error,unused-import from urllib import quote as url_quote, quote_plus as url_quote_plus from urlparse import urlparse as url_parse + from urlparse import urljoin as url_join from ConfigParser import RawConfigParser def iteritems_(adict): @@ -60,6 +61,7 @@ def next_(itr): return itr.next() else: from urllib.parse import urlparse as url_parse # pylint: disable=wrong-import-position,no-name-in-module,import-error,ungrouped-imports + from urllib.parse import urljoin as url_join # pylint: disable=wrong-import-position,no-name-in-module,import-error,ungrouped-imports from urllib.parse import quote as url_quote # pylint: disable=wrong-import-position,no-name-in-module,import-error,ungrouped-imports from urllib.parse import quote_plus as url_quote_plus # pylint: disable=wrong-import-position,no-name-in-module,import-error,ungrouped-imports from configparser import RawConfigParser # pylint: disable=wrong-import-position,no-name-in-module,import-error diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 8cb9f75f..b2a06483 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -62,6 +62,21 @@ def cloudant(user, passwd, **kwargs): yield cloudant_session cloudant_session.disconnect() +@contextlib.contextmanager +def cloudant_iam(api_key, account_name, **kwargs): + """ + Provides a context manager to create a Cloudant session and provide access + to databases, docs etc. + + :param api_key: IAM authentication API key. + :param account_name: Cloudant account name. + """ + cloudant_session = Cloudant(account_name, api_key, use_iam=True, **kwargs) + + cloudant_session.connect() + yield cloudant_session + cloudant_session.disconnect() + @contextlib.contextmanager def cloudant_bluemix(vcap_services, instance_name=None, **kwargs): """ diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 6dcf331d..0a2344ce 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -17,13 +17,15 @@ throughout the library. """ +import os import sys import platform from collections import Sequence import json -from requests import Session +from requests import RequestException, Session -from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_, url_parse +from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_, url_parse, \ + url_join from .error import CloudantArgumentError, CloudantException # Library Constants @@ -276,6 +278,7 @@ def append_response_error_content(response, **kwargs): # Classes + class _Code(str): """ Wraps a ``str`` object as a _Code object providing the means to handle @@ -287,66 +290,187 @@ def __new__(cls, code): return str.__new__(cls, code.encode('utf8')) return str.__new__(cls, code) -class InfiniteSession(Session): + +class ClientSession(Session): """ - This class provides for the ability to automatically renew session login - information in the event of expired session authentication. + This class extends Session and provides a default timeout. + """ + + def __init__(self, **kwargs): + super(ClientSession, self).__init__() + self._timeout = kwargs.get('timeout', None) + + def request(self, method, url, **kwargs): # pylint: disable=W0221 + """ + Overrides ``requests.Session.request`` to set the timeout. + """ + resp = super(ClientSession, self).request( + method, url, timeout=self._timeout, **kwargs) + + return resp + + +class CookieSession(ClientSession): + """ + This class extends ClientSession and provides cookie authentication. """ def __init__(self, username, password, server_url, **kwargs): - super(InfiniteSession, self).__init__() + super(CookieSession, self).__init__(**kwargs) self._username = username self._password = password - self._server_url = server_url - self._timeout = kwargs.get('timeout', None) + self._auto_renew = kwargs.get('auto_renew', False) + self._session_url = url_join(server_url, '_session') + + def info(self): + """ + Get cookie based login user information. + """ + resp = self.get(self._session_url) + resp.raise_for_status() + + return resp.json() + + def login(self): + """ + Perform cookie based user login. + """ + resp = super(CookieSession, self).request( + 'POST', + self._session_url, + data={'name': self._username, 'password': self._password}, + ) + resp.raise_for_status() + + def logout(self): + """ + Logout cookie based user. + """ + resp = super(CookieSession, self).request('DELETE', self._session_url) + resp.raise_for_status() def request(self, method, url, **kwargs): # pylint: disable=W0221 """ - Overrides ``requests.Session.request`` to perform a POST to the - _session endpoint to renew Session cookie authentication settings and - then retry the original request, if necessary. + Overrides ``requests.Session.request`` to renew the cookie and then + retry the original request (if required). """ - resp = super(InfiniteSession, self).request( - method, url, timeout=self._timeout, **kwargs) + resp = super(CookieSession, self).request(method, url, **kwargs) + path = url_parse(url).path.lower() post_to_session = method.upper() == 'POST' and path == '/_session' + + if not self._auto_renew or post_to_session: + return resp + is_expired = any(( resp.status_code == 403 and resp.json().get('error') == 'credentials_expired', resp.status_code == 401 )) - if not post_to_session and is_expired: - super(InfiniteSession, self).request( - 'POST', - '/'.join([self._server_url, '_session']), - data={'name': self._username, 'password': self._password}, - headers={'Content-Type': 'application/x-www-form-urlencoded'} - ) - resp = super(InfiniteSession, self).request( - method, url, timeout=self._timeout, **kwargs) + + if is_expired: + self.login() + resp = super(CookieSession, self).request(method, url, **kwargs) return resp -class ClientSession(Session): + +class IAMSession(ClientSession): """ - This class extends Session and provides a default timeout. + This class extends ClientSession and provides IAM authentication. """ - def __init__(self, username, password, server_url, **kwargs): - super(ClientSession, self).__init__() - self._username = username - self._password = password - self._server_url = server_url - self._timeout = kwargs.get('timeout', None) + def __init__(self, api_key, server_url, **kwargs): + super(IAMSession, self).__init__(**kwargs) + self._api_key = api_key + self._auto_renew = kwargs.get('auto_renew', False) + self._session_url = url_join(server_url, '_iam_session') + self._token_url = os.environ.get( + 'IAM_TOKEN_URL', 'https://iam.bluemix.net/oidc/token') + + def info(self): + """ + Get IAM cookie based login user information. + """ + resp = self.get(self._session_url) + resp.raise_for_status() + + return resp.json() + + def login(self): + """ + Perform IAM cookie based user login. + """ + access_token = self._get_access_token() + try: + super(IAMSession, self).request( + 'POST', + self._session_url, + headers={'Content-Type': 'application/json'}, + data=json.dumps({'access_token': access_token}) + ).raise_for_status() + + except RequestException: + raise CloudantException( + 'Failed to exchange IAM token with Cloudant') + + def logout(self): + """ + Logout IAM cookie based user. + """ + self.cookies.clear() def request(self, method, url, **kwargs): # pylint: disable=W0221 """ - Overrides ``requests.Session.request`` to set the timeout. + Overrides ``requests.Session.request`` to renew the IAM cookie + and then retry the original request (if required). """ - resp = super(ClientSession, self).request( - method, url, timeout=self._timeout, **kwargs) + resp = super(IAMSession, self).request(method, url, **kwargs) + + if not self._auto_renew or url in [self._session_url, self._token_url]: + return resp + + is_expired = any(( + resp.status_code == 403 and + resp.json().get('error') == 'credentials_expired', + resp.status_code == 401 + )) + + if is_expired: + self.login() + resp = super(IAMSession, self).request(method, url, **kwargs) + return resp + def _get_access_token(self): + """ + Get IAM access token using API key. + """ + err = 'Failed to contact IAM token service' + try: + resp = super(IAMSession, self).request( + 'POST', + self._token_url, + auth=('bx', 'bx'), # required for user API keys + headers={'Accepts': 'application/json'}, + data={ + 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', + 'response_type': 'cloud_iam', + 'apikey': self._api_key + } + ) + err = resp.json().get('errorMessage', err) + resp.raise_for_status() + + return resp.json()['access_token'] + + except KeyError: + raise CloudantException('Invalid response from IAM token service') + + except RequestException: + raise CloudantException(err) + + class CloudFoundryService(object): """ Manages Cloud Foundry service configuration. """ diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 7c69c461..3bb95c51 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -31,8 +31,9 @@ append_response_error_content, InfiniteSession, ClientSession, - CloudFoundryService) - + CloudFoundryService, + CookieSession, + IAMSession) class CouchDB(dict): """ @@ -83,6 +84,7 @@ def __init__(self, user, auth_token, admin_party=False, **kwargs): self._timeout = kwargs.get('timeout', None) self.r_session = None self._auto_renew = kwargs.get('auto_renew', False) + self._use_iam = kwargs.get('use_iam', False) connect_to_couch = kwargs.get('connect', False) if connect_to_couch and self._DATABASE_CLASS == CouchDatabase: self.connect() @@ -95,26 +97,32 @@ def connect(self): if self.r_session: return - if self._auto_renew and not self.admin_party: - self.r_session = InfiniteSession( - self._user, + if self.admin_party: + self.r_session = ClientSession(timeout=self._timeout) + elif self._use_iam: + self.r_session = IAMSession( self._auth_token, self.server_url, + auto_renew=self._auto_renew, timeout=self._timeout ) else: - self.r_session = ClientSession( + self.r_session = CookieSession( self._user, self._auth_token, self.server_url, + auto_renew=self._auto_renew, timeout=self._timeout ) + # If a Transport Adapter was supplied add it to the session if self.adapter is not None: self.r_session.mount(self.server_url, self.adapter) if self._client_user_header is not None: self.r_session.headers.update(self._client_user_header) - self.session_login(self._user, self._auth_token) + + self.session_login() + self._client_session = self.session() # Utilize an event hook to append to the response message # using :func:`~cloudant.common_util.append_response_error_content` @@ -137,11 +145,16 @@ def session(self): """ if self.admin_party: return None +<<<<<<< HEAD sess_url = '/'.join((self.server_url, '_session')) resp = self.r_session.get(sess_url) resp.raise_for_status() sess_data = resp.json() return sess_data +======= + + return self.r_session.info() +>>>>>>> Add IAM authentication support def session_cookie(self): """ @@ -153,16 +166,14 @@ def session_cookie(self): return None return self.r_session.cookies.get('AuthSession') - def session_login(self, user, passwd): + def session_login(self): """ Performs a session login by posting the auth information to the _session endpoint. - - :param str user: Username used to connect. - :param str passwd: Passcode used to connect. """ if self.admin_party: return +<<<<<<< HEAD sess_url = '/'.join((self.server_url, '_session')) resp = self.r_session.post( sess_url, @@ -173,6 +184,10 @@ def session_login(self, user, passwd): headers={'Content-Type': 'application/x-www-form-urlencoded'} ) resp.raise_for_status() +======= + + self.r_session.login() +>>>>>>> Add IAM authentication support def session_logout(self): """ @@ -181,9 +196,14 @@ def session_logout(self): """ if self.admin_party: return +<<<<<<< HEAD sess_url = '/'.join((self.server_url, '_session')) resp = self.r_session.delete(sess_url) resp.raise_for_status() +======= + + self.r_session.logout() +>>>>>>> Add IAM authentication support def basic_auth_str(self): """ From cdba134c853b416bd1d9afb9fd2edea7a8a31b16 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 27 Jun 2017 10:11:20 +0100 Subject: [PATCH 025/185] Rename InfiniteSession -> CookieSession in unit tests --- tests/unit/auth_renewal_tests.py | 12 ++++++------ tests/unit/client_tests.py | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/unit/auth_renewal_tests.py b/tests/unit/auth_renewal_tests.py index 799783d3..4fdf0a6c 100644 --- a/tests/unit/auth_renewal_tests.py +++ b/tests/unit/auth_renewal_tests.py @@ -23,14 +23,14 @@ import requests import time -from cloudant._common_util import InfiniteSession +from cloudant._common_util import CookieSession from .unit_t_db_base import UnitTestDbBase @unittest.skipIf(os.environ.get('ADMIN_PARTY') == 'true', 'Skipping - Admin Party mode') class AuthRenewalTests(UnitTestDbBase): """ - Auto renewal tests primarily testing the InfiniteSession functionality + Auto renewal tests primarily testing the CookieSession functionality """ def setUp(self): @@ -62,10 +62,10 @@ def test_client_db_doc_stack_success(self): db_2_auth_session = db_2.r_session.cookies.get('AuthSession') doc_auth_session = doc.r_session.cookies.get('AuthSession') - self.assertIsInstance(self.client.r_session, InfiniteSession) - self.assertIsInstance(db.r_session, InfiniteSession) - self.assertIsInstance(db_2.r_session, InfiniteSession) - self.assertIsInstance(doc.r_session, InfiniteSession) + self.assertIsInstance(self.client.r_session, CookieSession) + self.assertIsInstance(db.r_session, CookieSession) + self.assertIsInstance(db_2.r_session, CookieSession) + self.assertIsInstance(doc.r_session, CookieSession) self.assertIsNotNone(auth_session) self.assertTrue( auth_session == diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index b2dbcbf0..15a12777 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -34,7 +34,7 @@ from cloudant.client import Cloudant, CouchDB from cloudant.error import CloudantArgumentError, CloudantClientException from cloudant.feed import Feed, InfiniteFeed -from cloudant._common_util import InfiniteSession +from cloudant._common_util import CookieSession from .unit_t_db_base import UnitTestDbBase from .. import bytes_, str_ @@ -163,7 +163,7 @@ def test_multiple_connect(self): def test_auto_renew_enabled(self): """ - Test that InfiniteSession is used when auto_renew is enabled. + Test that CookieSession is used when auto_renew is enabled. """ try: self.set_up_client(auto_renew=True) @@ -171,13 +171,13 @@ def test_auto_renew_enabled(self): if os.environ.get('ADMIN_PARTY') == 'true': self.assertIsInstance(self.client.r_session, requests.Session) else: - self.assertIsInstance(self.client.r_session, InfiniteSession) + self.assertIsInstance(self.client.r_session, CookieSession) finally: self.client.disconnect() def test_auto_renew_enabled_with_auto_connect(self): """ - Test that InfiniteSession is used when auto_renew is enabled along with + Test that CookieSession is used when auto_renew is enabled along with an auto_connect. """ try: @@ -185,7 +185,7 @@ def test_auto_renew_enabled_with_auto_connect(self): if os.environ.get('ADMIN_PARTY') == 'true': self.assertIsInstance(self.client.r_session, requests.Session) else: - self.assertIsInstance(self.client.r_session, InfiniteSession) + self.assertIsInstance(self.client.r_session, CookieSession) finally: self.client.disconnect() From c828b857c310fcf046ff9517d0544587c021231c Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 27 Jun 2017 14:20:29 +0100 Subject: [PATCH 026/185] Add IAM authentication tests --- src/cloudant/_2to3.py | 2 + tests/unit/iam_auth_tests.py | 334 +++++++++++++++++++++++++++++++++++ 2 files changed, 336 insertions(+) create mode 100644 tests/unit/iam_auth_tests.py diff --git a/src/cloudant/_2to3.py b/src/cloudant/_2to3.py index 26f995ac..6cbd79a3 100644 --- a/src/cloudant/_2to3.py +++ b/src/cloudant/_2to3.py @@ -41,6 +41,7 @@ from urlparse import urlparse as url_parse from urlparse import urljoin as url_join from ConfigParser import RawConfigParser + from cookielib import Cookie def iteritems_(adict): """ @@ -65,6 +66,7 @@ def next_(itr): from urllib.parse import quote as url_quote # pylint: disable=wrong-import-position,no-name-in-module,import-error,ungrouped-imports from urllib.parse import quote_plus as url_quote_plus # pylint: disable=wrong-import-position,no-name-in-module,import-error,ungrouped-imports from configparser import RawConfigParser # pylint: disable=wrong-import-position,no-name-in-module,import-error + from http.cookiejar import Cookie # pylint: disable=wrong-import-position,no-name-in-module,import-error def iteritems_(adict): """ diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py new file mode 100644 index 00000000..e2d79e48 --- /dev/null +++ b/tests/unit/iam_auth_tests.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python +# Copyright (c) 2017 IBM. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Unit tests for IAM authentication. """ +import time +import unittest +import json +import mock + +from cloudant._2to3 import Cookie +from cloudant._common_util import IAMSession +from cloudant.client import Cloudant + +MOCK_API_KEY = 'CqbrIYzdO3btWV-5t4teJLY_etfT_dkccq-vO-5vCXSo' + +MOCK_ACCESS_TOKEN = ('eyJraWQiOiIyMDE3MDQwMi0wMDowMDowMCIsImFsZyI6IlJTMjU2In0.e' + 'yJpYW1faWQiOiJJQk1pZC0yNzAwMDdHRjBEIiwiaWQiOiJJQk1pZC0yNz' + 'AwMDdHRjBEIiwicmVhbG1pZCI6IklCTWlkIiwiaWRlbnRpZmllciI6IjI' + '3MDAwN0dGMEQiLCJnaXZlbl9uYW1lIjoiVG9tIiwiZmFtaWx5X25hbWUi' + 'OiJCbGVuY2giLCJuYW1lIjoiVG9tIEJsZW5jaCIsImVtYWlsIjoidGJsZ' + 'W5jaEB1ay5pYm0uY29tIiwic3ViIjoidGJsZW5jaEB1ay5pYm0uY29tIi' + 'wiYWNjb3VudCI6eyJic3MiOiI1ZTM1ZTZhMjlmYjJlZWNhNDAwYWU0YzN' + 'lMWZhY2Y2MSJ9LCJpYXQiOjE1MDA0NjcxMDIsImV4cCI6MTUwMDQ3MDcw' + 'MiwiaXNzIjoiaHR0cHM6Ly9pYW0ubmcuYmx1ZW1peC5uZXQvb2lkYy90b' + '2tlbiIsImdyYW50X3R5cGUiOiJ1cm46aWJtOnBhcmFtczpvYXV0aDpncm' + 'FudC10eXBlOmFwaWtleSIsInNjb3BlIjoib3BlbmlkIiwiY2xpZW50X2l' + 'kIjoiZGVmYXVsdCJ9.XAPdb5K4n2nYih-JWTWBGoKkxTXM31c1BB1g-Ci' + 'auc2LxuoNXVTyz_mNqf1zQL07FUde1Cb_dwrbotjickNcxVPost6byQzt' + 'fc0mRF1x2S6VR8tn7SGiRmXBjLofkTh1JQq-jutp2MS315XbTG6K6m16u' + 'YzL9qfMnRvQHxsZWErzfPiJx-Trg_j7OX-qNFjdNUGnRpU7FmULy0r7Rx' + 'Ld8mhG-M1yxVzRBAZzvM63s0XXfMnk1oLi-BuUUTqVOdrM0KyYMWfD0Q7' + '2PTo4Exa17V-R_73Nq8VPCwpOvZcwKRA2sPTVgTMzU34max8b5kpTzVGJ' + '6SXSItTVOUdAygZBng') + +MOCK_OIDC_TOKEN_RESPONSE = { + 'access_token': MOCK_ACCESS_TOKEN, + 'refresh_token': ('MO61FKNvVRWkSa4vmBZqYv_Jt1kkGMUc-XzTcNnR-GnIhVKXHUWxJVV3' + 'RddE8Kqh3X_TZRmyK8UySIWKxoJ2t6obUSUalPm90SBpTdoXtaljpNyo' + 'rmqCCYPROnk6JBym72ikSJqKHHEZVQkT0B5ggZCwPMnKagFj0ufs-VIh' + 'CF97xhDxDKcIPMWG02xxPuESaSTJJug7e_dUDoak_ZXm9xxBmOTRKwOx' + 'n5sTKthNyvVpEYPE7jIHeiRdVDOWhN5LomgCn3TqFCLpMErnqwgNYbyC' + 'Bd9rNm-alYKDb6Jle4njuIBpXxQPb4euDwLd1osApaSME3nEarFWqRBz' + 'hjoqCe1Kv564s_rY7qzD1nHGvKOdpSa0ZkMcfJ0LbXSQPs7gBTSVrBFZ' + 'qwlg-2F-U3Cto62-9qRR_cEu_K9ZyVwL4jWgOlngKmxV6Ku4L5mHp4Kg' + 'EJSnY_78_V2nm64E--i2ZA1FhiKwIVHDOivVNhggE9oabxg54vd63glp' + '4GfpNnmZsMOUYG9blJJpH4fDX4Ifjbw-iNBD7S2LRpP8b8vG9pb4WioG' + 'zN43lE5CysveKYWrQEZpThznxXlw1snDu_A48JiL3Lrvo1LobLhF3zFV' + '-kQ='), + 'token_type': 'Bearer', + 'expires_in': 3600, # 60mins + 'expiration': 1500470702 # Wed Jul 19 14:25:02 2017 +} + + +class IAMAuthTests(unittest.TestCase): + """ Unit tests for IAM authentication. """ + + @staticmethod + def _mock_cookie(expires_secs=300): + return Cookie( + version=0, + name='IAMSession', + value=('SQJCaUQxMqEfMEAyRKU6UopLVXceS0c9RPuQgDArCEYoN3l_TEY4gdf-DJ7' + '4sHfjcNEUVjfdOvA'), + port=None, + port_specified=False, + domain='localhost', + domain_specified=False, + domain_initial_dot=False, + path="/", + path_specified=True, + secure=True, + expires=int(time.time() + expires_secs), + discard=False, + comment=None, + comment_url=None, + rest={'HttpOnly': None}, + rfc2109=True) + + @mock.patch('cloudant._common_util.ClientSession.request') + def test_iam_get_access_token(self, m_req): + m_response = mock.MagicMock() + m_response.json.return_value = MOCK_OIDC_TOKEN_RESPONSE + m_req.return_value = m_response + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') + access_token = iam._get_access_token() + + m_req.assert_called_once_with( + 'POST', + iam._token_url, + auth=('bx', 'bx'), + headers={'Accepts': 'application/json'}, + data={ + 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', + 'response_type': 'cloud_iam', + 'apikey': MOCK_API_KEY + } + ) + + self.assertEqual(access_token, MOCK_ACCESS_TOKEN) + self.assertTrue(m_response.raise_for_status.called) + self.assertTrue(m_response.json.called) + + @mock.patch('cloudant._common_util.ClientSession.request') + @mock.patch('cloudant._common_util.IAMSession._get_access_token') + def test_iam_login(self, m_token, m_req): + m_token.return_value = MOCK_ACCESS_TOKEN + m_response = mock.MagicMock() + m_req.return_value = m_response + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') + iam.login() + + m_req.assert_called_once_with( + 'POST', + iam._session_url, + headers={'Content-Type': 'application/json'}, + data=json.dumps({'access_token': MOCK_ACCESS_TOKEN}) + ) + + self.assertEqual(m_token.call_count, 1) + self.assertTrue(m_response.raise_for_status.called) + + def test_iam_logout(self): + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') + # add a valid cookie to jar + iam.cookies.set_cookie(self._mock_cookie()) + self.assertEqual(len(iam.cookies.keys()), 1) + iam.logout() + self.assertEqual(len(iam.cookies.keys()), 0) + + @mock.patch('cloudant._common_util.ClientSession.get') + def test_iam_get_session_info(self, m_get): + m_info = {'ok': True, 'info': {'authentication_db': '_users'}} + + m_response = mock.MagicMock() + m_response.json.return_value = m_info + m_get.return_value = m_response + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') + info = iam.info() + + m_get.assert_called_once_with(iam._session_url) + + self.assertEqual(info, m_info) + self.assertTrue(m_response.raise_for_status.called) + + @mock.patch('cloudant._common_util.IAMSession.login') + @mock.patch('cloudant._common_util.ClientSession.request') + def test_iam_first_request(self, m_req, m_login): + # mock 200 + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + m_response_ok.json.return_value = {'ok': True} + + m_req.return_value = m_response_ok + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', auto_renew=True) + iam.login() + + self.assertEqual(m_login.call_count, 1) + self.assertEqual(m_req.call_count, 0) + + # add a valid cookie to jar + iam.cookies.set_cookie(self._mock_cookie()) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + + self.assertEqual(m_login.call_count, 1) + self.assertEqual(m_req.call_count, 1) + self.assertEqual(resp.status_code, 200) + + @mock.patch('cloudant._common_util.IAMSession.login') + @mock.patch('cloudant._common_util.ClientSession.request') + def test_iam_renew_cookie_on_expiry(self, m_req, m_login): + # mock 200 + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + m_response_ok.json.return_value = {'ok': True} + + m_req.return_value = m_response_ok + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', auto_renew=True) + iam.login() + + # add an expired cookie to jar + iam.cookies.set_cookie(self._mock_cookie(expires_secs=-300)) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + + self.assertEqual(m_login.call_count, 2) + self.assertEqual(m_req.call_count, 1) + self.assertEqual(resp.status_code, 200) + + @mock.patch('cloudant._common_util.IAMSession.login') + @mock.patch('cloudant._common_util.ClientSession.request') + def test_iam_renew_cookie_on_401_success(self, m_req, m_login): + # mock 200 + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + m_response_ok.json.return_value = {'ok': True} + # mock 401 + m_response_bad = mock.MagicMock() + type(m_response_bad).status_code = mock.PropertyMock(return_value=401) + + m_req.side_effect = [m_response_bad, m_response_ok, m_response_ok] + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', auto_renew=True) + iam.login() + self.assertEqual(m_login.call_count, 1) + + # add a valid cookie to jar + iam.cookies.set_cookie(self._mock_cookie()) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + self.assertEqual(resp.status_code, 200) + self.assertEqual(m_login.call_count, 2) + self.assertEqual(m_req.call_count, 2) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + self.assertEqual(resp.status_code, 200) + self.assertEqual(m_login.call_count, 2) + self.assertEqual(m_req.call_count, 3) + + + @mock.patch('cloudant._common_util.IAMSession.login') + @mock.patch('cloudant._common_util.ClientSession.request') + def test_iam_renew_cookie_on_403(self, m_req, m_login): + # mock 200 + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + m_response_ok.json.return_value = {'ok': True} + # mock 403 + m_response_bad = mock.MagicMock() + type(m_response_bad).status_code = mock.PropertyMock(return_value=403) + m_response_bad.json.return_value = {'error': 'credentials_expired'} + + m_req.side_effect = [m_response_bad, m_response_ok] + + iam = IAMSession('foo', 'http://127.0.0.1:5984', auto_renew=True) + iam.login() + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + + self.assertEqual(m_login.call_count, 2) + self.assertTrue(resp.json()['ok']) + + @mock.patch('cloudant._common_util.IAMSession.login') + @mock.patch('cloudant._common_util.ClientSession.request') + def test_iam_renew_cookie_on_401_failure(self, m_req, m_login): + # mock 401 + m_response_bad = mock.MagicMock() + type(m_response_bad).status_code = mock.PropertyMock(return_value=401) + + m_req.return_value = m_response_bad + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', auto_renew=True) + iam.login() + self.assertEqual(m_login.call_count, 1) + + # add a valid cookie to jar + iam.cookies.set_cookie(self._mock_cookie()) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + self.assertEqual(resp.status_code, 401) + self.assertEqual(m_login.call_count, 2) + self.assertEqual(m_req.call_count, 2) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + self.assertEqual(resp.status_code, 401) + self.assertEqual(m_login.call_count, 3) + self.assertEqual(m_req.call_count, 4) + + @mock.patch('cloudant._common_util.IAMSession.login') + @mock.patch('cloudant._common_util.ClientSession.request') + def test_iam_renew_cookie_disabled(self, m_req, m_login): + # mock 401 + m_response_bad = mock.MagicMock() + type(m_response_bad).status_code = mock.PropertyMock(return_value=401) + + m_req.return_value = m_response_bad + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', auto_renew=False) + iam.login() + self.assertEqual(m_login.call_count, 1) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + self.assertEqual(resp.status_code, 401) + self.assertEqual(m_login.call_count, 1) # no attempt to renew + self.assertEqual(m_req.call_count, 1) + + resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') + self.assertEqual(resp.status_code, 401) + self.assertEqual(m_login.call_count, 1) # no attempt to renew + self.assertEqual(m_req.call_count, 2) + + @mock.patch('cloudant._common_util.IAMSession.login') + @mock.patch('cloudant._common_util.ClientSession.request') + def test_iam_client_create(self, m_req, m_login): + # mock 200 + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + m_response_ok.json.return_value = ['animaldb'] + + m_req.return_value = m_response_ok + + # create IAM client + client = Cloudant.iam('foo', MOCK_API_KEY) + client.connect() + + # add a valid cookie to jar + client.r_session.cookies.set_cookie(self._mock_cookie()) + + dbs = client.all_dbs() + + self.assertEqual(m_login.call_count, 1) + self.assertEqual(m_req.call_count, 1) + self.assertEqual(dbs, ['animaldb']) + + +if __name__ == '__main__': + unittest.main() From 0f6176ba2167278320c9841088b31a7d9a74d242 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 28 Jun 2017 12:59:33 +0100 Subject: [PATCH 027/185] Add IAM notes to getting_started.rst --- docs/getting_started.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 17fe3e60..fc5d10be 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -88,6 +88,26 @@ following statements hold true: connect=True, auto_renew=True) + +************************************ +Identity and Access Management (IAM) +************************************ + +IBM Cloud Identity & Access Management enables you to securely authenticate +users and control access to all cloud resources consistently in the IBM Bluemix +Cloud Platform. + +See `IBM Cloud Identity and Access Management `_ +for more information. + +You can easily connect to your Cloudant account using an IAM API key: + +.. code-block:: python + + # Authenticate using an IAM API key + client = Cloudant.iam(ACCOUNT_NAME, API_KEY, connect=True) + + **************** Resource sharing **************** From 275057a296323cdb24e8e702f34ba17f8f6efa1f Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 28 Jun 2017 12:59:13 +0100 Subject: [PATCH 028/185] Allow multiple calls to client .connect() --- src/cloudant/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 3bb95c51..02dc0837 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -95,7 +95,7 @@ def connect(self): authentication if necessary. """ if self.r_session: - return + self.session_logout() if self.admin_party: self.r_session = ClientSession(timeout=self._timeout) @@ -132,7 +132,9 @@ def disconnect(self): """ Ends a client authentication session, performs a logout and a clean up. """ - self.session_logout() + if self.r_session: + self.session_logout() + self.r_session = None self.clear() From 59ca68f3152e5ed75789861f9d29051e7158aff8 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 20 Jul 2017 15:29:30 +0100 Subject: [PATCH 029/185] Renew IAM token on 401 status code or cookie expiry --- src/cloudant/_common_util.py | 12 +++++------- tests/unit/iam_auth_tests.py | 23 ----------------------- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 0a2344ce..5ebec71f 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -425,18 +425,16 @@ def request(self, method, url, **kwargs): # pylint: disable=W0221 Overrides ``requests.Session.request`` to renew the IAM cookie and then retry the original request (if required). """ + self.cookies.clear_expired_cookies() + if self._auto_renew and 'IAMSession' not in self.cookies.keys(): + self.login() + resp = super(IAMSession, self).request(method, url, **kwargs) if not self._auto_renew or url in [self._session_url, self._token_url]: return resp - is_expired = any(( - resp.status_code == 403 and - resp.json().get('error') == 'credentials_expired', - resp.status_code == 401 - )) - - if is_expired: + if resp.status_code == 401: self.login() resp = super(IAMSession, self).request(method, url, **kwargs) diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py index e2d79e48..fd181dad 100644 --- a/tests/unit/iam_auth_tests.py +++ b/tests/unit/iam_auth_tests.py @@ -234,29 +234,6 @@ def test_iam_renew_cookie_on_401_success(self, m_req, m_login): self.assertEqual(m_login.call_count, 2) self.assertEqual(m_req.call_count, 3) - - @mock.patch('cloudant._common_util.IAMSession.login') - @mock.patch('cloudant._common_util.ClientSession.request') - def test_iam_renew_cookie_on_403(self, m_req, m_login): - # mock 200 - m_response_ok = mock.MagicMock() - type(m_response_ok).status_code = mock.PropertyMock(return_value=200) - m_response_ok.json.return_value = {'ok': True} - # mock 403 - m_response_bad = mock.MagicMock() - type(m_response_bad).status_code = mock.PropertyMock(return_value=403) - m_response_bad.json.return_value = {'error': 'credentials_expired'} - - m_req.side_effect = [m_response_bad, m_response_ok] - - iam = IAMSession('foo', 'http://127.0.0.1:5984', auto_renew=True) - iam.login() - - resp = iam.request('GET', 'http://127.0.0.1:5984/mydb1') - - self.assertEqual(m_login.call_count, 2) - self.assertTrue(resp.json()['ok']) - @mock.patch('cloudant._common_util.IAMSession.login') @mock.patch('cloudant._common_util.ClientSession.request') def test_iam_renew_cookie_on_401_failure(self, m_req, m_login): From 6b85dc31815a713bd5bd186b1d7fa327caff5ca0 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 20 Jul 2017 16:36:12 +0100 Subject: [PATCH 030/185] Add IAM class method to Cloudant class --- src/cloudant/__init__.py | 24 ++++++++++++++++++----- src/cloudant/client.py | 42 ++++++++++++++-------------------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index b2a06483..75c22413 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -63,15 +63,29 @@ def cloudant(user, passwd, **kwargs): cloudant_session.disconnect() @contextlib.contextmanager -def cloudant_iam(api_key, account_name, **kwargs): +def cloudant_iam(account_name, api_key, **kwargs): """ - Provides a context manager to create a Cloudant session and provide access - to databases, docs etc. + Provides a context manager to create a Cloudant session using IAM + authentication and provide access to databases, docs etc. - :param api_key: IAM authentication API key. :param account_name: Cloudant account name. + :param api_key: IAM authentication API key. + + For example: + + .. code-block:: python + + # cloudant context manager + from cloudant import cloudant_iam + + with cloudant_iam(ACCOUNT_NAME, API_KEY) as client: + # Context handles connect() and disconnect() for you. + # Perform library operations within this context. Such as: + print client.all_dbs() + # ... + """ - cloudant_session = Cloudant(account_name, api_key, use_iam=True, **kwargs) + cloudant_session = Cloudant.iam(account_name, api_key, **kwargs) cloudant_session.connect() yield cloudant_session diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 02dc0837..7e39a303 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -29,7 +29,6 @@ from ._common_util import ( USER_AGENT, append_response_error_content, - InfiniteSession, ClientSession, CloudFoundryService, CookieSession, @@ -68,6 +67,10 @@ class CouchDB(dict): `Requests library timeout argument `_. but will apply to every request made using this client. + :param bool use_iam: Keyword argument, if set to True performs + IAM authentication with server. Default is False. + Use :func:`~cloudant.client.CouchDB.iam` to construct an IAM + authenticated client. """ _DATABASE_CLASS = CouchDatabase @@ -147,16 +150,8 @@ def session(self): """ if self.admin_party: return None -<<<<<<< HEAD - sess_url = '/'.join((self.server_url, '_session')) - resp = self.r_session.get(sess_url) - resp.raise_for_status() - sess_data = resp.json() - return sess_data -======= return self.r_session.info() ->>>>>>> Add IAM authentication support def session_cookie(self): """ @@ -175,21 +170,8 @@ def session_login(self): """ if self.admin_party: return -<<<<<<< HEAD - sess_url = '/'.join((self.server_url, '_session')) - resp = self.r_session.post( - sess_url, - data={ - 'name': user, - 'password': passwd - }, - headers={'Content-Type': 'application/x-www-form-urlencoded'} - ) - resp.raise_for_status() -======= self.r_session.login() ->>>>>>> Add IAM authentication support def session_logout(self): """ @@ -198,14 +180,8 @@ def session_logout(self): """ if self.admin_party: return -<<<<<<< HEAD - sess_url = '/'.join((self.server_url, '_session')) - resp = self.r_session.delete(sess_url) - resp.raise_for_status() -======= self.r_session.logout() ->>>>>>> Add IAM authentication support def basic_auth_str(self): """ @@ -805,3 +781,13 @@ def bluemix(cls, vcap_services, instance_name=None, **kwargs): service.password, url=service.url, **kwargs) + + @classmethod + def iam(cls, account_name, api_key, **kwargs): + """ + Create a Cloudant client that uses IAM authentication. + + :param account_name: Cloudant account name. + :param api_key: IAM authentication API key. + """ + return cls(None, api_key, account=account_name, use_iam=True, **kwargs) From c115f9c0c7a85570ddc8132d55ba31965d862848 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 20 Jul 2017 16:50:55 +0100 Subject: [PATCH 031/185] Always use auto_renew=True by default for IAM sessions --- src/cloudant/client.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 7e39a303..f05d3af1 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -790,4 +790,9 @@ def iam(cls, account_name, api_key, **kwargs): :param account_name: Cloudant account name. :param api_key: IAM authentication API key. """ - return cls(None, api_key, account=account_name, use_iam=True, **kwargs) + return cls(None, + api_key, + account=account_name, + auto_renew=kwargs.get('auto_renew', True), + use_iam=True, + **kwargs) From 54d8fbef04689086cbc3e42a46dc8094b39bc181 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Fri, 21 Jul 2017 14:57:01 +0100 Subject: [PATCH 032/185] Remove session endpoint checks These checks are redundant. Session endpoint requests are always made using the base request method. --- src/cloudant/_common_util.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 5ebec71f..a7f2bddc 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -24,8 +24,7 @@ import json from requests import RequestException, Session -from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_, url_parse, \ - url_join +from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_, url_join from .error import CloudantArgumentError, CloudantException # Library Constants @@ -356,10 +355,7 @@ def request(self, method, url, **kwargs): # pylint: disable=W0221 """ resp = super(CookieSession, self).request(method, url, **kwargs) - path = url_parse(url).path.lower() - post_to_session = method.upper() == 'POST' and path == '/_session' - - if not self._auto_renew or post_to_session: + if not self._auto_renew: return resp is_expired = any(( @@ -431,7 +427,7 @@ def request(self, method, url, **kwargs): # pylint: disable=W0221 resp = super(IAMSession, self).request(method, url, **kwargs) - if not self._auto_renew or url in [self._session_url, self._token_url]: + if not self._auto_renew: return resp if resp.status_code == 401: From 5f4935a65937eb42a781c3b21fa4b2282c725845 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 26 Jul 2017 09:55:06 +0100 Subject: [PATCH 033/185] Remove unused CouchDB._client_session --- src/cloudant/client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/cloudant/client.py b/src/cloudant/client.py index f05d3af1..3e52f338 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -78,7 +78,6 @@ def __init__(self, user, auth_token, admin_party=False, **kwargs): super(CouchDB, self).__init__() self._user = user self._auth_token = auth_token - self._client_session = None self.server_url = kwargs.get('url') self._client_user_header = None self.admin_party = admin_party @@ -126,7 +125,6 @@ def connect(self): self.session_login() - self._client_session = self.session() # Utilize an event hook to append to the response message # using :func:`~cloudant.common_util.append_response_error_content` self.r_session.hooks['response'].append(append_response_error_content) From 550b00b6c82b404533320fd14930e443088cb5a2 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 26 Jul 2017 10:32:29 +0100 Subject: [PATCH 034/185] Update year in copyright headers --- src/cloudant/_2to3.py | 2 +- src/cloudant/__init__.py | 2 +- src/cloudant/_common_util.py | 2 +- src/cloudant/client.py | 2 +- tests/unit/auth_renewal_tests.py | 2 +- tests/unit/client_tests.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cloudant/_2to3.py b/src/cloudant/_2to3.py index 6cbd79a3..5c7d412b 100644 --- a/src/cloudant/_2to3.py +++ b/src/cloudant/_2to3.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (c) 2016, 2017 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 75c22413..04131db7 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (c) 2015, 2017 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index a7f2bddc..e22e458b 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2016, 2017 IBM Corp. All rights reserved. +# Copyright (c) 2015, 2017 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 3e52f338..f53d6c75 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2016, 2017 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2017 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/unit/auth_renewal_tests.py b/tests/unit/auth_renewal_tests.py index 4fdf0a6c..3d9b7cc6 100644 --- a/tests/unit/auth_renewal_tests.py +++ b/tests/unit/auth_renewal_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (c) 2016, 2017 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 15a12777..33171417 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2016, 2017 IBM Corp. All rights reserved. +# Copyright (c) 2015, 2017 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From fdb08de75002aa3acc72f31ae6ae01928fc5521a Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Fri, 28 Jul 2017 15:10:02 +0100 Subject: [PATCH 035/185] Add IAM_TOKEN_URL env var note to docs/getting_started.rst --- docs/getting_started.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index fc5d10be..f4ce1303 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -100,6 +100,10 @@ Cloud Platform. See `IBM Cloud Identity and Access Management `_ for more information. +The production IAM token service at *https://iam.bluemix.net/oidc/token* is used +by default. You can set an ``IAM_TOKEN_URL`` environment variable to override +this. + You can easily connect to your Cloudant account using an IAM API key: .. code-block:: python From fcbe6301f87e0687055076445498898fcbb89e85 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 8 Aug 2017 10:20:01 +0100 Subject: [PATCH 036/185] Add code comment about discarding expired IAMSession cookies --- src/cloudant/_common_util.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index e22e458b..dce99494 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -421,7 +421,14 @@ def request(self, method, url, **kwargs): # pylint: disable=W0221 Overrides ``requests.Session.request`` to renew the IAM cookie and then retry the original request (if required). """ + # The CookieJar API prevents callers from getting an individual Cookie + # object by name. + # We are forced to use the only exposed method of discarding expired + # cookies from the CookieJar. Internally this involves iterating over + # the entire CookieJar and calling `.is_expired()` on each Cookie + # object. self.cookies.clear_expired_cookies() + if self._auto_renew and 'IAMSession' not in self.cookies.keys(): self.login() From 1a905fb6057ba34927104f418c321a415f464746 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 29 Aug 2017 17:14:46 +0100 Subject: [PATCH 037/185] Allow `CouchDB.session_login` to take credentials as arguments --- src/cloudant/_common_util.py | 23 ++++++++++++++++++++ src/cloudant/client.py | 3 ++- tests/unit/client_tests.py | 27 +++++++++++++++++++++++- tests/unit/iam_auth_tests.py | 41 ++++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 2 deletions(-) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index dce99494..fe2e0680 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -370,6 +370,19 @@ def request(self, method, url, **kwargs): # pylint: disable=W0221 return resp + def set_credentials(self, username, password): + """ + Set a new username and password. + + :param str username: New username. + :param str password: New password. + """ + if username is not None: + self._username = username + + if password is not None: + self._password = password + class IAMSession(ClientSession): """ @@ -443,6 +456,16 @@ def request(self, method, url, **kwargs): # pylint: disable=W0221 return resp + def set_credentials(self, username, api_key): + """ + Set a new IAM API key. + + :param str username: Username parameter is unused. + :param str api_key: New IAM API key. + """ + if api_key is not None: + self._api_key = api_key + def _get_access_token(self): """ Get IAM access token using API key. diff --git a/src/cloudant/client.py b/src/cloudant/client.py index f53d6c75..3a1360cc 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -161,7 +161,7 @@ def session_cookie(self): return None return self.r_session.cookies.get('AuthSession') - def session_login(self): + def session_login(self, user=None, passwd=None): """ Performs a session login by posting the auth information to the _session endpoint. @@ -169,6 +169,7 @@ def session_login(self): if self.admin_party: return + self.r_session.set_credentials(user, passwd) self.r_session.login() def session_logout(self): diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 33171417..796e5fce 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -28,7 +28,8 @@ import os import datetime -from requests import ConnectTimeout +from requests import ConnectTimeout, HTTPError +from time import sleep from cloudant import cloudant, cloudant_bluemix, couchdb, couchdb_admin_party from cloudant.client import Cloudant, CouchDB @@ -492,6 +493,30 @@ class CloudantClientTests(UnitTestDbBase): Cloudant specific client unit tests """ + def test_cloudant_session_login(self): + """ + Test that the Cloudant client session successfully authenticates. + """ + self.client.connect() + old_cookie = self.client.session_cookie() + + sleep(5) # ensure we get a different cookie back + + self.client.session_login() + self.assertNotEqual(self.client.session_cookie(), old_cookie) + + def test_cloudant_session_login_with_new_credentials(self): + """ + Test that the Cloudant client session fails to authenticate when + passed incorrect credentials. + """ + self.client.connect() + + with self.assertRaises(HTTPError) as cm: + self.client.session_login('invalid-user-123', 'pa$$w0rd01') + + self.assertTrue(str(cm.exception).find('Name or password is incorrect')) + def test_cloudant_context_helper(self): """ Test that the cloudant context helper works as expected. diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py index fd181dad..a65d3f04 100644 --- a/tests/unit/iam_auth_tests.py +++ b/tests/unit/iam_auth_tests.py @@ -88,6 +88,15 @@ def _mock_cookie(expires_secs=300): rest={'HttpOnly': None}, rfc2109=True) + def test_iam_set_credentials(self): + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') + self.assertEquals(iam._api_key, MOCK_API_KEY) + + new_api_key = 'some_new_api_key' + iam.set_credentials(None, new_api_key) + + self.assertEquals(iam._api_key, new_api_key) + @mock.patch('cloudant._common_util.ClientSession.request') def test_iam_get_access_token(self, m_req): m_response = mock.MagicMock() @@ -306,6 +315,38 @@ def test_iam_client_create(self, m_req, m_login): self.assertEqual(m_req.call_count, 1) self.assertEqual(dbs, ['animaldb']) + @mock.patch('cloudant._common_util.IAMSession.login') + @mock.patch('cloudant._common_util.IAMSession.set_credentials') + def test_iam_client_session_login(self, m_set, m_login): + # create IAM client + client = Cloudant.iam('foo', MOCK_API_KEY) + client.connect() + + # add a valid cookie to jar + client.r_session.cookies.set_cookie(self._mock_cookie()) + + client.session_login() + + m_set.assert_called_with(None, None) + self.assertEqual(m_login.call_count, 2) + self.assertEqual(m_set.call_count, 2) + + @mock.patch('cloudant._common_util.IAMSession.login') + @mock.patch('cloudant._common_util.IAMSession.set_credentials') + def test_iam_client_session_login_with_new_credentials(self, m_set, m_login): + # create IAM client + client = Cloudant.iam('foo', MOCK_API_KEY) + client.connect() + + # add a valid cookie to jar + client.r_session.cookies.set_cookie(self._mock_cookie()) + + client.session_login('bar', 'baz') # new creds + + m_set.assert_called_with('bar', 'baz') + self.assertEqual(m_login.call_count, 2) + self.assertEqual(m_set.call_count, 2) + if __name__ == '__main__': unittest.main() From 7846b2e63b15da54409010d137f26d43a644e68a Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 15 Sep 2017 16:05:01 -0400 Subject: [PATCH 038/185] Fixed DCO link --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c7520ee7..b4b41f99 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -8,7 +8,7 @@ Developer Certificate of Origin =============================== In order for us to accept pull-requests, the contributor must sign-off a -[Developer Certificate of Origin (DCO)](DCO1.1.txt). This clarifies the +`Developer Certificate of Origin (DCO) `_. This clarifies the intellectual property license granted with any contribution. It is for your protection as a Contributor as well as the protection of IBM and its customers; it does not change your rights to use your own Contributions for any other From 125bede8c23f04da00a7e42acba1b0a0631d5788 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Thu, 12 Oct 2017 11:58:02 -0400 Subject: [PATCH 039/185] Updated deprecated docs.cloudant.com links to the latest Bluemix Cloudant doc links - Added learning center link --- CHANGES.rst | 1 + README.rst | 4 ++-- src/cloudant/database.py | 9 +++++---- src/cloudant/design_document.py | 10 +++++----- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index cfae6908..eab05572 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,7 @@ Unreleased ========== - [NEW] Added ``Result.all()`` convenience method. - [IMPROVED] Updated ``posixpath.join`` references to use ``'/'.join`` when concatenating URL parts. +- [IMPROVED] Updated documentation by replacing deprecated Cloudant links with the latest Bluemix links. 2.6.0 (2017-08-10) ================== diff --git a/README.rst b/README.rst index 52736488..5c801d05 100644 --- a/README.rst +++ b/README.rst @@ -55,8 +55,8 @@ Related Documentation ===================== * `Cloudant Python client library docs (readthedocs.io) `_ -* `Cloudant documentation `_ -* `Cloudant for developers `_ +* `Cloudant documentation `_ +* `Cloudant Learning Center `_ =========== Development diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 47c2a1fd..1992d61e 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -811,7 +811,7 @@ def get_list_function_result(self, ddoc_id, list_name, view_name, **kwargs): # Process data (in text format). For more detail on list functions, refer to the - `Cloudant list documentation `_. :param str ddoc_id: Design document id used to get result. @@ -849,7 +849,7 @@ def get_show_function_result(self, ddoc_id, show_name, doc_id): # Process data (in text format). For more detail on show functions, refer to the - `Cloudant show documentation `_. :param str ddoc_id: Design document id used to get the result. @@ -897,7 +897,7 @@ def update_handler_result(self, ddoc_id, handler_name, doc_id=None, data=None, * data={'month': 'July'}) For more details, see the `update handlers documentation - `_. + `_. :param str ddoc_id: Design document id used to get result. :param str handler_name: Name used in part to identify the @@ -970,7 +970,8 @@ def share_database(self, username, roles=None): :param str username: Cloudant user to share the database with. :param list roles: A list of - `roles `_ + `roles + `_ to grant to the named user. :returns: Share database status in JSON format diff --git a/src/cloudant/design_document.py b/src/cloudant/design_document.py index b3f5a187..3e460741 100644 --- a/src/cloudant/design_document.py +++ b/src/cloudant/design_document.py @@ -68,7 +68,7 @@ def validate_doc_update(self): ddoc.save() For more details, see the `Update Validators documentation - `_. + `_. :returns: Dictionary containing update validator functions """ @@ -104,7 +104,7 @@ def filters(self): :func:`~cloudant.database.CouchDatabase.changes` For more details, see the `Filter functions documentation - `_. + `_. :returns: Dictionary containing filter function names and functions as key/value @@ -179,10 +179,10 @@ def st_indexes(self): Once the Cloudant Geo index is saved to the remote database, you can query the index with a GET request. To issue a request against the ``_geo`` endpoint, see the steps outlined in the `endpoint access - documentation `_. + `_ section. For more details, see the `Cloudant Geospatial - documentation `_. + documentation `_. :return: Dictionary containing Cloudant Geo names and index objects as key/value @@ -241,7 +241,7 @@ def rewrites(self): rewritten as ``/$DATABASE/_design/doc/_rewrite/new?k=v``. For more details on URL rewriting, see the `rewrite rules - documentation `_. :returns: List of dictionaries containing rewrite rules as key/value From e62faabc09466847eac78b837b9210875f96fa2f Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Fri, 20 Oct 2017 11:40:08 +0100 Subject: [PATCH 040/185] Fix index tests following Mango maximum key value patch Following a patch to Mango the returned index JSON now includes additional keys/values. To ensure the tests continue to pass on both old & new versions we opt to inspect the returned JSON rather than assert on the object in its entirety. --- tests/unit/database_tests.py | 196 +++++++++++++++++---------------- tests/unit/index_tests.py | 207 ++++++++++++++++------------------- 2 files changed, 196 insertions(+), 207 deletions(-) diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index 1ea6bec6..94e6fea8 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -1198,22 +1198,22 @@ def test_create_json_index(self): """ index = self.db.create_query_index(fields=['name', 'age']) self.assertIsInstance(index, Index) + ddoc = self.db[index.design_document_id] + + self.assertEquals(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': index.design_document_id, - '_rev': ddoc['_rev'], - 'indexes': {}, - 'lists': {}, - 'shows': {}, - 'language': 'query', - 'views': {index.name: {'map': {'fields': {'name': 'asc', - 'age': 'asc'}}, - 'reduce': '_count', - 'options': {'def': {'fields': ['name', - 'age']}, - }}}} - ) + + self.assertEquals(ddoc['indexes'], {}) + self.assertEquals(ddoc['language'], 'query') + self.assertEquals(ddoc['lists'], {}) + self.assertEquals(ddoc['shows'], {}) + + index = ddoc['views'][index.name] + self.assertEquals(index['map']['fields']['age'], 'asc') + self.assertEquals(index['map']['fields']['name'], 'asc') + self.assertEquals(index['options']['def']['fields'], ['name', 'age']) + self.assertEquals(index['reduce'], '_count') def test_create_text_index(self): """ @@ -1225,25 +1225,26 @@ def test_create_text_index(self): {'name': 'age', 'type':'number'}] ) self.assertIsInstance(index, TextIndex) + ddoc = self.db[index.design_document_id] + + self.assertEquals(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': index.design_document_id, - '_rev': ddoc['_rev'], - 'language': 'query', - 'views': {}, - 'lists': {}, - 'shows': {}, - 'indexes': {index.name: {'index': {'index_array_lengths': True, - 'fields': [{'name': 'name', 'type': 'string'}, - {'name': 'age', 'type': 'number'}], - 'default_field': {}, - 'default_analyzer': 'keyword', - 'selector': {}}, - 'analyzer': {'name': 'perfield', - 'default': 'keyword', - 'fields': {'$default': 'standard'}}}}} - ) + + self.assertEquals(ddoc['language'], 'query') + self.assertEquals(ddoc['lists'], {}) + self.assertEquals(ddoc['shows'], {}) + self.assertEquals(ddoc['views'], {}) + + text_index = ddoc['indexes'][index.name] + self.assertEquals(text_index['analyzer']['default'], 'keyword') + self.assertEquals(text_index['analyzer']['fields']['$default'], 'standard') + self.assertEquals(text_index['analyzer']['name'], 'perfield') + self.assertEquals(text_index['index']['default_analyzer'], 'keyword') + self.assertEquals(text_index['index']['default_field'], {}) + self.assertEquals(text_index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) + self.assertEquals(text_index['index']['selector'], {}) + self.assertTrue(text_index['index']['index_array_lengths']) def test_create_all_fields_text_index(self): """ @@ -1251,36 +1252,38 @@ def test_create_all_fields_text_index(self): """ index = self.db.create_query_index(index_type='text') self.assertIsInstance(index, TextIndex) + ddoc = self.db[index.design_document_id] + + self.assertEquals(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': index.design_document_id, - '_rev': ddoc['_rev'], - 'language': 'query', - 'views': {}, - 'lists': {}, - 'shows': {}, - 'indexes': {index.name: {'index': {'index_array_lengths': True, - 'fields': 'all_fields', - 'default_field': {}, - 'default_analyzer': 'keyword', - 'selector': {}}, - 'analyzer': {'name': 'perfield', - 'default': 'keyword', - 'fields': {'$default': 'standard'}}}}} - ) + + self.assertEquals(ddoc['language'], 'query') + self.assertEquals(ddoc['lists'], {}) + self.assertEquals(ddoc['shows'], {}) + self.assertEquals(ddoc['views'], {}) + + index = ddoc['indexes'][index.name] + self.assertEquals(index['analyzer']['default'], 'keyword') + self.assertEquals(index['analyzer']['fields'], {'$default': 'standard'}) + self.assertEquals(index['analyzer']['name'], 'perfield') + self.assertEquals(index['index']['default_analyzer'], 'keyword') + self.assertEquals(index['index']['default_field'], {}) + self.assertEquals(index['index']['fields'], 'all_fields') + self.assertEquals(index['index']['selector'], {}) + self.assertTrue(index['index']['index_array_lengths']) def test_create_multiple_indexes_one_ddoc(self): """ Tests that multiple indexes of different types can be stored in one design document. """ - json_index = self.db.create_query_index( + index = self.db.create_query_index( 'ddoc001', 'json-index-001', fields=['name', 'age'] ) - self.assertIsInstance(json_index, Index) + self.assertIsInstance(index, Index) search_index = self.db.create_query_index( 'ddoc001', 'text-index-001', @@ -1289,32 +1292,31 @@ def test_create_multiple_indexes_one_ddoc(self): {'name': 'age', 'type':'number'}] ) self.assertIsInstance(search_index, TextIndex) + ddoc = self.db['_design/ddoc001'] + + self.assertEquals(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('2-')) - self.assertEqual(ddoc, - {'_id': '_design/ddoc001', - '_rev': ddoc['_rev'], - 'language': 'query', - 'lists': {}, - 'shows': {}, - 'views': {'json-index-001': { - 'map': {'fields': {'name': 'asc', - 'age': 'asc'}}, - 'reduce': '_count', - 'options': {'def': {'fields': ['name', - 'age']}, - }}}, - 'indexes': {'text-index-001': { - 'index': {'index_array_lengths': True, - 'fields': [{'name': 'name', 'type': 'string'}, - {'name': 'age', 'type': 'number'}], - 'default_field': {}, - 'default_analyzer': 'keyword', - 'selector': {}}, - 'analyzer': {'name': 'perfield', - 'default': 'keyword', - 'fields': {'$default': 'standard'}}}}} - ) + + self.assertEquals(ddoc['language'], 'query') + self.assertEquals(ddoc['lists'], {}) + self.assertEquals(ddoc['shows'], {}) + + json_index = ddoc['views']['json-index-001'] + self.assertEquals(json_index['map']['fields']['age'], 'asc') + self.assertEquals(json_index['map']['fields']['name'], 'asc') + self.assertEquals(json_index['options']['def']['fields'], ['name', 'age']) + self.assertEquals(json_index['reduce'], '_count') + + text_index = ddoc['indexes']['text-index-001'] + self.assertEquals(text_index['analyzer']['default'], 'keyword') + self.assertEquals(text_index['analyzer']['fields']['$default'], 'standard') + self.assertEquals(text_index['analyzer']['name'], 'perfield') + self.assertEquals(text_index['index']['default_analyzer'], 'keyword') + self.assertEquals(text_index['index']['default_field'], {}) + self.assertEquals(text_index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) + self.assertEquals(text_index['index']['selector'], {}) + self.assertTrue(text_index['index']['index_array_lengths']) def test_create_query_index_failure(self): """ @@ -1381,28 +1383,32 @@ def test_get_query_indexes_raw(self): """ self.db.create_query_index('ddoc001', 'json-idx-001', fields=['name', 'age']) self.db.create_query_index('ddoc001', 'text-idx-001', 'text') - self.assertEqual( - self.db.get_query_indexes(raw_result=True), - {'indexes': [ - {'ddoc': None, - 'name': '_all_docs', - 'type': 'special', - 'def': {'fields': [{'_id': 'asc'}]}}, - {'ddoc': '_design/ddoc001', - 'name': 'json-idx-001', - 'type': 'json', - 'def': {'fields': [{'name': 'asc'}, {'age': 'asc'}]}}, - {'ddoc': '_design/ddoc001', - 'name': 'text-idx-001', - 'type': 'text', - 'def': {'index_array_lengths': True, - 'fields': [], - 'default_field': {}, - 'default_analyzer': 'keyword', - 'selector': {}}} - ], - 'total_rows' : 3} - ) + + indexes = self.db.get_query_indexes(raw_result=True) + + self.assertEquals(indexes['total_rows'], 3) + + all_docs_index = indexes['indexes'][0] + self.assertEquals(all_docs_index['ddoc'], None) + self.assertEquals(all_docs_index['def']['fields'], [{'_id': 'asc'}]) + self.assertEquals(all_docs_index['name'], '_all_docs') + self.assertEquals(all_docs_index['type'], 'special') + + json_index = indexes['indexes'][1] + self.assertEquals(json_index['ddoc'], '_design/ddoc001') + self.assertEquals(json_index['def']['fields'], [{'name': 'asc'}, {'age': 'asc'}]) + self.assertEquals(json_index['name'], 'json-idx-001') + self.assertEquals(json_index['type'], 'json') + + text_index = indexes['indexes'][2] + self.assertEquals(text_index['ddoc'], '_design/ddoc001') + self.assertEquals(text_index['def']['default_analyzer'], 'keyword') + self.assertEquals(text_index['def']['default_field'], {}) + self.assertEquals(text_index['def']['fields'], []) + self.assertEquals(text_index['def']['selector'], {}) + self.assertEquals(text_index['name'], 'text-idx-001') + self.assertEquals(text_index['type'], 'text') + self.assertTrue(text_index['def']['index_array_lengths']) def test_get_query_indexes(self): """ diff --git a/tests/unit/index_tests.py b/tests/unit/index_tests.py index 28696286..98dbf096 100644 --- a/tests/unit/index_tests.py +++ b/tests/unit/index_tests.py @@ -159,25 +159,23 @@ def test_create_an_index_using_ddoc_index_name(self): self.assertEqual(index.design_document_id, '_design/ddoc001') self.assertEqual(index.name, 'index001') with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEqual(ddoc['language'], 'query') - self.assertListEqual(list(ddoc['views'].keys()), ['index001']) - self.assertIsInstance(ddoc.get_view('index001'), QueryIndexView) + self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) + + self.assertEquals(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': '_design/ddoc001', - '_rev': ddoc['_rev'], - 'indexes': {}, - 'language': 'query', - 'views': {'index001': {'map': {'fields': {'name': 'asc', - 'age': 'asc'}}, - 'reduce': '_count', - 'options': {'def': {'fields': ['name', - 'age']}, - }}}, - 'lists': {}, - 'shows': {} - } - ) + + self.assertEquals(ddoc['indexes'], {}) + self.assertEquals(ddoc['language'], 'query') + self.assertEquals(ddoc['lists'], {}) + self.assertEquals(ddoc['shows'], {}) + + self.assertListEqual(list(ddoc['views'].keys()), ['index001']) + + view = ddoc['views'][index.name] + self.assertEquals(view['map']['fields']['age'], 'asc') + self.assertEquals(view['map']['fields']['name'], 'asc') + self.assertEquals(view['options']['def']['fields'], ['name', 'age']) + self.assertEquals(view['reduce'], '_count') def test_create_an_index_without_ddoc_index_name(self): """ @@ -189,25 +187,23 @@ def test_create_an_index_without_ddoc_index_name(self): self.assertTrue(index.design_document_id.startswith('_design/')) self.assertIsNotNone(index.name) with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEqual(ddoc['language'], 'query') - self.assertListEqual(list(ddoc['views'].keys()), [index.name]) self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) + + self.assertEquals(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': index.design_document_id, - '_rev': ddoc['_rev'], - 'indexes': {}, - 'language': 'query', - 'views': {index.name: {'map': {'fields': {'name': 'asc', - 'age': 'asc'}}, - 'reduce': '_count', - 'options': {'def': {'fields': ['name', - 'age']}, - }}}, - 'lists': {}, - 'shows': {} - } - ) + + self.assertEquals(ddoc['indexes'], {}) + self.assertEquals(ddoc['language'], 'query') + self.assertEquals(ddoc['lists'], {}) + self.assertEquals(ddoc['shows'], {}) + + self.assertListEqual(list(ddoc['views'].keys()), [index.name]) + + view = ddoc['views'][index.name] + self.assertEquals(view['map']['fields']['age'], 'asc') + self.assertEquals(view['map']['fields']['name'], 'asc') + self.assertEquals(view['options']['def']['fields'], ['name', 'age']) + self.assertEquals(view['reduce'], '_count') def test_create_an_index_with_empty_ddoc_index_name(self): """ @@ -219,25 +215,23 @@ def test_create_an_index_with_empty_ddoc_index_name(self): self.assertTrue(index.design_document_id.startswith('_design/')) self.assertIsNotNone(index.name) with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEqual(ddoc['language'], 'query') - self.assertListEqual(list(ddoc['views'].keys()), [index.name]) self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) + + self.assertEquals(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': index.design_document_id, - '_rev': ddoc['_rev'], - 'indexes': {}, - 'language': 'query', - 'views': {index.name: {'map': {'fields': {'name': 'asc', - 'age': 'asc'}}, - 'reduce': '_count', - 'options': {'def': {'fields': ['name', - 'age']}, - }}}, - 'lists': {}, - 'shows': {} - } - ) + + self.assertEquals(ddoc['indexes'], {}) + self.assertEquals(ddoc['language'], 'query') + self.assertEquals(ddoc['lists'], {}) + self.assertEquals(ddoc['shows'], {}) + + self.assertListEqual(list(ddoc['views'].keys()), [index.name]) + + view = ddoc['views'][index.name] + self.assertEquals(view['map']['fields']['age'], 'asc') + self.assertEquals(view['map']['fields']['name'], 'asc') + self.assertEquals(view['options']['def']['fields'], ['name', 'age']) + self.assertEquals(view['reduce'], '_count') def test_create_an_index_using_design_prefix(self): """ @@ -249,25 +243,23 @@ def test_create_an_index_using_design_prefix(self): self.assertEqual(index.design_document_id, '_design/ddoc001') self.assertEqual(index.name, 'index001') with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEqual(ddoc['language'], 'query') - self.assertListEqual(list(ddoc['views'].keys()), ['index001']) - self.assertIsInstance(ddoc.get_view('index001'), QueryIndexView) + self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) + + self.assertEquals(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': '_design/ddoc001', - '_rev': ddoc['_rev'], - 'indexes': {}, - 'language': 'query', - 'views': {'index001': {'map': {'fields': {'name': 'asc', - 'age': 'asc'}}, - 'reduce': '_count', - 'options': {'def': {'fields': ['name', - 'age']}, - }}}, - 'lists': {}, - 'shows': {} - } - ) + + self.assertEquals(ddoc['indexes'], {}) + self.assertEquals(ddoc['language'], 'query') + self.assertEquals(ddoc['lists'], {}) + self.assertEquals(ddoc['shows'], {}) + + self.assertListEqual(list(ddoc['views'].keys()), [index.name]) + + view = ddoc['views'][index.name] + self.assertEquals(view['map']['fields']['age'], 'asc') + self.assertEquals(view['map']['fields']['name'], 'asc') + self.assertEquals(view['options']['def']['fields'], ['name', 'age']) + self.assertEquals(view['reduce'], '_count') def test_create_uses_custom_encoder(self): """ @@ -457,27 +449,23 @@ def test_create_a_search_index_no_kwargs(self): self.assertEqual(index.design_document_id, '_design/ddoc001') self.assertEqual(index.name, 'index001') with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEqual(ddoc['language'], 'query') - self.assertListEqual(list(ddoc['indexes'].keys()), ['index001']) + self.assertEquals(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': '_design/ddoc001', - '_rev': ddoc['_rev'], - 'language': 'query', - 'views': {}, - 'indexes': {'index001': - {'index': {'index_array_lengths': True, - 'fields': 'all_fields', - 'default_field': {}, - 'default_analyzer': 'keyword', - 'selector': {}}, - 'analyzer': {'name': 'perfield', - 'default': 'keyword', - 'fields': {'$default': 'standard'}}}}, - 'lists': {}, - 'shows': {} - } - ) + + self.assertEquals(ddoc['language'], 'query') + self.assertEquals(ddoc['lists'], {}) + self.assertEquals(ddoc['shows'], {}) + self.assertEquals(ddoc['views'], {}) + + index = ddoc['indexes']['index001'] + self.assertEquals(index['analyzer']['default'], 'keyword') + self.assertEquals(index['analyzer']['fields']['$default'], 'standard') + self.assertEquals(index['analyzer']['name'], 'perfield') + self.assertEquals(index['index']['default_analyzer'], 'keyword') + self.assertEquals(index['index']['default_field'], {}) + self.assertEquals(index['index']['fields'], 'all_fields') + self.assertEquals(index['index']['selector'], {}) + self.assertTrue(index['index']['index_array_lengths']) def test_create_a_search_index_with_kwargs(self): """ @@ -495,29 +483,24 @@ def test_create_a_search_index_with_kwargs(self): self.assertEqual(index.design_document_id, '_design/ddoc001') self.assertEqual(index.name, 'index001') with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEqual(ddoc['language'], 'query') - self.assertListEqual(list(ddoc['indexes'].keys()), ['index001']) + self.assertEquals(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEqual(ddoc, - {'_id': '_design/ddoc001', - '_rev': ddoc['_rev'], - 'language': 'query', - 'views': {}, - 'indexes': {'index001': - {'index': {'index_array_lengths': True, - 'fields': [{'name': 'name', 'type': 'string'}, - {'name': 'age', 'type': 'number'}], - 'default_field': {'enabled': True, - 'analyzer': 'german'}, - 'default_analyzer': 'keyword', - 'selector': {}}, - 'analyzer': {'name': 'perfield', - 'default': 'keyword', - 'fields': {'$default': 'german'}}}}, - 'lists': {}, - 'shows': {} - } - ) + + self.assertEquals(ddoc['language'], 'query') + self.assertEquals(ddoc['lists'], {}) + self.assertEquals(ddoc['shows'], {}) + self.assertEquals(ddoc['views'], {}) + + index = ddoc['indexes']['index001'] + self.assertEquals(index['analyzer']['default'], 'keyword') + self.assertEquals(index['analyzer']['fields']['$default'], 'german') + self.assertEquals(index['analyzer']['name'], 'perfield') + self.assertEquals(index['index']['default_analyzer'], 'keyword') + self.assertEquals(index['index']['default_field']['analyzer'], 'german') + self.assertEquals(index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) + self.assertEquals(index['index']['selector'], {}) + self.assertTrue(index['index']['default_field']['enabled']) + self.assertTrue(index['index']['index_array_lengths']) def test_create_a_search_index_invalid_argument(self): """ From 8e89338c18110f5623304dfae93a1726611fba48 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 16 Oct 2017 14:49:42 +0100 Subject: [PATCH 041/185] Allow service name to be specified when instantiating from VCAP --- CHANGES.rst | 1 + src/cloudant/__init__.py | 12 ++--- src/cloudant/_common_util.py | 8 ++-- src/cloudant/client.py | 8 +++- tests/unit/client_tests.py | 28 +++++++++++ tests/unit/cloud_foundry_tests.py | 77 +++++++++++++++++++++++++------ 6 files changed, 108 insertions(+), 26 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index eab05572..c683e446 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,7 @@ Unreleased ========== - [NEW] Added ``Result.all()`` convenience method. +- [NEW] Allow ``service_name`` to be specified when instantiating from a Bluemix VCAP_SERVICES environment variable. - [IMPROVED] Updated ``posixpath.join`` references to use ``'/'.join`` when concatenating URL parts. - [IMPROVED] Updated documentation by replacing deprecated Cloudant links with the latest Bluemix links. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 04131db7..7b1ba55a 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -92,7 +92,7 @@ def cloudant_iam(account_name, api_key, **kwargs): cloudant_session.disconnect() @contextlib.contextmanager -def cloudant_bluemix(vcap_services, instance_name=None, **kwargs): +def cloudant_bluemix(vcap_services, instance_name=None, service_name=None, **kwargs): """ Provides a context manager to create a Cloudant session and provide access to databases, docs etc. @@ -101,6 +101,7 @@ def cloudant_bluemix(vcap_services, instance_name=None, **kwargs): :type vcap_services: dict or str :param str instance_name: Optional Bluemix instance name. Only required if multiple Cloudant instances are available. + :param str service_name: Optional Bluemix service name. :param str encoder: Optional json Encoder object used to encode documents for storage. Defaults to json.JSONEncoder. @@ -149,11 +150,10 @@ def cloudant_bluemix(vcap_services, instance_name=None, **kwargs): print client.all_dbs() # ... """ - service = CloudFoundryService(vcap_services, instance_name) - cloudant_session = Cloudant( - service.username, - service.password, - url=service.url, + cloudant_session = Cloudant.bluemix( + vcap_services, + instance_name=instance_name, + service_name=service_name, **kwargs ) cloudant_session.connect() diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index fe2e0680..05e3bd3c 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -498,18 +498,18 @@ def _get_access_token(self): class CloudFoundryService(object): """ Manages Cloud Foundry service configuration. """ - def __init__(self, vcap_services, name=None): + def __init__(self, vcap_services, instance_name=None, service_name=None): try: services = vcap_services if not isinstance(vcap_services, dict): services = json.loads(vcap_services) - cloudant_services = services.get('cloudantNoSQLDB', []) + cloudant_services = services.get(service_name, []) # use first service if no name given and only one service present - use_first = name is None and len(cloudant_services) == 1 + use_first = instance_name is None and len(cloudant_services) == 1 for service in cloudant_services: - if use_first or service.get('name') == name: + if use_first or service.get('name') == instance_name: credentials = service['credentials'] self._host = credentials['host'] self._name = service.get('name') diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 3a1360cc..ce7d4930 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -754,7 +754,7 @@ def _write_cors_configuration(self, config): return resp.json() @classmethod - def bluemix(cls, vcap_services, instance_name=None, **kwargs): + def bluemix(cls, vcap_services, instance_name=None, service_name=None, **kwargs): """ Create a Cloudant session using a VCAP_SERVICES environment variable. @@ -762,6 +762,7 @@ def bluemix(cls, vcap_services, instance_name=None, **kwargs): :type vcap_services: dict or str :param str instance_name: Optional Bluemix instance name. Only required if multiple Cloudant instances are available. + :param str service_name: Optional Bluemix service name. Example usage: @@ -775,7 +776,10 @@ def bluemix(cls, vcap_services, instance_name=None, **kwargs): print client.all_dbs() """ - service = CloudFoundryService(vcap_services, instance_name) + service_name = service_name or 'cloudantNoSQLDB' # default service + service = CloudFoundryService(vcap_services, + instance_name=instance_name, + service_name=service_name) return Cloudant(service.username, service.password, url=service.url, diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 796e5fce..db78861c 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -552,6 +552,34 @@ def test_cloudant_bluemix_context_helper(self): except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) + def test_cloudant_bluemix_dedicated_context_helper(self): + """ + Test that the cloudant_bluemix context helper works as expected when + specifying a service name. + """ + instance_name = 'Cloudant NoSQL DB-wq' + service_name = 'cloudantNoSQLDB Dedicated' + vcap_services = {service_name: [{ + 'credentials': { + 'username': self.user, + 'password': self.pwd, + 'host': '{0}.cloudant.com'.format(self.account), + 'port': 443, + 'url': self.url + }, + 'name': instance_name, + }]} + + try: + with cloudant_bluemix(vcap_services, + instance_name=instance_name, + service_name=service_name) as c: + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + self.assertEquals(c.session()['userCtx']['name'], self.user) + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + def test_constructor_with_account(self): """ Test instantiating a client object using an account name diff --git a/tests/unit/cloud_foundry_tests.py b/tests/unit/cloud_foundry_tests.py index 043949f7..43249b75 100644 --- a/tests/unit/cloud_foundry_tests.py +++ b/tests/unit/cloud_foundry_tests.py @@ -91,68 +91,104 @@ def __init__(self, *args, **kwargs): ] } ]}) + self._test_vcap_services_dedicated = json.dumps({ + 'cloudantNoSQLDB Dedicated': [ # dedicated service name + { + 'name': 'Cloudant NoSQL DB 1', # valid service + 'credentials': { + 'host': 'example.cloudant.com', + 'password': 'pa$$w0rd01', + 'port': 1234, + 'username': 'example' + } + } + ] + }) def test_get_vcap_service_default_success(self): - service = CloudFoundryService(self._test_vcap_services_single) + service = CloudFoundryService( + self._test_vcap_services_single, + service_name='cloudantNoSQLDB' + ) self.assertEqual('Cloudant NoSQL DB 1', service.name) def test_get_vcap_service_default_success_as_dict(self): service = CloudFoundryService( - json.loads(self._test_vcap_services_single) + json.loads(self._test_vcap_services_single), + service_name='cloudantNoSQLDB' ) self.assertEqual('Cloudant NoSQL DB 1', service.name) def test_get_vcap_service_default_failure_multiple_services(self): with self.assertRaises(CloudantException) as cm: - CloudFoundryService(self._test_vcap_services_multiple) + CloudFoundryService( + self._test_vcap_services_multiple, + service_name='cloudantNoSQLDB' + ) self.assertEqual('Missing service in VCAP_SERVICES', str(cm.exception)) def test_get_vcap_service_instance_host(self): service = CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 1' + self._test_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 1', + service_name='cloudantNoSQLDB' ) self.assertEqual('example.cloudant.com', service.host) def test_get_vcap_service_instance_password(self): service = CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 1' + self._test_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 1', + service_name='cloudantNoSQLDB' ) self.assertEqual('pa$$w0rd01', service.password) def test_get_vcap_service_instance_port(self): service = CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 1' + self._test_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 1', + service_name='cloudantNoSQLDB' ) self.assertEqual('1234', service.port) def test_get_vcap_service_instance_port_default(self): service = CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 2' + self._test_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 2', + service_name='cloudantNoSQLDB' ) self.assertEqual('443', service.port) def test_get_vcap_service_instance_url(self): service = CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 1' + self._test_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 1', + service_name='cloudantNoSQLDB' ) self.assertEqual('https://example.cloudant.com:1234', service.url) def test_get_vcap_service_instance_username(self): service = CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 1' + self._test_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 1', + service_name='cloudantNoSQLDB' ) self.assertEqual('example', service.username) def test_raise_error_for_missing_host(self): with self.assertRaises(CloudantException): CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 3' + self._test_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 3', + service_name='cloudantNoSQLDB' ) def test_raise_error_for_missing_password(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 4' + self._test_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 4', + service_name='cloudantNoSQLDB' ) self.assertEqual( "Invalid service: 'password' missing", @@ -162,7 +198,9 @@ def test_raise_error_for_missing_password(self): def test_raise_error_for_missing_username(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 5' + self._test_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 5', + service_name='cloudantNoSQLDB' ) self.assertEqual( "Invalid service: 'username' missing", @@ -172,7 +210,9 @@ def test_raise_error_for_missing_username(self): def test_raise_error_for_invalid_credentials_type(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 6' + self._test_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 6', + service_name='cloudantNoSQLDB' ) self.assertEqual( 'Failed to decode VCAP_SERVICES service credentials', @@ -182,7 +222,9 @@ def test_raise_error_for_invalid_credentials_type(self): def test_raise_error_for_missing_service(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, 'Cloudant NoSQL DB 7' + self._test_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 7', + service_name='cloudantNoSQLDB' ) self.assertEqual('Missing service in VCAP_SERVICES', str(cm.exception)) @@ -190,3 +232,10 @@ def test_raise_error_for_invalid_vcap(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService('{', 'Cloudant NoSQL DB 1') # invalid JSON self.assertEqual('Failed to decode VCAP_SERVICES JSON', str(cm.exception)) + + def test_get_vcap_service_with_dedicated_service_name_success(self): + service = CloudFoundryService( + self._test_vcap_services_dedicated, + service_name='cloudantNoSQLDB Dedicated' + ) + self.assertEqual('Cloudant NoSQL DB 1', service.name) From a41a9606fe66b4a17259742bdc0ace561e05af2c Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 5 Sep 2017 11:42:40 +0100 Subject: [PATCH 042/185] Move client session objects to new module --- src/cloudant/_common_util.py | 209 +--------------------------- src/cloudant/client.py | 6 +- src/cloudant/client_session.py | 231 +++++++++++++++++++++++++++++++ tests/unit/auth_renewal_tests.py | 2 +- tests/unit/client_tests.py | 2 +- tests/unit/iam_auth_tests.py | 42 +++--- 6 files changed, 258 insertions(+), 234 deletions(-) create mode 100644 src/cloudant/client_session.py diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 05e3bd3c..e44df3d5 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -17,14 +17,12 @@ throughout the library. """ -import os import sys import platform from collections import Sequence import json -from requests import RequestException, Session -from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_, url_join +from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_ from .error import CloudantArgumentError, CloudantException # Library Constants @@ -290,211 +288,6 @@ def __new__(cls, code): return str.__new__(cls, code) -class ClientSession(Session): - """ - This class extends Session and provides a default timeout. - """ - - def __init__(self, **kwargs): - super(ClientSession, self).__init__() - self._timeout = kwargs.get('timeout', None) - - def request(self, method, url, **kwargs): # pylint: disable=W0221 - """ - Overrides ``requests.Session.request`` to set the timeout. - """ - resp = super(ClientSession, self).request( - method, url, timeout=self._timeout, **kwargs) - - return resp - - -class CookieSession(ClientSession): - """ - This class extends ClientSession and provides cookie authentication. - """ - - def __init__(self, username, password, server_url, **kwargs): - super(CookieSession, self).__init__(**kwargs) - self._username = username - self._password = password - self._auto_renew = kwargs.get('auto_renew', False) - self._session_url = url_join(server_url, '_session') - - def info(self): - """ - Get cookie based login user information. - """ - resp = self.get(self._session_url) - resp.raise_for_status() - - return resp.json() - - def login(self): - """ - Perform cookie based user login. - """ - resp = super(CookieSession, self).request( - 'POST', - self._session_url, - data={'name': self._username, 'password': self._password}, - ) - resp.raise_for_status() - - def logout(self): - """ - Logout cookie based user. - """ - resp = super(CookieSession, self).request('DELETE', self._session_url) - resp.raise_for_status() - - def request(self, method, url, **kwargs): # pylint: disable=W0221 - """ - Overrides ``requests.Session.request`` to renew the cookie and then - retry the original request (if required). - """ - resp = super(CookieSession, self).request(method, url, **kwargs) - - if not self._auto_renew: - return resp - - is_expired = any(( - resp.status_code == 403 and - resp.json().get('error') == 'credentials_expired', - resp.status_code == 401 - )) - - if is_expired: - self.login() - resp = super(CookieSession, self).request(method, url, **kwargs) - - return resp - - def set_credentials(self, username, password): - """ - Set a new username and password. - - :param str username: New username. - :param str password: New password. - """ - if username is not None: - self._username = username - - if password is not None: - self._password = password - - -class IAMSession(ClientSession): - """ - This class extends ClientSession and provides IAM authentication. - """ - - def __init__(self, api_key, server_url, **kwargs): - super(IAMSession, self).__init__(**kwargs) - self._api_key = api_key - self._auto_renew = kwargs.get('auto_renew', False) - self._session_url = url_join(server_url, '_iam_session') - self._token_url = os.environ.get( - 'IAM_TOKEN_URL', 'https://iam.bluemix.net/oidc/token') - - def info(self): - """ - Get IAM cookie based login user information. - """ - resp = self.get(self._session_url) - resp.raise_for_status() - - return resp.json() - - def login(self): - """ - Perform IAM cookie based user login. - """ - access_token = self._get_access_token() - try: - super(IAMSession, self).request( - 'POST', - self._session_url, - headers={'Content-Type': 'application/json'}, - data=json.dumps({'access_token': access_token}) - ).raise_for_status() - - except RequestException: - raise CloudantException( - 'Failed to exchange IAM token with Cloudant') - - def logout(self): - """ - Logout IAM cookie based user. - """ - self.cookies.clear() - - def request(self, method, url, **kwargs): # pylint: disable=W0221 - """ - Overrides ``requests.Session.request`` to renew the IAM cookie - and then retry the original request (if required). - """ - # The CookieJar API prevents callers from getting an individual Cookie - # object by name. - # We are forced to use the only exposed method of discarding expired - # cookies from the CookieJar. Internally this involves iterating over - # the entire CookieJar and calling `.is_expired()` on each Cookie - # object. - self.cookies.clear_expired_cookies() - - if self._auto_renew and 'IAMSession' not in self.cookies.keys(): - self.login() - - resp = super(IAMSession, self).request(method, url, **kwargs) - - if not self._auto_renew: - return resp - - if resp.status_code == 401: - self.login() - resp = super(IAMSession, self).request(method, url, **kwargs) - - return resp - - def set_credentials(self, username, api_key): - """ - Set a new IAM API key. - - :param str username: Username parameter is unused. - :param str api_key: New IAM API key. - """ - if api_key is not None: - self._api_key = api_key - - def _get_access_token(self): - """ - Get IAM access token using API key. - """ - err = 'Failed to contact IAM token service' - try: - resp = super(IAMSession, self).request( - 'POST', - self._token_url, - auth=('bx', 'bx'), # required for user API keys - headers={'Accepts': 'application/json'}, - data={ - 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', - 'response_type': 'cloud_iam', - 'apikey': self._api_key - } - ) - err = resp.json().get('errorMessage', err) - resp.raise_for_status() - - return resp.json()['access_token'] - - except KeyError: - raise CloudantException('Invalid response from IAM token service') - - except RequestException: - raise CloudantException(err) - - class CloudFoundryService(object): """ Manages Cloud Foundry service configuration. """ diff --git a/src/cloudant/client.py b/src/cloudant/client.py index ce7d4930..a7a0205c 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -29,10 +29,10 @@ from ._common_util import ( USER_AGENT, append_response_error_content, - ClientSession, CloudFoundryService, - CookieSession, - IAMSession) + ) +from client_session import ClientSession, CookieSession, IAMSession + class CouchDB(dict): """ diff --git a/src/cloudant/client_session.py b/src/cloudant/client_session.py new file mode 100644 index 00000000..114b3164 --- /dev/null +++ b/src/cloudant/client_session.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python +# Copyright (c) 2015, 2017 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Module containing client session classes. +""" + +import json +import os + +from requests import RequestException, Session + +from ._2to3 import url_join +from .error import CloudantException + + +class ClientSession(Session): + """ + This class extends Session and provides a default timeout. + """ + + def __init__(self, **kwargs): + super(ClientSession, self).__init__() + self._timeout = kwargs.get('timeout', None) + + def request(self, method, url, **kwargs): # pylint: disable=W0221 + """ + Overrides ``requests.Session.request`` to set the timeout. + """ + resp = super(ClientSession, self).request( + method, url, timeout=self._timeout, **kwargs) + + return resp + + +class CookieSession(ClientSession): + """ + This class extends ClientSession and provides cookie authentication. + """ + + def __init__(self, username, password, server_url, **kwargs): + super(CookieSession, self).__init__(**kwargs) + self._username = username + self._password = password + self._auto_renew = kwargs.get('auto_renew', False) + self._session_url = url_join(server_url, '_session') + + def info(self): + """ + Get cookie based login user information. + """ + resp = self.get(self._session_url) + resp.raise_for_status() + + return resp.json() + + def login(self): + """ + Perform cookie based user login. + """ + resp = super(CookieSession, self).request( + 'POST', + self._session_url, + data={'name': self._username, 'password': self._password}, + ) + resp.raise_for_status() + + def logout(self): + """ + Logout cookie based user. + """ + resp = super(CookieSession, self).request('DELETE', self._session_url) + resp.raise_for_status() + + def request(self, method, url, **kwargs): # pylint: disable=W0221 + """ + Overrides ``requests.Session.request`` to renew the cookie and then + retry the original request (if required). + """ + resp = super(CookieSession, self).request(method, url, **kwargs) + + if not self._auto_renew: + return resp + + is_expired = any(( + resp.status_code == 403 and + resp.json().get('error') == 'credentials_expired', + resp.status_code == 401 + )) + + if is_expired: + self.login() + resp = super(CookieSession, self).request(method, url, **kwargs) + + return resp + + def set_credentials(self, username, password): + """ + Set a new username and password. + + :param str username: New username. + :param str password: New password. + """ + if username is not None: + self._username = username + + if password is not None: + self._password = password + + +class IAMSession(ClientSession): + """ + This class extends ClientSession and provides IAM authentication. + """ + + def __init__(self, api_key, server_url, **kwargs): + super(IAMSession, self).__init__(**kwargs) + self._api_key = api_key + self._auto_renew = kwargs.get('auto_renew', False) + self._session_url = url_join(server_url, '_iam_session') + self._token_url = os.environ.get( + 'IAM_TOKEN_URL', 'https://iam.bluemix.net/oidc/token') + + def info(self): + """ + Get IAM cookie based login user information. + """ + resp = self.get(self._session_url) + resp.raise_for_status() + + return resp.json() + + def login(self): + """ + Perform IAM cookie based user login. + """ + access_token = self._get_access_token() + try: + super(IAMSession, self).request( + 'POST', + self._session_url, + headers={'Content-Type': 'application/json'}, + data=json.dumps({'access_token': access_token}) + ).raise_for_status() + + except RequestException: + raise CloudantException( + 'Failed to exchange IAM token with Cloudant') + + def logout(self): + """ + Logout IAM cookie based user. + """ + self.cookies.clear() + + def request(self, method, url, **kwargs): # pylint: disable=W0221 + """ + Overrides ``requests.Session.request`` to renew the IAM cookie + and then retry the original request (if required). + """ + # The CookieJar API prevents callers from getting an individual Cookie + # object by name. + # We are forced to use the only exposed method of discarding expired + # cookies from the CookieJar. Internally this involves iterating over + # the entire CookieJar and calling `.is_expired()` on each Cookie + # object. + self.cookies.clear_expired_cookies() + + if self._auto_renew and 'IAMSession' not in self.cookies.keys(): + self.login() + + resp = super(IAMSession, self).request(method, url, **kwargs) + + if not self._auto_renew: + return resp + + if resp.status_code == 401: + self.login() + resp = super(IAMSession, self).request(method, url, **kwargs) + + return resp + + # pylint: disable=arguments-differ + def set_credentials(self, username, api_key): + """ + Set a new IAM API key. + + :param str username: Username parameter is unused. + :param str api_key: New IAM API key. + """ + if api_key is not None: + self._api_key = api_key + + def _get_access_token(self): + """ + Get IAM access token using API key. + """ + err = 'Failed to contact IAM token service' + try: + resp = super(IAMSession, self).request( + 'POST', + self._token_url, + auth=('bx', 'bx'), # required for user API keys + headers={'Accepts': 'application/json'}, + data={ + 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', + 'response_type': 'cloud_iam', + 'apikey': self._api_key + } + ) + err = resp.json().get('errorMessage', err) + resp.raise_for_status() + + return resp.json()['access_token'] + + except KeyError: + raise CloudantException('Invalid response from IAM token service') + + except RequestException: + raise CloudantException(err) diff --git a/tests/unit/auth_renewal_tests.py b/tests/unit/auth_renewal_tests.py index 3d9b7cc6..39001028 100644 --- a/tests/unit/auth_renewal_tests.py +++ b/tests/unit/auth_renewal_tests.py @@ -23,7 +23,7 @@ import requests import time -from cloudant._common_util import CookieSession +from cloudant.client_session import CookieSession from .unit_t_db_base import UnitTestDbBase diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index db78861c..b17c4f52 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -33,9 +33,9 @@ from cloudant import cloudant, cloudant_bluemix, couchdb, couchdb_admin_party from cloudant.client import Cloudant, CouchDB +from cloudant.client_session import CookieSession from cloudant.error import CloudantArgumentError, CloudantClientException from cloudant.feed import Feed, InfiniteFeed -from cloudant._common_util import CookieSession from .unit_t_db_base import UnitTestDbBase from .. import bytes_, str_ diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py index a65d3f04..18272d83 100644 --- a/tests/unit/iam_auth_tests.py +++ b/tests/unit/iam_auth_tests.py @@ -19,8 +19,8 @@ import mock from cloudant._2to3 import Cookie -from cloudant._common_util import IAMSession from cloudant.client import Cloudant +from cloudant.client_session import IAMSession MOCK_API_KEY = 'CqbrIYzdO3btWV-5t4teJLY_etfT_dkccq-vO-5vCXSo' @@ -97,7 +97,7 @@ def test_iam_set_credentials(self): self.assertEquals(iam._api_key, new_api_key) - @mock.patch('cloudant._common_util.ClientSession.request') + @mock.patch('cloudant.client_session.ClientSession.request') def test_iam_get_access_token(self, m_req): m_response = mock.MagicMock() m_response.json.return_value = MOCK_OIDC_TOKEN_RESPONSE @@ -122,8 +122,8 @@ def test_iam_get_access_token(self, m_req): self.assertTrue(m_response.raise_for_status.called) self.assertTrue(m_response.json.called) - @mock.patch('cloudant._common_util.ClientSession.request') - @mock.patch('cloudant._common_util.IAMSession._get_access_token') + @mock.patch('cloudant.client_session.ClientSession.request') + @mock.patch('cloudant.client_session.IAMSession._get_access_token') def test_iam_login(self, m_token, m_req): m_token.return_value = MOCK_ACCESS_TOKEN m_response = mock.MagicMock() @@ -150,7 +150,7 @@ def test_iam_logout(self): iam.logout() self.assertEqual(len(iam.cookies.keys()), 0) - @mock.patch('cloudant._common_util.ClientSession.get') + @mock.patch('cloudant.client_session.ClientSession.get') def test_iam_get_session_info(self, m_get): m_info = {'ok': True, 'info': {'authentication_db': '_users'}} @@ -166,8 +166,8 @@ def test_iam_get_session_info(self, m_get): self.assertEqual(info, m_info) self.assertTrue(m_response.raise_for_status.called) - @mock.patch('cloudant._common_util.IAMSession.login') - @mock.patch('cloudant._common_util.ClientSession.request') + @mock.patch('cloudant.client_session.IAMSession.login') + @mock.patch('cloudant.client_session.ClientSession.request') def test_iam_first_request(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() @@ -191,8 +191,8 @@ def test_iam_first_request(self, m_req, m_login): self.assertEqual(m_req.call_count, 1) self.assertEqual(resp.status_code, 200) - @mock.patch('cloudant._common_util.IAMSession.login') - @mock.patch('cloudant._common_util.ClientSession.request') + @mock.patch('cloudant.client_session.IAMSession.login') + @mock.patch('cloudant.client_session.ClientSession.request') def test_iam_renew_cookie_on_expiry(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() @@ -213,8 +213,8 @@ def test_iam_renew_cookie_on_expiry(self, m_req, m_login): self.assertEqual(m_req.call_count, 1) self.assertEqual(resp.status_code, 200) - @mock.patch('cloudant._common_util.IAMSession.login') - @mock.patch('cloudant._common_util.ClientSession.request') + @mock.patch('cloudant.client_session.IAMSession.login') + @mock.patch('cloudant.client_session.ClientSession.request') def test_iam_renew_cookie_on_401_success(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() @@ -243,8 +243,8 @@ def test_iam_renew_cookie_on_401_success(self, m_req, m_login): self.assertEqual(m_login.call_count, 2) self.assertEqual(m_req.call_count, 3) - @mock.patch('cloudant._common_util.IAMSession.login') - @mock.patch('cloudant._common_util.ClientSession.request') + @mock.patch('cloudant.client_session.IAMSession.login') + @mock.patch('cloudant.client_session.ClientSession.request') def test_iam_renew_cookie_on_401_failure(self, m_req, m_login): # mock 401 m_response_bad = mock.MagicMock() @@ -269,8 +269,8 @@ def test_iam_renew_cookie_on_401_failure(self, m_req, m_login): self.assertEqual(m_login.call_count, 3) self.assertEqual(m_req.call_count, 4) - @mock.patch('cloudant._common_util.IAMSession.login') - @mock.patch('cloudant._common_util.ClientSession.request') + @mock.patch('cloudant.client_session.IAMSession.login') + @mock.patch('cloudant.client_session.ClientSession.request') def test_iam_renew_cookie_disabled(self, m_req, m_login): # mock 401 m_response_bad = mock.MagicMock() @@ -292,8 +292,8 @@ def test_iam_renew_cookie_disabled(self, m_req, m_login): self.assertEqual(m_login.call_count, 1) # no attempt to renew self.assertEqual(m_req.call_count, 2) - @mock.patch('cloudant._common_util.IAMSession.login') - @mock.patch('cloudant._common_util.ClientSession.request') + @mock.patch('cloudant.client_session.IAMSession.login') + @mock.patch('cloudant.client_session.ClientSession.request') def test_iam_client_create(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() @@ -315,8 +315,8 @@ def test_iam_client_create(self, m_req, m_login): self.assertEqual(m_req.call_count, 1) self.assertEqual(dbs, ['animaldb']) - @mock.patch('cloudant._common_util.IAMSession.login') - @mock.patch('cloudant._common_util.IAMSession.set_credentials') + @mock.patch('cloudant.client_session.IAMSession.login') + @mock.patch('cloudant.client_session.IAMSession.set_credentials') def test_iam_client_session_login(self, m_set, m_login): # create IAM client client = Cloudant.iam('foo', MOCK_API_KEY) @@ -331,8 +331,8 @@ def test_iam_client_session_login(self, m_set, m_login): self.assertEqual(m_login.call_count, 2) self.assertEqual(m_set.call_count, 2) - @mock.patch('cloudant._common_util.IAMSession.login') - @mock.patch('cloudant._common_util.IAMSession.set_credentials') + @mock.patch('cloudant.client_session.IAMSession.login') + @mock.patch('cloudant.client_session.IAMSession.set_credentials') def test_iam_client_session_login_with_new_credentials(self, m_set, m_login): # create IAM client client = Cloudant.iam('foo', MOCK_API_KEY) From 18478443540b4eef7303ee402e1d947f21320d08 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Sun, 10 Sep 2017 13:22:37 +0100 Subject: [PATCH 043/185] Added HTTP basic authentication support --- CHANGES.rst | 1 + src/cloudant/client.py | 55 +++++++------ src/cloudant/client_session.py | 138 ++++++++++++++++++++++----------- src/cloudant/database.py | 6 +- tests/unit/client_tests.py | 72 ++++++++++++++++- tests/unit/unit_t_db_base.py | 1 - 6 files changed, 203 insertions(+), 70 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c683e446..c9445dee 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,6 @@ Unreleased ========== +- [NEW] Added HTTP basic authentication support. - [NEW] Added ``Result.all()`` convenience method. - [NEW] Allow ``service_name`` to be specified when instantiating from a Bluemix VCAP_SERVICES environment variable. - [IMPROVED] Updated ``posixpath.join`` references to use ``'/'.join`` when concatenating URL parts. diff --git a/src/cloudant/client.py b/src/cloudant/client.py index a7a0205c..7e69a66e 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -16,10 +16,14 @@ Top level API module that maps to a Cloudant or CouchDB client connection instance. """ -import base64 import json -from ._2to3 import bytes_, unicode_ +from .client_session import ( + BasicSession, + ClientSession, + CookieSession, + IAMSession +) from .database import CloudantDatabase, CouchDatabase from .feed import Feed, InfiniteFeed from .error import ( @@ -31,7 +35,6 @@ append_response_error_content, CloudFoundryService, ) -from client_session import ClientSession, CookieSession, IAMSession class CouchDB(dict): @@ -67,6 +70,8 @@ class CouchDB(dict): `Requests library timeout argument `_. but will apply to every request made using this client. + :param bool use_basic_auth: Keyword argument, if set to True performs basic + access authentication with server. Default is False. :param bool use_iam: Keyword argument, if set to True performs IAM authentication with server. Default is False. Use :func:`~cloudant.client.CouchDB.iam` to construct an IAM @@ -86,7 +91,9 @@ def __init__(self, user, auth_token, admin_party=False, **kwargs): self._timeout = kwargs.get('timeout', None) self.r_session = None self._auto_renew = kwargs.get('auto_renew', False) + self._use_basic_auth = kwargs.get('use_basic_auth', False) self._use_iam = kwargs.get('use_iam', False) + connect_to_couch = kwargs.get('connect', False) if connect_to_couch and self._DATABASE_CLASS == CouchDatabase: self.connect() @@ -100,7 +107,16 @@ def connect(self): self.session_logout() if self.admin_party: - self.r_session = ClientSession(timeout=self._timeout) + self.r_session = ClientSession( + timeout=self._timeout + ) + elif self._use_basic_auth: + self.r_session = BasicSession( + self._user, + self._auth_token, + self.server_url, + timeout=self._timeout + ) elif self._use_iam: self.r_session = IAMSession( self._auth_token, @@ -146,9 +162,6 @@ def session(self): :returns: Dictionary of session info for the current session. """ - if self.admin_party: - return None - return self.r_session.info() def session_cookie(self): @@ -157,19 +170,26 @@ def session_cookie(self): :returns: Session cookie for the current session """ - if self.admin_party: - return None return self.r_session.cookies.get('AuthSession') def session_login(self, user=None, passwd=None): """ Performs a session login by posting the auth information to the _session endpoint. + + :param str user: Username used to connect to CouchDB. + :param str auth_token: Authentication token used to connect to CouchDB. """ - if self.admin_party: - return + self.change_credentials(user=user, auth_token=passwd) + + def change_credentials(self, user=None, auth_token=None): + """ + Change login credentials. - self.r_session.set_credentials(user, passwd) + :param str user: Username used to connect to CouchDB. + :param str auth_token: Authentication token used to connect to CouchDB. + """ + self.r_session.set_credentials(user, auth_token) self.r_session.login() def session_logout(self): @@ -177,9 +197,6 @@ def session_logout(self): Performs a session logout and clears the current session by sending a delete request to the _session endpoint. """ - if self.admin_party: - return - self.r_session.logout() def basic_auth_str(self): @@ -189,13 +206,7 @@ def basic_auth_str(self): :returns: Basic http authentication string """ - if self.admin_party: - return None - hash_ = base64.urlsafe_b64encode(bytes_("{username}:{password}".format( - username=self._user, - password=self._auth_token - ))) - return "Basic {0}".format(unicode_(hash_)) + return self.r_session.base64_user_pass() def all_dbs(self): """ diff --git a/src/cloudant/client_session.py b/src/cloudant/client_session.py index 114b3164..c1d91951 100644 --- a/src/cloudant/client_session.py +++ b/src/cloudant/client_session.py @@ -15,13 +15,13 @@ """ Module containing client session classes. """ - +import base64 import json import os from requests import RequestException, Session -from ._2to3 import url_join +from ._2to3 import bytes_, unicode_, url_join from .error import CloudantException @@ -30,11 +30,34 @@ class ClientSession(Session): This class extends Session and provides a default timeout. """ - def __init__(self, **kwargs): + def __init__(self, username=None, password=None, session_url=None, **kwargs): super(ClientSession, self).__init__() + + self._username = username + self._password = password + self._session_url = session_url + + self._auto_renew = kwargs.get('auto_renew', False) self._timeout = kwargs.get('timeout', None) - def request(self, method, url, **kwargs): # pylint: disable=W0221 + def base64_user_pass(self): + """ + Composes a basic http auth string, suitable for use with the + _replicator database, and other places that need it. + + :returns: Basic http authentication string + """ + if self._username is None or self._password is None: + return None + + hash_ = base64.urlsafe_b64encode(bytes_("{username}:{password}".format( + username=self._username, + password=self._password + ))) + return "Basic {0}".format(unicode_(hash_)) + + # pylint: disable=arguments-differ + def request(self, method, url, **kwargs): """ Overrides ``requests.Session.request`` to set the timeout. """ @@ -43,27 +66,75 @@ def request(self, method, url, **kwargs): # pylint: disable=W0221 return resp + def info(self): + """ + Get session information. + """ + if self._session_url is None: + return None -class CookieSession(ClientSession): + resp = self.get(self._session_url) + resp.raise_for_status() + return resp.json() + + def set_credentials(self, username, password): + """ + Set a new username and password. + + :param str username: New username. + :param str password: New password. + """ + if username is not None: + self._username = username + + if password is not None: + self._password = password + + def login(self): + """ + No-op method - not implemented here. + """ + pass + + def logout(self): + """ + No-op method - not implemented here. + """ + pass + + +class BasicSession(ClientSession): """ - This class extends ClientSession and provides cookie authentication. + This class extends ClientSession to provide basic access authentication. """ def __init__(self, username, password, server_url, **kwargs): - super(CookieSession, self).__init__(**kwargs) - self._username = username - self._password = password - self._auto_renew = kwargs.get('auto_renew', False) - self._session_url = url_join(server_url, '_session') + super(BasicSession, self).__init__( + username=username, + password=password, + session_url=url_join(server_url, '_session'), + **kwargs) - def info(self): + def request(self, method, url, **kwargs): """ - Get cookie based login user information. + Overrides ``requests.Session.request`` to provide basic access + authentication. """ - resp = self.get(self._session_url) - resp.raise_for_status() + return super(BasicSession, self).request( + method, url, auth=(self._username, self._password), **kwargs) - return resp.json() + +class CookieSession(ClientSession): + """ + This class extends ClientSession and provides cookie authentication. + """ + + def __init__(self, username, password, server_url, **kwargs): + super(CookieSession, self).__init__( + username=username, + password=password, + session_url=url_join(server_url, '_session'), + **kwargs) def login(self): """ @@ -83,7 +154,7 @@ def logout(self): resp = super(CookieSession, self).request('DELETE', self._session_url) resp.raise_for_status() - def request(self, method, url, **kwargs): # pylint: disable=W0221 + def request(self, method, url, **kwargs): """ Overrides ``requests.Session.request`` to renew the cookie and then retry the original request (if required). @@ -105,19 +176,6 @@ def request(self, method, url, **kwargs): # pylint: disable=W0221 return resp - def set_credentials(self, username, password): - """ - Set a new username and password. - - :param str username: New username. - :param str password: New password. - """ - if username is not None: - self._username = username - - if password is not None: - self._password = password - class IAMSession(ClientSession): """ @@ -125,22 +183,14 @@ class IAMSession(ClientSession): """ def __init__(self, api_key, server_url, **kwargs): - super(IAMSession, self).__init__(**kwargs) + super(IAMSession, self).__init__( + session_url=url_join(server_url, '_iam_session'), + **kwargs) + self._api_key = api_key - self._auto_renew = kwargs.get('auto_renew', False) - self._session_url = url_join(server_url, '_iam_session') self._token_url = os.environ.get( 'IAM_TOKEN_URL', 'https://iam.bluemix.net/oidc/token') - def info(self): - """ - Get IAM cookie based login user information. - """ - resp = self.get(self._session_url) - resp.raise_for_status() - - return resp.json() - def login(self): """ Perform IAM cookie based user login. @@ -164,7 +214,7 @@ def logout(self): """ self.cookies.clear() - def request(self, method, url, **kwargs): # pylint: disable=W0221 + def request(self, method, url, **kwargs): """ Overrides ``requests.Session.request`` to renew the IAM cookie and then retry the original request (if required). @@ -191,7 +241,7 @@ def request(self, method, url, **kwargs): # pylint: disable=W0221 return resp - # pylint: disable=arguments-differ + # pylint: disable=arguments-differ, unused-argument def set_credentials(self, username, api_key): """ Set a new IAM API key. diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 1992d61e..cf53f6f1 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -94,11 +94,13 @@ def creds(self): :returns: Dictionary containing authentication information """ - if self.admin_party: + session = self.client.session() + if session is None: return None + return { "basic_auth": self.client.basic_auth_str(), - "user_ctx": self.client.session()['userCtx'] + "user_ctx": session.get('userCtx') } def exists(self): diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index b17c4f52..e2b85eb0 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -27,13 +27,14 @@ import sys import os import datetime +import mock from requests import ConnectTimeout, HTTPError from time import sleep from cloudant import cloudant, cloudant_bluemix, couchdb, couchdb_admin_party from cloudant.client import Cloudant, CouchDB -from cloudant.client_session import CookieSession +from cloudant.client_session import BasicSession, CookieSession from cloudant.error import CloudantArgumentError, CloudantClientException from cloudant.feed import Feed, InfiniteFeed @@ -219,6 +220,75 @@ def test_session_cookie(self): finally: self.client.disconnect() + @mock.patch('cloudant.client_session.Session.request') + def test_session_basic(self, m_req): + """ + Test using basic access authentication. + """ + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + m_response_ok.json.return_value = ['animaldb'] + m_req.return_value = m_response_ok + + client = Cloudant(self.user, self.pwd, url=self.url, use_basic_auth=True) + client.connect() + self.assertIsInstance(client.r_session, BasicSession) + + all_dbs = client.all_dbs() + + m_req.assert_called_once_with( + 'GET', + self.url + '/_all_dbs', + allow_redirects=True, + auth=(self.user, self.pwd), # uses HTTP Basic Auth + timeout=None + ) + + self.assertEquals(all_dbs, ['animaldb']) + + @mock.patch('cloudant.client_session.Session.request') + def test_change_credentials_basic(self, m_req): + """ + Test changing credentials when using basic access authentication. + """ + # mock 200 + m_response_ok = mock.MagicMock() + m_response_ok.json.return_value = ['animaldb'] + + # mock 401 + m_response_bad = mock.MagicMock() + m_response_bad.raise_for_status.side_effect = HTTPError('401 Unauthorized') + + m_req.side_effect = [m_response_bad, m_response_ok] + + client = Cloudant('foo', 'bar', url=self.url, use_basic_auth=True) + client.connect() + self.assertIsInstance(client.r_session, BasicSession) + + with self.assertRaises(HTTPError): + client.all_dbs() # expected 401 + + m_req.assert_called_with( + 'GET', + self.url + '/_all_dbs', + allow_redirects=True, + auth=('foo', 'bar'), # uses HTTP Basic Auth + timeout=None + ) + + # use valid credentials + client.change_credentials('baz', 'qux') + all_dbs = client.all_dbs() + + m_req.assert_called_with( + 'GET', + self.url + '/_all_dbs', + allow_redirects=True, + auth=('baz', 'qux'), # uses HTTP Basic Auth + timeout=None + ) + self.assertEquals(all_dbs, ['animaldb']) + def test_basic_auth_str(self): """ Test getting the basic authentication string. diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index c0b62b7a..8ad3f0b2 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -169,7 +169,6 @@ def set_up_client(self, auto_connect=False, auto_renew=False, encoder=None, timeout=timeout ) - def tearDown(self): """ Ensure the client is new for each test From 73e268f6a889e2ca0d95f874f0483ac5f29da33c Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 19 Sep 2017 18:03:22 +0100 Subject: [PATCH 044/185] Don't authenticate basic access requests if credentials are missing --- src/cloudant/client_session.py | 6 +++++- tests/unit/client_tests.py | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/cloudant/client_session.py b/src/cloudant/client_session.py index c1d91951..289b1317 100644 --- a/src/cloudant/client_session.py +++ b/src/cloudant/client_session.py @@ -120,8 +120,12 @@ def request(self, method, url, **kwargs): Overrides ``requests.Session.request`` to provide basic access authentication. """ + auth = None + if self._username is not None and self._password is not None: + auth = (self._username, self._password) + return super(BasicSession, self).request( - method, url, auth=(self._username, self._password), **kwargs) + method, url, auth=auth, **kwargs) class CookieSession(ClientSession): diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index e2b85eb0..5fdbd874 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -35,6 +35,7 @@ from cloudant import cloudant, cloudant_bluemix, couchdb, couchdb_admin_party from cloudant.client import Cloudant, CouchDB from cloudant.client_session import BasicSession, CookieSession +from cloudant.database import CloudantDatabase from cloudant.error import CloudantArgumentError, CloudantClientException from cloudant.feed import Feed, InfiniteFeed @@ -246,6 +247,31 @@ def test_session_basic(self, m_req): self.assertEquals(all_dbs, ['animaldb']) + @mock.patch('cloudant.client_session.Session.request') + def test_session_basic_with_no_credentials(self, m_req): + """ + Test using basic access authentication with no credentials. + """ + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + m_req.return_value = m_response_ok + + client = Cloudant(None, None, url=self.url, use_basic_auth=True) + client.connect() + self.assertIsInstance(client.r_session, BasicSession) + + db = client['animaldb'] + + m_req.assert_called_once_with( + 'HEAD', + self.url + '/animaldb', + allow_redirects=False, + auth=None, # ensure no authentication specified + timeout=None + ) + + self.assertIsInstance(db, CloudantDatabase) + @mock.patch('cloudant.client_session.Session.request') def test_change_credentials_basic(self, m_req): """ From 541ed5052535570eab5009ea3bb1f9a39565a2f5 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 20 Sep 2017 09:56:02 +0100 Subject: [PATCH 045/185] Use mock credentials in basic session test --- tests/unit/client_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 5fdbd874..1ba09266 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -231,7 +231,7 @@ def test_session_basic(self, m_req): m_response_ok.json.return_value = ['animaldb'] m_req.return_value = m_response_ok - client = Cloudant(self.user, self.pwd, url=self.url, use_basic_auth=True) + client = Cloudant('foo', 'bar', url=self.url, use_basic_auth=True) client.connect() self.assertIsInstance(client.r_session, BasicSession) @@ -241,7 +241,7 @@ def test_session_basic(self, m_req): 'GET', self.url + '/_all_dbs', allow_redirects=True, - auth=(self.user, self.pwd), # uses HTTP Basic Auth + auth=('foo', 'bar'), # uses HTTP Basic Auth timeout=None ) From 37dbaf55407d16fc549f7889c67dc2ed59379b54 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 11 Sep 2017 15:50:09 +0100 Subject: [PATCH 046/185] Test with an IAM authenticated client --- Jenkinsfile | 31 ++++++++++++- tests/unit/auth_renewal_tests.py | 5 +- tests/unit/client_tests.py | 24 +++++++++- tests/unit/database_tests.py | 9 +++- tests/unit/replicator_tests.py | 6 ++- tests/unit/unit_t_db_base.py | 80 ++++++++++++++++++++------------ 6 files changed, 119 insertions(+), 36 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b89cb5c7..be2d170f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,6 +26,33 @@ def test_python(pythonVersion) } } +def test_python_iam(pythonVersion) +{ + node { + // Unstash the source on this node + unstash name: 'source' + // Set up the environment and test + withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'iam-testy023', usernameVariable: 'DB_USER', passwordVariable: 'IAM_API_KEY']]) { + try { + sh """ virtualenv tmp -p /usr/local/lib/python${pythonVersion}/bin/${pythonVersion.startsWith('3') ? "python3" : "python"} + . ./tmp/bin/activate + echo \$DB_USER + export RUN_CLOUDANT_TESTS=1 + export CLOUDANT_ACCOUNT=\$DB_USER + # Temporarily disable the _db_updates tests pending resolution of case 71610 + export SKIP_DB_UPDATES=1 + pip install -r requirements.txt + pip install -r test-requirements.txt + pylint ./src/cloudant + nosetests -w ./tests/unit --with-xunit""" + } finally { + // Load the test results + junit 'nosetests.xml' + } + } + } +} + // Start of build stage('Checkout'){ // Checkout and stash the source @@ -38,6 +65,8 @@ stage('Test'){ // Run tests in parallel for multiple python versions parallel( Python2: {test_python('2.7.12')}, - Python3: {test_python('3.5.2')} + Python3: {test_python('3.5.2')}, + 'Python2-IAM': {test_python_iam('2.7.12')}, + 'Python3-IAM': {test_python_iam('3.5.2')} ) } diff --git a/tests/unit/auth_renewal_tests.py b/tests/unit/auth_renewal_tests.py index 39001028..605a61e1 100644 --- a/tests/unit/auth_renewal_tests.py +++ b/tests/unit/auth_renewal_tests.py @@ -25,7 +25,7 @@ from cloudant.client_session import CookieSession -from .unit_t_db_base import UnitTestDbBase +from .unit_t_db_base import skip_for_iam, UnitTestDbBase @unittest.skipIf(os.environ.get('ADMIN_PARTY') == 'true', 'Skipping - Admin Party mode') class AuthRenewalTests(UnitTestDbBase): @@ -44,7 +44,8 @@ def tearDown(self): Override UnitTestDbBase.tearDown() with no tear down """ pass - + + @skip_for_iam def test_client_db_doc_stack_success(self): """ Ensure that auto renewal of cookie auth happens as expected and applies diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 1ba09266..1d0a1c17 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -39,7 +39,7 @@ from cloudant.error import CloudantArgumentError, CloudantClientException from cloudant.feed import Feed, InfiniteFeed -from .unit_t_db_base import UnitTestDbBase +from .unit_t_db_base import skip_for_iam, UnitTestDbBase from .. import bytes_, str_ class CloudantClientExceptionTests(unittest.TestCase): @@ -164,6 +164,7 @@ def test_multiple_connect(self): self.client.disconnect() self.assertIsNone(self.client.r_session) + @skip_for_iam def test_auto_renew_enabled(self): """ Test that CookieSession is used when auto_renew is enabled. @@ -178,6 +179,7 @@ def test_auto_renew_enabled(self): finally: self.client.disconnect() + @skip_for_iam def test_auto_renew_enabled_with_auto_connect(self): """ Test that CookieSession is used when auto_renew is enabled along with @@ -192,6 +194,7 @@ def test_auto_renew_enabled_with_auto_connect(self): finally: self.client.disconnect() + @skip_for_iam def test_session(self): """ Test getting session information. @@ -207,6 +210,7 @@ def test_session(self): finally: self.client.disconnect() + @skip_for_iam def test_session_cookie(self): """ Test getting the session cookie. @@ -315,6 +319,7 @@ def test_change_credentials_basic(self, m_req): ) self.assertEquals(all_dbs, ['animaldb']) + @skip_for_iam def test_basic_auth_str(self): """ Test getting the basic authentication string. @@ -589,6 +594,7 @@ class CloudantClientTests(UnitTestDbBase): Cloudant specific client unit tests """ + @skip_for_iam def test_cloudant_session_login(self): """ Test that the Cloudant client session successfully authenticates. @@ -601,6 +607,7 @@ def test_cloudant_session_login(self): self.client.session_login() self.assertNotEqual(self.client.session_cookie(), old_cookie) + @skip_for_iam def test_cloudant_session_login_with_new_credentials(self): """ Test that the Cloudant client session fails to authenticate when @@ -613,6 +620,7 @@ def test_cloudant_session_login_with_new_credentials(self): self.assertTrue(str(cm.exception).find('Name or password is incorrect')) + @skip_for_iam def test_cloudant_context_helper(self): """ Test that the cloudant context helper works as expected. @@ -624,6 +632,7 @@ def test_cloudant_context_helper(self): except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) + @skip_for_iam def test_cloudant_bluemix_context_helper(self): """ Test that the cloudant_bluemix context helper works as expected. @@ -688,6 +697,7 @@ def test_constructor_with_account(self): 'https://{0}.cloudant.com'.format(self.account) ) + @skip_for_iam def test_bluemix_constructor(self): """ Test instantiating a client object using a VCAP_SERVICES environment @@ -720,6 +730,7 @@ def test_bluemix_constructor(self): finally: c.disconnect() + @skip_for_iam def test_bluemix_constructor_specify_instance_name(self): """ Test instantiating a client object using a VCAP_SERVICES environment @@ -752,6 +763,7 @@ def test_bluemix_constructor_specify_instance_name(self): finally: c.disconnect() + @skip_for_iam def test_bluemix_constructor_with_multiple_services(self): """ Test instantiating a client object using a VCAP_SERVICES environment @@ -819,6 +831,7 @@ def test_connect_headers(self): finally: self.client.disconnect() + @skip_for_iam def test_connect_timeout(self): """ Test that a connect timeout occurs when instantiating @@ -845,6 +858,7 @@ def test_db_updates_infinite_feed_call(self): finally: self.client.disconnect() + @skip_for_iam def test_billing_data(self): """ Test the retrieval of billing data @@ -939,6 +953,7 @@ def test_set_year_with_invalid_month_for_billing_data(self): finally: self.client.disconnect() + @skip_for_iam def test_volume_usage_data(self): """ Test the retrieval of volume usage data @@ -1030,6 +1045,7 @@ def test_set_year_with_invalid_month_for_volume_usage_data(self): finally: self.client.disconnect() + @skip_for_iam def test_requests_usage_data(self): """ Test the retrieval of requests usage data @@ -1121,6 +1137,7 @@ def test_set_year_with_invalid_month_for_requests_usage_data(self): finally: self.client.disconnect() + @skip_for_iam def test_shared_databases(self): """ Test the retrieval of shared database list @@ -1131,6 +1148,7 @@ def test_shared_databases(self): finally: self.client.disconnect() + @skip_for_iam def test_generate_api_key(self): """ Test the generation of an API key for this client account @@ -1144,6 +1162,7 @@ def test_generate_api_key(self): finally: self.client.disconnect() + @skip_for_iam def test_cors_configuration(self): """ Test the retrieval of the current CORS configuration for this client @@ -1157,6 +1176,7 @@ def test_cors_configuration(self): finally: self.client.disconnect() + @skip_for_iam def test_cors_origins(self): """ Test the retrieval of the CORS origins list @@ -1168,6 +1188,7 @@ def test_cors_origins(self): finally: self.client.disconnect() + @skip_for_iam def test_disable_cors(self): """ Test disabling CORS (assuming CORS is enabled) @@ -1188,6 +1209,7 @@ def test_disable_cors(self): finally: self.client.disconnect() + @skip_for_iam def test_update_cors_configuration(self): """ Test updating CORS configuration diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index 94e6fea8..9cd54b7a 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -38,7 +38,7 @@ from cloudant.feed import Feed, InfiniteFeed from tests.unit._test_util import LONG_NUMBER -from .unit_t_db_base import UnitTestDbBase +from .unit_t_db_base import skip_for_iam, UnitTestDbBase from .. import unicode_ class CloudantDatabaseExceptionTests(unittest.TestCase): @@ -151,6 +151,7 @@ def test_retrieve_db_url(self): '/'.join((self.client.server_url, self.test_dbname)) ) + @skip_for_iam def test_retrieve_creds(self): """ Test retrieving client credentials. The client credentials are None if @@ -370,6 +371,7 @@ def test_retrieve_design_document(self): ddoc = self.db.get_design_document('_design/ddoc01') self.assertEqual(ddoc, local_ddoc) + @skip_for_iam def test_get_security_document(self): """ Test retrieving the database security document @@ -995,6 +997,7 @@ def test_unshare_database_uses_custom_encoder(self): with self.assertRaises(TypeError): database.unshare_database(share) + @skip_for_iam def test_security_document(self): """ Test the retrieval of the security document. @@ -1004,6 +1007,7 @@ def test_security_document(self): expected = {'cloudant': {share: ['_reader']}} self.assertDictEqual(self.db.security_document(), expected) + @skip_for_iam def test_share_database_default_permissions(self): """ Test the sharing of a database applying default permissions. @@ -1014,6 +1018,7 @@ def test_share_database_default_permissions(self): expected = {'cloudant': {share: ['_reader']}} self.assertDictEqual(self.db.security_document(), expected) + @skip_for_iam def test_share_database(self): """ Test the sharing of a database. @@ -1024,6 +1029,7 @@ def test_share_database(self): expected = {'cloudant': {share: ['_writer']}} self.assertDictEqual(self.db.security_document(), expected) + @skip_for_iam def test_share_database_with_redundant_role_entries(self): """ Test the sharing of a database works when the list of roles contains @@ -1066,6 +1072,7 @@ def test_share_database_empty_role_list(self): '\'_db_updates\', \'_design\', \'_shards\', \'_security\']' ) + @skip_for_iam def test_unshare_database(self): """ Test the un-sharing of a database from a specified user. diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index 74d08465..eb7118a2 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -34,7 +34,7 @@ from cloudant.document import Document from cloudant.error import CloudantReplicatorException, CloudantClientException -from .unit_t_db_base import UnitTestDbBase +from .unit_t_db_base import skip_for_iam, UnitTestDbBase from .. import unicode_ class CloudantReplicatorExceptionTests(unittest.TestCase): @@ -157,6 +157,7 @@ def test_replication_with_generated_id(self): clone = Replicator(self.client) clone.create_replication(self.db, self.target_db) + @skip_for_iam @flaky(max_runs=3) def test_create_replication(self): """ @@ -300,6 +301,7 @@ def test_list_replications(self): match = [repl_id for repl_id in all_repl_ids if repl_id in repl_ids] self.assertEqual(set(repl_ids), set(match)) + @skip_for_iam def test_retrieve_replication_state(self): """ Test that the replication state can be retrieved for a replication @@ -341,6 +343,7 @@ def test_retrieve_replication_state_using_invalid_id(self): ) self.assertIsNone(repl_state) + @skip_for_iam def test_stop_replication(self): """ Test that a replication can be stopped. @@ -376,6 +379,7 @@ def test_stop_replication_using_invalid_id(self): 'Replication with id {} not found.'.format(repl_id) ) + @skip_for_iam def test_follow_replication(self): """ Test that follow_replication(...) properly iterates updated diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index 8ad3f0b2..5842383d 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -68,6 +68,15 @@ from .. import unicode_ + +def skip_for_iam(f): + def wrapper(*args): + if args[0].use_iam: + raise unittest.SkipTest('Test does not support IAM clients') + return f(*args) + return wrapper + + class UnitTestDbBase(unittest.TestCase): """ The base class for all unit tests targeting a database @@ -133,14 +142,18 @@ def setUp(self): def set_up_client(self, auto_connect=False, auto_renew=False, encoder=None, timeout=(30,300)): + self.user = os.environ.get('DB_USER', None) + self.pwd = os.environ.get('DB_PASSWORD', None) + self.use_iam = False + if os.environ.get('RUN_CLOUDANT_TESTS') is None: + self.url = os.environ['DB_URL'] + admin_party = False - if (os.environ.get('ADMIN_PARTY') and - os.environ.get('ADMIN_PARTY') == 'true'): + if os.environ.get('ADMIN_PARTY') == 'true': admin_party = True - self.user = os.environ.get('DB_USER', None) - self.pwd = os.environ.get('DB_PASSWORD', None) - self.url = os.environ['DB_URL'] + + # construct Cloudant client (using admin party mode) self.client = CouchDB( self.user, self.pwd, @@ -153,21 +166,36 @@ def set_up_client(self, auto_connect=False, auto_renew=False, encoder=None, ) else: self.account = os.environ.get('CLOUDANT_ACCOUNT') - self.user = os.environ.get('DB_USER') - self.pwd = os.environ.get('DB_PASSWORD') self.url = os.environ.get( 'DB_URL', 'https://{0}.cloudant.com'.format(self.account)) - self.client = Cloudant( - self.user, - self.pwd, - url=self.url, - x_cloudant_user=self.account, - connect=auto_connect, - auto_renew=auto_renew, - encoder=encoder, - timeout=timeout - ) + + if os.environ.get('IAM_API_KEY') is None: + # construct Cloudant client (using cookie authentication) + self.client = Cloudant( + self.user, + self.pwd, + url=self.url, + x_cloudant_user=self.account, + connect=auto_connect, + auto_renew=auto_renew, + encoder=encoder, + timeout=timeout + ) + else: + # construct Cloudant client (using IAM authentication) + self.use_iam = True + self.client = Cloudant( + None, # username is not required + os.environ.get('IAM_API_KEY'), + url=self.url, + x_cloudant_user=self.account, + connect=auto_connect, + auto_renew=auto_renew, + encoder=encoder, + timeout=timeout, + use_iam=True, + ) def tearDown(self): """ @@ -284,17 +312,9 @@ def load_security_document_data(self): 'bar2': ['_reader'] } } - if os.environ.get('ADMIN_PARTY') == 'true': - resp = requests.put( - '/'.join([self.db.database_url, '_security']), - data=json.dumps(self.sdoc), - headers={'Content-Type': 'application/json'} - ) - else: - resp = requests.put( - '/'.join([self.db.database_url, '_security']), - auth=(self.user, self.pwd), - data=json.dumps(self.sdoc), - headers={'Content-Type': 'application/json'} - ) + resp = self.client.r_session.put( + '/'.join([self.db.database_url, '_security']), + data=json.dumps(self.sdoc), + headers={'Content-Type': 'application/json'} + ) self.assertEqual(resp.status_code, 200) From 9411642ca15d83b0168c8685ecf2871f867a0ba9 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 11 Sep 2017 17:08:07 +0100 Subject: [PATCH 047/185] Test with a basic access authenticated client --- Jenkinsfile | 38 +++++++++++++++++++++++--- tests/unit/auth_renewal_tests.py | 5 ++-- tests/unit/client_tests.py | 46 ++++++++++++++++---------------- tests/unit/database_tests.py | 16 +++++------ tests/unit/replicator_tests.py | 10 +++---- tests/unit/unit_t_db_base.py | 33 ++++++++++++++++------- 6 files changed, 98 insertions(+), 50 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index be2d170f..b7e4ed63 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,35 @@ // Define the test routine for different python versions -def test_python(pythonVersion) + +def test_python_basic(pythonVersion) +{ + node { + // Unstash the source on this node + unstash name: 'source' + // Set up the environment and test + withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'clientlibs-test', usernameVariable: 'DB_USER', passwordVariable: 'DB_PASSWORD']]) { + try { + sh """ virtualenv tmp -p /usr/local/lib/python${pythonVersion}/bin/${pythonVersion.startsWith('3') ? "python3" : "python"} + . ./tmp/bin/activate + echo \$DB_USER + export RUN_CLOUDANT_TESTS=1 + export RUN_BASIC_AUTH_TESTS=1 + export CLOUDANT_ACCOUNT=\$DB_USER + # Temporarily disable the _db_updates tests pending resolution of case 71610 + export SKIP_DB_UPDATES=1 + pip install -r requirements.txt + pip install -r test-requirements.txt + pylint ./src/cloudant + nosetests -w ./tests/unit --with-xunit""" + } finally { + // Load the test results + junit 'nosetests.xml' + } + } + } +} + + +def test_python_cookie(pythonVersion) { node { // Unstash the source on this node @@ -64,8 +94,10 @@ stage('Checkout'){ stage('Test'){ // Run tests in parallel for multiple python versions parallel( - Python2: {test_python('2.7.12')}, - Python3: {test_python('3.5.2')}, + 'Python2-BASIC': {test_python_basic('2.7.12')}, + 'Python3-BASIC': {test_python_basic('3.5.2')}, + 'Python2-COOKIE': {test_python_cookie('2.7.12')}, + 'Python3-COOKIE': {test_python_cookie('3.5.2')}, 'Python2-IAM': {test_python_iam('2.7.12')}, 'Python3-IAM': {test_python_iam('3.5.2')} ) diff --git a/tests/unit/auth_renewal_tests.py b/tests/unit/auth_renewal_tests.py index 605a61e1..faf5f105 100644 --- a/tests/unit/auth_renewal_tests.py +++ b/tests/unit/auth_renewal_tests.py @@ -25,7 +25,7 @@ from cloudant.client_session import CookieSession -from .unit_t_db_base import skip_for_iam, UnitTestDbBase +from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase @unittest.skipIf(os.environ.get('ADMIN_PARTY') == 'true', 'Skipping - Admin Party mode') class AuthRenewalTests(UnitTestDbBase): @@ -45,7 +45,7 @@ def tearDown(self): """ pass - @skip_for_iam + @skip_if_not_cookie_auth def test_client_db_doc_stack_success(self): """ Ensure that auto renewal of cookie auth happens as expected and applies @@ -110,6 +110,7 @@ def test_client_db_doc_stack_success(self): self.client.disconnect() del self.client + @skip_if_not_cookie_auth def test_client_db_doc_stack_failure(self): """ Ensure that when the regular requests.Session is used that diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 1d0a1c17..d917182a 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -39,7 +39,7 @@ from cloudant.error import CloudantArgumentError, CloudantClientException from cloudant.feed import Feed, InfiniteFeed -from .unit_t_db_base import skip_for_iam, UnitTestDbBase +from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase from .. import bytes_, str_ class CloudantClientExceptionTests(unittest.TestCase): @@ -164,7 +164,7 @@ def test_multiple_connect(self): self.client.disconnect() self.assertIsNone(self.client.r_session) - @skip_for_iam + @skip_if_not_cookie_auth def test_auto_renew_enabled(self): """ Test that CookieSession is used when auto_renew is enabled. @@ -179,7 +179,7 @@ def test_auto_renew_enabled(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_auto_renew_enabled_with_auto_connect(self): """ Test that CookieSession is used when auto_renew is enabled along with @@ -194,7 +194,7 @@ def test_auto_renew_enabled_with_auto_connect(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_session(self): """ Test getting session information. @@ -210,7 +210,7 @@ def test_session(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_session_cookie(self): """ Test getting the session cookie. @@ -319,7 +319,7 @@ def test_change_credentials_basic(self, m_req): ) self.assertEquals(all_dbs, ['animaldb']) - @skip_for_iam + @skip_if_not_cookie_auth def test_basic_auth_str(self): """ Test getting the basic authentication string. @@ -594,7 +594,7 @@ class CloudantClientTests(UnitTestDbBase): Cloudant specific client unit tests """ - @skip_for_iam + @skip_if_not_cookie_auth def test_cloudant_session_login(self): """ Test that the Cloudant client session successfully authenticates. @@ -607,7 +607,7 @@ def test_cloudant_session_login(self): self.client.session_login() self.assertNotEqual(self.client.session_cookie(), old_cookie) - @skip_for_iam + @skip_if_not_cookie_auth def test_cloudant_session_login_with_new_credentials(self): """ Test that the Cloudant client session fails to authenticate when @@ -620,7 +620,7 @@ def test_cloudant_session_login_with_new_credentials(self): self.assertTrue(str(cm.exception).find('Name or password is incorrect')) - @skip_for_iam + @skip_if_not_cookie_auth def test_cloudant_context_helper(self): """ Test that the cloudant context helper works as expected. @@ -632,7 +632,7 @@ def test_cloudant_context_helper(self): except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) - @skip_for_iam + @skip_if_not_cookie_auth def test_cloudant_bluemix_context_helper(self): """ Test that the cloudant_bluemix context helper works as expected. @@ -697,7 +697,7 @@ def test_constructor_with_account(self): 'https://{0}.cloudant.com'.format(self.account) ) - @skip_for_iam + @skip_if_not_cookie_auth def test_bluemix_constructor(self): """ Test instantiating a client object using a VCAP_SERVICES environment @@ -730,7 +730,7 @@ def test_bluemix_constructor(self): finally: c.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_bluemix_constructor_specify_instance_name(self): """ Test instantiating a client object using a VCAP_SERVICES environment @@ -763,7 +763,7 @@ def test_bluemix_constructor_specify_instance_name(self): finally: c.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_bluemix_constructor_with_multiple_services(self): """ Test instantiating a client object using a VCAP_SERVICES environment @@ -831,7 +831,7 @@ def test_connect_headers(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_connect_timeout(self): """ Test that a connect timeout occurs when instantiating @@ -858,7 +858,7 @@ def test_db_updates_infinite_feed_call(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_billing_data(self): """ Test the retrieval of billing data @@ -953,7 +953,7 @@ def test_set_year_with_invalid_month_for_billing_data(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_volume_usage_data(self): """ Test the retrieval of volume usage data @@ -1045,7 +1045,7 @@ def test_set_year_with_invalid_month_for_volume_usage_data(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_requests_usage_data(self): """ Test the retrieval of requests usage data @@ -1137,7 +1137,7 @@ def test_set_year_with_invalid_month_for_requests_usage_data(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_shared_databases(self): """ Test the retrieval of shared database list @@ -1148,7 +1148,7 @@ def test_shared_databases(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_generate_api_key(self): """ Test the generation of an API key for this client account @@ -1162,7 +1162,7 @@ def test_generate_api_key(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_cors_configuration(self): """ Test the retrieval of the current CORS configuration for this client @@ -1176,7 +1176,7 @@ def test_cors_configuration(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_cors_origins(self): """ Test the retrieval of the CORS origins list @@ -1188,7 +1188,7 @@ def test_cors_origins(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_disable_cors(self): """ Test disabling CORS (assuming CORS is enabled) @@ -1209,7 +1209,7 @@ def test_disable_cors(self): finally: self.client.disconnect() - @skip_for_iam + @skip_if_not_cookie_auth def test_update_cors_configuration(self): """ Test updating CORS configuration diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index 9cd54b7a..9a35c3ed 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -38,7 +38,7 @@ from cloudant.feed import Feed, InfiniteFeed from tests.unit._test_util import LONG_NUMBER -from .unit_t_db_base import skip_for_iam, UnitTestDbBase +from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase from .. import unicode_ class CloudantDatabaseExceptionTests(unittest.TestCase): @@ -151,7 +151,7 @@ def test_retrieve_db_url(self): '/'.join((self.client.server_url, self.test_dbname)) ) - @skip_for_iam + @skip_if_not_cookie_auth def test_retrieve_creds(self): """ Test retrieving client credentials. The client credentials are None if @@ -371,7 +371,7 @@ def test_retrieve_design_document(self): ddoc = self.db.get_design_document('_design/ddoc01') self.assertEqual(ddoc, local_ddoc) - @skip_for_iam + @skip_if_not_cookie_auth def test_get_security_document(self): """ Test retrieving the database security document @@ -997,7 +997,7 @@ def test_unshare_database_uses_custom_encoder(self): with self.assertRaises(TypeError): database.unshare_database(share) - @skip_for_iam + @skip_if_not_cookie_auth def test_security_document(self): """ Test the retrieval of the security document. @@ -1007,7 +1007,7 @@ def test_security_document(self): expected = {'cloudant': {share: ['_reader']}} self.assertDictEqual(self.db.security_document(), expected) - @skip_for_iam + @skip_if_not_cookie_auth def test_share_database_default_permissions(self): """ Test the sharing of a database applying default permissions. @@ -1018,7 +1018,7 @@ def test_share_database_default_permissions(self): expected = {'cloudant': {share: ['_reader']}} self.assertDictEqual(self.db.security_document(), expected) - @skip_for_iam + @skip_if_not_cookie_auth def test_share_database(self): """ Test the sharing of a database. @@ -1029,7 +1029,7 @@ def test_share_database(self): expected = {'cloudant': {share: ['_writer']}} self.assertDictEqual(self.db.security_document(), expected) - @skip_for_iam + @skip_if_not_cookie_auth def test_share_database_with_redundant_role_entries(self): """ Test the sharing of a database works when the list of roles contains @@ -1072,7 +1072,7 @@ def test_share_database_empty_role_list(self): '\'_db_updates\', \'_design\', \'_shards\', \'_security\']' ) - @skip_for_iam + @skip_if_not_cookie_auth def test_unshare_database(self): """ Test the un-sharing of a database from a specified user. diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index eb7118a2..5c23140f 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -34,7 +34,7 @@ from cloudant.document import Document from cloudant.error import CloudantReplicatorException, CloudantClientException -from .unit_t_db_base import skip_for_iam, UnitTestDbBase +from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase from .. import unicode_ class CloudantReplicatorExceptionTests(unittest.TestCase): @@ -157,7 +157,7 @@ def test_replication_with_generated_id(self): clone = Replicator(self.client) clone.create_replication(self.db, self.target_db) - @skip_for_iam + @skip_if_not_cookie_auth @flaky(max_runs=3) def test_create_replication(self): """ @@ -301,7 +301,7 @@ def test_list_replications(self): match = [repl_id for repl_id in all_repl_ids if repl_id in repl_ids] self.assertEqual(set(repl_ids), set(match)) - @skip_for_iam + @skip_if_not_cookie_auth def test_retrieve_replication_state(self): """ Test that the replication state can be retrieved for a replication @@ -343,7 +343,7 @@ def test_retrieve_replication_state_using_invalid_id(self): ) self.assertIsNone(repl_state) - @skip_for_iam + @skip_if_not_cookie_auth def test_stop_replication(self): """ Test that a replication can be stopped. @@ -379,7 +379,7 @@ def test_stop_replication_using_invalid_id(self): 'Replication with id {} not found.'.format(repl_id) ) - @skip_for_iam + @skip_if_not_cookie_auth def test_follow_replication(self): """ Test that follow_replication(...) properly iterates updated diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index 5842383d..ccec69ad 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -69,10 +69,10 @@ from .. import unicode_ -def skip_for_iam(f): +def skip_if_not_cookie_auth(f): def wrapper(*args): - if args[0].use_iam: - raise unittest.SkipTest('Test does not support IAM clients') + if not args[0].use_cookie_auth: + raise unittest.SkipTest('Test only supports cookie authentication') return f(*args) return wrapper @@ -144,7 +144,7 @@ def set_up_client(self, auto_connect=False, auto_renew=False, encoder=None, timeout=(30,300)): self.user = os.environ.get('DB_USER', None) self.pwd = os.environ.get('DB_PASSWORD', None) - self.use_iam = False + self.use_cookie_auth = True if os.environ.get('RUN_CLOUDANT_TESTS') is None: self.url = os.environ['DB_URL'] @@ -153,6 +153,7 @@ def set_up_client(self, auto_connect=False, auto_renew=False, encoder=None, if os.environ.get('ADMIN_PARTY') == 'true': admin_party = True + self.use_cookie_auth = False # construct Cloudant client (using admin party mode) self.client = CouchDB( self.user, @@ -170,8 +171,9 @@ def set_up_client(self, auto_connect=False, auto_renew=False, encoder=None, 'DB_URL', 'https://{0}.cloudant.com'.format(self.account)) - if os.environ.get('IAM_API_KEY') is None: - # construct Cloudant client (using cookie authentication) + if os.environ.get('RUN_BASIC_AUTH_TESTS'): + self.use_cookie_auth = False + # construct Cloudant client (using basic access authentication) self.client = Cloudant( self.user, self.pwd, @@ -180,11 +182,12 @@ def set_up_client(self, auto_connect=False, auto_renew=False, encoder=None, connect=auto_connect, auto_renew=auto_renew, encoder=encoder, - timeout=timeout + timeout=timeout, + use_basic_auth=True, ) - else: + elif os.environ.get('IAM_API_KEY'): + self.use_cookie_auth = False # construct Cloudant client (using IAM authentication) - self.use_iam = True self.client = Cloudant( None, # username is not required os.environ.get('IAM_API_KEY'), @@ -196,6 +199,18 @@ def set_up_client(self, auto_connect=False, auto_renew=False, encoder=None, timeout=timeout, use_iam=True, ) + else: + # construct Cloudant client (using cookie authentication) + self.client = Cloudant( + self.user, + self.pwd, + url=self.url, + x_cloudant_user=self.account, + connect=auto_connect, + auto_renew=auto_renew, + encoder=encoder, + timeout=timeout + ) def tearDown(self): """ From 0cc6f637f533984cbfc7c0f9628b874504bf93fe Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 18 Sep 2017 15:50:52 +0100 Subject: [PATCH 048/185] Refactor JenkinsFile --- Jenkinsfile | 125 +++++++++++++++++++--------------------------------- 1 file changed, 45 insertions(+), 80 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b7e4ed63..2bc5d887 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,83 +1,48 @@ -// Define the test routine for different python versions - -def test_python_basic(pythonVersion) -{ - node { - // Unstash the source on this node - unstash name: 'source' - // Set up the environment and test - withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'clientlibs-test', usernameVariable: 'DB_USER', passwordVariable: 'DB_PASSWORD']]) { - try { - sh """ virtualenv tmp -p /usr/local/lib/python${pythonVersion}/bin/${pythonVersion.startsWith('3') ? "python3" : "python"} - . ./tmp/bin/activate - echo \$DB_USER - export RUN_CLOUDANT_TESTS=1 - export RUN_BASIC_AUTH_TESTS=1 - export CLOUDANT_ACCOUNT=\$DB_USER - # Temporarily disable the _db_updates tests pending resolution of case 71610 - export SKIP_DB_UPDATES=1 - pip install -r requirements.txt - pip install -r test-requirements.txt - pylint ./src/cloudant - nosetests -w ./tests/unit --with-xunit""" - } finally { - // Load the test results - junit 'nosetests.xml' - } - } - } -} - - -def test_python_cookie(pythonVersion) -{ - node { - // Unstash the source on this node - unstash name: 'source' - // Set up the environment and test - withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'clientlibs-test', usernameVariable: 'DB_USER', passwordVariable: 'DB_PASSWORD']]) { - try { - sh """ virtualenv tmp -p /usr/local/lib/python${pythonVersion}/bin/${pythonVersion.startsWith('3') ? "python3" : "python"} - . ./tmp/bin/activate - echo \$DB_USER - export RUN_CLOUDANT_TESTS=1 - export CLOUDANT_ACCOUNT=\$DB_USER - # Temporarily disable the _db_updates tests pending resolution of case 71610 - export SKIP_DB_UPDATES=1 - pip install -r requirements.txt - pip install -r test-requirements.txt - pylint ./src/cloudant - nosetests -w ./tests/unit --with-xunit""" - } finally { - // Load the test results - junit 'nosetests.xml' - } - } +def getEnvForSuite(suiteName) { + // Base environment variables + def envVars = [ + "CLOUDANT_ACCOUNT=$DB_USER", + "RUN_CLOUDANT_TESTS=1", + "SKIP_DB_UPDATES=1" // Disable pending resolution of case 71610 + ] + // Add test suite specific environment variables + switch(suiteName) { + case 'basic': + envVars.add("RUN_BASIC_AUTH_TESTS=1") + break + case 'cookie': + break + case 'iam': + // Setting IAM_API_KEY forces tests to run using an IAM enabled client. + envVars.add("IAM_API_KEY=$DB_IAM_API_KEY") + break + default: + error("Unknown test suite environment ${suiteName}") } + return envVars } -def test_python_iam(pythonVersion) -{ +def setupPythonAndTest(pythonVersion, testSuite) { node { // Unstash the source on this node unstash name: 'source' // Set up the environment and test - withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'iam-testy023', usernameVariable: 'DB_USER', passwordVariable: 'IAM_API_KEY']]) { - try { - sh """ virtualenv tmp -p /usr/local/lib/python${pythonVersion}/bin/${pythonVersion.startsWith('3') ? "python3" : "python"} - . ./tmp/bin/activate - echo \$DB_USER - export RUN_CLOUDANT_TESTS=1 - export CLOUDANT_ACCOUNT=\$DB_USER - # Temporarily disable the _db_updates tests pending resolution of case 71610 - export SKIP_DB_UPDATES=1 - pip install -r requirements.txt - pip install -r test-requirements.txt - pylint ./src/cloudant - nosetests -w ./tests/unit --with-xunit""" - } finally { - // Load the test results - junit 'nosetests.xml' + withCredentials([usernamePassword(credentialsId: 'clientlibs-test', usernameVariable: 'DB_USER', passwordVariable: 'DB_PASSWORD'), + string(credentialsId: 'clientlibs-test-iam', variable: 'DB_IAM_API_KEY')]) { + withEnv(getEnvForSuite("${testSuite}")) { + try { + sh """ + virtualenv tmp -p /usr/local/lib/python${pythonVersion}/bin/${pythonVersion.startsWith('3') ? "python3" : "python"} + . ./tmp/bin/activate + pip install -r requirements.txt + pip install -r test-requirements.txt + pylint ./src/cloudant + nosetests -w ./tests/unit --with-xunit + """ + } finally { + // Load the test results + junit 'nosetests.xml' + } } } } @@ -91,14 +56,14 @@ stage('Checkout'){ stash name: 'source' } } + stage('Test'){ - // Run tests in parallel for multiple python versions parallel( - 'Python2-BASIC': {test_python_basic('2.7.12')}, - 'Python3-BASIC': {test_python_basic('3.5.2')}, - 'Python2-COOKIE': {test_python_cookie('2.7.12')}, - 'Python3-COOKIE': {test_python_cookie('3.5.2')}, - 'Python2-IAM': {test_python_iam('2.7.12')}, - 'Python3-IAM': {test_python_iam('3.5.2')} + 'Python2-BASIC': { setupPythonAndTest('2.7.12', 'basic') }, + 'Python3-BASIC': { setupPythonAndTest('3.5.2', 'basic') }, + 'Python2-COOKIE': { setupPythonAndTest('2.7.12', 'cookie') }, + 'Python3-COOKIE': { setupPythonAndTest('3.5.2', 'cookie') }, + 'Python2-IAM': { setupPythonAndTest('2.7.12', 'iam') }, + 'Python3-IAM': { setupPythonAndTest('3.5.2', 'iam') } ) } From b1645da4ec20b072725bb1a6a06306496378736c Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 24 Oct 2017 11:18:28 +0100 Subject: [PATCH 049/185] Make client session module private --- .../{client_session.py => _client_session.py} | 0 src/cloudant/client.py | 2 +- tests/unit/auth_renewal_tests.py | 2 +- tests/unit/client_tests.py | 8 ++-- tests/unit/iam_auth_tests.py | 42 +++++++++---------- 5 files changed, 27 insertions(+), 27 deletions(-) rename src/cloudant/{client_session.py => _client_session.py} (100%) diff --git a/src/cloudant/client_session.py b/src/cloudant/_client_session.py similarity index 100% rename from src/cloudant/client_session.py rename to src/cloudant/_client_session.py diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 7e69a66e..84644415 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -18,7 +18,7 @@ """ import json -from .client_session import ( +from ._client_session import ( BasicSession, ClientSession, CookieSession, diff --git a/tests/unit/auth_renewal_tests.py b/tests/unit/auth_renewal_tests.py index faf5f105..b5fb48b6 100644 --- a/tests/unit/auth_renewal_tests.py +++ b/tests/unit/auth_renewal_tests.py @@ -23,7 +23,7 @@ import requests import time -from cloudant.client_session import CookieSession +from cloudant._client_session import CookieSession from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index d917182a..114a9e3a 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -34,7 +34,7 @@ from cloudant import cloudant, cloudant_bluemix, couchdb, couchdb_admin_party from cloudant.client import Cloudant, CouchDB -from cloudant.client_session import BasicSession, CookieSession +from cloudant._client_session import BasicSession, CookieSession from cloudant.database import CloudantDatabase from cloudant.error import CloudantArgumentError, CloudantClientException from cloudant.feed import Feed, InfiniteFeed @@ -225,7 +225,7 @@ def test_session_cookie(self): finally: self.client.disconnect() - @mock.patch('cloudant.client_session.Session.request') + @mock.patch('cloudant._client_session.Session.request') def test_session_basic(self, m_req): """ Test using basic access authentication. @@ -251,7 +251,7 @@ def test_session_basic(self, m_req): self.assertEquals(all_dbs, ['animaldb']) - @mock.patch('cloudant.client_session.Session.request') + @mock.patch('cloudant._client_session.Session.request') def test_session_basic_with_no_credentials(self, m_req): """ Test using basic access authentication with no credentials. @@ -276,7 +276,7 @@ def test_session_basic_with_no_credentials(self, m_req): self.assertIsInstance(db, CloudantDatabase) - @mock.patch('cloudant.client_session.Session.request') + @mock.patch('cloudant._client_session.Session.request') def test_change_credentials_basic(self, m_req): """ Test changing credentials when using basic access authentication. diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py index 18272d83..d6b3c04c 100644 --- a/tests/unit/iam_auth_tests.py +++ b/tests/unit/iam_auth_tests.py @@ -20,7 +20,7 @@ from cloudant._2to3 import Cookie from cloudant.client import Cloudant -from cloudant.client_session import IAMSession +from cloudant._client_session import IAMSession MOCK_API_KEY = 'CqbrIYzdO3btWV-5t4teJLY_etfT_dkccq-vO-5vCXSo' @@ -97,7 +97,7 @@ def test_iam_set_credentials(self): self.assertEquals(iam._api_key, new_api_key) - @mock.patch('cloudant.client_session.ClientSession.request') + @mock.patch('cloudant._client_session.ClientSession.request') def test_iam_get_access_token(self, m_req): m_response = mock.MagicMock() m_response.json.return_value = MOCK_OIDC_TOKEN_RESPONSE @@ -122,8 +122,8 @@ def test_iam_get_access_token(self, m_req): self.assertTrue(m_response.raise_for_status.called) self.assertTrue(m_response.json.called) - @mock.patch('cloudant.client_session.ClientSession.request') - @mock.patch('cloudant.client_session.IAMSession._get_access_token') + @mock.patch('cloudant._client_session.ClientSession.request') + @mock.patch('cloudant._client_session.IAMSession._get_access_token') def test_iam_login(self, m_token, m_req): m_token.return_value = MOCK_ACCESS_TOKEN m_response = mock.MagicMock() @@ -150,7 +150,7 @@ def test_iam_logout(self): iam.logout() self.assertEqual(len(iam.cookies.keys()), 0) - @mock.patch('cloudant.client_session.ClientSession.get') + @mock.patch('cloudant._client_session.ClientSession.get') def test_iam_get_session_info(self, m_get): m_info = {'ok': True, 'info': {'authentication_db': '_users'}} @@ -166,8 +166,8 @@ def test_iam_get_session_info(self, m_get): self.assertEqual(info, m_info) self.assertTrue(m_response.raise_for_status.called) - @mock.patch('cloudant.client_session.IAMSession.login') - @mock.patch('cloudant.client_session.ClientSession.request') + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.ClientSession.request') def test_iam_first_request(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() @@ -191,8 +191,8 @@ def test_iam_first_request(self, m_req, m_login): self.assertEqual(m_req.call_count, 1) self.assertEqual(resp.status_code, 200) - @mock.patch('cloudant.client_session.IAMSession.login') - @mock.patch('cloudant.client_session.ClientSession.request') + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.ClientSession.request') def test_iam_renew_cookie_on_expiry(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() @@ -213,8 +213,8 @@ def test_iam_renew_cookie_on_expiry(self, m_req, m_login): self.assertEqual(m_req.call_count, 1) self.assertEqual(resp.status_code, 200) - @mock.patch('cloudant.client_session.IAMSession.login') - @mock.patch('cloudant.client_session.ClientSession.request') + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.ClientSession.request') def test_iam_renew_cookie_on_401_success(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() @@ -243,8 +243,8 @@ def test_iam_renew_cookie_on_401_success(self, m_req, m_login): self.assertEqual(m_login.call_count, 2) self.assertEqual(m_req.call_count, 3) - @mock.patch('cloudant.client_session.IAMSession.login') - @mock.patch('cloudant.client_session.ClientSession.request') + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.ClientSession.request') def test_iam_renew_cookie_on_401_failure(self, m_req, m_login): # mock 401 m_response_bad = mock.MagicMock() @@ -269,8 +269,8 @@ def test_iam_renew_cookie_on_401_failure(self, m_req, m_login): self.assertEqual(m_login.call_count, 3) self.assertEqual(m_req.call_count, 4) - @mock.patch('cloudant.client_session.IAMSession.login') - @mock.patch('cloudant.client_session.ClientSession.request') + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.ClientSession.request') def test_iam_renew_cookie_disabled(self, m_req, m_login): # mock 401 m_response_bad = mock.MagicMock() @@ -292,8 +292,8 @@ def test_iam_renew_cookie_disabled(self, m_req, m_login): self.assertEqual(m_login.call_count, 1) # no attempt to renew self.assertEqual(m_req.call_count, 2) - @mock.patch('cloudant.client_session.IAMSession.login') - @mock.patch('cloudant.client_session.ClientSession.request') + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.ClientSession.request') def test_iam_client_create(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() @@ -315,8 +315,8 @@ def test_iam_client_create(self, m_req, m_login): self.assertEqual(m_req.call_count, 1) self.assertEqual(dbs, ['animaldb']) - @mock.patch('cloudant.client_session.IAMSession.login') - @mock.patch('cloudant.client_session.IAMSession.set_credentials') + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.IAMSession.set_credentials') def test_iam_client_session_login(self, m_set, m_login): # create IAM client client = Cloudant.iam('foo', MOCK_API_KEY) @@ -331,8 +331,8 @@ def test_iam_client_session_login(self, m_set, m_login): self.assertEqual(m_login.call_count, 2) self.assertEqual(m_set.call_count, 2) - @mock.patch('cloudant.client_session.IAMSession.login') - @mock.patch('cloudant.client_session.IAMSession.set_credentials') + @mock.patch('cloudant._client_session.IAMSession.login') + @mock.patch('cloudant._client_session.IAMSession.set_credentials') def test_iam_client_session_login_with_new_credentials(self, m_set, m_login): # create IAM client client = Cloudant.iam('foo', MOCK_API_KEY) From 4de8914ec60e03bcdf7da6e46642f58216523978 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 24 Oct 2017 11:21:38 +0100 Subject: [PATCH 050/185] Make login method docstrings service generic --- src/cloudant/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 84644415..abf3f5f5 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -177,8 +177,8 @@ def session_login(self, user=None, passwd=None): Performs a session login by posting the auth information to the _session endpoint. - :param str user: Username used to connect to CouchDB. - :param str auth_token: Authentication token used to connect to CouchDB. + :param str user: Username used to connect to server. + :param str auth_token: Authentication token used to connect to server. """ self.change_credentials(user=user, auth_token=passwd) @@ -186,8 +186,8 @@ def change_credentials(self, user=None, auth_token=None): """ Change login credentials. - :param str user: Username used to connect to CouchDB. - :param str auth_token: Authentication token used to connect to CouchDB. + :param str user: Username used to connect to server. + :param str auth_token: Authentication token used to connect to server. """ self.r_session.set_credentials(user, auth_token) self.r_session.login() From d3b59a9061f3539750f0d304079d600951e0d4e5 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 25 Oct 2017 16:20:58 +0100 Subject: [PATCH 051/185] Update CHANGES.rst for IAM support --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index c9445dee..d429d65e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,6 @@ Unreleased ========== +- [NEW] Added API for upcoming Bluemix Identity and Access Management support for Cloudant on Bluemix. Note: IAM API key support is not yet enabled in the service. - [NEW] Added HTTP basic authentication support. - [NEW] Added ``Result.all()`` convenience method. - [NEW] Allow ``service_name`` to be specified when instantiating from a Bluemix VCAP_SERVICES environment variable. From 06de4c853ce33c895fb6d78c09dc3a2f54d2b3a5 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 26 Oct 2017 13:49:38 +0100 Subject: [PATCH 052/185] Prepare for 2.7.0 release --- CHANGES.rst | 4 ++-- docs/conf.py | 4 ++-- setup.py | 2 +- src/cloudant/__init__.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index d429d65e..6010c3a1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,5 @@ -Unreleased -========== +2.7.0 (2017-10-31) +================== - [NEW] Added API for upcoming Bluemix Identity and Access Management support for Cloudant on Bluemix. Note: IAM API key support is not yet enabled in the service. - [NEW] Added HTTP basic authentication support. - [NEW] Added ``Result.all()`` convenience method. diff --git a/docs/conf.py b/docs/conf.py index d993fc17..42782096 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.6.1.dev' +version = '2.7.0' # The full version, including alpha/beta/rc tags. -release = '2.6.1.dev' +release = '2.7.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index d381c2a5..9bf6529f 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ 'include_package_data': True, 'install_requires': requirements, 'name': 'cloudant', - 'version': '2.6.1.dev', + 'version': '2.7.0', 'author': 'IBM', 'author_email': 'alfinkel@us.ibm.com', 'url': 'https://github.com/cloudant/python-cloudant', diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 7b1ba55a..3e9c19d4 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.6.1.dev' +__version__ = '2.7.0' # pylint: disable=wrong-import-position import contextlib From 8f262c94874df956b7b35be3bf7a90f9881fdced Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 31 Oct 2017 15:08:36 +0000 Subject: [PATCH 053/185] Start next development phase at 2.8.0 --- CHANGES.rst | 3 +++ docs/conf.py | 4 ++-- setup.py | 2 +- src/cloudant/__init__.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6010c3a1..e1402f87 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,6 @@ +2.8.0 (Unreleased) +================== + 2.7.0 (2017-10-31) ================== - [NEW] Added API for upcoming Bluemix Identity and Access Management support for Cloudant on Bluemix. Note: IAM API key support is not yet enabled in the service. diff --git a/docs/conf.py b/docs/conf.py index 42782096..7f8c1883 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.7.0' +version = '2.8.0-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.7.0' +release = '2.8.0-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 9bf6529f..705f2c45 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ 'include_package_data': True, 'install_requires': requirements, 'name': 'cloudant', - 'version': '2.7.0', + 'version': '2.8.0-SNAPSHOT', 'author': 'IBM', 'author_email': 'alfinkel@us.ibm.com', 'url': 'https://github.com/cloudant/python-cloudant', diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 3e9c19d4..eef4e184 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.7.0' +__version__ = '2.8.0-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From bc7cc41163d2c7fbb8bff760aa57c148483a4d6f Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 1 Dec 2017 15:20:48 -0500 Subject: [PATCH 054/185] Removed source and target optional parameters from replicator API - Removed integration test that used source/target parameters - Deleted replication document in test_replication_with_generated_id test --- CHANGES.rst | 1 + src/cloudant/replicator.py | 43 ++++++++++------------- tests/integration/replicator_test.py | 51 ---------------------------- tests/unit/replicator_tests.py | 6 +++- 4 files changed, 23 insertions(+), 78 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index e1402f87..cec958df 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,6 @@ 2.8.0 (Unreleased) ================== +- [REMOVED] Removed broken source and target parameters that constantly threw ``AttributeError`` when creating a replication document. 2.7.0 (2017-10-31) ================== diff --git a/src/cloudant/replicator.py b/src/cloudant/replicator.py index 06e91b0a..7e0b0db8 100644 --- a/src/cloudant/replicator.py +++ b/src/cloudant/replicator.py @@ -50,14 +50,6 @@ def create_replication(self, source_db=None, target_db=None, ``CouchDatabase`` or ``CloudantDatabase`` instance. :param str repl_id: Optional replication id. Generated internally if not explicitly set. - :param source: Optional ``str`` or ``dict`` representing the source - database, along with authentication info, if any. Composed - internally if not explicitly set and not in CouchDB Admin Party - mode. - :param target: Optional ``str`` or ``dict`` representing the - target database, possibly including authentication info. Composed - internally if not explicitly set and not in CouchDB Admin Party - mode. :param dict user_ctx: Optional user to act as. Composed internally if not explicitly set and not in CouchDB Admin Party mode. @@ -74,26 +66,25 @@ def create_replication(self, source_db=None, target_db=None, **kwargs ) - if not data.get('source'): - if source_db is None: - raise CloudantReplicatorException(101) - data['source'] = {'url': source_db.database_url} - if not source_db.admin_party: - data['source'].update( - {'headers': {'Authorization': source_db.creds['basic_auth']}} - ) - - if not data.get('target'): - if target_db is None: - raise CloudantReplicatorException(102) - data['target'] = {'url': target_db.database_url} - if not target_db.admin_party: - data['target'].update( - {'headers': {'Authorization': target_db.creds['basic_auth']}} - ) + if source_db is None: + raise CloudantReplicatorException(101) + data['source'] = {'url': source_db.database_url} + if not source_db.admin_party: + data['source'].update( + {'headers': {'Authorization': source_db.creds['basic_auth']}} + ) + + if target_db is None: + raise CloudantReplicatorException(102) + data['target'] = {'url': target_db.database_url} + if not target_db.admin_party: + data['target'].update( + {'headers': {'Authorization': target_db.creds['basic_auth']}} + ) if not data.get('user_ctx'): - if not target_db.admin_party: + if (target_db and not target_db.admin_party or + self.database.creds): data['user_ctx'] = self.database.creds['user_ctx'] return self.database.create_document(data, throw_on_exists=True) diff --git a/tests/integration/replicator_test.py b/tests/integration/replicator_test.py index c0542e0f..4fc3e759 100644 --- a/tests/integration/replicator_test.py +++ b/tests/integration/replicator_test.py @@ -190,57 +190,6 @@ def test_follow_replication(self): self.assertTrue(len(updates) > 0) self.assertEqual(updates[-1]['_replication_state'], 'completed') - @unittest.skip("Doesn't reliably get into error state on couch side.") - def test_follow_replication_with_errors(self): - """ - _test_follow_replication_with_errors_ - - Test to make sure that we exit the follow loop when we submit - a bad replication. - - """ - dbsource = unicode_("test_follow_replication_source_error_{}".format( - unicode_(uuid.uuid4()))) - dbtarget = unicode_("test_follow_replication_target_error_{}".format( - unicode_(uuid.uuid4()))) - - self.dbs = [dbsource, dbtarget] - - with cloudant(self.user, self.passwd, account=self.user) as c: - dbs = c.create_database(dbsource) - dbt = c.create_database(dbtarget) - - doc1 = dbs.create_document( - {"_id": "doc1", "testing": "document 1"} - ) - doc2 = dbs.create_document( - {"_id": "doc2", "testing": "document 1"} - ) - doc3 = dbs.create_document( - {"_id": "doc3", "testing": "document 1"} - ) - - replicator = Replicator(c) - repl_id = unicode_("test_follow_replication_{}".format( - unicode_(uuid.uuid4()))) - self.replication_ids.append(repl_id) - - ret = replicator.create_replication( - source_db=dbs, - target_db=dbt, - # Deliberately override these good params with bad params - source=dbsource + "foo", - target=dbtarget + "foo", - repl_id=repl_id, - continuous=False, - ) - updates = [ - update for update in replicator.follow_replication(repl_id) - ] - self.assertTrue(len(updates) > 0) - self.assertEqual(updates[-1]['_replication_state'], 'error') - - def test_replication_state(self): """ _test_replication_state_ diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index 5c23140f..cd79a7a1 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -155,7 +155,11 @@ def test_constructor_failure(self): def test_replication_with_generated_id(self): clone = Replicator(self.client) - clone.create_replication(self.db, self.target_db) + repl_id = clone.create_replication( + self.db, + self.target_db + ) + self.replication_ids.append(repl_id['_id']) @skip_if_not_cookie_auth @flaky(max_runs=3) From dd50b0131acf248d78824ce9a3b156697bf15120 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 15 Dec 2017 15:51:10 +0000 Subject: [PATCH 055/185] Pinned pylint 1.7.4 and astroid 1.5.3 Hold pylint versions at: pylint 1.7.4 astroid 1.5.3 --- test-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 098c70e5..3b2d990c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,6 @@ mock==1.3.0 nose sphinx -pylint +pylint==1.7.4 +astroid==1.5.3 flaky From 83fa622ef16329d22a7129e18437af7087e8c363 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Tue, 19 Dec 2017 15:03:51 +0000 Subject: [PATCH 056/185] Updated CHANGES/VERSION for auto tagging. Reformatted and renamed `CHANGES` file from `rst` to `md`. Added `VERSION` file. Updated `setup.py` to read `VERSION` file. --- CHANGES.md | 165 ++++++++++++++++++++++++++++++++++++++++++++++++++++ CHANGES.rst | 165 ---------------------------------------------------- VERSION | 1 + setup.py | 6 +- 4 files changed, 171 insertions(+), 166 deletions(-) create mode 100644 CHANGES.md delete mode 100644 CHANGES.rst create mode 100644 VERSION diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..25f4d492 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,165 @@ +# 2.8.0 (Unreleased) + +- [REMOVED] Removed broken source and target parameters that constantly threw `AttributeError` when creating a replication document. + +# 2.7.0 (2017-10-31) + +- [NEW] Added API for upcoming Bluemix Identity and Access Management support for Cloudant on Bluemix. Note: IAM API key support is not yet enabled in the service. +- [NEW] Added HTTP basic authentication support. +- [NEW] Added `Result.all()` convenience method. +- [NEW] Allow `service_name` to be specified when instantiating from a Bluemix VCAP_SERVICES environment variable. +- [IMPROVED] Updated `posixpath.join` references to use `'/'.join` when concatenating URL parts. +- [IMPROVED] Updated documentation by replacing deprecated Cloudant links with the latest Bluemix links. + +# 2.6.0 (2017-08-10) + +- [NEW] Added `Cloudant.bluemix()` class method to the Cloudant client allowing service credentials to be passed using the CloudFoundry VCAP_SERVICES environment variable. +- [FIXED] Fixed client construction in `cloudant_bluemix` context manager. +- [FIXED] Fixed validation for feed options to accept zero as a valid value. + +# 2.5.0 (2017-07-06) + +- [FIXED] Fixed crash caused by non-UTF8 chars in design documents. +- [FIXED] Fixed `TypeError` when setting revision limits on Python>=3.6. +- [FIXED] Fixed the `exists()` double check on `client.py` and `database.py`. +- [FIXED] Fixed Cloudant exception code 409 with 412 when creating a database that already exists. +- [FIXED] Catch error if `throw_on_exists` flag is `False` for creating a document. +- [FIXED] Fixed /_all_docs call where `keys` is an empty list. +- [FIXED] Issue where docs with IDs that sorted lower than 0 were not returned when iterating through _all_docs. + +# 2.4.0 (2017-02-14) + +- [NEW] Added `timeout` option to the client constructor for setting a timeout on a HTTP connection or a response. +- [NEW] Added `cloudant_bluemix` method to the Cloudant client allowing service credentials to be passed using the CloudFoundry VCAP_SERVICES environment variable. +- [IMPROVED] Updated non-response related errors with additional status code and improved error message for easier debugging. + All non-response error are handled using either CloudantException or CloudantArgumentError. +- [FIXED] Support `long` type argument when executing in Python 2. + +# 2.3.1 (2016-11-30) + +- [FIXED] Resolved issue where generated UUIDs for replication documents would not be converted to strings. +- [FIXED] Resolved issue where CouchDatabase.infinite_changes() method can cause a stack overflow. + +# 2.3.0 (2016-11-02) + +- [FIXED] Resolved issue where the custom JSON encoder was at times not used when transforming data. +- [NEW] Added support for managing the database security document through the SecurityDocument class and CouchDatabase convenience method `get_security_document`. +- [NEW] Added `auto_renewal` option to the client constructor to handle the automatic renewal of an expired session cookie auth. + +# 2.2.0 (2016-10-20) + +- [NEW] Added auto connect feature to the client constructor.
 +- [FIXED] Requests session is no longer valid after disconnect. + +# 2.1.1 (2016-10-03) + +- [FIXED] HTTPError is now raised when 4xx or 5xx codes are encountered. + +# 2.1.0 (2016-08-31) + +- [NEW] Added support for Cloudant Search execution. +- [NEW] Added support for Cloudant Search index management. +- [NEW] Added support for managing and querying list functions. +- [NEW] Added support for managing and querying show functions. +- [NEW] Added support for querying update handlers. +- [NEW] Added `rewrites` accessor property for URL rewriting. +- [NEW] Added `st_indexes` accessor property for Cloudant Geospatial indexes. +- [NEW] Added support for DesignDocument `_info` and `_search_info` endpoints. +- [NEW] Added `validate_doc_update` accessor property for update validators. +- [NEW] Added support for a custom `requests.HTTPAdapter` to be configured using an optional `adapter` arg e.g. + `Cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME, adapter=Replay429Adapter())`. +- [IMPROVED] Made the 429 response code backoff optional and configurable. To enable the backoff add + an `adapter` arg of a `Replay429Adapter` with the desired number of retries and initial backoff. To replicate + the 2.0.0 behaviour use: `adapter=Replay429Adapter(retries=10, initialBackoff=0.25)`. If `retries` or + `initialBackoff` are not specified they will default to 3 retries and a 0.25 s initial backoff. +- [IMPROVED] Additional error reason details appended to HTTP response message errors. +- [FIX] `415 Client Error: Unsupported Media Type` when using keys with `db.all_docs`. +- [FIX] Allowed strings as well as lists for search `group_sort` arguments. + +# 2.0.3 (2016-06-03) + +- [FIX] Fixed the python-cloudant readthedocs documentation home page to resolve correctly. + +# 2.0.2 (2016-06-02) + +- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to python-cloudant.readthedocs.io. +- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the user-agent string. +- [FIX] Fixed readthedocs link in README.rst to resolve to documentation home page. + +# 2.0.1 (2016-06-02) + +- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to python-cloudant.readthedocs.io. +- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the user-agent string. +- [FIX] Fixed readthedocs link in README.rst to resolve to documentation home page. + +# 2.0.0 (2016-05-02) + +- [BREAKING] Renamed modules account.py, errors.py, indexes.py, views.py, to client.py, error.py, index.py, and view.py. +- [BREAKING] Removed the `make_result` method from `View` and `Query` classes. If you need to make a query or view result, use `CloudantDatabase.get_query_result`, `CouchDatabase.get_view_result`, or the `View.custom_result` context manager. Additionally, the `Result` and `QueryResult` classes can be called directly to construct a result object. +- [BREAKING] Refactored the `SearchIndex` class to now be the `TextIndex` class. Also renamed the `CloudantDatabase` convenience methods of `get_all_indexes`, `create_index`, and `delete_index` as `get_query_indexes`, `create_query_index`, and `delete_query_index` respectively. These changes were made to clarify that the changed class and the changed methods were specific to query index processing only. +- [BREAKING] Replace "session" and "url" feed constructor arguments with "source" which can be either a client or a database object. Changes also made to the client `db_updates` method signature and the database `changes` method signature. +- [BREAKING] Fixed `CloudantDatabase.share_database` to accept all valid permission roles. Changed the method signature to accept roles as a list argument. +- [BREAKING] Removed credentials module from the API and moved it to the tests folder since the functionality is outside of the scope of this library but is still be useful in unit/integration tests. +- [IMPROVED] Changed the handling of queries using the keys argument to issue a http POST request instead of a http GET request so that the request is no longer bound by any URL length limitation. +- [IMPROVED] Added support for Result/QueryResult data access via index value and added validation logic to `Result.__getitem__()`. +- [IMPROVED] Updated feed functionality to process `_changes` and `_db_updates` with their supported options. Also added an infinite feed option. +- [NEW] Handled HTTP status code `429 Too Many Requests` with blocking backoff and retries. +- [NEW] Added support for CouchDB Admin Party mode. This library can now be used with CouchDB instances where everyone is Admin. +- [FIX] Fixed `Document.get_attachment` method to successfully create text and binary files based on http response Content-Type. The method also returns text, binary, and json content based on http response Content-Type. +- [FIX] Added validation to `Cloudant.bill`, `Cloudant.volume_usage`, and `Cloudant.requests_usage` methods to ensure that a valid year/month combination or neither are used as arguments. +- [FIX] Fixed the handling of empty views in the DesignDocument. +- [FIX] The `CouchDatabase.create_document` method now handles documents and design documents correctly. If the document created is a design document then the locally cached object will be a DesignDocument otherwise it will be a Document. +- [CHANGE] Moved internal `Code` class, functions like `python_to_couch` and `type_or_none`, and constants into a _common_util module. +- [CHANGE] Updated User-Agent header format to be `python-cloudant//Python///`. +- [CHANGE] Completed the addition of unit tests that target a database server. Removed all mocked unit tests. + +# 2.0.0b2 (2016-02-24) + +- [FIX] Remove the fields parameter from required Query parameters. +- [NEW] Add Python 3 support. + +# 2.0.0b1 (2016-01-11) + + +- [NEW] Added support for Cloudant Query execution. +- [NEW] Added support for Cloudant Query index management. +- [FIX] DesignDocument content is no longer limited to just views. +- [FIX] Document url encoding is now enforced. +- [FIX] Database iterator now yields Document/DesignDocument objects with valid document urls. + +# 2.0.0a4 (2015-12-03) + + +- [FIX] Fixed incorrect readme reference to current library being Alpha 2. + +# 2.0.0a3 (2015-12-03) + + +- [NEW] Added API documentation hosted on readthedocs.org. + +# 2.0.0a2 (2015-11-19) + + +- [NEW] Added unit tests targeting CouchDB and Cloudant databases. +- [FIX] Fixed bug in database create validation check to work if response code is either 201 (created) or 202 (accepted). +- [FIX] Fixed database iterator infinite loop problem and to now yield a Document object. +- [BREAKING] Removed previous bulk_docs method from the CouchDatabase class and renamed the previous bulk_insert method as bulk_docs. The previous bulk_docs functionality is available through the all_docs method using the "keys" parameter. +- [FIX] Made missing_revisions, revisions_diff, get_revision_limit, set_revision_limit, and view_cleanup API methods available for CouchDB as well as Cloudant. +- [BREAKING] Moved the db_update method to the account module. +- [FIX] Fixed missing_revisions to key on 'missing_revs'. +- [FIX] Fixed set_revision_limit to encode the request data payload correctly. +- [FIX] `Document.create()` will no longer update an existing document. +- [BREAKING] Renamed Document `field_append` method to `list_field_append`. +- [BREAKING] Renamed Document `field_remove` method to `list_field_remove`. +- [BREAKING] Renamed Document `field_replace` method to `field_set`. +- [FIX] The Document local dictionary `_id` key is now synched with `_document_id` private attribute. +- [FIX] The Document local dictionary is now refreshed after an add/update/delete of an attachment. +- [FIX] The Document `fetch()` method now refreshes the Document local dictionary content correctly. +- [BREAKING] Replace the ReplicatorDatabase class with the Replicator class. A Replicator object has a database attribute that represents the _replicator database. This allows the Replicator to work for both a CloudantDatabase and a CouchDatabase. +- [REMOVED] Removed "not implemented" methods from the DesignDocument. +- [FIX] Add implicit "_design/" prefix for DesignDocument document ids. + +# 2.0.0a1 (2015-10-13) + + +- Initial release (2.0.0a1). diff --git a/CHANGES.rst b/CHANGES.rst deleted file mode 100644 index cec958df..00000000 --- a/CHANGES.rst +++ /dev/null @@ -1,165 +0,0 @@ -2.8.0 (Unreleased) -================== -- [REMOVED] Removed broken source and target parameters that constantly threw ``AttributeError`` when creating a replication document. - -2.7.0 (2017-10-31) -================== -- [NEW] Added API for upcoming Bluemix Identity and Access Management support for Cloudant on Bluemix. Note: IAM API key support is not yet enabled in the service. -- [NEW] Added HTTP basic authentication support. -- [NEW] Added ``Result.all()`` convenience method. -- [NEW] Allow ``service_name`` to be specified when instantiating from a Bluemix VCAP_SERVICES environment variable. -- [IMPROVED] Updated ``posixpath.join`` references to use ``'/'.join`` when concatenating URL parts. -- [IMPROVED] Updated documentation by replacing deprecated Cloudant links with the latest Bluemix links. - -2.6.0 (2017-08-10) -================== -- [NEW] Added ``Cloudant.bluemix()`` class method to the Cloudant client allowing service credentials to be passed using the CloudFoundry VCAP_SERVICES environment variable. -- [FIXED] Fixed client construction in ``cloudant_bluemix`` context manager. -- [FIXED] Fixed validation for feed options to accept zero as a valid value. - -2.5.0 (2017-07-06) -================== -- [FIXED] Fixed crash caused by non-UTF8 chars in design documents. -- [FIXED] Fixed ``TypeError`` when setting revision limits on Python>=3.6. -- [FIXED] Fixed the ``exists()`` double check on ``client.py`` and ``database.py``. -- [FIXED] Fixed Cloudant exception code 409 with 412 when creating a database that already exists. -- [FIXED] Catch error if ``throw_on_exists`` flag is ``False`` for creating a document. -- [FIXED] Fixed /_all_docs call where ``keys`` is an empty list. -- [FIXED] Issue where docs with IDs that sorted lower than 0 were not returned when iterating through _all_docs. - -2.4.0 (2017-02-14) -================== -- [NEW] Added ``timeout`` option to the client constructor for setting a timeout on a HTTP connection or a response. -- [NEW] Added ``cloudant_bluemix`` method to the Cloudant client allowing service credentials to be passed using the CloudFoundry VCAP_SERVICES environment variable. -- [IMPROVED] Updated non-response related errors with additional status code and improved error message for easier debugging. - All non-response error are handled using either CloudantException or CloudantArgumentError. -- [FIXED] Support ``long`` type argument when executing in Python 2. - -2.3.1 (2016-11-30) -================== -- [FIXED] Resolved issue where generated UUIDs for replication documents would not be converted to strings. -- [FIXED] Resolved issue where CouchDatabase.infinite_changes() method can cause a stack overflow. - -2.3.0 (2016-11-02) -================== -- [FIXED] Resolved issue where the custom JSON encoder was at times not used when transforming data. -- [NEW] Added support for managing the database security document through the SecurityDocument class and CouchDatabase convenience method ``get_security_document``. -- [NEW] Added ``auto_renewal`` option to the client constructor to handle the automatic renewal of an expired session cookie auth. - -2.2.0 (2016-10-20) -================== -- [NEW] Added auto connect feature to the client constructor.
 -- [FIXED] Requests session is no longer valid after disconnect. - -2.1.1 (2016-10-03) -================== -- [FIXED] HTTPError is now raised when 4xx or 5xx codes are encountered. - -2.1.0 (2016-08-31) -================== -- [NEW] Added support for Cloudant Search execution. -- [NEW] Added support for Cloudant Search index management. -- [NEW] Added support for managing and querying list functions. -- [NEW] Added support for managing and querying show functions. -- [NEW] Added support for querying update handlers. -- [NEW] Added ``rewrites`` accessor property for URL rewriting. -- [NEW] Added ``st_indexes`` accessor property for Cloudant Geospatial indexes. -- [NEW] Added support for DesignDocument ``_info`` and ``_search_info`` endpoints. -- [NEW] Added ``validate_doc_update`` accessor property for update validators. -- [NEW] Added support for a custom ``requests.HTTPAdapter`` to be configured using an optional ``adapter`` arg e.g. - ``Cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME, adapter=Replay429Adapter())``. -- [IMPROVED] Made the 429 response code backoff optional and configurable. To enable the backoff add - an ``adapter`` arg of a ``Replay429Adapter`` with the desired number of retries and initial backoff. To replicate - the 2.0.0 behaviour use: ``adapter=Replay429Adapter(retries=10, initialBackoff=0.25)``. If ``retries`` or - ``initialBackoff`` are not specified they will default to 3 retries and a 0.25 s initial backoff. -- [IMPROVED] Additional error reason details appended to HTTP response message errors. -- [FIX] ``415 Client Error: Unsupported Media Type`` when using keys with ``db.all_docs``. -- [FIX] Allowed strings as well as lists for search ``group_sort`` arguments. - -2.0.3 (2016-06-03) -================== -- [FIX] Fixed the python-cloudant readthedocs documentation home page to resolve correctly. - -2.0.2 (2016-06-02) -================== -- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to python-cloudant.readthedocs.io. -- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the user-agent string. -- [FIX] Fixed readthedocs link in README.rst to resolve to documentation home page. - -2.0.1 (2016-06-02) -================== -- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to python-cloudant.readthedocs.io. -- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the user-agent string. -- [FIX] Fixed readthedocs link in README.rst to resolve to documentation home page. - -2.0.0 (2016-05-02) -================== -- [BREAKING] Renamed modules account.py, errors.py, indexes.py, views.py, to client.py, error.py, index.py, and view.py. -- [BREAKING] Removed the ``make_result`` method from ``View`` and ``Query`` classes. If you need to make a query or view result, use ``CloudantDatabase.get_query_result``, ``CouchDatabase.get_view_result``, or the ``View.custom_result`` context manager. Additionally, the ``Result`` and ``QueryResult`` classes can be called directly to construct a result object. -- [BREAKING] Refactored the ``SearchIndex`` class to now be the ``TextIndex`` class. Also renamed the ``CloudantDatabase`` convenience methods of ``get_all_indexes``, ``create_index``, and ``delete_index`` as ``get_query_indexes``, ``create_query_index``, and ``delete_query_index`` respectively. These changes were made to clarify that the changed class and the changed methods were specific to query index processing only. -- [BREAKING] Replace "session" and "url" feed constructor arguments with "source" which can be either a client or a database object. Changes also made to the client ``db_updates`` method signature and the database ``changes`` method signature. -- [BREAKING] Fixed ``CloudantDatabase.share_database`` to accept all valid permission roles. Changed the method signature to accept roles as a list argument. -- [BREAKING] Removed credentials module from the API and moved it to the tests folder since the functionality is outside of the scope of this library but is still be useful in unit/integration tests. -- [IMPROVED] Changed the handling of queries using the keys argument to issue a http POST request instead of a http GET request so that the request is no longer bound by any URL length limitation. -- [IMPROVED] Added support for Result/QueryResult data access via index value and added validation logic to ``Result.__getitem__()``. -- [IMPROVED] Updated feed functionality to process ``_changes`` and ``_db_updates`` with their supported options. Also added an infinite feed option. -- [NEW] Handled HTTP status code ``429 Too Many Requests`` with blocking backoff and retries. -- [NEW] Added support for CouchDB Admin Party mode. This library can now be used with CouchDB instances where everyone is Admin. -- [FIX] Fixed ``Document.get_attachment`` method to successfully create text and binary files based on http response Content-Type. The method also returns text, binary, and json content based on http response Content-Type. -- [FIX] Added validation to ``Cloudant.bill``, ``Cloudant.volume_usage``, and ``Cloudant.requests_usage`` methods to ensure that a valid year/month combination or neither are used as arguments. -- [FIX] Fixed the handling of empty views in the DesignDocument. -- [FIX] The ``CouchDatabase.create_document`` method now handles documents and design documents correctly. If the document created is a design document then the locally cached object will be a DesignDocument otherwise it will be a Document. -- [CHANGE] Moved internal ``Code`` class, functions like ``python_to_couch`` and ``type_or_none``, and constants into a _common_util module. -- [CHANGE] Updated User-Agent header format to be ``python-cloudant//Python///``. -- [CHANGE] Completed the addition of unit tests that target a database server. Removed all mocked unit tests. - -2.0.0b2 (2016-02-24) -==================== -- [FIX] Remove the fields parameter from required Query parameters. -- [NEW] Add Python 3 support. - -2.0.0b1 (2016-01-11) -==================== - -- [NEW] Added support for Cloudant Query execution. -- [NEW] Added support for Cloudant Query index management. -- [FIX] DesignDocument content is no longer limited to just views. -- [FIX] Document url encoding is now enforced. -- [FIX] Database iterator now yields Document/DesignDocument objects with valid document urls. - -2.0.0a4 (2015-12-03) -==================== - -- [FIX] Fixed incorrect readme reference to current library being Alpha 2. - -2.0.0a3 (2015-12-03) -==================== - -- [NEW] Added API documentation hosted on readthedocs.org. - -2.0.0a2 (2015-11-19) -==================== - -- [NEW] Added unit tests targeting CouchDB and Cloudant databases. -- [FIX] Fixed bug in database create validation check to work if response code is either 201 (created) or 202 (accepted). -- [FIX] Fixed database iterator infinite loop problem and to now yield a Document object. -- [BREAKING] Removed previous bulk_docs method from the CouchDatabase class and renamed the previous bulk_insert method as bulk_docs. The previous bulk_docs functionality is available through the all_docs method using the "keys" parameter. -- [FIX] Made missing_revisions, revisions_diff, get_revision_limit, set_revision_limit, and view_cleanup API methods available for CouchDB as well as Cloudant. -- [BREAKING] Moved the db_update method to the account module. -- [FIX] Fixed missing_revisions to key on 'missing_revs'. -- [FIX] Fixed set_revision_limit to encode the request data payload correctly. -- [FIX] ``Document.create()`` will no longer update an existing document. -- [BREAKING] Renamed Document ``field_append`` method to ``list_field_append``. -- [BREAKING] Renamed Document ``field_remove`` method to ``list_field_remove``. -- [BREAKING] Renamed Document ``field_replace`` method to ``field_set``. -- [FIX] The Document local dictionary ``_id`` key is now synched with ``_document_id`` private attribute. -- [FIX] The Document local dictionary is now refreshed after an add/update/delete of an attachment. -- [FIX] The Document ``fetch()`` method now refreshes the Document local dictionary content correctly. -- [BREAKING] Replace the ReplicatorDatabase class with the Replicator class. A Replicator object has a database attribute that represents the _replicator database. This allows the Replicator to work for both a CloudantDatabase and a CouchDatabase. -- [REMOVED] Removed "not implemented" methods from the DesignDocument. -- [FIX] Add implicit "_design/" prefix for DesignDocument document ids. - -2.0.0a1 (2015-10-13) -==================== - -- Initial release (2.0.0a1). diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..ceb9bd34 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +2.8.0-SNAPSHOT diff --git a/setup.py b/setup.py index 705f2c45..cac89fd5 100644 --- a/setup.py +++ b/setup.py @@ -23,13 +23,17 @@ requirements_file = open('requirements.txt') requirements = requirements_file.read().strip().split('\n') +requirements_file.close() +version_file = open('VERSION') +version = version_file.read().strip() +version_file.close() setup_args = { 'description': 'Cloudant / CouchDB Client Library', 'include_package_data': True, 'install_requires': requirements, 'name': 'cloudant', - 'version': '2.8.0-SNAPSHOT', + 'version': version, 'author': 'IBM', 'author_email': 'alfinkel@us.ibm.com', 'url': 'https://github.com/cloudant/python-cloudant', From d8fe082af9328bc73f63f08a3635c14082e54d60 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Tue, 19 Dec 2017 15:05:07 +0000 Subject: [PATCH 057/185] Added loops for generating test axes. --- Jenkinsfile | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2bc5d887..61eb4dd9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -58,12 +58,11 @@ stage('Checkout'){ } stage('Test'){ - parallel( - 'Python2-BASIC': { setupPythonAndTest('2.7.12', 'basic') }, - 'Python3-BASIC': { setupPythonAndTest('3.5.2', 'basic') }, - 'Python2-COOKIE': { setupPythonAndTest('2.7.12', 'cookie') }, - 'Python3-COOKIE': { setupPythonAndTest('3.5.2', 'cookie') }, - 'Python2-IAM': { setupPythonAndTest('2.7.12', 'iam') }, - 'Python3-IAM': { setupPythonAndTest('3.5.2', 'iam') } - ) + axes = [:] + ['2.7.12','3.5.2'].each { version -> + ['basic','cookie','iam'].each { auth -> + axes.put("Python${version}-${auth}", {setupPythonAndTest(version, auth)}) + } + } + parallel(axes) } From 475a4ec2b6b037ba84b8cc706bb743dd2767126b Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Tue, 19 Dec 2017 15:05:24 +0000 Subject: [PATCH 058/185] Added publish stage. --- Jenkinsfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 61eb4dd9..f1c24d9c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,3 +66,10 @@ stage('Test'){ } parallel(axes) } + +stage('Publish') { + gitTagAndPublish { + isDraft=true + releaseApiUrl='https://api.github.com/repos/cloudant/python-cloudant/releases' + } +} From 243ff755929cc1cad3cba3dea5a118e8983e699e Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Tue, 19 Dec 2017 17:13:19 +0000 Subject: [PATCH 059/185] Updated PR template for new CHANGES file name. --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 184340f6..e80303d9 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,7 +2,7 @@ Thanks for your hard work, please ensure all items are complete before opening. - [ ] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](https://github.com/cloudant/python-cloudant/blob/master/DCO1.1.txt) - [ ] You have added tests for any code changes -- [ ] You have updated the [CHANGES.rst](https://github.com/cloudant/python-cloudant/blob/master/CHANGES.rst) +- [ ] You have updated the [CHANGES.md](https://github.com/cloudant/python-cloudant/blob/master/CHANGES.md) - [ ] You have completed the PR template below: ## What From e7debd7e2eee29e987670024e58f5a01d8fd9948 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 22 Dec 2017 12:36:59 -0500 Subject: [PATCH 060/185] Replaced README.rst with README.md - Updated sub headers and sections to have the same outline as java-cloudant - Added examples of this library in other open source projects --- README.md | 87 +++++++++++++++++++++++++++++++++++++++++++++++++ README.rst | 95 ------------------------------------------------------ 2 files changed, 87 insertions(+), 95 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/README.md b/README.md new file mode 100644 index 00000000..617802a2 --- /dev/null +++ b/README.md @@ -0,0 +1,87 @@ +# Cloudant Python Client + +[![Build Status](https://travis-ci.org/cloudant/python-cloudant.svg?branch=master)](https://travis-ci.org/cloudant/java-cloudant) +[![Readthedocs](https://readthedocs.org/projects/pip/badge/)](http://python-cloudant.readthedocs.io) +[![Compatibility](https://img.shields.io/badge/python-2.7%2C%203.5-blue.svg)](http://python-cloudant.readthedocs.io/en/latest/compatibility.html) +[![pypi](https://img.shields.io/pypi/v/cloudant.svg)](https://pypi.python.org/pypi/cloudant) + +This is the official Cloudant library for Python. + +* [Installation and Usage](#installation-and-usage) +* [Getting Started](#getting-started) +* [API Reference](http://www.javadoc.io/doc/com.cloudant/cloudant-client/) +* [Related Documentation](#related-documentation) +* [Development](#development) + * [Contributing](CONTRIBUTING.rst) + * [Test Suite](CONTRIBUTING.rst#running-the-tests) + * [Using in Other Projects](#using-in-other-projects) + * [License](#license) + * [Issues](#issues) + +## Installation and Usage + + +Released versions of this library are [hosted on PyPI](https://pypi.python.org/pypi/cloudant) and can be installed with `pip`. + +In order to install the latest version, execute + + pip install cloudant + +## Getting started + + +See [Getting started (readthedocs.io)](http://python-cloudant.readthedocs.io/en/latest/getting_started.html) + +## API Reference + +See [API reference docs (readthedocs.io)](http://python-cloudant.readthedocs.io/en/latest/cloudant.html) + +## Related Documentation + +* [Cloudant Python client library docs (readthedocs.io)](http://python-cloudant.readthedocs.io) +* [Cloudant documentation](https://console.bluemix.net/docs/services/Cloudant/cloudant.html#overview) +* [Cloudant Learning Center](https://developer.ibm.com/clouddataservices/cloudant-learning-center/) + +## Development + +See [CONTRIBUTING.rst](https://github.com/cloudant/python-cloudant/blob/master/CONTRIBUTING.rst) + +## Using in other projects + +The preferred approach for using `python-cloudant` in other projects is to use the PyPI as described above. + +#### Examples in open source projects + +[Movie Recommender Demo](https://github.com/snowch/movie-recommender-demo): +- [Update and check if documents exist](https://github.com/snowch/movie-recommender-demo/blob/master/web_app/app/dao.py#L162-L168) +- [Connect to Cloudant using 429 backoff with 10 retries](https://github.com/snowch/movie-recommender-demo/blob/master/web_app/app/cloudant_db.py#L17-L18) + +[Watson Recipe Bot](https://github.com/ibm-watson-data-lab/watson-recipe-bot-python-cloudant): +- [Use Cloudant Query to find design docs](https://github.com/ibm-watson-data-lab/watson-recipe-bot-python-cloudant/blob/master/souschef/cloudant_recipe_store.py#L33-L77) + +## License + +Copyright © 2015 IBM. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Issues + +Before opening a new issue please consider the following: +* Only the latest release is supported. If at all possible please try to reproduce the issue using +the latest version. +* Please check the [existing issues](https://github.com/cloudant/python-cloudant/issues) +to see if the problem has already been reported. Note that the default search +includes only open issues, but it may already have been closed. +* Cloudant customers should contact Cloudant support for urgent issues. +* When opening a new issue [here in github](../../issues) please complete the template fully. diff --git a/README.rst b/README.rst deleted file mode 100644 index 5c801d05..00000000 --- a/README.rst +++ /dev/null @@ -1,95 +0,0 @@ -Cloudant Python Client -====================== - -|build-status| |docs| |compatibility| - -.. |build-status| image:: https://travis-ci.org/cloudant/python-cloudant.png - :alt: build status - :scale: 100% - :target: https://travis-ci.org/cloudant/python-cloudant - -.. |docs| image:: https://readthedocs.org/projects/pip/badge/ - :alt: docs - :scale: 100% - :target: http://python-cloudant.readthedocs.io - -.. |compatibility| image:: https://img.shields.io/badge/python-2.7%2C%203.5-blue.svg - :alt: compatibility - :scale: 100% - :target: http://python-cloudant.readthedocs.io/en/latest/compatibility.html - -This is the official Cloudant library for Python. - -.. contents:: - :local: - :depth: 2 - :backlinks: none - -====================== -Installation and Usage -====================== - -Released versions of this library are `hosted on PyPI `_ -and can be installed with ``pip``. - -In order to install the latest version, execute - -.. code-block:: bash - - pip install cloudant - -=============== -Getting started -=============== - -See `Getting started (readthedocs.io) `_ - -============= -API Reference -============= - -See `API reference docs (readthedocs.io) `_ - -===================== -Related Documentation -===================== - -* `Cloudant Python client library docs (readthedocs.io) `_ -* `Cloudant documentation `_ -* `Cloudant Learning Center `_ - -=========== -Development -=========== - -See `CONTRIBUTING.rst `_ - -********** -Test Suite -********** - -Content coming soon... - -*********************** -Using in other projects -*********************** - -Content coming soon... - -******* -License -******* - -Copyright © 2015 IBM. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. From 89d9053c9d8e621ba9566e59c4c7d0e65400e901 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Wed, 20 Dec 2017 15:45:03 -0500 Subject: [PATCH 061/185] =?UTF-8?q?Fixed=20pylint=201.8.1=20warnings=20-?= =?UTF-8?q?=20Disabled=20=E2=80=98keyword-arg-before-vararg=E2=80=99=20pyl?= =?UTF-8?q?int=20warning=20-=20Removed=20return=20statement=20to=20fix=20p?= =?UTF-8?q?ylint=20=E2=80=98inconsistent-return-statements=E2=80=99=20-=20?= =?UTF-8?q?Revert=20pinned=20pylint=201.7.4=20and=20astroid=201.5.3=20-=20?= =?UTF-8?q?Replaced=20StopIteration=20with=20return?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pylintrc | 2 +- src/cloudant/database.py | 4 ++-- src/cloudant/document.py | 4 ++-- src/cloudant/replicator.py | 6 +++--- test-requirements.txt | 3 +-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pylintrc b/pylintrc index 0e625d99..4cfbabaf 100644 --- a/pylintrc +++ b/pylintrc @@ -66,7 +66,7 @@ confidence= # Disable "redefined-variable-type" refactor warning messages # Disable "too-many-..." and "too-few-..." refactor warning messages # Disable "locally-disabled" message -disable=R0204,R0901,R0902,R0903,R0904,R0913,R0914,R0915,locally-disabled +disable=R0204,R0901,R0902,R0903,R0904,R0913,R0914,R0915,locally-disabled,keyword-arg-before-vararg [REPORTS] diff --git a/src/cloudant/database.py b/src/cloudant/database.py index cf53f6f1..930673f2 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -660,7 +660,7 @@ def __iter__(self, remote=True): super(CouchDatabase, self).__setitem__(doc['id'], document) yield document - raise StopIteration + return def bulk_docs(self, docs): """ diff --git a/src/cloudant/document.py b/src/cloudant/document.py index 8b93c92f..55abc5c2 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -257,7 +257,7 @@ def _update_field(self, action, field, value, max_tries, tries=0): self.save() except requests.HTTPError as ex: if tries < max_tries and ex.response.status_code == 409: - return self._update_field( + self._update_field( action, field, value, max_tries, tries=tries+1) raise diff --git a/src/cloudant/replicator.py b/src/cloudant/replicator.py index 7e0b0db8..c9b6a40c 100644 --- a/src/cloudant/replicator.py +++ b/src/cloudant/replicator.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -158,7 +158,7 @@ def update_state(): if repl_doc: yield repl_doc if state is not None and state in ['error', 'completed']: - raise StopIteration + return # Now listen on changes feed for the state for change in self.database.changes(): @@ -167,7 +167,7 @@ def update_state(): if repl_doc is not None: yield repl_doc if state is not None and state in ['error', 'completed']: - raise StopIteration + return def stop_replication(self, repl_id): """ diff --git a/test-requirements.txt b/test-requirements.txt index 3b2d990c..098c70e5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,5 @@ mock==1.3.0 nose sphinx -pylint==1.7.4 -astroid==1.5.3 +pylint flaky From 59d07476ef85e3d2faf488800693a8136f8a89c7 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 12 Jan 2018 00:35:22 -0500 Subject: [PATCH 062/185] Replaced java-cloudant Build Status and API references with python-cloudant - Fixed header level --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 617802a2..7dec7036 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Cloudant Python Client -[![Build Status](https://travis-ci.org/cloudant/python-cloudant.svg?branch=master)](https://travis-ci.org/cloudant/java-cloudant) +[![Build Status](https://travis-ci.org/cloudant/python-cloudant.svg?branch=master)](https://travis-ci.org/cloudant/python-cloudant) [![Readthedocs](https://readthedocs.org/projects/pip/badge/)](http://python-cloudant.readthedocs.io) [![Compatibility](https://img.shields.io/badge/python-2.7%2C%203.5-blue.svg)](http://python-cloudant.readthedocs.io/en/latest/compatibility.html) [![pypi](https://img.shields.io/pypi/v/cloudant.svg)](https://pypi.python.org/pypi/cloudant) @@ -9,7 +9,7 @@ This is the official Cloudant library for Python. * [Installation and Usage](#installation-and-usage) * [Getting Started](#getting-started) -* [API Reference](http://www.javadoc.io/doc/com.cloudant/cloudant-client/) +* [API Reference](http://python-cloudant.readthedocs.io/en/latest/cloudant.html) * [Related Documentation](#related-documentation) * [Development](#development) * [Contributing](CONTRIBUTING.rst) @@ -50,7 +50,7 @@ See [CONTRIBUTING.rst](https://github.com/cloudant/python-cloudant/blob/master/C The preferred approach for using `python-cloudant` in other projects is to use the PyPI as described above. -#### Examples in open source projects +### Examples in open source projects [Movie Recommender Demo](https://github.com/snowch/movie-recommender-demo): - [Update and check if documents exist](https://github.com/snowch/movie-recommender-demo/blob/master/web_app/app/dao.py#L162-L168) From 7e348a2948891dc05b478f79e81b415a888387f6 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Wed, 17 Jan 2018 10:52:36 -0500 Subject: [PATCH 063/185] Added Bluemix tutorial link - Added sample python flask app --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7dec7036..34e777e8 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ In order to install the latest version, execute ## Getting started - See [Getting started (readthedocs.io)](http://python-cloudant.readthedocs.io/en/latest/getting_started.html) ## API Reference @@ -41,6 +40,7 @@ See [API reference docs (readthedocs.io)](http://python-cloudant.readthedocs.io/ * [Cloudant Python client library docs (readthedocs.io)](http://python-cloudant.readthedocs.io) * [Cloudant documentation](https://console.bluemix.net/docs/services/Cloudant/cloudant.html#overview) * [Cloudant Learning Center](https://developer.ibm.com/clouddataservices/cloudant-learning-center/) +* [Tutorial for creating and populating a database on IBM Cloud](https://console.bluemix.net/docs/services/Cloudant/tutorials/create_database.html#creating-and-populating-a-simple-cloudant-nosql-db-database-on-ibm-cloud) ## Development @@ -52,6 +52,8 @@ The preferred approach for using `python-cloudant` in other projects is to use t ### Examples in open source projects +[Getting Started with Python Flask on IBM Cloud](https://github.com/IBM-Cloud/get-started-python) + [Movie Recommender Demo](https://github.com/snowch/movie-recommender-demo): - [Update and check if documents exist](https://github.com/snowch/movie-recommender-demo/blob/master/web_app/app/dao.py#L162-L168) - [Connect to Cloudant using 429 backoff with 10 retries](https://github.com/snowch/movie-recommender-demo/blob/master/web_app/app/cloudant_db.py#L17-L18) From 337a91b10f34135271ae3bd4b01dca34087c0918 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Tue, 23 Jan 2018 16:06:22 +0000 Subject: [PATCH 064/185] Updated default IAM token URL --- CHANGES.md | 1 + docs/getting_started.rst | 2 +- src/cloudant/_client_session.py | 2 +- tests/unit/iam_auth_tests.py | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 25f4d492..3a734fa9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # 2.8.0 (Unreleased) +- [FIXED] Updated default IBM Cloud Identity and Access Management token URL. - [REMOVED] Removed broken source and target parameters that constantly threw `AttributeError` when creating a replication document. # 2.7.0 (2017-10-31) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index f4ce1303..b65dcca4 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -100,7 +100,7 @@ Cloud Platform. See `IBM Cloud Identity and Access Management `_ for more information. -The production IAM token service at *https://iam.bluemix.net/oidc/token* is used +The production IAM token service at *https://iam.bluemix.net/identity/token* is used by default. You can set an ``IAM_TOKEN_URL`` environment variable to override this. diff --git a/src/cloudant/_client_session.py b/src/cloudant/_client_session.py index 289b1317..93df4575 100644 --- a/src/cloudant/_client_session.py +++ b/src/cloudant/_client_session.py @@ -193,7 +193,7 @@ def __init__(self, api_key, server_url, **kwargs): self._api_key = api_key self._token_url = os.environ.get( - 'IAM_TOKEN_URL', 'https://iam.bluemix.net/oidc/token') + 'IAM_TOKEN_URL', 'https://iam.bluemix.net/identity/token') def login(self): """ diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py index d6b3c04c..6e0c7670 100644 --- a/tests/unit/iam_auth_tests.py +++ b/tests/unit/iam_auth_tests.py @@ -43,7 +43,7 @@ '2PTo4Exa17V-R_73Nq8VPCwpOvZcwKRA2sPTVgTMzU34max8b5kpTzVGJ' '6SXSItTVOUdAygZBng') -MOCK_OIDC_TOKEN_RESPONSE = { +MOCK_IAM_TOKEN_RESPONSE = { 'access_token': MOCK_ACCESS_TOKEN, 'refresh_token': ('MO61FKNvVRWkSa4vmBZqYv_Jt1kkGMUc-XzTcNnR-GnIhVKXHUWxJVV3' 'RddE8Kqh3X_TZRmyK8UySIWKxoJ2t6obUSUalPm90SBpTdoXtaljpNyo' @@ -100,7 +100,7 @@ def test_iam_set_credentials(self): @mock.patch('cloudant._client_session.ClientSession.request') def test_iam_get_access_token(self, m_req): m_response = mock.MagicMock() - m_response.json.return_value = MOCK_OIDC_TOKEN_RESPONSE + m_response.json.return_value = MOCK_IAM_TOKEN_RESPONSE m_req.return_value = m_response iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') From 0ea71883ab6067160032a2cc0a3c47f18591590e Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 13 Feb 2018 11:50:41 +0000 Subject: [PATCH 065/185] Support '/_search_disk_size' search endpoint --- CHANGES.md | 2 +- src/cloudant/design_document.py | 12 +++++++++ tests/unit/design_document_tests.py | 40 +++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3a734fa9..a1db5435 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ # 2.8.0 (Unreleased) - +- [NEW] Added support for `/_search_disk_size` endpoint which retrieves disk size information for a specific search index. - [FIXED] Updated default IBM Cloud Identity and Access Management token URL. - [REMOVED] Removed broken source and target parameters that constantly threw `AttributeError` when creating a replication document. diff --git a/src/cloudant/design_document.py b/src/cloudant/design_document.py index 3e460741..b32c5e33 100644 --- a/src/cloudant/design_document.py +++ b/src/cloudant/design_document.py @@ -699,3 +699,15 @@ def search_info(self, search_index): '/'.join([self.document_url, '_search_info', search_index])) ddoc_search_info.raise_for_status() return ddoc_search_info.json() + + def search_disk_size(self, search_index): + """ + Retrieves disk size information about a specified search index within + the design document, returns dictionary + + GET databasename/_design/{ddoc}/_search_disk_size/{search_index} + """ + ddoc_search_disk_size = self.r_session.get( + '/'.join([self.document_url, '_search_disk_size', search_index])) + ddoc_search_disk_size.raise_for_status() + return ddoc_search_disk_size.json() diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 44ede8fa..3fef4d0d 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -841,6 +841,46 @@ def test_get_search_info(self): self.assertTrue(search_index_metadata['pending_seq'] <= 101, 'The pending_seq should be 101 or fewer.') self.assertTrue(search_index_metadata['disk_size'] >0, 'The disk_size should be greater than 0.') + @unittest.skipUnless( + os.environ.get('RUN_CLOUDANT_TESTS') is not None, + 'Skipping Cloudant _search_disk_size endpoint test' + ) + def test_get_search_disk_size(self): + """ + Test retrieval of search_disk_size endpoint from the DesignDocument. + """ + self.populate_db_with_documents(100) + ddoc = DesignDocument(self.db, '_design/ddoc001') + ddoc.add_search_index( + 'search001', + 'function (doc) {\n index("default", doc._id); ' + 'if (doc._id) {index("name", doc.name, {"store": true}); }\n}' + ) + ddoc.save() + + ddoc_remote = DesignDocument(self.db, '_design/ddoc001') + ddoc_remote.fetch() + + ddoc_remote.search_info('search001') # trigger index build + + search_disk_size = ddoc_remote.search_disk_size('search001') + + self.assertEqual( + sorted(search_disk_size.keys()), ['name', 'search_index'], + 'The search disk size should contain only keys "name" and "search_index"') + self.assertEqual( + search_disk_size['name'], '_design/ddoc001/search001', + 'The search index "name" should be correct.') + self.assertEqual( + sorted(search_disk_size['search_index'].keys()), ['disk_size'], + 'The search index should contain only key "disk_size"') + self.assertTrue( + isinstance(search_disk_size['search_index']['disk_size'], int), + 'The "disk_size" value should be an integer.') + self.assertTrue( + search_disk_size['search_index']['disk_size'] > 0, + 'The "disk_size" should be greater than 0.') + @unittest.skipUnless( os.environ.get('RUN_CLOUDANT_TESTS') is not None, 'Skipping Cloudant _search_info raises HTTPError test' From 8192907852c5b529cf31c5b47cdd5087bfdb85b3 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 15 Feb 2018 10:43:23 +0000 Subject: [PATCH 066/185] Prepared for 2.8.0 release --- CHANGES.md | 3 ++- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a1db5435..d7fe0645 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,5 @@ -# 2.8.0 (Unreleased) +# 2.8.0 (2018-02-15) + - [NEW] Added support for `/_search_disk_size` endpoint which retrieves disk size information for a specific search index. - [FIXED] Updated default IBM Cloud Identity and Access Management token URL. - [REMOVED] Removed broken source and target parameters that constantly threw `AttributeError` when creating a replication document. diff --git a/VERSION b/VERSION index ceb9bd34..834f2629 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.8.0-SNAPSHOT +2.8.0 diff --git a/docs/conf.py b/docs/conf.py index 7f8c1883..d69d32db 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.8.0-SNAPSHOT' +version = '2.8.0' # The full version, including alpha/beta/rc tags. -release = '2.8.0-SNAPSHOT' +release = '2.8.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index eef4e184..36541aa9 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2017 IBM. All rights reserved. +# Copyright (c) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.8.0-SNAPSHOT' +__version__ = '2.8.0' # pylint: disable=wrong-import-position import contextlib From b7e4ccfc2f165f0185afd89e362c2811b525fb3e Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 15 Feb 2018 14:53:46 +0000 Subject: [PATCH 067/185] Updated version to 2.8.1-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 834f2629..f56f5bc1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.8.0 +2.8.1-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index d69d32db..7ec4258f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.8.0' +version = '2.8.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.8.0' +release = '2.8.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 36541aa9..4836de09 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.8.0' +__version__ = '2.8.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From 0f9cd5f869107defd58e3537ae3ced486b570304 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 16 Feb 2018 10:15:32 +0000 Subject: [PATCH 068/185] Added VERSION file to manifest --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index f9bd1455..96ebb5e1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include requirements.txt +include requirements.txt VERSION From 33622086f7cce5ace48fc994717a53ae6f76975e Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 16 Feb 2018 10:21:19 +0000 Subject: [PATCH 069/185] Prepared for version 2.8.1 release --- CHANGES.md | 4 ++++ VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d7fe0645..8a904e30 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# 2.8.1 (2018-02-16) + +- [FIXED] Installation failures of 2.8.0 caused by missing VERSION file in distribution. + # 2.8.0 (2018-02-15) - [NEW] Added support for `/_search_disk_size` endpoint which retrieves disk size information for a specific search index. diff --git a/VERSION b/VERSION index f56f5bc1..dbe59006 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.8.1-SNAPSHOT +2.8.1 diff --git a/docs/conf.py b/docs/conf.py index 7ec4258f..76e07274 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.8.1-SNAPSHOT' +version = '2.8.1' # The full version, including alpha/beta/rc tags. -release = '2.8.1-SNAPSHOT' +release = '2.8.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 4836de09..3fab4840 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.8.1-SNAPSHOT' +__version__ = '2.8.1' # pylint: disable=wrong-import-position import contextlib From d995bb6d9affa0ed85c150b531c560239bf213f8 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 16 Feb 2018 12:30:11 +0000 Subject: [PATCH 070/185] Updated version to 2.8.2-SNAPSHOT --- CHANGES.md | 2 ++ VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8a904e30..4f764fa7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,5 @@ +# Unreleased + # 2.8.1 (2018-02-16) - [FIXED] Installation failures of 2.8.0 caused by missing VERSION file in distribution. diff --git a/VERSION b/VERSION index dbe59006..dc3cd3a8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.8.1 +2.8.2-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index 76e07274..c4e0c7f8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.8.1' +version = '2.8.2-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.8.1' +release = '2.8.2-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 3fab4840..5bb47179 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.8.1' +__version__ = '2.8.2-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From a1631485091337ca8bd4550f36c0b475f732135a Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 20 Feb 2018 15:00:48 +0000 Subject: [PATCH 071/185] Add enforcing TLS 1.2 note to getting_started.rst --- docs/getting_started.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index b65dcca4..c825d4fe 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -476,3 +476,15 @@ Cloudant/CouchDB server. This example assumes that either a ``Cloudant`` or a # Display the response content print(response.json()) + +*************** +TLS 1.2 Support +*************** + +The TLS protocol is used to encrypt communications across a network to ensure +that transmitted data remains private. There are three released versions of TLS: +1.0, 1.1, and 1.2. All HTTPS connections use TLS. + +If your server enforces the use of TLS 1.2 then the python-cloudant client will +continue to work as expected (assuming you're running a version of +Python/OpenSSL that supports TLS 1.2). From b563aa4f51e76a674968eb93875a54e484fc45c6 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Mon, 23 Apr 2018 14:53:58 -0400 Subject: [PATCH 072/185] Updated Travis CI and unit tests to run against CouchDB 2.1.1 --- .travis.yml | 15 ++++- CHANGES.md | 2 + tests/unit/changes_tests.py | 41 +++++------- tests/unit/db_updates_tests.py | 113 +++++++++++++++++++-------------- tests/unit/unit_t_db_base.py | 33 +++++++++- tests/unit/view_tests.py | 22 +++---- 6 files changed, 135 insertions(+), 91 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4b2f9ae4..00fe9573 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ +sudo: required + language: python python: @@ -9,9 +11,20 @@ env: - ADMIN_PARTY=false services: - - couchdb + - docker + +before_install: + - docker pull apache/couchdb:2.1.1 + - docker run -d -it -p 5984:5984 apache/couchdb:2.1.1 --with-admin-party-please --with-haproxy install: "pip install -r requirements.txt && pip install -r test-requirements.txt" + +before_script: + # Make sure CouchDB is up + - while [ $? -ne 0 ]; do sleep 1 && curl -v http://localhost:5984; done + - curl -X PUT http://localhost:5984/_users + - curl -X PUT http://localhost:5984/_replicator + # command to run tests script: - pylint ./src/cloudant diff --git a/CHANGES.md b/CHANGES.md index 4f764fa7..b0a7d3f9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Unreleased +- [IMPROVED] Updated Travis CI and unit tests to run against CouchDB 2.1.1. + # 2.8.1 (2018-02-16) - [FIXED] Installation failures of 2.8.0 caused by missing VERSION file in distribution. diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index 8558c156..6f284221 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ from cloudant.feed import Feed from cloudant.document import Document from cloudant.design_document import DesignDocument -from cloudant.error import CloudantArgumentError, CloudantException +from cloudant.error import CloudantArgumentError from cloudant._2to3 import unicode_ from .unit_t_db_base import UnitTestDbBase @@ -100,12 +100,7 @@ def test_get_raw_content(self): self.assertIsInstance(raw_line, BYTETYPE) raw_content.append(raw_line) changes = json.loads(''.join([unicode_(x) for x in raw_content])) - if self.cloudant_test: - self.assertSetEqual( - set(changes.keys()), set(['results', 'last_seq', 'pending'])) - else: - self.assertSetEqual( - set(changes.keys()), set(['results', 'last_seq'])) + self.assertSetEqual(set(changes.keys()), set(['results', 'last_seq', 'pending'])) results = list() for result in changes['results']: self.assertSetEqual(set(result.keys()), set(['seq', 'changes', 'id'])) @@ -223,12 +218,11 @@ def test_get_raw_feed_with_heartbeat(self): def test_get_feed_descending(self): """ - Test getting content back for a descending feed. When testing with - Cloudant the sequence identifier is in the form of - -. Often times the number prefix sorts - as expected when using descending but sometimes the number prefix is - repeated. In these cases the check is to see if the following random - character sequence suffix is longer than its predecessor. + Test getting content back for a descending feed. When testing, the sequence + identifier is in the form of -. Often times + the number prefix sorts as expected when using descending but sometimes the + number prefix is repeated. In these cases the check is to see if the following + random character sequence suffix is longer than its predecessor. """ self.populate_db_with_documents(50) feed = Feed(self.db, descending=True) @@ -236,16 +230,13 @@ def test_get_feed_descending(self): last_seq = None for change in feed: if last_seq: - if self.cloudant_test: - current = int(change['seq'][0: change['seq'].find('-')]) - last = int(last_seq[0:last_seq.find('-')]) - try: - self.assertTrue(current < last) - except AssertionError: - self.assertEqual(current, last) - self.assertTrue(len(change['seq']) > len(last_seq)) - else: - self.assertTrue(change['seq'] < last_seq) + current = int(change['seq'][0: change['seq'].find('-')]) + last = int(last_seq[0:last_seq.find('-')]) + try: + self.assertTrue(current < last) + except AssertionError: + self.assertEqual(current, last) + self.assertTrue(len(change['seq']) > len(last_seq)) seq_list.append(change['seq']) last_seq = change['seq'] self.assertEqual(len(seq_list), 50) @@ -303,8 +294,6 @@ def test_get_feed_using_style_all_docs(self): changes = list() for change in feed: self.assertSetEqual(set(change.keys()), set(['seq', 'changes', 'id'])) - if not self.cloudant_test: - self.assertEqual(len(change['changes']), 2) changes.append(change) expected = set(['julia000', 'julia001', 'julia002']) self.assertSetEqual(set([x['id'] for x in changes]), expected) diff --git a/tests/unit/db_updates_tests.py b/tests/unit/db_updates_tests.py index 711b1937..c1df0d63 100644 --- a/tests/unit/db_updates_tests.py +++ b/tests/unit/db_updates_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,9 +22,7 @@ import os from cloudant.feed import Feed -from cloudant.document import Document -from cloudant.design_document import DesignDocument -from cloudant.error import CloudantArgumentError, CloudantException +from cloudant.error import CloudantArgumentError from cloudant._2to3 import unicode_ from .unit_t_db_base import UnitTestDbBase @@ -34,25 +32,63 @@ class DbUpdatesTestsBase(UnitTestDbBase): """ Common _db_updates tests methods """ - + def setUp(self): """ Set up test attributes """ super(DbUpdatesTestsBase, self).setUp() self.client.connect() + self.db_names = list() self.new_dbs = list() + self.create_db_updates() + self.create_dbs() def tearDown(self): """ Reset test attributes """ + test_dbs_deleted = False + changes = list() [db.delete() for db in self.new_dbs] + # Check the changes in the _db_updates feed to assert that the test databases are deleted + while not test_dbs_deleted: + feed = Feed(self.client, timeout=1000) + for change in feed: + if change['db_name'] in self.db_names and change['type'] == 'deleted': + changes.append(change) + if len(changes) == 2: + test_dbs_deleted = True + feed.stop() + self.delete_db_updates() self.client.disconnect() super(DbUpdatesTestsBase, self).tearDown() - def create_dbs(self, count=3): - self.new_dbs += [(self.client.create_database(self.dbname())) for x in range(count)] + def create_dbs(self): + self.db_names = [self.dbname() for x in range(2)] + self.new_dbs += [self.client.create_database(dbname) for dbname in self.db_names] + # Verify that all created databases are listed in _db_updates + all_dbs_exist = False + while not all_dbs_exist: + changes = list() + feed = Feed(self.client, timeout=1000) + for change in feed: + changes.append(change) + if len(changes) == 3: + all_dbs_exist = True + feed.stop() + + def assert_changes_in_db_updates_feed(self, changes): + """ + Assert that databases created in setup for db_updates_tests exist when looping through _db_updates feed + Note: During the creation of _global_changes database, a doc called '_dbs' is created and seen in _db_updates + """ + self.dbs = ['_dbs', self.new_dbs[0].database_name, self.new_dbs[1].database_name] + types = ['created', 'updated'] + for doc in changes: + self.assertIsNotNone(doc['seq']) + self.assertTrue(doc['db_name'] in self.dbs) + self.assertTrue(doc['type'] in types) @unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS'), 'Skipping CouchDB _db_updates feed tests') @@ -80,45 +116,25 @@ def test_stop_iteration_of_continuous_feed_with_heartbeat(self): feed = Feed(self.client, feed='continuous', timeout=100) changes = list() for change in feed: - if not change: - if not self.new_dbs: - self.create_dbs(5) - else: - continue - else: - changes.append(change) - if len(changes) == 3: - feed.stop() - self.assertEqual(len(self.new_dbs), 5) + changes.append(change) + if len(changes) == 3: + feed.stop() + self.assert_changes_in_db_updates_feed(changes) self.assertEqual(len(changes), 3) - self.assertDictEqual( - changes[0], {'db_name': self.new_dbs[0].database_name, 'type': 'created'}) - self.assertDictEqual( - changes[1], {'db_name': self.new_dbs[1].database_name, 'type': 'created'}) - self.assertDictEqual( - changes[2], {'db_name': self.new_dbs[2].database_name, 'type': 'created'}) def test_get_raw_content(self): """ Test getting raw feed content """ - feed = Feed(self.client, raw_data='True', feed='continuous', timeout=100) + feed = Feed(self.client, raw_data=True, feed='continuous', timeout=100) raw_content = list() for raw_line in feed: self.assertIsInstance(raw_line, BYTETYPE) - if not raw_line: - self.create_dbs(3) - else: - raw_content.append(raw_line) - if len(raw_content) == 3: - feed.stop() + raw_content.append(raw_line) + if len(raw_content) == 3: + feed.stop() changes = [json.loads(unicode_(x)) for x in raw_content] - self.assertDictEqual( - changes[0], {'db_name': self.new_dbs[0].database_name, 'type': 'created'}) - self.assertDictEqual( - changes[1], {'db_name': self.new_dbs[1].database_name, 'type': 'created'}) - self.assertDictEqual( - changes[2], {'db_name': self.new_dbs[2].database_name, 'type': 'created'}) + self.assert_changes_in_db_updates_feed(changes) def test_get_longpoll_feed_as_default(self): """ @@ -127,10 +143,10 @@ def test_get_longpoll_feed_as_default(self): feed = Feed(self.client, timeout=1000) changes = list() for change in feed: - self.assertIsNone(change) + self.assertIsNotNone(change) changes.append(change) - self.assertEqual(len(changes), 1) - self.assertIsNone(changes[0]) + self.assert_changes_in_db_updates_feed(changes) + self.assertEqual(len(changes), 3) def test_get_longpoll_feed_explicit(self): """ @@ -140,10 +156,10 @@ def test_get_longpoll_feed_explicit(self): feed = Feed(self.client, timeout=1000, feed='longpoll') changes = list() for change in feed: - self.assertIsNone(change) + self.assertIsNotNone(change) changes.append(change) - self.assertEqual(len(changes), 1) - self.assertIsNone(changes[0]) + self.assert_changes_in_db_updates_feed(changes) + self.assertEqual(len(changes), 3) def test_get_continuous_with_timeout(self): """ @@ -151,7 +167,14 @@ def test_get_continuous_with_timeout(self): and no heartbeat """ feed = Feed(self.client, feed='continuous', heartbeat=False, timeout=1000) - self.assertListEqual([x for x in feed], []) + changes = list() + for change in feed: + self.assertIsNotNone(change) + changes.append(change) + if len(changes) == 3: + feed.stop() + self.assert_changes_in_db_updates_feed(changes) + self.assertEqual(len(changes), 3) def test_invalid_argument(self): """ @@ -249,7 +272,6 @@ def test_stop_iteration_of_continuous_feed_using_since_now(self): feed = Feed(self.client, feed='continuous', since='now') count = 0 changes = list() - self.create_dbs(3) for change in feed: self.assertTrue(all(x in change for x in ('seq', 'type'))) changes.append(change) @@ -281,7 +303,6 @@ def test_get_normal_feed_default(self): Test getting content back for a "normal" feed without feed option. Also using limit since we don't know how many updates have occurred on client. """ - self.create_dbs(3) feed = Feed(self.client, limit=3) changes = list() for change in feed: @@ -296,7 +317,6 @@ def test_get_normal_feed_explicit(self): Test getting content back for a "normal" feed using feed option. Also using limit since we don't know how many updates have occurred on client. """ - self.create_dbs(3) feed = Feed(self.client, feed='normal', limit=3) changes = list() for change in feed: @@ -310,7 +330,6 @@ def test_get_longpoll_feed(self): """ Test getting content back for a "longpoll" feed """ - self.create_dbs(3) feed = Feed(self.client, feed='longpoll', limit=3) changes = list() for change in feed: diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index ccec69ad..38639cf1 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2016, 2017 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -65,6 +65,7 @@ from cloudant.client import CouchDB, Cloudant from cloudant.design_document import DesignDocument +from cloudant.error import CloudantClientException from .. import unicode_ @@ -100,14 +101,18 @@ def setUpClass(cls): return if os.environ.get('DB_USER') is None: + # Get couchdb docker node name + os.environ['NODENAME'] = requests.get( + '{0}/_membership'.format(os.environ['DB_URL'])).json()['all_nodes'][0] os.environ['DB_USER_CREATED'] = '1' os.environ['DB_USER'] = 'user-{0}'.format( unicode_(uuid.uuid4()) ) os.environ['DB_PASSWORD'] = 'password' resp = requests.put( - '{0}/_config/admins/{1}'.format( + '{0}/_node/{1}/_config/admins/{2}'.format( os.environ['DB_URL'], + os.environ['NODENAME'], os.environ['DB_USER'] ), data='"{0}"'.format(os.environ['DB_PASSWORD']) @@ -122,11 +127,12 @@ def tearDownClass(cls): if (os.environ.get('RUN_CLOUDANT_TESTS') is None and os.environ.get('DB_USER_CREATED') is not None): resp = requests.delete( - '{0}://{1}:{2}@{3}/_config/admins/{4}'.format( + '{0}://{1}:{2}@{3}/_node/{4}/_config/admins/{5}'.format( os.environ['DB_URL'].split('://', 1)[0], os.environ['DB_USER'], os.environ['DB_PASSWORD'], os.environ['DB_URL'].split('://', 1)[1], + os.environ['NODENAME'], os.environ['DB_USER'] ) ) @@ -333,3 +339,24 @@ def load_security_document_data(self): headers={'Content-Type': 'application/json'} ) self.assertEqual(resp.status_code, 200) + + def create_db_updates(self): + """ + Create '_global_changes' system database required for testing against _db_updates + """ + self.DB_UPDATES = '_global_changes' + try: + self.client.create_database(self.DB_UPDATES, throw_on_exists=True) + except CloudantClientException: + self.delete_db_updates() + self.create_db_updates() + + def delete_db_updates(self): + """ + Delete '_global_changes' system database used for _db_updates testing + """ + try: + self.client.delete_database(self.DB_UPDATES) + except CloudantClientException: + pass + diff --git a/tests/unit/view_tests.py b/tests/unit/view_tests.py index 35bac6e3..5e0f492e 100644 --- a/tests/unit/view_tests.py +++ b/tests/unit/view_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -309,19 +309,13 @@ def test_view_callable_with_invalid_javascript(self): 'view001', 'This is not valid Javascript' ) - ddoc.save() - # Verify that the ddoc and view were saved remotely - # along with the invalid Javascript - del ddoc - ddoc = DesignDocument(self.db, 'ddoc001') - ddoc.fetch() - view = ddoc.get_view('view001') - self.assertEqual(view.map, 'This is not valid Javascript') - try: - for row in view.result: - self.fail('Above statement should raise an Exception') - except requests.HTTPError as err: - self.assertEqual(err.response.status_code, 500) + with self.assertRaises(requests.HTTPError) as cm: + ddoc.save() + err = cm.exception + self.assertTrue(str(err).startswith( + '400 Client Error: Bad Request compilation_error Compilation of the map function ' + 'in the \'view001\' view failed: Expression does not eval to a function.' + )) def test_custom_result_context_manager(self): """ From 7769375f3c6ff32fdf07a8c38aee43518a820ff0 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Thu, 26 Apr 2018 00:04:33 -0400 Subject: [PATCH 073/185] Added skip_if_iam to skip tests when running with IAM auth --- tests/unit/database_tests.py | 5 +++-- tests/unit/design_document_tests.py | 3 ++- tests/unit/unit_t_db_base.py | 7 +++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index 9a35c3ed..3c71bd3a 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -38,7 +38,7 @@ from cloudant.feed import Feed, InfiniteFeed from tests.unit._test_util import LONG_NUMBER -from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase +from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase, skip_if_iam from .. import unicode_ class CloudantDatabaseExceptionTests(unittest.TestCase): @@ -863,7 +863,7 @@ def test_get_show_result(self): resp, 'Hello from doc001!' ) - + @skip_if_iam def test_create_doc_with_update_handler(self): """ Test update_handler_result executes an update handler function @@ -884,6 +884,7 @@ def test_create_doc_with_update_handler(self): 'Created new doc: {"message":"hello","_id":"testDoc"}' ) + @skip_if_iam def test_update_doc_with_update_handler(self): """ Test update_handler_result executes an update handler function diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 3fef4d0d..6cb14281 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -32,7 +32,7 @@ from cloudant.view import View, QueryIndexView from cloudant.error import CloudantArgumentError, CloudantDesignDocumentException -from .unit_t_db_base import UnitTestDbBase +from .unit_t_db_base import UnitTestDbBase, skip_if_iam class CloudantDesignDocumentExceptionTests(unittest.TestCase): """ @@ -1252,6 +1252,7 @@ def test_get_search_index(self): '{"store": true}); }\n}'} ) + @skip_if_iam def test_rewrite_rule(self): """ Test that design document URL is rewritten to the expected test document. diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index 38639cf1..d1b51457 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -78,6 +78,13 @@ def wrapper(*args): return wrapper +def skip_if_iam(f): + def wrapper(*args): + if os.environ.get('IAM_API_KEY'): + raise unittest.SkipTest('Test only supports non-IAM authentication') + return f(*args) + return wrapper + class UnitTestDbBase(unittest.TestCase): """ The base class for all unit tests targeting a database From 3e54fcd6187e3e9be601beaf640cf12a99b57154 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Tue, 1 May 2018 13:58:22 -0400 Subject: [PATCH 074/185] Added support for IAM API key in cloudant_bluemix method --- CHANGES.md | 1 + src/cloudant/__init__.py | 1 + src/cloudant/_common_util.py | 16 ++++- src/cloudant/_messages.py | 1 + src/cloudant/client.py | 18 +++-- tests/unit/client_tests.py | 93 +++++++++++++++++++++++-- tests/unit/cloud_foundry_tests.py | 110 +++++++++++++++++++++++------- tests/unit/unit_t_db_base.py | 5 +- 8 files changed, 203 insertions(+), 42 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b0a7d3f9..f2d6da36 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # Unreleased +- [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. - [IMPROVED] Updated Travis CI and unit tests to run against CouchDB 2.1.1. # 2.8.1 (2018-02-16) diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 5bb47179..51018794 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -116,6 +116,7 @@ def cloudant_bluemix(vcap_services, instance_name=None, service_name=None, **kwa "cloudantNoSQLDB": [ { "credentials": { + "apikey": "some123api456key" "username": "example", "password": "xxxxxxx", "host": "example.cloudant.com", diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index e44df3d5..9e20d226 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2017 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import json from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_ -from .error import CloudantArgumentError, CloudantException +from .error import CloudantArgumentError, CloudantException, CloudantClientException # Library Constants @@ -306,9 +306,14 @@ def __init__(self, vcap_services, instance_name=None, service_name=None): credentials = service['credentials'] self._host = credentials['host'] self._name = service.get('name') - self._password = credentials['password'] self._port = credentials.get('port', 443) self._username = credentials['username'] + if 'apikey' in credentials: + self._iam_api_key = credentials['apikey'] + elif 'username' in credentials and 'password' in credentials: + self._password = credentials['password'] + else: + raise CloudantClientException(103) break else: raise CloudantException('Missing service in VCAP_SERVICES') @@ -355,3 +360,8 @@ def url(self): def username(self): """ Return service username. """ return self._username + + @property + def iam_api_key(self): + """ Return service IAM API key. """ + return self._iam_api_key diff --git a/src/cloudant/_messages.py b/src/cloudant/_messages.py index 2b6e9257..07a23d62 100644 --- a/src/cloudant/_messages.py +++ b/src/cloudant/_messages.py @@ -72,6 +72,7 @@ 100: 'A general Cloudant client exception was raised.', 101: 'Value must be set to a Database object. Found type: {0}', 102: 'You must provide a url or an account.', + 103: 'Invalid service: IAM API key or username/password credentials are required.', 404: 'Database {0} does not exist. Verify that the client is valid and try again.', 412: 'Database {0} already exists.' } diff --git a/src/cloudant/client.py b/src/cloudant/client.py index abf3f5f5..59d25b2c 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2017 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ from .error import ( CloudantArgumentError, CloudantClientException, - CloudantDatabaseException) + CloudantDatabaseException, CloudantException) from ._common_util import ( USER_AGENT, append_response_error_content, @@ -788,9 +788,17 @@ def bluemix(cls, vcap_services, instance_name=None, service_name=None, **kwargs) print client.all_dbs() """ service_name = service_name or 'cloudantNoSQLDB' # default service - service = CloudFoundryService(vcap_services, - instance_name=instance_name, - service_name=service_name) + try: + service = CloudFoundryService(vcap_services, + instance_name=instance_name, + service_name=service_name) + except CloudantException: + raise CloudantClientException(103) + + if hasattr(service, 'iam_api_key'): + return Cloudant.iam(service.username, + service.iam_api_key, + url=service.url) return Cloudant(service.username, service.password, url=service.url, diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 114a9e3a..d79e0c6f 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -633,9 +633,9 @@ def test_cloudant_context_helper(self): self.fail('Exception {0} was raised.'.format(str(err))) @skip_if_not_cookie_auth - def test_cloudant_bluemix_context_helper(self): + def test_cloudant_bluemix_context_helper_with_legacy_creds(self): """ - Test that the cloudant_bluemix context helper works as expected. + Test that the cloudant_bluemix context helper with legacy creds works as expected. """ instance_name = 'Cloudant NoSQL DB-lv' vcap_services = {'cloudantNoSQLDB': [{ @@ -657,6 +657,56 @@ def test_cloudant_bluemix_context_helper(self): except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) + @unittest.skipUnless(os.environ.get('IAM_API_KEY'), + 'Skipping Cloudant Bluemix context helper with IAM test') + def test_cloudant_bluemix_context_helper_with_iam(self): + """ + Test that the cloudant_bluemix context helper with IAM works as expected. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [{ + 'credentials': { + 'apikey': self.iam_api_key, + 'username': self.user, + 'host': '{0}.cloudant.com'.format(self.account), + 'port': 443, + 'url': self.url + }, + 'name': instance_name, + }]} + + try: + with cloudant_bluemix(vcap_services, instance_name=instance_name) as c: + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + + def test_cloudant_bluemix_context_helper_raise_error_for_missing_iam_and_creds(self): + """ + Test that the cloudant_bluemix context helper raises a CloudantClientException + when the IAM key, username, and password are missing in the VCAP_SERVICES env variable. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [{ + 'credentials': { + 'host': '{0}.cloudant.com'.format(self.account), + 'port': 443, + 'url': self.url + }, + 'name': instance_name, + }]} + + try: + with cloudant_bluemix(vcap_services, instance_name=instance_name) as c: + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + except CloudantClientException as err: + self.assertEqual( + 'Invalid service: IAM API key or username/password credentials are required.', + str(err) + ) + def test_cloudant_bluemix_dedicated_context_helper(self): """ Test that the cloudant_bluemix context helper works as expected when @@ -698,7 +748,7 @@ def test_constructor_with_account(self): ) @skip_if_not_cookie_auth - def test_bluemix_constructor(self): + def test_bluemix_constructor_with_legacy_creds(self): """ Test instantiating a client object using a VCAP_SERVICES environment variable. @@ -730,7 +780,39 @@ def test_bluemix_constructor(self): finally: c.disconnect() - @skip_if_not_cookie_auth + + @unittest.skipUnless(os.environ.get('IAM_API_KEY'), + 'Skipping Cloudant Bluemix constructor with IAM test') + def test_bluemix_constructor_with_iam(self): + """ + Test instantiating a client object using a VCAP_SERVICES environment + variable. + """ + instance_name = 'Cloudant NoSQL DB-lv' + vcap_services = {'cloudantNoSQLDB': [{ + 'credentials': { + 'apikey': self.iam_api_key, + 'username': self.user, + 'host': '{0}.cloudant.com'.format(self.account), + 'port': 443 + }, + 'name': instance_name + }]} + + # create Cloudant Bluemix client + c = Cloudant.bluemix(vcap_services) + + try: + c.connect() + self.assertIsInstance(c, Cloudant) + self.assertIsInstance(c.r_session, requests.Session) + + except Exception as err: + self.fail('Exception {0} was raised.'.format(str(err))) + + finally: + c.disconnect() + def test_bluemix_constructor_specify_instance_name(self): """ Test instantiating a client object using a VCAP_SERVICES environment @@ -773,8 +855,7 @@ def test_bluemix_constructor_with_multiple_services(self): vcap_services = {'cloudantNoSQLDB': [ { 'credentials': { - 'username': self.user, - 'password': self.pwd, + 'apikey': '1234api', 'host': '{0}.cloudant.com'.format(self.account), 'port': 443, 'url': self.url diff --git a/tests/unit/cloud_foundry_tests.py b/tests/unit/cloud_foundry_tests.py index 43249b75..a832e4ce 100644 --- a/tests/unit/cloud_foundry_tests.py +++ b/tests/unit/cloud_foundry_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,20 +29,30 @@ class CloudFoundryServiceTests(unittest.TestCase): def __init__(self, *args, **kwargs): super(CloudFoundryServiceTests, self).__init__(*args, **kwargs) - self._test_vcap_services_single = json.dumps({'cloudantNoSQLDB': [ - { + self._test_vcap_services_single_legacy_credentials_enabled = json.dumps({'cloudantNoSQLDB': [{ + 'name': 'Cloudant NoSQL DB 1', # valid service with legacy creds enabled + 'credentials': { + 'apikey': '1234api', + 'username': 'user-bluemix', + 'password': 'password', + 'port': 443, + 'host': 'user-bluemix.cloudant.com' + } + } + ]}) + self._test_vcap_services_single = json.dumps({'cloudantNoSQLDB': [{ 'name': 'Cloudant NoSQL DB 1', # valid service 'credentials': { - 'host': 'example.cloudant.com', - 'password': 'pa$$w0rd01', - 'port': 1234, - 'username': 'example' + 'apikey': '1234api', + 'username': 'user-bluemix', + 'port': 443, + 'host': 'user-bluemix.cloudant.com' } } ]}) - self._test_vcap_services_multiple = json.dumps({'cloudantNoSQLDB': [ + self._test_legacy_vcap_services_multiple = json.dumps({'cloudantNoSQLDB': [ { - 'name': 'Cloudant NoSQL DB 1', # valid service + 'name': 'Cloudant NoSQL DB 1', # valid legacy service 'credentials': { 'host': 'example.cloudant.com', 'password': 'pa$$w0rd01', @@ -89,7 +99,24 @@ def __init__(self, *args, **kwargs): 'pa$$w0rd01', 'example' ] - } + }, + { + 'name': 'Cloudant NoSQL DB 7', # missing iam api key and creds + 'credentials': { + 'host': 'example.cloudant.com', + 'port': 1234, + 'username': 'example' + } + }, + { + 'name': 'Cloudant NoSQL DB 8', # valid service with IAM api + 'credentials': { + 'apikey': '1234api', + 'username': 'example', + 'host': 'example.cloudant.com', + 'port': 1234 + } + }, ]}) self._test_vcap_services_dedicated = json.dumps({ 'cloudantNoSQLDB Dedicated': [ # dedicated service name @@ -105,16 +132,27 @@ def __init__(self, *args, **kwargs): ] }) - def test_get_vcap_service_default_success(self): + def test_get_vcap_service_legacy_creds_success(self): + service = CloudFoundryService( + self._test_vcap_services_single_legacy_credentials_enabled, + service_name='cloudantNoSQLDB' + ) + self.assertEqual('Cloudant NoSQL DB 1', service.name) + + def test_get_vcap_service_iam_api_no_creds_success(self): service = CloudFoundryService( self._test_vcap_services_single, service_name='cloudantNoSQLDB' ) self.assertEqual('Cloudant NoSQL DB 1', service.name) + self.assertEqual('1234api', service.iam_api_key) + with self.assertRaises(AttributeError) as cm: + service.password + self.assertEqual("'CloudFoundryService' object has no attribute '_password'", str(cm.exception)) def test_get_vcap_service_default_success_as_dict(self): service = CloudFoundryService( - json.loads(self._test_vcap_services_single), + json.loads(self._test_vcap_services_single_legacy_credentials_enabled), service_name='cloudantNoSQLDB' ) self.assertEqual('Cloudant NoSQL DB 1', service.name) @@ -122,14 +160,14 @@ def test_get_vcap_service_default_success_as_dict(self): def test_get_vcap_service_default_failure_multiple_services(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, + self._test_legacy_vcap_services_multiple, service_name='cloudantNoSQLDB' ) self.assertEqual('Missing service in VCAP_SERVICES', str(cm.exception)) def test_get_vcap_service_instance_host(self): service = CloudFoundryService( - self._test_vcap_services_multiple, + self._test_legacy_vcap_services_multiple, instance_name='Cloudant NoSQL DB 1', service_name='cloudantNoSQLDB' ) @@ -137,7 +175,7 @@ def test_get_vcap_service_instance_host(self): def test_get_vcap_service_instance_password(self): service = CloudFoundryService( - self._test_vcap_services_multiple, + self._test_legacy_vcap_services_multiple, instance_name='Cloudant NoSQL DB 1', service_name='cloudantNoSQLDB' ) @@ -145,7 +183,7 @@ def test_get_vcap_service_instance_password(self): def test_get_vcap_service_instance_port(self): service = CloudFoundryService( - self._test_vcap_services_multiple, + self._test_legacy_vcap_services_multiple, instance_name='Cloudant NoSQL DB 1', service_name='cloudantNoSQLDB' ) @@ -153,7 +191,7 @@ def test_get_vcap_service_instance_port(self): def test_get_vcap_service_instance_port_default(self): service = CloudFoundryService( - self._test_vcap_services_multiple, + self._test_legacy_vcap_services_multiple, instance_name='Cloudant NoSQL DB 2', service_name='cloudantNoSQLDB' ) @@ -161,7 +199,7 @@ def test_get_vcap_service_instance_port_default(self): def test_get_vcap_service_instance_url(self): service = CloudFoundryService( - self._test_vcap_services_multiple, + self._test_legacy_vcap_services_multiple, instance_name='Cloudant NoSQL DB 1', service_name='cloudantNoSQLDB' ) @@ -169,16 +207,24 @@ def test_get_vcap_service_instance_url(self): def test_get_vcap_service_instance_username(self): service = CloudFoundryService( - self._test_vcap_services_multiple, + self._test_legacy_vcap_services_multiple, instance_name='Cloudant NoSQL DB 1', service_name='cloudantNoSQLDB' ) self.assertEqual('example', service.username) + def test_get_vcap_service_instance_iam_api_key(self): + service = CloudFoundryService( + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 8', + service_name='cloudantNoSQLDB' + ) + self.assertEqual('1234api', service.iam_api_key) + def test_raise_error_for_missing_host(self): with self.assertRaises(CloudantException): CloudFoundryService( - self._test_vcap_services_multiple, + self._test_legacy_vcap_services_multiple, instance_name='Cloudant NoSQL DB 3', service_name='cloudantNoSQLDB' ) @@ -186,19 +232,19 @@ def test_raise_error_for_missing_host(self): def test_raise_error_for_missing_password(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, + self._test_legacy_vcap_services_multiple, instance_name='Cloudant NoSQL DB 4', service_name='cloudantNoSQLDB' ) self.assertEqual( - "Invalid service: 'password' missing", + 'Invalid service: IAM API key or username/password credentials are required.', str(cm.exception) ) def test_raise_error_for_missing_username(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, + self._test_legacy_vcap_services_multiple, instance_name='Cloudant NoSQL DB 5', service_name='cloudantNoSQLDB' ) @@ -210,7 +256,7 @@ def test_raise_error_for_missing_username(self): def test_raise_error_for_invalid_credentials_type(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, + self._test_legacy_vcap_services_multiple, instance_name='Cloudant NoSQL DB 6', service_name='cloudantNoSQLDB' ) @@ -219,13 +265,25 @@ def test_raise_error_for_invalid_credentials_type(self): str(cm.exception) ) - def test_raise_error_for_missing_service(self): + def test_raise_error_for_missing_iam_api_key_and_credentials(self): with self.assertRaises(CloudantException) as cm: CloudFoundryService( - self._test_vcap_services_multiple, + self._test_legacy_vcap_services_multiple, instance_name='Cloudant NoSQL DB 7', service_name='cloudantNoSQLDB' ) + self.assertEqual( + 'Invalid service: IAM API key or username/password credentials are required.', + str(cm.exception) + ) + + def test_raise_error_for_missing_service(self): + with self.assertRaises(CloudantException) as cm: + CloudFoundryService( + self._test_legacy_vcap_services_multiple, + instance_name='Cloudant NoSQL DB 9', + service_name='cloudantNoSQLDB' + ) self.assertEqual('Missing service in VCAP_SERVICES', str(cm.exception)) def test_raise_error_for_invalid_vcap(self): diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index d1b51457..fc41ae9d 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -158,6 +158,7 @@ def set_up_client(self, auto_connect=False, auto_renew=False, encoder=None, self.user = os.environ.get('DB_USER', None) self.pwd = os.environ.get('DB_PASSWORD', None) self.use_cookie_auth = True + self.iam_api_key = os.environ.get('IAM_API_KEY', None) if os.environ.get('RUN_CLOUDANT_TESTS') is None: self.url = os.environ['DB_URL'] @@ -198,12 +199,12 @@ def set_up_client(self, auto_connect=False, auto_renew=False, encoder=None, timeout=timeout, use_basic_auth=True, ) - elif os.environ.get('IAM_API_KEY'): + elif self.iam_api_key: self.use_cookie_auth = False # construct Cloudant client (using IAM authentication) self.client = Cloudant( None, # username is not required - os.environ.get('IAM_API_KEY'), + self.iam_api_key, url=self.url, x_cloudant_user=self.account, connect=auto_connect, From bb729e6277bca216b1772cd90f201dad081ebbc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Sun, 11 Mar 2018 17:06:34 +0400 Subject: [PATCH 075/185] Move create_query_index and others to CouchDatabase class, add support for 'key in db' [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x. [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict `__contains__` and checking in the remote database. --- CHANGES.md | 2 + docs/getting_started.rst | 13 ++ src/cloudant/database.py | 277 +++++++++++++++++++---------------- tests/unit/database_tests.py | 88 ++++++----- 4 files changed, 218 insertions(+), 162 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f2d6da36..27f5b9b6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Unreleased +- [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x. +- [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict `__contains__` and checking in the remote database. - [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. - [IMPROVED] Updated Travis CI and unit tests to run against CouchDB 2.1.1. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index c825d4fe..e899eff4 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -270,6 +270,19 @@ classes are sub-classes of ``dict``, this is accomplished through standard # Display the document print(my_document) +Checking if a document exists +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can check if a document exists in a database the same way you would check +if a ``dict`` has a key-value pair by key. + +.. code-block:: python + + doc_exists = 'julia30' in my_database + + if doc_exists: + print('document with _id julia30 exists') + Retrieve all documents ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 930673f2..e5e94f1e 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -608,6 +608,32 @@ def __getitem__(self, key): else: raise KeyError(key) + def __contains__(self, key): + """ + Overrides dictionary __contains__ behavior to check if a document + by key exists in the current cached or remote database. + + For example: + + .. code-block:: python + + if key in database: + doc = database[key] + # Do something with doc + + :param str key: Document id used to check if it exists in the database. + + :returns: True if the document exists in the local or remote + database, otherwise False. + """ + if key in list(self.keys()): + return True + if key.startswith('_design/'): + doc = DesignDocument(self, key) + else: + doc = Document(self, key) + return doc.exists() + def __iter__(self, remote=True): """ Overrides dictionary __iter__ behavior to provide iterable Document @@ -921,131 +947,6 @@ def update_handler_result(self, ddoc_id, handler_name, doc_id=None, data=None, * resp.raise_for_status() return resp.text -class CloudantDatabase(CouchDatabase): - """ - Encapsulates a Cloudant database. A CloudantDatabase object is - instantiated with a reference to a client/session. - It supports accessing the documents, and various database - features such as the document indexes, changes feed, design documents, etc. - - :param Cloudant client: Client instance used by the database. - :param str database_name: Database name used to reference the database. - :param int fetch_limit: Optional fetch limit used to set the max number of - documents to fetch per query during iteration cycles. Defaults to 100. - """ - def __init__(self, client, database_name, fetch_limit=100): - super(CloudantDatabase, self).__init__( - client, - database_name, - fetch_limit=fetch_limit - ) - - def security_document(self): - """ - Retrieves the security document for the current database - containing information about the users that the database - is shared with. - - :returns: Security document as a ``dict`` - """ - return dict(self.get_security_document()) - - @property - def security_url(self): - """ - Constructs and returns the security document URL. - - :returns: Security document URL - """ - url = '/'.join((self._database_host, '_api', 'v2', 'db', - self.database_name, '_security')) - return url - - def share_database(self, username, roles=None): - """ - Shares the current remote database with the username provided. - You can grant varying degrees of access rights, - default is to share read-only, but additional - roles can be added by providing the specific roles as a - ``list`` argument. If the user already has this database shared with - them then it will modify/overwrite the existing permissions. - - :param str username: Cloudant user to share the database with. - :param list roles: A list of - `roles - `_ - to grant to the named user. - - :returns: Share database status in JSON format - """ - if roles is None: - roles = ['_reader'] - valid_roles = [ - '_reader', - '_writer', - '_admin', - '_replicator', - '_db_updates', - '_design', - '_shards', - '_security' - ] - doc = self.security_document() - data = doc.get('cloudant', {}) - perms = [] - if all(role in valid_roles for role in roles): - perms = list(set(roles)) - - if not perms: - raise CloudantArgumentError(102, roles, valid_roles) - - data[username] = perms - doc['cloudant'] = data - resp = self.r_session.put( - self.security_url, - data=json.dumps(doc, cls=self.client.encoder), - headers={'Content-Type': 'application/json'} - ) - resp.raise_for_status() - return resp.json() - - def unshare_database(self, username): - """ - Removes all sharing with the named user for the current remote database. - This will remove the entry for the user from the security document. - To modify permissions, use the - :func:`~cloudant.database.CloudantDatabase.share_database` method - instead. - - :param str username: Cloudant user to unshare the database from. - - :returns: Unshare database status in JSON format - """ - doc = self.security_document() - data = doc.get('cloudant', {}) - if username in data: - del data[username] - doc['cloudant'] = data - resp = self.r_session.put( - self.security_url, - data=json.dumps(doc, cls=self.client.encoder), - headers={'Content-Type': 'application/json'} - ) - resp.raise_for_status() - return resp.json() - - def shards(self): - """ - Retrieves information about the shards in the current remote database. - - :returns: Shard information retrieval status in JSON format - """ - url = '/'.join((self.database_url, '_shards')) - resp = self.r_session.get(url) - resp.raise_for_status() - - return resp.json() - def get_query_indexes(self, raw_result=False): """ Retrieves query indexes from the remote database. @@ -1242,6 +1143,132 @@ def get_query_result(self, selector, fields=None, raw_result=False, return query.result + +class CloudantDatabase(CouchDatabase): + """ + Encapsulates a Cloudant database. A CloudantDatabase object is + instantiated with a reference to a client/session. + It supports accessing the documents, and various database + features such as the document indexes, changes feed, design documents, etc. + + :param Cloudant client: Client instance used by the database. + :param str database_name: Database name used to reference the database. + :param int fetch_limit: Optional fetch limit used to set the max number of + documents to fetch per query during iteration cycles. Defaults to 100. + """ + def __init__(self, client, database_name, fetch_limit=100): + super(CloudantDatabase, self).__init__( + client, + database_name, + fetch_limit=fetch_limit + ) + + def security_document(self): + """ + Retrieves the security document for the current database + containing information about the users that the database + is shared with. + + :returns: Security document as a ``dict`` + """ + return dict(self.get_security_document()) + + @property + def security_url(self): + """ + Constructs and returns the security document URL. + + :returns: Security document URL + """ + url = '/'.join((self._database_host, '_api', 'v2', 'db', + self.database_name, '_security')) + return url + + def share_database(self, username, roles=None): + """ + Shares the current remote database with the username provided. + You can grant varying degrees of access rights, + default is to share read-only, but additional + roles can be added by providing the specific roles as a + ``list`` argument. If the user already has this database shared with + them then it will modify/overwrite the existing permissions. + + :param str username: Cloudant user to share the database with. + :param list roles: A list of + `roles + `_ + to grant to the named user. + + :returns: Share database status in JSON format + """ + if roles is None: + roles = ['_reader'] + valid_roles = [ + '_reader', + '_writer', + '_admin', + '_replicator', + '_db_updates', + '_design', + '_shards', + '_security' + ] + doc = self.security_document() + data = doc.get('cloudant', {}) + perms = [] + if all(role in valid_roles for role in roles): + perms = list(set(roles)) + + if not perms: + raise CloudantArgumentError(102, roles, valid_roles) + + data[username] = perms + doc['cloudant'] = data + resp = self.r_session.put( + self.security_url, + data=json.dumps(doc, cls=self.client.encoder), + headers={'Content-Type': 'application/json'} + ) + resp.raise_for_status() + return resp.json() + + def unshare_database(self, username): + """ + Removes all sharing with the named user for the current remote database. + This will remove the entry for the user from the security document. + To modify permissions, use the + :func:`~cloudant.database.CloudantDatabase.share_database` method + instead. + + :param str username: Cloudant user to unshare the database from. + + :returns: Unshare database status in JSON format + """ + doc = self.security_document() + data = doc.get('cloudant', {}) + if username in data: + del data[username] + doc['cloudant'] = data + resp = self.r_session.put( + self.security_url, + data=json.dumps(doc, cls=self.client.encoder), + headers={'Content-Type': 'application/json'} + ) + resp.raise_for_status() + return resp.json() + + def shards(self): + """ + Retrieves information about the shards in the current remote database. + + :returns: Shard information retrieval status in JSON format + """ + url = '/'.join((self.database_url, '_shards')) + resp = self.r_session.get(url) + resp.raise_for_status() + + return resp.json() + def get_search_result(self, ddoc_id, index_name, **query_params): """ Retrieves the raw JSON content from the remote database based on the diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index 3c71bd3a..bd5bd4fd 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -502,6 +502,20 @@ def test_keys(self): ['julia000', 'julia001', 'julia002'] ) + def test_doc_id_in_db(self): + """ + Test checking if a document exists in a DB with in operator + """ + self.populate_db_with_documents(1) + self.assertTrue('julia000' in self.db) + + def test_doc_id_not_in_db(self): + """ + Test checking if a document exists in a DB with in operator + """ + self.populate_db_with_documents(1) + self.assertFalse('julia001' in self.db) + def test_get_non_existing_doc_via_getitem(self): """ Test __getitem__ when retrieving a non-existing document @@ -954,6 +968,43 @@ def test_database_request_fails_after_client_disconnects(self): finally: self.client.connect() + def test_create_json_index(self): + """ + Ensure that a JSON index is created as expected. + """ + index = self.db.create_query_index(fields=['name', 'age']) + self.assertIsInstance(index, Index) + + ddoc = self.db[index.design_document_id] + + self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertTrue(ddoc['_rev'].startswith('1-')) + + self.assertEquals(ddoc['indexes'], {}) + self.assertEquals(ddoc['language'], 'query') + self.assertEquals(ddoc['lists'], {}) + self.assertEquals(ddoc['shows'], {}) + + index = ddoc['views'][index.name] + self.assertEquals(index['map']['fields']['age'], 'asc') + self.assertEquals(index['map']['fields']['name'], 'asc') + self.assertEquals(index['options']['def']['fields'], ['name', 'age']) + self.assertEquals(index['reduce'], '_count') + + def test_delete_json_index(self): + """ + Ensure that a JSON index is deleted as expected. + """ + index = self.db.create_query_index( + 'ddoc001', + 'index001', + fields=['name', 'age']) + self.assertIsInstance(index, Index) + ddoc = self.db['_design/ddoc001'] + self.assertTrue(ddoc.exists()) + self.db.delete_query_index('ddoc001', 'json', 'index001') + self.assertFalse(ddoc.exists()) + @unittest.skipUnless( os.environ.get('RUN_CLOUDANT_TESTS') is not None, 'Skipping Cloudant specific Database tests' @@ -1200,29 +1251,6 @@ def test_get_query_result_with_empty_fields_list(self): ['julia001', 'julia002', 'julia003', 'julia004'] ) - def test_create_json_index(self): - """ - Ensure that a JSON index is created as expected. - """ - index = self.db.create_query_index(fields=['name', 'age']) - self.assertIsInstance(index, Index) - - ddoc = self.db[index.design_document_id] - - self.assertEquals(ddoc['_id'], index.design_document_id) - self.assertTrue(ddoc['_rev'].startswith('1-')) - - self.assertEquals(ddoc['indexes'], {}) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) - - index = ddoc['views'][index.name] - self.assertEquals(index['map']['fields']['age'], 'asc') - self.assertEquals(index['map']['fields']['name'], 'asc') - self.assertEquals(index['options']['def']['fields'], ['name', 'age']) - self.assertEquals(index['reduce'], '_count') - def test_create_text_index(self): """ Ensure that a text index is created as expected. @@ -1345,20 +1373,6 @@ def test_create_query_index_failure(self): 'Index type must be either \"json\" or \"text\".' ) - def test_delete_json_index(self): - """ - Ensure that a JSON index is deleted as expected. - """ - index = self.db.create_query_index( - 'ddoc001', - 'index001', - fields=['name', 'age']) - self.assertIsInstance(index, Index) - ddoc = self.db['_design/ddoc001'] - self.assertTrue(ddoc.exists()) - self.db.delete_query_index('ddoc001', 'json', 'index001') - self.assertFalse(ddoc.exists()) - def test_delete_text_index(self): """ Ensure that a text index is deleted as expected. From cb6998ca86e9416e276fc0d1802fa5fd518508b6 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 13 Mar 2018 07:31:43 +0000 Subject: [PATCH 076/185] Support IAM authentication in replication documents --- CHANGES.md | 3 +- src/cloudant/_client_session.py | 11 +- src/cloudant/client.py | 11 ++ src/cloudant/replicator.py | 48 +++++--- tests/unit/replicator_mock_tests.py | 175 ++++++++++++++++++++++++++++ tests/unit/replicator_tests.py | 10 +- 6 files changed, 235 insertions(+), 23 deletions(-) create mode 100644 tests/unit/replicator_mock_tests.py diff --git a/CHANGES.md b/CHANGES.md index 27f5b9b6..ab805d4e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,8 @@ # Unreleased -- [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x. - [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict `__contains__` and checking in the remote database. +- [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x. +- [NEW] Support IAM authentication in replication documents. - [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. - [IMPROVED] Updated Travis CI and unit tests to run against CouchDB 2.1.1. diff --git a/src/cloudant/_client_session.py b/src/cloudant/_client_session.py index 93df4575..0e8fd386 100644 --- a/src/cloudant/_client_session.py +++ b/src/cloudant/_client_session.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2017 IBM Corp. All rights reserved. +# Copyright (c) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -195,6 +195,15 @@ def __init__(self, api_key, server_url, **kwargs): self._token_url = os.environ.get( 'IAM_TOKEN_URL', 'https://iam.bluemix.net/identity/token') + @property + def get_api_key(self): + """ + Get IAM API key. + + :return: IAM API key. + """ + return self._api_key + def login(self): """ Perform IAM cookie based user login. diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 59d25b2c..de88112f 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -98,6 +98,15 @@ def __init__(self, user, auth_token, admin_party=False, **kwargs): if connect_to_couch and self._DATABASE_CLASS == CouchDatabase: self.connect() + @property + def is_iam_authenticated(self): + """ + Show if a client has authenticated using an IAM API key. + + :return: True if client is IAM authenticated. False otherwise. + """ + return self._use_iam + def connect(self): """ Starts up an authentication session for the client using cookie @@ -107,10 +116,12 @@ def connect(self): self.session_logout() if self.admin_party: + self._use_iam = False self.r_session = ClientSession( timeout=self._timeout ) elif self._use_basic_auth: + self._use_iam = False self.r_session = BasicSession( self._user, self._auth_token, diff --git a/src/cloudant/replicator.py b/src/cloudant/replicator.py index c9b6a40c..39c2f1b9 100644 --- a/src/cloudant/replicator.py +++ b/src/cloudant/replicator.py @@ -60,31 +60,51 @@ def create_replication(self, source_db=None, target_db=None, :returns: Replication document as a Document instance """ + if source_db is None: + raise CloudantReplicatorException(101) + + if target_db is None: + raise CloudantReplicatorException(102) data = dict( _id=repl_id if repl_id else str(uuid.uuid4()), **kwargs ) - if source_db is None: - raise CloudantReplicatorException(101) + # replication source + data['source'] = {'url': source_db.database_url} - if not source_db.admin_party: - data['source'].update( - {'headers': {'Authorization': source_db.creds['basic_auth']}} - ) + if source_db.admin_party: + pass # no credentials required + elif source_db.client.is_iam_authenticated: + data['source'].update({'auth': { + 'iam': {'api_key': source_db.client.r_session.get_api_key} + }}) + else: + data['source'].update({'headers': { + 'Authorization': source_db.creds['basic_auth'] + }}) + + # replication target - if target_db is None: - raise CloudantReplicatorException(102) data['target'] = {'url': target_db.database_url} - if not target_db.admin_party: - data['target'].update( - {'headers': {'Authorization': target_db.creds['basic_auth']}} - ) + if target_db.admin_party: + pass # no credentials required + elif target_db.client.is_iam_authenticated: + data['target'].update({'auth': { + 'iam': {'api_key': target_db.client.r_session.get_api_key} + }}) + else: + data['target'].update({'headers': { + 'Authorization': target_db.creds['basic_auth'] + }}) + + # add user context delegation if not data.get('user_ctx'): - if (target_db and not target_db.admin_party or - self.database.creds): + if target_db and target_db.admin_party: + pass # noop - not required for admin party mode + elif self.database.creds and self.database.creds.get('user_ctx'): data['user_ctx'] = self.database.creds['user_ctx'] return self.database.create_document(data, throw_on_exists=True) diff --git a/tests/unit/replicator_mock_tests.py b/tests/unit/replicator_mock_tests.py new file mode 100644 index 00000000..96589f9e --- /dev/null +++ b/tests/unit/replicator_mock_tests.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +# Copyright (C) 2018 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +_replicator_mock_tests_ + +replicator module - Mock unit tests for the Replicator class +""" + +import mock +import unittest + +from cloudant.database import CouchDatabase +from cloudant.replicator import Replicator + +from tests.unit.iam_auth_tests import MOCK_API_KEY + + +class ReplicatorDocumentValidationMockTests(unittest.TestCase): + """ + Replicator document validation tests + """ + + def setUp(self): + self.repl_id = 'rep_test' + + self.server_url = 'http://localhost:5984' + self.user_ctx = { + 'name': 'foo', + 'roles': ['erlanger', 'researcher'] + } + + self.source_db = 'source_db' + self.target_db = 'target_db' + + def setUpClientMocks(self, admin_party=False, iam_api_key=None): + m_client = mock.MagicMock() + type(m_client).server_url = mock.PropertyMock( + return_value=self.server_url) + + type(m_client).admin_party = mock.PropertyMock( + return_value=admin_party) + + iam_authenticated = False + + if iam_api_key is not None: + iam_authenticated = True + + m_session = mock.MagicMock() + type(m_session).get_api_key = mock.PropertyMock( + return_value=iam_api_key) + + type(m_client).r_session = mock.PropertyMock( + return_value=m_session) + + type(m_client).is_iam_authenticated = mock.PropertyMock( + return_value=iam_authenticated) + + return m_client + + def test_using_admin_party_source_and_target(self): + m_admin_party_client = self.setUpClientMocks(admin_party=True) + + m_replicator = mock.MagicMock() + type(m_replicator).creds = mock.PropertyMock(return_value=None) + m_admin_party_client.__getitem__.return_value = m_replicator + + # create source/target databases + src = CouchDatabase(m_admin_party_client, self.source_db) + tgt = CouchDatabase(m_admin_party_client, self.target_db) + + # trigger replication + rep = Replicator(m_admin_party_client) + rep.create_replication(src, tgt, repl_id=self.repl_id) + + kcall = m_replicator.create_document.call_args_list + self.assertEquals(len(kcall), 1) + args, kwargs = kcall[0] + self.assertEquals(len(args), 1) + + expected_doc = { + '_id': self.repl_id, + 'source': {'url': '/'.join((self.server_url, self.source_db))}, + 'target': {'url': '/'.join((self.server_url, self.target_db))} + } + + self.assertDictEqual(args[0], expected_doc) + self.assertTrue(kwargs['throw_on_exists']) + + def test_using_basic_auth_source_and_target(self): + test_basic_auth_header = 'abc' + + m_basic_auth_client = self.setUpClientMocks() + + m_replicator = mock.MagicMock() + m_basic_auth_client.__getitem__.return_value = m_replicator + m_basic_auth_client.basic_auth_str.return_value = test_basic_auth_header + + # create source/target databases + src = CouchDatabase(m_basic_auth_client, self.source_db) + tgt = CouchDatabase(m_basic_auth_client, self.target_db) + + # trigger replication + rep = Replicator(m_basic_auth_client) + rep.create_replication( + src, tgt, repl_id=self.repl_id, user_ctx=self.user_ctx) + + kcall = m_replicator.create_document.call_args_list + self.assertEquals(len(kcall), 1) + args, kwargs = kcall[0] + self.assertEquals(len(args), 1) + + expected_doc = { + '_id': self.repl_id, + 'user_ctx': self.user_ctx, + 'source': { + 'headers': {'Authorization': test_basic_auth_header}, + 'url': '/'.join((self.server_url, self.source_db)) + }, + 'target': { + 'headers': {'Authorization': test_basic_auth_header}, + 'url': '/'.join((self.server_url, self.target_db)) + } + } + + self.assertDictEqual(args[0], expected_doc) + self.assertTrue(kwargs['throw_on_exists']) + + def test_using_iam_auth_source_and_target(self): + m_iam_auth_client = self.setUpClientMocks(iam_api_key=MOCK_API_KEY) + + m_replicator = mock.MagicMock() + m_iam_auth_client.__getitem__.return_value = m_replicator + + # create source/target databases + src = CouchDatabase(m_iam_auth_client, self.source_db) + tgt = CouchDatabase(m_iam_auth_client, self.target_db) + + # trigger replication + rep = Replicator(m_iam_auth_client) + rep.create_replication( + src, tgt, repl_id=self.repl_id, user_ctx=self.user_ctx) + + kcall = m_replicator.create_document.call_args_list + self.assertEquals(len(kcall), 1) + args, kwargs = kcall[0] + self.assertEquals(len(args), 1) + + expected_doc = { + '_id': self.repl_id, + 'user_ctx': self.user_ctx, + 'source': { + 'auth': {'iam': {'api_key': MOCK_API_KEY}}, + 'url': '/'.join((self.server_url, self.source_db)) + }, + 'target': { + 'auth': {'iam': {'api_key': MOCK_API_KEY}}, + 'url': '/'.join((self.server_url, self.target_db)) + } + } + + self.assertDictEqual(args[0], expected_doc) + self.assertTrue(kwargs['throw_on_exists']) diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index cd79a7a1..121f1aee 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2016, 2017 IBM Corp. All rights reserved. +# Copyright (c) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -161,7 +161,6 @@ def test_replication_with_generated_id(self): ) self.replication_ids.append(repl_id['_id']) - @skip_if_not_cookie_auth @flaky(max_runs=3) def test_create_replication(self): """ @@ -180,7 +179,7 @@ def test_create_replication(self): # Test that the replication document was created expected_keys = ['_id', '_rev', 'source', 'target', 'user_ctx'] # If Admin Party mode then user_ctx will not be in the key list - if self.client.admin_party: + if self.client.admin_party or self.client.is_iam_authenticated: expected_keys.pop() self.assertTrue(all(x in list(repl_doc.keys()) for x in expected_keys)) self.assertEqual(repl_doc['_id'], repl_id) @@ -238,7 +237,7 @@ def test_timeout_in_create_replication(self): # Test that the replication document was created expected_keys = ['_id', '_rev', 'source', 'target', 'user_ctx'] # If Admin Party mode then user_ctx will not be in the key list - if self.client.admin_party: + if self.client.admin_party or self.client.is_iam_authenticated: expected_keys.pop() self.assertTrue(all(x in list(repl_doc.keys()) for x in expected_keys)) self.assertEqual(repl_doc['_id'], repl_id) @@ -305,7 +304,6 @@ def test_list_replications(self): match = [repl_id for repl_id in all_repl_ids if repl_id in repl_ids] self.assertEqual(set(repl_ids), set(match)) - @skip_if_not_cookie_auth def test_retrieve_replication_state(self): """ Test that the replication state can be retrieved for a replication @@ -347,7 +345,6 @@ def test_retrieve_replication_state_using_invalid_id(self): ) self.assertIsNone(repl_state) - @skip_if_not_cookie_auth def test_stop_replication(self): """ Test that a replication can be stopped. @@ -383,7 +380,6 @@ def test_stop_replication_using_invalid_id(self): 'Replication with id {} not found.'.format(repl_id) ) - @skip_if_not_cookie_auth def test_follow_replication(self): """ Test that follow_replication(...) properly iterates updated From ca1a71e1c9e885aeb79db123f93a19b4012b3924 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 1 May 2018 11:57:37 +0100 Subject: [PATCH 077/185] Retry replication stop test on HTTP 409 response status code --- tests/unit/replicator_tests.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index 121f1aee..504500b6 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -356,7 +356,16 @@ def test_stop_replication(self): self.target_db, repl_id ) - self.replicator.stop_replication(repl_id) + max_retry = 3 + while True: + try: + max_retry -= 1 + self.replicator.stop_replication(repl_id) + break + except requests.HTTPError as err: + self.assertEqual(err.response.status_code, 409) + if max_retry == 0: + self.fail('Failed to stop replication: {0}'.format(err)) try: # The .fetch() will fail since the replication has been stopped # and the replication document has been removed from the db. From 8cbee2a8b72f46bb6f9685a135426f04c18cc6a6 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 1 May 2018 13:44:35 +0100 Subject: [PATCH 078/185] Always set user_ctx in replication documents --- src/cloudant/replicator.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/cloudant/replicator.py b/src/cloudant/replicator.py index 39c2f1b9..7f24407e 100644 --- a/src/cloudant/replicator.py +++ b/src/cloudant/replicator.py @@ -51,8 +51,7 @@ def create_replication(self, source_db=None, target_db=None, :param str repl_id: Optional replication id. Generated internally if not explicitly set. :param dict user_ctx: Optional user to act as. Composed internally - if not explicitly set and not in CouchDB Admin Party - mode. + if not explicitly set. :param bool create_target: Specifies whether or not to create the target, if it does not already exist. :param bool continuous: If set to True then the replication will be @@ -101,11 +100,9 @@ def create_replication(self, source_db=None, target_db=None, # add user context delegation - if not data.get('user_ctx'): - if target_db and target_db.admin_party: - pass # noop - not required for admin party mode - elif self.database.creds and self.database.creds.get('user_ctx'): - data['user_ctx'] = self.database.creds['user_ctx'] + if not data.get('user_ctx') and self.database.creds and \ + self.database.creds.get('user_ctx'): + data['user_ctx'] = self.database.creds['user_ctx'] return self.database.create_document(data, throw_on_exists=True) From 26029910907750177389813188009f42c9ee180d Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 11 May 2018 16:28:51 -0400 Subject: [PATCH 079/185] Removed user and password creds from URL property --- CHANGES.md | 1 + src/cloudant/client.py | 23 +++++++++++++++++------ tests/unit/client_tests.py | 27 +++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ab805d4e..e6c6a008 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ - [NEW] Support IAM authentication in replication documents. - [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. - [IMPROVED] Updated Travis CI and unit tests to run against CouchDB 2.1.1. +- [IMPROVED] Shortened length of client URLs by removing username and password. # 2.8.1 (2018-02-16) diff --git a/src/cloudant/client.py b/src/cloudant/client.py index de88112f..e3151cd2 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -17,6 +17,7 @@ instance. """ import json +from ._2to3 import url_parse from ._client_session import ( BasicSession, @@ -93,6 +94,20 @@ def __init__(self, user, auth_token, admin_party=False, **kwargs): self._auto_renew = kwargs.get('auto_renew', False) self._use_basic_auth = kwargs.get('use_basic_auth', False) self._use_iam = kwargs.get('use_iam', False) + # If user/pass exist in URL, remove and set variables + if not self._use_basic_auth and self.server_url: + parsed_url = url_parse(kwargs.get('url')) + # Note: To prevent conflicts with field names, the method + # and attribute names of `url_parse` start with an underscore + if parsed_url.port is None: + self.server_url = parsed_url._replace( + netloc="{}".format(parsed_url.hostname)).geturl() + else: + self.server_url = parsed_url._replace( + netloc="{}:{}".format(parsed_url.hostname, parsed_url.port)).geturl() + if (not user and not auth_token) and (parsed_url.username and parsed_url.password): + self._user = parsed_url.username + self._auth_token = parsed_url.password connect_to_couch = kwargs.get('connect', False) if connect_to_couch and self._DATABASE_CLASS == CouchDatabase: @@ -450,14 +465,10 @@ def __init__(self, cloudant_user, auth_token, **kwargs): super(Cloudant, self).__init__(cloudant_user, auth_token, **kwargs) self._client_user_header = {'User-Agent': USER_AGENT} account = kwargs.get('account') - url = kwargs.get('url') - x_cloudant_user = kwargs.get('x_cloudant_user') if account is not None: self.server_url = 'https://{0}.cloudant.com'.format(account) - elif kwargs.get('url') is not None: - self.server_url = url - if x_cloudant_user is not None: - self._client_user_header['X-Cloudant-User'] = x_cloudant_user + if kwargs.get('x_cloudant_user') is not None: + self._client_user_header['X-Cloudant-User'] = kwargs.get('x_cloudant_user') if self.server_url is None: raise CloudantClientException(102) diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index d79e0c6f..a65e729f 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -128,6 +128,20 @@ def test_constructor_with_url(self): self.assertEqual(self.client.encoder, json.JSONEncoder) self.assertIsNone(self.client.r_session) + def test_constructor_with_creds_removed_from_url(self): + """ + Test instantiating a client object using a URL + """ + client = CouchDB(None, None, url='http://a9a9a9a9-a9a9-a9a9-a9a9-a9a9a9a9a9a9-bluemix' + ':a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9' + 'a9a9a9a9a9a9@d8a01891-e4d2-4102-b5f8-751fb735ce31-' + 'bluemix.couchdb.local:5984') + self.assertEqual(client.server_url, 'http://d8a01891-e4d2-4102-b5f8-751fb735ce31-' + 'bluemix.couchdb.local:5984') + self.assertEqual(client._user, 'a9a9a9a9-a9a9-a9a9-a9a9-a9a9a9a9a9a9-bluemix') + self.assertEqual(client._auth_token, 'a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a' + '9a9a9a9a9a9a9a9a9a9a9a9a9') + def test_connect(self): """ Test connect and disconnect functionality. @@ -594,6 +608,19 @@ class CloudantClientTests(UnitTestDbBase): Cloudant specific client unit tests """ + def test_constructor_with_creds_removed_from_url(self): + """ + Test instantiating a client object using a URL + """ + client = Cloudant(None, None, url='https://a9a9a9a9-a9a9-a9a9-a9a9-a9a9a9a9a9a9-bluemix' + ':a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9' + 'a9a9a9a9a9a9@d8a01891-e4d2-4102-b5f8-751fb735ce31-' + 'bluemix.cloudant.com') + self.assertEqual(client.server_url, 'https://d8a01891-e4d2-4102-b5f8-751fb735ce31-' + 'bluemix.cloudant.com') + self.assertEqual(client._user, 'a9a9a9a9-a9a9-a9a9-a9a9-a9a9a9a9a9a9-bluemix') + self.assertEqual(client._auth_token, 'a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a9a' + '9a9a9a9a9a9a9a9a9a9a9a9a9') @skip_if_not_cookie_auth def test_cloudant_session_login(self): """ From 9ed98331dc8b9e91b1441911fd85cf258b35801a Mon Sep 17 00:00:00 2001 From: Tom Blench Date: Wed, 23 May 2018 09:03:21 +0100 Subject: [PATCH 080/185] Add support for _scheduler endpoints (#379) * Add support for _scheduler endpoints * PR updates: - add get_doc for replication docs by docid - fix methods so they return the json, not the whole response * Fetch features/metadata, update replication_state * Copyrights, docstrings * Copyright * Fix TODO # Local Variables: * Attempt to fix broken test * Add new expected state for scheduler endpoint * Add new expected state for scheduler endpoint --- src/cloudant/client.py | 23 ++++ src/cloudant/replicator.py | 43 +++++-- src/cloudant/scheduler.py | 81 ++++++++++++ tests/unit/replicator_tests.py | 12 +- tests/unit/scheduler_tests.py | 225 +++++++++++++++++++++++++++++++++ 5 files changed, 368 insertions(+), 16 deletions(-) create mode 100644 src/cloudant/scheduler.py create mode 100644 tests/unit/scheduler_tests.py diff --git a/src/cloudant/client.py b/src/cloudant/client.py index e3151cd2..6c9b5f33 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -108,6 +108,7 @@ def __init__(self, user, auth_token, admin_party=False, **kwargs): if (not user and not auth_token) and (parsed_url.username and parsed_url.password): self._user = parsed_url.username self._auth_token = parsed_url.password + self._features = None connect_to_couch = kwargs.get('connect', False) if connect_to_couch and self._DATABASE_CLASS == CouchDatabase: @@ -122,6 +123,18 @@ def is_iam_authenticated(self): """ return self._use_iam + def features(self): + """ + lazy fetch and cache features + """ + if self._features is None: + metadata = self.metadata() + if "features" in metadata: + self._features = metadata["features"] + else: + self._features = [] + return self._features + def connect(self): """ Starts up an authentication session for the client using cookie @@ -324,6 +337,16 @@ def db_updates(self, raw_data=False, **kwargs): """ return Feed(self, raw_data, **kwargs) + def metadata(self): + """ + Retrieves the remote server metadata dictionary. + + :returns: Dictionary containing server metadata details + """ + resp = self.r_session.get(self.server_url) + resp.raise_for_status() + return resp.json() + def keys(self, remote=False): """ Returns the database names for this client. Default is diff --git a/src/cloudant/replicator.py b/src/cloudant/replicator.py index 7f24407e..e3ec3550 100644 --- a/src/cloudant/replicator.py +++ b/src/cloudant/replicator.py @@ -18,8 +18,11 @@ import uuid +from requests.exceptions import HTTPError + from .error import CloudantReplicatorException, CloudantClientException from .document import Document +from .scheduler import Scheduler class Replicator(object): """ @@ -34,6 +37,7 @@ class Replicator(object): def __init__(self, client): repl_db = '_replicator' + self.client = client try: self.database = client[repl_db] except Exception: @@ -133,12 +137,20 @@ def replication_state(self, repl_id): :returns: Replication state as a ``str`` """ - try: - repl_doc = self.database[repl_id] - except KeyError: - raise CloudantReplicatorException(404, repl_id) - repl_doc.fetch() - return repl_doc.get('_replication_state') + if "scheduler" in self.client.features(): + try: + repl_doc = Scheduler(self.client).get_doc(repl_id) + except HTTPError as err: + raise CloudantReplicatorException(err.response.status_code, repl_id) + state = repl_doc['state'] + else: + try: + repl_doc = self.database[repl_id] + except KeyError: + raise CloudantReplicatorException(404, repl_id) + repl_doc.fetch() + state = repl_doc.get('_replication_state') + return state def follow_replication(self, repl_id): """ @@ -161,12 +173,19 @@ def update_state(): """ Retrieves the replication state. """ - try: - arepl_doc = self.database[repl_id] - arepl_doc.fetch() - return arepl_doc, arepl_doc.get('_replication_state') - except KeyError: - return None, None + if "scheduler" in self.client.features(): + try: + arepl_doc = Scheduler(self.client).get_doc(repl_id) + return arepl_doc, arepl_doc['state'] + except HTTPError: + return None, None + else: + try: + arepl_doc = self.database[repl_id] + arepl_doc.fetch() + return arepl_doc, arepl_doc.get('_replication_state') + except KeyError: + return None, None while True: # Make sure we fetch the state up front, just in case it moves diff --git a/src/cloudant/scheduler.py b/src/cloudant/scheduler.py new file mode 100644 index 00000000..76cdc101 --- /dev/null +++ b/src/cloudant/scheduler.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# Copyright (C) 2018 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +API module for interacting with scheduler endpoints +""" + +class Scheduler(object): + """ + API for retrieving scheduler jobs and documents. + + :param client: Client instance used by the database. Can either be a + ``CouchDB`` or ``Cloudant`` client instance. + """ + + def __init__(self, client): + self._client = client + self._r_session = client.r_session + self._scheduler = '/'.join([self._client.server_url, '_scheduler']) + + def list_docs(self, limit=None, skip=None): + """ + Lists replication documents. Includes information + about all the documents, even in completed and failed + states. For each document it returns the document ID, the + database, the replication ID, source and target, and other + information. + + :param limit: How many results to return. + :param skip: How many result to skip starting at the beginning, if ordered by document ID. + """ + params = dict() + if limit != None: + params["limit"] = limit + if skip != None: + params["skip"] = skip + resp = self._r_session.get('/'.join([self._scheduler, 'docs']), params=params) + resp.raise_for_status() + return resp.json() + + def get_doc(self, doc_id): + """ + Get replication document state for a given replication document ID. + """ + resp = self._r_session.get('/'.join([self._scheduler, 'docs', '_replicator', doc_id])) + resp.raise_for_status() + return resp.json() + + + def list_jobs(self, limit=None, skip=None): + """ + Lists replication jobs. Includes replications created via + /_replicate endpoint as well as those created from replication + documents. Does not include replications which have completed + or have failed to start because replication documents were + malformed. Each job description will include source and target + information, replication id, a history of recent event, and a + few other things. + + :param limit: How many results to return. + :param skip: How many result to skip starting at the beginning, if ordered by document ID. + """ + params = dict() + if limit != None: + params["limit"] = limit + if skip != None: + params["skip"] = skip + resp = self._r_session.get('/'.join([self._scheduler, 'jobs']), params=params) + resp.raise_for_status() + return resp.json() diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index 504500b6..538adbd2 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -317,7 +317,7 @@ def test_retrieve_replication_state(self): ) self.replication_ids.append(repl_id) repl_state = None - valid_states = ['completed', 'error', 'triggered', None] + valid_states = ['completed', 'error', 'triggered', 'running', None] finished = False for _ in range(300): repl_state = self.replicator.replication_state(repl_id) @@ -402,11 +402,15 @@ def test_follow_replication(self): repl_id ) self.replication_ids.append(repl_id) - valid_states = ('completed', 'error', 'triggered', None) + valid_states = ('completed', 'error', 'triggered', 'running', None) repl_states = [] + if 'scheduler' in self.client.features(): + state_key = 'state' + else: + state_key = '_replication_state' for doc in self.replicator.follow_replication(repl_id): - self.assertIn(doc.get('_replication_state'), valid_states) - repl_states.append(doc.get('_replication_state')) + self.assertIn(doc.get(state_key), valid_states) + repl_states.append(doc.get(state_key)) self.assertTrue(len(repl_states) > 0) self.assertEqual(repl_states[-1], 'completed') self.assertNotIn('error', repl_states) diff --git a/tests/unit/scheduler_tests.py b/tests/unit/scheduler_tests.py new file mode 100644 index 00000000..97b5046f --- /dev/null +++ b/tests/unit/scheduler_tests.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python +# Copyright (C) 2018 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Unit tests for the Scheduler class +""" + +import unittest +import requests +import json +import mock + +from cloudant.scheduler import Scheduler + +from .unit_t_db_base import UnitTestDbBase + +class SchedulerTests(UnitTestDbBase): + + def setUp(self): + """ + Set up test attributes + """ + super(SchedulerTests, self).setUp() + self.db_set_up() + + def tearDown(self): + """ + Reset test attributes + """ + self.db_tear_down() + super(SchedulerTests, self).tearDown() + + def test_scheduler_docs(self): + """ + Test scheduler docs + """ + # set up mock response using a real captured response + m_response_ok = requests.Response() + m_response_ok.status_code = 200 + m_response_ok.json = mock.Mock() + m_response_ok.json.return_value = {"total_rows":6,"offset":0,"docs":[ + {"database":"tomblench/_replicator", + "doc_id":"296e48244e003eba8764b2156b3bf302", + "id":None, + "source":"https://tomblench.cloudant.com/animaldb/", + "target":"https://tomblench.cloudant.com/animaldb_copy/", + "state":"completed", + "error_count":0, + "info":{"revisions_checked":15, + "missing_revisions_found":2, + "docs_read":2, + "docs_written":2, + "changes_pending":None, + "doc_write_failures":0, + "checkpointed_source_seq":"19-g1AAAAGjeJyVz10KwjAMB_BoJ4KX8AZF2tWPJ3eVpqnO0XUg27PeTG9Wa_VhwmT6kkDIPz_iACArGcGS0DRnWxDmHE9HdJ3lxjUdad9yb1sXF6cacB9CqEqmZ3UczKUh2uGhHxeD8U9i_Z3AIla8vJVJUlBIZYTqX5A_KMM7SfFZrHCNLUK3p7RIkl5tSRD-K6kx6f6S0k8sScpYJTb5uFQ9AI9Ch9c"}, + "start_time":None, + "last_updated":"2017-04-13T14:53:50+00:00"}, + {"database":"tomblench/_replicator", + "doc_id":"3b749f320867d703550b0f758a4000ae", + "id":None, + "source":"https://examples.cloudant.com/animaldb/", + "target":"https://tomblench.cloudant.com/animaldb/", + "state":"completed", + "error_count":0, + "info":{"revisions_checked":15, + "missing_revisions_found":15, + "docs_read":15, + "docs_written":15, + "changes_pending":None, + "doc_write_failures":0, + "checkpointed_source_seq":"56-g1AAAAGveJzLYWBgYMlgTmFQSElKzi9KdUhJstDLTS3KLElMT9VLzskvTUnMK9HLSy3JAapkSmRIsv___39WBnMiby5QgN04JS3FLDUJWb8Jdv0gSxThigyN8diS5AAkk-qhFvFALEo2MTEwMSXGDDSbTPHYlMcCJBkagBTQsv0g28TBtpkbGCQapaF4C4cxJFt2AGIZ2GscYMuMDEzMUizMkC0zw25MFgBKoovi"}, + "start_time":None, + "last_updated":"2017-04-27T12:28:44+00:00"}, + {"database":"tomblench/_replicator", + "doc_id":"ad8f7896480b8081c8f0a2267ffd1859", + "id":None, + "source":"https://tortytherlediffecareette:*****@mikerhodestesty008.cloudant.com/moviesdb/", + "target":"https://tomblench.cloudant.com/moviesdb_rep/", + "state":"completed", + "error_count":0, + "info":{"revisions_checked":5997, + "missing_revisions_found":5997, + "docs_read":5997, + "docs_written":5997, + "changes_pending":None, + "doc_write_failures":0, + "checkpointed_source_seq":"5997-g1AAAANreJy10UEKwjAQAMBgBcVP2BeUpEm1PdmfaDYJSKkVtB486U_0J_oBTz5AHyAI3jxIjUml1x7ayy67LDssmyKE-nNHIleCWK5ULIF6uVrnW4xDT6TLjeRZ7mUqT_VkhyMYFkWRzB3Q1XOhez3iczKKghor6jvg6giTiroYiuNQYYqbpeIfNa2oh72KhQGosFlq9qN2FfUyFPgUCKONoneXR7TXSWuHkvsYjjEWjQVvgTta7lRyV_szKgmRbVx3ttzNcs7AcEoKCHAb3N1y_9-9DYeBYzEiNTYlX3EcE0s"}, + "start_time":None, + "last_updated":"2016-08-23T13:11:26+00:00"}, + {"database":"tomblench/_replicator", + "doc_id":"b63c053ecd95a4047b55ed8847b046f1", + "id":None, + "source":"https://tomblench.cloudant.com/atestdb2/", + "target":"https://tomblench.cloudant.com/atestdb1/", + "state":"completed", + "error_count":0, + "info":{"revisions_checked":1, + "missing_revisions_found":1, + "docs_read":1, + "docs_written":1, + "changes_pending":None, + "doc_write_failures":0, + "checkpointed_source_seq":"2-g1AAAAFHeJyNjkEOgjAQRSdAYjyFN2jSFCtdyVU6nSKQWhJC13ozvVktsoEF0c2fTPL_-98BQNHmBCdCM4y2JuQMuxu6YJlxQyDtJ-bt5JIx04DXGGOvYRsR-xGsk-JjTrW5hnv6Dg0XplRngmPwZJvOW9ry5D7PF0nhmU5CvmZm9mVKVVacLr8pfy9fmt5L02q9qEhJbtbr-w-AQmfD"}, + "start_time":None, + "last_updated":"2017-05-16T16:25:22+00:00"}, + {"database":"tomblench/_replicator", + "doc_id":"c71c9e69e30a182dc91d8938277bc85e", + "id":None, + "source":"https://tomblench.cloudant.com/animaldb/", + "target":"https://tomblench.cloudant.com/animaldb_copy/", + "state":"completed", + "error_count":0, + "info":{"revisions_checked":15, + "missing_revisions_found":15, + "docs_read":15, + "docs_written":15, + "changes_pending":None, + "doc_write_failures":0, + "checkpointed_source_seq":"14-g1AAAAEueJzLYWBgYMlgTmGQSUlKzi9KdUhJMtTLTU1M0UvOyS9NScwr0ctLLckBqmJKZEiy____f1YGUyJrLlCAPdHEPCktJZk43UkOQDKpHmoAI9gAw2STxCTzJOIMyGMBkgwNQApoxv6sDGaoK0yN04wsk80IGEGKHQcgdoAdygxxaIplklFaWhYAu2FdOA"}, + "start_time":None, + "last_updated":"2015-05-12T11:47:33+00:00"}, + {"database":"tomblench/_replicator", + "doc_id":"e6242d1e9ce059b0388fc75af3116a39", + "id":None, + "source":"https://tomblench.cloudant.com/atestdb1/", + "target":"https://tomblench.cloudant.com/atestdb2/", + "state":"completed", + "error_count":0, + "info":{"revisions_checked":1, + "missing_revisions_found":1, + "docs_read":1, + "docs_written":1, + "changes_pending":None, + "doc_write_failures":0, + "checkpointed_source_seq":"1-g1AAAAFheJyFzkEOgjAQBdBRSIyn8AZNgEJgJVeZ6bQCqSUhdK0305th1Q1dEDYzyWTy_rcAkHYJw4VJjZNumQpB_Y2s10LZ0TO6WTg92_B4RKDrsixDlyDcw-FUVUiFahjO3rE2vdMcY9k2Rm2Y9Ig8bWqspdz25Lbn0jDhGVYgX1_z8DMblnlp8n0lTir3kt7_pFV7NE2WYbluP3wATr5vQA"}, + "start_time":None, + "last_updated":"2017-05-16T16:24:02+00:00"} + ]} + + self.client.r_session.get = mock.Mock(return_value=m_response_ok) + scheduler = Scheduler(self.client) + response = scheduler.list_docs(skip=0, limit=10) + # assert on request and response + self.client.r_session.get.assert_called_with( + self.url + '/_scheduler/docs', + params={"skip":0, "limit":10}, + ) + self.assertEqual(response["total_rows"], 6) + + def test_scheduler_doc(self): + """ + Test scheduler doc + """ + # set up mock response using a real captured response + m_response_ok = requests.Response() + m_response_ok.status_code = 200 + m_response_ok.json = mock.Mock() + m_response_ok.json.return_value = {"database":"tomblench/_replicator", + "doc_id":"296e48244e003eba8764b2156b3bf302", + "id":None, + "source":"https://tomblench.cloudant.com/animaldb/", + "target":"https://tomblench.cloudant.com/animaldb_copy/", + "state":"completed", + "error_count":0, + "info":{"revisions_checked":15, + "missing_revisions_found":2, + "docs_read":2, + "docs_written":2, + "changes_pending":None, + "doc_write_failures":0, + "checkpointed_source_seq":"19-g1AAAAGjeJyVz10KwjAMB_BoJ4KX8AZF2tWPJ3eVpqnO0XUg27PeTG9Wa_VhwmT6kkDIPz_iACArGcGS0DRnWxDmHE9HdJ3lxjUdad9yb1sXF6cacB9CqEqmZ3UczKUh2uGhHxeD8U9i_Z3AIla8vJVJUlBIZYTqX5A_KMM7SfFZrHCNLUK3p7RIkl5tSRD-K6kx6f6S0k8sScpYJTb5uFQ9AI9Ch9c"}, + "start_time":None, + "last_updated":"2017-04-13T14:53:50+00:00"}; + self.client.r_session.get = mock.Mock(return_value=m_response_ok) + scheduler = Scheduler(self.client) + response = scheduler.get_doc("296e48244e003eba8764b2156b3bf302") + # assert on request and response + self.client.r_session.get.assert_called_with( + self.url + '/_scheduler/docs/_replicator/296e48244e003eba8764b2156b3bf302', + ) + self.assertEqual(response["doc_id"], "296e48244e003eba8764b2156b3bf302") + + + def test_scheduler_jobs(self): + """ + Test scheduler jobs + """ + # set up mock response using a real captured response + m_response_ok = requests.Response() + m_response_ok.status_code = 200 + m_response_ok.json = mock.Mock() + m_response_ok.json.return_value = {"total_rows":1,"offset":0, + "jobs":[{"database":None, + "id":"f11105eaaded4981d21ff8ebf846f48b+create_target", + "pid":"<0.5866.6800>", + "source":"https://clientlibs-test:*****@clientlibs-test.cloudant.com/largedb1g/", + "target":"https://tomblench:*****@tomblench.cloudant.com/largedb1g/", + "user":"tomblench", + "doc_id":None, + "history":[{"timestamp":"2018-04-12T13:06:20Z", + "type":"started"}, + {"timestamp":"2018-04-12T13:06:20Z", + "type":"added"}], + "node":"dbcore@db2.bigblue.cloudant.net", + "start_time":"2018-04-12T13:06:20Z"}]} + self.client.r_session.get = mock.Mock(return_value=m_response_ok) + scheduler = Scheduler(self.client) + response = scheduler.list_jobs(skip=0, limit=10) + # assert on request and response + self.client.r_session.get.assert_called_with( + self.url + '/_scheduler/jobs', + params={"skip":0, "limit":10}, + ) + self.assertEqual(response["total_rows"], 1) From 461165d59d54059713ed929b83cfa484f1fe043e Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Thu, 19 Apr 2018 17:03:01 -0400 Subject: [PATCH 081/185] Updated Travis to run against CouchDB 1.7.1 and Python 3.6 --- .travis.yml | 12 +-- CHANGES.md | 2 +- tests/unit/changes_tests.py | 23 ++++-- tests/unit/database_tests.py | 10 ++- tests/unit/db_updates_tests.py | 145 +++++++++++++++++++++------------ tests/unit/document_tests.py | 8 +- tests/unit/unit_t_db_base.py | 65 +++++++++++---- tests/unit/view_tests.py | 24 ------ 8 files changed, 178 insertions(+), 111 deletions(-) diff --git a/.travis.yml b/.travis.yml index 00fe9573..ffd18c69 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,18 +4,20 @@ language: python python: - "2.7" - - "3.5" + - "3.6" env: - - ADMIN_PARTY=true - - ADMIN_PARTY=false + - ADMIN_PARTY=true COUCHDB_VERSION=2.1.1 + - ADMIN_PARTY=false COUCHDB_VERSION=2.1.1 + - ADMIN_PARTY=true COUCHDB_VERSION=1.7.1 + - ADMIN_PARTY=false COUCHDB_VERSION=1.7.1 services: - docker before_install: - - docker pull apache/couchdb:2.1.1 - - docker run -d -it -p 5984:5984 apache/couchdb:2.1.1 --with-admin-party-please --with-haproxy + - docker pull apache/couchdb:$COUCHDB_VERSION + - docker run -d -p 5984:5984 apache/couchdb:$COUCHDB_VERSION install: "pip install -r requirements.txt && pip install -r test-requirements.txt" diff --git a/CHANGES.md b/CHANGES.md index e6c6a008..a51dd720 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ - [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x. - [NEW] Support IAM authentication in replication documents. - [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. -- [IMPROVED] Updated Travis CI and unit tests to run against CouchDB 2.1.1. +- [IMPROVED] Verified library operation on Python 3.6.3. - [IMPROVED] Shortened length of client URLs by removing username and password. # 2.8.1 (2018-02-16) diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index 6f284221..ee3dce13 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -100,7 +100,11 @@ def test_get_raw_content(self): self.assertIsInstance(raw_line, BYTETYPE) raw_content.append(raw_line) changes = json.loads(''.join([unicode_(x) for x in raw_content])) - self.assertSetEqual(set(changes.keys()), set(['results', 'last_seq', 'pending'])) + if self.is_couchdb_1x_version() is True: + self.assertSetEqual( + set(changes.keys()), set(['results', 'last_seq'])) + else: + self.assertSetEqual(set(changes.keys()), set(['results', 'last_seq', 'pending'])) results = list() for result in changes['results']: self.assertSetEqual(set(result.keys()), set(['seq', 'changes', 'id'])) @@ -230,13 +234,16 @@ def test_get_feed_descending(self): last_seq = None for change in feed: if last_seq: - current = int(change['seq'][0: change['seq'].find('-')]) - last = int(last_seq[0:last_seq.find('-')]) - try: - self.assertTrue(current < last) - except AssertionError: - self.assertEqual(current, last) - self.assertTrue(len(change['seq']) > len(last_seq)) + if self.is_couchdb_1x_version() is True: + self.assertTrue(change['seq'] < last_seq) + else: + current = int(change['seq'][0: change['seq'].find('-')]) + last = int(last_seq[0:last_seq.find('-')]) + try: + self.assertTrue(current < last) + except AssertionError: + self.assertEqual(current, last) + self.assertTrue(len(change['seq']) > len(last_seq)) seq_list.append(change['seq']) last_seq = change['seq'] self.assertEqual(len(seq_list), 50) diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index bd5bd4fd..517736bc 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -968,6 +968,10 @@ def test_database_request_fails_after_client_disconnects(self): finally: self.client.connect() + @unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or + (os.environ.get('COUCHDB_VERSION') and + os.environ.get('COUCHDB_VERSION').startswith('1')), + 'Skipping test_create_json_index test') def test_create_json_index(self): """ Ensure that a JSON index is created as expected. @@ -991,6 +995,10 @@ def test_create_json_index(self): self.assertEquals(index['options']['def']['fields'], ['name', 'age']) self.assertEquals(index['reduce'], '_count') + @unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or + (os.environ.get('COUCHDB_VERSION') and + os.environ.get('COUCHDB_VERSION').startswith('1')), + 'Skipping test_create_json_index test') def test_delete_json_index(self): """ Ensure that a JSON index is deleted as expected. diff --git a/tests/unit/db_updates_tests.py b/tests/unit/db_updates_tests.py index c1df0d63..7cb8c956 100644 --- a/tests/unit/db_updates_tests.py +++ b/tests/unit/db_updates_tests.py @@ -28,6 +28,7 @@ from .unit_t_db_base import UnitTestDbBase from .. import BYTETYPE + class DbUpdatesTestsBase(UnitTestDbBase): """ Common _db_updates tests methods @@ -41,8 +42,9 @@ def setUp(self): self.client.connect() self.db_names = list() self.new_dbs = list() - self.create_db_updates() - self.create_dbs() + if not self.is_couchdb_1x_version(): + self.create_db_updates() + self.create_dbs() def tearDown(self): """ @@ -52,43 +54,55 @@ def tearDown(self): changes = list() [db.delete() for db in self.new_dbs] # Check the changes in the _db_updates feed to assert that the test databases are deleted - while not test_dbs_deleted: - feed = Feed(self.client, timeout=1000) - for change in feed: - if change['db_name'] in self.db_names and change['type'] == 'deleted': - changes.append(change) - if len(changes) == 2: - test_dbs_deleted = True - feed.stop() - self.delete_db_updates() + if not self.is_couchdb_1x_version(): + while not test_dbs_deleted and not self.is_couchdb_1x_version(): + feed = Feed(self.client, timeout=1000) + for change in feed: + if change['db_name'] in self.db_names and change['type'] == 'deleted': + changes.append(change) + if len(changes) == 2: + test_dbs_deleted = True + feed.stop() + self.delete_db_updates() self.client.disconnect() super(DbUpdatesTestsBase, self).tearDown() def create_dbs(self): - self.db_names = [self.dbname() for x in range(2)] - self.new_dbs += [self.client.create_database(dbname) for dbname in self.db_names] - # Verify that all created databases are listed in _db_updates - all_dbs_exist = False - while not all_dbs_exist: - changes = list() - feed = Feed(self.client, timeout=1000) - for change in feed: - changes.append(change) - if len(changes) == 3: - all_dbs_exist = True - feed.stop() + if not self.is_couchdb_1x_version(): + self.db_names = [self.dbname() for x in range(2)] + self.new_dbs += [self.client.create_database(dbname) for dbname in self.db_names] + # Verify that all created databases are listed in _db_updates + all_dbs_exist = False + while not all_dbs_exist: + changes = list() + feed = Feed(self.client, timeout=1000) + for change in feed: + changes.append(change) + if len(changes) == 3: + all_dbs_exist = True + feed.stop() + else: + self.new_dbs += [(self.client.create_database(self.dbname())) for x in range(3)] def assert_changes_in_db_updates_feed(self, changes): """ Assert that databases created in setup for db_updates_tests exist when looping through _db_updates feed Note: During the creation of _global_changes database, a doc called '_dbs' is created and seen in _db_updates """ - self.dbs = ['_dbs', self.new_dbs[0].database_name, self.new_dbs[1].database_name] - types = ['created', 'updated'] - for doc in changes: - self.assertIsNotNone(doc['seq']) - self.assertTrue(doc['db_name'] in self.dbs) - self.assertTrue(doc['type'] in types) + if not self.is_couchdb_1x_version(): + self.dbs = ['_dbs', self.new_dbs[0].database_name, self.new_dbs[1].database_name] + types = ['created', 'updated'] + for doc in changes: + self.assertIsNotNone(doc['seq']) + self.assertTrue(doc['db_name'] in self.dbs) + self.assertTrue(doc['type'] in types) + else: + self.assertDictEqual( + changes[0], {'db_name': self.new_dbs[0].database_name, 'type': 'created'}) + self.assertDictEqual( + changes[1], {'db_name': self.new_dbs[1].database_name, 'type': 'created'}) + self.assertDictEqual( + changes[2], {'db_name': self.new_dbs[2].database_name, 'type': 'created'}) @unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS'), 'Skipping CouchDB _db_updates feed tests') @@ -116,9 +130,12 @@ def test_stop_iteration_of_continuous_feed_with_heartbeat(self): feed = Feed(self.client, feed='continuous', timeout=100) changes = list() for change in feed: - changes.append(change) - if len(changes) == 3: - feed.stop() + if not change and self.is_couchdb_1x_version(): + self.create_dbs() + else: + changes.append(change) + if len(changes) == 3: + feed.stop() self.assert_changes_in_db_updates_feed(changes) self.assertEqual(len(changes), 3) @@ -130,9 +147,12 @@ def test_get_raw_content(self): raw_content = list() for raw_line in feed: self.assertIsInstance(raw_line, BYTETYPE) - raw_content.append(raw_line) - if len(raw_content) == 3: - feed.stop() + if not raw_line and self.is_couchdb_1x_version(): + self.create_dbs() + else: + raw_content.append(raw_line) + if len(raw_content) == 3: + feed.stop() changes = [json.loads(unicode_(x)) for x in raw_content] self.assert_changes_in_db_updates_feed(changes) @@ -142,11 +162,20 @@ def test_get_longpoll_feed_as_default(self): """ feed = Feed(self.client, timeout=1000) changes = list() - for change in feed: - self.assertIsNotNone(change) - changes.append(change) - self.assert_changes_in_db_updates_feed(changes) - self.assertEqual(len(changes), 3) + if self.is_couchdb_1x_version(): + for change in feed: + self.assertIsNone(change) + changes.append(change) + self.assertEqual(len(changes), 1) + self.assertIsNone(changes[0]) + else: + for change in feed: + self.assertIsNotNone(change) + changes.append(change) + if len(changes) == 3: + feed.stop() + self.assert_changes_in_db_updates_feed(changes) + self.assertEqual(len(changes), 3) def test_get_longpoll_feed_explicit(self): """ @@ -155,11 +184,20 @@ def test_get_longpoll_feed_explicit(self): """ feed = Feed(self.client, timeout=1000, feed='longpoll') changes = list() - for change in feed: - self.assertIsNotNone(change) - changes.append(change) - self.assert_changes_in_db_updates_feed(changes) - self.assertEqual(len(changes), 3) + if self.is_couchdb_1x_version(): + for change in feed: + self.assertIsNone(change) + changes.append(change) + self.assertEqual(len(changes), 1) + self.assertIsNone(changes[0]) + else: + for change in feed: + self.assertIsNotNone(change) + changes.append(change) + if len(changes) == 3: + feed.stop() + self.assert_changes_in_db_updates_feed(changes) + self.assertEqual(len(changes), 3) def test_get_continuous_with_timeout(self): """ @@ -168,13 +206,16 @@ def test_get_continuous_with_timeout(self): """ feed = Feed(self.client, feed='continuous', heartbeat=False, timeout=1000) changes = list() - for change in feed: - self.assertIsNotNone(change) - changes.append(change) - if len(changes) == 3: - feed.stop() - self.assert_changes_in_db_updates_feed(changes) - self.assertEqual(len(changes), 3) + if self.is_couchdb_1x_version(): + self.assertListEqual([x for x in feed], []) + else: + for change in feed: + self.assertIsNotNone(change) + changes.append(change) + if len(changes) == 3: + feed.stop() + self.assert_changes_in_db_updates_feed(changes) + self.assertEqual(len(changes), 3) def test_invalid_argument(self): """ diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 93e3659d..ea3c8d33 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -306,9 +306,9 @@ def test_appended_error_message_using_save_with_invalid_key(self): err = cm.exception # Should be a 400 error code, but CouchDB 1.6 issues a 500 if err.response.status_code == 500: - #Check this is CouchDB 1.6 - self.assertTrue(self.client.r_session.head(self.url).headers['Server'].find('CouchDB/1.6.') >= 0, - '500 returned but was not CouchDB 1.6.x') + # Check this is CouchDB 1.x + self.assertTrue(self.client.r_session.head(self.url).headers['Server'].find('CouchDB/1.') >= 0, + '500 returned but was not CouchDB 1.x') self.assertEqual( str(err.response.reason), 'Internal Server Error doc_validation Bad special document member: _invalid_key' diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index fc41ae9d..1ca9220f 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -109,20 +109,30 @@ def setUpClass(cls): if os.environ.get('DB_USER') is None: # Get couchdb docker node name - os.environ['NODENAME'] = requests.get( - '{0}/_membership'.format(os.environ['DB_URL'])).json()['all_nodes'][0] + if os.environ.get('COUCHDB_VERSION') == '2.1.1': + os.environ['NODENAME'] = requests.get( + '{0}/_membership'.format(os.environ['DB_URL'])).json()['all_nodes'][0] os.environ['DB_USER_CREATED'] = '1' os.environ['DB_USER'] = 'user-{0}'.format( unicode_(uuid.uuid4()) ) os.environ['DB_PASSWORD'] = 'password' - resp = requests.put( - '{0}/_node/{1}/_config/admins/{2}'.format( - os.environ['DB_URL'], - os.environ['NODENAME'], - os.environ['DB_USER'] + if os.environ.get('COUCHDB_VERSION') == '2.1.1': + resp = requests.put( + '{0}/_node/{1}/_config/admins/{2}'.format( + os.environ['DB_URL'], + os.environ['NODENAME'], + os.environ['DB_USER'] + ), + data='"{0}"'.format(os.environ['DB_PASSWORD']) + ) + else: + resp = requests.put( + '{0}/_config/admins/{1}'.format( + os.environ['DB_URL'], + os.environ['DB_USER'] ), - data='"{0}"'.format(os.environ['DB_PASSWORD']) + data='"{0}"'.format(os.environ['DB_PASSWORD']) ) resp.raise_for_status() @@ -133,14 +143,25 @@ def tearDownClass(cls): """ if (os.environ.get('RUN_CLOUDANT_TESTS') is None and os.environ.get('DB_USER_CREATED') is not None): - resp = requests.delete( - '{0}://{1}:{2}@{3}/_node/{4}/_config/admins/{5}'.format( - os.environ['DB_URL'].split('://', 1)[0], - os.environ['DB_USER'], - os.environ['DB_PASSWORD'], - os.environ['DB_URL'].split('://', 1)[1], - os.environ['NODENAME'], - os.environ['DB_USER'] + if os.environ.get('COUCHDB_VERSION') == '2.1.1': + resp = requests.delete( + '{0}://{1}:{2}@{3}/_node/{4}/_config/admins/{5}'.format( + os.environ['DB_URL'].split('://', 1)[0], + os.environ['DB_USER'], + os.environ['DB_PASSWORD'], + os.environ['DB_URL'].split('://', 1)[1], + os.environ['NODENAME'], + os.environ['DB_USER'] + ) + ) + else: + resp = requests.delete( + '{0}://{1}:{2}@{3}/_config/admins/{4}'.format( + os.environ['DB_URL'].split('://', 1)[0], + os.environ['DB_USER'], + os.environ['DB_PASSWORD'], + os.environ['DB_URL'].split('://', 1)[1], + os.environ['DB_USER'] ) ) del os.environ['DB_USER_CREATED'] @@ -368,3 +389,15 @@ def delete_db_updates(self): except CloudantClientException: pass + def is_couchdb_1x_version(self): + if os.environ.get('COUCHDB_VERSION') and os.environ.get('COUCHDB_VERSION').startswith('1'): + return True + else: + # Get version from server info + couchdb_info = json.loads(self.client.r_session.get(self.client.server_url).text) + if couchdb_info and couchdb_info['version'].startswith('1'): + return True + else: + return False + + diff --git a/tests/unit/view_tests.py b/tests/unit/view_tests.py index 5e0f492e..28be9cd4 100644 --- a/tests/unit/view_tests.py +++ b/tests/unit/view_tests.py @@ -293,30 +293,6 @@ def test_view_callable_with_non_existing_view(self): except requests.HTTPError as err: self.assertEqual(err.response.status_code, 404) - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is None, - 'Only execute as part of CouchDB tests') - def test_view_callable_with_invalid_javascript(self): - """ - Test error condition when Javascript errors exist. This test is only - valid for CouchDB because the map function Javascript is validated on - the Cloudant server when attempting to save a design document so invalid - Javascript is not possible there. - """ - self.populate_db_with_documents() - ddoc = DesignDocument(self.db, 'ddoc001') - ddoc.add_view( - 'view001', - 'This is not valid Javascript' - ) - with self.assertRaises(requests.HTTPError) as cm: - ddoc.save() - err = cm.exception - self.assertTrue(str(err).startswith( - '400 Client Error: Bad Request compilation_error Compilation of the map function ' - 'in the \'view001\' view failed: Expression does not eval to a function.' - )) - def test_custom_result_context_manager(self): """ Test that the context manager for custom results returns From ce47ce0b63bf8d23d62f72bc9d88825a2a274dad Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 25 May 2018 16:24:46 -0400 Subject: [PATCH 082/185] Added copyright to DCO1.1.txt --- DCO1.1.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/DCO1.1.txt b/DCO1.1.txt index f440e6fa..d5646050 100644 --- a/DCO1.1.txt +++ b/DCO1.1.txt @@ -1,3 +1,15 @@ +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +1 Letterman Drive +Suite D4700 +San Francisco, CA, 94129 + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: From 8ef838717cf2de07b82ff0c83e6c538043b854f1 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Mon, 4 Jun 2018 11:06:29 -0400 Subject: [PATCH 083/185] =?UTF-8?q?Updated=20test=20container=20name=20to?= =?UTF-8?q?=20=E2=80=98couchdb=E2=80=99=20The=20CouchDB=20users=20list=20s?= =?UTF-8?q?uggested=20that=20`couchdb`=20is=20preferred=20to=20`apache/cou?= =?UTF-8?q?chb`=20despite=20what=20the=20documentation=20currently=20says.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ffd18c69..5fabac04 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,8 @@ services: - docker before_install: - - docker pull apache/couchdb:$COUCHDB_VERSION - - docker run -d -p 5984:5984 apache/couchdb:$COUCHDB_VERSION + - docker pull couchdb:$COUCHDB_VERSION + - docker run -d -p 5984:5984 couchdb:$COUCHDB_VERSION install: "pip install -r requirements.txt && pip install -r test-requirements.txt" From 0191573ca84c8210e3b9dbb9facf844a9e80597e Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Wed, 30 May 2018 14:53:40 -0400 Subject: [PATCH 084/185] Fixed case where `Document` context manager would throw instead of creating a new document if no `_id` was provided --- CHANGES.md | 1 + docs/getting_started.rst | 15 ++++++++++----- src/cloudant/document.py | 3 +++ tests/unit/document_tests.py | 23 +++++++++++++++++++++++ 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a51dd720..28a3d55f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ - [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. - [IMPROVED] Verified library operation on Python 3.6.3. - [IMPROVED] Shortened length of client URLs by removing username and password. +- [FIXED] Case where `Document` context manager would throw instead of creating a new document if no `_id` was provided. # 2.8.1 (2018-02-16) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index e899eff4..caf0b1ab 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -430,11 +430,16 @@ context manager. my_database = client.create_database('my_database') - # Performs a fetch upon entry and a save upon exit of this block - with Document(my_database, 'julia30') as doc: - doc['name'] = 'Julia' - doc['age'] = 30 - doc['pets'] = ['cat', 'dog', 'frog'] + # Upon entry into the document context, fetches the document from the + # remote database, if it exists. Upon exit from the context, saves the + # document to the remote database with changes made within the context + # or creates a new document. + with Document(database, 'julia006') as document: + # If document exists, it's fetched from the remote database + # Changes are made locally + document['name'] = 'Julia' + document['age'] = 6 + # The document is saved to the remote database # Display a Document print(my_database['julia30']) diff --git a/src/cloudant/document.py b/src/cloudant/document.py index 55abc5c2..e8412c3c 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -332,6 +332,9 @@ def __enter__(self): except HTTPError as error: if error.response.status_code != 404: raise + except CloudantDocumentException as error: + if error.status_code != 101: + raise return self diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index ea3c8d33..3881326a 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -540,6 +540,29 @@ def test_document_context_manager(self): self.assertTrue(doc['_rev'].startswith('2-')) self.assertEqual(self.db['julia006'], doc) + def test_document_context_manager_no_doc_id(self): + """ + Test that the __enter__ and __exit__ methods perform as expected + with no document id when initiated through a document context manager + """ + with Document(self.db) as doc: + doc['_id'] = 'julia006' + doc['name'] = 'julia' + doc['age'] = 6 + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(self.db['julia006'], doc) + + def test_document_context_manager_doc_create(self): + """ + Test that the document context manager will create a doc if it does + not yet exist. + """ + with Document(self.db, 'julia006') as doc: + doc['name'] = 'julia' + doc['age'] = 6 + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(self.db['julia006'], doc) + def test_setting_id(self): """ Ensure that proper processing occurs when setting the _id From 6f5edd958da40c85f6ea956124b7241ae40fba92 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 13 Jun 2018 13:50:14 +0100 Subject: [PATCH 085/185] Prepare for version 2.9.0 release --- CHANGES.md | 6 +++--- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 28a3d55f..a3aa4a69 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,12 +1,12 @@ -# Unreleased +# 2.9.0 (2018-06-13) - [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict `__contains__` and checking in the remote database. - [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x. - [NEW] Support IAM authentication in replication documents. +- [FIXED] Case where `Document` context manager would throw instead of creating a new document if no `_id` was provided. - [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. -- [IMPROVED] Verified library operation on Python 3.6.3. - [IMPROVED] Shortened length of client URLs by removing username and password. -- [FIXED] Case where `Document` context manager would throw instead of creating a new document if no `_id` was provided. +- [IMPROVED] Verified library operation on Python 3.6.3. # 2.8.1 (2018-02-16) diff --git a/VERSION b/VERSION index dc3cd3a8..c8e38b61 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.8.2-SNAPSHOT +2.9.0 diff --git a/docs/conf.py b/docs/conf.py index c4e0c7f8..a59073af 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.8.2-SNAPSHOT' +version = '2.9.0' # The full version, including alpha/beta/rc tags. -release = '2.8.2-SNAPSHOT' +release = '2.9.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 51018794..646cc41b 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.8.2-SNAPSHOT' +__version__ = '2.9.0' # pylint: disable=wrong-import-position import contextlib From 0369755d3200967afd2cd61df31b35ae5c967e0b Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 13 Jun 2018 15:32:25 +0100 Subject: [PATCH 086/185] Update version to 2.9.1-SNAPSHOT --- CHANGES.md | 2 ++ VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a3aa4a69..45a88f0d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,5 @@ +# Unreleased + # 2.9.0 (2018-06-13) - [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict `__contains__` and checking in the remote database. diff --git a/VERSION b/VERSION index c8e38b61..43870570 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.9.0 +2.9.1-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index a59073af..21304f8e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.9.0' +version = '2.9.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.9.0' +release = '2.9.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 646cc41b..e34873a0 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.9.0' +__version__ = '2.9.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From 271c4bdfbefdd942b36b3465dd1bd3e22f72b8d9 Mon Sep 17 00:00:00 2001 From: Tom Blench Date: Tue, 19 Jun 2018 11:56:36 +0100 Subject: [PATCH 087/185] Support new view parameters (#388) * Support new view parameters: `stable` and `update` * Copyrights * Grammar (full stop) * Add deprecation note. --- CHANGES.md | 1 + src/cloudant/_common_util.py | 4 ++- src/cloudant/database.py | 11 ++++++- src/cloudant/result.py | 20 +++++++++--- tests/unit/param_translation_tests.py | 10 +++++- tests/unit/view_execution_tests.py | 46 +++++++++++++++++++++++++-- 6 files changed, 81 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 45a88f0d..6aa70c70 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ - [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict `__contains__` and checking in the remote database. - [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x. - [NEW] Support IAM authentication in replication documents. +- [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. - [FIXED] Case where `Document` context manager would throw instead of creating a new document if no `_id` was provided. - [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. - [IMPROVED] Shortened length of client URLs by removing username and password. diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 9e20d226..6d9cca61 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -60,9 +60,11 @@ 'limit': (int, LONGTYPE, NONETYPE,), 'reduce': (bool,), 'skip': (int, LONGTYPE, NONETYPE,), + 'stable': (bool,), 'stale': (STRTYPE,), 'startkey': (int, LONGTYPE, STRTYPE, Sequence,), 'startkey_docid': (STRTYPE,), + 'update': (STRTYPE,), } # pylint: disable=unnecessary-lambda @@ -196,7 +198,7 @@ def _py_to_couch_translate(key, val): equivalent. """ try: - if key in ['keys', 'endkey_docid', 'startkey_docid', 'stale']: + if key in ['keys', 'endkey_docid', 'startkey_docid', 'stale', 'update']: return {key: val} elif val is None: return {key: None} diff --git a/src/cloudant/database.py b/src/cloudant/database.py index e5e94f1e..85cb0313 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -312,15 +312,24 @@ def get_view_result(self, ddoc_id, view_name, raw_result=False, **kwargs): :param bool reduce: True to use the reduce function, false otherwise. :param int skip: Skip this number of rows from the start. Not valid when used with :class:`~cloudant.result.Result` iteration. + :param bool stable: Whether or not the view results should be returned + from a "stable" set of shards. :param str stale: Allow the results from a stale view to be used. This makes the request return immediately, even if the view has not been completely built yet. If this parameter is not given, a response is - returned only after the view has been built. + returned only after the view has been built. Note that this + parameter is deprecated and the appropriate combination of `stable` + and `update` should be used instead. :param startkey: Return records starting with the specified key. Not valid when used with :class:`~cloudant.result.Result` key access and key slicing. :param str startkey_docid: Return records starting with the specified document ID. + :param str update: Determine whether the view in question should be + updated prior to or after responding to the user. Valid values are: + false: return results before updating the view; true: Return results + after updating the view; lazy: Return the view results without + waiting for an update, but update them immediately after the request. :returns: The result content either wrapped in a QueryResult or as the raw response JSON content diff --git a/src/cloudant/result.py b/src/cloudant/result.py index f4be5f1f..230811f1 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -150,14 +150,24 @@ class Result(object): :param bool reduce: True to use the reduce function, false otherwise. :param int skip: Skip this number of rows from the start. Not valid when used with key iteration. - :param str stale: Allow the results from a stale view to be used. This - makes the request return immediately, even if the view has not been - completely built yet. If this parameter is not given, a response is - returned only after the view has been built. + :param bool stable: Whether or not the view results should be returned from + a "stable" set of shards. + :param str stale: Allow the results from a stale view to be used. This makes + the request return immediately, even if the view has not been completely + built yet. If this parameter is not given, a response is returned only + after the view has been built. Note that this parameter is deprecated + and the appropriate combination of `stable` and `update` should be used + instead. :param startkey: Return records starting with the specified key. Not valid when used with key access and key slicing. :param str startkey_docid: Return records starting with the specified document ID. + :param str update: Determine whether the view in question should be + updated prior to or after responding to the user. Valid values are: + false: return results before updating the view; true: Return results + after updating the view; lazy: Return the view results without + waiting for an update, but update them immediately after the request. + """ def __init__(self, method_ref, **options): self.options = options diff --git a/tests/unit/param_translation_tests.py b/tests/unit/param_translation_tests.py index 0fca5e09..4da0d05d 100644 --- a/tests/unit/param_translation_tests.py +++ b/tests/unit/param_translation_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -215,6 +215,14 @@ def test_valid_startkey_docid(self): {'startkey_docid': 'foo'} ) + def test_valid_update(self): + """ + Test lazy translation is successful. + """ + self.assertEqual(python_to_couch({'update': 'true'}), {'update': 'true'}) + self.assertEqual(python_to_couch({'update': 'false'}), {'update': 'false'}) + self.assertEqual(python_to_couch({'update': 'lazy'}), {'update': 'lazy'}) + def test_invalid_argument(self): """ Test translation fails when an invalid argument is passed in. diff --git a/tests/unit/view_execution_tests.py b/tests/unit/view_execution_tests.py index 680508f5..4ac97dc5 100644 --- a/tests/unit/view_execution_tests.py +++ b/tests/unit/view_execution_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -467,7 +467,7 @@ def test_stale_ok(self): try: self.view001(stale='ok') except Exception as err: - self.assertFail(str(err), 'An unexpected error was encountered.') + self.fail('An unexpected error was encountered: '+str(err)) def test_stale_update_after(self): """ @@ -480,8 +480,48 @@ def test_stale_update_after(self): try: self.view001(stale='update_after') except Exception as err: - self.assertFail(str(err), 'An unexpected error was encountered.') + self.fail('An unexpected error was encountered:' +str(err)) + + def test_stable_true(self): + """ + Test view query using the stable parameter set to true + + + Since there is no way to know whether the view will return a response from a stable set of + shards or not the test here focuses on ensuring that the call itself is successful. + + """ + try: + self.view001(stable=True) + except Exception as err: + self.fail('An unexpected error was encountered: '+str(err)) + + def test_stable_update_lazy(self): + """ + Test view query using the update parameter set to lazy + Since there is no way to know whether the view will update lazily or not the test here + focuses on ensuring that the call itself is successful. + + """ + try: + self.view001(update='lazy') + except Exception as err: + self.fail('An unexpected error was encountered: '+str(err)) + + def test_stable_update_true(self): + """ + Test view query using the update parameter set to true + + Since there is no way to know whether the view will update or not the test here focuses on + ensuring that the call itself is successful. + + """ + try: + self.view001(update='true') + except Exception as err: + self.fail('An unexpected error was encountered: '+str(err)) + def test_startkey_int(self): """ Test view query using startkey parameter as an integer. From d15ea5105ef99496768294c68405234ca5d8212f Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 20 Jun 2018 17:01:44 +0100 Subject: [PATCH 088/185] Template updates (#391) * Updated issue and PR templates * Updated CONTRIBUTING information --- .github/ISSUE_TEMPLATE.md | 26 ++++++-- .github/PULL_REQUEST_TEMPLATE.md | 103 +++++++++++++++++++++++++++---- CONTRIBUTING.md | 84 +++++++++++++++++++++++++ CONTRIBUTING.rst | 75 ---------------------- 4 files changed, 197 insertions(+), 91 deletions(-) create mode 100644 CONTRIBUTING.md delete mode 100644 CONTRIBUTING.rst diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index ba295fa1..95d86bf6 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,5 +1,23 @@ -Please include the following information in your ticket. +Please [read these guidelines](http://ibm.biz/cdt-issue-guide) before opening an issue. -- Cloudant (python-cloudant) version(s) that are affected by this issue. -- Python version -- A small code sample that demonstrates the issue. + + +## Bug Description + +### 1. Steps to reproduce and the simplest code sample possible to demonstrate the issue + + +### 2. What you expected to happen + +### 3. What actually happened + +## Environment details + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e80303d9..8a9c853a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,23 +1,102 @@ + +## Checklist -- [ ] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](https://github.com/cloudant/python-cloudant/blob/master/DCO1.1.txt) -- [ ] You have added tests for any code changes -- [ ] You have updated the [CHANGES.md](https://github.com/cloudant/python-cloudant/blob/master/CHANGES.md) -- [ ] You have completed the PR template below: +- [ ] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](../blob/master/DCO1.1.txt) +- [ ] Added tests for code changes _or_ test/build only changes +- [ ] Updated the change log file (`CHANGES.md`|`CHANGELOG.md`) _or_ test/build only changes +- [ ] Completed the PR template below: -## What +## Description + + +## Approach + + + +## Schema & API Changes + + + +## Security and Privacy + + ## Testing -How to test your changes work, not required for documentation changes. + + +## Monitoring and Logging + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..21890316 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,84 @@ +# Contributing + +## Issues + +Please [read these guidelines](http://ibm.biz/cdt-issue-guide) before opening an issue. +If you still need to open an issue then we ask that you complete the template as +fully as possible. + +## Pull requests + +We welcome pull requests, but ask contributors to keep in mind the following: + +* Only PRs with the template completed will be accepted +* We will not accept PRs for user specific functionality + +### Developer Certificate of Origin + +In order for us to accept pull-requests, the contributor must sign-off a +[Developer Certificate of Origin (DCO)](DCO1.1.txt). This clarifies the +intellectual property license granted with any contribution. It is for your +protection as a Contributor as well as the protection of IBM and its customers; +it does not change your rights to use your own Contributions for any other purpose. + +Please read the agreement and acknowledge it by ticking the appropriate box in the PR + text, for example: + +- [x] Tick to sign-off your agreement to the Developer Certificate of Origin (DCO) 1.1 + +## General information + +Python-Cloudant Client Library is written in Python. + +## Requirements + +- Python +- pip + +It is recommended to use a [virtual environment](https://virtualenv.pypa.io/en/latest) during development. The +python-cloudant dependencies can be installed via the `requirements.txt` file using pip. + +For example to create a virtualenv and install requirements: + +```sh +virtualenv . +./bin/activate +pip install -r requirements.txt +pip install -r test-requirements.txt +``` + +## Testing + +The tests need an Apache CouchDB or Cloudant service to run against. + +The tests create databases in your CouchDB instance, these are `db-`. +They also create and delete documents in the `_replicator` database. + +The tests are run with the `nosetests` runner. In this example the `ADMIN_PARTY` environment variable is used to tell + the tests not to use any authentication. See below for the full set of variables that can be used. + +```sh +$ ADMIN_PARTY=true nosetests -w ./tests/unit +``` + +There are several environment variables which affect +test behaviour: + +- `RUN_CLOUDANT_TESTS`: set this to run the tests that use Cloudant-specific features. If + you set this, you must set one of the following combinations of other variables: + - `DB_URL`, `DB_USER` and `DB_PASSWORD`. + - `CLOUDANT_ACCOUNT`, `DB_USER` and `DB_PASSWORD`. + - If you set both `DB_URL` and `CLOUDANT_ACCOUNT`, `DB_URL` is used as the + URL to make requests to and `CLOUDANT_ACCOUNT` is inserted into the `X-Cloudant-User` + header. +- Without `RUN_CLOUDANT_TESTS`, the following environment variables have an effect: + - Set `DB_URL` to set the root URL of the CouchDB/Cloudant instance. It defaults + to `http://localhost:5984`. + - Set `ADMIN_PARTY` to `true` to not use any authentication details. + - Without `ADMIN_PARTY`, set `DB_USER` and `DB_PASSWORD` to use those + credentials to access the database. + - Without `ADMIN_PARTY` and `DB_USER`, the tests assume CouchDB is in + admin party mode, but create a user via `_config` to run tests as. + This user is deleted at the end of the test run, but beware it'll + break other applications using the CouchDB instance that rely on + admin party mode being in effect while the tests are running. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst deleted file mode 100644 index b4b41f99..00000000 --- a/CONTRIBUTING.rst +++ /dev/null @@ -1,75 +0,0 @@ -Developing this library -======================= - -Python-Cloudant Client Library is written in Python. - -=============================== -Developer Certificate of Origin -=============================== - -In order for us to accept pull-requests, the contributor must sign-off a -`Developer Certificate of Origin (DCO) `_. This clarifies the -intellectual property license granted with any contribution. It is for your -protection as a Contributor as well as the protection of IBM and its customers; -it does not change your rights to use your own Contributions for any other -purpose. - -Please read the agreement and acknowledge it by ticking the appropriate box in -the PR text, for example: - -- [x] Tick to sign-off your agreement to the Developer Certificate of Origin (DCO) 1.1 - -====================== -Development Quickstart -====================== - -Clone the repo into a folder, set up a `virtual environment `_, -install the requirements: - -.. code-block:: bash - - $ git clone git clone git@github.com:cloudant/python-cloudant.git - $ cd python-cloudant - $ virtualenv . - $ ./bin/activate - $ pip install -r requirements.txt - $ pip install -r test-requirements.txt - -Before running the tests, start CouchDB: - -.. code-block:: bash - - $ couchdb - -The tests create databases in your CouchDB instance, these are `db-`. -They also create and delete documents in the `_replicator` database. - -Now, run the tests. Here, I use the ``ADMIN_PARTY`` environment variable to -tell the tests not to use any authentication. See below for the full set of -variables that can be used. - -.. code-block:: bash - - $ ADMIN_PARTY=true nosetests -w ./tests/unit - -There are several environment variables which affect -test behaviour: - -- ``RUN_CLOUDANT_TESTS``: set this to run the tests that use Cloudant-specific features. If - you set this, you must set one of the following combinations of other variables: - - ``DB_URL``, ``DB_USER`` and ``DB_PASSWORD``. - - ``CLOUDANT_ACCOUNT``, ``DB_USER`` and ``DB_PASSWORD``. - - If you set both ``DB_URL`` and ``CLOUDANT_ACCOUNT``, ``DB_URL`` is used as the - URL to make requests to and ``CLOUDANT_ACCOUNT`` is inserted into the ``X-Cloudant-User`` - header. -- Without ``RUN_CLOUDANT_TESTS``, the following environment variables have an effect: - - Set ``DB_URL`` to set the root URL of the CouchDB/Cloudant instance. It defaults - to ``http://localhost:5984``. - - Set ``ADMIN_PARTY`` to ``true`` to not use any authentication details. - - Without ``ADMIN_PARTY``, set ``DB_USER`` and ``DB_PASSWORD`` to use those - credentials to access the database. - - Without ``ADMIN_PARTY`` and ``DB_USER``, the tests assume CouchDB is in - admin party mode, but create a user via ``_config`` to run tests as. - This user is deleted at the end of the test run, but beware it'll - break other applications using the CouchDB instance that rely on - admin party mode being in effect while the tests are running. From cb0c1a4f161d2b428c588b91e351790069ddd880 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Thu, 28 Jun 2018 12:34:47 -0400 Subject: [PATCH 089/185] Fixed CONTRIBUTING links --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 34e777e8..0cc5edcf 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ This is the official Cloudant library for Python. * [API Reference](http://python-cloudant.readthedocs.io/en/latest/cloudant.html) * [Related Documentation](#related-documentation) * [Development](#development) - * [Contributing](CONTRIBUTING.rst) - * [Test Suite](CONTRIBUTING.rst#running-the-tests) + * [Contributing](CONTRIBUTING.md) + * [Test Suite](CONTRIBUTING.md#running-the-tests) * [Using in Other Projects](#using-in-other-projects) * [License](#license) * [Issues](#issues) @@ -44,7 +44,7 @@ See [API reference docs (readthedocs.io)](http://python-cloudant.readthedocs.io/ ## Development -See [CONTRIBUTING.rst](https://github.com/cloudant/python-cloudant/blob/master/CONTRIBUTING.rst) +See [CONTRIBUTING.md](https://github.com/cloudant/python-cloudant/blob/master/CONTRIBUTING.md) ## Using in other projects From f0d00100412d84452d69f5464054be0d1cf74557 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Tue, 3 Jul 2018 13:21:32 +0100 Subject: [PATCH 090/185] Corrected bad merge of CHANGES --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 6aa70c70..432ae241 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,11 +1,12 @@ # Unreleased +- [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. + # 2.9.0 (2018-06-13) - [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict `__contains__` and checking in the remote database. - [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x. - [NEW] Support IAM authentication in replication documents. -- [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. - [FIXED] Case where `Document` context manager would throw instead of creating a new document if no `_id` was provided. - [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. - [IMPROVED] Shortened length of client URLs by removing username and password. From fa1cf06ef36fd4a8270c7b477d76601b0b399b91 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Mon, 2 Jul 2018 17:01:26 +0100 Subject: [PATCH 091/185] Stopped raising exception on conflict retry success Moved exception raise into else block. Added tests for conflict retry cases: i) max_tries exhuastion ii) success on retry --- CHANGES.md | 1 + src/cloudant/document.py | 3 +- tests/unit/document_tests.py | 56 ++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 432ae241..1ca11aa0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # Unreleased - [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. +- [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. # 2.9.0 (2018-06-13) diff --git a/src/cloudant/document.py b/src/cloudant/document.py index e8412c3c..768ecca5 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -259,7 +259,8 @@ def _update_field(self, action, field, value, max_tries, tries=0): if tries < max_tries and ex.response.status_code == 409: self._update_field( action, field, value, max_tries, tries=tries+1) - raise + else: + raise def update_field(self, action, field, value, max_tries=10): """ diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 3881326a..dd443ccf 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -474,6 +474,62 @@ def test_update_field(self): self.assertTrue(doc['_rev'].startswith('2-')) self.assertEqual(doc['pets'], ['cat', 'dog', 'fish']) + @mock.patch('cloudant.document.Document.save') + def test_update_field_maxretries(self, m_save): + """ + Test that conflict retries work for updating a single field. + """ + # Create a doc + doc = Document(self.db, 'julia006') + doc['name'] = 'julia' + doc['age'] = 6 + doc.create() + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(doc['age'], 6) + # Mock conflicts when saving updates + m_save.side_effect = requests.HTTPError(response=mock.Mock(status_code=409, reason='conflict')) + # Tests that failing on retry eventually throws + with self.assertRaises(requests.HTTPError) as cm: + doc.update_field(doc.field_set, 'age', 7, max_tries=2) + + # There is an off-by-one error for "max_tries" + # It really means max_retries i.e. 1 attempt + # followed by a max of 2 retries + self.assertEqual(m_save.call_count, 3) + self.assertEqual(cm.exception.response.status_code, 409) + self.assertEqual(cm.exception.response.reason, 'conflict') + # Fetch again before asserting, otherwise we assert against + # the locally updated age field + doc.fetch() + self.assertFalse(doc['_rev'].startswith('2-')) + self.assertNotEqual(doc['age'], 7) + + def test_update_field_success_on_retry(self): + """ + Test that conflict retries work for updating a single field. + """ + # Create a doc + doc = Document(self.db, 'julia006') + doc['name'] = 'julia' + doc['age'] = 6 + doc.create() + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(doc['age'], 6) + + # Mock when saving the document + # 1st call throw a 409 + # 2nd call delegate to the real doc.save() + with mock.patch('cloudant.document.Document.save', + side_effect=[requests.HTTPError(response=mock.Mock(status_code=409, reason='conflict')), + doc.save()]) as m_save: + # A list of side effects containing only 1 element + doc.update_field(doc.field_set, 'age', 7, max_tries=1) + # Two calls to save, one with a 409 and one that succeeds + self.assertEqual(m_save.call_count, 2) + # Check that the _rev and age field were updated + self.assertTrue(doc['_rev'].startswith('2-')) + self.assertEqual(doc['age'], 7) + def test_delete_document_failure(self): """ Test failure condition when attempting to remove a document From 82017411beb12717780d036f7308086d8fe37ef5 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 9 Jul 2018 16:00:46 +0100 Subject: [PATCH 092/185] Add custom JSON encoder/decoder option to `Document` constructor --- CHANGES.md | 1 + src/cloudant/document.py | 9 +++++--- tests/unit/document_tests.py | 44 ++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1ca11aa0..3c3e925a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # Unreleased +- [NEW] Add custom JSON encoder/decoder option to `Document` constructor. - [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. - [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. diff --git a/src/cloudant/document.py b/src/cloudant/document.py index 768ecca5..7feda18a 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -53,8 +53,10 @@ class Document(dict): :param database: A database instance used by the Document. Can be either a ``CouchDatabase`` or ``CloudantDatabase`` instance. :param str document_id: Optional document id used to identify the document. + :param str encoder: Optional JSON encoder object. + :param str decoder: Optional JSON decoder object. """ - def __init__(self, database, document_id=None): + def __init__(self, database, document_id=None, **kwargs): super(Document, self).__init__() self._client = database.client self._database = database @@ -63,7 +65,8 @@ def __init__(self, database, document_id=None): self._document_id = document_id if self._document_id is not None: self['_id'] = self._document_id - self.encoder = self._client.encoder + self.encoder = kwargs.get('encoder') or self._client.encoder + self.decoder = kwargs.get('decoder') or json.JSONDecoder @property def r_session(self): @@ -165,7 +168,7 @@ def fetch(self): resp = self.r_session.get(self.document_url) resp.raise_for_status() self.clear() - self.update(resp.json()) + self.update(resp.json(cls=self.decoder)) def save(self): """ diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index dd443ccf..3c7b5afc 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -30,6 +30,8 @@ import uuid import inspect +from datetime import datetime + from cloudant.document import Document from cloudant.error import CloudantDocumentException @@ -859,5 +861,47 @@ def test_document_request_fails_after_client_disconnects(self): finally: self.client.connect() + def test_document_custom_json_encoder_and_decoder(self): + dt_format = '%Y-%m-%dT%H:%M:%S' + + class DTEncoder(json.JSONEncoder): + + def default(self, obj): + if isinstance(obj, datetime): + return { + '_type': 'datetime', + 'value': obj.strftime(dt_format) + } + return super(DTEncoder, self).default(obj) + + class DTDecoder(json.JSONDecoder): + + def __init__(self, *args, **kwargs): + json.JSONDecoder.__init__(self, object_hook=self.object_hook, + *args, **kwargs) + + def object_hook(self, obj): + if '_type' not in obj: + return obj + if obj['_type'] == 'datetime': + return datetime.strptime(obj['value'], dt_format) + return obj + + doc = Document(self.db, encoder=DTEncoder) + doc['name'] = 'julia' + doc['dt'] = datetime(2018, 7, 9, 15, 11, 10, 0) + doc.save() + + raw_doc = self.db.all_docs(include_docs=True)['rows'][0]['doc'] + + self.assertEquals(raw_doc['name'], 'julia') + self.assertEquals(raw_doc['dt']['_type'], 'datetime') + self.assertEquals(raw_doc['dt']['value'], '2018-07-09T15:11:10') + + doc2 = Document(self.db, doc['_id'], decoder=DTDecoder) + doc2.fetch() + + self.assertEquals(doc2['dt'], doc['dt']) + if __name__ == '__main__': unittest.main() From 9bd77189a47527ac8f7d157ba88c90c552c7be87 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Wed, 18 Jul 2018 13:33:00 -0400 Subject: [PATCH 093/185] Added connect parameter to CouchDB client example Issue #374 --- docs/getting_started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index caf0b1ab..a1cd43f7 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -38,7 +38,7 @@ Connecting with a client # Use CouchDB to create a CouchDB client # from cloudant.client import CouchDB - # client = CouchDB(USERNAME, PASSWORD, url='http://127.0.0.1:5984') + # client = CouchDB(USERNAME, PASSWORD, url='http://127.0.0.1:5984', connect=True) # Use Cloudant to create a Cloudant client using account from cloudant.client import Cloudant From a01e95ae7574fdbdf4e3c96ab4330e3ef668ab21 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Mon, 13 Aug 2018 18:41:19 -0400 Subject: [PATCH 094/185] Fixed pylint error messages - useless-object-inheritance warning message added to the disable list in pylintrc --- pylintrc | 3 ++- src/cloudant/_common_util.py | 4 ++-- src/cloudant/client.py | 2 +- src/cloudant/database.py | 4 ++-- src/cloudant/document.py | 14 ++++++-------- src/cloudant/feed.py | 3 +-- src/cloudant/index.py | 2 -- src/cloudant/result.py | 10 +++++----- src/cloudant/scheduler.py | 8 ++++---- 9 files changed, 23 insertions(+), 27 deletions(-) diff --git a/pylintrc b/pylintrc index 4cfbabaf..0b4f402d 100644 --- a/pylintrc +++ b/pylintrc @@ -66,7 +66,8 @@ confidence= # Disable "redefined-variable-type" refactor warning messages # Disable "too-many-..." and "too-few-..." refactor warning messages # Disable "locally-disabled" message -disable=R0204,R0901,R0902,R0903,R0904,R0913,R0914,R0915,locally-disabled,keyword-arg-before-vararg +# Disable Python 3 "useless-object-inheritance" message +disable=R0204,R0901,R0902,R0903,R0904,R0913,R0914,R0915,locally-disabled,keyword-arg-before-vararg,useless-object-inheritance [REPORTS] diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 6d9cca61..2af49b34 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -147,7 +147,7 @@ def feed_arg_types(feed_type): """ if feed_type == 'Cloudant': return _DB_UPDATES_ARG_TYPES - elif feed_type == 'CouchDB': + if feed_type == 'CouchDB': return _COUCH_DB_UPDATES_ARG_TYPES return _CHANGES_ARG_TYPES @@ -200,7 +200,7 @@ def _py_to_couch_translate(key, val): try: if key in ['keys', 'endkey_docid', 'startkey_docid', 'stale', 'update']: return {key: val} - elif val is None: + if val is None: return {key: None} arg_converter = TYPE_CONVERTERS.get(type(val)) return {key: arg_converter(val)} diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 6c9b5f33..06a6092c 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -387,9 +387,9 @@ def __getitem__(self, key): db = self._DATABASE_CLASS(self, key) if db.exists(): super(CouchDB, self).__setitem__(key, db) - return db else: raise KeyError(key) + return db def __delitem__(self, key, remote=False): """ diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 85cb0313..ec6c8b7a 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -337,7 +337,7 @@ def get_view_result(self, ddoc_id, view_name, raw_result=False, **kwargs): view = View(DesignDocument(self, ddoc_id), view_name) if raw_result: return view(**kwargs) - elif kwargs: + if kwargs: return Result(view, **kwargs) return view.result @@ -613,9 +613,9 @@ def __getitem__(self, key): if doc.exists(): doc.fetch() super(CouchDatabase, self).__setitem__(key, doc) - return doc else: raise KeyError(key) + return doc def __contains__(self, key): """ diff --git a/src/cloudant/document.py b/src/cloudant/document.py index 7feda18a..ac08ee65 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -112,10 +112,10 @@ def exists(self): """ if self._document_id is None: return False - else: - resp = self.r_session.head(self.document_url) - if resp.status_code not in [200, 404]: - resp.raise_for_status() + + resp = self.r_session.head(self.document_url) + if resp.status_code not in [200, 404]: + resp.raise_for_status() return resp.status_code == 200 @@ -154,7 +154,6 @@ def create(self): self._document_id = data['id'] super(Document, self).__setitem__('_id', data['id']) super(Document, self).__setitem__('_rev', data['rev']) - return def fetch(self): """ @@ -320,7 +319,6 @@ def delete(self): del_resp.raise_for_status() self.clear() self.__setitem__('_id', self._document_id) - return def __enter__(self): """ @@ -413,13 +411,13 @@ def get_attachment( attachment_type = 'binary' if write_to is not None: - if attachment_type == 'text' or attachment_type == 'json': + if attachment_type in ('text', 'json'): write_to.write(resp.text) else: write_to.write(resp.content) if attachment_type == 'text': return resp.text - elif attachment_type == 'json': + if attachment_type == 'json': return resp.json() return resp.content diff --git a/src/cloudant/feed.py b/src/cloudant/feed.py index 44fa3525..4a93fc16 100644 --- a/src/cloudant/feed.py +++ b/src/cloudant/feed.py @@ -164,8 +164,7 @@ def _process_data(self, line): skip = False if self._raw_data: return skip, line - else: - line = unicode_(line) + line = unicode_(line) if not line: if (self._options.get('heartbeat', False) and self._options.get('feed') in ('continuous', 'longpoll') and diff --git a/src/cloudant/index.py b/src/cloudant/index.py index 5f34b76b..3d0d49c3 100644 --- a/src/cloudant/index.py +++ b/src/cloudant/index.py @@ -145,7 +145,6 @@ def create(self): resp.raise_for_status() self._ddoc_id = resp.json()['id'] self._name = resp.json()['name'] - return def _def_check(self): """ @@ -168,7 +167,6 @@ def delete(self): url = '/'.join((self.index_url, ddoc_id, self._type, self._name)) resp = self._r_session.delete(url) resp.raise_for_status() - return class TextIndex(Index): """ diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 230811f1..53718c7e 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -268,13 +268,13 @@ def _handle_result_by_idx_slice(self, idx_slice): start = idx_slice.start stop = idx_slice.stop data = None - if (start is not None and stop is not None and - start >= 0 and stop >= 0 and start < stop): + # start and stop cannot be None and both must be greater than 0 + if all(i is not None and i >= 0 for i in [start, stop]) and start < stop: if limit is not None: if start >= limit: # Result is out of range return dict() - elif stop > limit: + if stop > limit: # Ensure that slice does not extend past original limit return self._ref(skip=skip+start, limit=limit-start, **opts) data = self._ref(skip=skip+start, limit=stop-start, **opts) @@ -498,8 +498,8 @@ def __getitem__(self, arg): type_or_none(int, arg.start) and type_or_none(int, arg.stop))): return super(QueryResult, self).__getitem__(arg) - else: - raise ResultException(101, arg) + + raise ResultException(101, arg) def _parse_data(self, data): """ diff --git a/src/cloudant/scheduler.py b/src/cloudant/scheduler.py index 76cdc101..723635b2 100644 --- a/src/cloudant/scheduler.py +++ b/src/cloudant/scheduler.py @@ -41,9 +41,9 @@ def list_docs(self, limit=None, skip=None): :param skip: How many result to skip starting at the beginning, if ordered by document ID. """ params = dict() - if limit != None: + if limit is not None: params["limit"] = limit - if skip != None: + if skip is not None: params["skip"] = skip resp = self._r_session.get('/'.join([self._scheduler, 'docs']), params=params) resp.raise_for_status() @@ -72,9 +72,9 @@ def list_jobs(self, limit=None, skip=None): :param skip: How many result to skip starting at the beginning, if ordered by document ID. """ params = dict() - if limit != None: + if limit is not None: params["limit"] = limit - if skip != None: + if skip is not None: params["skip"] = skip resp = self._r_session.get('/'.join([self._scheduler, 'jobs']), params=params) resp.raise_for_status() From e1c4846c8e1acd66a79ef42a4011f2863425b1aa Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Tue, 4 Sep 2018 17:06:37 -0400 Subject: [PATCH 095/185] Added if statement to break from while loop when the result set is less than page size - Converted page_size to int once and removed any additional conversions --- CHANGES.md | 1 + src/cloudant/result.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3c3e925a..64b5f9c0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,7 @@ - [NEW] Add custom JSON encoder/decoder option to `Document` constructor. - [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. - [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. +- [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the 'page_size' value # 2.9.0 (2018-06-13) diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 53718c7e..302b7128 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -341,7 +341,8 @@ def __iter__(self): raise ResultException(103, invalid_options, self.options) try: - if int(self._page_size) <= 0: + self._page_size = int(self._page_size) + if self._page_size <= 0: raise ResultException(104, self._page_size) except ValueError: raise ResultException(104, self._page_size) @@ -349,15 +350,17 @@ def __iter__(self): skip = 0 while True: response = self._ref( - limit=int(self._page_size), + limit=self._page_size, skip=skip, **self.options ) result = self._parse_data(response) - skip += int(self._page_size) + skip += self._page_size if result: for row in result: yield row + if len(result) < self._page_size: + break del result else: break From 03385dc195d554a86dc2e143514ab59fd0c4b96e Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 11 Sep 2018 16:06:57 +0100 Subject: [PATCH 096/185] Allow query parameters to be passed to custom changes filters --- CHANGES.md | 1 + src/cloudant/_common_util.py | 4 ++++ src/cloudant/feed.py | 21 ++++++++++++++------- tests/unit/changes_tests.py | 18 ++++++++++++------ 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 64b5f9c0..5efcf9b6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ - [NEW] Add custom JSON encoder/decoder option to `Document` constructor. - [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. +- [NEW] Allow arbitrary query parameters to be passed to custom changes filters. - [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. - [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the 'page_size' value diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 2af49b34..1904a0a4 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -47,6 +47,9 @@ # Argument Types +ANY_ARG = object() +ANY_TYPE = object() + RESULT_ARG_TYPES = { 'descending': (bool,), 'endkey': (int, LONGTYPE, STRTYPE, Sequence,), @@ -101,6 +104,7 @@ 'filter': (STRTYPE,), 'include_docs': (bool,), 'style': (STRTYPE,), + ANY_ARG: ANY_TYPE # pass arbitrary query parameters to a custom filter } _CHANGES_ARG_TYPES.update(_DB_UPDATES_ARG_TYPES) diff --git a/src/cloudant/feed.py b/src/cloudant/feed.py index 4a93fc16..b02b79d6 100644 --- a/src/cloudant/feed.py +++ b/src/cloudant/feed.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2016 IBM. All rights reserved. +# Copyright (c) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ from ._2to3 import iteritems_, next_, unicode_, STRTYPE, NONETYPE from .error import CloudantArgumentError, CloudantFeedException -from ._common_util import feed_arg_types, TYPE_CONVERTERS +from ._common_util import ANY_ARG, ANY_TYPE, feed_arg_types, TYPE_CONVERTERS class Feed(object): """ @@ -111,11 +111,18 @@ def _validate(self, key, val, arg_types): Ensures that the key and the value are valid arguments to be used with the feed. """ - if key not in arg_types: - raise CloudantArgumentError(116, key) - if (not isinstance(val, arg_types[key]) or - (isinstance(val, bool) and int in arg_types[key])): - raise CloudantArgumentError(117, key, arg_types[key]) + if key in arg_types: + arg_type = arg_types[key] + else: + if ANY_ARG not in arg_types: + raise CloudantArgumentError(116, key) + arg_type = arg_types[ANY_ARG] + + if arg_type == ANY_TYPE: + return + if (not isinstance(val, arg_type) or + (isinstance(val, bool) and int in arg_type)): + raise CloudantArgumentError(117, key, arg_type) if isinstance(val, int) and val < 0 and not isinstance(val, bool): raise CloudantArgumentError(118, key, val) if key == 'feed': diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index ee3dce13..007e07a6 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -465,14 +465,20 @@ def test_get_feed_using_doc_ids(self): self.assertSetEqual(set([x['id'] for x in changes]), expected) self.assertTrue(str(feed.last_seq).startswith('100')) - def test_invalid_argument(self): + def test_get_feed_with_custom_filter_query_params(self): """ - Test that an invalid argument is caught and an exception is raised + Test using feed with custom filter query parameters. """ - feed = Feed(self.db, foo='bar') - with self.assertRaises(CloudantArgumentError) as cm: - invalid_feed = [x for x in feed] - self.assertEqual(str(cm.exception), 'Invalid argument foo') + feed = Feed( + self.db, + filter='mailbox/new_mail', + foo='bar', # query parameters to a custom filter + include_docs=False + ) + params = feed._translate(feed._options) + self.assertEquals(params['filter'], 'mailbox/new_mail') + self.assertEquals(params['foo'], 'bar') + self.assertEquals(params['include_docs'], 'false') def test_invalid_argument_type(self): """ From c74b7d8e0e37ba6b64fd4f9657157b220c550455 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 13 Sep 2018 11:10:22 +0100 Subject: [PATCH 097/185] Use json.dumps as default type converter --- src/cloudant/feed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloudant/feed.py b/src/cloudant/feed.py index b02b79d6..ef2c90fc 100644 --- a/src/cloudant/feed.py +++ b/src/cloudant/feed.py @@ -100,7 +100,7 @@ def _translate(self, options): if isinstance(val, STRTYPE): translation[key] = val elif not isinstance(val, NONETYPE): - arg_converter = TYPE_CONVERTERS.get(type(val)) + arg_converter = TYPE_CONVERTERS.get(type(val), json.dumps) translation[key] = arg_converter(val) except Exception as ex: raise CloudantArgumentError(115, key, ex) From c28a4b5fe1fe806c2ce60d06b4d2b048c15cf579 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 19 Sep 2018 09:47:37 +0100 Subject: [PATCH 098/185] Prepare for version 2.10.0 release --- CHANGES.md | 4 ++-- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5efcf9b6..e1ddefd2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,10 @@ -# Unreleased +# 2.10.0 (2018-09-19) - [NEW] Add custom JSON encoder/decoder option to `Document` constructor. - [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. - [NEW] Allow arbitrary query parameters to be passed to custom changes filters. - [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. -- [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the 'page_size' value +- [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the `page_size` value. # 2.9.0 (2018-06-13) diff --git a/VERSION b/VERSION index 43870570..10c2c0c3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.9.1-SNAPSHOT +2.10.0 diff --git a/docs/conf.py b/docs/conf.py index 21304f8e..688375cc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.9.1-SNAPSHOT' +version = '2.10.0' # The full version, including alpha/beta/rc tags. -release = '2.9.1-SNAPSHOT' +release = '2.10.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index e34873a0..04dd3a06 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.9.1-SNAPSHOT' +__version__ = '2.10.0' # pylint: disable=wrong-import-position import contextlib From a54349d712d12cd2c8cafdb1937736f5533afada Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 20 Sep 2018 16:52:52 +0100 Subject: [PATCH 099/185] Update version to 2.11.0-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 10c2c0c3..d33489b3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10.0 +2.11.0-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index 688375cc..72e556ae 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.10.0' +version = '2.11.0-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.10.0' +release = '2.11.0-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 04dd3a06..ffb150c2 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.10.0' +__version__ = '2.11.0-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From 48b10c720f28d7a8eda4cbb2239c7c0da214b4a3 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 4 Oct 2018 15:08:07 +0100 Subject: [PATCH 100/185] Added test attributes Tagged tests that require a real database: * database name `cloudant` or `couch` * couchapi for tests that only run on a specific api level Added test filtering by attributes. Removed test skips now controlled by attributes. Added test attribute information to contributing. --- .travis.yml | 2 +- CONTRIBUTING.md | 9 +++++++ Jenkinsfile | 2 +- tests/unit/auth_renewal_tests.py | 9 ++++--- tests/unit/changes_tests.py | 16 +++++------ tests/unit/client_tests.py | 34 +++++++++++------------ tests/unit/database_tests.py | 39 +++++++++++---------------- tests/unit/db_updates_tests.py | 16 +++++------ tests/unit/design_document_tests.py | 36 +++++++++---------------- tests/unit/document_tests.py | 14 +++++----- tests/unit/index_tests.py | 27 +++++++++---------- tests/unit/infinite_feed_tests.py | 24 ++++++++--------- tests/unit/query_result_tests.py | 7 ++--- tests/unit/query_tests.py | 14 +++++----- tests/unit/replicator_tests.py | 16 ++++++----- tests/unit/result_tests.py | 8 +++--- tests/unit/security_document_tests.py | 8 +++--- tests/unit/view_execution_tests.py | 4 +++ tests/unit/view_tests.py | 12 +++++---- 19 files changed, 149 insertions(+), 148 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5fabac04..8a88780e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ before_script: # command to run tests script: - pylint ./src/cloudant - - nosetests -w ./tests/unit + - nosetests -A 'not db or ((db is "couch" or "couch" in db) and (not couchapi or couchapi <='${COUCHDB_VERSION:0:1}'))' -w ./tests/unit notifications: email: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 21890316..7f9e5300 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,3 +82,12 @@ test behaviour: This user is deleted at the end of the test run, but beware it'll break other applications using the CouchDB instance that rely on admin party mode being in effect while the tests are running. + +### Test attributes + +Database tests also have node attributes. Currently there are these attributes: +`db` - `cloudant` and/or `couch` +`couchapi` - Apache CouchDB major version number (i.e. API level) e.g. `2` + +Example to run database tests that require CouchDB version 1 API and no Cloudant features: +`nosetests -A 'db and ((db is "couch" or "couch" in db) and (not couchapi or couchapi <=1))' -w ./tests/unit` diff --git a/Jenkinsfile b/Jenkinsfile index f1c24d9c..a806771f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,7 +37,7 @@ def setupPythonAndTest(pythonVersion, testSuite) { pip install -r requirements.txt pip install -r test-requirements.txt pylint ./src/cloudant - nosetests -w ./tests/unit --with-xunit + nosetests -A 'not db or (db is "cloudant" or "cloudant" in db)' -w ./tests/unit --with-xunit """ } finally { // Load the test results diff --git a/tests/unit/auth_renewal_tests.py b/tests/unit/auth_renewal_tests.py index b5fb48b6..e32f7459 100644 --- a/tests/unit/auth_renewal_tests.py +++ b/tests/unit/auth_renewal_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016, 2017 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,15 +18,18 @@ See configuration options for environment variables in unit_t_db_base module docstring. """ -import unittest import os -import requests import time +import unittest +import requests from cloudant._client_session import CookieSession +from nose.plugins.attrib import attr from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase + +@attr(db=['cloudant','couch']) @unittest.skipIf(os.environ.get('ADMIN_PARTY') == 'true', 'Skipping - Admin Party mode') class AuthRenewalTests(UnitTestDbBase): """ diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index 007e07a6..d39ab159 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -16,20 +16,23 @@ Unit tests for _changes feed """ -import unittest -from requests import Session import json import os +import unittest -from cloudant.feed import Feed -from cloudant.document import Document +from cloudant._2to3 import unicode_ from cloudant.design_document import DesignDocument +from cloudant.document import Document from cloudant.error import CloudantArgumentError -from cloudant._2to3 import unicode_ +from cloudant.feed import Feed +from nose.plugins.attrib import attr +from requests import Session from .unit_t_db_base import UnitTestDbBase from .. import BYTETYPE + +@attr(db=['cloudant','couch']) class ChangesTests(UnitTestDbBase): """ _changes feed unit tests @@ -41,7 +44,6 @@ def setUp(self): """ super(ChangesTests, self).setUp() self.db_set_up() - self.cloudant_test = os.environ.get('RUN_CLOUDANT_TESTS') is not None def tearDown(self): """ @@ -448,8 +450,6 @@ def test_get_feed_using_conflicts_false(self): self.assertSetEqual(set([x['id'] for x in changes]), expected) self.assertTrue(str(feed.last_seq).startswith('3')) - @unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping since _doc_ids filter is not supported on all Cloudant clusters') def test_get_feed_using_doc_ids(self): """ Test getting content back for a feed using doc_ids diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index a65e729f..e110e5fa 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2017 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,28 +20,29 @@ """ -import unittest -import requests -import json import base64 -import sys -import os import datetime -import mock - -from requests import ConnectTimeout, HTTPError +import json +import os +import sys +import unittest from time import sleep +import mock +import requests from cloudant import cloudant, cloudant_bluemix, couchdb, couchdb_admin_party -from cloudant.client import Cloudant, CouchDB from cloudant._client_session import BasicSession, CookieSession +from cloudant.client import Cloudant, CouchDB from cloudant.database import CloudantDatabase from cloudant.error import CloudantArgumentError, CloudantClientException from cloudant.feed import Feed, InfiniteFeed +from nose.plugins.attrib import attr +from requests import ConnectTimeout, HTTPError from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase from .. import bytes_, str_ + class CloudantClientExceptionTests(unittest.TestCase): """ Ensure CloudantClientException functions as expected. @@ -86,10 +87,10 @@ class ClientTests(UnitTestDbBase): """ @unittest.skipIf( - (os.environ.get('RUN_CLOUDANT_TESTS') is not None or - (os.environ.get('ADMIN_PARTY') and os.environ.get('ADMIN_PARTY') == 'true')), + ((os.environ.get('ADMIN_PARTY') and os.environ.get('ADMIN_PARTY') == 'true')), 'Skipping couchdb context manager test' ) + @attr(db='couch') def test_couchdb_context_helper(self): """ Test that the couchdb context helper works as expected. @@ -102,10 +103,10 @@ def test_couchdb_context_helper(self): self.fail('Exception {0} was raised.'.format(str(err))) @unittest.skipUnless( - (os.environ.get('RUN_CLOUDANT_TESTS') is None and - (os.environ.get('ADMIN_PARTY') and os.environ.get('ADMIN_PARTY') == 'true')), + ((os.environ.get('ADMIN_PARTY') and os.environ.get('ADMIN_PARTY') == 'true')), 'Skipping couchdb_admin_party context manager test' ) + @attr(db='couch') def test_couchdb_admin_party_context_helper(self): """ Test that the couchdb_admin_party context helper works as expected. @@ -599,10 +600,7 @@ def test_db_updates_feed_call(self): finally: self.client.disconnect() -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant client specific tests' -) +@attr(db='cloudant') class CloudantClientTests(UnitTestDbBase): """ Cloudant specific client unit tests diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index 517736bc..5881a92d 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -22,25 +22,27 @@ """ -import unittest -import mock -import requests import os +import unittest import uuid +import mock +import requests from cloudant._2to3 import UNICHR -from cloudant.result import Result, QueryResult -from cloudant.error import CloudantArgumentError, CloudantDatabaseException -from cloudant.document import Document from cloudant.design_document import DesignDocument -from cloudant.security_document import SecurityDocument -from cloudant.index import Index, TextIndex, SpecialIndex +from cloudant.document import Document +from cloudant.error import CloudantArgumentError, CloudantDatabaseException from cloudant.feed import Feed, InfiniteFeed -from tests.unit._test_util import LONG_NUMBER +from cloudant.index import Index, TextIndex, SpecialIndex +from cloudant.result import Result, QueryResult +from cloudant.security_document import SecurityDocument +from nose.plugins.attrib import attr +from tests.unit._test_util import LONG_NUMBER from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase, skip_if_iam from .. import unicode_ + class CloudantDatabaseExceptionTests(unittest.TestCase): """ Ensure CloudantDatabaseException functions as expected. @@ -79,6 +81,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantDatabaseException(400, 'foo') self.assertEqual(cm.exception.status_code, 400) +@attr(db=['cloudant','couch']) class DatabaseTests(UnitTestDbBase): """ CouchDatabase/CloudantDatabase unit tests @@ -780,8 +783,7 @@ def test_get_set_revision_limit(self): self.assertNotEqual(new_limit, limit) self.assertEqual(new_limit, 1234) - @unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS'), - 'Skipping since view cleanup is automatic in Cloudant.') + @attr(db='couch') def test_view_clean_up(self): """ Test cleaning up old view files @@ -968,10 +970,7 @@ def test_database_request_fails_after_client_disconnects(self): finally: self.client.connect() - @unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - (os.environ.get('COUCHDB_VERSION') and - os.environ.get('COUCHDB_VERSION').startswith('1')), - 'Skipping test_create_json_index test') + @attr(couchapi=2) def test_create_json_index(self): """ Ensure that a JSON index is created as expected. @@ -995,10 +994,7 @@ def test_create_json_index(self): self.assertEquals(index['options']['def']['fields'], ['name', 'age']) self.assertEquals(index['reduce'], '_count') - @unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - (os.environ.get('COUCHDB_VERSION') and - os.environ.get('COUCHDB_VERSION').startswith('1')), - 'Skipping test_create_json_index test') + @attr(couchapi=2) def test_delete_json_index(self): """ Ensure that a JSON index is deleted as expected. @@ -1013,10 +1009,7 @@ def test_delete_json_index(self): self.db.delete_query_index('ddoc001', 'json', 'index001') self.assertFalse(ddoc.exists()) -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant specific Database tests' -) +@attr(db='cloudant') class CloudantDatabaseTests(UnitTestDbBase): """ Cloudant specific Database unit tests diff --git a/tests/unit/db_updates_tests.py b/tests/unit/db_updates_tests.py index 7cb8c956..0c83e75f 100644 --- a/tests/unit/db_updates_tests.py +++ b/tests/unit/db_updates_tests.py @@ -16,14 +16,15 @@ Unit tests for _db_updates feed """ -import unittest -from requests import Session import json import os +import unittest -from cloudant.feed import Feed -from cloudant.error import CloudantArgumentError from cloudant._2to3 import unicode_ +from cloudant.error import CloudantArgumentError +from cloudant.feed import Feed +from nose.plugins.attrib import attr +from requests import Session from .unit_t_db_base import UnitTestDbBase from .. import BYTETYPE @@ -104,8 +105,7 @@ def assert_changes_in_db_updates_feed(self, changes): self.assertDictEqual( changes[2], {'db_name': self.new_dbs[2].database_name, 'type': 'created'}) -@unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS'), - 'Skipping CouchDB _db_updates feed tests') +@attr(db='couch') class CouchDbUpdatesTests(DbUpdatesTestsBase): """ CouchDB _db_updates feed unit tests @@ -276,8 +276,8 @@ def test_invalid_feed_value(self): self.assertTrue(str(cm.exception).startswith( 'Invalid value (normal) for feed option.')) -@unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') +@attr(db='cloudant') +@unittest.skipIf(os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') class CloudantDbUpdatesTests(DbUpdatesTestsBase): """ Cloudant _db_updates feed unit tests diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 6cb14281..43714727 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,16 +24,18 @@ import json import os import unittest + import mock import requests - -from cloudant.document import Document from cloudant.design_document import DesignDocument -from cloudant.view import View, QueryIndexView +from cloudant.document import Document from cloudant.error import CloudantArgumentError, CloudantDesignDocumentException +from cloudant.view import View, QueryIndexView +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase, skip_if_iam + class CloudantDesignDocumentExceptionTests(unittest.TestCase): """ Ensure CloudantDesignDocumentException functions as expected. @@ -72,6 +74,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantDesignDocumentException(104, 'foo') self.assertEqual(cm.exception.status_code, 104) +@attr(db=['cloudant','couch']) class DesignDocumentTests(UnitTestDbBase): """ DesignDocument unit tests @@ -360,10 +363,7 @@ def test_fetch_map_reduce(self): self.assertIsInstance(ddoc_remote['views']['view001'], View) self.assertIsInstance(ddoc_remote['views']['view003'], View) - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant fetch dbcopy test' - ) + @attr(db='cloudant') def test_fetch_dbcopy(self): """ Ensure that the document fetch from the database returns the @@ -810,10 +810,7 @@ def test_get_info_raises_httperror(self): self.client.r_session.get.assert_called_with( '/'.join([ddoc.document_url, '_info'])) - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant _search_info endpoint test' - ) + @attr(db='cloudant') def test_get_search_info(self): """ Test retrieval of search_info endpoint from the DesignDocument. @@ -841,10 +838,7 @@ def test_get_search_info(self): self.assertTrue(search_index_metadata['pending_seq'] <= 101, 'The pending_seq should be 101 or fewer.') self.assertTrue(search_index_metadata['disk_size'] >0, 'The disk_size should be greater than 0.') - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant _search_disk_size endpoint test' - ) + @attr(db='cloudant') def test_get_search_disk_size(self): """ Test retrieval of search_disk_size endpoint from the DesignDocument. @@ -881,10 +875,7 @@ def test_get_search_disk_size(self): search_disk_size['search_index']['disk_size'] > 0, 'The "disk_size" should be greater than 0.') - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant _search_info raises HTTPError test' - ) + @attr(db='cloudant') def test_get_search_info_raises_httperror(self): """ Test get_search_info raises an HTTPError. @@ -1525,10 +1516,7 @@ def test_get_list_function(self): 'html += \'\'; return html; }); }' ) - @unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant specific Cloudant Geo tests' - ) + @attr(db='cloudant') def test_geospatial_index(self): """ Test retrieval and query of Cloudant Geo indexes from the DesignDocument. diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 3c7b5afc..3676c2b1 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -22,21 +22,22 @@ """ -import unittest -import mock +import inspect import json -import requests import os +import unittest import uuid -import inspect - from datetime import datetime +import mock +import requests from cloudant.document import Document from cloudant.error import CloudantDocumentException +from nose.plugins.attrib import attr -from .. import StringIO, unicode_ from .unit_t_db_base import UnitTestDbBase +from .. import StringIO, unicode_ + def find_fixture(name): import tests.unit.fixtures as fixtures @@ -82,6 +83,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantDocumentException(102, 'foo') self.assertEqual(cm.exception.status_code, 102) +@attr(db=['cloudant','couch']) class DocumentTests(UnitTestDbBase): """ Document unit tests diff --git a/tests/unit/index_tests.py b/tests/unit/index_tests.py index 98dbf096..59a952d6 100644 --- a/tests/unit/index_tests.py +++ b/tests/unit/index_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,20 +22,22 @@ """ from __future__ import absolute_import +import os import unittest + import mock -import os import requests - -from cloudant.index import Index, TextIndex, SpecialIndex -from cloudant.query import Query -from cloudant.view import QueryIndexView from cloudant.design_document import DesignDocument from cloudant.document import Document from cloudant.error import CloudantArgumentError, CloudantIndexException +from cloudant.index import Index, TextIndex, SpecialIndex +from cloudant.query import Query +from cloudant.view import QueryIndexView +from nose.plugins.attrib import attr -from .. import PY2 from .unit_t_db_base import UnitTestDbBase +from .. import PY2 + class CloudantIndexExceptionTests(unittest.TestCase): """ @@ -66,10 +68,8 @@ def test_raise_with_proper_code_and_args(self): raise CloudantIndexException(101) self.assertEqual(cm.exception.status_code, 101) -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant Index tests' - ) +@attr(db=['cloudant','couch']) +@attr(couchapi=2) class IndexTests(UnitTestDbBase): """ Index unit tests @@ -392,10 +392,7 @@ def test_index_usage_via_query(self): selector={'age': {'$eq': 6}}, raw_result=True) self.assertTrue(str(result['warning']).startswith("no matching index found")) -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant Text Index tests' - ) +@attr(db='cloudant') class TextIndexTests(UnitTestDbBase): """ Search Index unit tests diff --git a/tests/unit/infinite_feed_tests.py b/tests/unit/infinite_feed_tests.py index 544aca60..7891cb91 100644 --- a/tests/unit/infinite_feed_tests.py +++ b/tests/unit/infinite_feed_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,18 +16,18 @@ feed module - Unit tests for Feed class """ -import unittest -from requests import Session -import json import os +import unittest from time import sleep -from cloudant.feed import InfiniteFeed, Feed -from cloudant.client import CouchDB from cloudant.error import CloudantArgumentError, CloudantFeedException +from cloudant.feed import InfiniteFeed, Feed +from nose.plugins.attrib import attr +from requests import Session from .unit_t_db_base import UnitTestDbBase + class MethodCallCount(object): """ This callable class is used as a proxy by the infinite feed tests to wrap @@ -71,6 +71,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantFeedException(101) self.assertEqual(cm.exception.status_code, 101) +@attr(db=['cloudant','couch']) class InfiniteFeedTests(UnitTestDbBase): """ Infinite Feed unit tests @@ -126,8 +127,7 @@ def test_constructor_with_invalid_feed_option(self): 'Invalid infinite feed option: longpoll. Must be set to continuous.' ) - @unittest.skipIf(os.environ.get('RUN_CLOUDANT_TESTS'), - 'Skipping since test is possible only when testing against CouchDB.') + @attr(db='couch') def test_invalid_source_couchdb(self): """ Ensure that a CouchDB client cannot be used with an infinite feed. @@ -137,8 +137,8 @@ def test_invalid_source_couchdb(self): self.assertEqual(str(cm.exception), 'Infinite _db_updates feed not supported for CouchDB.') - @unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') + @unittest.skipIf(os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') + @attr(db='cloudant') def test_constructor_db_updates(self): """ Test constructing an infinite _db_updates feed. @@ -185,8 +185,8 @@ def test_infinite_feed(self): # the continuous feed was started/restarted 3 separate times. self.assertEqual(feed._start.called_count, 3) - @unittest.skipIf(not os.environ.get('RUN_CLOUDANT_TESTS') or - os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') + @unittest.skipIf(os.environ.get('SKIP_DB_UPDATES'), 'Skipping Cloudant _db_updates feed tests') + @attr(db='cloudant') def test_infinite_db_updates_feed(self): """ Test that an _db_updates infinite feed will continue to issue multiple diff --git a/tests/unit/query_result_tests.py b/tests/unit/query_result_tests.py index d290f930..9f4fd170 100644 --- a/tests/unit/query_result_tests.py +++ b/tests/unit/query_result_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,11 +26,12 @@ from cloudant.query import Query from cloudant.result import QueryResult from cloudant.error import ResultException +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase -@unittest.skipUnless(os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant QueryResult tests') +@attr(db=['cloudant','couch']) +@attr(couchapi=2) class QueryResultTests(UnitTestDbBase): """ QueryResult unit tests diff --git a/tests/unit/query_tests.py b/tests/unit/query_tests.py index 72fd35f1..30f78bcd 100644 --- a/tests/unit/query_tests.py +++ b/tests/unit/query_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,19 +20,19 @@ """ -import unittest import os +import unittest +from cloudant.error import CloudantArgumentError from cloudant.query import Query from cloudant.result import QueryResult -from cloudant.error import CloudantArgumentError +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase -@unittest.skipUnless( - os.environ.get('RUN_CLOUDANT_TESTS') is not None, - 'Skipping Cloudant Query tests' - ) + +@attr(db=['cloudant','couch']) +@attr(couchapi=2) class QueryTests(UnitTestDbBase): """ Query unit tests diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index 538adbd2..610d3588 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,21 +22,22 @@ """ +import time import unittest import uuid -import time -from flaky import flaky import requests -from requests import ConnectionError - -from cloudant.replicator import Replicator from cloudant.document import Document from cloudant.error import CloudantReplicatorException, CloudantClientException +from cloudant.replicator import Replicator +from flaky import flaky +from nose.plugins.attrib import attr +from requests import ConnectionError -from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase +from .unit_t_db_base import UnitTestDbBase from .. import unicode_ + class CloudantReplicatorExceptionTests(unittest.TestCase): """ Ensure CloudantReplicatorException functions as expected. @@ -75,6 +76,7 @@ def test_raise_with_proper_code_and_args(self): raise CloudantReplicatorException(404, 'foo') self.assertEqual(cm.exception.status_code, 404) +@attr(db=['cloudant','couch']) class ReplicatorTests(UnitTestDbBase): """ Replicator unit tests diff --git a/tests/unit/result_tests.py b/tests/unit/result_tests.py index a7ede996..f56387ad 100644 --- a/tests/unit/result_tests.py +++ b/tests/unit/result_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,14 +16,15 @@ result module - Unit tests for Result class """ import unittest -import os -from requests.exceptions import HTTPError from cloudant.error import ResultException from cloudant.result import Result, ResultByKey +from nose.plugins.attrib import attr +from requests.exceptions import HTTPError from .unit_t_db_base import UnitTestDbBase + class ResultExceptionTests(unittest.TestCase): """ Ensure ResultException functions as expected. @@ -72,6 +73,7 @@ def test_raise_with_proper_code_and_args(self): raise ResultException(102, 'foo', 'bar') self.assertEqual(cm.exception.status_code, 102) +@attr(db=['cloudant','couch']) class ResultTests(UnitTestDbBase): """ Result unit tests diff --git a/tests/unit/security_document_tests.py b/tests/unit/security_document_tests.py index 4e8d1ed6..0e2ee024 100644 --- a/tests/unit/security_document_tests.py +++ b/tests/unit/security_document_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,16 +19,16 @@ module docstring. """ -import unittest -import requests import json -import os +import unittest from cloudant.security_document import SecurityDocument +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase +@attr(db=['cloudant','couch']) class SecurityDocumentTests(UnitTestDbBase): """ SecurityDocument unit tests diff --git a/tests/unit/view_execution_tests.py b/tests/unit/view_execution_tests.py index 4ac97dc5..45a082c6 100644 --- a/tests/unit/view_execution_tests.py +++ b/tests/unit/view_execution_tests.py @@ -17,8 +17,12 @@ """ import unittest +from nose.plugins.attrib import attr + from .unit_t_db_base import UnitTestDbBase + +@attr(db=['cloudant','couch']) class QueryParmExecutionTests(UnitTestDbBase): """ Test cases for the execution of views queries using translated parameters. diff --git a/tests/unit/view_tests.py b/tests/unit/view_tests.py index 28be9cd4..a362fa77 100644 --- a/tests/unit/view_tests.py +++ b/tests/unit/view_tests.py @@ -23,18 +23,19 @@ """ import unittest + import mock import requests -import os - -from cloudant.design_document import DesignDocument -from cloudant.view import View, QueryIndexView from cloudant._common_util import _Code -from cloudant.result import Result +from cloudant.design_document import DesignDocument from cloudant.error import CloudantArgumentError, CloudantViewException +from cloudant.result import Result +from cloudant.view import View, QueryIndexView +from nose.plugins.attrib import attr from .unit_t_db_base import UnitTestDbBase + class CodeTests(unittest.TestCase): """ _Code class unit test @@ -78,6 +79,7 @@ def test_raise_with_proper_code(self): raise CloudantViewException(101) self.assertEqual(cm.exception.status_code, 101) +@attr(db=['cloudant','couch']) class ViewTests(UnitTestDbBase): """ View class unit tests From cfe4ae174b3913ecb3391a8151b8b01b05118bc6 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Mon, 12 Nov 2018 14:12:14 +0000 Subject: [PATCH 101/185] Replaced usage of requests `response.json()` method Replaced `response.json()` calls with `_common_util.response_to_json_dict()`. Added CHANGES entry. Clarified types of JSON encoder and decoder in doc strings. Updated tests mocking `response.json()` with `response.text`. Added simplejson test axis. --- CHANGES.md | 6 + Jenkinsfile | 13 +- src/cloudant/_client_session.py | 11 +- src/cloudant/_common_util.py | 13 +- src/cloudant/client.py | 15 +- src/cloudant/database.py | 36 ++-- src/cloudant/design_document.py | 10 +- src/cloudant/document.py | 19 +- src/cloudant/index.py | 7 +- src/cloudant/query.py | 5 +- src/cloudant/scheduler.py | 8 +- src/cloudant/security_document.py | 5 +- src/cloudant/view.py | 6 +- tests/unit/client_tests.py | 5 +- tests/unit/database_tests.py | 3 +- tests/unit/design_document_tests.py | 13 +- tests/unit/iam_auth_tests.py | 53 +++--- tests/unit/scheduler_tests.py | 270 ++++++++++++++-------------- 18 files changed, 263 insertions(+), 235 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e1ddefd2..47e621f0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +# Unreleased + +- [FIXED] Unexpected keyword argument errors when using the library with the + `simplejson` module present in the environment caused by `requests` preferentially + loading it over the system `json` module. + # 2.10.0 (2018-09-19) - [NEW] Add custom JSON encoder/decoder option to `Document` constructor. diff --git a/Jenkinsfile b/Jenkinsfile index a806771f..1ab03a60 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,12 +10,13 @@ def getEnvForSuite(suiteName) { case 'basic': envVars.add("RUN_BASIC_AUTH_TESTS=1") break - case 'cookie': - break case 'iam': // Setting IAM_API_KEY forces tests to run using an IAM enabled client. envVars.add("IAM_API_KEY=$DB_IAM_API_KEY") break + case 'cookie': + case 'simplejson': + break default: error("Unknown test suite environment ${suiteName}") } @@ -36,6 +37,7 @@ def setupPythonAndTest(pythonVersion, testSuite) { . ./tmp/bin/activate pip install -r requirements.txt pip install -r test-requirements.txt + ${'simplejson'.equals(testSuite) ? 'pip install simplejson' : ''} pylint ./src/cloudant nosetests -A 'not db or (db is "cloudant" or "cloudant" in db)' -w ./tests/unit --with-xunit """ @@ -58,12 +60,15 @@ stage('Checkout'){ } stage('Test'){ - axes = [:] - ['2.7.12','3.5.2'].each { version -> + def py2 = '2.7.12' + def py3 = '3.5.2' + def axes = [:] + [py2, py3].each { version -> ['basic','cookie','iam'].each { auth -> axes.put("Python${version}-${auth}", {setupPythonAndTest(version, auth)}) } } + axes.put("Python${py3}-simplejson", {setupPythonAndTest(py3, 'simplejson')}) parallel(axes) } diff --git a/src/cloudant/_client_session.py b/src/cloudant/_client_session.py index 0e8fd386..820bc532 100644 --- a/src/cloudant/_client_session.py +++ b/src/cloudant/_client_session.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ from requests import RequestException, Session from ._2to3 import bytes_, unicode_, url_join +from ._common_util import response_to_json_dict from .error import CloudantException @@ -75,7 +76,7 @@ def info(self): resp = self.get(self._session_url) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def set_credentials(self, username, password): """ @@ -170,7 +171,7 @@ def request(self, method, url, **kwargs): is_expired = any(( resp.status_code == 403 and - resp.json().get('error') == 'credentials_expired', + response_to_json_dict(resp).get('error') == 'credentials_expired', resp.status_code == 401 )) @@ -282,10 +283,10 @@ def _get_access_token(self): 'apikey': self._api_key } ) - err = resp.json().get('errorMessage', err) + err = response_to_json_dict(resp).get('errorMessage', err) resp.raise_for_status() - return resp.json()['access_token'] + return response_to_json_dict(resp)['access_token'] except KeyError: raise CloudantException('Invalid response from IAM token service') diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 1904a0a4..7a373815 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -270,7 +270,7 @@ def append_response_error_content(response, **kwargs): """ if response.status_code >= 400: try: - resp_dict = response.json() + resp_dict = response_to_json_dict(response) error = resp_dict.get('error', '') reason = resp_dict.get('reason', '') # Append to the existing response's reason @@ -279,6 +279,17 @@ def append_response_error_content(response, **kwargs): pass return response +def response_to_json_dict(response, **kwargs): + """ + Standard place to convert responses to JSON. + + :param response: requests response object + :param **kwargs: arguments accepted by json.loads + + :returns: dict of JSON response + """ + return json.loads(response.text, **kwargs) + # Classes diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 06a6092c..9b22d0d7 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -35,6 +35,7 @@ USER_AGENT, append_response_error_content, CloudFoundryService, + response_to_json_dict, ) @@ -256,7 +257,7 @@ def all_dbs(self): url = '/'.join((self.server_url, '_all_dbs')) resp = self.r_session.get(url) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def create_database(self, dbname, **kwargs): """ @@ -345,7 +346,7 @@ def metadata(self): """ resp = self.r_session.get(self.server_url) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def keys(self, remote=False): """ @@ -622,7 +623,7 @@ def _usage_endpoint(self, endpoint, year=None, month=None): raise CloudantArgumentError(101, year, month) else: resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def bill(self, year=None, month=None): """ @@ -687,7 +688,7 @@ def shared_databases(self): self.server_url, '_api', 'v2', 'user', 'shared_databases')) resp = self.r_session.get(endpoint) resp.raise_for_status() - data = resp.json() + data = response_to_json_dict(resp) return data.get('shared_databases', []) def generate_api_key(self): @@ -699,7 +700,7 @@ def generate_api_key(self): endpoint = '/'.join((self.server_url, '_api', 'v2', 'api_keys')) resp = self.r_session.post(endpoint) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def cors_configuration(self): """ @@ -712,7 +713,7 @@ def cors_configuration(self): resp = self.r_session.get(endpoint) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def disable_cors(self): """ @@ -807,7 +808,7 @@ def _write_cors_configuration(self, config): ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) @classmethod def bluemix(cls, vcap_services, instance_name=None, service_name=None, **kwargs): diff --git a/src/cloudant/database.py b/src/cloudant/database.py index ec6c8b7a..51ff2430 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -26,7 +26,9 @@ SEARCH_INDEX_ARGS, SPECIAL_INDEX_TYPE, TEXT_INDEX_TYPE, - get_docs) + get_docs, + response_to_json_dict, + ) from .document import Document from .design_document import DesignDocument from .security_document import SecurityDocument @@ -123,7 +125,7 @@ def metadata(self): """ resp = self.r_session.get(self.database_url) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def doc_count(self): """ @@ -192,7 +194,7 @@ def design_documents(self): query = "startkey=\"_design\"&endkey=\"_design0\"&include_docs=true" resp = self.r_session.get(url, params=query) resp.raise_for_status() - data = resp.json() + data = response_to_json_dict(resp) return data['rows'] def list_design_documents(self): @@ -206,7 +208,7 @@ def list_design_documents(self): query = "startkey=\"_design\"&endkey=\"_design0\"" resp = self.r_session.get(url, params=query) resp.raise_for_status() - data = resp.json() + data = response_to_json_dict(resp) return [x.get('key') for x in data.get('rows', [])] def get_design_document(self, ddoc_id): @@ -403,7 +405,7 @@ def all_docs(self, **kwargs): '/'.join([self.database_url, '_all_docs']), self.client.encoder, **kwargs) - return resp.json() + return response_to_json_dict(resp) @contextlib.contextmanager def custom_result(self, **options): @@ -718,7 +720,7 @@ def bulk_docs(self, docs): headers=headers ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def missing_revisions(self, doc_id, *revisions): """ @@ -742,7 +744,7 @@ def missing_revisions(self, doc_id, *revisions): ) resp.raise_for_status() - resp_json = resp.json() + resp_json = response_to_json_dict(resp) missing_revs = resp_json['missing_revs'].get(doc_id) if missing_revs is None: missing_revs = [] @@ -771,7 +773,7 @@ def revisions_diff(self, doc_id, *revisions): ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def get_revision_limit(self): """ @@ -787,7 +789,7 @@ def get_revision_limit(self): try: ret = int(resp.text) except ValueError: - raise CloudantDatabaseException(400, resp.json()) + raise CloudantDatabaseException(400, response_to_json_dict(resp)) return ret @@ -806,7 +808,7 @@ def set_revision_limit(self, limit): resp = self.r_session.put(url, data=json.dumps(limit, cls=self.client.encoder)) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def view_cleanup(self): """ @@ -822,7 +824,7 @@ def view_cleanup(self): ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def get_list_function_result(self, ddoc_id, list_name, view_name, **kwargs): """ @@ -974,10 +976,10 @@ def get_query_indexes(self, raw_result=False): resp.raise_for_status() if raw_result: - return resp.json() + return response_to_json_dict(resp) indexes = [] - for data in resp.json().get('indexes', []): + for data in response_to_json_dict(resp).get('indexes', []): if data.get('type') == JSON_INDEX_TYPE: indexes.append(Index( self, @@ -1239,7 +1241,7 @@ def share_database(self, username, roles=None): headers={'Content-Type': 'application/json'} ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def unshare_database(self, username): """ @@ -1264,7 +1266,7 @@ def unshare_database(self, username): headers={'Content-Type': 'application/json'} ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def shards(self): """ @@ -1276,7 +1278,7 @@ def shards(self): resp = self.r_session.get(url) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def get_search_result(self, ddoc_id, index_name, **query_params): """ @@ -1399,4 +1401,4 @@ def get_search_result(self, ddoc_id, index_name, **query_params): data=json.dumps(query_params, cls=self.client.encoder) ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) diff --git a/src/cloudant/design_document.py b/src/cloudant/design_document.py index b32c5e33..66a9789d 100644 --- a/src/cloudant/design_document.py +++ b/src/cloudant/design_document.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ API module/class for interacting with a design document in a database. """ from ._2to3 import iteritems_, STRTYPE -from ._common_util import QUERY_LANGUAGE, codify +from ._common_util import QUERY_LANGUAGE, codify, response_to_json_dict from .document import Document from .view import View, QueryIndexView from .error import CloudantArgumentError, CloudantDesignDocumentException @@ -686,7 +686,7 @@ def info(self): ddoc_info = self.r_session.get( '/'.join([self.document_url, '_info'])) ddoc_info.raise_for_status() - return ddoc_info.json() + return response_to_json_dict(ddoc_info) def search_info(self, search_index): """ @@ -698,7 +698,7 @@ def search_info(self, search_index): ddoc_search_info = self.r_session.get( '/'.join([self.document_url, '_search_info', search_index])) ddoc_search_info.raise_for_status() - return ddoc_search_info.json() + return response_to_json_dict(ddoc_search_info) def search_disk_size(self, search_index): """ @@ -710,4 +710,4 @@ def search_disk_size(self, search_index): ddoc_search_disk_size = self.r_session.get( '/'.join([self.document_url, '_search_disk_size', search_index])) ddoc_search_disk_size.raise_for_status() - return ddoc_search_disk_size.json() + return response_to_json_dict(ddoc_search_disk_size) diff --git a/src/cloudant/document.py b/src/cloudant/document.py index ac08ee65..426b0656 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -20,6 +20,7 @@ from requests.exceptions import HTTPError from ._2to3 import url_quote, url_quote_plus +from ._common_util import response_to_json_dict from .error import CloudantDocumentException @@ -53,8 +54,8 @@ class Document(dict): :param database: A database instance used by the Document. Can be either a ``CouchDatabase`` or ``CloudantDatabase`` instance. :param str document_id: Optional document id used to identify the document. - :param str encoder: Optional JSON encoder object. - :param str decoder: Optional JSON decoder object. + :param str encoder: Optional JSON encoder object (extending json.JSONEncoder). + :param str decoder: Optional JSON decoder object (extending json.JSONDecoder). """ def __init__(self, database, document_id=None, **kwargs): super(Document, self).__init__() @@ -150,7 +151,7 @@ def create(self): data=json.dumps(doc, cls=self.encoder) ) resp.raise_for_status() - data = resp.json() + data = response_to_json_dict(resp) self._document_id = data['id'] super(Document, self).__setitem__('_id', data['id']) super(Document, self).__setitem__('_rev', data['rev']) @@ -167,7 +168,7 @@ def fetch(self): resp = self.r_session.get(self.document_url) resp.raise_for_status() self.clear() - self.update(resp.json(cls=self.decoder)) + self.update(response_to_json_dict(resp, cls=self.decoder)) def save(self): """ @@ -189,7 +190,7 @@ def save(self): headers=headers ) put_resp.raise_for_status() - data = put_resp.json() + data = response_to_json_dict(put_resp) super(Document, self).__setitem__('_rev', data['rev']) return @@ -418,7 +419,7 @@ def get_attachment( if attachment_type == 'text': return resp.text if attachment_type == 'json': - return resp.json() + return response_to_json_dict(resp) return resp.content @@ -447,7 +448,7 @@ def delete_attachment(self, attachment, headers=None): headers=headers ) resp.raise_for_status() - super(Document, self).__setitem__('_rev', resp.json()['rev']) + super(Document, self).__setitem__('_rev', response_to_json_dict(resp)['rev']) # Execute logic only if attachment metadata exists locally if self.get('_attachments'): # Remove the attachment metadata for the specified attachment @@ -457,7 +458,7 @@ def delete_attachment(self, attachment, headers=None): if not self['_attachments']: super(Document, self).__delitem__('_attachments') - return resp.json() + return response_to_json_dict(resp) def put_attachment(self, attachment, content_type, data, headers=None): """ @@ -494,4 +495,4 @@ def put_attachment(self, attachment, content_type, data, headers=None): ) resp.raise_for_status() self.fetch() - return resp.json() + return response_to_json_dict(resp) diff --git a/src/cloudant/index.py b/src/cloudant/index.py index 3d0d49c3..40274ce1 100644 --- a/src/cloudant/index.py +++ b/src/cloudant/index.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ from ._common_util import TEXT_INDEX_TYPE from ._common_util import SPECIAL_INDEX_TYPE from ._common_util import TEXT_INDEX_ARGS +from ._common_util import response_to_json_dict from .error import CloudantArgumentError, CloudantIndexException class Index(object): @@ -143,8 +144,8 @@ def create(self): headers=headers ) resp.raise_for_status() - self._ddoc_id = resp.json()['id'] - self._name = resp.json()['name'] + self._ddoc_id = response_to_json_dict(resp)['id'] + self._name = response_to_json_dict(resp)['name'] def _def_check(self): """ diff --git a/src/cloudant/query.py b/src/cloudant/query.py index b5bd2b58..ed9ec35e 100644 --- a/src/cloudant/query.py +++ b/src/cloudant/query.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ from .result import QueryResult from .error import CloudantArgumentError from ._common_util import QUERY_ARG_TYPES +from ._common_util import response_to_json_dict class Query(dict): """ @@ -172,7 +173,7 @@ def __call__(self, **kwargs): data=json.dumps(data, cls=self._encoder) ) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) @contextlib.contextmanager def custom_result(self, **options): diff --git a/src/cloudant/scheduler.py b/src/cloudant/scheduler.py index 723635b2..012272fe 100644 --- a/src/cloudant/scheduler.py +++ b/src/cloudant/scheduler.py @@ -16,6 +16,8 @@ API module for interacting with scheduler endpoints """ +from ._common_util import response_to_json_dict + class Scheduler(object): """ API for retrieving scheduler jobs and documents. @@ -47,7 +49,7 @@ def list_docs(self, limit=None, skip=None): params["skip"] = skip resp = self._r_session.get('/'.join([self._scheduler, 'docs']), params=params) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def get_doc(self, doc_id): """ @@ -55,7 +57,7 @@ def get_doc(self, doc_id): """ resp = self._r_session.get('/'.join([self._scheduler, 'docs', '_replicator', doc_id])) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) def list_jobs(self, limit=None, skip=None): @@ -78,4 +80,4 @@ def list_jobs(self, limit=None, skip=None): params["skip"] = skip resp = self._r_session.get('/'.join([self._scheduler, 'jobs']), params=params) resp.raise_for_status() - return resp.json() + return response_to_json_dict(resp) diff --git a/src/cloudant/security_document.py b/src/cloudant/security_document.py index 22b0f79c..87b77ca3 100644 --- a/src/cloudant/security_document.py +++ b/src/cloudant/security_document.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright (C) 2016, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import json from ._2to3 import url_quote_plus +from ._common_util import response_to_json_dict class SecurityDocument(dict): """ @@ -101,7 +102,7 @@ def fetch(self): resp = self.r_session.get(self.document_url) resp.raise_for_status() self.clear() - self.update(resp.json()) + self.update(response_to_json_dict(resp)) def save(self): """ diff --git a/src/cloudant/view.py b/src/cloudant/view.py index 685600e5..3a0e63fc 100644 --- a/src/cloudant/view.py +++ b/src/cloudant/view.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015 IBM. All rights reserved. +# Copyright (C) 2015, 2018 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ import contextlib from ._2to3 import STRTYPE -from ._common_util import codify, get_docs +from ._common_util import codify, get_docs, response_to_json_dict from .result import Result from .error import CloudantArgumentError, CloudantViewException @@ -227,7 +227,7 @@ def __call__(self, **kwargs): self.url, self.design_doc.encoder, **kwargs) - return resp.json() + return response_to_json_dict(resp) @contextlib.contextmanager def custom_result(self, **options): diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index e110e5fa..0a544a15 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -247,7 +247,7 @@ def test_session_basic(self, m_req): """ m_response_ok = mock.MagicMock() type(m_response_ok).status_code = mock.PropertyMock(return_value=200) - m_response_ok.json.return_value = ['animaldb'] + type(m_response_ok).text = mock.PropertyMock(return_value='["animaldb"]') m_req.return_value = m_response_ok client = Cloudant('foo', 'bar', url=self.url, use_basic_auth=True) @@ -298,8 +298,7 @@ def test_change_credentials_basic(self, m_req): """ # mock 200 m_response_ok = mock.MagicMock() - m_response_ok.json.return_value = ['animaldb'] - + type(m_response_ok).text = mock.PropertyMock(return_value='["animaldb"]') # mock 401 m_response_bad = mock.MagicMock() m_response_bad.raise_for_status.side_effect = HTTPError('401 Unauthorized') diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index 5881a92d..f4c0e9aa 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -29,6 +29,7 @@ import mock import requests from cloudant._2to3 import UNICHR +from cloudant._common_util import response_to_json_dict from cloudant.design_document import DesignDocument from cloudant.document import Document from cloudant.error import CloudantArgumentError, CloudantDatabaseException @@ -237,7 +238,7 @@ def test_retrieve_db_metadata(self): """ resp = self.db.r_session.get( '/'.join((self.client.server_url, self.test_dbname))) - expected = resp.json() + expected = response_to_json_dict(resp) actual = self.db.metadata() self.assertListEqual(list(actual.keys()), list(expected.keys())) diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 43714727..c986e769 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -27,6 +27,7 @@ import mock import requests +from cloudant._common_util import response_to_json_dict from cloudant.design_document import DesignDocument from cloudant.document import Document from cloudant.error import CloudantArgumentError, CloudantDesignDocumentException @@ -693,7 +694,7 @@ def test_save_with_no_views(self): # Ensure that remotely saved design document does not # include a views sub-document. resp = self.client.r_session.get(ddoc.document_url) - raw_ddoc = resp.json() + raw_ddoc = response_to_json_dict(resp) self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1182,7 +1183,7 @@ def test_save_with_no_search_indexes(self): # Ensure that remotely saved design document does not # include a search indexes sub-document. resp = self.client.r_session.get(ddoc.document_url) - raw_ddoc = resp.json() + raw_ddoc = response_to_json_dict(resp) self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1263,7 +1264,7 @@ def test_rewrite_rule(self): doc.save() resp = self.client.r_session.get('/'.join([ddoc.document_url, '_rewrite'])) self.assertEquals( - resp.json(), + response_to_json_dict(resp), { '_id': 'rewrite_doc', '_rev': doc['_rev'] @@ -1451,7 +1452,7 @@ def test_save_with_no_list_functions(self): # Ensure that remotely saved design document does not # include a lists sub-document. resp = self.client.r_session.get(ddoc.document_url) - raw_ddoc = resp.json() + raw_ddoc = response_to_json_dict(resp) self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1752,7 +1753,7 @@ def test_save_with_no_show_functions(self): # Ensure that remotely saved design document does not # include a shows sub-document. resp = self.client.r_session.get(ddoc.document_url) - raw_ddoc = resp.json() + raw_ddoc = response_to_json_dict(resp) self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1834,7 +1835,7 @@ def test_update_validator(self): data=json.dumps({'_id': 'test001'}) ) self.assertEqual( - resp.json(), + response_to_json_dict(resp), {'reason': 'Document must have an address.', 'error': 'forbidden'} ) diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py index 6e0c7670..ec4fbd95 100644 --- a/tests/unit/iam_auth_tests.py +++ b/tests/unit/iam_auth_tests.py @@ -43,24 +43,22 @@ '2PTo4Exa17V-R_73Nq8VPCwpOvZcwKRA2sPTVgTMzU34max8b5kpTzVGJ' '6SXSItTVOUdAygZBng') -MOCK_IAM_TOKEN_RESPONSE = { - 'access_token': MOCK_ACCESS_TOKEN, - 'refresh_token': ('MO61FKNvVRWkSa4vmBZqYv_Jt1kkGMUc-XzTcNnR-GnIhVKXHUWxJVV3' - 'RddE8Kqh3X_TZRmyK8UySIWKxoJ2t6obUSUalPm90SBpTdoXtaljpNyo' - 'rmqCCYPROnk6JBym72ikSJqKHHEZVQkT0B5ggZCwPMnKagFj0ufs-VIh' - 'CF97xhDxDKcIPMWG02xxPuESaSTJJug7e_dUDoak_ZXm9xxBmOTRKwOx' - 'n5sTKthNyvVpEYPE7jIHeiRdVDOWhN5LomgCn3TqFCLpMErnqwgNYbyC' - 'Bd9rNm-alYKDb6Jle4njuIBpXxQPb4euDwLd1osApaSME3nEarFWqRBz' - 'hjoqCe1Kv564s_rY7qzD1nHGvKOdpSa0ZkMcfJ0LbXSQPs7gBTSVrBFZ' - 'qwlg-2F-U3Cto62-9qRR_cEu_K9ZyVwL4jWgOlngKmxV6Ku4L5mHp4Kg' - 'EJSnY_78_V2nm64E--i2ZA1FhiKwIVHDOivVNhggE9oabxg54vd63glp' - '4GfpNnmZsMOUYG9blJJpH4fDX4Ifjbw-iNBD7S2LRpP8b8vG9pb4WioG' - 'zN43lE5CysveKYWrQEZpThznxXlw1snDu_A48JiL3Lrvo1LobLhF3zFV' - '-kQ='), - 'token_type': 'Bearer', - 'expires_in': 3600, # 60mins - 'expiration': 1500470702 # Wed Jul 19 14:25:02 2017 -} +MOCK_IAM_TOKEN_RESPONSE = '{"access_token": "%s",\ + "refresh_token": "MO61FKNvVRWkSa4vmBZqYv_Jt1kkGMUc-XzTcNnR-GnIhVKXHUWxJVV3\ + RddE8Kqh3X_TZRmyK8UySIWKxoJ2t6obUSUalPm90SBpTdoXtaljpNyo\ + rmqCCYPROnk6JBym72ikSJqKHHEZVQkT0B5ggZCwPMnKagFj0ufs-VIh\ + CF97xhDxDKcIPMWG02xxPuESaSTJJug7e_dUDoak_ZXm9xxBmOTRKwOx\ + n5sTKthNyvVpEYPE7jIHeiRdVDOWhN5LomgCn3TqFCLpMErnqwgNYbyC\ + Bd9rNm-alYKDb6Jle4njuIBpXxQPb4euDwLd1osApaSME3nEarFWqRBz\ + hjoqCe1Kv564s_rY7qzD1nHGvKOdpSa0ZkMcfJ0LbXSQPs7gBTSVrBFZ\ + qwlg-2F-U3Cto62-9qRR_cEu_K9ZyVwL4jWgOlngKmxV6Ku4L5mHp4Kg\ + EJSnY_78_V2nm64E--i2ZA1FhiKwIVHDOivVNhggE9oabxg54vd63glp\ + 4GfpNnmZsMOUYG9blJJpH4fDX4Ifjbw-iNBD7S2LRpP8b8vG9pb4WioG\ + zN43lE5CysveKYWrQEZpThznxXlw1snDu_A48JiL3Lrvo1LobLhF3zFV\ + -kQ=",\ + "token_type": "Bearer",\ + "expires_in": 3600,\ + "expiration": 1500470702}'%(MOCK_ACCESS_TOKEN) class IAMAuthTests(unittest.TestCase): @@ -100,7 +98,8 @@ def test_iam_set_credentials(self): @mock.patch('cloudant._client_session.ClientSession.request') def test_iam_get_access_token(self, m_req): m_response = mock.MagicMock() - m_response.json.return_value = MOCK_IAM_TOKEN_RESPONSE + mock_token_response_text = mock.PropertyMock(return_value=MOCK_IAM_TOKEN_RESPONSE) + type(m_response).text = mock_token_response_text m_req.return_value = m_response iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') @@ -120,7 +119,7 @@ def test_iam_get_access_token(self, m_req): self.assertEqual(access_token, MOCK_ACCESS_TOKEN) self.assertTrue(m_response.raise_for_status.called) - self.assertTrue(m_response.json.called) + mock_token_response_text.assert_called_with() @mock.patch('cloudant._client_session.ClientSession.request') @mock.patch('cloudant._client_session.IAMSession._get_access_token') @@ -152,10 +151,10 @@ def test_iam_logout(self): @mock.patch('cloudant._client_session.ClientSession.get') def test_iam_get_session_info(self, m_get): - m_info = {'ok': True, 'info': {'authentication_db': '_users'}} + m_info = '{"ok": true, "info": {"authentication_db": "_users"}}' m_response = mock.MagicMock() - m_response.json.return_value = m_info + type(m_response).text = mock.PropertyMock(return_value=m_info) m_get.return_value = m_response iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') @@ -163,7 +162,7 @@ def test_iam_get_session_info(self, m_get): m_get.assert_called_once_with(iam._session_url) - self.assertEqual(info, m_info) + self.assertEqual(info, json.loads(m_info)) self.assertTrue(m_response.raise_for_status.called) @mock.patch('cloudant._client_session.IAMSession.login') @@ -172,7 +171,7 @@ def test_iam_first_request(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() type(m_response_ok).status_code = mock.PropertyMock(return_value=200) - m_response_ok.json.return_value = {'ok': True} + type(m_response_ok).text = mock.PropertyMock(return_value='{"ok": true}') m_req.return_value = m_response_ok @@ -197,7 +196,7 @@ def test_iam_renew_cookie_on_expiry(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() type(m_response_ok).status_code = mock.PropertyMock(return_value=200) - m_response_ok.json.return_value = {'ok': True} + type(m_response_ok).text = mock.PropertyMock(return_value='{"ok": true}') m_req.return_value = m_response_ok @@ -219,7 +218,7 @@ def test_iam_renew_cookie_on_401_success(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() type(m_response_ok).status_code = mock.PropertyMock(return_value=200) - m_response_ok.json.return_value = {'ok': True} + type(m_response_ok).text = mock.PropertyMock(return_value='{"ok": true}') # mock 401 m_response_bad = mock.MagicMock() type(m_response_bad).status_code = mock.PropertyMock(return_value=401) @@ -298,7 +297,7 @@ def test_iam_client_create(self, m_req, m_login): # mock 200 m_response_ok = mock.MagicMock() type(m_response_ok).status_code = mock.PropertyMock(return_value=200) - m_response_ok.json.return_value = ['animaldb'] + type(m_response_ok).text = mock.PropertyMock(return_value='["animaldb"]') m_req.return_value = m_response_ok diff --git a/tests/unit/scheduler_tests.py b/tests/unit/scheduler_tests.py index 97b5046f..5ef8cab9 100644 --- a/tests/unit/scheduler_tests.py +++ b/tests/unit/scheduler_tests.py @@ -46,107 +46,105 @@ def test_scheduler_docs(self): Test scheduler docs """ # set up mock response using a real captured response - m_response_ok = requests.Response() - m_response_ok.status_code = 200 - m_response_ok.json = mock.Mock() - m_response_ok.json.return_value = {"total_rows":6,"offset":0,"docs":[ - {"database":"tomblench/_replicator", - "doc_id":"296e48244e003eba8764b2156b3bf302", - "id":None, - "source":"https://tomblench.cloudant.com/animaldb/", - "target":"https://tomblench.cloudant.com/animaldb_copy/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":15, - "missing_revisions_found":2, - "docs_read":2, - "docs_written":2, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"19-g1AAAAGjeJyVz10KwjAMB_BoJ4KX8AZF2tWPJ3eVpqnO0XUg27PeTG9Wa_VhwmT6kkDIPz_iACArGcGS0DRnWxDmHE9HdJ3lxjUdad9yb1sXF6cacB9CqEqmZ3UczKUh2uGhHxeD8U9i_Z3AIla8vJVJUlBIZYTqX5A_KMM7SfFZrHCNLUK3p7RIkl5tSRD-K6kx6f6S0k8sScpYJTb5uFQ9AI9Ch9c"}, - "start_time":None, - "last_updated":"2017-04-13T14:53:50+00:00"}, - {"database":"tomblench/_replicator", - "doc_id":"3b749f320867d703550b0f758a4000ae", - "id":None, - "source":"https://examples.cloudant.com/animaldb/", - "target":"https://tomblench.cloudant.com/animaldb/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":15, - "missing_revisions_found":15, - "docs_read":15, - "docs_written":15, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"56-g1AAAAGveJzLYWBgYMlgTmFQSElKzi9KdUhJstDLTS3KLElMT9VLzskvTUnMK9HLSy3JAapkSmRIsv___39WBnMiby5QgN04JS3FLDUJWb8Jdv0gSxThigyN8diS5AAkk-qhFvFALEo2MTEwMSXGDDSbTPHYlMcCJBkagBTQsv0g28TBtpkbGCQapaF4C4cxJFt2AGIZ2GscYMuMDEzMUizMkC0zw25MFgBKoovi"}, - "start_time":None, - "last_updated":"2017-04-27T12:28:44+00:00"}, - {"database":"tomblench/_replicator", - "doc_id":"ad8f7896480b8081c8f0a2267ffd1859", - "id":None, - "source":"https://tortytherlediffecareette:*****@mikerhodestesty008.cloudant.com/moviesdb/", - "target":"https://tomblench.cloudant.com/moviesdb_rep/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":5997, - "missing_revisions_found":5997, - "docs_read":5997, - "docs_written":5997, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"5997-g1AAAANreJy10UEKwjAQAMBgBcVP2BeUpEm1PdmfaDYJSKkVtB486U_0J_oBTz5AHyAI3jxIjUml1x7ayy67LDssmyKE-nNHIleCWK5ULIF6uVrnW4xDT6TLjeRZ7mUqT_VkhyMYFkWRzB3Q1XOhez3iczKKghor6jvg6giTiroYiuNQYYqbpeIfNa2oh72KhQGosFlq9qN2FfUyFPgUCKONoneXR7TXSWuHkvsYjjEWjQVvgTta7lRyV_szKgmRbVx3ttzNcs7AcEoKCHAb3N1y_9-9DYeBYzEiNTYlX3EcE0s"}, - "start_time":None, - "last_updated":"2016-08-23T13:11:26+00:00"}, - {"database":"tomblench/_replicator", - "doc_id":"b63c053ecd95a4047b55ed8847b046f1", - "id":None, - "source":"https://tomblench.cloudant.com/atestdb2/", - "target":"https://tomblench.cloudant.com/atestdb1/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":1, - "missing_revisions_found":1, - "docs_read":1, - "docs_written":1, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"2-g1AAAAFHeJyNjkEOgjAQRSdAYjyFN2jSFCtdyVU6nSKQWhJC13ozvVktsoEF0c2fTPL_-98BQNHmBCdCM4y2JuQMuxu6YJlxQyDtJ-bt5JIx04DXGGOvYRsR-xGsk-JjTrW5hnv6Dg0XplRngmPwZJvOW9ry5D7PF0nhmU5CvmZm9mVKVVacLr8pfy9fmt5L02q9qEhJbtbr-w-AQmfD"}, - "start_time":None, - "last_updated":"2017-05-16T16:25:22+00:00"}, - {"database":"tomblench/_replicator", - "doc_id":"c71c9e69e30a182dc91d8938277bc85e", - "id":None, - "source":"https://tomblench.cloudant.com/animaldb/", - "target":"https://tomblench.cloudant.com/animaldb_copy/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":15, - "missing_revisions_found":15, - "docs_read":15, - "docs_written":15, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"14-g1AAAAEueJzLYWBgYMlgTmGQSUlKzi9KdUhJMtTLTU1M0UvOyS9NScwr0ctLLckBqmJKZEiy____f1YGUyJrLlCAPdHEPCktJZk43UkOQDKpHmoAI9gAw2STxCTzJOIMyGMBkgwNQApoxv6sDGaoK0yN04wsk80IGEGKHQcgdoAdygxxaIplklFaWhYAu2FdOA"}, - "start_time":None, - "last_updated":"2015-05-12T11:47:33+00:00"}, - {"database":"tomblench/_replicator", - "doc_id":"e6242d1e9ce059b0388fc75af3116a39", - "id":None, - "source":"https://tomblench.cloudant.com/atestdb1/", - "target":"https://tomblench.cloudant.com/atestdb2/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":1, - "missing_revisions_found":1, - "docs_read":1, - "docs_written":1, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"1-g1AAAAFheJyFzkEOgjAQBdBRSIyn8AZNgEJgJVeZ6bQCqSUhdK0305th1Q1dEDYzyWTy_rcAkHYJw4VJjZNumQpB_Y2s10LZ0TO6WTg92_B4RKDrsixDlyDcw-FUVUiFahjO3rE2vdMcY9k2Rm2Y9Ig8bWqspdz25Lbn0jDhGVYgX1_z8DMblnlp8n0lTir3kt7_pFV7NE2WYbluP3wATr5vQA"}, - "start_time":None, - "last_updated":"2017-05-16T16:24:02+00:00"} - ]} + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='{"total_rows":6,"offset":0,"docs":[\ + {"database":"tomblench/_replicator",\ + "doc_id":"296e48244e003eba8764b2156b3bf302",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/animaldb/",\ + "target":"https://tomblench.cloudant.com/animaldb_copy/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":15,\ + "missing_revisions_found":2,\ + "docs_read":2,\ + "docs_written":2,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"19-g1AAAAGjeJyVz10KwjAMB_BoJ4KX8AZF2tWPJ3eVpqnO0XUg27PeTG9Wa_VhwmT6kkDIPz_iACArGcGS0DRnWxDmHE9HdJ3lxjUdad9yb1sXF6cacB9CqEqmZ3UczKUh2uGhHxeD8U9i_Z3AIla8vJVJUlBIZYTqX5A_KMM7SfFZrHCNLUK3p7RIkl5tSRD-K6kx6f6S0k8sScpYJTb5uFQ9AI9Ch9c"},\ + "start_time":null,\ + "last_updated":"2017-04-13T14:53:50+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"3b749f320867d703550b0f758a4000ae",\ + "id":null,\ + "source":"https://examples.cloudant.com/animaldb/",\ + "target":"https://tomblench.cloudant.com/animaldb/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":15,\ + "missing_revisions_found":15,\ + "docs_read":15,\ + "docs_written":15,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"56-g1AAAAGveJzLYWBgYMlgTmFQSElKzi9KdUhJstDLTS3KLElMT9VLzskvTUnMK9HLSy3JAapkSmRIsv___39WBnMiby5QgN04JS3FLDUJWb8Jdv0gSxThigyN8diS5AAkk-qhFvFALEo2MTEwMSXGDDSbTPHYlMcCJBkagBTQsv0g28TBtpkbGCQapaF4C4cxJFt2AGIZ2GscYMuMDEzMUizMkC0zw25MFgBKoovi"},\ + "start_time":null,\ + "last_updated":"2017-04-27T12:28:44+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"ad8f7896480b8081c8f0a2267ffd1859",\ + "id":null,\ + "source":"https://tortytherlediffecareette:*****@mikerhodestesty008.cloudant.com/moviesdb/",\ + "target":"https://tomblench.cloudant.com/moviesdb_rep/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":5997,\ + "missing_revisions_found":5997,\ + "docs_read":5997,\ + "docs_written":5997,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"5997-g1AAAANreJy10UEKwjAQAMBgBcVP2BeUpEm1PdmfaDYJSKkVtB486U_0J_oBTz5AHyAI3jxIjUml1x7ayy67LDssmyKE-nNHIleCWK5ULIF6uVrnW4xDT6TLjeRZ7mUqT_VkhyMYFkWRzB3Q1XOhez3iczKKghor6jvg6giTiroYiuNQYYqbpeIfNa2oh72KhQGosFlq9qN2FfUyFPgUCKOnullXR7TXSWuHkvsYjjEWjQVvgTta7lRyV_szKgmRbVx3ttzNcs7AcEoKCHAb3N1y_9-9DYeBYzEiNTYlX3EcE0s"},\ + "start_time":null,\ + "last_updated":"2016-08-23T13:11:26+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"b63c053ecd95a4047b55ed8847b046f1",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/atestdb2/",\ + "target":"https://tomblench.cloudant.com/atestdb1/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":1,\ + "missing_revisions_found":1,\ + "docs_read":1,\ + "docs_written":1,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"2-g1AAAAFHeJyNjkEOgjAQRSdAYjyFN2jSFCtdyVU6nSKQWhJC13ozvVktsoEF0c2fTPL_-98BQNHmBCdCM4y2JuQMuxu6YJlxQyDtJ-bt5JIx04DXGGOvYRsR-xGsk-JjTrW5hnv6Dg0XplRngmPwZJvOW9ry5D7PF0nhmU5CvmZm9mVKVVacLr8pfy9fmt5L02q9qEhJbtbr-w-AQmfD"},\ + "start_time":null,\ + "last_updated":"2017-05-16T16:25:22+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"c71c9e69e30a182dc91d8938277bc85e",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/animaldb/",\ + "target":"https://tomblench.cloudant.com/animaldb_copy/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":15,\ + "missing_revisions_found":15,\ + "docs_read":15,\ + "docs_written":15,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"14-g1AAAAEueJzLYWBgYMlgTmGQSUlKzi9KdUhJMtTLTU1M0UvOyS9NScwr0ctLLckBqmJKZEiy____f1YGUyJrLlCAPdHEPCktJZk43UkOQDKpHmoAI9gAw2STxCTzJOIMyGMBkgwNQApoxv6sDGaoK0yN04wsk80IGEGKHQcgdoAdygxxaIplklFaWhYAu2FdOA"},\ + "start_time":null,\ + "last_updated":"2015-05-12T11:47:33+00:00"},\ + {"database":"tomblench/_replicator",\ + "doc_id":"e6242d1e9ce059b0388fc75af3116a39",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/atestdb1/",\ + "target":"https://tomblench.cloudant.com/atestdb2/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":1,\ + "missing_revisions_found":1,\ + "docs_read":1,\ + "docs_written":1,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"1-g1AAAAFheJyFzkEOgjAQBdBRSIyn8AZNgEJgJVeZ6bQCqSUhdK0305th1Q1dEDYzyWTy_rcAkHYJw4VJjZNumQpB_Y2s10LZ0TO6WTg92_B4RKDrsixDlyDcw-FUVUiFahjO3rE2vdMcY9k2Rm2Y9Ig8bWqspdz25Lbn0jDhGVYgX1_z8DMblnlp8n0lTir3kt7_pFV7NE2WYbluP3wATr5vQA"},\ + "start_time":null,\ + "last_updated":"2017-05-16T16:24:02+00:00"}]}') self.client.r_session.get = mock.Mock(return_value=m_response_ok) scheduler = Scheduler(self.client) @@ -163,25 +161,24 @@ def test_scheduler_doc(self): Test scheduler doc """ # set up mock response using a real captured response - m_response_ok = requests.Response() - m_response_ok.status_code = 200 - m_response_ok.json = mock.Mock() - m_response_ok.json.return_value = {"database":"tomblench/_replicator", - "doc_id":"296e48244e003eba8764b2156b3bf302", - "id":None, - "source":"https://tomblench.cloudant.com/animaldb/", - "target":"https://tomblench.cloudant.com/animaldb_copy/", - "state":"completed", - "error_count":0, - "info":{"revisions_checked":15, - "missing_revisions_found":2, - "docs_read":2, - "docs_written":2, - "changes_pending":None, - "doc_write_failures":0, - "checkpointed_source_seq":"19-g1AAAAGjeJyVz10KwjAMB_BoJ4KX8AZF2tWPJ3eVpqnO0XUg27PeTG9Wa_VhwmT6kkDIPz_iACArGcGS0DRnWxDmHE9HdJ3lxjUdad9yb1sXF6cacB9CqEqmZ3UczKUh2uGhHxeD8U9i_Z3AIla8vJVJUlBIZYTqX5A_KMM7SfFZrHCNLUK3p7RIkl5tSRD-K6kx6f6S0k8sScpYJTb5uFQ9AI9Ch9c"}, - "start_time":None, - "last_updated":"2017-04-13T14:53:50+00:00"}; + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='{"database":"tomblench/_replicator",\ + "doc_id":"296e48244e003eba8764b2156b3bf302",\ + "id":null,\ + "source":"https://tomblench.cloudant.com/animaldb/",\ + "target":"https://tomblench.cloudant.com/animaldb_copy/",\ + "state":"completed",\ + "error_count":0,\ + "info":{"revisions_checked":15,\ + "missing_revisions_found":2,\ + "docs_read":2,\ + "docs_written":2,\ + "changes_pending":null,\ + "doc_write_failures":0,\ + "checkpointed_source_seq":"19-g1AAAAGjeJyVz10KwjAMB_BoJ4KX8AZF2tWPJ3eVpqnO0XUg27PeTG9Wa_VhwmT6kkDIPz_iACArGcGS0DRnWxDmHE9HdJ3lxjUdad9yb1sXF6cacB9CqEqmZ3UczKUh2uGhHxeD8U9i_Z3AIla8vJVJUlBIZYTqX5A_KMM7SfFZrHCNLUK3p7RIkl5tSRD-K6kx6f6S0k8sScpYJTb5uFQ9AI9Ch9c"},\ + "start_time":null,\ + "last_updated":"2017-04-13T14:53:50+00:00"}') self.client.r_session.get = mock.Mock(return_value=m_response_ok) scheduler = Scheduler(self.client) response = scheduler.get_doc("296e48244e003eba8764b2156b3bf302") @@ -197,23 +194,22 @@ def test_scheduler_jobs(self): Test scheduler jobs """ # set up mock response using a real captured response - m_response_ok = requests.Response() - m_response_ok.status_code = 200 - m_response_ok.json = mock.Mock() - m_response_ok.json.return_value = {"total_rows":1,"offset":0, - "jobs":[{"database":None, - "id":"f11105eaaded4981d21ff8ebf846f48b+create_target", - "pid":"<0.5866.6800>", - "source":"https://clientlibs-test:*****@clientlibs-test.cloudant.com/largedb1g/", - "target":"https://tomblench:*****@tomblench.cloudant.com/largedb1g/", - "user":"tomblench", - "doc_id":None, - "history":[{"timestamp":"2018-04-12T13:06:20Z", - "type":"started"}, - {"timestamp":"2018-04-12T13:06:20Z", - "type":"added"}], - "node":"dbcore@db2.bigblue.cloudant.net", - "start_time":"2018-04-12T13:06:20Z"}]} + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + type(m_response_ok).text = mock.PropertyMock(return_value='{"total_rows":1,"offset":0,\ + "jobs":[{"database":null,\ + "id":"f11105eaaded4981d21ff8ebf846f48b+create_target",\ + "pid":"<0.5866.6800>",\ + "source":"https://clientlibs-test:*****@clientlibs-test.cloudant.com/largedb1g/",\ + "target":"https://tomblench:*****@tomblench.cloudant.com/largedb1g/",\ + "user":"tomblench",\ + "doc_id":null,\ + "history":[{"timestamp":"2018-04-12T13:06:20Z",\ + "type":"started"},\ + {"timestamp":"2018-04-12T13:06:20Z",\ + "type":"added"}],\ + "node":"dbcore@db2.bigblue.cloudant.net",\ + "start_time":"2018-04-12T13:06:20Z"}]}') self.client.r_session.get = mock.Mock(return_value=m_response_ok) scheduler = Scheduler(self.client) response = scheduler.list_jobs(skip=0, limit=10) From 38c047ecf8a2cf42f851f34c98d3af551bc66cf6 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 16 Nov 2018 10:15:26 +0000 Subject: [PATCH 102/185] Updated CHANGES for 2.10.1 release --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 47e621f0..bf15910c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# Unreleased +# 2.10.1 (2018-11-16) - [FIXED] Unexpected keyword argument errors when using the library with the `simplejson` module present in the environment caused by `requests` preferentially From 30cd7857bd18c818093238641a85ca3177059b58 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 16 Nov 2018 10:17:48 +0000 Subject: [PATCH 103/185] Updated version to 2.10.1 --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index d33489b3..8bbb6e40 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.11.0-SNAPSHOT +2.10.1 diff --git a/docs/conf.py b/docs/conf.py index 72e556ae..4224bff9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.11.0-SNAPSHOT' +version = '2.10.1' # The full version, including alpha/beta/rc tags. -release = '2.11.0-SNAPSHOT' +release = '2.10.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index ffb150c2..32db1baf 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.11.0-SNAPSHOT' +__version__ = '2.10.1' # pylint: disable=wrong-import-position import contextlib From c296a5a5208d8e5850df939232c65713576e3855 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 16 Nov 2018 15:47:51 +0000 Subject: [PATCH 104/185] Updated version to 2.10.2-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 8bbb6e40..4e9cca7c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10.1 +2.10.2-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index 4224bff9..562082d6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.10.1' +version = '2.10.2-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.10.1' +release = '2.10.2-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 32db1baf..74f57225 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.10.1' +__version__ = '2.10.2-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From 9caf232a5eba0d558a2045bd7b2cdc21a1f83879 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 12 Dec 2018 10:34:28 +0000 Subject: [PATCH 105/185] Set repsonse encoding to UTF-8 to avoid chardet The performance of the default chardet used by requests can be very poor and since the JSON is UTF-8 we can avoid going throught that detection. --- CHANGES.md | 4 ++++ src/cloudant/_common_util.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index bf15910c..6d8e112f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# Unreleased + +- [FIXED] A performance regression deserializing JSON in version 2.10.1. + # 2.10.1 (2018-11-16) - [FIXED] Unexpected keyword argument errors when using the library with the diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 7a373815..5460cd1c 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -288,6 +288,8 @@ def response_to_json_dict(response, **kwargs): :returns: dict of JSON response """ + if response.encoding is None: + response.encoding = 'utf-8' return json.loads(response.text, **kwargs) # Classes From 442526dc3eb0e6d4d69236e115b2a76109ebef9d Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 27 Nov 2018 13:08:43 +0000 Subject: [PATCH 106/185] Silence unnecessary-pass linter warnings --- src/cloudant/_client_session.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cloudant/_client_session.py b/src/cloudant/_client_session.py index 820bc532..232f821d 100644 --- a/src/cloudant/_client_session.py +++ b/src/cloudant/_client_session.py @@ -95,12 +95,14 @@ def login(self): """ No-op method - not implemented here. """ + # pylint: disable=unnecessary-pass pass def logout(self): """ No-op method - not implemented here. """ + # pylint: disable=unnecessary-pass pass From c13f42c61759f0c685338e8eb82d83116c292e82 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 18 Dec 2018 14:58:27 +0000 Subject: [PATCH 107/185] Updated version to 2.10.2 --- CHANGES.md | 2 +- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6d8e112f..2fc99a7f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# Unreleased +# 2.10.2 (2018-12-19) - [FIXED] A performance regression deserializing JSON in version 2.10.1. diff --git a/VERSION b/VERSION index 4e9cca7c..c6436a85 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10.2-SNAPSHOT +2.10.2 diff --git a/docs/conf.py b/docs/conf.py index 562082d6..7490396b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.10.2-SNAPSHOT' +version = '2.10.2' # The full version, including alpha/beta/rc tags. -release = '2.10.2-SNAPSHOT' +release = '2.10.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 74f57225..ef439a24 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.10.2-SNAPSHOT' +__version__ = '2.10.2' # pylint: disable=wrong-import-position import contextlib From 827f64684fb2d51f3770bce739e20623f38586fd Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 19 Dec 2018 11:07:18 +0000 Subject: [PATCH 108/185] Updated version to 2.10.3-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index c6436a85..2814d561 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10.2 +2.10.3-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index 7490396b..2ff46dea 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.10.2' +version = '2.10.3-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.10.2' +release = '2.10.3-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index ef439a24..3fac6b7e 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.10.2' +__version__ = '2.10.3-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From 2b3b14ae40256bb45577c1aa2d2c5707ad49c598 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 9 Jan 2019 10:41:36 +0000 Subject: [PATCH 109/185] Add option for client to authenticate with IAM token server. --- CHANGES.md | 4 ++++ src/cloudant/_client_session.py | 10 +++++++--- src/cloudant/client.py | 13 ++++++++++-- tests/unit/iam_auth_tests.py | 35 +++++++++++++++++++++++++++++++-- 4 files changed, 55 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2fc99a7f..ca875da9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# UNRELEASED + +- [NEW] Added option for client to authenticate with IAM token server. + # 2.10.2 (2018-12-19) - [FIXED] A performance regression deserializing JSON in version 2.10.1. diff --git a/src/cloudant/_client_session.py b/src/cloudant/_client_session.py index 232f821d..82f3c7ab 100644 --- a/src/cloudant/_client_session.py +++ b/src/cloudant/_client_session.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (c) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -189,7 +189,8 @@ class IAMSession(ClientSession): This class extends ClientSession and provides IAM authentication. """ - def __init__(self, api_key, server_url, **kwargs): + def __init__(self, api_key, server_url, client_id=None, client_secret=None, + **kwargs): super(IAMSession, self).__init__( session_url=url_join(server_url, '_iam_session'), **kwargs) @@ -197,6 +198,9 @@ def __init__(self, api_key, server_url, **kwargs): self._api_key = api_key self._token_url = os.environ.get( 'IAM_TOKEN_URL', 'https://iam.bluemix.net/identity/token') + self._token_auth = None + if client_id and client_secret: + self._token_auth = (client_id, client_secret) @property def get_api_key(self): @@ -277,7 +281,7 @@ def _get_access_token(self): resp = super(IAMSession, self).request( 'POST', self._token_url, - auth=('bx', 'bx'), # required for user API keys + auth=self._token_auth, headers={'Accepts': 'application/json'}, data={ 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 9b22d0d7..568f905e 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (c) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -78,6 +78,10 @@ class CouchDB(dict): IAM authentication with server. Default is False. Use :func:`~cloudant.client.CouchDB.iam` to construct an IAM authenticated client. + :param string iam_client_id: Keyword argument, client ID to use when + authenticating with the IAM token server. Default is ``None``. + :param string iam_client_secret: Keyword argument, client secret to use when + authenticating with the IAM token server. Default is ``None``. """ _DATABASE_CLASS = CouchDatabase @@ -95,6 +99,8 @@ def __init__(self, user, auth_token, admin_party=False, **kwargs): self._auto_renew = kwargs.get('auto_renew', False) self._use_basic_auth = kwargs.get('use_basic_auth', False) self._use_iam = kwargs.get('use_iam', False) + self._iam_client_id = kwargs.get('iam_client_id', None) + self._iam_client_secret = kwargs.get('iam_client_secret', None) # If user/pass exist in URL, remove and set variables if not self._use_basic_auth and self.server_url: parsed_url = url_parse(kwargs.get('url')) @@ -162,6 +168,8 @@ def connect(self): self._auth_token, self.server_url, auto_renew=self._auto_renew, + client_id=self._iam_client_id, + client_secret=self._iam_client_secret, timeout=self._timeout ) else: @@ -844,7 +852,8 @@ def bluemix(cls, vcap_services, instance_name=None, service_name=None, **kwargs) if hasattr(service, 'iam_api_key'): return Cloudant.iam(service.username, service.iam_api_key, - url=service.url) + url=service.url, + **kwargs) return Cloudant(service.username, service.password, url=service.url, diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py index ec4fbd95..f36af4e0 100644 --- a/tests/unit/iam_auth_tests.py +++ b/tests/unit/iam_auth_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2017 IBM. All rights reserved. +# Copyright (c) 2017, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -108,7 +108,38 @@ def test_iam_get_access_token(self, m_req): m_req.assert_called_once_with( 'POST', iam._token_url, - auth=('bx', 'bx'), + auth=None, + headers={'Accepts': 'application/json'}, + data={ + 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', + 'response_type': 'cloud_iam', + 'apikey': MOCK_API_KEY + } + ) + + self.assertEqual(access_token, MOCK_ACCESS_TOKEN) + self.assertTrue(m_response.raise_for_status.called) + mock_token_response_text.assert_called_with() + + @mock.patch('cloudant._client_session.ClientSession.request') + def test_iam_get_access_token_with_iam_client_id_and_secret(self, m_req): + m_response = mock.MagicMock() + mock_token_response_text = mock.PropertyMock(return_value=MOCK_IAM_TOKEN_RESPONSE) + type(m_response).text = mock_token_response_text + m_req.return_value = m_response + + iam_client_id = 'foo' + iam_client_secret = 'bar' + + iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984', + client_id=iam_client_id, + client_secret=iam_client_secret) + access_token = iam._get_access_token() + + m_req.assert_called_once_with( + 'POST', + iam._token_url, + auth=(iam_client_id, iam_client_secret), headers={'Accepts': 'application/json'}, data={ 'grant_type': 'urn:ibm:params:oauth:grant-type:apikey', From 90473f459c6b75923811382c54fc9e70fc288709 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 8 Jan 2019 15:34:05 +0000 Subject: [PATCH 110/185] Update the default IAM token server URL --- CHANGES.md | 1 + docs/getting_started.rst | 2 +- src/cloudant/_client_session.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ca875da9..cd925be6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # UNRELEASED - [NEW] Added option for client to authenticate with IAM token server. +- [FIXED] Updated the default IAM token server URL. # 2.10.2 (2018-12-19) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index a1cd43f7..04a46846 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -100,7 +100,7 @@ Cloud Platform. See `IBM Cloud Identity and Access Management `_ for more information. -The production IAM token service at *https://iam.bluemix.net/identity/token* is used +The production IAM token service at *https://iam.cloud.ibm.com/identity/token* is used by default. You can set an ``IAM_TOKEN_URL`` environment variable to override this. diff --git a/src/cloudant/_client_session.py b/src/cloudant/_client_session.py index 82f3c7ab..f6339ed2 100644 --- a/src/cloudant/_client_session.py +++ b/src/cloudant/_client_session.py @@ -197,7 +197,7 @@ def __init__(self, api_key, server_url, client_id=None, client_secret=None, self._api_key = api_key self._token_url = os.environ.get( - 'IAM_TOKEN_URL', 'https://iam.bluemix.net/identity/token') + 'IAM_TOKEN_URL', 'https://iam.cloud.ibm.com/identity/token') self._token_auth = None if client_id and client_secret: self._token_auth = (client_id, client_secret) From e3baf4746c485a0055ce2033e684e83dc5c3faec Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 21 Jan 2019 11:57:09 +0000 Subject: [PATCH 111/185] Prepare version 2.11.0 release --- CHANGES.md | 2 +- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cd925be6..3ed58be9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# UNRELEASED +# 2.11.0 (2019-01-21) - [NEW] Added option for client to authenticate with IAM token server. - [FIXED] Updated the default IAM token server URL. diff --git a/VERSION b/VERSION index 2814d561..46b81d81 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10.3-SNAPSHOT +2.11.0 diff --git a/docs/conf.py b/docs/conf.py index 2ff46dea..e0d01235 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.10.3-SNAPSHOT' +version = '2.11.0' # The full version, including alpha/beta/rc tags. -release = '2.10.3-SNAPSHOT' +release = '2.11.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 3fac6b7e..67a64efb 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.10.3-SNAPSHOT' +__version__ = '2.11.0' # pylint: disable=wrong-import-position import contextlib From 78ea0c476b2f1f3bf8844ea272f7691c47b9f806 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 21 Jan 2019 17:55:59 +0000 Subject: [PATCH 112/185] Update version to 2.11.1-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 46b81d81..57f9a3c9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.11.0 +2.11.1-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index e0d01235..9a6ab4d4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,9 +58,9 @@ # built documents. # # The short X.Y version. -version = '2.11.0' +version = '2.11.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.11.0' +release = '2.11.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 67a64efb..89d142f9 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.11.0' +__version__ = '2.11.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From e4a0224727b1b3ee5b9988801ddc2874484181f0 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 25 Jan 2019 11:32:25 -0500 Subject: [PATCH 113/185] Added get_query_result example to getting started (#423) Added get_query_result example to getting started --- CHANGES.md | 4 ++++ docs/getting_started.rst | 20 ++++++++++++++++++++ test-requirements.txt | 1 + 3 files changed, 25 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 3ed58be9..93f3b5e4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# Unreleased + +- [IMPROVED] Updated `Getting started` section with a `get_query_result` example. + # 2.11.0 (2019-01-21) - [NEW] Added option for client to authenticate with IAM token server. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 04a46846..8ad13cd8 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -372,6 +372,26 @@ object already exists. for result in result_collection: print(result) +This example retrieves the query result from the specified database based on the query parameters provided, updates the +document, and saves the document in the remote database. +By default, the result is returned as a ``QueryResult`` which uses the skip and limit query parameters internally to +handle slicing and iteration through the query result collection. For more detail on slicing and iteration, refer +to the :class:`~cloudant.result.QueryResult` documentation. + +.. code-block:: python + + # Retrieve documents where the name field is 'foo' + selector = {'name': {'$eq': 'foo'}} + docs = my_database.get_query_result(selector) + for doc in docs: + # Create Document object from dict + updated_doc = Document(my_database, doc['_id']) + updated_doc.update(doc) + # Update document field + updated_doc['name'] = 'new_name' + # Save document + updated_doc.save() + **************** Context managers **************** diff --git a/test-requirements.txt b/test-requirements.txt index 098c70e5..88c13ffb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,6 @@ mock==1.3.0 nose sphinx +sphinx_rtd_theme pylint flaky From 00b6cbb4e297477be214b7202d627bb99bc29223 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 1 Feb 2019 10:43:01 -0500 Subject: [PATCH 114/185] Fixed type in docstring for get_query_result (#425) Updated parameter `selector` from str to dict in docstring --- CHANGES.md | 1 + src/cloudant/database.py | 4 ++-- src/cloudant/query.py | 8 ++++---- src/cloudant/result.py | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 93f3b5e4..fed3ca54 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # Unreleased - [IMPROVED] Updated `Getting started` section with a `get_query_result` example. +- [FIXED] Fixed parameter type of `selector` in docstring. # 2.11.0 (2019-01-21) diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 51ff2430..da7b8a80 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -1112,7 +1112,7 @@ def get_query_result(self, selector, fields=None, raw_result=False, For more detail on slicing and iteration, refer to the :class:`~cloudant.result.QueryResult` documentation. - :param str selector: Dictionary object describing criteria used to + :param dict selector: Dictionary object describing criteria used to select documents. :param list fields: A list of fields to be returned by the query. :param bool raw_result: Dictates whether the query result is returned diff --git a/src/cloudant/query.py b/src/cloudant/query.py index ed9ec35e..791fd908 100644 --- a/src/cloudant/query.py +++ b/src/cloudant/query.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -75,7 +75,7 @@ class Query(dict): :param int limit: Maximum number of results returned. :param int r: Read quorum needed for the result. Each document is read from at least 'r' number of replicas before it is returned in the results. - :param str selector: Dictionary object describing criteria used to select + :param dict selector: Dictionary object describing criteria used to select documents. :param int skip: Skip the first 'n' results, where 'n' is the value specified. @@ -137,7 +137,7 @@ def __call__(self, **kwargs): :param int r: Read quorum needed for the result. Each document is read from at least 'r' number of replicas before it is returned in the results. - :param str selector: Dictionary object describing criteria used to + :param dict selector: Dictionary object describing criteria used to select documents. :param int skip: Skip the first 'n' results, where 'n' is the value specified. @@ -201,7 +201,7 @@ def custom_result(self, **options): :param int r: Read quorum needed for the result. Each document is read from at least 'r' number of replicas before it is returned in the results. - :param str selector: Dictionary object describing criteria used to + :param dict selector: Dictionary object describing criteria used to select documents. :param list sort: A list of fields to sort by. Optionally the list can contain elements that are single member dictionary structures that diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 302b7128..0fdcc472 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -454,7 +454,7 @@ class QueryResult(Result): :param int r: Read quorum needed for the result. Each document is read from at least 'r' number of replicas before it is returned in the results. - :param str selector: Dictionary object describing criteria used to + :param dict selector: Dictionary object describing criteria used to select documents. :param list sort: A list of fields to sort by. Optionally the list can contain elements that are single member dictionary structures that From 0bf212fe2f010b8dc47e66ee137f8b305e788a66 Mon Sep 17 00:00:00 2001 From: Alessandro Ogier Date: Sun, 10 Feb 2019 18:22:09 +0100 Subject: [PATCH 115/185] Fixing test_update_field_success_on_retry --- tests/unit/document_tests.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 3676c2b1..4fa0e85c 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import cloudant """ _document_tests_ @@ -523,9 +524,18 @@ def test_update_field_success_on_retry(self): # Mock when saving the document # 1st call throw a 409 # 2nd call delegate to the real doc.save() - with mock.patch('cloudant.document.Document.save', - side_effect=[requests.HTTPError(response=mock.Mock(status_code=409, reason='conflict')), - doc.save()]) as m_save: + + class SaveMock(object): + calls = 0 + def save(self): + if self.calls == 0: + self.calls += 1 + raise requests.HTTPError(response=mock.Mock(status_code=409, reason='conflict')) + else: + return cloudant.document.Document.save(doc) + + with mock.patch.object(doc, 'save', + side_effect=SaveMock().save) as m_save: # A list of side effects containing only 1 element doc.update_field(doc.field_set, 'age', 7, max_tries=1) # Two calls to save, one with a 409 and one that succeeds From 0c67682adc6f27a12abcab4d401c0abff719d8c1 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 12 Feb 2019 12:00:39 +0000 Subject: [PATCH 116/185] Mock server responses for DatabaseTests.test_get_set_revision_limit. The `PUT /db/_revs_limit` endpoint was disabled in the Cloudant service. --- tests/unit/database_tests.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index f4c0e9aa..d57ba654 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -773,16 +773,36 @@ def test_revisions_diff(self): # Test no differences self.assertEqual(self.db.revisions_diff('julia006', doc['_rev']), {}) - def test_get_set_revision_limit(self): + @mock.patch('cloudant._client_session.ClientSession.request') + def test_get_set_revision_limit(self, m_req): """ Test setting and getting revision limits """ - limit = self.db.get_revision_limit() - self.assertIsInstance(limit, int) + # Setup mock responses. + mock_200_get_1 = mock.MagicMock() + type(mock_200_get_1).status_code = mock.PropertyMock(return_value=200) + type(mock_200_get_1).text = mock.PropertyMock(return_value='4321') + + mock_200_get_2 = mock.MagicMock() + type(mock_200_get_2).status_code = mock.PropertyMock(return_value=200) + type(mock_200_get_2).text = mock.PropertyMock(return_value='1234') + + mock_200_set = mock.MagicMock() + type(mock_200_set).status_code = mock.PropertyMock(return_value=200) + type(mock_200_set).text = mock.PropertyMock(return_value='{"ok":true}') + + m_req.side_effect = [mock_200_get_1, mock_200_set, mock_200_get_2] + + # Get current revisions limit. + self.assertEqual(self.db.get_revision_limit(), 4321) + + # Set new revisions limit. self.assertEqual(self.db.set_revision_limit(1234), {'ok': True}) - new_limit = self.db.get_revision_limit() - self.assertNotEqual(new_limit, limit) - self.assertEqual(new_limit, 1234) + + # Get new revisions limit. + self.assertEqual(self.db.get_revision_limit(), 1234) + + self.assertEquals(m_req.call_count, 3) @attr(db='couch') def test_view_clean_up(self): From 467dff23b6ad3400bee12800293e8ca586d65d80 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 12 Feb 2019 16:06:21 +0000 Subject: [PATCH 117/185] Stop document context manager from saving if an error is raised. --- CHANGES.md | 4 +++- src/cloudant/document.py | 9 ++++---- tests/unit/document_tests.py | 41 ++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fed3ca54..c7d48b60 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,9 @@ # Unreleased -- [IMPROVED] Updated `Getting started` section with a `get_query_result` example. +- [FIXED] Bug where document context manager performed remote save despite + uncaught exceptions being raised inside `with` block. - [FIXED] Fixed parameter type of `selector` in docstring. +- [IMPROVED] Updated `Getting started` section with a `get_query_result` example. # 2.11.0 (2019-01-21) diff --git a/src/cloudant/document.py b/src/cloudant/document.py index 426b0656..ad630ac0 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -341,12 +341,13 @@ def __enter__(self): return self - def __exit__(self, *args): + def __exit__(self, exc_type, exc_value, traceback): """ - Support context like editing of document fields. Handles context exit - logic. Executes a Document.save() upon exit. + Support context like editing of document fields. Handles context exit + logic. Executes a `Document.save()` upon exit if no exception occurred. """ - self.save() + if exc_type is None: + self.save() def __setitem__(self, key, value): """ diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 4fa0e85c..b9d72a0f 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -622,6 +622,47 @@ def test_document_context_manager_no_doc_id(self): self.assertTrue(doc['_rev'].startswith('1-')) self.assertEqual(self.db['julia006'], doc) + def test_document_context_manager_creation_failure_on_error(self): + """ + Test that the document context manager skips document creation if there + is an error. + """ + with self.assertRaises(ZeroDivisionError), Document(self.db, 'julia006') as doc: + doc['name'] = 'julia' + doc['age'] = 6 + raise ZeroDivisionError() + + doc = Document(self.db, 'julia006') + try: + doc.fetch() + except requests.HTTPError as err: + self.assertEqual(err.response.status_code, 404) + else: + self.fail('Above statement should raise a HTTPError.') + + def test_document_context_manager_update_failure_on_error(self): + """ + Test that the document context manager skips document update if there + is an error. + """ + # Create the document. + doc = Document(self.db, 'julia006') + doc['name'] = 'julia' + doc['age'] = 6 + doc.save() + + # Make a document update and then raise an error. + with self.assertRaises(ZeroDivisionError), Document(self.db, 'julia006') as doc: + doc['age'] = 7 + raise ZeroDivisionError() + + # Assert the change persists locally. + self.assertEqual(doc['age'], 7) + + # Assert the document has not been saved to remote server. + self.assertTrue(doc['_rev'].startswith('1-')) + self.assertEqual(self.db['julia006']['age'], 6) + def test_document_context_manager_doc_create(self): """ Test that the document context manager will create a doc if it does From af07c04ae63754123a5a393061c0bde116c3f359 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 14 Feb 2019 14:54:44 +0000 Subject: [PATCH 118/185] Add warning to document CM documentation. --- docs/conf.py | 2 ++ docs/getting_started.rst | 54 +++++++++++++++++++++------------------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9a6ab4d4..3df55b2e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,6 +21,8 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('../src')) + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 8ad13cd8..072fe061 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -441,41 +441,45 @@ multiple updates to a single document. Note that we don't save to the server after each update. We only save once to the server upon exiting the ``Document`` context manager. - .. code-block:: python +.. warning:: Uncaught exceptions inside the ``with`` block will prevent your + document changes being saved to the remote server. However, changes + will still be applied to your local document object. - from cloudant import cloudant - from cloudant.document import Document +.. code-block:: python - with cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME) as client: + from cloudant import cloudant + from cloudant.document import Document - my_database = client.create_database('my_database') + with cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME) as client: - # Upon entry into the document context, fetches the document from the - # remote database, if it exists. Upon exit from the context, saves the - # document to the remote database with changes made within the context - # or creates a new document. - with Document(database, 'julia006') as document: - # If document exists, it's fetched from the remote database - # Changes are made locally - document['name'] = 'Julia' - document['age'] = 6 - # The document is saved to the remote database - - # Display a Document - print(my_database['julia30']) - - # Delete the database - client.delete_database('my_database') + my_database = client.create_database('my_database') - print('Databases: {0}'.format(client.all_dbs())) + # Upon entry into the document context, fetches the document from the + # remote database, if it exists. Upon exit from the context, saves the + # document to the remote database with changes made within the context + # or creates a new document. + with Document(database, 'julia006') as document: + # If document exists, it's fetched from the remote database + # Changes are made locally + document['name'] = 'Julia' + document['age'] = 6 + # The document is saved to the remote database + + # Display a Document + print(my_database['julia30']) + + # Delete the database + client.delete_database('my_database') + + print('Databases: {0}'.format(client.all_dbs())) Always use the ``_deleted`` document property to delete a document from within a ``Document`` context manager. For example: - .. code-block:: python +.. code-block:: python - with Document(my_database, 'julia30') as doc: - doc['_deleted'] = True + with Document(my_database, 'julia30') as doc: + doc['_deleted'] = True *You can also delete non underscore prefixed document keys to reduce the size of the request.* From e3e83d074528561cb296aeebdeaa0c190958156f Mon Sep 17 00:00:00 2001 From: Alessandro Ogier Date: Sun, 10 Feb 2019 14:01:40 +0100 Subject: [PATCH 119/185] removed _document_id logic (Fixes #430) --- CHANGES.md | 2 ++ src/cloudant/document.py | 38 +++++++++--------------------------- tests/unit/document_tests.py | 3 --- 3 files changed, 11 insertions(+), 32 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c7d48b60..0a3237d3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ uncaught exceptions being raised inside `with` block. - [FIXED] Fixed parameter type of `selector` in docstring. - [IMPROVED] Updated `Getting started` section with a `get_query_result` example. +- [FIXED] Removed internal `Document._document_id` property to allow a safe use of + dict's methods. # 2.11.0 (2019-01-21) diff --git a/src/cloudant/document.py b/src/cloudant/document.py index ad630ac0..38082054 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -63,9 +63,8 @@ def __init__(self, database, document_id=None, **kwargs): self._database = database self._database_host = self._client.server_url self._database_name = database.database_name - self._document_id = document_id - if self._document_id is not None: - self['_id'] = self._document_id + if document_id: + self['_id'] = document_id self.encoder = kwargs.get('encoder') or self._client.encoder self.decoder = kwargs.get('decoder') or json.JSONDecoder @@ -85,23 +84,23 @@ def document_url(self): :returns: Document URL """ - if self._document_id is None: + if '_id' not in self or self['_id'] is None: return None # handle design document url - if self._document_id.startswith('_design/'): + if self['_id'].startswith('_design/'): return '/'.join(( self._database_host, url_quote_plus(self._database_name), '_design', - url_quote(self._document_id[8:], safe='') + url_quote(self['_id'][8:], safe='') )) # handle document url return '/'.join(( self._database_host, url_quote_plus(self._database_name), - url_quote(self._document_id, safe='') + url_quote(self['_id'], safe='') )) def exists(self): @@ -111,7 +110,7 @@ def exists(self): :returns: True if the document exists in the remote database, otherwise False """ - if self._document_id is None: + if '_id' not in self or self['_id'] is None: return False resp = self.r_session.head(self.document_url) @@ -136,8 +135,6 @@ def create(self): updates the locally cached Document object with the ``_id`` and ``_rev`` returned as part of the successful response. """ - if self._document_id is not None: - self['_id'] = self._document_id # Ensure that an existing document will not be "updated" doc = dict(self) @@ -152,7 +149,6 @@ def create(self): ) resp.raise_for_status() data = response_to_json_dict(resp) - self._document_id = data['id'] super(Document, self).__setitem__('_id', data['id']) super(Document, self).__setitem__('_rev', data['rev']) @@ -318,8 +314,9 @@ def delete(self): params={"rev": self["_rev"]}, ) del_resp.raise_for_status() + _id = self['_id'] self.clear() - self.__setitem__('_id', self._document_id) + self['_id'] = _id def __enter__(self): """ @@ -349,23 +346,6 @@ def __exit__(self, exc_type, exc_value, traceback): if exc_type is None: self.save() - def __setitem__(self, key, value): - """ - Sets the _document_id when setting the '_id' field. - The _document_id is used to construct the document url. - """ - if key == '_id': - self._document_id = value - super(Document, self).__setitem__(key, value) - - def __delitem__(self, key): - """ - Sets the _document_id to None when deleting the '_id' field. - """ - if key == '_id': - self._document_id = None - super(Document, self).__delitem__(key) - def get_attachment( self, attachment, diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index b9d72a0f..61fa9fac 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -680,10 +680,8 @@ def test_setting_id(self): """ doc = Document(self.db) self.assertIsNone(doc.get('_id')) - self.assertEqual(doc._document_id, None) doc['_id'] = 'julia006' self.assertEqual(doc['_id'], 'julia006') - self.assertEqual(doc._document_id, 'julia006') def test_removing_id(self): """ @@ -693,7 +691,6 @@ def test_removing_id(self): doc['_id'] = 'julia006' del doc['_id'] self.assertIsNone(doc.get('_id')) - self.assertEqual(doc._document_id, None) def test_get_text_attachment(self): """ From cb44158d6b459c806d4760f03fc85e184567e6ba Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Mon, 11 Mar 2019 12:25:29 -0400 Subject: [PATCH 120/185] Fixed no-else-raise pylint warning (#438) --- src/cloudant/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 568f905e..79ce0993 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -629,9 +629,9 @@ def _usage_endpoint(self, endpoint, year=None, month=None): if err: raise CloudantArgumentError(101, year, month) - else: - resp.raise_for_status() - return response_to_json_dict(resp) + + resp.raise_for_status() + return response_to_json_dict(resp) def bill(self, year=None, month=None): """ From 1cb8abf9ccab6ff18c3d45dd8edcb32df1918f0a Mon Sep 17 00:00:00 2001 From: Alessandro Ogier Date: Mon, 18 Mar 2019 15:26:36 +0100 Subject: [PATCH 121/185] paginate by startkey (#437) * startkey/bookmark pagination --- CHANGES.md | 2 + src/cloudant/result.py | 95 +++++++++++++++++++++++++++----- tests/unit/query_result_tests.py | 14 +---- tests/unit/result_tests.py | 60 ++++++++++++++++---- tests/unit/unit_t_db_base.py | 5 ++ 5 files changed, 139 insertions(+), 37 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0a3237d3..3cbf6b60 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Unreleased +- [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and + queries' `bookmark`. - [FIXED] Bug where document context manager performed remote save despite uncaught exceptions being raised inside `with` block. - [FIXED] Fixed parameter type of `selector` in docstring. diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 0fdcc472..23fd3aac 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -15,6 +15,7 @@ """ API module for interacting with result collections. """ +from functools import partial from ._2to3 import STRTYPE from .error import ResultException from ._common_util import py_to_couch_validate, type_or_none @@ -327,16 +328,16 @@ def _handle_result_by_key_slice(self, key_slice): def __iter__(self): """ Provides iteration support, primarily for large data collections. - The iterator uses the ``skip`` and ``limit`` options to consume - data in chunks controlled by the ``page_size`` option. It retrieves - a batch of data from the result collection and then yields each - element. + The iterator uses the ``startkey``, ``startkey_docid``, and ``limit`` + options to consume data in chunks controlled by the ``page_size`` + option. It retrieves a batch of data from the result collection + and then yields each element. See :class:`~cloudant.result.Result` for Result iteration examples. :returns: Iterable data sequence """ - invalid_options = ('skip', 'limit') + invalid_options = ('limit', ) if any(x in invalid_options for x in self.options): raise ResultException(103, invalid_options, self.options) @@ -347,21 +348,60 @@ def __iter__(self): except ValueError: raise ResultException(104, self._page_size) - skip = 0 + init_opts = { + 'skip': self.options.pop('skip', None), + 'startkey': self.options.pop('startkey', None) + } + + self._call = partial(self._ref, #pylint: disable=attribute-defined-outside-init + limit=self._real_page_size, + **self.options) + + response = self._call(**{k: v + for k, v + in init_opts.items() + if v is not None}) + + return self._iterator(response) + + @property + def _real_page_size(self): + ''' + In views we paginate with N+1 items per page. + https://docs.couchdb.org/en/stable/ddocs/views/pagination.html#paging-alternate-method + ''' + return self._page_size + 1 + + def _iterator(self, response): + ''' + Iterate through view data. + ''' + while True: - response = self._ref( - limit=self._page_size, - skip=skip, - **self.options - ) result = self._parse_data(response) - skip += self._page_size if result: + doc_count = len(result) + last = result.pop() for row in result: yield row - if len(result) < self._page_size: + + # We expect doc_count = self._page_size + 1 results, if + # we have self._page_size or less it means we are on the + # last page and need to return the last result. + if doc_count < self._real_page_size: + yield last break del result + + # if we are in a view, keys could be duplicate so we + # need to start from the right docid + if last['id']: + response = self._call(startkey=last['key'], + startkey_docid=last['id']) + # reduce result keys are unique by definition + else: + response = self._call(startkey=last['key']) + else: break @@ -510,3 +550,32 @@ def _parse_data(self, data): query result JSON response content """ return data.get('docs', []) + + @property + def _real_page_size(self): + ''' + During queries iteration page size is user-specified + ''' + return self._page_size + + def _iterator(self, response): + ''' + Iterate through query data. + ''' + + while True: + result = self._parse_data(response) + bookmark = response.get('bookmark') + if result: + for row in result: + yield row + + del result + + if not bookmark: + break + + response = self._call(bookmark=bookmark) + + else: + break diff --git a/tests/unit/query_result_tests.py b/tests/unit/query_result_tests.py index 9f4fd170..98a16f3a 100644 --- a/tests/unit/query_result_tests.py +++ b/tests/unit/query_result_tests.py @@ -425,24 +425,14 @@ def test_get_item_index_slice_using_stop_only_limit_skip(self): def test_iteration_with_invalid_options(self): """ - Test that iteration raises an exception when "skip" and/or "limit" are - used as options for the result. + Test that iteration raises an exception when "limit" is + used as option for the result. """ - result = self.create_result(q_parms={'skip': 10}) - with self.assertRaises(ResultException) as cm: - invalid_result = [row for row in result] - self.assertEqual(cm.exception.status_code, 103) - result = self.create_result(q_parms={'limit': 10}) with self.assertRaises(ResultException) as cm: invalid_result = [row for row in result] self.assertEqual(cm.exception.status_code, 103) - result = self.create_result(q_parms={'limit': 10, 'skip': 10}) - with self.assertRaises(ResultException) as cm: - invalid_result = [row for row in result] - self.assertEqual(cm.exception.status_code, 103) - def test_iteration_invalid_page_size(self): """ Test that iteration raises an exception when and invalid "page_size" is diff --git a/tests/unit/result_tests.py b/tests/unit/result_tests.py index f56387ad..13b8eb7a 100644 --- a/tests/unit/result_tests.py +++ b/tests/unit/result_tests.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import mock """ result module - Unit tests for Result class """ @@ -19,6 +20,7 @@ from cloudant.error import ResultException from cloudant.result import Result, ResultByKey +from cloudant.view import View from nose.plugins.attrib import attr from requests.exceptions import HTTPError @@ -564,24 +566,14 @@ def test_get_item_key_slice_using_stop_only(self): def test_iteration_with_invalid_options(self): """ - Test that iteration raises an exception when "skip" and/or "limit" are - used as options for the result. + Test that iteration raises an exception when "limit" is + used as option for the result. """ - result = Result(self.view001, skip=10) - with self.assertRaises(ResultException) as cm: - invalid_result = [row for row in result] - self.assertEqual(cm.exception.status_code, 103) - result = Result(self.view001, limit=10) with self.assertRaises(ResultException) as cm: invalid_result = [row for row in result] self.assertEqual(cm.exception.status_code, 103) - result = Result(self.view001, skip=10, limit=10) - with self.assertRaises(ResultException) as cm: - invalid_result = [row for row in result] - self.assertEqual(cm.exception.status_code, 103) - def test_iteration_invalid_page_size(self): """ Test that iteration raises an exception when and invalid "page_size" is @@ -643,5 +635,49 @@ def test_iteration_no_data(self): result = Result(self.view001, startkey='ruby') self.assertEqual([x for x in result], []) + def test_iteration_integer_keys(self): + """ + Test that iteration works as expected when keys are integer. + """ + result = Result(self.view007, page_size=10) + self.assertEqual(len([x for x in result]), 100) + + def test_iteration_pagination(self): + """ + Test that iteration pagination works as expected. + """ + + class CallMock: + expected_calls = [ + {'limit': 28}, + {'limit': 28, 'startkey': 1, 'startkey_docid': 'julia027'}, + {'limit': 28, 'startkey': 1, 'startkey_docid': 'julia054'}, + {'limit': 28, 'startkey': 1, 'startkey_docid': 'julia081'}, + ] + + def __init__(self, outer): + self.outer = outer + self.expected_calls.reverse() + + def call(self, *args, **kwargs): + self.outer.assertEqual(dict(kwargs), + self.expected_calls.pop(), + 'pagination error') + return View.__call__(self.outer.view007, *args, **kwargs) + + with mock.patch.object(self, 'view007', + CallMock(self).call) as _: + + result = Result(self.view007, page_size=27) + + expected = [ + {'id': 'julia{0:03d}'.format(i), + 'key': 1, + 'value': 'julia'} + for i in range(100) + ] + self.assertEqual([x for x in result], expected) + + if __name__ == '__main__': unittest.main() diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index 1ca9220f..ad7eefd6 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -314,6 +314,10 @@ def create_views(self): 'function (doc) {\n emit([doc.name, doc.age], 1);\n}', '_count' ) + self.ddoc.add_view( + 'view007', + 'function (doc) {\n emit(1, doc.name);\n}' + ) self.ddoc.save() self.view001 = self.ddoc.get_view('view001') self.view002 = self.ddoc.get_view('view002') @@ -321,6 +325,7 @@ def create_views(self): self.view004 = self.ddoc.get_view('view004') self.view005 = self.ddoc.get_view('view005') self.view006 = self.ddoc.get_view('view006') + self.view007 = self.ddoc.get_view('view007') def create_search_index(self): """ From e1d4075f43627afae577506ab60d3a5a3eeb1720 Mon Sep 17 00:00:00 2001 From: Alessandro Ogier Date: Mon, 11 Mar 2019 17:16:05 +0100 Subject: [PATCH 122/185] slimming iterator --- src/cloudant/result.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 23fd3aac..f51e1699 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -15,6 +15,7 @@ """ API module for interacting with result collections. """ +from collections import deque from functools import partial from ._2to3 import STRTYPE from .error import ResultException @@ -378,12 +379,12 @@ def _iterator(self, response): ''' while True: - result = self._parse_data(response) + result = deque(self._parse_data(response)) if result: doc_count = len(result) last = result.pop() - for row in result: - yield row + while result: + yield result.popleft() # We expect doc_count = self._page_size + 1 results, if # we have self._page_size or less it means we are on the From 55e7b472f75d7deb8dffd48c9250fcb31dc446ec Mon Sep 17 00:00:00 2001 From: Alessandro Ogier Date: Sat, 9 Mar 2019 00:02:42 +0100 Subject: [PATCH 123/185] halves memory usage iterator side --- src/cloudant/result.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cloudant/result.py b/src/cloudant/result.py index 23fd3aac..b934114d 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -379,6 +379,7 @@ def _iterator(self, response): while True: result = self._parse_data(response) + del response if result: doc_count = len(result) last = result.pop() From 5f926a0a122051f7fa73d31a5ade246cf1f28efb Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 6 Feb 2019 16:43:46 +0000 Subject: [PATCH 124/185] Add partitioned databases feature. --- CHANGES.md | 1 + src/cloudant/_common_util.py | 5 +- src/cloudant/client.py | 6 +- src/cloudant/database.py | 199 +++++++++++++++++++++++-- src/cloudant/design_document.py | 27 +++- src/cloudant/index.py | 24 ++- src/cloudant/query.py | 13 +- src/cloudant/view.py | 14 +- tests/unit/database_partition_tests.py | 130 ++++++++++++++++ tests/unit/unit_t_db_base.py | 21 ++- 10 files changed, 410 insertions(+), 30 deletions(-) create mode 100644 tests/unit/database_partition_tests.py diff --git a/CHANGES.md b/CHANGES.md index 3cbf6b60..63255bb2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # Unreleased +- [NEW] Added partitioned database support. - [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and queries' `bookmark`. - [FIXED] Bug where document context manager performed remote save despite diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 5460cd1c..f7885f49 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -140,7 +140,8 @@ 'highlight_post_tag': STRTYPE, 'highlight_number': (int, LONGTYPE, NONETYPE), 'highlight_size': (int, LONGTYPE, NONETYPE), - 'include_fields': list + 'include_fields': list, + 'partition': STRTYPE } # Functions diff --git a/src/cloudant/client.py b/src/cloudant/client.py index 79ce0993..b1993530 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -267,7 +267,7 @@ def all_dbs(self): resp.raise_for_status() return response_to_json_dict(resp) - def create_database(self, dbname, **kwargs): + def create_database(self, dbname, partitioned=False, **kwargs): """ Creates a new database on the remote server with the name provided and adds the new database object to the client's locally cached @@ -279,10 +279,12 @@ def create_database(self, dbname, **kwargs): :param bool throw_on_exists: Boolean flag dictating whether or not to throw a CloudantClientException when attempting to create a database that already exists. + :param bool partitioned: Create as a partitioned database. Defaults to + ``False``. :returns: The newly created database object """ - new_db = self._DATABASE_CLASS(self, dbname) + new_db = self._DATABASE_CLASS(self, dbname, partitioned=partitioned) try: new_db.create(kwargs.get('throw_on_exists', False)) except CloudantDatabaseException as ex: diff --git a/src/cloudant/database.py b/src/cloudant/database.py index da7b8a80..baf2df67 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -26,9 +26,9 @@ SEARCH_INDEX_ARGS, SPECIAL_INDEX_TYPE, TEXT_INDEX_TYPE, + TYPE_CONVERTERS, get_docs, - response_to_json_dict, - ) + response_to_json_dict) from .document import Document from .design_document import DesignDocument from .security_document import SecurityDocument @@ -50,13 +50,17 @@ class CouchDatabase(dict): :param str database_name: Database name used to reference the database. :param int fetch_limit: Optional fetch limit used to set the max number of documents to fetch per query during iteration cycles. Defaults to 100. + :param bool partitioned: Create as a partitioned database. Defaults to + ``False``. """ - def __init__(self, client, database_name, fetch_limit=100): + def __init__(self, client, database_name, fetch_limit=100, + partitioned=False): super(CouchDatabase, self).__init__() self.client = client self._database_host = client.server_url self.database_name = database_name self._fetch_limit = fetch_limit + self._partitioned = partitioned self.result = Result(self.all_docs) @property @@ -105,6 +109,18 @@ def creds(self): "user_ctx": session.get('userCtx') } + def database_partition_url(self, partition_key): + """ + Get the URL of the database partition. + + :param str partition_key: Partition key. + :return: URL of the database partition. + :rtype: str + """ + return '/'.join((self.database_url, + '_partition', + url_quote_plus(partition_key))) + def exists(self): """ Performs an existence check on the remote database. @@ -127,6 +143,18 @@ def metadata(self): resp.raise_for_status() return response_to_json_dict(resp) + def partition_metadata(self, partition_key): + """ + Retrieves the metadata dictionary for the remote database partition. + + :param str partition_key: Partition key. + :returns: Metadata dictionary for the database partition. + :rtype: dict + """ + resp = self.r_session.get(self.database_partition_url(partition_key)) + resp.raise_for_status() + return response_to_json_dict(resp) + def doc_count(self): """ Retrieves the number of documents in the remote database @@ -244,6 +272,33 @@ def get_security_document(self): sdoc.fetch() return sdoc + def get_partitioned_view_result(self, partition_key, ddoc_id, view_name, + raw_result=False, **kwargs): + """ + Retrieves the partitioned view result based on the design document and + view name. + + See :func:`~cloudant.database.CouchDatabase.get_view_result` method for + further details. + + :param str partition_key: Partition key. + :param str ddoc_id: Design document id used to get result. + :param str view_name: Name of the view used to get result. + :param bool raw_result: Dictates whether the view result is returned + as a default Result object or a raw JSON response. + Defaults to False. + :param kwargs: See + :func:`~cloudant.database.CouchDatabase.get_view_result` method for + available keyword arguments. + :returns: The result content either wrapped in a QueryResult or + as the raw response JSON content. + :rtype: QueryResult, dict + """ + ddoc = DesignDocument(self, ddoc_id) + view = View(ddoc, view_name, partition_key=partition_key) + + return self._get_view_result(view, raw_result, **kwargs) + def get_view_result(self, ddoc_id, view_name, raw_result=False, **kwargs): """ Retrieves the view result based on the design document and view name. @@ -336,7 +391,14 @@ def get_view_result(self, ddoc_id, view_name, raw_result=False, **kwargs): :returns: The result content either wrapped in a QueryResult or as the raw response JSON content """ - view = View(DesignDocument(self, ddoc_id), view_name) + ddoc = DesignDocument(self, ddoc_id) + view = View(ddoc, view_name) + + return self._get_view_result(view, raw_result, **kwargs) + + @staticmethod + def _get_view_result(view, raw_result, **kwargs): + """ Get view results helper. """ if raw_result: return view(**kwargs) if kwargs: @@ -359,7 +421,9 @@ def create(self, throw_on_exists=False): if not throw_on_exists and self.exists(): return self - resp = self.r_session.put(self.database_url) + resp = self.r_session.put(self.database_url, params={ + 'partitioned': TYPE_CONVERTERS.get(bool)(self._partitioned) + }) if resp.status_code == 201 or resp.status_code == 202: return self @@ -407,6 +471,29 @@ def all_docs(self, **kwargs): **kwargs) return response_to_json_dict(resp) + def partitioned_all_docs(self, partition_key, **kwargs): + """ + Wraps the _all_docs primary index on the database partition, and returns + the results by value. + + See :func:`~cloudant.database.CouchDatabase.all_docs` method for further + details. + + :param str partition_key: Partition key. + :param kwargs: See :func:`~cloudant.database.CouchDatabase.all_docs` + method for available keyword arguments. + :returns: Raw JSON response content from ``_all_docs`` endpoint. + :rtype: dict + """ + resp = get_docs(self.r_session, + '/'.join([ + self.database_partition_url(partition_key), + '_all_docs' + ]), + self.client.encoder, + **kwargs) + return response_to_json_dict(resp) + @contextlib.contextmanager def custom_result(self, **options): """ @@ -985,6 +1072,7 @@ def get_query_indexes(self, raw_result=False): self, data.get('ddoc'), data.get('name'), + partitioned=data.get('partitioned', False), **data.get('def', {}) )) elif data.get('type') == TEXT_INDEX_TYPE: @@ -992,6 +1080,7 @@ def get_query_indexes(self, raw_result=False): self, data.get('ddoc'), data.get('name'), + partitioned=data.get('partitioned', False), **data.get('def', {}) )) elif data.get('type') == SPECIAL_INDEX_TYPE: @@ -999,6 +1088,7 @@ def get_query_indexes(self, raw_result=False): self, data.get('ddoc'), data.get('name'), + partitioned=data.get('partitioned', False), **data.get('def', {}) )) else: @@ -1010,6 +1100,7 @@ def create_query_index( design_document_id=None, index_name=None, index_type='json', + partitioned=False, **kwargs ): """ @@ -1047,9 +1138,11 @@ def create_query_index( remote database """ if index_type == JSON_INDEX_TYPE: - index = Index(self, design_document_id, index_name, **kwargs) + index = Index(self, design_document_id, index_name, + partitioned=partitioned, **kwargs) elif index_type == TEXT_INDEX_TYPE: - index = TextIndex(self, design_document_id, index_name, **kwargs) + index = TextIndex(self, design_document_id, index_name, + partitioned=partitioned, **kwargs) else: raise CloudantArgumentError(103, index_type) index.create() @@ -1074,6 +1167,36 @@ def delete_query_index(self, design_document_id, index_type, index_name): raise CloudantArgumentError(103, index_type) index.delete() + def get_partitioned_query_result(self, partition_key, selector, fields=None, + raw_result=False, **kwargs): + """ + Retrieves the partitioned query result from the specified database based + on the query parameters provided. + + See :func:`~cloudant.database.CouchDatabase.get_query_result` method for + further details. + + :param str partition_key: Partition key. + :param str selector: Dictionary object describing criteria used to + select documents. + :param list fields: A list of fields to be returned by the query. + :param bool raw_result: Dictates whether the query result is returned + wrapped in a QueryResult or if the response JSON is returned. + Defaults to False. + :param kwargs: See + :func:`~cloudant.database.CouchDatabase.get_query_result` method for + available keyword arguments. + :returns: The result content either wrapped in a QueryResult or + as the raw response JSON content. + :rtype: QueryResult, dict + """ + query = Query(self, + selector=selector, + fields=fields, + partition_key=partition_key) + + return self._get_query_result(query, raw_result, **kwargs) + def get_query_result(self, selector, fields=None, raw_result=False, **kwargs): """ @@ -1143,10 +1266,15 @@ def get_query_result(self, selector, fields=None, raw_result=False, :returns: The result content either wrapped in a QueryResult or as the raw response JSON content """ - if fields: - query = Query(self, selector=selector, fields=fields) - else: - query = Query(self, selector=selector) + query = Query(self, + selector=selector, + fields=fields) + + return self._get_query_result(query, raw_result, **kwargs) + + @staticmethod + def _get_query_result(query, raw_result, **kwargs): + """ Get query results helper. """ if raw_result: return query(**kwargs) if kwargs: @@ -1166,12 +1294,16 @@ class CloudantDatabase(CouchDatabase): :param str database_name: Database name used to reference the database. :param int fetch_limit: Optional fetch limit used to set the max number of documents to fetch per query during iteration cycles. Defaults to 100. + :param bool partitioned: Create as a partitioned database. Defaults to + ``False``. """ - def __init__(self, client, database_name, fetch_limit=100): + def __init__(self, client, database_name, fetch_limit=100, + partitioned=False): super(CloudantDatabase, self).__init__( client, database_name, - fetch_limit=fetch_limit + fetch_limit=fetch_limit, + partitioned=partitioned ) def security_document(self): @@ -1280,6 +1412,36 @@ def shards(self): return response_to_json_dict(resp) + def get_partitioned_search_result(self, partition_key, ddoc_id, index_name, + **query_params): + """ + Retrieves the raw JSON content from the remote database based on the + partitioned search index on the server, using the query_params provided + as query parameters. + + See :func:`~cloudant.database.CouchDatabase.get_search_result` method + for further details. + + :param str partition_key: Partition key. + :param str ddoc_id: Design document id used to get the search result. + :param str index_name: Name used in part to identify the index. + :param query_params: See + :func:`~cloudant.database.CloudantDatabase.get_search_result` method + for available keyword arguments. + :returns: Search query result data in JSON format. + :rtype: dict + """ + ddoc = DesignDocument(self, ddoc_id) + + return self._get_search_result( + '/'.join(( + ddoc.document_partition_url(partition_key), + '_search', + index_name + )), + **query_params + ) + def get_search_result(self, ddoc_id, index_name, **query_params): """ Retrieves the raw JSON content from the remote database based on the @@ -1380,6 +1542,14 @@ def get_search_result(self, ddoc_id, index_name, **query_params): :returns: Search query result data in JSON format """ + ddoc = DesignDocument(self, ddoc_id) + return self._get_search_result( + '/'.join((ddoc.document_url, '_search', index_name)), + **query_params + ) + + def _get_search_result(self, query_url, **query_params): + """ Get search results helper. """ param_q = query_params.get('q') param_query = query_params.get('query') # Either q or query parameter is required @@ -1394,9 +1564,8 @@ def get_search_result(self, ddoc_id, index_name, **query_params): raise CloudantArgumentError(106, key, SEARCH_INDEX_ARGS[key]) # Execute query search headers = {'Content-Type': 'application/json'} - ddoc = DesignDocument(self, ddoc_id) resp = self.r_session.post( - '/'.join([ddoc.document_url, '_search', index_name]), + query_url, headers=headers, data=json.dumps(query_params, cls=self.client.encoder) ) diff --git a/src/cloudant/design_document.py b/src/cloudant/design_document.py index 66a9789d..34594b95 100644 --- a/src/cloudant/design_document.py +++ b/src/cloudant/design_document.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ """ API module/class for interacting with a design document in a database. """ -from ._2to3 import iteritems_, STRTYPE +from ._2to3 import iteritems_, url_quote_plus, STRTYPE from ._common_util import QUERY_LANGUAGE, codify, response_to_json_dict from .document import Document from .view import View, QueryIndexView @@ -39,11 +39,18 @@ class DesignDocument(Document): either a ``CouchDatabase`` or ``CloudantDatabase`` instance. :param str document_id: Optional document id. If provided and does not start with ``_design/``, it will be prepended with ``_design/``. + :param bool partitioned: Optional. Create as a partitioned design document. + Defaults to ``False`` for both partitioned and non-partitioned + databases. """ - def __init__(self, database, document_id=None): + def __init__(self, database, document_id=None, partitioned=False): if document_id and not document_id.startswith('_design/'): document_id = '_design/{0}'.format(document_id) super(DesignDocument, self).__init__(database, document_id) + + if partitioned: + self.setdefault('options', {'partitioned': True}) + self._nested_object_names = frozenset(['views', 'indexes', 'lists', 'shows']) for prop in self._nested_object_names: self.setdefault(prop, dict()) @@ -269,6 +276,20 @@ def indexes(self): """ return self.get('indexes') + def document_partition_url(self, partition_key): + """ + Retrieve the design document partition URL. + + :param str partition_key: Partition key. + :return: Design document partition URL. + :rtype: str + """ + return '/'.join(( + self._database.database_partition_url(partition_key), + '_design', + url_quote_plus(self['_id'][8:], safe='') + )) + def add_view(self, view_name, map_func, reduce_func=None, **kwargs): """ Appends a MapReduce view to the locally cached DesignDocument View diff --git a/src/cloudant/index.py b/src/cloudant/index.py index 40274ce1..051c004b 100644 --- a/src/cloudant/index.py +++ b/src/cloudant/index.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,18 +40,21 @@ class Index(object): Index. :param str design_document_id: Optional identifier of the design document. :param str name: Optional name of the index. + :param bool partitioned: Optional. Create as a partitioned index. Defaults + to ``False`` for both partitioned and non-partitioned databases. :param kwargs: Options used to construct the index definition for the purposes of index creation. For more details on valid options See :func:`~cloudant.database.CloudantDatabase.create_query_index`. """ - def __init__(self, database, design_document_id=None, name=None, **kwargs): + def __init__(self, database, design_document_id=None, name=None, partitioned=False, **kwargs): self._database = database self._r_session = self._database.r_session self._ddoc_id = design_document_id self._name = name self._type = JSON_INDEX_TYPE self._def = kwargs + self._partitioned = partitioned @property def index_url(self): @@ -100,6 +103,17 @@ def definition(self): """ return self._def + @property + def partitioned(self): + """ + Check if this index is partitioned. + + :return: ``True`` if index is partitioned, else ``False``. + :rtype: bool + """ + + return self._partitioned + def as_a_dict(self): """ Displays the index as a dictionary. This includes the design document @@ -114,6 +128,9 @@ def as_a_dict(self): 'def': self._def } + if self._partitioned: + index_dict['partitioned'] = True + return index_dict def create(self): @@ -137,6 +154,9 @@ def create(self): self._def_check() payload['index'] = self._def + if self._partitioned: + payload['partitioned'] = True + headers = {'Content-Type': 'application/json'} resp = self._r_session.post( self.index_url, diff --git a/src/cloudant/query.py b/src/cloudant/query.py index 791fd908..d6a0aebc 100644 --- a/src/cloudant/query.py +++ b/src/cloudant/query.py @@ -87,13 +87,18 @@ class Query(dict): :param str use_index: Identifies a specific index for the query to run against, rather than using the Cloudant Query algorithm which finds what it believes to be the best index. + :param str partition_key: Optional. Specify a query partition key. Defaults + to ``None`` resulting in global queries. """ def __init__(self, database, **kwargs): super(Query, self).__init__() self._database = database + self._partition_key = kwargs.pop('partition_key', None) self._r_session = self._database.r_session self._encoder = self._database.client.encoder + if kwargs.get('fields', True) is None: + del kwargs['fields'] # delete `None` fields kwarg if kwargs: super(Query, self).update(kwargs) self.result = QueryResult(self) @@ -105,7 +110,13 @@ def url(self): :returns: Query URL """ - return '/'.join((self._database.database_url, '_find')) + if self._partition_key: + base_url = self._database.database_partition_url( + self._partition_key) + else: + base_url = self._database.database_url + + return base_url + '/_find' def __call__(self, **kwargs): """ diff --git a/src/cloudant/view.py b/src/cloudant/view.py index 3a0e63fc..7b76f2a5 100644 --- a/src/cloudant/view.py +++ b/src/cloudant/view.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM. All rights reserved. +# Copyright (C) 2015, 2019 IBM. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -87,6 +87,8 @@ class View(dict): :param str view_name: Name used in part to identify the view. :param str map_func: Optional Javascript map function. :param str reduce_func: Optional Javascript reduce function. + :param str partition_key: Optional. Specify a view partition key. Defaults + to ``None`` resulting in global queries. """ def __init__( self, @@ -94,6 +96,7 @@ def __init__( view_name, map_func=None, reduce_func=None, + partition_key=None, **kwargs ): super(View, self).__init__() @@ -104,6 +107,7 @@ def __init__( self['map'] = codify(map_func) if reduce_func is not None: self['reduce'] = codify(reduce_func) + self._partition_key = partition_key self.update(kwargs) self.result = Result(self) @@ -167,8 +171,14 @@ def url(self): :returns: View URL """ + if self._partition_key: + base_url = self.design_doc.document_partition_url( + self._partition_key) + else: + base_url = self.design_doc.document_url + return '/'.join(( - self.design_doc.document_url, + base_url, '_view', self.view_name )) diff --git a/tests/unit/database_partition_tests.py b/tests/unit/database_partition_tests.py new file mode 100644 index 00000000..2c595f65 --- /dev/null +++ b/tests/unit/database_partition_tests.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# Copyright (C) 2019 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +_database_partition_tests_ +""" + +from cloudant.design_document import DesignDocument +from cloudant.index import Index, SpecialIndex + +from nose.plugins.attrib import attr + +from .unit_t_db_base import UnitTestDbBase + + +@attr(db=['cloudant']) +class DatabasePartitionTests(UnitTestDbBase): + + def setUp(self): + super(DatabasePartitionTests, self).setUp() + self.db_set_up(partitioned=True) + + def tearDown(self): + self.db_tear_down() + super(DatabasePartitionTests, self).tearDown() + + def test_is_partitioned_database(self): + self.assertTrue(self.db.metadata()['props']['partitioned']) + + def test_create_partitioned_design_document(self): + ddoc_id = 'empty_ddoc' + + ddoc = DesignDocument(self.db, ddoc_id, partitioned=True) + ddoc.save() + + r = self.db.r_session.get(ddoc.document_url) + r.raise_for_status() + + self.assertTrue(r.json()['options']['partitioned']) + + def test_partitioned_all_docs(self): + for partition_key in self.populate_db_with_partitioned_documents(5, 25): + docs = self.db.partitioned_all_docs(partition_key) + self.assertEquals(len(docs['rows']), 25) + + for doc in docs['rows']: + self.assertTrue(doc['id'].startswith(partition_key + ':')) + + def test_partition_metadata(self): + for partition_key in self.populate_db_with_partitioned_documents(5, 25): + meta = self.db.partition_metadata(partition_key) + self.assertEquals(meta['partition'], partition_key) + self.assertEquals(meta['doc_count'], 25) + + def test_partitioned_search(self): + ddoc = DesignDocument(self.db, 'partitioned_search', partitioned=True) + ddoc.add_search_index( + 'search1', + 'function(doc) { index("id", doc._id, {"store": true}); }' + ) + ddoc.save() + + for partition_key in self.populate_db_with_partitioned_documents(2, 10): + results = self.db.get_partitioned_search_result( + partition_key, ddoc['_id'], 'search1', query='*:*') + + i = 0 + for result in results['rows']: + print(result) + self.assertTrue(result['id'].startswith(partition_key + ':')) + i += 1 + self.assertEquals(i, 10) + + def test_get_partitioned_index(self): + index_name = 'test_partitioned_index' + + self.db.create_query_index(index_name=index_name, fields=['foo']) + + results = self.db.get_query_indexes() + self.assertEquals(len(results), 2) + + index_all_docs = results[0] + self.assertEquals(index_all_docs.name, '_all_docs') + self.assertEquals(type(index_all_docs), SpecialIndex) + self.assertFalse(index_all_docs.partitioned) + + index_partitioned = results[1] + self.assertEquals(index_partitioned.name, index_name) + self.assertEquals(type(index_partitioned), Index) + self.assertTrue(index_partitioned.partitioned) + + def test_partitioned_query(self): + self.db.create_query_index(fields=['foo']) + + for partition_key in self.populate_db_with_partitioned_documents(2, 10): + results = self.db.get_partitioned_query_result( + partition_key, selector={'foo': {'$eq': 'bar'}}) + + i = 0 + for result in results: + self.assertTrue(result['_id'].startswith(partition_key + ':')) + i += 1 + self.assertEquals(i, 10) + + def test_partitioned_view(self): + ddoc = DesignDocument(self.db, 'partitioned_view', partitioned=True) + ddoc.add_view('view1', 'function(doc) { emit(doc._id, 1); }') + ddoc.save() + + for partition_key in self.populate_db_with_partitioned_documents(2, 10): + results = self.db.get_partitioned_view_result( + partition_key, ddoc['_id'], 'view1') + + i = 0 + for result in results: + self.assertTrue( + result['id'].startswith(partition_key + ':')) + i += 1 + self.assertEquals(i, 10) diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index ad7eefd6..913d7b20 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -253,13 +253,14 @@ def tearDown(self): """ del self.client - def db_set_up(self): + def db_set_up(self, partitioned=False): """ Set up test attributes for Database tests """ self.client.connect() self.test_dbname = self.dbname() - self.db = self.client._DATABASE_CLASS(self.client, self.test_dbname) + self.db = self.client._DATABASE_CLASS( + self.client, self.test_dbname, partitioned=partitioned) self.db.create() def db_tear_down(self): @@ -282,6 +283,20 @@ def populate_db_with_documents(self, doc_count=100, **kwargs): ] return self.db.bulk_docs(docs) + def populate_db_with_partitioned_documents(self, key_count, docs_per_partition): + partition_keys = [uuid.uuid4().hex.upper()[:8] for _ in range(key_count)] + for partition_key in partition_keys: + docs = [] + for i in range(docs_per_partition): + docs.append({ + '_id': '{0}:doc{1}'.format(partition_key, i), + 'foo': 'bar' + }) + + self.db.bulk_docs(docs) + + return partition_keys + def create_views(self): """ Create a design document with views for use with tests. From bb7cdb524925e11ffadb1f4dddfe66a59bff9775 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 7 Feb 2019 11:31:57 +0000 Subject: [PATCH 125/185] Add partitioned databases documentation. --- docs/getting_started.rst | 147 +++++++++++++++++++++++++++++++++++---- 1 file changed, 134 insertions(+), 13 deletions(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 072fe061..9cd4b65e 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -1,6 +1,6 @@ -=============== +############### Getting started -=============== +############### Now it's time to begin doing some work with Cloudant and Python. For working code samples of any of the API's please go to our test suite. @@ -32,7 +32,7 @@ a HTTP connection or a response on all requests. A timeout can be set using the ``timeout`` argument when constructing a client. Connecting with a client -^^^^^^^^^^^^^^^^^^^^^^^^ +======================== .. code-block:: python @@ -144,7 +144,7 @@ Note: Idle connections within the pool may be terminated by the server, so will indefinitely meaning that this will not completely remove the overhead of creating new connections. Using library in app server environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +======================================= This library can be used in an app server, and the example below shows how to use ``client`` in a ``flask`` app server. @@ -190,7 +190,7 @@ existing database, or delete a database. The following examples assume a client connection has already been established. Creating a database -^^^^^^^^^^^^^^^^^^^ +=================== .. code-block:: python @@ -203,7 +203,7 @@ Creating a database print('SUCCESS!!') Opening a database -^^^^^^^^^^^^^^^^^^ +================== Opening an existing database is done by supplying the name of an existing database to the client. Since the ``Cloudant`` and ``CouchDB`` classes are @@ -216,13 +216,134 @@ sub-classes of ``dict``, this can be accomplished through standard Python my_database = client['my_database'] Deleting a database -^^^^^^^^^^^^^^^^^^^ +=================== .. code-block:: python # Delete a database using an initialized client client.delete_database('my_database') + +Partitioned Databases +===================== + +Partitioned databases introduce the ability for a user to create logical groups +of documents called partitions by providing a partition key with each document. + +.. warning:: Your Cloudant cluster must have the ``partitions`` feature enabled. + A full list of enabled features can be retrieved by calling the + client :func:`~cloudant.client.CouchDB.metadata` method. + +Creating a partitioned database +------------------------------- + +.. code-block:: python + + db = client.create_database('mydb', partitioned=True) + +Handling documents +------------------ + +The document ID contains both the partition key and document key in the form +``:`` where: + +- Partition Key *(string)*. Must be non-empty. Must not contain colons (as this + is the partition key delimiter) or begin with an underscore. +- Document Key *(string)*. Must be non-empty. Must not begin with an underscore. + +Be aware that ``_design`` documents and ``_local`` documents must not contain a +partition key as they are global definitions. + +**Create a document** + +.. code-block:: python + + partition_key = 'Year2' + document_key = 'julia30' + db.create_document({ + '_id': ':'.join((partition_key, document_key)), + 'name': 'Jules', + 'age': 6 + }) + +**Get a document** + +.. code-block:: python + + doc = db[':'.join((partition_key, document_key))] + +Creating design documents +------------------------- + +To define partitioned indexes you must set the ``partitioned=True`` optional +when constructing the new ``DesignDocument`` class. + +.. code-block:: python + + ddoc = DesignDocument(db, document_id='view', partitioned=True) + ddoc.add_view('myview','function(doc) { emit(doc.foo, doc.bar); }') + ddoc.save() + +Similarly, to define a partitioned Cloudant Query index you must set the +``partitioned=True`` optional. + +.. code-block:: python + + index = db.create_query_index( + design_document_id='query', + index_name='foo-index', + fields=['foo'], + partitioned=True + ) + index.create() + +Querying data +------------- + +A partition key can be specified when querying data so that results can be +constrained to a specific database partition. + +.. warning:: To run partitioned queries the database itself must be partitioned. + +**Query** + +.. code-block:: python + + results = self.db.get_partitioned_query_result( + partition_key, selector={'foo': {'$eq': 'bar'}}) + + for result in results: + ... + +See :func:`~cloudant.database.CouchDatabase.get_partitioned_query_result` for a +full list of supported parameters. + +**Search** + +.. code-block:: python + + results = self.db.get_partitioned_search_result( + partition_key, search_ddoc['_id'], 'search1', query='*:*') + + for result in results['rows']: + .... + +See :func:`~cloudant.database.CloudantDatabase.get_partitioned_search_result` +for a full list of supported parameters. + +**Views (MapReduce)** + +.. code-block:: python + + results = self.db.get_partitioned_view_result( + partition_key, view_ddoc['_id'], 'view1') + + for result in results: + .... + +See :func:`~cloudant.database.CouchDatabase.get_partitioned_view_result` for a +full list of supported parameters. + ********* Documents ********* @@ -235,7 +356,7 @@ create, read, update, and delete a document. These examples assume that either a CloudantDatabase or a CouchDatabase object already exists. Creating a document -^^^^^^^^^^^^^^^^^^^ +=================== .. code-block:: python @@ -255,7 +376,7 @@ Creating a document print('SUCCESS!!') Retrieving a document -^^^^^^^^^^^^^^^^^^^^^ +===================== Accessing a document from a database is done by supplying the document identifier of an existing document to either a ``CloudantDatabase`` or a @@ -271,7 +392,7 @@ classes are sub-classes of ``dict``, this is accomplished through standard print(my_document) Checking if a document exists -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +============================= You can check if a document exists in a database the same way you would check if a ``dict`` has a key-value pair by key. @@ -284,7 +405,7 @@ if a ``dict`` has a key-value pair by key. print('document with _id julia30 exists') Retrieve all documents -^^^^^^^^^^^^^^^^^^^^^^ +====================== You can also iterate over a ``CloudantDatabase`` or a ``CouchDatabase`` object to retrieve all documents in a database. @@ -296,7 +417,7 @@ to retrieve all documents in a database. print(document) Update a document -^^^^^^^^^^^^^^^^^ +================= .. code-block:: python @@ -312,7 +433,7 @@ Update a document my_document.save() Delete a document -^^^^^^^^^^^^^^^^^ +================= .. code-block:: python From 2135d6f3cf63218a907643bc42e247c313408d4e Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Tue, 19 Mar 2019 10:36:22 +0000 Subject: [PATCH 126/185] Use system Python installs in Jenkinsfile. --- Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1ab03a60..7501c07e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,8 +33,9 @@ def setupPythonAndTest(pythonVersion, testSuite) { withEnv(getEnvForSuite("${testSuite}")) { try { sh """ - virtualenv tmp -p /usr/local/lib/python${pythonVersion}/bin/${pythonVersion.startsWith('3') ? "python3" : "python"} + virtualenv tmp -p ${pythonVersion.startsWith('3') ? "python3" : "python"} . ./tmp/bin/activate + python --version pip install -r requirements.txt pip install -r test-requirements.txt ${'simplejson'.equals(testSuite) ? 'pip install simplejson' : ''} @@ -60,8 +61,8 @@ stage('Checkout'){ } stage('Test'){ - def py2 = '2.7.12' - def py3 = '3.5.2' + def py2 = '2' + def py3 = '3' def axes = [:] [py2, py3].each { version -> ['basic','cookie','iam'].each { auth -> From 3a535a87d2f85456e517c295cae1fb5b543534d3 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 27 Mar 2019 14:57:39 +0000 Subject: [PATCH 127/185] Set CHANGES.md to 100 char limit. --- CHANGES.md | 168 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 111 insertions(+), 57 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 63255bb2..e2c4dedf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,14 +1,13 @@ # Unreleased - [NEW] Added partitioned database support. -- [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and - queries' `bookmark`. -- [FIXED] Bug where document context manager performed remote save despite - uncaught exceptions being raised inside `with` block. +- [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and queries' + `bookmark`. +- [FIXED] Bug where document context manager performed remote save despite uncaught exceptions being + raised inside `with` block. - [FIXED] Fixed parameter type of `selector` in docstring. - [IMPROVED] Updated `Getting started` section with a `get_query_result` example. -- [FIXED] Removed internal `Document._document_id` property to allow a safe use of - dict's methods. +- [FIXED] Removed internal `Document._document_id` property to allow a safe use of dict's methods. # 2.11.0 (2019-01-21) @@ -21,9 +20,9 @@ # 2.10.1 (2018-11-16) -- [FIXED] Unexpected keyword argument errors when using the library with the - `simplejson` module present in the environment caused by `requests` preferentially - loading it over the system `json` module. +- [FIXED] Unexpected keyword argument errors when using the library with the `simplejson` module + present in the environment caused by `requests` preferentially loading it over the system `json` + module. # 2.10.0 (2018-09-19) @@ -31,14 +30,18 @@ - [NEW] Add new view parameters, `stable` and `update`, as keyword arguments to `get_view_result`. - [NEW] Allow arbitrary query parameters to be passed to custom changes filters. - [FIXED] Case where an exception was raised after successful retry when using `doc.update_field`. -- [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the `page_size` value. +- [FIXED] Removed unnecessary request when retrieving a Result collection that is less than the + `page_size` value. # 2.9.0 (2018-06-13) -- [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict `__contains__` and checking in the remote database. -- [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the `_index`/`_find` API is available in CouchDB 2.x. +- [NEW] Added functionality to test if a key is in a database as in `key in db`, overriding dict + `__contains__` and checking in the remote database. +- [NEW] Moved `create_query_index` and other query related methods to `CouchDatabase` as the + `_index`/`_find` API is available in CouchDB 2.x. - [NEW] Support IAM authentication in replication documents. -- [FIXED] Case where `Document` context manager would throw instead of creating a new document if no `_id` was provided. +- [FIXED] Case where `Document` context manager would throw instead of creating a new document if no + `_id` was provided. - [IMPROVED] Added support for IAM API key in `cloudant_bluemix` method. - [IMPROVED] Shortened length of client URLs by removing username and password. - [IMPROVED] Verified library operation on Python 3.6.3. @@ -49,22 +52,28 @@ # 2.8.0 (2018-02-15) -- [NEW] Added support for `/_search_disk_size` endpoint which retrieves disk size information for a specific search index. +- [NEW] Added support for `/_search_disk_size` endpoint which retrieves disk size information for a + specific search index. - [FIXED] Updated default IBM Cloud Identity and Access Management token URL. -- [REMOVED] Removed broken source and target parameters that constantly threw `AttributeError` when creating a replication document. +- [REMOVED] Removed broken source and target parameters that constantly threw `AttributeError` when + creating a replication document. # 2.7.0 (2017-10-31) -- [NEW] Added API for upcoming Bluemix Identity and Access Management support for Cloudant on Bluemix. Note: IAM API key support is not yet enabled in the service. +- [NEW] Added API for upcoming Bluemix Identity and Access Management support for Cloudant on + Bluemix. Note: IAM API key support is not yet enabled in the service. - [NEW] Added HTTP basic authentication support. - [NEW] Added `Result.all()` convenience method. -- [NEW] Allow `service_name` to be specified when instantiating from a Bluemix VCAP_SERVICES environment variable. +- [NEW] Allow `service_name` to be specified when instantiating from a Bluemix VCAP_SERVICES + environment variable. - [IMPROVED] Updated `posixpath.join` references to use `'/'.join` when concatenating URL parts. -- [IMPROVED] Updated documentation by replacing deprecated Cloudant links with the latest Bluemix links. +- [IMPROVED] Updated documentation by replacing deprecated Cloudant links with the latest Bluemix + links. # 2.6.0 (2017-08-10) -- [NEW] Added `Cloudant.bluemix()` class method to the Cloudant client allowing service credentials to be passed using the CloudFoundry VCAP_SERVICES environment variable. +- [NEW] Added `Cloudant.bluemix()` class method to the Cloudant client allowing service credentials + to be passed using the CloudFoundry VCAP_SERVICES environment variable. - [FIXED] Fixed client construction in `cloudant_bluemix` context manager. - [FIXED] Fixed validation for feed options to accept zero as a valid value. @@ -76,26 +85,33 @@ - [FIXED] Fixed Cloudant exception code 409 with 412 when creating a database that already exists. - [FIXED] Catch error if `throw_on_exists` flag is `False` for creating a document. - [FIXED] Fixed /_all_docs call where `keys` is an empty list. -- [FIXED] Issue where docs with IDs that sorted lower than 0 were not returned when iterating through _all_docs. +- [FIXED] Issue where docs with IDs that sorted lower than 0 were not returned when iterating + through _all_docs. # 2.4.0 (2017-02-14) -- [NEW] Added `timeout` option to the client constructor for setting a timeout on a HTTP connection or a response. -- [NEW] Added `cloudant_bluemix` method to the Cloudant client allowing service credentials to be passed using the CloudFoundry VCAP_SERVICES environment variable. -- [IMPROVED] Updated non-response related errors with additional status code and improved error message for easier debugging. - All non-response error are handled using either CloudantException or CloudantArgumentError. +- [NEW] Added `timeout` option to the client constructor for setting a timeout on a HTTP connection + or a response. +- [NEW] Added `cloudant_bluemix` method to the Cloudant client allowing service credentials to be + passed using the CloudFoundry VCAP_SERVICES environment variable. +- [IMPROVED] Updated non-response related errors with additional status code and improved error + message for easier debugging. All non-response error are handled using either CloudantException + or CloudantArgumentError. - [FIXED] Support `long` type argument when executing in Python 2. # 2.3.1 (2016-11-30) -- [FIXED] Resolved issue where generated UUIDs for replication documents would not be converted to strings. +- [FIXED] Resolved issue where generated UUIDs for replication documents would not be converted to + strings. - [FIXED] Resolved issue where CouchDatabase.infinite_changes() method can cause a stack overflow. # 2.3.0 (2016-11-02) - [FIXED] Resolved issue where the custom JSON encoder was at times not used when transforming data. -- [NEW] Added support for managing the database security document through the SecurityDocument class and CouchDatabase convenience method `get_security_document`. -- [NEW] Added `auto_renewal` option to the client constructor to handle the automatic renewal of an expired session cookie auth. +- [NEW] Added support for managing the database security document through the SecurityDocument class + and CouchDatabase convenience method `get_security_document`. +- [NEW] Added `auto_renewal` option to the client constructor to handle the automatic renewal of an + expired session cookie auth. # 2.2.0 (2016-10-20) @@ -117,12 +133,14 @@ - [NEW] Added `st_indexes` accessor property for Cloudant Geospatial indexes. - [NEW] Added support for DesignDocument `_info` and `_search_info` endpoints. - [NEW] Added `validate_doc_update` accessor property for update validators. -- [NEW] Added support for a custom `requests.HTTPAdapter` to be configured using an optional `adapter` arg e.g. - `Cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME, adapter=Replay429Adapter())`. +- [NEW] Added support for a custom `requests.HTTPAdapter` to be configured using an optional + `adapter` arg e.g. `Cloudant(USERNAME, PASSWORD, account=ACCOUNT_NAME, + adapter=Replay429Adapter())`. - [IMPROVED] Made the 429 response code backoff optional and configurable. To enable the backoff add - an `adapter` arg of a `Replay429Adapter` with the desired number of retries and initial backoff. To replicate - the 2.0.0 behaviour use: `adapter=Replay429Adapter(retries=10, initialBackoff=0.25)`. If `retries` or - `initialBackoff` are not specified they will default to 3 retries and a 0.25 s initial backoff. + an `adapter` arg of a `Replay429Adapter` with the desired number of retries and initial + backoff. To replicate the 2.0.0 behaviour use: `adapter=Replay429Adapter(retries=10, + initialBackoff=0.25)`. If `retries` or `initialBackoff` are not specified they will default to 3 + retries and a 0.25 s initial backoff. - [IMPROVED] Additional error reason details appended to HTTP response message errors. - [FIX] `415 Client Error: Unsupported Media Type` when using keys with `db.all_docs`. - [FIX] Allowed strings as well as lists for search `group_sort` arguments. @@ -133,36 +151,65 @@ # 2.0.2 (2016-06-02) -- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to python-cloudant.readthedocs.io. -- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the user-agent string. +- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to + python-cloudant.readthedocs.io. +- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the + user-agent string. - [FIX] Fixed readthedocs link in README.rst to resolve to documentation home page. # 2.0.1 (2016-06-02) -- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to python-cloudant.readthedocs.io. -- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the user-agent string. +- [IMPROVED] Updated documentation links from python-cloudant.readthedocs.org to + python-cloudant.readthedocs.io. +- [FIX] Fixed issue with Windows platform compatibility,replaced usage of os.uname for the + user-agent string. - [FIX] Fixed readthedocs link in README.rst to resolve to documentation home page. # 2.0.0 (2016-05-02) -- [BREAKING] Renamed modules account.py, errors.py, indexes.py, views.py, to client.py, error.py, index.py, and view.py. -- [BREAKING] Removed the `make_result` method from `View` and `Query` classes. If you need to make a query or view result, use `CloudantDatabase.get_query_result`, `CouchDatabase.get_view_result`, or the `View.custom_result` context manager. Additionally, the `Result` and `QueryResult` classes can be called directly to construct a result object. -- [BREAKING] Refactored the `SearchIndex` class to now be the `TextIndex` class. Also renamed the `CloudantDatabase` convenience methods of `get_all_indexes`, `create_index`, and `delete_index` as `get_query_indexes`, `create_query_index`, and `delete_query_index` respectively. These changes were made to clarify that the changed class and the changed methods were specific to query index processing only. -- [BREAKING] Replace "session" and "url" feed constructor arguments with "source" which can be either a client or a database object. Changes also made to the client `db_updates` method signature and the database `changes` method signature. -- [BREAKING] Fixed `CloudantDatabase.share_database` to accept all valid permission roles. Changed the method signature to accept roles as a list argument. -- [BREAKING] Removed credentials module from the API and moved it to the tests folder since the functionality is outside of the scope of this library but is still be useful in unit/integration tests. -- [IMPROVED] Changed the handling of queries using the keys argument to issue a http POST request instead of a http GET request so that the request is no longer bound by any URL length limitation. -- [IMPROVED] Added support for Result/QueryResult data access via index value and added validation logic to `Result.__getitem__()`. -- [IMPROVED] Updated feed functionality to process `_changes` and `_db_updates` with their supported options. Also added an infinite feed option. +- [BREAKING] Renamed modules account.py, errors.py, indexes.py, views.py, to client.py, error.py, + index.py, and view.py. +- [BREAKING] Removed the `make_result` method from `View` and `Query` classes. If you need to make + a query or view result, use `CloudantDatabase.get_query_result`, `CouchDatabase.get_view_result`, + or the `View.custom_result` context manager. Additionally, the `Result` and `QueryResult` classes + can be called directly to construct a result object. +- [BREAKING] Refactored the `SearchIndex` class to now be the `TextIndex` class. Also renamed the + `CloudantDatabase` convenience methods of `get_all_indexes`, `create_index`, and `delete_index` as + `get_query_indexes`, `create_query_index`, and `delete_query_index` respectively. These changes + were made to clarify that the changed class and the changed methods were specific to query index + processing only. +- [BREAKING] Replace "session" and "url" feed constructor arguments with "source" which can be + either a client or a database object. Changes also made to the client `db_updates` method + signature and the database `changes` method signature. +- [BREAKING] Fixed `CloudantDatabase.share_database` to accept all valid permission roles. Changed + the method signature to accept roles as a list argument. +- [BREAKING] Removed credentials module from the API and moved it to the tests folder since the + functionality is outside of the scope of this library but is still be useful in unit/integration + tests. +- [IMPROVED] Changed the handling of queries using the keys argument to issue a http POST request + instead of a http GET request so that the request is no longer bound by any URL length limitation. +- [IMPROVED] Added support for Result/QueryResult data access via index value and added validation + logic to `Result.__getitem__()`. +- [IMPROVED] Updated feed functionality to process `_changes` and `_db_updates` with their supported + options. Also added an infinite feed option. - [NEW] Handled HTTP status code `429 Too Many Requests` with blocking backoff and retries. -- [NEW] Added support for CouchDB Admin Party mode. This library can now be used with CouchDB instances where everyone is Admin. -- [FIX] Fixed `Document.get_attachment` method to successfully create text and binary files based on http response Content-Type. The method also returns text, binary, and json content based on http response Content-Type. -- [FIX] Added validation to `Cloudant.bill`, `Cloudant.volume_usage`, and `Cloudant.requests_usage` methods to ensure that a valid year/month combination or neither are used as arguments. +- [NEW] Added support for CouchDB Admin Party mode. This library can now be used with CouchDB + instances where everyone is Admin. +- [FIX] Fixed `Document.get_attachment` method to successfully create text and binary files based on + http response Content-Type. The method also returns text, binary, and json content based on http + response Content-Type. +- [FIX] Added validation to `Cloudant.bill`, `Cloudant.volume_usage`, and `Cloudant.requests_usage` + methods to ensure that a valid year/month combination or neither are used as arguments. - [FIX] Fixed the handling of empty views in the DesignDocument. -- [FIX] The `CouchDatabase.create_document` method now handles documents and design documents correctly. If the document created is a design document then the locally cached object will be a DesignDocument otherwise it will be a Document. -- [CHANGE] Moved internal `Code` class, functions like `python_to_couch` and `type_or_none`, and constants into a _common_util module. -- [CHANGE] Updated User-Agent header format to be `python-cloudant//Python///`. -- [CHANGE] Completed the addition of unit tests that target a database server. Removed all mocked unit tests. +- [FIX] The `CouchDatabase.create_document` method now handles documents and design documents + correctly. If the document created is a design document then the locally cached object will be a + DesignDocument otherwise it will be a Document. +- [CHANGE] Moved internal `Code` class, functions like `python_to_couch` and `type_or_none`, and + constants into a _common_util module. +- [CHANGE] Updated User-Agent header format to be `python-cloudant//Python///`. +- [CHANGE] Completed the addition of unit tests that target a database server. Removed all mocked + unit tests. # 2.0.0b2 (2016-02-24) @@ -192,10 +239,14 @@ - [NEW] Added unit tests targeting CouchDB and Cloudant databases. -- [FIX] Fixed bug in database create validation check to work if response code is either 201 (created) or 202 (accepted). +- [FIX] Fixed bug in database create validation check to work if response code is either 201 + (created) or 202 (accepted). - [FIX] Fixed database iterator infinite loop problem and to now yield a Document object. -- [BREAKING] Removed previous bulk_docs method from the CouchDatabase class and renamed the previous bulk_insert method as bulk_docs. The previous bulk_docs functionality is available through the all_docs method using the "keys" parameter. -- [FIX] Made missing_revisions, revisions_diff, get_revision_limit, set_revision_limit, and view_cleanup API methods available for CouchDB as well as Cloudant. +- [BREAKING] Removed previous bulk_docs method from the CouchDatabase class and renamed the previous + bulk_insert method as bulk_docs. The previous bulk_docs functionality is available through the + all_docs method using the "keys" parameter. +- [FIX] Made missing_revisions, revisions_diff, get_revision_limit, set_revision_limit, and + view_cleanup API methods available for CouchDB as well as Cloudant. - [BREAKING] Moved the db_update method to the account module. - [FIX] Fixed missing_revisions to key on 'missing_revs'. - [FIX] Fixed set_revision_limit to encode the request data payload correctly. @@ -203,10 +254,13 @@ - [BREAKING] Renamed Document `field_append` method to `list_field_append`. - [BREAKING] Renamed Document `field_remove` method to `list_field_remove`. - [BREAKING] Renamed Document `field_replace` method to `field_set`. -- [FIX] The Document local dictionary `_id` key is now synched with `_document_id` private attribute. +- [FIX] The Document local dictionary `_id` key is now synched with `_document_id` private + attribute. - [FIX] The Document local dictionary is now refreshed after an add/update/delete of an attachment. - [FIX] The Document `fetch()` method now refreshes the Document local dictionary content correctly. -- [BREAKING] Replace the ReplicatorDatabase class with the Replicator class. A Replicator object has a database attribute that represents the _replicator database. This allows the Replicator to work for both a CloudantDatabase and a CouchDatabase. +- [BREAKING] Replace the ReplicatorDatabase class with the Replicator class. A Replicator object + has a database attribute that represents the _replicator database. This allows the Replicator to + work for both a CloudantDatabase and a CouchDatabase. - [REMOVED] Removed "not implemented" methods from the DesignDocument. - [FIX] Add implicit "_design/" prefix for DesignDocument document ids. From a0b92b3b5d02897cf417a74fb1a11c6dc8698555 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Wed, 27 Mar 2019 15:10:24 +0000 Subject: [PATCH 128/185] Prepare version 2.12.0 release. --- CHANGES.md | 10 ++++++---- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e2c4dedf..d2217438 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,13 +1,15 @@ -# Unreleased +# 2.12.0 (2019-03-28) - [NEW] Added partitioned database support. -- [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and queries' - `bookmark`. - [FIXED] Bug where document context manager performed remote save despite uncaught exceptions being raised inside `with` block. - [FIXED] Fixed parameter type of `selector` in docstring. -- [IMPROVED] Updated `Getting started` section with a `get_query_result` example. - [FIXED] Removed internal `Document._document_id` property to allow a safe use of dict's methods. +- [IMPROVED] Performance of `Result` iteration by releasing result objects immediately after they + are returned to the client. +- [IMPROVED] Updated `Getting started` section with a `get_query_result` example. +- [IMPROVED] Updated `Result` iteration by paginating with views' `startkey` and queries' + `bookmark`. # 2.11.0 (2019-01-21) diff --git a/VERSION b/VERSION index 57f9a3c9..d8b69897 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.11.1-SNAPSHOT +2.12.0 diff --git a/docs/conf.py b/docs/conf.py index 3df55b2e..c400d37b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.11.1-SNAPSHOT' +version = '2.12.0' # The full version, including alpha/beta/rc tags. -release = '2.11.1-SNAPSHOT' +release = '2.12.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 89d142f9..93752011 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.11.1-SNAPSHOT' +__version__ = '2.12.0' # pylint: disable=wrong-import-position import contextlib From 77d37f4c3b7e8cab181b2ec675b401718a408d5b Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 28 Mar 2019 10:41:43 +0000 Subject: [PATCH 129/185] Update version to 2.12.1-SNAPSHOT. --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index d8b69897..91d3a8d4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.12.0 +2.12.1-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index c400d37b..764be732 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.12.0' +version = '2.12.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.12.0' +release = '2.12.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 93752011..677df2ed 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.12.0' +__version__ = '2.12.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From 83b0a6557a42061dfc4150f062ae0a27e42b195a Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Thu, 28 Mar 2019 11:12:11 +0000 Subject: [PATCH 130/185] Include README in package metadata. --- setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup.py b/setup.py index cac89fd5..e02192cd 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ """ +from os import path from setuptools import setup, find_packages requirements_file = open('requirements.txt') @@ -28,8 +29,14 @@ version = version_file.read().strip() version_file.close() +this_directory = path.abspath(path.dirname(__file__)) +with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + setup_args = { 'description': 'Cloudant / CouchDB Client Library', + 'long_description': long_description, + 'long_description_content_type': 'text/markdown', 'include_package_data': True, 'install_requires': requirements, 'name': 'cloudant', From e0ba190f6ba07fe3522a668747128214ad573c7e Mon Sep 17 00:00:00 2001 From: Felix Date: Wed, 24 Apr 2019 20:44:33 +0200 Subject: [PATCH 131/185] Update documentation to match function signature (#445) s/get_list_result/get_list_function_result/ --- src/cloudant/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloudant/database.py b/src/cloudant/database.py index baf2df67..17978468 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -932,7 +932,7 @@ def get_list_function_result(self, ddoc_id, list_name, view_name, **kwargs): # Assuming that 'view001' exists as part of the # 'ddoc001' design document in the remote database... # Retrieve documents where the list function is 'list1' - resp = db.get_list_result('ddoc001', 'list1', 'view001', limit=10) + resp = db.get_list_function_result('ddoc001', 'list1', 'view001', limit=10) for row in resp['rows']: # Process data (in text format). From aef879cd71ac12350c0b7391b196499a26bc6348 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Mon, 20 May 2019 11:25:00 -0400 Subject: [PATCH 132/185] Updated 'create_database' to raise exception when database name is invalid (#447) --- CHANGES.md | 4 +++ src/cloudant/client.py | 1 + src/cloudant/error.py | 8 +++++- tests/unit/client_tests.py | 53 +++++++++++++++++++++++++++++++++++++- 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d2217438..3ba7cc7f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# Unreleased + +- [FIXED] Correctly raise exceptions from `create_database` calls. + # 2.12.0 (2019-03-28) - [NEW] Added partitioned database support. diff --git a/src/cloudant/client.py b/src/cloudant/client.py index b1993530..f09d3a69 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -290,6 +290,7 @@ def create_database(self, dbname, partitioned=False, **kwargs): except CloudantDatabaseException as ex: if ex.status_code == 412: raise CloudantClientException(412, dbname) + raise ex super(CouchDB, self).__setitem__(dbname, new_db) return new_db diff --git a/src/cloudant/error.py b/src/cloudant/error.py index a35345f2..348e1bf6 100644 --- a/src/cloudant/error.py +++ b/src/cloudant/error.py @@ -107,7 +107,13 @@ class CloudantDatabaseException(CloudantException): """ def __init__(self, code=100, *args): try: - msg = DATABASE[code].format(*args) + if code in DATABASE: + msg = DATABASE[code].format(*args) + elif isinstance(code, int): + msg = ' '.join(args) + else: + code = 100 + msg = DATABASE[code] except (KeyError, IndexError): code = 100 msg = DATABASE[code] diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 0a544a15..bfced219 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -34,7 +34,8 @@ from cloudant._client_session import BasicSession, CookieSession from cloudant.client import Cloudant, CouchDB from cloudant.database import CloudantDatabase -from cloudant.error import CloudantArgumentError, CloudantClientException +from cloudant.error import (CloudantArgumentError, CloudantClientException, + CloudantDatabaseException) from cloudant.feed import Feed, InfiniteFeed from nose.plugins.attrib import attr from requests import ConnectTimeout, HTTPError @@ -398,6 +399,56 @@ def test_create_existing_database(self): self.client.delete_database(dbname) self.client.disconnect() + def test_create_invalid_database_name(self): + """ + Test creation of database with an invalid name + """ + dbname = 'invalidDbName_' + self.client.connect() + with self.assertRaises(CloudantDatabaseException) as cm: + self.client.create_database(dbname) + self.assertEqual(cm.exception.status_code, 400) + self.client.disconnect() + + @skip_if_not_cookie_auth + @mock.patch('cloudant._client_session.Session.request') + def test_create_with_server_error(self, m_req): + """ + Test creation of database with a server error + """ + dbname = self.dbname() + # mock 200 for authentication + m_response_ok = mock.MagicMock() + type(m_response_ok).status_code = mock.PropertyMock(return_value=200) + + # mock 404 for head request when verifying if database exists + m_response_bad = mock.MagicMock() + type(m_response_bad).status_code = mock.PropertyMock(return_value=404) + + # mock 500 when trying to create the database + m_resp_service_error = mock.MagicMock() + type(m_resp_service_error).status_code = mock.PropertyMock( + return_value=500) + type(m_resp_service_error).text = mock.PropertyMock( + return_value='Internal Server Error') + + m_req.side_effect = [m_response_ok, m_response_bad, m_resp_service_error] + + self.client.connect() + with self.assertRaises(CloudantDatabaseException) as cm: + self.client.create_database(dbname) + + self.assertEqual(cm.exception.status_code, 500) + + self.assertEquals(m_req.call_count, 3) + m_req.assert_called_with( + 'PUT', + '/'.join([self.url, dbname]), + data=None, + params={'partitioned': 'false'}, + timeout=(30, 300) + ) + def test_delete_non_existing_database(self): """ Test deletion of non-existing database From 00abad732d01364e2c4e2960afb706214d8b50fe Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 22 May 2019 16:12:31 +0100 Subject: [PATCH 133/185] Added skip_if_iam to some tests that rely on a legacy password Some of the bluemix initialization tests make a connection to the service to test the init worked correctly. They use a username and password combination so can't work with an IAM API key without additional switching. While they pass in cases where both legacy creds and IAM creds are supplied, if only IAM creds are supplied they fail so should be disabled for IAM. --- tests/unit/client_tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index bfced219..f3ac4e76 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,7 +40,7 @@ from nose.plugins.attrib import attr from requests import ConnectTimeout, HTTPError -from .unit_t_db_base import skip_if_not_cookie_auth, UnitTestDbBase +from .unit_t_db_base import skip_if_iam, skip_if_not_cookie_auth, UnitTestDbBase from .. import bytes_, str_ @@ -782,6 +782,7 @@ def test_cloudant_bluemix_context_helper_raise_error_for_missing_iam_and_creds(s str(err) ) + @skip_if_iam def test_cloudant_bluemix_dedicated_context_helper(self): """ Test that the cloudant_bluemix context helper works as expected when @@ -888,6 +889,7 @@ def test_bluemix_constructor_with_iam(self): finally: c.disconnect() + @skip_if_iam def test_bluemix_constructor_specify_instance_name(self): """ Test instantiating a client object using a VCAP_SERVICES environment From f2c12984c325124d72cbdf5154a33b32538da941 Mon Sep 17 00:00:00 2001 From: tretinha Date: Wed, 9 Oct 2019 10:00:24 -0300 Subject: [PATCH 134/185] Fix "DeprecationWarning" from "collections" (#451) DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working. --- src/cloudant/_common_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index f7885f49..d47beb56 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -19,7 +19,7 @@ import sys import platform -from collections import Sequence +from collections.abc import Sequence import json from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_ From fec158e71b9d49fdae74cd1e59e2f74ad9186cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Tue, 15 Oct 2019 17:39:54 +0200 Subject: [PATCH 135/185] Fix broken Python 2 build (#453) f2c1298 accidentally broke the build as collections.abc is only available in Python 3.x. This commit adds a workaround that enables the import that works both on 2 and 3. --- src/cloudant/_common_util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index d47beb56..6e765c1d 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -19,12 +19,16 @@ import sys import platform -from collections.abc import Sequence import json from ._2to3 import LONGTYPE, STRTYPE, NONETYPE, UNITYPE, iteritems_ from .error import CloudantArgumentError, CloudantException, CloudantClientException +try: + from collections.abc import Sequence +except ImportError: + from collections import Sequence + # Library Constants USER_AGENT = '/'.join([ From 026f615ff389bbf99a562becd56df7c8d950189f Mon Sep 17 00:00:00 2001 From: James Mackenzie Date: Tue, 11 Feb 2020 14:13:41 +0000 Subject: [PATCH 136/185] Fixup the DesignDocumentTests::test_get_info and CloudantIndexExceptionTests::test_index_usage_via_query tests for couch3 --- tests/unit/design_document_tests.py | 5 +++-- tests/unit/index_tests.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index c986e769..35f46714 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -776,7 +776,8 @@ def test_get_info(self): info = ddoc_remote.info() # Remove variable fields to make equality easier to check info['view_index'].pop('signature') - info['view_index'].pop('disk_size') + if 'disk_size' in info['view_index']: + info['view_index'].pop('disk_size') # Remove Cloudant/Couch 2 fields if present to allow test to pass on Couch 1.6 if 'sizes' in info['view_index']: info['view_index'].pop('sizes') diff --git a/tests/unit/index_tests.py b/tests/unit/index_tests.py index 59a952d6..4361644f 100644 --- a/tests/unit/index_tests.py +++ b/tests/unit/index_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -390,7 +390,7 @@ def test_index_usage_via_query(self): self.populate_db_with_documents(100) result = self.db.get_query_result(fields=['name', 'age'], selector={'age': {'$eq': 6}}, raw_result=True) - self.assertTrue(str(result['warning']).startswith("no matching index found")) + self.assertTrue(str(result['warning']).lower().startswith("no matching index found")) @attr(db='cloudant') class TextIndexTests(UnitTestDbBase): From ae61f6a063f36f2b5cd1ad6341bd10e00c0ce4af Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 14 Feb 2020 09:34:39 +0000 Subject: [PATCH 137/185] Trigger search index builds before asserting metadata --- tests/unit/design_document_tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index c986e769..0bd27121 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -827,6 +827,9 @@ def test_get_search_info(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() + # Make a request to the search index to ensure it is built + self.db.get_search_result('_design/ddoc001', 'search001', query='name:julia*') + search_info = ddoc_remote.search_info('search001') # Check the search index name self.assertEqual(search_info['name'], '_design/ddoc001/search001', 'The search index name should be correct.') @@ -856,7 +859,8 @@ def test_get_search_disk_size(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() - ddoc_remote.search_info('search001') # trigger index build + # Make a request to the search index to ensure it is built + self.db.get_search_result('_design/ddoc001', 'search001', query='name:julia*') search_disk_size = ddoc_remote.search_disk_size('search001') From ab92f5e0b48238e549aa08b2fd3bb9a8d7e75c30 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 14 Feb 2020 11:57:34 +0000 Subject: [PATCH 138/185] Removed data_size from view info assertion The data_size field is no longer present in CouchDB 3.x so remove it from the assertion for compatibility. --- tests/unit/design_document_tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index f710e9e5..7ec3c10e 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -778,6 +778,8 @@ def test_get_info(self): info['view_index'].pop('signature') if 'disk_size' in info['view_index']: info['view_index'].pop('disk_size') + if 'data_size' in info['view_index']: + info['view_index'].pop('data_size') # Remove Cloudant/Couch 2 fields if present to allow test to pass on Couch 1.6 if 'sizes' in info['view_index']: info['view_index'].pop('sizes') @@ -790,8 +792,7 @@ def test_get_info(self): {'view_index': {'update_seq': 0, 'waiting_clients': 0, 'language': 'javascript', 'purge_seq': 0, 'compact_running': False, - 'waiting_commit': False, 'updater_running': False, - 'data_size': 0 + 'waiting_commit': False, 'updater_running': False }, 'name': name }) From 924c2097f2e68f7d31e553a123015ac249e524cb Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 19 Feb 2020 09:45:00 +0000 Subject: [PATCH 139/185] Extend test read timeout value --- tests/unit/replicator_tests.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index 610d3588..28d78fab 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -220,10 +220,13 @@ def test_create_replication(self): def test_timeout_in_create_replication(self): """ Test that a read timeout exception is thrown when creating a - replicator with a timeout value of 500 ms. + replicator with a read timeout value of 5 s. """ - # Setup client with a timeout - self.set_up_client(auto_connect=True, timeout=.5) + # Setup client with a read timeout (but the standard connect timeout) + # Note that this timeout applies to all connections from this client + # setting it too short can cause intermittent failures when responses + # are not quick enough. Setting it too long makes the test take longer. + self.set_up_client(auto_connect=True, timeout=(30,5)) self.db = self.client[self.test_target_dbname] self.target_db = self.client[self.test_dbname] # Construct a replicator with the updated client From 058b96762d63f20b3e018ed35aff2070ec228436 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 19 Feb 2020 11:12:37 +0000 Subject: [PATCH 140/185] Updated test matrix version Python 2 is EOL, removed from matrix. Updated CouchDB patch versions. --- .travis.yml | 11 +++++------ Jenkinsfile | 3 +-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a88780e..44abb089 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,13 @@ sudo: required language: python python: - - "2.7" - - "3.6" + - "3.8" env: - - ADMIN_PARTY=true COUCHDB_VERSION=2.1.1 - - ADMIN_PARTY=false COUCHDB_VERSION=2.1.1 - - ADMIN_PARTY=true COUCHDB_VERSION=1.7.1 - - ADMIN_PARTY=false COUCHDB_VERSION=1.7.1 + - ADMIN_PARTY=true COUCHDB_VERSION=2.3.1 + - ADMIN_PARTY=false COUCHDB_VERSION=2.3.1 + - ADMIN_PARTY=true COUCHDB_VERSION=1.7.2 + - ADMIN_PARTY=false COUCHDB_VERSION=1.7.2 services: - docker diff --git a/Jenkinsfile b/Jenkinsfile index 7501c07e..aa2f2c06 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -61,10 +61,9 @@ stage('Checkout'){ } stage('Test'){ - def py2 = '2' def py3 = '3' def axes = [:] - [py2, py3].each { version -> + [py3].each { version -> ['basic','cookie','iam'].each { auth -> axes.put("Python${version}-${auth}", {setupPythonAndTest(version, auth)}) } From 47b8a5ab4f05f036ca97e6e5019d4634b230e094 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 19 Feb 2020 12:00:00 +0000 Subject: [PATCH 141/185] Corrected syntax to eliminate warning in 3.8 --- .travis.yml | 2 +- Jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 44abb089..b49557a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ before_script: # command to run tests script: - pylint ./src/cloudant - - nosetests -A 'not db or ((db is "couch" or "couch" in db) and (not couchapi or couchapi <='${COUCHDB_VERSION:0:1}'))' -w ./tests/unit + - nosetests -A 'not db or ((db == "couch" or "couch" in db) and (not couchapi or couchapi <='${COUCHDB_VERSION:0:1}'))' -w ./tests/unit notifications: email: false diff --git a/Jenkinsfile b/Jenkinsfile index aa2f2c06..ddf21dd4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,7 @@ def setupPythonAndTest(pythonVersion, testSuite) { pip install -r test-requirements.txt ${'simplejson'.equals(testSuite) ? 'pip install simplejson' : ''} pylint ./src/cloudant - nosetests -A 'not db or (db is "cloudant" or "cloudant" in db)' -w ./tests/unit --with-xunit + nosetests -A 'not db or (db == "cloudant" or "cloudant" in db)' -w ./tests/unit --with-xunit """ } finally { // Load the test results From d84a0ea12d07688f0d6391616869b89792a11a1f Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 19 Feb 2020 14:43:11 +0000 Subject: [PATCH 142/185] Updated CouchDB version check in tests --- tests/unit/unit_t_db_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/unit_t_db_base.py b/tests/unit/unit_t_db_base.py index 913d7b20..2067af0c 100644 --- a/tests/unit/unit_t_db_base.py +++ b/tests/unit/unit_t_db_base.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -109,7 +109,7 @@ def setUpClass(cls): if os.environ.get('DB_USER') is None: # Get couchdb docker node name - if os.environ.get('COUCHDB_VERSION') == '2.1.1': + if os.environ.get('COUCHDB_VERSION') == '2.3.1': os.environ['NODENAME'] = requests.get( '{0}/_membership'.format(os.environ['DB_URL'])).json()['all_nodes'][0] os.environ['DB_USER_CREATED'] = '1' @@ -117,7 +117,7 @@ def setUpClass(cls): unicode_(uuid.uuid4()) ) os.environ['DB_PASSWORD'] = 'password' - if os.environ.get('COUCHDB_VERSION') == '2.1.1': + if os.environ.get('COUCHDB_VERSION') == '2.3.1': resp = requests.put( '{0}/_node/{1}/_config/admins/{2}'.format( os.environ['DB_URL'], @@ -143,7 +143,7 @@ def tearDownClass(cls): """ if (os.environ.get('RUN_CLOUDANT_TESTS') is None and os.environ.get('DB_USER_CREATED') is not None): - if os.environ.get('COUCHDB_VERSION') == '2.1.1': + if os.environ.get('COUCHDB_VERSION') == '2.3.1': resp = requests.delete( '{0}://{1}:{2}@{3}/_node/{4}/_config/admins/{5}'.format( os.environ['DB_URL'].split('://', 1)[0], From 8331efae8dc7f432267c7f7b96062af8a4cd5a59 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Wed, 19 Feb 2020 15:21:05 +0000 Subject: [PATCH 143/185] Fixed invalid DB name test The HEAD on an invalid DB name that doesn't exist returns a 400 in 1.7.2 but a 404 in 1.7.1 and 2.x --- tests/unit/client_tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index f3ac4e76..2726de33 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -405,9 +405,10 @@ def test_create_invalid_database_name(self): """ dbname = 'invalidDbName_' self.client.connect() - with self.assertRaises(CloudantDatabaseException) as cm: + with self.assertRaises((CloudantDatabaseException, HTTPError)) as cm: self.client.create_database(dbname) - self.assertEqual(cm.exception.status_code, 400) + code = cm.exception.status_code if hasattr(cm.exception, 'status_code') else cm.exception.response.status_code + self.assertEqual(code, 400) self.client.disconnect() @skip_if_not_cookie_auth From 65fcb81fadecdb27e3127b9ab2d9c9cf38cc92ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Tue, 14 Apr 2020 14:03:39 +0200 Subject: [PATCH 144/185] Updated CHANGES.md for 2.13.0 release --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3ba7cc7f..0a09653b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ -# Unreleased +# 2.13.0 (2020-04-16) - [FIXED] Correctly raise exceptions from `create_database` calls. +- [FIXED] Fix `DeprecationWarning` from `collections`. # 2.12.0 (2019-03-28) From 28cc072bc261b48716dcee1f091944258aecdf89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Tue, 14 Apr 2020 14:04:02 +0200 Subject: [PATCH 145/185] Updated version to 2.13.0 --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 91d3a8d4..fb2c0766 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.12.1-SNAPSHOT +2.13.0 diff --git a/docs/conf.py b/docs/conf.py index 764be732..beb4f08b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.12.1-SNAPSHOT' +version = '2.13.0' # The full version, including alpha/beta/rc tags. -release = '2.12.1-SNAPSHOT' +release = '2.13.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 677df2ed..0b8abc5f 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.12.1-SNAPSHOT' +__version__ = '2.13.0' # pylint: disable=wrong-import-position import contextlib From a45d21949baa9d753d899ce926ca68a3d1fd5371 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 16 Apr 2020 16:50:56 +0100 Subject: [PATCH 146/185] Updated version to 2.13.1-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index fb2c0766..ca3389d9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.13.0 +2.13.1-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index beb4f08b..ada059aa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.13.0' +version = '2.13.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.13.0' +release = '2.13.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 0b8abc5f..0b162070 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.13.0' +__version__ = '2.13.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From 06735afacadf8773517cbeaa9843dfa129c3b5da Mon Sep 17 00:00:00 2001 From: Dipjyoti Bisharad Date: Fri, 8 May 2020 20:19:30 +0530 Subject: [PATCH 147/185] Fixed design document creation when partitioned parameter is false (#467) Set default value for 'partitioned' parameter to false when creating a design document. Co-authored-by: dipjyoti bisharad #466 --- CHANGES.md | 4 ++++ src/cloudant/design_document.py | 2 ++ tests/unit/database_partition_tests.py | 11 +++++++++ tests/unit/database_tests.py | 2 +- tests/unit/design_document_tests.py | 31 ++++++++++++++++---------- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0a09653b..9e7155db 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# UNRELEASED + +- [FIXED] Set default value for `partitioned` parameter to false when creating a design document. + # 2.13.0 (2020-04-16) - [FIXED] Correctly raise exceptions from `create_database` calls. diff --git a/src/cloudant/design_document.py b/src/cloudant/design_document.py index 34594b95..d3ea387a 100644 --- a/src/cloudant/design_document.py +++ b/src/cloudant/design_document.py @@ -50,6 +50,8 @@ def __init__(self, database, document_id=None, partitioned=False): if partitioned: self.setdefault('options', {'partitioned': True}) + else: + self.setdefault('options', {'partitioned': False}) self._nested_object_names = frozenset(['views', 'indexes', 'lists', 'shows']) for prop in self._nested_object_names: diff --git a/tests/unit/database_partition_tests.py b/tests/unit/database_partition_tests.py index 2c595f65..52b76e0d 100644 --- a/tests/unit/database_partition_tests.py +++ b/tests/unit/database_partition_tests.py @@ -48,6 +48,17 @@ def test_create_partitioned_design_document(self): r.raise_for_status() self.assertTrue(r.json()['options']['partitioned']) + + def test_create_non_partitioned_design_document(self): + ddoc_id = 'empty_ddoc' + + ddoc = DesignDocument(self.db, ddoc_id, partitioned=False) + ddoc.save() + + r = self.db.r_session.get(ddoc.document_url) + r.raise_for_status() + + self.assertFalse(r.json()['options']['partitioned']) def test_partitioned_all_docs(self): for partition_key in self.populate_db_with_partitioned_documents(5, 25): diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index d57ba654..f054dd62 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -364,7 +364,7 @@ def test_retrieve_design_document(self): """ # Get an empty design document object that does not exist remotely local_ddoc = self.db.get_design_document('_design/ddoc01') - self.assertEqual(local_ddoc, {'_id': '_design/ddoc01', 'indexes': {}, + self.assertEqual(local_ddoc, {'_id': '_design/ddoc01', 'indexes': {}, 'options': {'partitioned': False}, 'views': {}, 'lists': {}, 'shows': {}}) # Add the design document to the database map_func = 'function(doc) {\n emit(doc._id, 1); \n}' diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 7ec3c10e..68a40505 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -353,6 +353,7 @@ def test_fetch_map_reduce(self): self.assertEqual(ddoc_remote, { '_id': '_design/ddoc001', '_rev': ddoc['_rev'], + 'options': {'partitioned': False}, 'lists': {}, 'shows': {}, 'indexes': {}, @@ -415,7 +416,7 @@ def test_fetch_no_views(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() self.assertEqual(set(ddoc_remote.keys()), - {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + {'_id', '_rev', 'indexes', 'views', 'options', 'lists', 'shows'}) self.assertEqual(ddoc_remote['_id'], '_design/ddoc001') self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote['_rev'], ddoc['_rev']) @@ -432,6 +433,7 @@ def test_fetch_query_views(self): data = { '_id': '_design/ddoc001', 'indexes': {}, + 'options': {'partitioned': False}, 'lists': {}, 'shows': {}, 'language': 'query', @@ -463,6 +465,7 @@ def test_fetch_text_indexes(self): data = { '_id': '_design/ddoc001', 'language': 'query', + 'options': {'partitioned': False}, 'lists': {}, 'shows': {}, 'indexes': {'index001': @@ -499,6 +502,7 @@ def test_fetch_text_indexes_and_query_views(self): 'language': 'query', 'lists': {}, 'shows': {}, + 'options': {'partitioned': False}, 'views': { 'view001': {'map': {'fields': {'name': 'asc', 'age': 'asc'}}, 'reduce': '_count', @@ -687,7 +691,7 @@ def test_save_with_no_views(self): ddoc.save() # Ensure that locally cached DesignDocument contains an # empty views dict. - self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'indexes', 'options', 'views', 'lists', 'shows'}) self.assertEqual(ddoc['_id'], '_design/ddoc001') self.assertTrue(ddoc['_rev'].startswith('1-')) self.assertEqual(ddoc.views, {}) @@ -695,7 +699,7 @@ def test_save_with_no_views(self): # include a views sub-document. resp = self.client.r_session.get(ddoc.document_url) raw_ddoc = response_to_json_dict(resp) - self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) + self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev','options'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1076,6 +1080,7 @@ def test_fetch_search_index(self): self.assertEqual(ddoc_remote, { '_id': '_design/ddoc001', '_rev': ddoc['_rev'], + 'options': {'partitioned': False}, 'indexes': { 'search001': {'index': search_index}, 'search002': {'index': search_index, 'analyzer': 'simple'}, @@ -1100,7 +1105,7 @@ def test_fetch_no_search_index(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() self.assertEqual(set(ddoc_remote.keys()), - {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + {'_id', '_rev', 'indexes', 'options', 'views', 'lists', 'shows'}) self.assertEqual(ddoc_remote['_id'], '_design/ddoc001') self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote['_rev'], ddoc['_rev']) @@ -1183,14 +1188,14 @@ def test_save_with_no_search_indexes(self): ddoc.save() # Ensure that locally cached DesignDocument contains an # empty search indexes and views dict. - self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'indexes','options', 'views', 'lists', 'shows'}) self.assertEqual(ddoc['_id'], '_design/ddoc001') self.assertTrue(ddoc['_rev'].startswith('1-')) # Ensure that remotely saved design document does not # include a search indexes sub-document. resp = self.client.r_session.get(ddoc.document_url) raw_ddoc = response_to_json_dict(resp) - self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) + self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev','options'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1413,6 +1418,7 @@ def test_fetch_list_functions(self): self.assertEqual(ddoc_remote, { '_id': '_design/ddoc001', '_rev': ddoc['_rev'], + 'options': {'partitioned': False}, 'lists': { 'list001': list_func, 'list002': list_func, @@ -1437,7 +1443,7 @@ def test_fetch_no_list_functions(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() self.assertEqual(set(ddoc_remote.keys()), - {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + {'_id', '_rev', 'options', 'indexes', 'views', 'lists', 'shows'}) self.assertEqual(ddoc_remote['_id'], '_design/ddoc001') self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote['_rev'], ddoc['_rev']) @@ -1452,14 +1458,14 @@ def test_save_with_no_list_functions(self): ddoc = DesignDocument(self.db, '_design/ddoc001') ddoc.save() # Ensure that locally cached DesignDocument contains lists dict - self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'lists', 'shows', 'indexes', 'views'}) + self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'lists', 'options', 'shows', 'indexes', 'views'}) self.assertEqual(ddoc['_id'], '_design/ddoc001') self.assertTrue(ddoc['_rev'].startswith('1-')) # Ensure that remotely saved design document does not # include a lists sub-document. resp = self.client.r_session.get(ddoc.document_url) raw_ddoc = response_to_json_dict(resp) - self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) + self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev','options'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) @@ -1714,6 +1720,7 @@ def test_fetch_show_functions(self): self.assertEqual(ddoc_remote, { '_id': '_design/ddoc001', '_rev': ddoc['_rev'], + 'options': {'partitioned': False}, 'lists': {}, 'shows': { 'show001': show_func, @@ -1738,7 +1745,7 @@ def test_fetch_no_show_functions(self): ddoc_remote = DesignDocument(self.db, '_design/ddoc001') ddoc_remote.fetch() self.assertEqual(set(ddoc_remote.keys()), - {'_id', '_rev', 'indexes', 'views', 'lists', 'shows'}) + {'_id', '_rev', 'indexes', 'options', 'views', 'lists', 'shows'}) self.assertEqual(ddoc_remote['_id'], '_design/ddoc001') self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote['_rev'], ddoc['_rev']) @@ -1753,14 +1760,14 @@ def test_save_with_no_show_functions(self): ddoc = DesignDocument(self.db, '_design/ddoc001') ddoc.save() # Ensure that locally cached DesignDocument contains shows dict - self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'lists', 'shows', 'indexes', 'views'}) + self.assertEqual(set(ddoc.keys()), {'_id', '_rev', 'lists','options', 'shows', 'indexes', 'views'}) self.assertEqual(ddoc['_id'], '_design/ddoc001') self.assertTrue(ddoc['_rev'].startswith('1-')) # Ensure that remotely saved design document does not # include a shows sub-document. resp = self.client.r_session.get(ddoc.document_url) raw_ddoc = response_to_json_dict(resp) - self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev'}) + self.assertEqual(set(raw_ddoc.keys()), {'_id', '_rev','options'}) self.assertEqual(raw_ddoc['_id'], ddoc['_id']) self.assertEqual(raw_ddoc['_rev'], ddoc['_rev']) From 37894ceb7bab61667a8c399470a1fcc8d461a27e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Mon, 11 May 2020 16:16:53 +0200 Subject: [PATCH 148/185] Disable false-positive linter error (#469) * Disable false-positive linter error The build has been failing recently with a false positive linter error. Pylint was complaining that a method is not callable, even though the method is defined right on the following line. As I could not find either the reason pylint was failing or a solution that resolves this issue, this commit disables the check for that specific line pylint was failing on. * Fix assertions for non-partitioned db * Pin pylint version --- src/cloudant/feed.py | 2 +- test-requirements.txt | 2 +- tests/unit/design_document_tests.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cloudant/feed.py b/src/cloudant/feed.py index ef2c90fc..f038ebfe 100644 --- a/src/cloudant/feed.py +++ b/src/cloudant/feed.py @@ -144,7 +144,7 @@ def __next__(self): """ Provides Python3 compatibility. """ - return self.next() + return self.next() # pylint: disable=not-callable def next(self): """ diff --git a/test-requirements.txt b/test-requirements.txt index 88c13ffb..8d95c01d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,5 +2,5 @@ mock==1.3.0 nose sphinx sphinx_rtd_theme -pylint +pylint==2.5.2 flaky diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 68a40505..52352bcc 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -390,7 +390,8 @@ def test_fetch_dbcopy(self): # before comparison also. Compare the removed values with # the expected content in each case. self.assertEqual(db_copy, ddoc['views']['view002'].pop('dbcopy')) - self.assertEqual({'epi': {'dbcopy': {'view002': db_copy}}}, ddoc_remote.pop('options')) + self.assertEqual({'epi': {'dbcopy': {'view002': db_copy}}, 'partitioned': False}, ddoc_remote.pop('options')) + self.assertEqual({'partitioned': False}, ddoc.pop('options')) self.assertEqual(ddoc_remote, ddoc) self.assertTrue(ddoc_remote['_rev'].startswith('1-')) self.assertEqual(ddoc_remote, { @@ -1553,7 +1554,8 @@ def test_geospatial_index(self): 'indexes': {}, 'views': {}, 'lists': {}, - 'shows': {} + 'shows': {}, + 'options': {'partitioned': False} }) # Document with geospatial point geodoc = Document(self.db, 'doc001') From dfa5a7bb11bafa44b50d4dd3edfbdf7793c8aa07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Wed, 13 May 2020 15:34:42 +0200 Subject: [PATCH 149/185] Remove `CHANGELOG.md` from the PR template (#471) This repo only has a `CHANGES.md` and `CHANGELOG.md` is not applicable. This PR removes the `CHANGELOG.md` from the template. --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8a9c853a..5b5697c2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,7 +5,7 @@ Thanks for your hard work, please ensure all items are complete before opening. - [ ] Tick to sign-off your agreement to the [Developer Certificate of Origin (DCO) 1.1](../blob/master/DCO1.1.txt) - [ ] Added tests for code changes _or_ test/build only changes -- [ ] Updated the change log file (`CHANGES.md`|`CHANGELOG.md`) _or_ test/build only changes +- [ ] Updated the change log file (`CHANGES.md`) _or_ test/build only changes - [ ] Completed the PR template below: ## Description From 14646b7c38caeceface6e2a35f94392356886b9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Thu, 14 May 2020 13:31:03 +0200 Subject: [PATCH 150/185] Make `create_query_index` always supply `partitioned` (#470) `create_query_index` does not always supply `partitioned` which is always required by the service. This commit changes that so that `partitioned` is now always present in the `create_query_index` requests when defined. This fixes #468 --- CHANGES.md | 1 + docs/getting_started.rst | 8 ++++++-- src/cloudant/database.py | 2 +- src/cloudant/index.py | 6 +++--- tests/unit/index_tests.py | 15 +++++++++++++++ 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9e7155db..6ea317b1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # UNRELEASED - [FIXED] Set default value for `partitioned` parameter to false when creating a design document. +- [FIXED] Corrected setting of `partitioned` flag for `create_query_index` requests. # 2.13.0 (2020-04-16) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 9cd4b65e..c92387ca 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -284,8 +284,12 @@ when constructing the new ``DesignDocument`` class. ddoc.add_view('myview','function(doc) { emit(doc.foo, doc.bar); }') ddoc.save() -Similarly, to define a partitioned Cloudant Query index you must set the -``partitioned=True`` optional. + +To define a partitioned Cloudant Query index you may set the +``partitioned=True`` optional, but it is not required as the index will be +partitioned by default in a partitioned database. Conversely, you must +set the ``partitioned=False`` optional if you wish to create a global +(non-partitioned) index in a partitioned database. .. code-block:: python diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 17978468..3586c93f 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -1100,7 +1100,7 @@ def create_query_index( design_document_id=None, index_name=None, index_type='json', - partitioned=False, + partitioned=None, **kwargs ): """ diff --git a/src/cloudant/index.py b/src/cloudant/index.py index 051c004b..c66c7ac2 100644 --- a/src/cloudant/index.py +++ b/src/cloudant/index.py @@ -47,7 +47,7 @@ class Index(object): :func:`~cloudant.database.CloudantDatabase.create_query_index`. """ - def __init__(self, database, design_document_id=None, name=None, partitioned=False, **kwargs): + def __init__(self, database, design_document_id=None, name=None, partitioned=None, **kwargs): self._database = database self._r_session = self._database.r_session self._ddoc_id = design_document_id @@ -154,8 +154,8 @@ def create(self): self._def_check() payload['index'] = self._def - if self._partitioned: - payload['partitioned'] = True + if self._partitioned is not None: + payload['partitioned'] = bool(self._partitioned) headers = {'Content-Type': 'application/json'} resp = self._r_session.post( diff --git a/tests/unit/index_tests.py b/tests/unit/index_tests.py index 4361644f..29a3d51c 100644 --- a/tests/unit/index_tests.py +++ b/tests/unit/index_tests.py @@ -554,6 +554,21 @@ def test_create_a_search_index_invalid_selector_value(self): '<{} \'dict\'>'.format('type' if PY2 else 'class') ) + def test_create_unpartitioned_query_index(self): + """ + Test that create_query_index works on an unpartitioned database + """ + ddoc = DesignDocument(self.db, document_id="unpartitioned_query_index_ddoc") + ddoc["language"] = "query" + ddoc.save() + index = self.db.create_query_index( + design_document_id="_design/unpartitioned_query_index_ddoc", + fields=["key"], + partitioned=False + ) + index.create() + self.assertGreater(len(self.db.get_query_indexes()), 0) + def test_search_index_via_query(self): """ Test that a created TEXT index will produce expected query results. From d080687766a3101b1e876957674983b956851a76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Fri, 12 Jun 2020 14:36:04 +0200 Subject: [PATCH 151/185] Add private endpoint usage with IAM to docs (#475) --- docs/getting_started.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index c92387ca..37c5b92c 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -111,6 +111,13 @@ You can easily connect to your Cloudant account using an IAM API key: # Authenticate using an IAM API key client = Cloudant.iam(ACCOUNT_NAME, API_KEY, connect=True) +If you need to authenticate to a server outside of the `cloudant.com` domain, you can use the `url` parameter: + +.. code-block:: python + + # Authenticate using an IAM API key to an account outside of the cloudant.com domain + client = Cloudant.iam(None, API_KEY, url='https://private.endpoint.example', connect=True) + **************** Resource sharing From 4e2c377f9c955f04b2d911d4480a00a308ce581e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Mon, 17 Aug 2020 10:36:58 +0200 Subject: [PATCH 152/185] Make setup.py compatible with python 2 (#477) --- CHANGES.md | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 6ea317b1..0fe2e075 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ - [FIXED] Set default value for `partitioned` parameter to false when creating a design document. - [FIXED] Corrected setting of `partitioned` flag for `create_query_index` requests. +- [FIXED] Added a workaround for installation on Python 2. # 2.13.0 (2020-04-16) diff --git a/setup.py b/setup.py index e02192cd..ac2e3876 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ """ +from io import open from os import path from setuptools import setup, find_packages From 296cf8110da40dfd2528f5431340a79d7e5fdc1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Mon, 17 Aug 2020 16:09:52 +0200 Subject: [PATCH 153/185] Prepare 2.14.0 release --- CHANGES.md | 2 ++ VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0fe2e075..93cc5339 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # UNRELEASED +# 2.14.0 (2020-08-17) + - [FIXED] Set default value for `partitioned` parameter to false when creating a design document. - [FIXED] Corrected setting of `partitioned` flag for `create_query_index` requests. - [FIXED] Added a workaround for installation on Python 2. diff --git a/VERSION b/VERSION index ca3389d9..edcfe40d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.13.1-SNAPSHOT +2.14.0 diff --git a/docs/conf.py b/docs/conf.py index ada059aa..44f27314 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.13.1-SNAPSHOT' +version = '2.14.0' # The full version, including alpha/beta/rc tags. -release = '2.13.1-SNAPSHOT' +release = '2.14.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 0b162070..193a099c 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.13.1-SNAPSHOT' +__version__ = '2.14.0' # pylint: disable=wrong-import-position import contextlib From 3f4ae5d7c3dc79a5b85a2b481b998570c262b0fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Mon, 17 Aug 2020 16:42:49 +0200 Subject: [PATCH 154/185] Remove "Unreleased" from CHANGES.md --- CHANGES.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 93cc5339..9188a90d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,3 @@ -# UNRELEASED - # 2.14.0 (2020-08-17) - [FIXED] Set default value for `partitioned` parameter to false when creating a design document. From 8c703409053fc2a47d25ba9e3da43b1527370baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Tue, 18 Aug 2020 17:15:37 +0200 Subject: [PATCH 155/185] Update version to 2.14.1-SNAPSHOT (#480) --- CHANGES.md | 2 ++ VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9188a90d..93cc5339 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,5 @@ +# UNRELEASED + # 2.14.0 (2020-08-17) - [FIXED] Set default value for `partitioned` parameter to false when creating a design document. diff --git a/VERSION b/VERSION index edcfe40d..20738ec2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.14.0 +2.14.1-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index 44f27314..1e338e6d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.14.0' +version = '2.14.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.14.0' +release = '2.14.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 193a099c..7e011096 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.14.0' +__version__ = '2.14.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From e80a07eb8a9723e2179872e832f0a47cd1a4fce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Tue, 18 Aug 2020 18:28:39 +0200 Subject: [PATCH 156/185] Remove Python 2 compatibility (#478) Python 2 was EoLed back in January. This commit removes it from the list of supported environments. --- CHANGES.md | 1 + README.md | 2 +- docs/compatibility.rst | 1 - setup.py | 2 -- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 93cc5339..b40e6fb1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,5 @@ # UNRELEASED +- [REMOVED] Removed Python 2 compatibility from the supported environments. # 2.14.0 (2020-08-17) diff --git a/README.md b/README.md index 0cc5edcf..7eaf83ba 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build Status](https://travis-ci.org/cloudant/python-cloudant.svg?branch=master)](https://travis-ci.org/cloudant/python-cloudant) [![Readthedocs](https://readthedocs.org/projects/pip/badge/)](http://python-cloudant.readthedocs.io) -[![Compatibility](https://img.shields.io/badge/python-2.7%2C%203.5-blue.svg)](http://python-cloudant.readthedocs.io/en/latest/compatibility.html) +[![Compatibility](https://img.shields.io/badge/python-3.5-blue.svg)](http://python-cloudant.readthedocs.io/en/latest/compatibility.html) [![pypi](https://img.shields.io/pypi/v/cloudant.svg)](https://pypi.python.org/pypi/cloudant) This is the official Cloudant library for Python. diff --git a/docs/compatibility.rst b/docs/compatibility.rst index 1bb98cce..b756a65e 100644 --- a/docs/compatibility.rst +++ b/docs/compatibility.rst @@ -11,5 +11,4 @@ Note that some features are Cloudant specific. This library has been tested with the following versions of Python -* `Python™ 2.7 `_ * `Python™ 3.5 `_ diff --git a/setup.py b/setup.py index ac2e3876..a8ac4ef8 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,6 @@ 'Topic :: Software Development :: Libraries', 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5' ] From 943ad117d0cb416ffb589c35e144b14eb32726f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bessenyei=20Bal=C3=A1zs=20Don=C3=A1t?= Date: Fri, 4 Sep 2020 11:10:36 +0200 Subject: [PATCH 157/185] Remove incorrect restriction from docs for bookmarks (#481) --- CHANGES.md | 1 + src/cloudant/database.py | 3 +-- src/cloudant/query.py | 9 +++------ src/cloudant/result.py | 3 +-- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b40e6fb1..e68156a0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # UNRELEASED - [REMOVED] Removed Python 2 compatibility from the supported environments. +- [FIXED] Fixed the documentation for `bookmarks`. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/database.py b/src/cloudant/database.py index 3586c93f..ccde12da 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -1242,8 +1242,7 @@ def get_query_result(self, selector, fields=None, raw_result=False, wrapped in a QueryResult or if the response JSON is returned. Defaults to False. :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param int limit: Maximum number of results returned. Only valid if used with ``raw_result=True``. :param int page_size: Sets the page size for result iteration. Default diff --git a/src/cloudant/query.py b/src/cloudant/query.py index d6a0aebc..2362f80d 100644 --- a/src/cloudant/query.py +++ b/src/cloudant/query.py @@ -69,8 +69,7 @@ class Query(dict): :param CloudantDatabase database: A Cloudant database instance used by the Query. :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param list fields: A list of fields to be returned by the query. :param int limit: Maximum number of results returned. :param int r: Read quorum needed for the result. Each document is read from @@ -141,8 +140,7 @@ def __call__(self, **kwargs): and set ``raw_result=True`` instead. :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param list fields: A list of fields to be returned by the query. :param int limit: Maximum number of results returned. :param int r: Read quorum needed for the result. Each document is read @@ -204,8 +202,7 @@ def custom_result(self, **options): data = rslt[100:200] :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param list fields: A list of fields to be returned by the query. :param int page_size: Sets the page size for result iteration. Default is 100. diff --git a/src/cloudant/result.py b/src/cloudant/result.py index d3dd9a09..e484cd07 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -488,8 +488,7 @@ class QueryResult(Result): :param query: A reference to the query callable that returns the JSON content result to be wrapped. :param str bookmark: A string that enables you to specify which page of - results you require. Only valid for queries using indexes of type - *text*. + results you require. :param list fields: A list of fields to be returned by the query. :param int page_size: Sets the page size for result iteration. Default is 100. From 1d634d69c48e23932a1ae7b0b5befefe1c274047 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 20 Nov 2020 12:18:30 +0000 Subject: [PATCH 158/185] Correct replicator states Add the `failed` state to `follow_replication`. The replicator tests were missing some other states introduced in CouchDB 2.1 scheduler. --- CHANGES.md | 1 + src/cloudant/replicator.py | 10 ++++++++-- tests/unit/replicator_tests.py | 9 ++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e68156a0..40299e13 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # UNRELEASED - [REMOVED] Removed Python 2 compatibility from the supported environments. - [FIXED] Fixed the documentation for `bookmarks`. +- [FIXED] Also exit `follow_replication` for `failed` state. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/replicator.py b/src/cloudant/replicator.py index e3ec3550..f1b44643 100644 --- a/src/cloudant/replicator.py +++ b/src/cloudant/replicator.py @@ -193,7 +193,12 @@ def update_state(): repl_doc, state = update_state() if repl_doc: yield repl_doc - if state is not None and state in ['error', 'completed']: + # This is a little awkward, since 2.1 the terminal states are + # "failed" and "completed", so those should be the exit states, but + # for backwards compatibility with older versions "error" is also + # needed. The code has always exited for "error" state even long + # after 2.1 was available so that behaviour is retained. + if state is not None and state in ['error', 'failed', 'completed']: return # Now listen on changes feed for the state @@ -202,7 +207,8 @@ def update_state(): repl_doc, state = update_state() if repl_doc is not None: yield repl_doc - if state is not None and state in ['error', 'completed']: + # See note about these states + if state is not None and state in ['error', 'failed', 'completed']: return def stop_replication(self, repl_id): diff --git a/tests/unit/replicator_tests.py b/tests/unit/replicator_tests.py index 28d78fab..9eb56b56 100644 --- a/tests/unit/replicator_tests.py +++ b/tests/unit/replicator_tests.py @@ -322,12 +322,14 @@ def test_retrieve_replication_state(self): ) self.replication_ids.append(repl_id) repl_state = None - valid_states = ['completed', 'error', 'triggered', 'running', None] + # note triggered is for versions prior to 2.1 + valid_states = ['completed', 'error', 'initializing', 'triggered', 'pending', 'running', 'failed', 'crashing', None] finished = False + # Wait for 5 minutes or a terminal replication state for _ in range(300): repl_state = self.replicator.replication_state(repl_id) self.assertTrue(repl_state in valid_states) - if repl_state in ('error', 'completed'): + if repl_state in ('error', 'failed', 'completed'): finished = True break time.sleep(1) @@ -407,7 +409,8 @@ def test_follow_replication(self): repl_id ) self.replication_ids.append(repl_id) - valid_states = ('completed', 'error', 'triggered', 'running', None) + # note triggered is for versions prior to 2.1 + valid_states = ['completed', 'error', 'initializing', 'triggered', 'pending', 'running', 'failed', 'crashing', None] repl_states = [] if 'scheduler' in self.client.features(): state_key = 'state' From a105ae7c191328ad8bdcc75bce12e9ba5f5ad72f Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 20 Nov 2020 13:04:15 -0500 Subject: [PATCH 159/185] Fix paging logic to check 'id' field exists for grouped view queries Added CHANGES entry Dict members aren't checked by `hasattr`. Use `get` instead and keep the value so we don't have to fetch it twice. --- CHANGES.md | 1 + src/cloudant/result.py | 5 +++-- tests/unit/database_tests.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 40299e13..ad1d3788 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ - [REMOVED] Removed Python 2 compatibility from the supported environments. - [FIXED] Fixed the documentation for `bookmarks`. - [FIXED] Also exit `follow_replication` for `failed` state. +- [FIXED] Fixed result paging for grouped view queries. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/result.py b/src/cloudant/result.py index e484cd07..319d7268 100644 --- a/src/cloudant/result.py +++ b/src/cloudant/result.py @@ -397,9 +397,10 @@ def _iterator(self, response): # if we are in a view, keys could be duplicate so we # need to start from the right docid - if last['id']: + last_doc_id = last.get('id') + if last_doc_id is not None: response = self._call(startkey=last['key'], - startkey_docid=last['id']) + startkey_docid=last_doc_id) # reduce result keys are unique by definition else: response = self._call(startkey=last['key']) diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index f054dd62..c8c5cfe0 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -408,6 +408,34 @@ def test_retrieve_view_results(self): self.assertIsInstance(rslt, Result) self.assertEqual(rslt[:1], rslt['julia099']) + def test_retrieve_grouped_view_result_with_page_size(self): + """ + Test retrieving Result wrapped output from a design document grouped view + that uses a custom page size + + The view used here along with group=True will generate rows of + data where each key will be grouped into groups of 2. Such as: + {'key': 0, 'value': 2}, + {'key': 1, 'value': 2}, + ... + """ + map_func = 'function(doc) {\n emit(Math.floor(doc.age / 2), 1); \n}' + data = {'_id': '_design/ddoc01','views': {'view01': {"map": map_func, "reduce": "_count"}}} + self.db.create_document(data) + self.populate_db_with_documents(5) + + rslt = self.db.get_view_result( + '_design/ddoc01', + 'view01', + group=True, + page_size=1) + self.assertIsInstance(rslt, Result) + i = 0 + for row in rslt: + self.assertIsNotNone(row) + self.assertEqual(row['key'], i) + i += 1 + def test_retrieve_raw_view_results(self): """ Test retrieving raw output from a design document view From 43254c75cafdc19b098c5b04f19907879352a197 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Fri, 8 Jan 2021 12:17:26 -0500 Subject: [PATCH 160/185] Add migration guide (#488) * Added migration guide to cloudant-python-sdk library * Updated README with migration guide section --- MIGRATION.md | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 5 +++ 2 files changed, 100 insertions(+) create mode 100644 MIGRATION.md diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 00000000..5503e2cc --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,95 @@ +# Migrating to the `cloudant-python-sdk` library +This document is to assist in migrating from the `python-cloudant` (module: `cloudant`) to the newly supported [`cloudant-python-sdk`](https://github.com/IBM/cloudant-python-sdk) (module: `ibmcloudant`). + +## Initializing the client connection +There are several ways to create a client connection in `cloudant-python-sdk`: +1. [Environment variables](https://github.com/IBM/cloudant-python-sdk#authentication-with-environment-variables) +2. [External configuration file](https://github.com/IBM/cloudant-python-sdk#authentication-with-external-configuration) +3. [Programmatically](https://github.com/IBM/cloudant-python-sdk#programmatic-authentication) + +[See the README](https://github.com/IBM/cloudant-python-sdk#code-examples) for code examples on using environment variables. + +## Other differences +1. The `cloudant-python-sdk` library does not support local dictionary caching of database and document objects. +1. There are no context managers in `cloudant-python-sdk`. To reproduce the behaviour of the `python-cloudant` +context managers in `cloudant-python-sdk` users need to explicitly call the specific operations against the +remote HTTP API. For example, in the case of the document context manager, this would mean doing both a `get_document` +to fetch and a `put_document` to save. +1. In `cloudant-python-sdk` View, Search, and Query (aka `_find` endpoint) operation responses contain raw JSON +content like using `raw_result=True` in `python-cloudant`. + +## Request mapping +Here's a list of the top 5 most frequently used `python-cloudant` operations and the `cloudant-python-sdk` equivalent API operation documentation link: + +| `python-cloudant` operation | `cloudant-python-sdk` API operation documentation link | +|---------------------------------------|---------------------------------| +|`Document('db_name', 'docid').fetch()` |[`getDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdocument)| +|`db.get_view_result()` |[`postView`](https://cloud.ibm.com/apidocs/cloudant?code=python#postview)| +|`db.get_query_result()` |[`postFind`](https://cloud.ibm.com/apidocs/cloudant?code=python#postfind)| +| `doc.exists()` |[`headDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#headdocument)| +|`Document('db_name', 'docid').save()` |[`putDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#putdocument)| + + +[A table](#reference-table) with the whole list of operations is provided at the end of this guide. + +The `cloudant-python-sdk` library is generated from a more complete API spec and provides a significant number of operations that do not exist in `python-cloudant`. See [the IBM Cloud API Documentation](https://cloud.ibm.com/apidocs/cloudant) to review request parameter and body options, code examples, and additional details for every endpoint. + +## Known Issues +There's an [outline of known issues](https://github.com/IBM/cloudant-python-sdk/blob/master/KNOWN_ISSUES.md) in the `cloudant-python-sdk` repository. + +## Reference table +The table below contains a list of `python-cloudant` functions and the `cloudant-python-sdk` equivalent API operation documentation link. The `cloudant-python-sdk` operation documentation link will contain the new function in a code sample e.g. `getServerInformation` link will contain a code example with `get_server_information()`. + +**Note:** There are many API operations included in the new `cloudant-python-sdk` that are not available in the `python-cloudant` library. The [API documentation](https://cloud.ibm.com/apidocs/cloudant?code=python) contains the full list of operations. + +| `python-cloudant` function | `cloudant-python-sdk` API operation documentation link | +|-----------------|---------------------| +|`metadata()`|[getServerInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getserverinformation)| +|`all_dbs()`|[getAllDbs](https://cloud.ibm.com/apidocs/cloudant?code=python#getalldbs)| +|`db_updates()/infinite_db_updates()`|[getDbUpdates](https://cloud.ibm.com/apidocs/cloudant?code=python#getdbupdates)| +|`Replicator.create_replication()`|[postReplicate](https://cloud.ibm.com/apidocs/cloudant?code=python#postreplicate)| +|`Scheduler.get_doc()`|[getSchedulerDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocument)| +|`Scheduler.list_jobs()`|[getSchedulerJobs](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerjobs)| +|`session()`|[getSessionInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getsessioninformation)| +|`uuids()`|[getUuids](https://cloud.ibm.com/apidocs/cloudant?code=python#getuuids)| +|`db.delete()`|[deleteDatabase](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedatabase)| +|`db.metadata()`|[getDatabaseInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getdatabaseinformation)| +|`db.create_document()`|[postDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#postdocument)| +|`db.create()`|[putDatabase](https://cloud.ibm.com/apidocs/cloudant?code=python#putdatabase)| +|`db.all_docs()/db.keys()`|[postAllDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postalldocs)| +|`db.bulk_docs()`|[postBulkDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postbulkdocs)| +|`db.changes()/db.infinite_changes()`|[postChanges](https://cloud.ibm.com/apidocs/cloudant?code=python#postchanges)| +|`DesignDocument(db, '_design/doc').delete()`|[deleteDesignDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedesigndocument)| +|`db.get_design_document()/DesignDocument(db, '_design/doc').fetch()`|[getDesignDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocument)| +|`DesignDocument(db, '_design/doc').save()`|[putDesignDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putdesigndocument)| +|`DesignDocument(db, '_design/doc').info()`|[getDesignDocumentInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocumentinformation)| +|`db.get_search_result()`|[postSearch](https://cloud.ibm.com/apidocs/cloudant?code=python#postsearch)| +|`db.get_view_result()`|[postView](https://cloud.ibm.com/apidocs/cloudant?code=python#postview)| +|`db.list_design_documents()`|[postDesignDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postdesigndocs)| +|`db.get_query_result()`|[postFind](https://cloud.ibm.com/apidocs/cloudant?code=python#postfind)| +|`db.get_query_indexes()`|[getIndexesInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getindexesinformation)| +|`db.create_query_index()`|[postIndex](https://cloud.ibm.com/apidocs/cloudant?code=python#postindex)| +|`db.delete_query_index()`|[deleteIndex](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteindex)| +|`Document(db, 'docid').delete()`|[getLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| +|`Document(db, 'docid').create()`|[putLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putlocaldocument)| +|`db.missing_revisions()`|[postMissingRevs](https://cloud.ibm.com/apidocs/cloudant?code=python#postmissingrevs)| +|`db.partition_metadata()`|[getPartitionInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getpartitioninformation)| +|`db.partitioned_all_docs()`|[postPartitionAllDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionalldocs)| +|`db.get_partitioned_search_result()`|[postPartitionSearch](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionsearch)| +|`db.get_partitioned_view_result()`|[postPartitionView](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionview)| +|`db.get_partitioned_query_result()`|[postPartitionFind](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionfind)| +|`db.revisions_diff()`|[postRevsDiff](https://cloud.ibm.com/apidocs/cloudant?code=python#postrevsdiff)| +|`db.get_security_document()/db.security_document()`|[getSecurity](https://cloud.ibm.com/apidocs/cloudant?code=python#getsecurity)| +|`db.share_database()`|[putSecurity](https://cloud.ibm.com/apidocs/cloudant?code=python#putsecurity)| +|`db.shards()`|[getShardsInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getshardsinformation)| +|`Document(db, 'docid').delete()`|[getLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| +|`Document(db, 'docid').fetch()`|[getDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getdocument)| +|`Document(db, 'docid').exists()`|[headDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#headdocument)| +|`Document(db, 'docid').save()`|[putDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putdocument)| +|`Document(db, 'docid').delete_attachment()`|[deleteAttachment](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteattachment)| +|`Document(db, 'docid').get_attachment()`|[getAttachment](https://cloud.ibm.com/apidocs/cloudant?code=python#getattachment)| +|`Document(db, 'docid').put_attachment()`|[putAttachment](https://cloud.ibm.com/apidocs/cloudant?code=python#putattachment)| +|`generate_api_key()`|[postApiKeys](https://cloud.ibm.com/apidocs/cloudant?code=python#postapikeys)| +|`SecurityDocument().save()`|[putCloudantSecurityConfiguration](https://cloud.ibm.com/apidocs/cloudant?code=python#putcloudantsecurityconfiguration)| +|`cors_configuration()/cors_origin()`|[getCorsInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getcorsinformation)| +|`update_cors_configuration()`|[putCorsConfiguration](https://cloud.ibm.com/apidocs/cloudant?code=python#putcorsconfiguration)| diff --git a/README.md b/README.md index 7eaf83ba..f12b2814 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ This is the official Cloudant library for Python. * [Using in Other Projects](#using-in-other-projects) * [License](#license) * [Issues](#issues) +* [Migrating to `cloudant-python-sdk` library](#migrating-to-cloudant-python-sdk-library) ## Installation and Usage @@ -87,3 +88,7 @@ to see if the problem has already been reported. Note that the default search includes only open issues, but it may already have been closed. * Cloudant customers should contact Cloudant support for urgent issues. * When opening a new issue [here in github](../../issues) please complete the template fully. + +## Migrating to `cloudant-python-sdk` library +We have a newly supported Cloudant Python SDK named [cloudant-python-sdk](https://github.com/IBM/cloudant-python-sdk). +For advice on migrating from this module see [MIGRATION.md](MIGRATION.md). From 20cefd5ddd6c2975224ddfd4831de98a238f6c2f Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Thu, 28 Jan 2021 10:43:30 -0500 Subject: [PATCH 161/185] Update local document operations in migration guide (#489) --- MIGRATION.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 5503e2cc..53845e7d 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -70,8 +70,9 @@ The table below contains a list of `python-cloudant` functions and the `cloudant |`db.get_query_indexes()`|[getIndexesInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getindexesinformation)| |`db.create_query_index()`|[postIndex](https://cloud.ibm.com/apidocs/cloudant?code=python#postindex)| |`db.delete_query_index()`|[deleteIndex](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteindex)| -|`Document(db, 'docid').delete()`|[getLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| -|`Document(db, 'docid').create()`|[putLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putlocaldocument)| +|`Document(db, '_local/docid').fetch()`|[getLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| +|`Document(db, '_local/docid').save()`|[putLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putlocaldocument)| +|`Document(db, '_local/docid').delete()`|[deleteLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletelocaldocument)| |`db.missing_revisions()`|[postMissingRevs](https://cloud.ibm.com/apidocs/cloudant?code=python#postmissingrevs)| |`db.partition_metadata()`|[getPartitionInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getpartitioninformation)| |`db.partitioned_all_docs()`|[postPartitionAllDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionalldocs)| @@ -82,7 +83,7 @@ The table below contains a list of `python-cloudant` functions and the `cloudant |`db.get_security_document()/db.security_document()`|[getSecurity](https://cloud.ibm.com/apidocs/cloudant?code=python#getsecurity)| |`db.share_database()`|[putSecurity](https://cloud.ibm.com/apidocs/cloudant?code=python#putsecurity)| |`db.shards()`|[getShardsInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getshardsinformation)| -|`Document(db, 'docid').delete()`|[getLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| +|`Document(db, 'docid').delete()`|[deleteDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedocument)| |`Document(db, 'docid').fetch()`|[getDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getdocument)| |`Document(db, 'docid').exists()`|[headDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#headdocument)| |`Document(db, 'docid').save()`|[putDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putdocument)| From 33781463ff85c8bb535bd6ede2049e4d3bec6c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Wed, 10 Feb 2021 16:47:14 +0100 Subject: [PATCH 162/185] Add replication and scheduler operations to the migration guide (#490) --- MIGRATION.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 53845e7d..29be9810 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -1,4 +1,4 @@ -# Migrating to the `cloudant-python-sdk` library +# Migrating to the `cloudant-python-sdk` library This document is to assist in migrating from the `python-cloudant` (module: `cloudant`) to the newly supported [`cloudant-python-sdk`](https://github.com/IBM/cloudant-python-sdk) (module: `ibmcloudant`). ## Initializing the client connection @@ -11,11 +11,11 @@ There are several ways to create a client connection in `cloudant-python-sdk`: ## Other differences 1. The `cloudant-python-sdk` library does not support local dictionary caching of database and document objects. -1. There are no context managers in `cloudant-python-sdk`. To reproduce the behaviour of the `python-cloudant` -context managers in `cloudant-python-sdk` users need to explicitly call the specific operations against the -remote HTTP API. For example, in the case of the document context manager, this would mean doing both a `get_document` +1. There are no context managers in `cloudant-python-sdk`. To reproduce the behaviour of the `python-cloudant` +context managers in `cloudant-python-sdk` users need to explicitly call the specific operations against the +remote HTTP API. For example, in the case of the document context manager, this would mean doing both a `get_document` to fetch and a `put_document` to save. -1. In `cloudant-python-sdk` View, Search, and Query (aka `_find` endpoint) operation responses contain raw JSON +1. In `cloudant-python-sdk` View, Search, and Query (aka `_find` endpoint) operation responses contain raw JSON content like using `raw_result=True` in `python-cloudant`. ## Request mapping @@ -47,8 +47,11 @@ The table below contains a list of `python-cloudant` functions and the `cloudant |`metadata()`|[getServerInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getserverinformation)| |`all_dbs()`|[getAllDbs](https://cloud.ibm.com/apidocs/cloudant?code=python#getalldbs)| |`db_updates()/infinite_db_updates()`|[getDbUpdates](https://cloud.ibm.com/apidocs/cloudant?code=python#getdbupdates)| -|`Replicator.create_replication()`|[postReplicate](https://cloud.ibm.com/apidocs/cloudant?code=python#postreplicate)| +|`Replicator.stop_replication()`|[deleteReplicationDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletereplicationdocument)| +|`Replicator.replication_state()`|[getReplicationDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getreplicationdocument)| +|`Replicator.create_replication()`|[putReplicationDocument](https://cloud.ibm.com/apidocs/cloudant?code=#putreplicationdocument)| |`Scheduler.get_doc()`|[getSchedulerDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocument)| +|`Scheduler.list_docs()`|[getSchedulerDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocs)| |`Scheduler.list_jobs()`|[getSchedulerJobs](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerjobs)| |`session()`|[getSessionInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getsessioninformation)| |`uuids()`|[getUuids](https://cloud.ibm.com/apidocs/cloudant?code=python#getuuids)| From 18245a94a8b6551517cf5134efefc5a84b516f59 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 14 May 2021 15:51:34 +0100 Subject: [PATCH 163/185] Make IAM token server configurable in tests --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index ddf21dd4..d88f017c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,6 +13,7 @@ def getEnvForSuite(suiteName) { case 'iam': // Setting IAM_API_KEY forces tests to run using an IAM enabled client. envVars.add("IAM_API_KEY=$DB_IAM_API_KEY") + envVars.add("IAM_TOKEN_URL=$SDKS_TEST_IAM_URL") break case 'cookie': case 'simplejson': From c409805ccb148d2f0da2e3a2bd5e72bbed1ffed9 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Fri, 14 May 2021 16:01:03 +0100 Subject: [PATCH 164/185] Add new test server env vars --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d88f017c..e655a45c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ def getEnvForSuite(suiteName) { // Base environment variables def envVars = [ - "CLOUDANT_ACCOUNT=$DB_USER", + "DB_URL=${SDKS_TEST_SERVER_URL}", "RUN_CLOUDANT_TESTS=1", "SKIP_DB_UPDATES=1" // Disable pending resolution of case 71610 ] @@ -29,8 +29,8 @@ def setupPythonAndTest(pythonVersion, testSuite) { // Unstash the source on this node unstash name: 'source' // Set up the environment and test - withCredentials([usernamePassword(credentialsId: 'clientlibs-test', usernameVariable: 'DB_USER', passwordVariable: 'DB_PASSWORD'), - string(credentialsId: 'clientlibs-test-iam', variable: 'DB_IAM_API_KEY')]) { + withCredentials([usernamePassword(credentialsId: 'testServerLegacy', usernameVariable: 'DB_USER', passwordVariable: 'DB_PASSWORD'), + string(credentialsId: 'testServerIamApiKey', variable: 'DB_IAM_API_KEY')]) { withEnv(getEnvForSuite("${testSuite}")) { try { sh """ From ebcb98c544c2e24b284e0c10970ba3d72c1783d7 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Mon, 17 May 2021 12:04:34 +0100 Subject: [PATCH 165/185] Fix client tests to not always use account --- tests/unit/client_tests.py | 40 ++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 2726de33..07c4159b 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2020 IBM Corp. All rights reserved. +# Copyright (C) 2015, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ import sys import unittest from time import sleep +from urllib.parse import urlparse import mock import requests @@ -213,7 +214,7 @@ def test_auto_renew_enabled_with_auto_connect(self): @skip_if_not_cookie_auth def test_session(self): """ - Test getting session information. + Test getting session information. Session info is None if CouchDB Admin Party mode was selected. """ try: @@ -563,7 +564,7 @@ def test_get_cached_db_object_via_get(self): self.client.connect() # Default returns None self.assertIsNone(self.client.get('no_such_db')) - # Creates the database remotely and adds it to the + # Creates the database remotely and adds it to the # client database cache db = self.client.create_database(dbname) # Locally cached database object is returned @@ -702,7 +703,7 @@ def test_cloudant_context_helper(self): Test that the cloudant context helper works as expected. """ try: - with cloudant(self.user, self.pwd, account=self.account) as c: + with cloudant(self.user, self.pwd, url=self.url) as c: self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) except Exception as err: @@ -718,7 +719,7 @@ def test_cloudant_bluemix_context_helper_with_legacy_creds(self): 'credentials': { 'username': self.user, 'password': self.pwd, - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -744,7 +745,7 @@ def test_cloudant_bluemix_context_helper_with_iam(self): 'credentials': { 'apikey': self.iam_api_key, 'username': self.user, - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -766,7 +767,7 @@ def test_cloudant_bluemix_context_helper_raise_error_for_missing_iam_and_creds(s instance_name = 'Cloudant NoSQL DB-lv' vcap_services = {'cloudantNoSQLDB': [{ 'credentials': { - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -795,7 +796,7 @@ def test_cloudant_bluemix_dedicated_context_helper(self): 'credentials': { 'username': self.user, 'password': self.pwd, - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -818,10 +819,10 @@ def test_constructor_with_account(self): """ # Ensure that the client is new del self.client - self.client = Cloudant(self.user, self.pwd, account=self.account) + self.client = Cloudant('user', 'pass', account='foo') self.assertEqual( self.client.server_url, - 'https://{0}.cloudant.com'.format(self.account) + 'https://foo.cloudant.com' ) @skip_if_not_cookie_auth @@ -835,7 +836,7 @@ def test_bluemix_constructor_with_legacy_creds(self): 'credentials': { 'username': self.user, 'password': self.pwd, - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -870,7 +871,7 @@ def test_bluemix_constructor_with_iam(self): 'credentials': { 'apikey': self.iam_api_key, 'username': self.user, - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443 }, 'name': instance_name @@ -901,7 +902,7 @@ def test_bluemix_constructor_specify_instance_name(self): 'credentials': { 'username': self.user, 'password': self.pwd, - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -934,7 +935,7 @@ def test_bluemix_constructor_with_multiple_services(self): { 'credentials': { 'apikey': '1234api', - 'host': '{0}.cloudant.com'.format(self.account), + 'host': urlparse(self.url).hostname, 'port': 443, 'url': self.url }, @@ -973,10 +974,11 @@ def test_connect_headers(self): """ try: self.client.connect() - self.assertEqual( - self.client.r_session.headers['X-Cloudant-User'], - self.account - ) + if (self.account): + self.assertEqual( + self.client.r_session.headers['X-Cloudant-User'], + self.account + ) agent = self.client.r_session.headers.get('User-Agent') ua_parts = agent.split('/') self.assertEqual(len(ua_parts), 6) @@ -1413,4 +1415,4 @@ def test_update_cors_configuration(self): self.client.disconnect() if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 0f031cde8a1ea6c6b91ecfc353d9aad9242a8e50 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 20 May 2021 14:38:45 +0100 Subject: [PATCH 166/185] Correct use of username as account name in `Cloudant.bluemix()` --- CHANGES.md | 2 ++ src/cloudant/client.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ad1d3788..8885ae26 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,8 @@ - [FIXED] Fixed the documentation for `bookmarks`. - [FIXED] Also exit `follow_replication` for `failed` state. - [FIXED] Fixed result paging for grouped view queries. +- [FIXED] Incorrect use of username as account name in `Cloudant.bluemix()`. +- [IMPROVED] Documented use of None account name and url override for `Cloudant.iam()`. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/client.py b/src/cloudant/client.py index f09d3a69..47c8555c 100755 --- a/src/cloudant/client.py +++ b/src/cloudant/client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015, 2019 IBM Corp. All rights reserved. +# Copyright (c) 2015, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -853,7 +853,7 @@ def bluemix(cls, vcap_services, instance_name=None, service_name=None, **kwargs) raise CloudantClientException(103) if hasattr(service, 'iam_api_key'): - return Cloudant.iam(service.username, + return Cloudant.iam(None, service.iam_api_key, url=service.url, **kwargs) @@ -867,7 +867,7 @@ def iam(cls, account_name, api_key, **kwargs): """ Create a Cloudant client that uses IAM authentication. - :param account_name: Cloudant account name. + :param account_name: Cloudant account name; or use None and a url kwarg. :param api_key: IAM authentication API key. """ return cls(None, From 5f3ca45c442847046f3a5dfd34f6cfcd5b5929f5 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sun, 30 May 2021 16:45:38 +0200 Subject: [PATCH 167/185] Include LICENSE in pip package --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 96ebb5e1..7c050d91 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include requirements.txt VERSION +include requirements.txt VERSION LICENSE From 105fe2055562adbeb7c5ae41f97712794bf8716b Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Thu, 3 Jun 2021 09:28:45 -0400 Subject: [PATCH 168/185] Remove CouchDB 1.x from Travis CI (#502) --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b49557a3..9f770aac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,6 @@ python: env: - ADMIN_PARTY=true COUCHDB_VERSION=2.3.1 - ADMIN_PARTY=false COUCHDB_VERSION=2.3.1 - - ADMIN_PARTY=true COUCHDB_VERSION=1.7.2 - - ADMIN_PARTY=false COUCHDB_VERSION=1.7.2 services: - docker From f0c0883080a4dcf403a8f3b6e717a4de224f965b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Wed, 16 Jun 2021 15:05:39 +0200 Subject: [PATCH 169/185] Use custom encoder (if provided) for all view `key` params not just `keys` (#501) --- CHANGES.md | 1 + src/cloudant/_common_util.py | 11 +++++++---- tests/unit/database_tests.py | 9 +++++++++ tests/unit/param_translation_tests.py | 14 +++++++------- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8885ae26..522d5ede 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ - [FIXED] Fixed result paging for grouped view queries. - [FIXED] Incorrect use of username as account name in `Cloudant.bluemix()`. - [IMPROVED] Documented use of None account name and url override for `Cloudant.iam()`. +- [FIXED] Use custom encoder (if provided) for all view `key` params not just `keys`. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 6e765c1d..962f1dec 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -160,7 +160,7 @@ def feed_arg_types(feed_type): return _COUCH_DB_UPDATES_ARG_TYPES return _CHANGES_ARG_TYPES -def python_to_couch(options): +def python_to_couch(options, encoder=None): """ Translates query options from python style options into CouchDB/Cloudant query options. For example ``{'include_docs': True}`` will @@ -171,13 +171,14 @@ def python_to_couch(options): :func:`~cloudant.view.View.__call__` callable, both used to retrieve data. :param dict options: Python style parameters to be translated. + :param encoder: Custom encoder, defaults to None :returns: Dictionary of translated CouchDB/Cloudant query parameters """ translation = dict() for key, val in iteritems_(options): py_to_couch_validate(key, val) - translation.update(_py_to_couch_translate(key, val)) + translation.update(_py_to_couch_translate(key, val, encoder)) return translation def py_to_couch_validate(key, val): @@ -201,7 +202,7 @@ def py_to_couch_validate(key, val): if val not in ('ok', 'update_after'): raise CloudantArgumentError(135, val) -def _py_to_couch_translate(key, val): +def _py_to_couch_translate(key, val, encoder=None): """ Performs the conversion of the Python parameter value to its CouchDB equivalent. @@ -209,6 +210,8 @@ def _py_to_couch_translate(key, val): try: if key in ['keys', 'endkey_docid', 'startkey_docid', 'stale', 'update']: return {key: val} + if key in ['endkey', 'key', 'startkey']: + return {key: json.dumps(val, cls=encoder)} if val is None: return {key: None} arg_converter = TYPE_CONVERTERS.get(type(val)) @@ -249,7 +252,7 @@ def get_docs(r_session, url, encoder=None, headers=None, **params): keys = None if keys_list is not None: keys = json.dumps({'keys': keys_list}, cls=encoder) - f_params = python_to_couch(params) + f_params = python_to_couch(params, encoder) resp = None if keys is not None: # If we're using POST we are sending JSON so add the header diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index c8c5cfe0..a4e4bad1 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -512,6 +512,15 @@ def test_all_docs_get_with_long_type(self): data = self.db.all_docs(limit=1, skip=LONG_NUMBER) self.assertEqual(len(data.get('rows')), 1) + def test_all_docs_get_uses_custom_encoder(self): + """ + Test that all_docs uses the custom encoder. + """ + self.set_up_client(auto_connect=True, encoder="AEncoder") + database = self.client[self.test_dbname] + with self.assertRaises(CloudantArgumentError): + database.all_docs(endkey=['foo', 10]) + def test_custom_result_context_manager(self): """ Test using the database custom result context manager diff --git a/tests/unit/param_translation_tests.py b/tests/unit/param_translation_tests.py index 4da0d05d..6aac7210 100644 --- a/tests/unit/param_translation_tests.py +++ b/tests/unit/param_translation_tests.py @@ -36,7 +36,7 @@ def test_valid_descending(self): {'descending': 'true'} ) self.assertEqual( - python_to_couch({'descending': False}), + python_to_couch({'descending': False}), {'descending': 'false'} ) @@ -44,9 +44,9 @@ def test_valid_endkey(self): """ Test endkey translation is successful. """ - self.assertEqual(python_to_couch({'endkey': 10}), {'endkey': 10}) + self.assertEqual(python_to_couch({'endkey': 10}), {'endkey': '10'}) # Test with long type - self.assertEqual(python_to_couch({'endkey': LONG_NUMBER}), {'endkey': LONG_NUMBER}) + self.assertEqual(python_to_couch({'endkey': LONG_NUMBER}), {'endkey': str(LONG_NUMBER)}) self.assertEqual( python_to_couch({'endkey': 'foo'}), {'endkey': '"foo"'} @@ -120,9 +120,9 @@ def test_valid_key(self): """ Test key translation is successful. """ - self.assertEqual(python_to_couch({'key': 10}), {'key': 10}) + self.assertEqual(python_to_couch({'key': 10}), {'key': '10'}) # Test with long type - self.assertEqual(python_to_couch({'key': LONG_NUMBER}), {'key': LONG_NUMBER}) + self.assertEqual(python_to_couch({'key': LONG_NUMBER}), {'key': str(LONG_NUMBER)}) self.assertEqual(python_to_couch({'key': 'foo'}), {'key': '"foo"'}) self.assertEqual( python_to_couch({'key': ['foo', 10]}), @@ -194,9 +194,9 @@ def test_valid_startkey(self): """ Test startkey translation is successful. """ - self.assertEqual(python_to_couch({'startkey': 10}), {'startkey': 10}) + self.assertEqual(python_to_couch({'startkey': 10}), {'startkey': '10'}) # Test with long type - self.assertEqual(python_to_couch({'startkey': LONG_NUMBER}), {'startkey': LONG_NUMBER}) + self.assertEqual(python_to_couch({'startkey': LONG_NUMBER}), {'startkey': str(LONG_NUMBER)}) self.assertEqual( python_to_couch({'startkey': 'foo'}), {'startkey': '"foo"'} From 19b14ab9adb38d84885587b71c56a66a7789893c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Mon, 21 Jun 2021 09:48:15 +0200 Subject: [PATCH 170/185] Support boolean type for `key`, `endkey`, and `startkey` in view requests (#504) --- CHANGES.md | 1 + src/cloudant/_common_util.py | 11 ++++++----- tests/unit/param_translation_tests.py | 18 +++++++++++++++--- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 522d5ede..56700503 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ - [FIXED] Incorrect use of username as account name in `Cloudant.bluemix()`. - [IMPROVED] Documented use of None account name and url override for `Cloudant.iam()`. - [FIXED] Use custom encoder (if provided) for all view `key` params not just `keys`. +- [FIXED] Support boolean type for `key`, `endkey`, and `startkey` in view requests. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index 962f1dec..adce56fa 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -56,20 +56,20 @@ RESULT_ARG_TYPES = { 'descending': (bool,), - 'endkey': (int, LONGTYPE, STRTYPE, Sequence,), + 'endkey': (int, LONGTYPE, STRTYPE, Sequence, bool,), 'endkey_docid': (STRTYPE,), 'group': (bool,), 'group_level': (int, LONGTYPE, NONETYPE,), 'include_docs': (bool,), 'inclusive_end': (bool,), - 'key': (int, LONGTYPE, STRTYPE, Sequence,), + 'key': (int, LONGTYPE, STRTYPE, Sequence, bool,), 'keys': (list,), 'limit': (int, LONGTYPE, NONETYPE,), 'reduce': (bool,), 'skip': (int, LONGTYPE, NONETYPE,), 'stable': (bool,), 'stale': (STRTYPE,), - 'startkey': (int, LONGTYPE, STRTYPE, Sequence,), + 'startkey': (int, LONGTYPE, STRTYPE, Sequence, bool,), 'startkey_docid': (STRTYPE,), 'update': (STRTYPE,), } @@ -191,12 +191,13 @@ def py_to_couch_validate(key, val): # Validate argument values and ensure that a boolean is not passed in # if an integer is expected if (not isinstance(val, RESULT_ARG_TYPES[key]) or - (type(val) is bool and int in RESULT_ARG_TYPES[key])): + (type(val) is bool and bool not in RESULT_ARG_TYPES[key] and + int in RESULT_ARG_TYPES[key])): raise CloudantArgumentError(117, key, RESULT_ARG_TYPES[key]) if key == 'keys': for key_list_val in val: if (not isinstance(key_list_val, RESULT_ARG_TYPES['key']) or - type(key_list_val) is bool): + isinstance(key_list_val, bool)): raise CloudantArgumentError(134, RESULT_ARG_TYPES['key']) if key == 'stale': if val not in ('ok', 'update_after'): diff --git a/tests/unit/param_translation_tests.py b/tests/unit/param_translation_tests.py index 6aac7210..fda3c002 100644 --- a/tests/unit/param_translation_tests.py +++ b/tests/unit/param_translation_tests.py @@ -55,6 +55,10 @@ def test_valid_endkey(self): python_to_couch({'endkey': ['foo', 10]}), {'endkey': '["foo", 10]'} ) + self.assertEqual( + python_to_couch({'endkey': True}), + {'endkey': 'true'} + ) def test_valid_endkey_docid(self): """ @@ -128,6 +132,10 @@ def test_valid_key(self): python_to_couch({'key': ['foo', 10]}), {'key': '["foo", 10]'} ) + self.assertEqual( + python_to_couch({'key': True}), + {'key': 'true'} + ) def test_valid_keys(self): """ @@ -205,6 +213,10 @@ def test_valid_startkey(self): python_to_couch({'startkey': ['foo', 10]}), {'startkey': '["foo", 10]'} ) + self.assertEqual( + python_to_couch({'startkey': True}), + {'startkey': 'true'} + ) def test_valid_startkey_docid(self): """ @@ -247,7 +259,7 @@ def test_invalid_endkey(self): """ msg = 'Argument endkey not instance of expected type:' with self.assertRaises(CloudantArgumentError) as cm: - python_to_couch({'endkey': True}) + python_to_couch({'endkey': {'foo': 'bar'}}) self.assertTrue(str(cm.exception).startswith(msg)) def test_invalid_endkey_docid(self): @@ -302,7 +314,7 @@ def test_invalid_key(self): """ msg = 'Argument key not instance of expected type:' with self.assertRaises(CloudantArgumentError) as cm: - python_to_couch({'key': True}) + python_to_couch({'key': {'foo': 'bar'}}) self.assertTrue(str(cm.exception).startswith(msg)) def test_invalid_keys_not_list(self): @@ -372,7 +384,7 @@ def test_invalid_startkey(self): """ msg = 'Argument startkey not instance of expected type:' with self.assertRaises(CloudantArgumentError) as cm: - python_to_couch({'startkey': True}) + python_to_couch({'startkey': {'foo': 'bar'}}) self.assertTrue(str(cm.exception).startswith(msg)) def test_invalid_startkey_docid(self): From 436eddb45093a6f8aab26f778a7ed5aabc6aba5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Mon, 21 Jun 2021 16:02:48 +0200 Subject: [PATCH 171/185] Override dict's get method (#505) Co-authored-by: dominickj-tdi <49956725+dominickj-tdi@users.noreply.github.com> --- CHANGES.md | 1 + src/cloudant/database.py | 18 ++++++++++++++ tests/unit/database_tests.py | 46 ++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 56700503..5b641708 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ - [IMPROVED] Documented use of None account name and url override for `Cloudant.iam()`. - [FIXED] Use custom encoder (if provided) for all view `key` params not just `keys`. - [FIXED] Support boolean type for `key`, `endkey`, and `startkey` in view requests. +- [NEW] Override `dict.get` method for `CouchDatabase` to add `remote` parameter allowing it to retrieve a remote document if specified. # 2.14.0 (2020-08-17) diff --git a/src/cloudant/database.py b/src/cloudant/database.py index ccde12da..4ad93f90 100644 --- a/src/cloudant/database.py +++ b/src/cloudant/database.py @@ -706,6 +706,24 @@ def __getitem__(self, key): raise KeyError(key) return doc + def get(self, key, remote=False): + """ + Overrides dict's get method. This gets an item from the database or cache + like __getitem__, but instead of throwing an exception if the item is not + found, it simply returns None. + + :param bool remote: Dictates whether a remote request is made to + retrieve the doc, if it is not present in the local cache. + Defaults to False. + """ + if remote: + try: + return self.__getitem__(key) + except KeyError: + return None + else: + return super(CouchDatabase, self).get(key) + def __contains__(self, key): """ Overrides dictionary __contains__ behavior to check if a document diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index a4e4bad1..3ba12584 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -256,6 +256,8 @@ def test_create_document_with_id(self): data = {'_id': 'julia06', 'name': 'julia', 'age': 6} doc = self.db.create_document(data) self.assertEqual(self.db['julia06'], doc) + self.assertEqual(self.db.get('julia06'), doc) + self.assertEqual(self.db.get('julia06', remote=True), doc) self.assertEqual(doc['_id'], data['_id']) self.assertTrue(doc['_rev'].startswith('1-')) self.assertEqual(doc['name'], data['name']) @@ -271,6 +273,42 @@ def test_create_document_with_id(self): 'Document with id julia06 already exists.' ) + def test_get_non_existing_document_from_remote(self): + """ + Test dict's get on non existing document from remote. + """ + doc = self.db.get('non-existing', remote=True) + self.assertIsNone(doc) + + def test_get_non_existing_document_from_cache(self): + """ + Test dict's get on non existing document from cache. + """ + doc = self.db.get('non-existing') + self.assertIsNone(doc) + + def test_get_document_from_cache(self): + """ + Test dict's get on a document from cache. + """ + doc = Document(self.db, document_id='julia06') + self.db['julia06'] = doc + self.assertEqual(self.db.get('julia06'), doc) + # doc is fetched from the local dict preferentially to remote even with remote=True + self.assertEqual(self.db.get('julia06', remote=True), doc) + self.assertEqual(self.db['julia06'], doc) + + def test_get_document_from_remote(self): + """ + Test dict's get on a document from remote. + """ + data = {'_id': 'julia06','name': 'julia06', 'age': 6} + doc = self.db.create_document(data) + self.db.clear() + self.assertIsNone(self.db.get('julia06')) + self.assertEqual(self.db.get('julia06', remote=True), doc) + self.assertEqual(self.db['julia06'], doc) + def test_create_document_that_already_exists(self): """ Test creating a document that already exists @@ -278,6 +316,8 @@ def test_create_document_that_already_exists(self): data = {'_id': 'julia'} doc = self.db.create_document(data) self.assertEqual(self.db['julia'], doc) + self.assertEqual(self.db.get('julia'), doc) + self.assertEqual(self.db.get('julia', remote=True), doc) self.assertTrue(doc['_rev'].startswith('1-')) # attempt to recreate document self.db.create_document(data, throw_on_exists=False) @@ -289,6 +329,8 @@ def test_create_document_without_id(self): data = {'name': 'julia', 'age': 6} doc = self.db.create_document(data) self.assertEqual(self.db[doc['_id']], doc) + self.assertEqual(self.db.get(doc['_id']), doc) + self.assertEqual(self.db.get(doc['_id'], remote=True), doc) self.assertTrue(doc['_rev'].startswith('1-')) self.assertEqual(doc['name'], data['name']) self.assertEqual(doc['age'], data['age']) @@ -302,6 +344,8 @@ def test_create_design_document(self): data = {'_id': '_design/julia06', 'name': 'julia', 'age': 6} doc = self.db.create_document(data) self.assertEqual(self.db['_design/julia06'], doc) + self.assertEqual(self.db.get('_design/julia06'), doc) + self.assertEqual(self.db.get('_design/julia06', remote=True), doc) self.assertEqual(doc['_id'], data['_id']) self.assertTrue(doc['_rev'].startswith('1-')) self.assertEqual(doc['name'], data['name']) @@ -316,6 +360,8 @@ def test_create_empty_document(self): """ empty_doc = self.db.new_document() self.assertEqual(self.db[empty_doc['_id']], empty_doc) + self.assertEqual(self.db.get(empty_doc['_id']), empty_doc) + self.assertEqual(self.db.get(empty_doc['_id'], remote=True), empty_doc) self.assertTrue(all(x in ['_id', '_rev'] for x in empty_doc.keys())) self.assertTrue(empty_doc['_rev'].startswith('1-')) From 66e4b188030bfa8ed4a71a0a8980f34fd8f34153 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 3 Jun 2021 16:30:22 +0100 Subject: [PATCH 172/185] Add deprecation warnings Co-authored-by: Esteban Laver --- CHANGES.md | 10 ++++++---- README.md | 14 ++++++++++++++ src/cloudant/__init__.py | 4 ++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5b641708..21fa1a50 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,13 +1,15 @@ # UNRELEASED -- [REMOVED] Removed Python 2 compatibility from the supported environments. +- [NEW] Override `dict.get` method for `CouchDatabase` to add `remote` parameter allowing it to + retrieve a remote document if specified. - [FIXED] Fixed the documentation for `bookmarks`. - [FIXED] Also exit `follow_replication` for `failed` state. - [FIXED] Fixed result paging for grouped view queries. - [FIXED] Incorrect use of username as account name in `Cloudant.bluemix()`. -- [IMPROVED] Documented use of None account name and url override for `Cloudant.iam()`. - [FIXED] Use custom encoder (if provided) for all view `key` params not just `keys`. - [FIXED] Support boolean type for `key`, `endkey`, and `startkey` in view requests. -- [NEW] Override `dict.get` method for `CouchDatabase` to add `remote` parameter allowing it to retrieve a remote document if specified. +- [DEPRECATED] This library is now deprecated and will be EOL on Dec 31 2021. +- [REMOVED] Removed Python 2 compatibility from the supported environments. +- [IMPROVED] Documented use of `None` account name and url override for `Cloudant.iam()`. # 2.14.0 (2020-08-17) @@ -139,7 +141,7 @@ # 2.2.0 (2016-10-20) -- [NEW] Added auto connect feature to the client constructor.
 +- [NEW] Added auto connect feature to the client constructor. - [FIXED] Requests session is no longer valid after disconnect. # 2.1.1 (2016-10-03) diff --git a/README.md b/README.md index f12b2814..029edf07 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,17 @@ +# DEPRECATED + +**This library is now deprecated and will be end-of-life on Dec 31 2021.** + +The library remains supported until the end-of-life date, +but will receive only _critical_ maintenance updates. + +Please see the [Migration Guide](./MIGRATION.md) for advice +about migrating to our replacement library +[cloudant-python-sdk](https://github.com/IBM/cloudant-python-sdk). + +For FAQs and additional information please refer to the +[Cloudant blog](https://blog.cloudant.com/2021/06/30/Cloudant-SDK-Transition.html). + # Cloudant Python Client [![Build Status](https://travis-ci.org/cloudant/python-cloudant.svg?branch=master)](https://travis-ci.org/cloudant/python-cloudant) diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 7e011096..f3910174 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -19,10 +19,14 @@ # pylint: disable=wrong-import-position import contextlib +import warnings # pylint: disable=wrong-import-position from .client import Cloudant, CouchDB from ._common_util import CloudFoundryService +warnings.warn('The module cloudant is now deprecated. The replacement is ibmcloudant.', + DeprecationWarning) + @contextlib.contextmanager def cloudant(user, passwd, **kwargs): """ From a166650f795852e506adf3c758a612300fb28392 Mon Sep 17 00:00:00 2001 From: Esteban Laver Date: Wed, 28 Jul 2021 00:34:32 -0400 Subject: [PATCH 173/185] Add validation for doc ID and attachment names --- CHANGES.md | 5 + Jenkinsfile | 2 +- src/cloudant/_common_util.py | 28 +- src/cloudant/_messages.py | 6 +- src/cloudant/design_document.py | 9 +- src/cloudant/document.py | 24 +- tests/unit/document_validation_tests.py | 909 ++++++++++++++++++++++++ 7 files changed, 973 insertions(+), 10 deletions(-) create mode 100644 tests/unit/document_validation_tests.py diff --git a/CHANGES.md b/CHANGES.md index 21fa1a50..66d1f820 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,11 @@ - [DEPRECATED] This library is now deprecated and will be EOL on Dec 31 2021. - [REMOVED] Removed Python 2 compatibility from the supported environments. - [IMPROVED] Documented use of `None` account name and url override for `Cloudant.iam()`. +- [IMPROVED] - Document IDs and attachment names are now rejected if they could cause an unexpected + Cloudant request. We have seen that some applications pass unsantized document IDs to SDK functions + (e.g. direct from user requests). In response to this we have updated many functions to reject + obviously invalid paths. However, for complete safety applications must still validate that + document IDs and attachment names match expected patterns. # 2.14.0 (2020-08-17) diff --git a/Jenkinsfile b/Jenkinsfile index e655a45c..cc2d0ad4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,7 +25,7 @@ def getEnvForSuite(suiteName) { } def setupPythonAndTest(pythonVersion, testSuite) { - node { + node('sdks-executor') { // Unstash the source on this node unstash name: 'source' // Set up the environment and test diff --git a/src/cloudant/_common_util.py b/src/cloudant/_common_util.py index adce56fa..433e09a6 100644 --- a/src/cloudant/_common_util.py +++ b/src/cloudant/_common_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2019 IBM Corp. All rights reserved. +# Copyright © 2015, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,6 +31,8 @@ # Library Constants +DESIGN_PREFIX = '_design/' +LOCAL_PREFIX = '_local/' USER_AGENT = '/'.join([ 'python-cloudant', sys.modules['cloudant'].__version__, @@ -301,6 +303,30 @@ def response_to_json_dict(response, **kwargs): response.encoding = 'utf-8' return json.loads(response.text, **kwargs) +def assert_document_type_id(docid): + """ + Validate the document ID. Raises an error if the ID is an `_` prefixed name + that isn't either `_design` or `_local`. + :return: + """ + invalid = False + if docid.startswith('_'): + if docid.startswith(DESIGN_PREFIX) and DESIGN_PREFIX != docid: + invalid = False + elif docid.startswith(LOCAL_PREFIX) and LOCAL_PREFIX != docid: + invalid = False + else: + invalid = True + if invalid: + raise CloudantArgumentError(137, docid) + +def assert_attachment_name(attname): + """ + Validate the document attachment's name. Raises an error if `_` prefixed name exists. + :return: + """ + if attname.startswith('_'): + raise CloudantArgumentError(138, attname) # Classes diff --git a/src/cloudant/_messages.py b/src/cloudant/_messages.py index 07a23d62..b428998c 100644 --- a/src/cloudant/_messages.py +++ b/src/cloudant/_messages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2016 IBM. All rights reserved. +# Copyright © 2016, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -65,7 +65,9 @@ # Common_util 134: 'Key list element not of expected type: {0}', 135: 'Invalid value for stale option {0} must be ok or update_after.', - 136: 'Error converting argument {0}: {1}' + 136: 'Error converting argument {0}: {1}', + 137: 'Invalid document ID: {0}', + 138: 'Invalid attachment name: {0}' } CLIENT = { diff --git a/src/cloudant/design_document.py b/src/cloudant/design_document.py index d3ea387a..df9fb44d 100644 --- a/src/cloudant/design_document.py +++ b/src/cloudant/design_document.py @@ -16,7 +16,8 @@ API module/class for interacting with a design document in a database. """ from ._2to3 import iteritems_, url_quote_plus, STRTYPE -from ._common_util import QUERY_LANGUAGE, codify, response_to_json_dict +from ._common_util import QUERY_LANGUAGE, codify, response_to_json_dict, \ + assert_document_type_id, DESIGN_PREFIX from .document import Document from .view import View, QueryIndexView from .error import CloudantArgumentError, CloudantDesignDocumentException @@ -44,8 +45,10 @@ class DesignDocument(Document): databases. """ def __init__(self, database, document_id=None, partitioned=False): - if document_id and not document_id.startswith('_design/'): - document_id = '_design/{0}'.format(document_id) + if document_id: + assert_document_type_id(document_id) + if document_id and not document_id.startswith(DESIGN_PREFIX): + document_id = '{0}{1}'.format(DESIGN_PREFIX, document_id) super(DesignDocument, self).__init__(database, document_id) if partitioned: diff --git a/src/cloudant/document.py b/src/cloudant/document.py index 38082054..96267bc9 100644 --- a/src/cloudant/document.py +++ b/src/cloudant/document.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015, 2018 IBM Corp. All rights reserved. +# Copyright © 2015, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ from requests.exceptions import HTTPError from ._2to3 import url_quote, url_quote_plus -from ._common_util import response_to_json_dict +from ._common_util import response_to_json_dict, assert_document_type_id, assert_attachment_name from .error import CloudantDocumentException @@ -96,6 +96,15 @@ def document_url(self): url_quote(self['_id'][8:], safe='') )) + # handle _local document url + if self['_id'].startswith('_local/'): + return '/'.join(( + self._database_host, + url_quote_plus(self._database_name), + '_local', + url_quote(self['_id'][7:], safe='') + )) + # handle document url return '/'.join(( self._database_host, @@ -113,6 +122,8 @@ def exists(self): if '_id' not in self or self['_id'] is None: return False + assert_document_type_id(self['_id']) + resp = self.r_session.head(self.document_url) if resp.status_code not in [200, 404]: resp.raise_for_status() @@ -161,6 +172,8 @@ def fetch(self): """ if self.document_url is None: raise CloudantDocumentException(101) + if '_id' in self: + assert_document_type_id(self['_id']) resp = self.r_session.get(self.document_url) resp.raise_for_status() self.clear() @@ -309,6 +322,8 @@ def delete(self): if not self.get("_rev"): raise CloudantDocumentException(103) + assert_document_type_id(self['_id']) + del_resp = self.r_session.delete( self.document_url, params={"rev": self["_rev"]}, @@ -375,7 +390,8 @@ def get_attachment( """ # need latest rev self.fetch() - attachment_url = '/'.join((self.document_url, attachment)) + assert_attachment_name(attachment) + attachment_url = '/'.join((self.document_url, url_quote(attachment, safe=''))) if headers is None: headers = {'If-Match': self['_rev']} else: @@ -418,6 +434,7 @@ def delete_attachment(self, attachment, headers=None): """ # need latest rev self.fetch() + assert_attachment_name(attachment) attachment_url = '/'.join((self.document_url, attachment)) if headers is None: headers = {'If-Match': self['_rev']} @@ -459,6 +476,7 @@ def put_attachment(self, attachment, content_type, data, headers=None): """ # need latest rev self.fetch() + assert_attachment_name(attachment) attachment_url = '/'.join((self.document_url, attachment)) if headers is None: headers = { diff --git a/tests/unit/document_validation_tests.py b/tests/unit/document_validation_tests.py new file mode 100644 index 00000000..021d8d6f --- /dev/null +++ b/tests/unit/document_validation_tests.py @@ -0,0 +1,909 @@ +#!/usr/bin/env python +# Copyright © 2021 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +from enum import Enum +from unittest.mock import Mock, patch + +from mock import create_autospec + +import requests +from urllib.parse import urlparse + +from cloudant import database +from cloudant.design_document import DesignDocument +from cloudant.document import Document +from cloudant.error import CloudantArgumentError + +class ValidationExceptionMsg(Enum): + DOC = 'Invalid document ID:' + ATTACHMENT = 'Invalid attachment name:' + +class Expect(Enum): + VALIDATION_EXCEPTION_DOCID = CloudantArgumentError(137, '') + VALIDATION_EXCEPTION_ATT = CloudantArgumentError(138, '') + RESPONSE_404 = 404 + RESPONSE_200 = 200 + RESPONSE_201 = 201 + + +class ValidationTests(unittest.TestCase): + """ + Document validation unit tests + """ + def setUp(self): + self.doc_r_session_patcher = patch('cloudant.document.Document.r_session') + self.requests_get_patcher = patch('requests.get') + + self.addCleanup(patch.stopall) + + self.doc_r_session_mock = self.doc_r_session_patcher.start() + self.requests_get_mock = self.requests_get_patcher.start() + + self.db = create_autospec(database) + self.db.client = Mock() + self.db.client.server_url = 'http://mocked.url.com' + self.db.database_url = 'http://mocked.url.com/my_db' + self.db.database_name = 'mydb' + + def teardown(self): + self.addCleanup(patch.stopall) + del self.db + del self.doc_r_session_patcher + del self.requests_get_patcher + del self.doc_r_session_mock + del self.requests_get_mock + + # GET and HEAD _all_docs + # EXPECTED: validation failure + def test_get_invalid_all_docs(self): + """ + Test GET/HEAD request for invalid '_all_docs' document ID + """ + self.get_document_variants('_all_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _design/foo + # EXPECTED: 200 + def test_get_valid_ddoc(self): + """ + Test GET/HEAD request for valid '_design/foo' document ID + """ + self.get_document_variants('_design/foo', Expect.RESPONSE_200.value, path_segment_count=3) + self.get_document_variants('_design/foo', Expect.RESPONSE_200.value, True, path_segment_count=3) + + # GET and HEAD _design + # EXPECTED: Validation exception + def test_get_invalid_design(self): + """ + Test GET/HEAD request for invalid '_design' document ID + """ + self.get_document_variants('_design', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_document_variants('_design', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET and HEAD /_design/foo with a slash + # EXPECTED: 404 + def test_get_missing_ddoc_with_slash(self): + """ + Test GET/HEAD request for missing '/_design/foo' document ID + """ + self.get_document_variants('/_design/foo', Expect.RESPONSE_404.value, path_segment_count=2) + + # GET and HEAD _design/foo/_view/bar + # EXPECTED: 404 + def test_get_invalid_view(self): + """ + Test GET/HEAD request for missing '_design/foo' document ID + """ + self.get_document_variants('_design/foo/_view/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_view/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _design/foo/_info + # EXPECTED: 404 + def test_get_invalid_view_info(self): + """ + Test GET/HEAD request for missing '_design/foo/_info' document ID + """ + self.get_document_variants('_design/foo/_info', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_info', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _design/foo/_search/bar + # EXPECTED: 404 + def test_get_invalid_search(self): + """ + Test GET/HEAD request for missing '_design/foo/_search/bar' document ID + """ + self.get_document_variants('_design/foo/_search/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_search/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _design/foo/_search_info/bar + # EXPECTED: 404 + def test_get_invalid_search_info(self): + """ + Test GET/HEAD request for missing '_design/foo/_search_info/bar' document ID + """ + self.get_document_variants('_design/foo/_search_info/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_search_info/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _design/foo/_geo/bar + # EXPECTED: 404 + def test_get_missing_geo(self): + """ + Test GET/HEAD request for missing '_design/foo/_geo/bar' document ID + """ + self.get_document_variants('_design/foo/_geo/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_geo/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + # with a parameter + self.get_document_variants('_design/foo/_geo/bar?bbox=-50.52,-4.46,54.59,1.45', Expect.RESPONSE_404.value, + path_segment_count=3) + self.get_document_variants('_design/foo/_geo/bar?bbox=-50.52,-4.46,54.59,1.45', Expect.RESPONSE_404.value, True, + path_segment_count=3) + + # GET and HEAD _design/foo/_geo_info/bar + # EXPECTED: 404 + def test_get_missing_geo_info(self): + """ + Test GET/HEAD request for missing '_design/foo/_geo_info/bar' document ID + """ + self.get_document_variants('_design/foo/_geo_info/bar', Expect.RESPONSE_404.value, path_segment_count=3) + self.get_document_variants('_design/foo/_geo_info/bar', Expect.RESPONSE_404.value, True, path_segment_count=3) + + # GET and HEAD _local/foo + # EXPECTED: 200 + def test_get_local_doc(self): + """ + Test GET/HEAD request for valid '_local/foo' document ID + """ + self.get_document_variants('_local/foo', Expect.RESPONSE_200.value, path_segment_count=3) + + # GET and HEAD _local + # EXPECTED: Validation exception + def test_get_invalid_local(self): + """ + Test GET/HEAD request for invalid '_local' document ID + """ + self.get_document_variants('_local', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _local_docs + # EXPECTED: Validation exception + def test_get_invalid_local_docs(self): + """ + Test GET/HEAD request for invalid '_local_docs' document ID + """ + self.get_document_variants('_local_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _design_docs + # EXPECTED: Validation exception + def test_get_invalid_design_docs(self): + """ + Test GET/HEAD request for invalid '_design_docs' document ID + """ + self.get_document_variants('_design_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _changes + # EXPECTED: Validation exception + def test_get_invalid_changes(self): + """ + Test GET/HEAD request for invalid '_changes' document ID + """ + self.get_document_variants('_changes', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _ensure_full_commit + # EXPECTED: Validation exception + def test_get_invalid_ensure_full_commit(self): + """ + Test GET/HEAD request for invalid '_ensure_full_commit' document ID + """ + self.get_document_variants('_ensure_full_commit', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _index + # EXPECTED: Validation exception + def test_get_invalid_index(self): + """ + Test GET/HEAD request for invalid '_index' document ID + """ + self.get_document_variants('_index', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _revs_limit + # EXPECTED: Validation exception + def test_get_invalid_revs_limit(self): + """ + Test GET/HEAD request for invalid '_revs_limit' document ID + """ + self.get_document_variants('_revs_limit', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _security + # EXPECTED: Validation exception + def test_get_invalid_security(self): + """ + Test GET/HEAD request for invalid '_security' document ID + """ + self.get_document_variants('_security', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET and HEAD _shards + # EXPECTED: Validation exception + def test_get_invalid_shards(self): + """ + Test GET/HEAD request for invalid '_shards' document ID + """ + self.get_document_variants('_shards', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _index/_design/foo/json/bar + # EXPECTED: Validation exception + def test_delete_invalid_index(self): + """ + Test DELETE request for invalid '_index/_design/foo/json/bar' document ID + """ + self.delete_document_variants('_index/_design/foo/json/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _design/foo + # EXPECTED: 200 + def test_delete_valid_ddoc(self): + """ + Test DELETE request for valid '_design/foo' document ID + """ + self.delete_document_variants('_design/foo', Expect.RESPONSE_200.value, path_segment_count=3) + + # DELETE _design + # EXPECTED: Validation exception + def test_delete_invalid_ddoc(self): + """ + Test DELETE request for invalid '_design' document ID + """ + # no trailing '/' on _design prefix + self.delete_document_variants('_design', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _local/foo + # EXPECTED: 200 + def test_delete_valid_local_doc(self): + """ + Test DELETE request for valid '_local/foo' document ID + """ + self.delete_document_variants('_local/foo', Expect.RESPONSE_200.value, path_segment_count=3) + + # DELETE _local + # EXPECTED: Validation exception + def test_delete_invalid_local(self): + """ + Test DELETE request for invalid '_local' document ID + """ + # no trailing '/' on _local prefix + self.delete_document_variants('_local', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _design/foo + # EXPECTED: 201 + def test_put_valid_ddoc(self): + """ + Test PUT request for valid '_design/foo' document ID + """ + self.put_document_variants('_design/foo', Expect.RESPONSE_201.value, path_segment_count=3) + + # PUT _design + # EXPECTED: Validation exception + def test_put_invalid_ddoc(self): + """ + Test PUT request for invalid '_design' document ID + """ + self.put_document_variants('_design', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _local/foo + # EXPECTED: 201 + def test_put_valid_local_doc(self): + """ + Test PUT request for valid '_local/foo' document ID + """ + self.put_document_variants('_local/foo', Expect.RESPONSE_201.value, path_segment_count=3) + + # PUT _local + # EXPECTED: Validation exception + def test_put_invalid_local_doc(self): + """ + Test PUT request for invalid '_local' document ID + """ + self.put_document_variants('_local', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _revs_limit + # EXPECTED: Validation exception + def test_put_invalid_revs_limit(self): + """ + Test PUT request for invalid '_revs_limit' document ID + """ + self.put_document_variants('_revs_limit', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _security + # EXPECTED: Validation exception + def test_put_invalid_security(self): + """ + Test PUT request for invalid '_security' document ID + """ + self.put_document_variants('_security', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET _design/foo/bar + # EXPECTED: 200 + def test_get_valid_ddoc_attachment(self): + """ + Test PUT request for valid '_design/foo/bar' document ID + """ + self.get_doc_attachment_variants('_design/foo', 'bar', Expect.RESPONSE_200.value, True, path_segment_count=4) + + # PUT _design/foo/bar + # EXPECTED: 201 + def test_put_valid_ddoc_attachment(self): + """ + Test PUT request for valid '_design/foo/bar' document ID + """ + self.put_doc_attachment_variants('_design/foo', 'bar', Expect.RESPONSE_201.value, True, path_segment_count=4) + + # DELETE _design/foo/bar + # EXPECTED: 200 + def test_delete_valid_ddoc_attachment(self): + """ + Test DELETE request for valid '_design/foo/bar' document ID + """ + self.delete_doc_attachment_variants('_design/foo', 'bar', Expect.RESPONSE_200.value, True, path_segment_count=4) + + # GET _design/foo + # EXPECTED: Validation exception + def test_get_invalid_ddoc_attachment(self): + """ + Test GET request for invalid '_design/foo' document ID + """ + # with ddoc option enabled + self.get_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.get_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # PUT _design/foo + # EXPECTED: Validation exception + def test_put_invalid_ddoc_attachment(self): + """ + Test PUT request for invalid '_design/foo' document ID + """ + # with ddoc option enabled + self.put_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.put_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _design/foo + # EXPECTED: Validation exception + def test_delete_invalid_ddoc_attachment(self): + """ + Test DELETE request for invalid '_design/foo' document ID + """ + # with ddoc option enabled + self.delete_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_design', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # DELETE _index/_design/foo/json/bar + # EXPECTED: Validation exception + def test_delete_index_via_attachment(self): + """ + Test DELETE requests for invalid '_index/_design/foo/json/bar' + """ + self.delete_doc_attachment_variants('_index', '_design/foo/json/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_index', '_design/foo/json/bar', + Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_index/_design', 'foo/json/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_index/_design', 'foo/json/bar', + Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_index/_design/foo', 'json/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_index/_design/foo', 'json/bar', + Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_index/_design/foo/json', 'bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_index/_design/foo/json', 'bar', + Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET _design/foo/_view/bar + def test_get_view_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_view/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_view', 'bar', Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_view', 'bar', Expect.RESPONSE_404.value, True, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo', '/_view/bar', Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo', '/_view/bar', Expect.RESPONSE_404.value, True, path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.get_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.get_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # PUT _design/foo/_view/bar + def test_put_view_via_ddoc_attachment(self): + """ + Test PUT requests for '_design/foo/_view/bar' + """ + # EXPECTED: Validation exception + self.put_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.put_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.put_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.put_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.put_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.put_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # DELETE _design/foo/_view/bar + def test_delete_view_via_ddoc_attachment(self): + """ + Test DELETE requests for '_design/foo/_view/bar' + """ + # EXPECTED: Validation exception + self.delete_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.delete_doc_attachment_variants('_design/foo', '_view/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.delete_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_design', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.delete_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.delete_doc_attachment_variants('_design/', 'foo/_view/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET _design/foo/_info + def test_get_view_info_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_info' + """ + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_info', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_info', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.get_doc_attachment_variants('_design', 'foo/_info', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design', 'foo/_info', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.get_doc_attachment_variants('_design/', 'foo/_info', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design/', 'foo/_info', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET _design/foo/_search/bar + def test_get_search_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_search/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_search', 'bar', Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_search', 'bar', Expect.RESPONSE_404.value, True, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_search', 'bar?q=*.*', Expect.RESPONSE_404.value, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_search', 'bar?q=*.*', Expect.RESPONSE_404.value, True, + path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_search/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_search/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + self.get_doc_attachment_variants('_design', 'foo/_search/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design', 'foo/_search/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + self.get_doc_attachment_variants('_design/', 'foo/_search/bar', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_design/', 'foo/_search/bar', Expect.VALIDATION_EXCEPTION_DOCID.value, True) + + # GET _design/foo/_search_info/bar + def test_get_search_info_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_search_info/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_search_info', 'bar', Expect.RESPONSE_404.value, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_search_info', 'bar', Expect.RESPONSE_404.value, True, + path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_search_info/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_search_info/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + + # GET _design/foo/_geo/bar + def test_get_geo_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_geo/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_geo', 'bar', Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_geo', 'bar', Expect.RESPONSE_404.value, True, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_geo', 'bar?bbox=-50.52,-4.46,54.59,1.45', + Expect.RESPONSE_404.value, path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_geo', 'bar?bbox=-50.52,-4.46,54.59,1.45', + Expect.RESPONSE_404.value, True, path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_geo/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_geo/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + + # GET _design/foo/_geo_info/bar + def test_get_geo_info_via_ddoc_attachment(self): + """ + Test GET requests for '_design/foo/_geo_info/bar' + """ + # EXPECTED: 404 + self.get_doc_attachment_variants('_design/foo/_geo_info', 'bar', Expect.RESPONSE_404.value, + path_segment_count=4) + self.get_doc_attachment_variants('_design/foo/_geo_info', 'bar', Expect.RESPONSE_404.value, True, + path_segment_count=4) + # EXPECTED: Validation exception + self.get_doc_attachment_variants('_design/foo', '_geo_info/bar', Expect.VALIDATION_EXCEPTION_ATT.value) + self.get_doc_attachment_variants('_design/foo', '_geo_info/bar', Expect.VALIDATION_EXCEPTION_ATT.value, True) + + # GET _partition/foo + # EXPECTED: Validation exception + def test_get_invalid_partition_info(self): + """ + Test GET requests for '_partition/foo' + """ + self.get_document_variants('_partition/foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET _partition/foo + # EXPECTED: Validation exception + def test_get_invalid_partition_info_via_attachment(self): + """ + Test GET requests for '_partition/foo' + """ + self.get_doc_attachment_variants('_partition', 'foo', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET _partition/foo/_all_docs + # EXPECTED: Validation exception + def test_get_partition_info(self): + """ + Test GET requests for '_partition/foo/_all_docs' + """ + self.get_document_variants('_partition/foo/_all_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + # GET _partition/foo/_all_docs + # EXPECTED: Validation exception + def test_get_invalid_partition_all_docs_via_attachment(self): + """ + Test GET requests for '_partition/foo/_all_docs' + """ + self.get_doc_attachment_variants('_partition', 'foo/_all_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + self.get_doc_attachment_variants('_partition/foo', '_all_docs', Expect.VALIDATION_EXCEPTION_DOCID.value) + + """UTIL FUNCTIONS""" + def mocked_get_requests(self, rev=None, override_status_code=None): + """ + Create a mock GET request for documents with the expected status code + :param rev: the doc's revision (default None) + :param override_status_code: override the status code for handling + inner `fetch` request call within `get_attachment` + :return: mocked Response object + """ + resp_mock = create_autospec(requests.Response) + if override_status_code is not None: + resp_mock.status_code = override_status_code + else: + resp_mock.status_code = self.expected_enum + if (resp_mock.status_code == 200 or resp_mock.status_code == 201 + and self.doc_id is not None): + if rev is not None: + resp_mock.text = f"""{{"_id": "{self.doc_id}", "_rev": "{rev}"}}""" + else: + resp_mock.text = f"""{{"_id": "{self.doc_id}", "_rev": "1-abc"}}""" + elif resp_mock.status_code == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + resp_mock.encoding = None + + return resp_mock + + def mocked_get_att_requests(self): + """ + Create a mock GET request for attachments with the expected status code + """ + self.expected_att_content = f"""this is a text attachment""" + # first fetch doc call with rev + fetch_mock = self.mocked_get_requests(rev=None, override_status_code=200) + # second get to attachment + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + if self.expected_enum == 200 and self.doc_id is not None and self.att_name is not None: + resp_mock.text = self.expected_att_content + if self.expected_enum == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + self.doc_r_session_mock.get.side_effect = [fetch_mock, resp_mock] + + def mocked_head_requests(self, override_status_code=None): + """ + Create a mock HEAD request for documents and attachments with the expected status code + """ + resp_mock = create_autospec(requests.Response) + if override_status_code is not None: + resp_mock.status_code = override_status_code + else: + resp_mock.status_code = self.expected_enum + self.doc_r_session_mock.head = Mock(return_value=resp_mock) + + def mocked_delete_requests(self): + """ + Create a mock DELETE request for documents with the expected status code + """ + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + if self.expected_enum == 201 and self.doc_id is not None: + resp_mock.text = f"""{{"id": "{self.doc_id}", "rev": "2-abc", "ok": true}}""" + self.doc_r_session_mock.delete = Mock(return_value=resp_mock) + + def mocked_delete_att_requests(self): + """ + Create a mock DELETE request for attachments with the expected status code + """ + # first `fetch` document call with rev + self.doc_r_session_mock.get = Mock(return_value=self.mocked_get_requests(rev=None, override_status_code=200)) + # second delete to attachment + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + resp_mock.encoding = None + if self.expected_enum == 200 and self.doc_id is not None and self.att_name is not None: + resp_mock.text = f"""{{"id": "{self.doc_id}", "rev": "2-abc", "ok": true}}""" + elif self.expected_enum == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + + self.doc_r_session_mock.delete = Mock(return_value=resp_mock) + + def mocked_put_doc_requests(self): + """ + Create a mock PUT request for documents with the expected status code + """ + # mock 'doc.exists' request call within 'doc.save' function + self.mocked_head_requests(200) + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + resp_mock.encoding = None + if self.expected_enum == 201 and self.doc_id is not None: + resp_mock.text = f"""{{"id": "{self.doc_id}", "rev": "1-abc", "ok": true}}""" + if self.expected_enum == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + self.doc_r_session_mock.put = Mock(return_value=resp_mock) + + def mocked_put_att_requests(self): + """ + Create a mock PUT request for attachments with the expected status code + """ + # first `fetch` document call within `put_attachment` + fetch_mock = self.mocked_get_requests(rev=None, override_status_code=200) + # create Response object for PUT attachment + resp_mock = create_autospec(requests.Response) + resp_mock.status_code = self.expected_enum + resp_mock.encoding = None + if self.expected_enum == 201 and self.doc_id is not None: + resp_mock.text = f"""{{"id": "{self.doc_id}", "rev": "2-def", "ok": true}}""" + if self.expected_enum == 404: + resp_mock.raise_for_status.side_effect = requests.exceptions.HTTPError + # final fetch doc call + second_fetch_mock = self.mocked_get_requests(rev='2-def', override_status_code=200) + self.doc_r_session_mock.get.side_effect = [fetch_mock, second_fetch_mock] + self.doc_r_session_mock.put = Mock(return_value=resp_mock) + + def get_document_variants(self, doc_id, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute GET/HEAD document requests + """ + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_head_requests() + self.head_document() + self.doc_r_session_mock.get.return_value = self.mocked_get_requests() + self.fetch_document() + self.assert_path_segments(self.doc_r_session_mock.get.call_args_list, path_segment_count) + + def get_doc_attachment_variants(self, doc_id, att_name, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute GET attachment requests + """ + self.att_name = att_name + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_get_att_requests() + self.get_doc_attachment() + self.assert_path_segments(self.doc_r_session_mock.get.call_args_list, path_segment_count) + + def put_document_variants(self, doc_id, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute PUT document requests + """ + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_put_doc_requests() + self.put_document() + self.assert_path_segments(self.doc_r_session_mock.put.call_args_list, path_segment_count) + + def put_doc_attachment_variants(self, doc_id, att_name, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute PUT attachment requests + """ + self.att_name = att_name + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_put_att_requests() + self.put_doc_attachment() + self.assert_path_segments(self.doc_r_session_mock.put.call_args_list, path_segment_count) + + def delete_document_variants(self, doc_id, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute DELETE document requests + """ + self.doc_id = doc_id + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_delete_requests() + self.delete_document() + self.assert_path_segments(self.doc_r_session_mock.delete.call_args_list, path_segment_count) + + def delete_doc_attachment_variants(self, doc_id, attname, expected_enum, is_ddoc=False, + path_segment_count=None): + """ + Function to setup mock requests and execute DELETE attachment requests + """ + self.doc_id = doc_id + self.att_name = attname + self.expected_enum = expected_enum + self.is_ddoc = is_ddoc + self.mocked_delete_att_requests() + self.delete_doc_attachment() + self.assert_path_segments(self.doc_r_session_mock.delete.call_args_list, path_segment_count) + + """HTTP REQUEST FUNCTIONS""" + def head_document(self): + try: + resp = self.create_doc(self.doc_id, self.is_ddoc).exists() + if self.expected_enum == 200 or self.expected_enum == 201: + self.assertTrue(resp) + elif self.expected_enum == 404: + self.assertFalse(resp) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + + def delete_document(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + doc['_rev'] = '1-abc' + doc.delete() + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertFalse('rev' in doc) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def fetch_document(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + doc.fetch() + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertIsNotNone(doc['_rev']) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def put_document(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + doc.save() + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertIsNotNone(doc['_rev']) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def delete_doc_attachment(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + doc['_rev'] = '1-abc' + resp = doc.delete_attachment(self.att_name) + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertEqual(self.doc_id, resp['id']) + self.assertEqual(doc['_rev'], resp['rev']) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def get_doc_attachment(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + resp_att = doc.get_attachment(self.att_name, attachment_type='text') + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, doc['_id']) + self.assertIsNotNone(resp_att) + self.assertEqual(resp_att, self.expected_att_content) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + def put_doc_attachment(self): + try: + doc = self.create_doc(self.doc_id, self.is_ddoc) + resp_att = doc.put_attachment(self.att_name, content_type='utf-8', data='test') + self.assertIsNotNone(resp_att) + self.assertTrue(isinstance(self.expected_enum, int), + f"""Expected value {self.expected_enum} is not an int status code.""") + self.assertTrue(self.expected_enum < 400, + f"""Expected value {self.expected_enum} is not a successful status code.""") + self.assertEqual(self.doc_id, resp_att['id']) + self.assertEqual(resp_att['id'], doc['_id']) + self.assertEqual(doc['_rev'], resp_att['rev']) + self.assertEqual(resp_att['ok'], True) + except CloudantArgumentError as cae: + self.assert_exception_msg(cae) + except requests.exceptions.HTTPError as err: + self.assertTrue(id(self.expected_enum), id(err)) + + """HELPER FUNCTIONS""" + def create_doc(self, doc_id=None, is_ddoc=False): + """ + Function to create and return a Document or DesignDocument object. + """ + if is_ddoc: + if doc_id is not None: + doc = DesignDocument(self.db, doc_id) + else: + doc = DesignDocument(self.db) + elif doc_id is not None: + doc = Document(self.db, doc_id) + else: + doc = Document(self.db) + self.assertIsNone(doc.get('_rev')) + return doc + + def assert_exception_msg(self, cae): + """ + Function to assert whether the exception message is for an invalid + document ID or an attachment name. + """ + self.assertTrue(id(self.expected_enum), id(cae)) + # Check that actual exception message starts with the expected msg + if str(cae).startswith(str(self.expected_enum)): + # Figure out which exception msg to assert against + if str(cae).startswith(ValidationExceptionMsg.ATTACHMENT.value): + self.assertEqual(str(cae), f"""{ValidationExceptionMsg.ATTACHMENT.value} {self.att_name}""") + elif str(cae).startswith(ValidationExceptionMsg.DOC.value): + self.assertEqual(str(cae), f"""{ValidationExceptionMsg.DOC.value} {self.doc_id}""") + else: + self.fail('Expected CloudantArgumentError message should equal actual error message.') + + def assert_path_segments(self, actual_call_args_list, exp_segment_count): + """ + Function to assert the number of path segments from a mock request + """ + # If there's no segment count, verify that the test case expects an argument error + if exp_segment_count is None: + self.assertTrue(isinstance(self.expected_enum, CloudantArgumentError), 'Path segment count should exist ' + 'when testing against valid ' + 'document or attachment names.') + else: + # get latest call in list + url, headers = actual_call_args_list[len(actual_call_args_list) - 1] + # there should only be one mocked url + self.assertEqual(len(url), 1) + # parse path of url and remove first / path segment + path = urlparse(url[0]).path[1:] + actual_segment_count = len(path.split('/')) + self.assertEqual(actual_segment_count, exp_segment_count) From f05f0921716684c4aac0c61bedbe7cb7b0823655 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 26 Aug 2021 10:53:20 +0100 Subject: [PATCH 174/185] Update CHANGES for 2.15.0 --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 66d1f820..d5cf74e5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -# UNRELEASED +# 2.15.0 (2021-08-26) - [NEW] Override `dict.get` method for `CouchDatabase` to add `remote` parameter allowing it to retrieve a remote document if specified. - [FIXED] Fixed the documentation for `bookmarks`. From 760c8c99373b7b02dc32cc2ff994071ed4938997 Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 26 Aug 2021 10:55:33 +0100 Subject: [PATCH 175/185] Update version to 2.15.0 --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 20738ec2..68e69e40 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.14.1-SNAPSHOT +2.15.0 diff --git a/docs/conf.py b/docs/conf.py index 1e338e6d..403f5bd7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.14.1-SNAPSHOT' +version = '2.15.0' # The full version, including alpha/beta/rc tags. -release = '2.14.1-SNAPSHOT' +release = '2.15.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index f3910174..720b8709 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.14.1-SNAPSHOT' +__version__ = '2.15.0' # pylint: disable=wrong-import-position import contextlib From 3bb26d75fa255802a5f308bbf9cff1ba3b34439b Mon Sep 17 00:00:00 2001 From: Rich Ellis Date: Thu, 26 Aug 2021 12:17:13 +0100 Subject: [PATCH 176/185] Update verson to 2.15.1-SNAPSHOT --- VERSION | 2 +- docs/conf.py | 4 ++-- src/cloudant/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 68e69e40..3a7d90b0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.15.0 +2.15.1-SNAPSHOT diff --git a/docs/conf.py b/docs/conf.py index 403f5bd7..c8d4ad26 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,9 +60,9 @@ # built documents. # # The short X.Y version. -version = '2.15.0' +version = '2.15.1-SNAPSHOT' # The full version, including alpha/beta/rc tags. -release = '2.15.0' +release = '2.15.1-SNAPSHOT' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 720b8709..66769dad 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -15,7 +15,7 @@ """ Cloudant / CouchDB Python client library API package """ -__version__ = '2.15.0' +__version__ = '2.15.1-SNAPSHOT' # pylint: disable=wrong-import-position import contextlib From c2ffcd6393e9115acc02c81135424dd9325cdfd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Tue, 21 Dec 2021 10:56:10 +0100 Subject: [PATCH 177/185] Improve migration guide (#509) --- MIGRATION.md | 109 ++++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 53 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 29be9810..21c9ae5f 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -17,6 +17,10 @@ remote HTTP API. For example, in the case of the document context manager, this to fetch and a `put_document` to save. 1. In `cloudant-python-sdk` View, Search, and Query (aka `_find` endpoint) operation responses contain raw JSON content like using `raw_result=True` in `python-cloudant`. +1. Replay adapters are replaced by the [automatic retries](https://github. + com/IBM/ibm-cloud-sdk-common/#automatic-retries) feature for failed requests. +1. Error handling is not transferable from `python-cloudant` to `cloudant-python-sdk`. For more information go to the [Error handling section](https://cloud.ibm.com/apidocs/cloudant?code=python#error-handling) in our API docs. +1. Custom HTTP client configurations in `python-cloudant` are not transferable to `python-java-sdk`. For more information go to the [Configuring the HTTP client section(https://githubcom/IBM/ibm-cloud-sdk-common/#configuring-the-http-client) in the IBM Cloud SDK Common README. ## Request mapping Here's a list of the top 5 most frequently used `python-cloudant` operations and the `cloudant-python-sdk` equivalent API operation documentation link: @@ -44,56 +48,55 @@ The table below contains a list of `python-cloudant` functions and the `cloudant | `python-cloudant` function | `cloudant-python-sdk` API operation documentation link | |-----------------|---------------------| -|`metadata()`|[getServerInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getserverinformation)| -|`all_dbs()`|[getAllDbs](https://cloud.ibm.com/apidocs/cloudant?code=python#getalldbs)| -|`db_updates()/infinite_db_updates()`|[getDbUpdates](https://cloud.ibm.com/apidocs/cloudant?code=python#getdbupdates)| -|`Replicator.stop_replication()`|[deleteReplicationDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletereplicationdocument)| -|`Replicator.replication_state()`|[getReplicationDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getreplicationdocument)| -|`Replicator.create_replication()`|[putReplicationDocument](https://cloud.ibm.com/apidocs/cloudant?code=#putreplicationdocument)| -|`Scheduler.get_doc()`|[getSchedulerDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocument)| -|`Scheduler.list_docs()`|[getSchedulerDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocs)| -|`Scheduler.list_jobs()`|[getSchedulerJobs](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerjobs)| -|`session()`|[getSessionInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getsessioninformation)| -|`uuids()`|[getUuids](https://cloud.ibm.com/apidocs/cloudant?code=python#getuuids)| -|`db.delete()`|[deleteDatabase](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedatabase)| -|`db.metadata()`|[getDatabaseInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getdatabaseinformation)| -|`db.create_document()`|[postDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#postdocument)| -|`db.create()`|[putDatabase](https://cloud.ibm.com/apidocs/cloudant?code=python#putdatabase)| -|`db.all_docs()/db.keys()`|[postAllDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postalldocs)| -|`db.bulk_docs()`|[postBulkDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postbulkdocs)| -|`db.changes()/db.infinite_changes()`|[postChanges](https://cloud.ibm.com/apidocs/cloudant?code=python#postchanges)| -|`DesignDocument(db, '_design/doc').delete()`|[deleteDesignDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedesigndocument)| -|`db.get_design_document()/DesignDocument(db, '_design/doc').fetch()`|[getDesignDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocument)| -|`DesignDocument(db, '_design/doc').save()`|[putDesignDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putdesigndocument)| -|`DesignDocument(db, '_design/doc').info()`|[getDesignDocumentInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocumentinformation)| -|`db.get_search_result()`|[postSearch](https://cloud.ibm.com/apidocs/cloudant?code=python#postsearch)| -|`db.get_view_result()`|[postView](https://cloud.ibm.com/apidocs/cloudant?code=python#postview)| -|`db.list_design_documents()`|[postDesignDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postdesigndocs)| -|`db.get_query_result()`|[postFind](https://cloud.ibm.com/apidocs/cloudant?code=python#postfind)| -|`db.get_query_indexes()`|[getIndexesInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getindexesinformation)| -|`db.create_query_index()`|[postIndex](https://cloud.ibm.com/apidocs/cloudant?code=python#postindex)| -|`db.delete_query_index()`|[deleteIndex](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteindex)| -|`Document(db, '_local/docid').fetch()`|[getLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| -|`Document(db, '_local/docid').save()`|[putLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putlocaldocument)| -|`Document(db, '_local/docid').delete()`|[deleteLocalDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletelocaldocument)| -|`db.missing_revisions()`|[postMissingRevs](https://cloud.ibm.com/apidocs/cloudant?code=python#postmissingrevs)| -|`db.partition_metadata()`|[getPartitionInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getpartitioninformation)| -|`db.partitioned_all_docs()`|[postPartitionAllDocs](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionalldocs)| -|`db.get_partitioned_search_result()`|[postPartitionSearch](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionsearch)| -|`db.get_partitioned_view_result()`|[postPartitionView](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionview)| -|`db.get_partitioned_query_result()`|[postPartitionFind](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionfind)| -|`db.revisions_diff()`|[postRevsDiff](https://cloud.ibm.com/apidocs/cloudant?code=python#postrevsdiff)| -|`db.get_security_document()/db.security_document()`|[getSecurity](https://cloud.ibm.com/apidocs/cloudant?code=python#getsecurity)| -|`db.share_database()`|[putSecurity](https://cloud.ibm.com/apidocs/cloudant?code=python#putsecurity)| -|`db.shards()`|[getShardsInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getshardsinformation)| -|`Document(db, 'docid').delete()`|[deleteDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedocument)| -|`Document(db, 'docid').fetch()`|[getDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#getdocument)| -|`Document(db, 'docid').exists()`|[headDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#headdocument)| -|`Document(db, 'docid').save()`|[putDocument](https://cloud.ibm.com/apidocs/cloudant?code=python#putdocument)| -|`Document(db, 'docid').delete_attachment()`|[deleteAttachment](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteattachment)| -|`Document(db, 'docid').get_attachment()`|[getAttachment](https://cloud.ibm.com/apidocs/cloudant?code=python#getattachment)| -|`Document(db, 'docid').put_attachment()`|[putAttachment](https://cloud.ibm.com/apidocs/cloudant?code=python#putattachment)| -|`generate_api_key()`|[postApiKeys](https://cloud.ibm.com/apidocs/cloudant?code=python#postapikeys)| -|`SecurityDocument().save()`|[putCloudantSecurityConfiguration](https://cloud.ibm.com/apidocs/cloudant?code=python#putcloudantsecurityconfiguration)| -|`cors_configuration()/cors_origin()`|[getCorsInformation](https://cloud.ibm.com/apidocs/cloudant?code=python#getcorsinformation)| -|`update_cors_configuration()`|[putCorsConfiguration](https://cloud.ibm.com/apidocs/cloudant?code=python#putcorsconfiguration)| +|`metadata()`|[`getServerInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getserverinformation)| +|`all_dbs()`|[`getAllDbs`](https://cloud.ibm.com/apidocs/cloudant?code=python#getalldbs)| +|`db_updates()/infinite_db_updates()`|[`getDbUpdates`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdbupdates)| +|`Replicator.stop_replication()`|[`deleteReplicationDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletereplicationdocument)| +|`Replicator.replication_state()`|[`getReplicationDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getreplicationdocument)| +|`Replicator.create_replication()`|[`putReplicationDocument`](https://cloud.ibm.com/apidocs/cloudant?code=#putreplicationdocument)| +|`Scheduler.get_doc()`|[`getSchedulerDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocument)| +|`Scheduler.list_docs()`|[`getSchedulerDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerdocs)| +|`Scheduler.list_jobs()`|[`getSchedulerJobs`](https://cloud.ibm.com/apidocs/cloudant?code=python#getschedulerjobs)| +|`session()`|[`getSessionInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getsessioninformation)| +|`uuids()`|[`getUuids`](https://cloud.ibm.com/apidocs/cloudant?code=python#getuuids)| +|`db.delete()`|[`deleteDatabase`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedatabase)| +|`db.metadata()`|[`getDatabaseInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdatabaseinformation)| +|`db.create_document()`|[`postDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#postdocument)| +|`db.create()`|[`putDatabase`](https://cloud.ibm.com/apidocs/cloudant?code=python#putdatabase)| +|`db.all_docs()/db.keys()`|[`postAllDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#postalldocs)| +|`db.bulk_docs()`|[`postBulkDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#postbulkdocs)| +|`db.changes()/db.infinite_changes()`|[`postChanges`](https://cloud.ibm.com/apidocs/cloudant?code=python#postchanges-databases)| +|`DesignDocument(db, '_design/doc').delete()`|[`deleteDesignDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedesigndocument)| +|`db.get_design_document()/DesignDocument(db, '_design/doc').fetch()`|[`getDesignDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocument)| +|`DesignDocument(db, '_design/doc').save()`|[`putDesignDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#putdesigndocument)| +|`DesignDocument(db, '_design/doc').info()`|[`getDesignDocumentInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdesigndocumentinformation)| +|`db.get_search_result()`|[`postSearch`](https://cloud.ibm.com/apidocs/cloudant?code=python#postsearch)| +|`db.get_view_result()`|[`postView`](https://cloud.ibm.com/apidocs/cloudant?code=python#postview)| +|`db.list_design_documents()`|[`postDesignDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#postdesigndocs)| +|`db.get_query_result()`|[`postFind`](https://cloud.ibm.com/apidocs/cloudant?code=python#postfind)| +|`db.get_query_indexes()`|[`getIndexesInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getindexesinformation)| +|`db.create_query_index()`|[`postIndex`](https://cloud.ibm.com/apidocs/cloudant?code=python#postindex)| +|`db.delete_query_index()`|[`deleteIndex`](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteindex)| +|`Document(db, '_local/docid').fetch()`|[`getLocalDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getlocaldocument)| +|`Document(db, '_local/docid').save()`|[`putLocalDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#putlocaldocument)| +|`Document(db, '_local/docid').delete()`|[`deleteLocalDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletelocaldocument)| +|`db.missing_revisions()/db.revisions_diff()`|[`postRevsDiff`](https://cloud.ibm.com/apidocs/cloudant?code=python#postrevsdiff)| +|`db.partition_metadata()`|[`getPartitionInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getpartitioninformation)| +|`db.partitioned_all_docs()`|[`postPartitionAllDocs`](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionalldocs)| +|`db.get_partitioned_search_result()`|[`postPartitionSearch`](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionsearch)| +|`db.get_partitioned_view_result()`|[`postPartitionView`](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionview)| +|`db.get_partitioned_query_result()`|[`postPartitionFind`](https://cloud.ibm.com/apidocs/cloudant?code=python#postpartitionfind-partitioned-databases)| +|`db.get_security_document()/db.security_document()`|[`getSecurity`](https://cloud.ibm.com/apidocs/cloudant?code=python#getsecurity)| +|`db.share_database()`|[`putSecurity`](https://cloud.ibm.com/apidocs/cloudant?code=python#putsecurity)| +|`db.shards()`|[`getShardsInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getshardsinformation)| +|`Document(db, 'docid').delete()`|[`deleteDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#deletedocument)| +|`Document(db, 'docid').fetch()`|[`getDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#getdocument)| +|`Document(db, 'docid').exists()`|[`headDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#headdocument)| +|`Document(db, 'docid').save()`|[`putDocument`](https://cloud.ibm.com/apidocs/cloudant?code=python#putdocument)| +|`Document(db, 'docid').delete_attachment()`|[`deleteAttachment`](https://cloud.ibm.com/apidocs/cloudant?code=python#deleteattachment)| +|`Document(db, 'docid').get_attachment()`|[`getAttachment`](https://cloud.ibm.com/apidocs/cloudant?code=python#getattachment)| +|`Document(db, 'docid').put_attachment()`|[`putAttachment`](https://cloud.ibm.com/apidocs/cloudant?code=python#putattachment)| +|`generate_api_key()`|[`postApiKeys`](https://cloud.ibm.com/apidocs/cloudant?code=python#postapikeys)| +|`SecurityDocument().save()`|[`putCloudantSecurityConfiguration`](https://cloud.ibm.com/apidocs/cloudant?code=python#putcloudantsecurity)| +|`cors_configuration()/cors_origin()`|[`getCorsInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getcorsinformation)| +|`update_cors_configuration()`|[`putCorsConfiguration`](https://cloud.ibm.com/apidocs/cloudant?code=python#putcorsconfiguration)| From a92c6593ba5db69fda64b462a9847d7e06cc9cf4 Mon Sep 17 00:00:00 2001 From: Ildiko Eigel Date: Tue, 21 Dec 2021 10:58:05 +0100 Subject: [PATCH 178/185] Remove assertion from test_get_feed_descending --- tests/unit/changes_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index d39ab159..8d937214 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -245,7 +245,6 @@ def test_get_feed_descending(self): self.assertTrue(current < last) except AssertionError: self.assertEqual(current, last) - self.assertTrue(len(change['seq']) > len(last_seq)) seq_list.append(change['seq']) last_seq = change['seq'] self.assertEqual(len(seq_list), 50) @@ -522,7 +521,7 @@ def test_invalid_style_value(self): with self.assertRaises(CloudantArgumentError) as cm: invalid_feed = [x for x in feed] self.assertEqual( - str(cm.exception), + str(cm.exception), 'Invalid value (foo) for style option. Must be main_only, or all_docs.') if __name__ == '__main__': From e113b8f0da43d5cb9f90a84502eb5cf32d3ba1de Mon Sep 17 00:00:00 2001 From: Ildiko Eigel Date: Tue, 21 Dec 2021 10:58:54 +0100 Subject: [PATCH 179/185] Update date in copyright header --- tests/unit/changes_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index 8d937214..5ec2f02e 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2016, 2018 IBM Corp. All rights reserved. +# Copyright (C) 2016, 2021 IBM Corp. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 8d968f4c048a320c2398436333b225d792837762 Mon Sep 17 00:00:00 2001 From: Ildiko Eigel Date: Tue, 21 Dec 2021 11:34:34 +0100 Subject: [PATCH 180/185] Update comment --- tests/unit/changes_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index 5ec2f02e..b0e9cb3e 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -227,8 +227,7 @@ def test_get_feed_descending(self): Test getting content back for a descending feed. When testing, the sequence identifier is in the form of -. Often times the number prefix sorts as expected when using descending but sometimes the - number prefix is repeated. In these cases the check is to see if the following - random character sequence suffix is longer than its predecessor. + number prefix is repeated. """ self.populate_db_with_documents(50) feed = Feed(self.db, descending=True) From ab941812c84cfbcf90c85ae63fb034fbe136780d Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Wed, 22 Dec 2021 14:19:22 +0000 Subject: [PATCH 181/185] Use assertEqual instead of assertEquals for Python 3.11 compatibility. --- tests/unit/changes_tests.py | 6 +- tests/unit/client_tests.py | 16 +-- tests/unit/database_partition_tests.py | 22 ++--- tests/unit/database_tests.py | 130 ++++++++++++------------- tests/unit/design_document_tests.py | 6 +- tests/unit/document_tests.py | 8 +- tests/unit/iam_auth_tests.py | 4 +- tests/unit/index_tests.py | 120 +++++++++++------------ tests/unit/replicator_mock_tests.py | 12 +-- 9 files changed, 162 insertions(+), 162 deletions(-) diff --git a/tests/unit/changes_tests.py b/tests/unit/changes_tests.py index b0e9cb3e..3df0cef3 100644 --- a/tests/unit/changes_tests.py +++ b/tests/unit/changes_tests.py @@ -474,9 +474,9 @@ def test_get_feed_with_custom_filter_query_params(self): include_docs=False ) params = feed._translate(feed._options) - self.assertEquals(params['filter'], 'mailbox/new_mail') - self.assertEquals(params['foo'], 'bar') - self.assertEquals(params['include_docs'], 'false') + self.assertEqual(params['filter'], 'mailbox/new_mail') + self.assertEqual(params['foo'], 'bar') + self.assertEqual(params['include_docs'], 'false') def test_invalid_argument_type(self): """ diff --git a/tests/unit/client_tests.py b/tests/unit/client_tests.py index 07c4159b..35139168 100644 --- a/tests/unit/client_tests.py +++ b/tests/unit/client_tests.py @@ -266,7 +266,7 @@ def test_session_basic(self, m_req): timeout=None ) - self.assertEquals(all_dbs, ['animaldb']) + self.assertEqual(all_dbs, ['animaldb']) @mock.patch('cloudant._client_session.Session.request') def test_session_basic_with_no_credentials(self, m_req): @@ -333,7 +333,7 @@ def test_change_credentials_basic(self, m_req): auth=('baz', 'qux'), # uses HTTP Basic Auth timeout=None ) - self.assertEquals(all_dbs, ['animaldb']) + self.assertEqual(all_dbs, ['animaldb']) @skip_if_not_cookie_auth def test_basic_auth_str(self): @@ -442,7 +442,7 @@ def test_create_with_server_error(self, m_req): self.assertEqual(cm.exception.status_code, 500) - self.assertEquals(m_req.call_count, 3) + self.assertEqual(m_req.call_count, 3) m_req.assert_called_with( 'PUT', '/'.join([self.url, dbname]), @@ -730,7 +730,7 @@ def test_cloudant_bluemix_context_helper_with_legacy_creds(self): with cloudant_bluemix(vcap_services, instance_name=instance_name) as c: self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) - self.assertEquals(c.session()['userCtx']['name'], self.user) + self.assertEqual(c.session()['userCtx']['name'], self.user) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) @@ -809,7 +809,7 @@ def test_cloudant_bluemix_dedicated_context_helper(self): service_name=service_name) as c: self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) - self.assertEquals(c.session()['userCtx']['name'], self.user) + self.assertEqual(c.session()['userCtx']['name'], self.user) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) @@ -850,7 +850,7 @@ def test_bluemix_constructor_with_legacy_creds(self): c.connect() self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) - self.assertEquals(c.session()['userCtx']['name'], self.user) + self.assertEqual(c.session()['userCtx']['name'], self.user) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) @@ -916,7 +916,7 @@ def test_bluemix_constructor_specify_instance_name(self): c.connect() self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) - self.assertEquals(c.session()['userCtx']['name'], self.user) + self.assertEqual(c.session()['userCtx']['name'], self.user) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) @@ -960,7 +960,7 @@ def test_bluemix_constructor_with_multiple_services(self): c.connect() self.assertIsInstance(c, Cloudant) self.assertIsInstance(c.r_session, requests.Session) - self.assertEquals(c.session()['userCtx']['name'], self.user) + self.assertEqual(c.session()['userCtx']['name'], self.user) except Exception as err: self.fail('Exception {0} was raised.'.format(str(err))) diff --git a/tests/unit/database_partition_tests.py b/tests/unit/database_partition_tests.py index 52b76e0d..8b3690dd 100644 --- a/tests/unit/database_partition_tests.py +++ b/tests/unit/database_partition_tests.py @@ -63,7 +63,7 @@ def test_create_non_partitioned_design_document(self): def test_partitioned_all_docs(self): for partition_key in self.populate_db_with_partitioned_documents(5, 25): docs = self.db.partitioned_all_docs(partition_key) - self.assertEquals(len(docs['rows']), 25) + self.assertEqual(len(docs['rows']), 25) for doc in docs['rows']: self.assertTrue(doc['id'].startswith(partition_key + ':')) @@ -71,8 +71,8 @@ def test_partitioned_all_docs(self): def test_partition_metadata(self): for partition_key in self.populate_db_with_partitioned_documents(5, 25): meta = self.db.partition_metadata(partition_key) - self.assertEquals(meta['partition'], partition_key) - self.assertEquals(meta['doc_count'], 25) + self.assertEqual(meta['partition'], partition_key) + self.assertEqual(meta['doc_count'], 25) def test_partitioned_search(self): ddoc = DesignDocument(self.db, 'partitioned_search', partitioned=True) @@ -91,7 +91,7 @@ def test_partitioned_search(self): print(result) self.assertTrue(result['id'].startswith(partition_key + ':')) i += 1 - self.assertEquals(i, 10) + self.assertEqual(i, 10) def test_get_partitioned_index(self): index_name = 'test_partitioned_index' @@ -99,16 +99,16 @@ def test_get_partitioned_index(self): self.db.create_query_index(index_name=index_name, fields=['foo']) results = self.db.get_query_indexes() - self.assertEquals(len(results), 2) + self.assertEqual(len(results), 2) index_all_docs = results[0] - self.assertEquals(index_all_docs.name, '_all_docs') - self.assertEquals(type(index_all_docs), SpecialIndex) + self.assertEqual(index_all_docs.name, '_all_docs') + self.assertEqual(type(index_all_docs), SpecialIndex) self.assertFalse(index_all_docs.partitioned) index_partitioned = results[1] - self.assertEquals(index_partitioned.name, index_name) - self.assertEquals(type(index_partitioned), Index) + self.assertEqual(index_partitioned.name, index_name) + self.assertEqual(type(index_partitioned), Index) self.assertTrue(index_partitioned.partitioned) def test_partitioned_query(self): @@ -122,7 +122,7 @@ def test_partitioned_query(self): for result in results: self.assertTrue(result['_id'].startswith(partition_key + ':')) i += 1 - self.assertEquals(i, 10) + self.assertEqual(i, 10) def test_partitioned_view(self): ddoc = DesignDocument(self.db, 'partitioned_view', partitioned=True) @@ -138,4 +138,4 @@ def test_partitioned_view(self): self.assertTrue( result['id'].startswith(partition_key + ':')) i += 1 - self.assertEquals(i, 10) + self.assertEqual(i, 10) diff --git a/tests/unit/database_tests.py b/tests/unit/database_tests.py index 3ba12584..ae9898c7 100644 --- a/tests/unit/database_tests.py +++ b/tests/unit/database_tests.py @@ -885,7 +885,7 @@ def test_get_set_revision_limit(self, m_req): # Get new revisions limit. self.assertEqual(self.db.get_revision_limit(), 1234) - self.assertEquals(m_req.call_count, 3) + self.assertEqual(m_req.call_count, 3) @attr(db='couch') def test_view_clean_up(self): @@ -1084,19 +1084,19 @@ def test_create_json_index(self): ddoc = self.db[index.design_document_id] - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['indexes'], {}) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) index = ddoc['views'][index.name] - self.assertEquals(index['map']['fields']['age'], 'asc') - self.assertEquals(index['map']['fields']['name'], 'asc') - self.assertEquals(index['options']['def']['fields'], ['name', 'age']) - self.assertEquals(index['reduce'], '_count') + self.assertEqual(index['map']['fields']['age'], 'asc') + self.assertEqual(index['map']['fields']['name'], 'asc') + self.assertEqual(index['options']['def']['fields'], ['name', 'age']) + self.assertEqual(index['reduce'], '_count') @attr(couchapi=2) def test_delete_json_index(self): @@ -1369,22 +1369,22 @@ def test_create_text_index(self): ddoc = self.db[index.design_document_id] - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) - self.assertEquals(ddoc['views'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertEqual(ddoc['views'], {}) text_index = ddoc['indexes'][index.name] - self.assertEquals(text_index['analyzer']['default'], 'keyword') - self.assertEquals(text_index['analyzer']['fields']['$default'], 'standard') - self.assertEquals(text_index['analyzer']['name'], 'perfield') - self.assertEquals(text_index['index']['default_analyzer'], 'keyword') - self.assertEquals(text_index['index']['default_field'], {}) - self.assertEquals(text_index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) - self.assertEquals(text_index['index']['selector'], {}) + self.assertEqual(text_index['analyzer']['default'], 'keyword') + self.assertEqual(text_index['analyzer']['fields']['$default'], 'standard') + self.assertEqual(text_index['analyzer']['name'], 'perfield') + self.assertEqual(text_index['index']['default_analyzer'], 'keyword') + self.assertEqual(text_index['index']['default_field'], {}) + self.assertEqual(text_index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) + self.assertEqual(text_index['index']['selector'], {}) self.assertTrue(text_index['index']['index_array_lengths']) def test_create_all_fields_text_index(self): @@ -1396,22 +1396,22 @@ def test_create_all_fields_text_index(self): ddoc = self.db[index.design_document_id] - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) - self.assertEquals(ddoc['views'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertEqual(ddoc['views'], {}) index = ddoc['indexes'][index.name] - self.assertEquals(index['analyzer']['default'], 'keyword') - self.assertEquals(index['analyzer']['fields'], {'$default': 'standard'}) - self.assertEquals(index['analyzer']['name'], 'perfield') - self.assertEquals(index['index']['default_analyzer'], 'keyword') - self.assertEquals(index['index']['default_field'], {}) - self.assertEquals(index['index']['fields'], 'all_fields') - self.assertEquals(index['index']['selector'], {}) + self.assertEqual(index['analyzer']['default'], 'keyword') + self.assertEqual(index['analyzer']['fields'], {'$default': 'standard'}) + self.assertEqual(index['analyzer']['name'], 'perfield') + self.assertEqual(index['index']['default_analyzer'], 'keyword') + self.assertEqual(index['index']['default_field'], {}) + self.assertEqual(index['index']['fields'], 'all_fields') + self.assertEqual(index['index']['selector'], {}) self.assertTrue(index['index']['index_array_lengths']) def test_create_multiple_indexes_one_ddoc(self): @@ -1436,27 +1436,27 @@ def test_create_multiple_indexes_one_ddoc(self): ddoc = self.db['_design/ddoc001'] - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('2-')) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) json_index = ddoc['views']['json-index-001'] - self.assertEquals(json_index['map']['fields']['age'], 'asc') - self.assertEquals(json_index['map']['fields']['name'], 'asc') - self.assertEquals(json_index['options']['def']['fields'], ['name', 'age']) - self.assertEquals(json_index['reduce'], '_count') + self.assertEqual(json_index['map']['fields']['age'], 'asc') + self.assertEqual(json_index['map']['fields']['name'], 'asc') + self.assertEqual(json_index['options']['def']['fields'], ['name', 'age']) + self.assertEqual(json_index['reduce'], '_count') text_index = ddoc['indexes']['text-index-001'] - self.assertEquals(text_index['analyzer']['default'], 'keyword') - self.assertEquals(text_index['analyzer']['fields']['$default'], 'standard') - self.assertEquals(text_index['analyzer']['name'], 'perfield') - self.assertEquals(text_index['index']['default_analyzer'], 'keyword') - self.assertEquals(text_index['index']['default_field'], {}) - self.assertEquals(text_index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) - self.assertEquals(text_index['index']['selector'], {}) + self.assertEqual(text_index['analyzer']['default'], 'keyword') + self.assertEqual(text_index['analyzer']['fields']['$default'], 'standard') + self.assertEqual(text_index['analyzer']['name'], 'perfield') + self.assertEqual(text_index['index']['default_analyzer'], 'keyword') + self.assertEqual(text_index['index']['default_field'], {}) + self.assertEqual(text_index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) + self.assertEqual(text_index['index']['selector'], {}) self.assertTrue(text_index['index']['index_array_lengths']) def test_create_query_index_failure(self): @@ -1513,28 +1513,28 @@ def test_get_query_indexes_raw(self): indexes = self.db.get_query_indexes(raw_result=True) - self.assertEquals(indexes['total_rows'], 3) + self.assertEqual(indexes['total_rows'], 3) all_docs_index = indexes['indexes'][0] - self.assertEquals(all_docs_index['ddoc'], None) - self.assertEquals(all_docs_index['def']['fields'], [{'_id': 'asc'}]) - self.assertEquals(all_docs_index['name'], '_all_docs') - self.assertEquals(all_docs_index['type'], 'special') + self.assertEqual(all_docs_index['ddoc'], None) + self.assertEqual(all_docs_index['def']['fields'], [{'_id': 'asc'}]) + self.assertEqual(all_docs_index['name'], '_all_docs') + self.assertEqual(all_docs_index['type'], 'special') json_index = indexes['indexes'][1] - self.assertEquals(json_index['ddoc'], '_design/ddoc001') - self.assertEquals(json_index['def']['fields'], [{'name': 'asc'}, {'age': 'asc'}]) - self.assertEquals(json_index['name'], 'json-idx-001') - self.assertEquals(json_index['type'], 'json') + self.assertEqual(json_index['ddoc'], '_design/ddoc001') + self.assertEqual(json_index['def']['fields'], [{'name': 'asc'}, {'age': 'asc'}]) + self.assertEqual(json_index['name'], 'json-idx-001') + self.assertEqual(json_index['type'], 'json') text_index = indexes['indexes'][2] - self.assertEquals(text_index['ddoc'], '_design/ddoc001') - self.assertEquals(text_index['def']['default_analyzer'], 'keyword') - self.assertEquals(text_index['def']['default_field'], {}) - self.assertEquals(text_index['def']['fields'], []) - self.assertEquals(text_index['def']['selector'], {}) - self.assertEquals(text_index['name'], 'text-idx-001') - self.assertEquals(text_index['type'], 'text') + self.assertEqual(text_index['ddoc'], '_design/ddoc001') + self.assertEqual(text_index['def']['default_analyzer'], 'keyword') + self.assertEqual(text_index['def']['default_field'], {}) + self.assertEqual(text_index['def']['fields'], []) + self.assertEqual(text_index['def']['selector'], {}) + self.assertEqual(text_index['name'], 'text-idx-001') + self.assertEqual(text_index['type'], 'text') self.assertTrue(text_index['def']['index_array_lengths']) def test_get_query_indexes(self): diff --git a/tests/unit/design_document_tests.py b/tests/unit/design_document_tests.py index 52352bcc..86a2b5f1 100644 --- a/tests/unit/design_document_tests.py +++ b/tests/unit/design_document_tests.py @@ -843,9 +843,9 @@ def test_get_search_info(self): # Validate the metadata search_index_metadata = search_info['search_index'] self.assertIsNotNone(search_index_metadata) - self.assertEquals(search_index_metadata['doc_del_count'], 0, 'There should be no deleted docs.') + self.assertEqual(search_index_metadata['doc_del_count'], 0, 'There should be no deleted docs.') self.assertTrue(search_index_metadata['doc_count'] <= 100, 'There should be 100 or fewer docs.') - self.assertEquals(search_index_metadata['committed_seq'], 0, 'The committed_seq should be 0.') + self.assertEqual(search_index_metadata['committed_seq'], 0, 'The committed_seq should be 0.') self.assertTrue(search_index_metadata['pending_seq'] <= 101, 'The pending_seq should be 101 or fewer.') self.assertTrue(search_index_metadata['disk_size'] >0, 'The disk_size should be greater than 0.') @@ -1275,7 +1275,7 @@ def test_rewrite_rule(self): doc = Document(self.db, 'rewrite_doc') doc.save() resp = self.client.r_session.get('/'.join([ddoc.document_url, '_rewrite'])) - self.assertEquals( + self.assertEqual( response_to_json_dict(resp), { '_id': 'rewrite_doc', diff --git a/tests/unit/document_tests.py b/tests/unit/document_tests.py index 61fa9fac..09cb09c4 100644 --- a/tests/unit/document_tests.py +++ b/tests/unit/document_tests.py @@ -944,14 +944,14 @@ def object_hook(self, obj): raw_doc = self.db.all_docs(include_docs=True)['rows'][0]['doc'] - self.assertEquals(raw_doc['name'], 'julia') - self.assertEquals(raw_doc['dt']['_type'], 'datetime') - self.assertEquals(raw_doc['dt']['value'], '2018-07-09T15:11:10') + self.assertEqual(raw_doc['name'], 'julia') + self.assertEqual(raw_doc['dt']['_type'], 'datetime') + self.assertEqual(raw_doc['dt']['value'], '2018-07-09T15:11:10') doc2 = Document(self.db, doc['_id'], decoder=DTDecoder) doc2.fetch() - self.assertEquals(doc2['dt'], doc['dt']) + self.assertEqual(doc2['dt'], doc['dt']) if __name__ == '__main__': unittest.main() diff --git a/tests/unit/iam_auth_tests.py b/tests/unit/iam_auth_tests.py index f36af4e0..3e0c7cf2 100644 --- a/tests/unit/iam_auth_tests.py +++ b/tests/unit/iam_auth_tests.py @@ -88,12 +88,12 @@ def _mock_cookie(expires_secs=300): def test_iam_set_credentials(self): iam = IAMSession(MOCK_API_KEY, 'http://127.0.0.1:5984') - self.assertEquals(iam._api_key, MOCK_API_KEY) + self.assertEqual(iam._api_key, MOCK_API_KEY) new_api_key = 'some_new_api_key' iam.set_credentials(None, new_api_key) - self.assertEquals(iam._api_key, new_api_key) + self.assertEqual(iam._api_key, new_api_key) @mock.patch('cloudant._client_session.ClientSession.request') def test_iam_get_access_token(self, m_req): diff --git a/tests/unit/index_tests.py b/tests/unit/index_tests.py index 29a3d51c..f12cf158 100644 --- a/tests/unit/index_tests.py +++ b/tests/unit/index_tests.py @@ -161,21 +161,21 @@ def test_create_an_index_using_ddoc_index_name(self): with DesignDocument(self.db, index.design_document_id) as ddoc: self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['indexes'], {}) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) self.assertListEqual(list(ddoc['views'].keys()), ['index001']) view = ddoc['views'][index.name] - self.assertEquals(view['map']['fields']['age'], 'asc') - self.assertEquals(view['map']['fields']['name'], 'asc') - self.assertEquals(view['options']['def']['fields'], ['name', 'age']) - self.assertEquals(view['reduce'], '_count') + self.assertEqual(view['map']['fields']['age'], 'asc') + self.assertEqual(view['map']['fields']['name'], 'asc') + self.assertEqual(view['options']['def']['fields'], ['name', 'age']) + self.assertEqual(view['reduce'], '_count') def test_create_an_index_without_ddoc_index_name(self): """ @@ -189,21 +189,21 @@ def test_create_an_index_without_ddoc_index_name(self): with DesignDocument(self.db, index.design_document_id) as ddoc: self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['indexes'], {}) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) self.assertListEqual(list(ddoc['views'].keys()), [index.name]) view = ddoc['views'][index.name] - self.assertEquals(view['map']['fields']['age'], 'asc') - self.assertEquals(view['map']['fields']['name'], 'asc') - self.assertEquals(view['options']['def']['fields'], ['name', 'age']) - self.assertEquals(view['reduce'], '_count') + self.assertEqual(view['map']['fields']['age'], 'asc') + self.assertEqual(view['map']['fields']['name'], 'asc') + self.assertEqual(view['options']['def']['fields'], ['name', 'age']) + self.assertEqual(view['reduce'], '_count') def test_create_an_index_with_empty_ddoc_index_name(self): """ @@ -217,21 +217,21 @@ def test_create_an_index_with_empty_ddoc_index_name(self): with DesignDocument(self.db, index.design_document_id) as ddoc: self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['indexes'], {}) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) self.assertListEqual(list(ddoc['views'].keys()), [index.name]) view = ddoc['views'][index.name] - self.assertEquals(view['map']['fields']['age'], 'asc') - self.assertEquals(view['map']['fields']['name'], 'asc') - self.assertEquals(view['options']['def']['fields'], ['name', 'age']) - self.assertEquals(view['reduce'], '_count') + self.assertEqual(view['map']['fields']['age'], 'asc') + self.assertEqual(view['map']['fields']['name'], 'asc') + self.assertEqual(view['options']['def']['fields'], ['name', 'age']) + self.assertEqual(view['reduce'], '_count') def test_create_an_index_using_design_prefix(self): """ @@ -245,21 +245,21 @@ def test_create_an_index_using_design_prefix(self): with DesignDocument(self.db, index.design_document_id) as ddoc: self.assertIsInstance(ddoc.get_view(index.name), QueryIndexView) - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['indexes'], {}) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) + self.assertEqual(ddoc['indexes'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) self.assertListEqual(list(ddoc['views'].keys()), [index.name]) view = ddoc['views'][index.name] - self.assertEquals(view['map']['fields']['age'], 'asc') - self.assertEquals(view['map']['fields']['name'], 'asc') - self.assertEquals(view['options']['def']['fields'], ['name', 'age']) - self.assertEquals(view['reduce'], '_count') + self.assertEqual(view['map']['fields']['age'], 'asc') + self.assertEqual(view['map']['fields']['name'], 'asc') + self.assertEqual(view['options']['def']['fields'], ['name', 'age']) + self.assertEqual(view['reduce'], '_count') def test_create_uses_custom_encoder(self): """ @@ -446,22 +446,22 @@ def test_create_a_search_index_no_kwargs(self): self.assertEqual(index.design_document_id, '_design/ddoc001') self.assertEqual(index.name, 'index001') with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) - self.assertEquals(ddoc['views'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertEqual(ddoc['views'], {}) index = ddoc['indexes']['index001'] - self.assertEquals(index['analyzer']['default'], 'keyword') - self.assertEquals(index['analyzer']['fields']['$default'], 'standard') - self.assertEquals(index['analyzer']['name'], 'perfield') - self.assertEquals(index['index']['default_analyzer'], 'keyword') - self.assertEquals(index['index']['default_field'], {}) - self.assertEquals(index['index']['fields'], 'all_fields') - self.assertEquals(index['index']['selector'], {}) + self.assertEqual(index['analyzer']['default'], 'keyword') + self.assertEqual(index['analyzer']['fields']['$default'], 'standard') + self.assertEqual(index['analyzer']['name'], 'perfield') + self.assertEqual(index['index']['default_analyzer'], 'keyword') + self.assertEqual(index['index']['default_field'], {}) + self.assertEqual(index['index']['fields'], 'all_fields') + self.assertEqual(index['index']['selector'], {}) self.assertTrue(index['index']['index_array_lengths']) def test_create_a_search_index_with_kwargs(self): @@ -480,22 +480,22 @@ def test_create_a_search_index_with_kwargs(self): self.assertEqual(index.design_document_id, '_design/ddoc001') self.assertEqual(index.name, 'index001') with DesignDocument(self.db, index.design_document_id) as ddoc: - self.assertEquals(ddoc['_id'], index.design_document_id) + self.assertEqual(ddoc['_id'], index.design_document_id) self.assertTrue(ddoc['_rev'].startswith('1-')) - self.assertEquals(ddoc['language'], 'query') - self.assertEquals(ddoc['lists'], {}) - self.assertEquals(ddoc['shows'], {}) - self.assertEquals(ddoc['views'], {}) + self.assertEqual(ddoc['language'], 'query') + self.assertEqual(ddoc['lists'], {}) + self.assertEqual(ddoc['shows'], {}) + self.assertEqual(ddoc['views'], {}) index = ddoc['indexes']['index001'] - self.assertEquals(index['analyzer']['default'], 'keyword') - self.assertEquals(index['analyzer']['fields']['$default'], 'german') - self.assertEquals(index['analyzer']['name'], 'perfield') - self.assertEquals(index['index']['default_analyzer'], 'keyword') - self.assertEquals(index['index']['default_field']['analyzer'], 'german') - self.assertEquals(index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) - self.assertEquals(index['index']['selector'], {}) + self.assertEqual(index['analyzer']['default'], 'keyword') + self.assertEqual(index['analyzer']['fields']['$default'], 'german') + self.assertEqual(index['analyzer']['name'], 'perfield') + self.assertEqual(index['index']['default_analyzer'], 'keyword') + self.assertEqual(index['index']['default_field']['analyzer'], 'german') + self.assertEqual(index['index']['fields'], [{'name': 'name', 'type': 'string'}, {'name': 'age', 'type': 'number'}]) + self.assertEqual(index['index']['selector'], {}) self.assertTrue(index['index']['default_field']['enabled']) self.assertTrue(index['index']['index_array_lengths']) diff --git a/tests/unit/replicator_mock_tests.py b/tests/unit/replicator_mock_tests.py index 96589f9e..f49b04d4 100644 --- a/tests/unit/replicator_mock_tests.py +++ b/tests/unit/replicator_mock_tests.py @@ -86,9 +86,9 @@ def test_using_admin_party_source_and_target(self): rep.create_replication(src, tgt, repl_id=self.repl_id) kcall = m_replicator.create_document.call_args_list - self.assertEquals(len(kcall), 1) + self.assertEqual(len(kcall), 1) args, kwargs = kcall[0] - self.assertEquals(len(args), 1) + self.assertEqual(len(args), 1) expected_doc = { '_id': self.repl_id, @@ -118,9 +118,9 @@ def test_using_basic_auth_source_and_target(self): src, tgt, repl_id=self.repl_id, user_ctx=self.user_ctx) kcall = m_replicator.create_document.call_args_list - self.assertEquals(len(kcall), 1) + self.assertEqual(len(kcall), 1) args, kwargs = kcall[0] - self.assertEquals(len(args), 1) + self.assertEqual(len(args), 1) expected_doc = { '_id': self.repl_id, @@ -154,9 +154,9 @@ def test_using_iam_auth_source_and_target(self): src, tgt, repl_id=self.repl_id, user_ctx=self.user_ctx) kcall = m_replicator.create_document.call_args_list - self.assertEquals(len(kcall), 1) + self.assertEqual(len(kcall), 1) args, kwargs = kcall[0] - self.assertEquals(len(args), 1) + self.assertEqual(len(args), 1) expected_doc = { '_id': self.repl_id, From 5b1ecc215b2caea22ccc2d7310462df56be6e848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Tue, 11 Jan 2022 14:21:54 +0100 Subject: [PATCH 182/185] [MIGRATION] Add note about authentication errors and server errors (#512) --- MIGRATION.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index 21c9ae5f..a122d7dd 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -20,8 +20,21 @@ content like using `raw_result=True` in `python-cloudant`. 1. Replay adapters are replaced by the [automatic retries](https://github. com/IBM/ibm-cloud-sdk-common/#automatic-retries) feature for failed requests. 1. Error handling is not transferable from `python-cloudant` to `cloudant-python-sdk`. For more information go to the [Error handling section](https://cloud.ibm.com/apidocs/cloudant?code=python#error-handling) in our API docs. -1. Custom HTTP client configurations in `python-cloudant` are not transferable to `python-java-sdk`. For more information go to the [Configuring the HTTP client section(https://githubcom/IBM/ibm-cloud-sdk-common/#configuring-the-http-client) in the IBM Cloud SDK Common README. - +1. Custom HTTP client configurations in `python-cloudant` can be set differently in + `cloudant-python-sdk`. For more information go to the + [Configuring the HTTP client section](https://github.com/IBM/ibm-cloud-sdk-common/#configuring-the-http-client) + in the IBM Cloud SDK Common README. + +### Troubleshooting +1. Authentication errors occur during service instantiation. For example, the code `service = + CloudantV1.new_instance(service_name="EXAMPLE")` will fail with `ValueError: At least one of + iam_profile_name or iam_profile_id must be specified.` if required environment variables + prefixed with `EXAMPLE` are not set. +1. Server errors occur when running a request against the service. We suggest to + check server errors with + [`getServerInformation`](https://cloud.ibm.com/apidocs/cloudant?code=python#getserverinformation) + which is the new alternative of `metadata()`. + ## Request mapping Here's a list of the top 5 most frequently used `python-cloudant` operations and the `cloudant-python-sdk` equivalent API operation documentation link: From fe141fcc3b3a3614e9b780317f3fb6efdebceb99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eigel=20Ildik=C3=B3?= Date: Thu, 27 Jan 2022 15:59:05 +0100 Subject: [PATCH 183/185] doc: note full deprecation (#513) --- CHANGES.md | 3 +++ README.md | 7 +++---- src/cloudant/__init__.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d5cf74e5..75402ba5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,6 @@ +# UNRELEASED +- [DEPRECATED] This library is end-of-life and no longer supported. + # 2.15.0 (2021-08-26) - [NEW] Override `dict.get` method for `CouchDatabase` to add `remote` parameter allowing it to retrieve a remote document if specified. diff --git a/README.md b/README.md index 029edf07..467c9dd3 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ -# DEPRECATED +# :warning: NO LONGER MAINTAINED :warning: -**This library is now deprecated and will be end-of-life on Dec 31 2021.** +**This library is end-of-life and no longer supported.** -The library remains supported until the end-of-life date, -but will receive only _critical_ maintenance updates. +This repository will not be updated. The repository will be kept available in read-only mode. Please see the [Migration Guide](./MIGRATION.md) for advice about migrating to our replacement library diff --git a/src/cloudant/__init__.py b/src/cloudant/__init__.py index 66769dad..8e4d7e33 100644 --- a/src/cloudant/__init__.py +++ b/src/cloudant/__init__.py @@ -24,7 +24,7 @@ from .client import Cloudant, CouchDB from ._common_util import CloudFoundryService -warnings.warn('The module cloudant is now deprecated. The replacement is ibmcloudant.', +warnings.warn('The module cloudant is now end-of-life. The replacement is ibmcloudant.', DeprecationWarning) @contextlib.contextmanager From 3636169ef2212f70668c70d169533653817a9ae7 Mon Sep 17 00:00:00 2001 From: Ivan Date: Wed, 9 Mar 2022 10:17:33 +0000 Subject: [PATCH 184/185] Update MIGRATION.md Fixing small typo that prevented a markdown link to render properly. --- MIGRATION.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MIGRATION.md b/MIGRATION.md index a122d7dd..3323e1d8 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -17,8 +17,7 @@ remote HTTP API. For example, in the case of the document context manager, this to fetch and a `put_document` to save. 1. In `cloudant-python-sdk` View, Search, and Query (aka `_find` endpoint) operation responses contain raw JSON content like using `raw_result=True` in `python-cloudant`. -1. Replay adapters are replaced by the [automatic retries](https://github. - com/IBM/ibm-cloud-sdk-common/#automatic-retries) feature for failed requests. +1. Replay adapters are replaced by the [automatic retries](https://github.com/IBM/ibm-cloud-sdk-common/#automatic-retries) feature for failed requests. 1. Error handling is not transferable from `python-cloudant` to `cloudant-python-sdk`. For more information go to the [Error handling section](https://cloud.ibm.com/apidocs/cloudant?code=python#error-handling) in our API docs. 1. Custom HTTP client configurations in `python-cloudant` can be set differently in `cloudant-python-sdk`. For more information go to the From 1fb0bb577e8fba6dbf95919d1e5f6326057e0b01 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Fri, 30 Aug 2024 16:36:34 +0000 Subject: [PATCH 185/185] deprecation warning --- docs/index.rst | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 0f642eed..a26ea611 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,14 +1 @@ -python-cloudant documentation -============================= - -This is the official Cloudant client library for Python. - -.. toctree:: - :maxdepth: 4 - - compatibility - getting_started - cloudant - -* :ref:`genindex` - +This library is end-of-life and no longer supported.